From 934c7178850e40da16799792de858c92936a85f8 Mon Sep 17 00:00:00 2001 From: Lucki Date: Fri, 30 Jul 2021 13:22:18 +0200 Subject: [PATCH] write files asap --- src/data/sources/epicgames/EpicAnalysis.vala | 103 ++++++++++------ .../sources/epicgames/EpicDownloader.vala | 24 ++-- src/data/sources/epicgames/EpicGame.vala | 4 +- src/data/sources/epicgames/EpicInstaller.vala | 114 ++++++++++++++++-- 4 files changed, 181 insertions(+), 64 deletions(-) diff --git a/src/data/sources/epicgames/EpicAnalysis.vala b/src/data/sources/epicgames/EpicAnalysis.vala index e4e5994b..790a3d12 100644 --- a/src/data/sources/epicgames/EpicAnalysis.vala +++ b/src/data/sources/epicgames/EpicAnalysis.vala @@ -13,11 +13,11 @@ namespace GameHub.Data.Sources.EpicGames // FIXME: There are a lot of things related to Legendarys memory management we probably don't even need private class Analysis { - internal AnalysisResult? result { get; default = null; } - internal ArrayList tasks { get; default = new ArrayList(); } - internal LinkedList chunks_to_dl { get; default = new LinkedList(); } - internal Manifest.ChunkDataList chunk_data_list { get; default = null; } - internal string? base_url { get; default = null; } + internal AnalysisResult? result { get; default = null; } + internal ArrayList> tasks { get; default = new ArrayList>(); } + internal LinkedList chunks_to_dl { get; default = new LinkedList(); } + internal Manifest.ChunkDataList chunk_data_list { get; default = null; } + internal string? base_url { get; default = null; } private File? resume_file { get; default = null; } private HashMap hash_map { get; default = new HashMap(); } @@ -70,15 +70,15 @@ namespace GameHub.Data.Sources.EpicGames private uint32 removed { get; default = 0; } private uint32 uncompressed_dl_size { get; default = 0; } - internal AnalysisResult(Manifest new_manifest, - string download_dir, - ref HashMap hash_map, - ref LinkedList chunks_to_dl, - ref ArrayList tasks, - out Manifest.ChunkDataList chunk_data_list, - Manifest? old_manifest = null, - File? resume_file = null, - string[]? file_install_tags = null) + internal AnalysisResult(Manifest new_manifest, + string download_dir, + ref HashMap hash_map, + ref LinkedList chunks_to_dl, + ref ArrayList> tasks, + out Manifest.ChunkDataList chunk_data_list, + Manifest? old_manifest = null, + File? resume_file = null, + string[]? file_install_tags = null) { foreach(var element in new_manifest.file_manifest_list.elements) { @@ -369,7 +369,9 @@ namespace GameHub.Data.Sources.EpicGames } else if(current_file.chunk_parts.size == 0) { - tasks.add(new FileTask.empty_file(current_file.filename)); + var task_list = new ArrayList(); + task_list.add(new FileTask.empty_file(current_file.filename)); + tasks.add(task_list); continue; } @@ -438,21 +440,26 @@ namespace GameHub.Data.Sources.EpicGames { if(log_analysis) debug(@"[Sources.EpicGames.AnalysisResult] Reusing $reused chunks from: $(current_file.filename)"); + var task_list = new ArrayList(); // open temporary file that will contain download + old file contents - tasks.add(new FileTask.open(current_file.filename + ".tmp")); - tasks.add_all(chunk_tasks); - tasks.add(new FileTask.close(current_file.filename + ".tmp")); + task_list.add(new FileTask.open(current_file.filename + ".tmp")); + task_list.add_all(chunk_tasks); + task_list.add(new FileTask.close(current_file.filename + ".tmp")); // delete old file and rename temporary - tasks.add(new FileTask.rename(current_file.filename, - current_file.filename + ".tmp", - true)); + task_list.add(new FileTask.rename(current_file.filename, + current_file.filename + ".tmp", + true)); + + tasks.add(task_list); } else { - tasks.add(new FileTask.open(current_file.filename)); - tasks.add_all(chunk_tasks); - tasks.add(new FileTask.close(current_file.filename)); + var task_list = new ArrayList(); + task_list.add(new FileTask.open(current_file.filename)); + task_list.add_all(chunk_tasks); + task_list.add(new FileTask.close(current_file.filename)); + tasks.add(task_list); } // check if runtime cache size has changed @@ -487,10 +494,12 @@ namespace GameHub.Data.Sources.EpicGames // add jobs to remove files foreach(var filename in manifest_comparison.removed) { - tasks.add(new FileTask.delete(filename)); + var task_list = new ArrayList(); + task_list.add(new FileTask.delete(filename)); + tasks.add(task_list); } - tasks.add_all(additional_deletion_tasks); + tasks.add(additional_deletion_tasks); _num_chunks_cache = dl_cache_guids.size; chunk_data_list = new_manifest.chunk_data_list; @@ -533,7 +542,7 @@ namespace GameHub.Data.Sources.EpicGames // so that the tasks order stays in the correct position internal abstract class Task { - internal async abstract bool process(FileOutputStream? iostream, File install_dir, EpicGame game); + internal abstract bool process(ref FileOutputStream? iostream, File install_dir, EpicGame game); } /** @@ -598,10 +607,11 @@ namespace GameHub.Data.Sources.EpicGames _del = dele; } - internal async override bool process(FileOutputStream? iostream, File install_dir, EpicGame game) + internal override bool process(ref FileOutputStream? iostream, File install_dir, EpicGame game) { // make directories var full_path = File.new_build_filename(install_dir.get_path(), filename); + debug("Path: " + full_path.get_path()); Utils.FS.mkdir(full_path.get_parent().get_path()); try @@ -622,13 +632,13 @@ namespace GameHub.Data.Sources.EpicGames if(full_path.query_exists()) { - iostream = yield full_path.replace_async(null, - false, - FileCreateFlags.REPLACE_DESTINATION); + iostream = full_path.replace(null, + false, + FileCreateFlags.REPLACE_DESTINATION); } else { - iostream = yield full_path.create_async(FileCreateFlags.NONE); + iostream = full_path.create(FileCreateFlags.NONE); } } else if(fclose) @@ -654,7 +664,20 @@ namespace GameHub.Data.Sources.EpicGames path = path[0 : path.length - 4]; } - var file_hash = yield Utils.compute_file_checksum(full_path, ChecksumType.SHA1); + // var file_hash = yield Utils.compute_file_checksum(full_path, ChecksumType.SHA1); + // This is basically Utils.compute_file_checksum() but I need this in sync to be able to use ref iostream + Checksum checksum = new Checksum(ChecksumType.SHA1); + FileStream stream = FileStream.open(full_path.get_path(), "rb"); + uint8 buf[4096]; + size_t size; + + while((size = stream.read(buf)) > 0) + { + checksum.update(buf, size); + } + + var file_hash = checksum.get_string(); + // var tmp = ""; // if(((Analysis.FileTask)file_task).filename[((Analysis.FileTask)file_task).filename.length - 4 : ((Analysis.FileTask)file_task).filename.length] == ".tmp") @@ -738,7 +761,7 @@ namespace GameHub.Data.Sources.EpicGames _chunk_size = chunk_size; } - internal async override bool process(FileOutputStream? iostream, File install_dir, EpicGame game) + internal override bool process(ref FileOutputStream? iostream, File install_dir, EpicGame game) { var downloaded_chunk = Utils.FS.file(Utils.FS.Paths.EpicGames.Cache + "/chunks/" + game.id + "/" + chunk_guid.to_string()); @@ -751,19 +774,19 @@ namespace GameHub.Data.Sources.EpicGames assert(File.new_build_filename(install_dir.get_path(), chunk_file).query_exists()); old_stream = File.new_build_filename(install_dir.get_path(), chunk_file).read(); old_stream.seek(chunk_offset, SeekType.SET); - var bytes = yield old_stream.read_bytes_async(chunk_size); - yield iostream.write_bytes_async(bytes); + var bytes = old_stream.read_bytes(chunk_size); + iostream.write_bytes(bytes); old_stream.close(); old_stream = null; } else if(downloaded_chunk.query_exists()) { - var chunk = new Chunk.from_byte_stream(new DataInputStream(yield downloaded_chunk.read_async())); + var chunk = new Chunk.from_byte_stream(new DataInputStream(downloaded_chunk.read())); // debug(@"chunk data length $(chunk.data.length)"); // debug("chunk %s hash: %s", // hunk_guid.to_string(), // Checksum.compute_for_bytes(ChecksumType.SHA1, chunk.data)); - yield iostream.write_bytes_async(chunk.data[chunk_offset : chunk_offset + chunk_size]); + iostream.write_bytes(chunk.data[chunk_offset: chunk_offset + chunk_size]); // debug(@"written $size bytes"); if(cleanup) @@ -771,6 +794,10 @@ namespace GameHub.Data.Sources.EpicGames Utils.FS.rm(downloaded_chunk.get_path()); } } + else + { + assert_not_reached(); + } } catch (Error e) { diff --git a/src/data/sources/epicgames/EpicDownloader.vala b/src/data/sources/epicgames/EpicDownloader.vala index 6509d439..13584356 100644 --- a/src/data/sources/epicgames/EpicDownloader.vala +++ b/src/data/sources/epicgames/EpicDownloader.vala @@ -54,7 +54,6 @@ namespace GameHub.Data.Sources.EpicGames // TODO: a lot of small files, we should probably handle this in parallel internal async bool download(Installer installer) { - var files = new ArrayList(); var game = installer.game; var download = get_game_download(game); @@ -138,8 +137,11 @@ namespace GameHub.Data.Sources.EpicGames debug("[SoupDownloader] '%s' is already downloaded", part.remote.get_uri()); } - files.add(part.local); - download.downloaded_parts.offer(part); + if(!yield installer.write_file(part.chunk_info.guid_num)) + { + throw new Error(0, 0, "Error"); + } + current_part++; continue; } @@ -159,8 +161,6 @@ namespace GameHub.Data.Sources.EpicGames // var file = yield download(part.remote, part.local, new Downloader.DownloadInfo.for_runnable(task.runnable, partDesc), false); if(part.local != null && part.local.query_exists()) { - files.add(part.local); - download.downloaded_parts.offer(part); // TODO: uncompress, compare hash // https://github.com/derrod/legendary/blob/a2280edea8f7f8da9a080fd3fb2bafcabf9ee33d/legendary/downloader/workers.py#L99 // var chunk = new Chunk.from_file(new DataInputStream(file.read())); @@ -210,6 +210,11 @@ namespace GameHub.Data.Sources.EpicGames // } } + if(!yield installer.write_file(part.chunk_info.guid_num)) + { + throw new Error(0, 0, "Error"); + } + current_part++; } @@ -242,7 +247,7 @@ namespace GameHub.Data.Sources.EpicGames download.status = new FileDownload.Status(Download.State.FINISHED); lock (downloads) downloads.remove(game.id); lock (dl_info) dl_info.remove(game.id); - // lock (dl_queue) dl_queue.remove(game.id); + lock (dl_queue) dl_queue.remove(game.id); } // download_manager().disconnect(ds_id); @@ -543,14 +548,10 @@ namespace GameHub.Data.Sources.EpicGames public File local_tmp; public Manifest.ChunkDataList.ChunkInfo chunk_info; - public EpicPart(string id, Analysis analysis) - { - // base(id, analysis); - } + // public EpicPart(string id, Analysis analysis) {} public EpicPart.from_chunk_guid(string id, Analysis analysis, uint32 chunk_guid) { - // base(id, analysis); chunk_info = analysis.chunk_data_list.get_chunk_by_number(chunk_guid); remote = File.new_for_uri(analysis.base_url + "/" + chunk_info.path); local = Utils.FS.file(Utils.FS.Paths.EpicGames.Cache + "/chunks/" + id + "/" + chunk_info.guid_num.to_string()); @@ -565,7 +566,6 @@ namespace GameHub.Data.Sources.EpicGames public weak Message? message; public bool is_cancelled = false; public ArrayQueue parts { get; default = new ArrayQueue(); } - public ArrayQueue downloaded_parts { get; default = new ArrayQueue(); } public EpicDownload(string id, Analysis analysis) { diff --git a/src/data/sources/epicgames/EpicGame.vala b/src/data/sources/epicgames/EpicGame.vala index 05487bb3..7e6224e1 100644 --- a/src/data/sources/epicgames/EpicGame.vala +++ b/src/data/sources/epicgames/EpicGame.vala @@ -641,8 +641,10 @@ namespace GameHub.Data.Sources.EpicGames var gh_marker = (this is DLC) ? get_file(@"$(FS.GAMEHUB_DIR)/$id.version") : get_file(@"$(FS.GAMEHUB_DIR)/version"); gh_marker.delete(); } - catch (Error e) {} + catch (Error e) + {} + _manifest = null; // Forget cached manifest update_status(); } diff --git a/src/data/sources/epicgames/EpicInstaller.vala b/src/data/sources/epicgames/EpicInstaller.vala index fce10ce7..a44b482d 100644 --- a/src/data/sources/epicgames/EpicInstaller.vala +++ b/src/data/sources/epicgames/EpicInstaller.vala @@ -12,6 +12,8 @@ namespace GameHub.Data.Sources.EpicGames internal EpicGame game { get; private set; } internal InstallTask? install_task { get; default = null; } + private ArrayList> file_tasks { get; } + internal Installer(EpicGame game, Platform platform) { _game = game; @@ -53,7 +55,8 @@ namespace GameHub.Data.Sources.EpicGames debug("starting installation"); debug("preparing download"); - _analysis = game.prepare_download(install_task); + _analysis = game.prepare_download(install_task); + _file_tasks = analysis.tasks; // game is either up to date or hasn't changed, so we have nothing to do if(analysis.result.dl_size < 1) @@ -73,9 +76,26 @@ namespace GameHub.Data.Sources.EpicGames } else { - if(!yield EpicDownloader.instance.download(this)) assert_not_reached(); + if(!yield EpicDownloader.instance.download(this)) + { + debug("downloading failed"); + task.status = new InstallTask.Status(InstallTask.State.NONE); + game.status = new Game.Status(Game.State.UNINSTALLED, this.game); - if(!yield write_files(game, install_task.install_dir, analysis.tasks)) assert_not_reached(); + return false; + } + + if(!file_tasks.is_empty) + { + if(!yield write_files(file_tasks)) + { + debug("downloading failed"); + task.status = new InstallTask.Status(InstallTask.State.NONE); + game.status = new Game.Status(Game.State.UNINSTALLED, this.game); + + return false; + } + } } update_game_info(); @@ -134,27 +154,95 @@ namespace GameHub.Data.Sources.EpicGames game.update_status(); } - private async bool write_files(EpicGame game, File install_dir, ArrayList tasks) + private async bool write_files(ArrayList> tasks) { // download_task should be available here with all required information // tasks should be in the correct order: open -> write chunk -> close FileOutputStream? iostream = null; - - foreach(var file_task in tasks) + foreach(var task_list in tasks) { - if(file_task is Analysis.FileTask) + foreach(var task in task_list) + { + if(task is Analysis.FileTask) + { + return_val_if_fail(task.process(ref iostream, install_task.install_dir, game), false); + continue; + } + + // We should only be here with a valid iostream + return_val_if_fail(task is Analysis.ChunkTask, false); + assert_nonnull(iostream); + + return_val_if_fail(task.process(ref iostream, install_task.install_dir, game), false); + } + } + + return true; + } + + /** Write file if we have all required chunks */ + internal async bool write_file(uint32 guid_num) + { + var current_file_tasks = new ArrayList>(); + + // Get all tasks with the current guid and process it if we also have all other chunks + lock (file_tasks) { + foreach(var task_list in file_tasks) { - return_val_if_fail(yield file_task.process(iostream, install_dir, game), false); - continue; + if(task_list.first_match(() => + { + foreach(var task in task_list) + { + if(task is Analysis.ChunkTask + && ((Analysis.ChunkTask) task).chunk_guid == guid_num) + { + return true; + } + } + }) == null) + { + // This task set does not include this guid + continue; + } + + var list_complete = true; + foreach(var task in task_list) + { + // Check if other downloaded chunks are available + if(task is Analysis.FileTask + || (task is Analysis.ChunkTask + && ((Analysis.ChunkTask) task).chunk_file != null)) + { + continue; + } + + if(!Utils.FS.file(Utils.FS.Paths.EpicGames.Cache + "/chunks/" + game.id + "/" + ((Analysis.ChunkTask) task).chunk_guid.to_string()).query_exists()) + { + list_complete = false; + break; + } + } + + if(list_complete) + { + // FIXME: We may have lists here already which includes cleanup of our chunk + // while others still depend on it being available + current_file_tasks.add(task_list); + } } - // We should only be here with a valid iostream - return_val_if_fail(file_task is Analysis.ChunkTask, false); - assert_nonnull(iostream); + file_tasks.remove_all(current_file_tasks); + } + + if(current_file_tasks.is_empty) + { + debug("Nothing to do yet…"); - return_val_if_fail(yield file_task.process(iostream, install_dir, game), false); + return true; } + return_val_if_fail(yield write_files(current_file_tasks), false); + return true; } }