Skip to content

Commit

Permalink
write files asap
Browse files Browse the repository at this point in the history
  • Loading branch information
@Lucki
Lucki committed Jul 30, 2021
1 parent ce4a597 commit 934c717
Show file tree
Hide file tree
Showing 4 changed files with 181 additions and 64 deletions.
103 changes: 65 additions & 38 deletions src/data/sources/epicgames/EpicAnalysis.vala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,11 +13,11 @@ namespace GameHub.Data.Sources.EpicGames
// FIXME: There are a lot of things related to Legendarys memory management we probably don't even need
private class Analysis
{
internal AnalysisResult? result { get; default = null; }
internal ArrayList<Task> tasks { get; default = new ArrayList<Task>(); }
internal LinkedList<uint32> chunks_to_dl { get; default = new LinkedList<uint32>(); }
internal Manifest.ChunkDataList chunk_data_list { get; default = null; }
internal string? base_url { get; default = null; }
internal AnalysisResult? result { get; default = null; }
internal ArrayList<ArrayList<Task>> tasks { get; default = new ArrayList<ArrayList<Task>>(); }
internal LinkedList<uint32> chunks_to_dl { get; default = new LinkedList<uint32>(); }
internal Manifest.ChunkDataList chunk_data_list { get; default = null; }
internal string? base_url { get; default = null; }

private File? resume_file { get; default = null; }
private HashMap<string, string> hash_map { get; default = new HashMap<string, string>(); }
Expand Down Expand Up @@ -70,15 +70,15 @@ namespace GameHub.Data.Sources.EpicGames
private uint32 removed { get; default = 0; }
private uint32 uncompressed_dl_size { get; default = 0; }

internal AnalysisResult(Manifest new_manifest,
string download_dir,
ref HashMap<string, string> hash_map,
ref LinkedList<uint32> chunks_to_dl,
ref ArrayList<Task> tasks,
out Manifest.ChunkDataList chunk_data_list,
Manifest? old_manifest = null,
File? resume_file = null,
string[]? file_install_tags = null)
internal AnalysisResult(Manifest new_manifest,
string download_dir,
ref HashMap<string, string> hash_map,
ref LinkedList<uint32> chunks_to_dl,
ref ArrayList<ArrayList<Task>> tasks,
out Manifest.ChunkDataList chunk_data_list,
Manifest? old_manifest = null,
File? resume_file = null,
string[]? file_install_tags = null)
{
foreach(var element in new_manifest.file_manifest_list.elements)
{
Expand Down Expand Up @@ -369,7 +369,9 @@ namespace GameHub.Data.Sources.EpicGames
}
else if(current_file.chunk_parts.size == 0)
{
tasks.add(new FileTask.empty_file(current_file.filename));
var task_list = new ArrayList<Task>();
task_list.add(new FileTask.empty_file(current_file.filename));
tasks.add(task_list);
continue;
}

Expand Down Expand Up @@ -438,21 +440,26 @@ namespace GameHub.Data.Sources.EpicGames
{
if(log_analysis) debug(@"[Sources.EpicGames.AnalysisResult] Reusing $reused chunks from: $(current_file.filename)");

var task_list = new ArrayList<Task>();
// open temporary file that will contain download + old file contents
tasks.add(new FileTask.open(current_file.filename + ".tmp"));
tasks.add_all(chunk_tasks);
tasks.add(new FileTask.close(current_file.filename + ".tmp"));
task_list.add(new FileTask.open(current_file.filename + ".tmp"));
task_list.add_all(chunk_tasks);
task_list.add(new FileTask.close(current_file.filename + ".tmp"));

// delete old file and rename temporary
tasks.add(new FileTask.rename(current_file.filename,
current_file.filename + ".tmp",
true));
task_list.add(new FileTask.rename(current_file.filename,
current_file.filename + ".tmp",
true));

tasks.add(task_list);
}
else
{
tasks.add(new FileTask.open(current_file.filename));
tasks.add_all(chunk_tasks);
tasks.add(new FileTask.close(current_file.filename));
var task_list = new ArrayList<Task>();
task_list.add(new FileTask.open(current_file.filename));
task_list.add_all(chunk_tasks);
task_list.add(new FileTask.close(current_file.filename));
tasks.add(task_list);
}

// check if runtime cache size has changed
Expand Down Expand Up @@ -487,10 +494,12 @@ namespace GameHub.Data.Sources.EpicGames
// add jobs to remove files
foreach(var filename in manifest_comparison.removed)
{
tasks.add(new FileTask.delete(filename));
var task_list = new ArrayList<Task>();
task_list.add(new FileTask.delete(filename));
tasks.add(task_list);
}

tasks.add_all(additional_deletion_tasks);
tasks.add(additional_deletion_tasks);

_num_chunks_cache = dl_cache_guids.size;
chunk_data_list = new_manifest.chunk_data_list;
Expand Down Expand Up @@ -533,7 +542,7 @@ namespace GameHub.Data.Sources.EpicGames
// so that the tasks order stays in the correct position
internal abstract class Task
{
internal async abstract bool process(FileOutputStream? iostream, File install_dir, EpicGame game);
internal abstract bool process(ref FileOutputStream? iostream, File install_dir, EpicGame game);
}

/**
Expand Down Expand Up @@ -598,10 +607,11 @@ namespace GameHub.Data.Sources.EpicGames
_del = dele;
}

internal async override bool process(FileOutputStream? iostream, File install_dir, EpicGame game)
internal override bool process(ref FileOutputStream? iostream, File install_dir, EpicGame game)
{
// make directories
var full_path = File.new_build_filename(install_dir.get_path(), filename);
debug("Path: " + full_path.get_path());
Utils.FS.mkdir(full_path.get_parent().get_path());

try
Expand All @@ -622,13 +632,13 @@ namespace GameHub.Data.Sources.EpicGames

if(full_path.query_exists())
{
iostream = yield full_path.replace_async(null,
false,
FileCreateFlags.REPLACE_DESTINATION);
iostream = full_path.replace(null,
false,
FileCreateFlags.REPLACE_DESTINATION);
}
else
{
iostream = yield full_path.create_async(FileCreateFlags.NONE);
iostream = full_path.create(FileCreateFlags.NONE);
}
}
else if(fclose)
Expand All @@ -654,7 +664,20 @@ namespace GameHub.Data.Sources.EpicGames
path = path[0 : path.length - 4];
}

var file_hash = yield Utils.compute_file_checksum(full_path, ChecksumType.SHA1);
// var file_hash = yield Utils.compute_file_checksum(full_path, ChecksumType.SHA1);
// This is basically Utils.compute_file_checksum() but I need this in sync to be able to use ref iostream
Checksum checksum = new Checksum(ChecksumType.SHA1);
FileStream stream = FileStream.open(full_path.get_path(), "rb");
uint8 buf[4096];
size_t size;

while((size = stream.read(buf)) > 0)
{
checksum.update(buf, size);
}

var file_hash = checksum.get_string();

// var tmp = "";

// if(((Analysis.FileTask)file_task).filename[((Analysis.FileTask)file_task).filename.length - 4 : ((Analysis.FileTask)file_task).filename.length] == ".tmp")
Expand Down Expand Up @@ -738,7 +761,7 @@ namespace GameHub.Data.Sources.EpicGames
_chunk_size = chunk_size;
}

internal async override bool process(FileOutputStream? iostream, File install_dir, EpicGame game)
internal override bool process(ref FileOutputStream? iostream, File install_dir, EpicGame game)
{
var downloaded_chunk = Utils.FS.file(Utils.FS.Paths.EpicGames.Cache + "/chunks/" + game.id + "/" + chunk_guid.to_string());

Expand All @@ -751,26 +774,30 @@ namespace GameHub.Data.Sources.EpicGames
assert(File.new_build_filename(install_dir.get_path(), chunk_file).query_exists());
old_stream = File.new_build_filename(install_dir.get_path(), chunk_file).read();
old_stream.seek(chunk_offset, SeekType.SET);
var bytes = yield old_stream.read_bytes_async(chunk_size);
yield iostream.write_bytes_async(bytes);
var bytes = old_stream.read_bytes(chunk_size);
iostream.write_bytes(bytes);
old_stream.close();
old_stream = null;
}
else if(downloaded_chunk.query_exists())
{
var chunk = new Chunk.from_byte_stream(new DataInputStream(yield downloaded_chunk.read_async()));
var chunk = new Chunk.from_byte_stream(new DataInputStream(downloaded_chunk.read()));
// debug(@"chunk data length $(chunk.data.length)");
// debug("chunk %s hash: %s",
// hunk_guid.to_string(),
// Checksum.compute_for_bytes(ChecksumType.SHA1, chunk.data));
yield iostream.write_bytes_async(chunk.data[chunk_offset : chunk_offset + chunk_size]);
iostream.write_bytes(chunk.data[chunk_offset: chunk_offset + chunk_size]);
// debug(@"written $size bytes");

if(cleanup)
{
Utils.FS.rm(downloaded_chunk.get_path());
}
}
else
{
assert_not_reached();
}
}
catch (Error e)
{
Expand Down
24 changes: 12 additions & 12 deletions src/data/sources/epicgames/EpicDownloader.vala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,6 @@ namespace GameHub.Data.Sources.EpicGames
// TODO: a lot of small files, we should probably handle this in parallel
internal async bool download(Installer installer)
{
var files = new ArrayList<File>();
var game = installer.game;
var download = get_game_download(game);

Expand Down Expand Up @@ -138,8 +137,11 @@ namespace GameHub.Data.Sources.EpicGames
debug("[SoupDownloader] '%s' is already downloaded", part.remote.get_uri());
}

files.add(part.local);
download.downloaded_parts.offer(part);
if(!yield installer.write_file(part.chunk_info.guid_num))
{
throw new Error(0, 0, "Error");
}

current_part++;
continue;
}
Expand All @@ -159,8 +161,6 @@ namespace GameHub.Data.Sources.EpicGames
// var file = yield download(part.remote, part.local, new Downloader.DownloadInfo.for_runnable(task.runnable, partDesc), false);
if(part.local != null && part.local.query_exists())
{
files.add(part.local);
download.downloaded_parts.offer(part);
// TODO: uncompress, compare hash
// https://github.com/derrod/legendary/blob/a2280edea8f7f8da9a080fd3fb2bafcabf9ee33d/legendary/downloader/workers.py#L99
// var chunk = new Chunk.from_file(new DataInputStream(file.read()));
Expand Down Expand Up @@ -210,6 +210,11 @@ namespace GameHub.Data.Sources.EpicGames
// }
}

if(!yield installer.write_file(part.chunk_info.guid_num))
{
throw new Error(0, 0, "Error");
}

current_part++;
}

Expand Down Expand Up @@ -242,7 +247,7 @@ namespace GameHub.Data.Sources.EpicGames
download.status = new FileDownload.Status(Download.State.FINISHED);
lock (downloads) downloads.remove(game.id);
lock (dl_info) dl_info.remove(game.id);
// lock (dl_queue) dl_queue.remove(game.id);
lock (dl_queue) dl_queue.remove(game.id);
}

// download_manager().disconnect(ds_id);
Expand Down Expand Up @@ -543,14 +548,10 @@ namespace GameHub.Data.Sources.EpicGames
public File local_tmp;
public Manifest.ChunkDataList.ChunkInfo chunk_info;

public EpicPart(string id, Analysis analysis)
{
// base(id, analysis);
}
// public EpicPart(string id, Analysis analysis) {}

public EpicPart.from_chunk_guid(string id, Analysis analysis, uint32 chunk_guid)
{
// base(id, analysis);
chunk_info = analysis.chunk_data_list.get_chunk_by_number(chunk_guid);
remote = File.new_for_uri(analysis.base_url + "/" + chunk_info.path);
local = Utils.FS.file(Utils.FS.Paths.EpicGames.Cache + "/chunks/" + id + "/" + chunk_info.guid_num.to_string());
Expand All @@ -565,7 +566,6 @@ namespace GameHub.Data.Sources.EpicGames
public weak Message? message;
public bool is_cancelled = false;
public ArrayQueue<EpicPart> parts { get; default = new ArrayQueue<EpicPart>(); }
public ArrayQueue<EpicPart> downloaded_parts { get; default = new ArrayQueue<EpicPart>(); }

public EpicDownload(string id, Analysis analysis)
{
Expand Down
4 changes: 3 additions & 1 deletion src/data/sources/epicgames/EpicGame.vala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -641,8 +641,10 @@ namespace GameHub.Data.Sources.EpicGames
var gh_marker = (this is DLC) ? get_file(@"$(FS.GAMEHUB_DIR)/$id.version") : get_file(@"$(FS.GAMEHUB_DIR)/version");
gh_marker.delete();
}
catch (Error e) {}
catch (Error e)
{}

_manifest = null; // Forget cached manifest
update_status();
}

Expand Down
Loading

0 comments on commit 934c717

Please sign in to comment.