From 79e55095ce73002e437c0df3cf6e6fcc7cafd40f Mon Sep 17 00:00:00 2001 From: oureveryday Date: Fri, 20 Dec 2024 17:13:07 +0800 Subject: [PATCH] Update to 2.7.4 --- DepotDownloader/Ansi.cs | 5 + DepotDownloader/AnsiDetector.cs | 83 +++--- DepotDownloader/ContentDownloader.cs | 256 ++++++++---------- DepotDownloader/DepotDownloaderMod.csproj | 9 +- DepotDownloader/DownloadConfig.cs | 1 + DepotDownloader/NativeMethods.txt | 3 + DepotDownloader/PlatformUtilities.cs | 1 - DepotDownloader/Program.cs | 18 +- DepotDownloader/ProtoManifest.cs | 35 +++ DepotDownloader/Steam3Session.cs | 308 ++++++++-------------- DepotDownloader/Util.cs | 101 ++++++- README.md | 87 +++--- global.json | 2 +- 13 files changed, 466 insertions(+), 443 deletions(-) diff --git a/DepotDownloader/Ansi.cs b/DepotDownloader/Ansi.cs index b17b5972f..da369b3a3 100644 --- a/DepotDownloader/Ansi.cs +++ b/DepotDownloader/Ansi.cs @@ -30,6 +30,11 @@ public static void Init() { return; } + + if (OperatingSystem.IsLinux()) + { + return; + } var (supportsAnsi, legacyConsole) = AnsiDetector.Detect(stdError: false, upgrade: true); diff --git a/DepotDownloader/AnsiDetector.cs b/DepotDownloader/AnsiDetector.cs index 2110d9ccb..f7f8a4204 100644 --- a/DepotDownloader/AnsiDetector.cs +++ b/DepotDownloader/AnsiDetector.cs @@ -7,6 +7,8 @@ using System.Linq; using System.Runtime.InteropServices; using System.Text.RegularExpressions; +using Windows.Win32; +using Windows.Win32.System.Console; namespace Spectre.Console; @@ -45,7 +47,7 @@ public static (bool SupportsAnsi, bool LegacyConsole) Detect(bool stdError, bool return (true, false); } - var supportsAnsi = Windows.SupportsAnsi(upgrade, stdError, out var legacyConsole); + var supportsAnsi = WindowsSupportsAnsi(upgrade, stdError, out var legacyConsole); return (supportsAnsi, legacyConsole); } @@ -67,68 +69,49 @@ private static (bool SupportsAnsi, bool LegacyConsole) DetectFromTerm() return (false, true); } - private static class Windows + private static bool WindowsSupportsAnsi(bool upgrade, bool stdError, out bool isLegacy) { - private const int STD_OUTPUT_HANDLE = -11; - private const int STD_ERROR_HANDLE = -12; - private const uint ENABLE_VIRTUAL_TERMINAL_PROCESSING = 0x0004; - private const uint DISABLE_NEWLINE_AUTO_RETURN = 0x0008; + isLegacy = false; - [DllImport("kernel32.dll")] - private static extern bool GetConsoleMode(IntPtr hConsoleHandle, out uint lpMode); - - [DllImport("kernel32.dll")] - private static extern bool SetConsoleMode(IntPtr hConsoleHandle, uint dwMode); - - [DllImport("kernel32.dll", SetLastError = true)] - private static extern IntPtr GetStdHandle(int nStdHandle); + try + { + var @out = PInvoke.GetStdHandle_SafeHandle(stdError ? STD_HANDLE.STD_ERROR_HANDLE :STD_HANDLE.STD_OUTPUT_HANDLE); - [DllImport("kernel32.dll")] - public static extern uint GetLastError(); + if (!PInvoke.GetConsoleMode(@out, out var mode)) + { + // Could not get console mode, try TERM (set in cygwin, WSL-Shell). + var (ansiFromTerm, legacyFromTerm) = DetectFromTerm(); - public static bool SupportsAnsi(bool upgrade, bool stdError, out bool isLegacy) - { - isLegacy = false; + isLegacy = ansiFromTerm ? legacyFromTerm : isLegacy; + return ansiFromTerm; + } - try + if ((mode & CONSOLE_MODE.ENABLE_VIRTUAL_TERMINAL_PROCESSING) == 0||true) { - var @out = GetStdHandle(stdError ? STD_ERROR_HANDLE : STD_OUTPUT_HANDLE); - if (!GetConsoleMode(@out, out var mode)) - { - // Could not get console mode, try TERM (set in cygwin, WSL-Shell). - var (ansiFromTerm, legacyFromTerm) = DetectFromTerm(); + isLegacy = true; - isLegacy = ansiFromTerm ? legacyFromTerm : isLegacy; - return ansiFromTerm; + if (!upgrade) + { + return false; } - if ((mode & ENABLE_VIRTUAL_TERMINAL_PROCESSING) == 0) + // Try enable ANSI support. + mode |= CONSOLE_MODE.ENABLE_VIRTUAL_TERMINAL_PROCESSING | CONSOLE_MODE.DISABLE_NEWLINE_AUTO_RETURN; + if (!PInvoke.SetConsoleMode(@out, mode)) { - isLegacy = true; - - if (!upgrade) - { - return false; - } - - // Try enable ANSI support. - mode |= ENABLE_VIRTUAL_TERMINAL_PROCESSING | DISABLE_NEWLINE_AUTO_RETURN; - if (!SetConsoleMode(@out, mode)) - { - // Enabling failed. - return false; - } - - isLegacy = false; + // Enabling failed. + return false; } - return true; - } - catch - { - // All we know here is that we don't support ANSI. - return false; + isLegacy = false; } + + return true; + } + catch + { + // All we know here is that we don't support ANSI. + return false; } } } diff --git a/DepotDownloader/ContentDownloader.cs b/DepotDownloader/ContentDownloader.cs index 4adc36312..a193c824a 100644 --- a/DepotDownloader/ContentDownloader.cs +++ b/DepotDownloader/ContentDownloader.cs @@ -106,7 +106,7 @@ static bool TestIsFileIncluded(string filename) return false; } - static bool AccountHasAccess(uint depotId) + static async Task AccountHasAccess(uint appId, uint depotId) { if (steam3 == null || steam3.steamUser.SteamID == null || (steam3.Licenses == null && steam3.steamUser.SteamID.AccountType != EAccountType.AnonUser)) return false; @@ -121,7 +121,7 @@ static bool AccountHasAccess(uint depotId) licenseQuery = steam3.Licenses.Select(x => x.PackageID).Distinct(); } - steam3.RequestPackageInfo(licenseQuery); + await steam3.RequestPackageInfo(licenseQuery); foreach (var license in licenseQuery) { @@ -184,7 +184,21 @@ static uint GetSteam3AppBuildNumber(uint appId, string branch) return uint.Parse(buildid.Value); } - static ulong GetSteam3DepotManifest(uint depotId, uint appId, string branch) + static uint GetSteam3DepotProxyAppId(uint depotId, uint appId) + { + var depots = GetSteam3AppSection(appId, EAppInfoSection.Depots); + var depotChild = depots[depotId.ToString()]; + + if (depotChild == KeyValue.Invalid) + return INVALID_APP_ID; + + if (depotChild["depotfromapp"] == KeyValue.Invalid) + return INVALID_APP_ID; + + return depotChild["depotfromapp"].AsUnsignedInteger(); + } + + static async Task GetSteam3DepotManifest(uint depotId, uint appId, string branch) { var depots = GetSteam3AppSection(appId, EAppInfoSection.Depots); var depotChild = depots[depotId.ToString()]; @@ -206,9 +220,9 @@ static ulong GetSteam3DepotManifest(uint depotId, uint appId, string branch) return INVALID_MANIFEST_ID; } - steam3.RequestAppInfo(otherAppId); + await steam3.RequestAppInfo(otherAppId); - return GetSteam3DepotManifest(depotId, otherAppId, branch); + return await GetSteam3DepotManifest(depotId, otherAppId, branch); } var manifests = depotChild["manifests"]; @@ -236,7 +250,7 @@ static ulong GetSteam3DepotManifest(uint depotId, uint appId, string branch) if (encrypted_gid != KeyValue.Invalid) { // Submit the password to Steam now to get encryption keys - steam3.CheckAppBetaPassword(appId, Config.BetaPassword); + await steam3.CheckAppBetaPassword(appId, Config.BetaPassword); if (!steam3.AppBetaPasswords.TryGetValue(branch, out var appBetaPassword)) { @@ -307,6 +321,8 @@ public static bool InitializeSteam3(string username, string password) return false; } + Task.Run(steam3.TickCallbacks); + return true; } @@ -326,7 +342,7 @@ public static void ShutdownSteam3() public static async Task DownloadPubfileAsync(uint appId, ulong publishedFileId) { - var details = steam3.GetPublishedFileDetails(appId, publishedFileId); + var details = await steam3.GetPublishedFileDetails(appId, publishedFileId); if (!string.IsNullOrEmpty(details?.file_url)) { @@ -348,7 +364,7 @@ public static async Task DownloadUGCAsync(uint appId, ulong ugcId) if (steam3.steamUser.SteamID.AccountType != EAccountType.AnonUser) { - details = steam3.GetUGCDetails(ugcId); + details = await steam3.GetUGCDetails(ugcId); } else { @@ -361,7 +377,7 @@ public static async Task DownloadUGCAsync(uint appId, ulong ugcId) } else { - await DownloadAppAsync(appId, new List<(uint, ulong)> { (appId, ugcId) }, DEFAULT_BRANCH, null, null, null, false, true); + await DownloadAppAsync(appId, [(appId, ugcId)], DEFAULT_BRANCH, null, null, null, false, true); } } @@ -410,16 +426,16 @@ public static async Task DownloadAppAsync(uint appId, List<(uint depotId, ulong Directory.CreateDirectory(Path.Combine(configPath, CONFIG_DIR)); DepotConfigStore.LoadFromFile(Path.Combine(configPath, CONFIG_DIR, "depot.config")); - steam3?.RequestAppInfo(appId); + await steam3?.RequestAppInfo(appId); /* - if (!AccountHasAccess(appId)) + if (!await AccountHasAccess(appId)) { - if (steam3.RequestFreeAppLicense(appId)) + if (await steam3.RequestFreeAppLicense(appId)) { Console.WriteLine("Obtained FreeOnDemand license for app {0}", appId); // Fetch app info again in case we didn't get it fully without a license. - steam3.RequestAppInfo(appId, true); + await steam3.RequestAppInfo(appId, true); } else { @@ -476,7 +492,8 @@ public static async Task DownloadAppAsync(uint appId, List<(uint depotId, ulong continue; } - if (depotConfig["osarch"] != KeyValue.Invalid && + if (!Config.DownloadAllArchs && + depotConfig["osarch"] != KeyValue.Invalid && !string.IsNullOrWhiteSpace(depotConfig["osarch"].Value)) { var depotArch = depotConfig["osarch"].Value; @@ -523,7 +540,7 @@ public static async Task DownloadAppAsync(uint appId, List<(uint depotId, ulong foreach (var (depotId, manifestId) in depotManifestIds) { - var info = GetDepotInfo(depotId, appId, manifestId, branch); + var info = await GetDepotInfo(depotId, appId, manifestId, branch); if (info != null) { infos.Add(info); @@ -541,13 +558,15 @@ public static async Task DownloadAppAsync(uint appId, List<(uint depotId, ulong } } - static DepotDownloadInfo GetDepotInfo(uint depotId, uint appId, ulong manifestId, string branch) + static async Task GetDepotInfo(uint depotId, uint appId, ulong manifestId, string branch) { if (steam3 != null && appId != INVALID_APP_ID) - steam3.RequestAppInfo(appId); + { + await steam3.RequestAppInfo(appId); + } /* - if (!AccountHasAccess(depotId)) + if (!await AccountHasAccess(depotId)) { Console.WriteLine("Depot {0} is not available from this account.", depotId); @@ -557,12 +576,12 @@ static DepotDownloadInfo GetDepotInfo(uint depotId, uint appId, ulong manifestId if (manifestId == INVALID_MANIFEST_ID) { - manifestId = GetSteam3DepotManifest(depotId, appId, branch); + manifestId = await GetSteam3DepotManifest(depotId, appId, branch); if (manifestId == INVALID_MANIFEST_ID && !string.Equals(branch, DEFAULT_BRANCH, StringComparison.OrdinalIgnoreCase)) { Console.WriteLine("Warning: Depot {0} does not have branch named \"{1}\". Trying {2} branch.", depotId, branch, DEFAULT_BRANCH); branch = DEFAULT_BRANCH; - manifestId = GetSteam3DepotManifest(depotId, appId, branch); + manifestId = await GetSteam3DepotManifest(depotId, appId, branch); } if (manifestId == INVALID_MANIFEST_ID) @@ -580,7 +599,7 @@ static DepotDownloadInfo GetDepotInfo(uint depotId, uint appId, ulong manifestId } else { - steam3.RequestDepotKey(depotId, appId); + await steam3.RequestDepotKey(depotId, appId); } if (!steam3.DepotKeys.TryGetValue(depotId, out depotKey)) { @@ -596,13 +615,18 @@ static DepotDownloadInfo GetDepotInfo(uint depotId, uint appId, ulong manifestId return null; } - return new DepotDownloadInfo(depotId, appId, manifestId, branch, installDir, depotKey); + // For depots that are proxied through depotfromapp, we still need to resolve the proxy app id + var containingAppId = appId; + var proxyAppId = GetSteam3DepotProxyAppId(depotId, appId); + if (proxyAppId != INVALID_APP_ID) containingAppId = proxyAppId; + + return new DepotDownloadInfo(depotId, containingAppId, manifestId, branch, installDir, depotKey); } - private class ChunkMatch(ProtoManifest.ChunkData oldChunk, ProtoManifest.ChunkData newChunk) + private class ChunkMatch(DepotManifest.ChunkData oldChunk, DepotManifest.ChunkData newChunk) { - public ProtoManifest.ChunkData OldChunk { get; } = oldChunk; - public ProtoManifest.ChunkData NewChunk { get; } = newChunk; + public DepotManifest.ChunkData OldChunk { get; } = oldChunk; + public DepotManifest.ChunkData NewChunk { get; } = newChunk; } private class DepotFilesData @@ -610,9 +634,9 @@ private class DepotFilesData public DepotDownloadInfo depotDownloadInfo; public DepotDownloadCounter depotCounter; public string stagingDir; - public ProtoManifest manifest; - public ProtoManifest previousManifest; - public List filteredFiles; + public DepotManifest manifest; + public DepotManifest previousManifest; + public List filteredFiles; public HashSet allFileNames; } @@ -695,8 +719,8 @@ private static async Task ProcessDepotManifestAndFiles(Cancellat Console.WriteLine("Processing depot {0}", depot.DepotId); - ProtoManifest oldProtoManifest = null; - ProtoManifest newProtoManifest = null; + DepotManifest oldManifest = null; + DepotManifest newManifest = null; var configDir = Path.Combine(depot.InstallDir, CONFIG_DIR); var lastManifestId = INVALID_MANIFEST_ID; @@ -708,78 +732,34 @@ private static async Task ProcessDepotManifestAndFiles(Cancellat if (lastManifestId != INVALID_MANIFEST_ID) { - var oldManifestFileName = Path.Combine(configDir, string.Format("{0}_{1}.bin", depot.DepotId, lastManifestId)); - - if (File.Exists(oldManifestFileName)) - { - byte[] expectedChecksum; - - try - { - expectedChecksum = File.ReadAllBytes(oldManifestFileName + ".sha"); - } - catch (IOException) - { - expectedChecksum = null; - } - - oldProtoManifest = ProtoManifest.LoadFromFile(oldManifestFileName, out var currentChecksum); - - if (expectedChecksum == null || !expectedChecksum.SequenceEqual(currentChecksum)) - { - // We only have to show this warning if the old manifest ID was different - if (lastManifestId != depot.ManifestId) - Console.WriteLine("Manifest {0} on disk did not match the expected checksum.", lastManifestId); - //oldProtoManifest = null; - } - } + // We only have to show this warning if the old manifest ID was different + var badHashWarning = (lastManifestId != depot.ManifestId); + oldManifest = Util.LoadManifestFromFile(configDir, depot.DepotId, lastManifestId, badHashWarning); } if (Config.UseManifestFile) { lastManifestId = depot.ManifestId; - oldProtoManifest = new ProtoManifest(DepotManifest.LoadFromFile(Config.ManifestFile), 0); + oldManifest = DepotManifest.LoadFromFile(Config.ManifestFile); } - if (lastManifestId == depot.ManifestId && oldProtoManifest != null) + if (lastManifestId == depot.ManifestId && oldManifest != null) { - newProtoManifest = oldProtoManifest; + newManifest = oldManifest; Console.WriteLine("Already have manifest {0} for depot {1}.", depot.ManifestId, depot.DepotId); } else { - var newManifestFileName = Path.Combine(configDir, string.Format("{0}_{1}.bin", depot.DepotId, depot.ManifestId)); - if (newManifestFileName != null) - { - byte[] expectedChecksum; + newManifest = Util.LoadManifestFromFile(configDir, depot.DepotId, depot.ManifestId, true); - try - { - expectedChecksum = File.ReadAllBytes(newManifestFileName + ".sha"); - } - catch (IOException) - { - expectedChecksum = null; - } - - newProtoManifest = ProtoManifest.LoadFromFile(newManifestFileName, out var currentChecksum); - - if (newProtoManifest != null && (expectedChecksum == null || !expectedChecksum.SequenceEqual(currentChecksum))) - { - Console.WriteLine("Manifest {0} on disk did not match the expected checksum.", depot.ManifestId); - //newProtoManifest = null; - } - } - - if (newProtoManifest != null) + if (newManifest != null) { Console.WriteLine("Already have manifest {0} for depot {1}.", depot.ManifestId, depot.DepotId); } else { - Console.Write("Downloading depot manifest... "); + Console.WriteLine($"Downloading depot {depot.DepotId} manifest"); - DepotManifest depotManifest = null; ulong manifestRequestCode = 0; var manifestRequestCodeExpiration = DateTime.MinValue; @@ -817,7 +797,6 @@ private static async Task ProcessDepotManifestAndFiles(Cancellat // If we could not get the manifest code, this is a fatal error if (manifestRequestCode == 0) { - Console.WriteLine("No manifest request code was returned for {0} {1}", depot.DepotId, depot.ManifestId); cts.Cancel(); } } @@ -827,7 +806,7 @@ private static async Task ProcessDepotManifestAndFiles(Cancellat depot.ManifestId, connection, cdnPool.ProxyServer != null ? cdnPool.ProxyServer : "no proxy"); - depotManifest = await cdnPool.CDNClient.DownloadManifestAsync( + newManifest = await cdnPool.CDNClient.DownloadManifestAsync( depot.DepotId, depot.ManifestId, manifestRequestCode, @@ -879,9 +858,9 @@ private static async Task ProcessDepotManifestAndFiles(Cancellat cdnPool.ReturnBrokenConnection(connection); Console.WriteLine("Encountered error downloading manifest for depot {0} {1}: {2}", depot.DepotId, depot.ManifestId, e.Message); } - } while (depotManifest == null); + } while (newManifest == null); - if (depotManifest == null) + if (newManifest == null) { Console.WriteLine("\nUnable to download manifest {0} for depot {1}", depot.ManifestId, depot.DepotId); cts.Cancel(); @@ -890,28 +869,21 @@ private static async Task ProcessDepotManifestAndFiles(Cancellat // Throw the cancellation exception if requested so that this task is marked failed cts.Token.ThrowIfCancellationRequested(); - - newProtoManifest = new ProtoManifest(depotManifest, depot.ManifestId); - newProtoManifest.SaveToFile(newManifestFileName, out var checksum); - File.WriteAllBytes(newManifestFileName + ".sha", checksum); - - Console.WriteLine(" Done!"); + Util.SaveManifestToFile(configDir, newManifest); } } - newProtoManifest.Files.Sort((x, y) => string.Compare(x.FileName, y.FileName, StringComparison.Ordinal)); - - Console.WriteLine("Manifest {0} ({1})", depot.ManifestId, newProtoManifest.CreationTime); + Console.WriteLine("Manifest {0} ({1})", depot.ManifestId, newManifest.CreationTime); if (Config.DownloadManifestOnly) { - DumpManifestToTextFile(depot, newProtoManifest); + DumpManifestToTextFile(depot, newManifest); return null; } var stagingDir = Path.Combine(depot.InstallDir, STAGING_DIR); - var filesAfterExclusions = newProtoManifest.Files.AsParallel().Where(f => TestIsFileIncluded(f.FileName)).ToList(); + var filesAfterExclusions = newManifest.Files.AsParallel().Where(f => TestIsFileIncluded(f.FileName)).ToList(); var allFileNames = new HashSet(filesAfterExclusions.Count); // Pre-process @@ -943,8 +915,8 @@ private static async Task ProcessDepotManifestAndFiles(Cancellat depotDownloadInfo = depot, depotCounter = depotCounter, stagingDir = stagingDir, - manifest = newProtoManifest, - previousManifest = oldProtoManifest, + manifest = newManifest, + previousManifest = oldManifest, filteredFiles = filesAfterExclusions, allFileNames = allFileNames }; @@ -959,7 +931,7 @@ private static async Task DownloadSteam3AsyncDepotFiles(CancellationTokenSource Console.WriteLine("Downloading depot {0}", depot.DepotId); var files = depotFilesData.filteredFiles.Where(f => !f.Flags.HasFlag(EDepotFileFlag.Directory)).ToArray(); - var networkChunkQueue = new ConcurrentQueue<(FileStreamData fileStreamData, ProtoManifest.FileData fileData, ProtoManifest.ChunkData chunk)>(); + var networkChunkQueue = new ConcurrentQueue<(FileStreamData fileStreamData, DepotManifest.FileData fileData, DepotManifest.ChunkData chunk)>(); await Util.InvokeAsync( files.Select(file => new Func(async () => @@ -1013,8 +985,8 @@ private static void DownloadSteam3AsyncDepotFile( CancellationTokenSource cts, GlobalDownloadCounter downloadCounter, DepotFilesData depotFilesData, - ProtoManifest.FileData file, - ConcurrentQueue<(FileStreamData, ProtoManifest.FileData, ProtoManifest.ChunkData)> networkChunkQueue) + DepotManifest.FileData file, + ConcurrentQueue<(FileStreamData, DepotManifest.FileData, DepotManifest.ChunkData)> networkChunkQueue) { cts.Token.ThrowIfCancellationRequested(); @@ -1022,7 +994,7 @@ private static void DownloadSteam3AsyncDepotFile( var stagingDir = depotFilesData.stagingDir; var depotDownloadCounter = depotFilesData.depotCounter; var oldProtoManifest = depotFilesData.previousManifest; - ProtoManifest.FileData oldManifestFile = null; + DepotManifest.FileData oldManifestFile = null; if (oldProtoManifest != null) { oldManifestFile = oldProtoManifest.Files.SingleOrDefault(f => f.FileName == file.FileName); @@ -1037,7 +1009,7 @@ private static void DownloadSteam3AsyncDepotFile( File.Delete(fileStagingPath); } - List neededChunks; + List neededChunks; var fi = new FileInfo(fileFinalPath); var fileDidExist = fi.Exists; if (!fileDidExist) @@ -1055,7 +1027,7 @@ private static void DownloadSteam3AsyncDepotFile( throw new ContentDownloaderException(string.Format("Failed to allocate file {0}: {1}", fileFinalPath, ex.Message)); } - neededChunks = new List(file.Chunks); + neededChunks = new List(file.Chunks); } else { @@ -1099,7 +1071,7 @@ private static void DownloadSteam3AsyncDepotFile( fsOld.Seek((long)match.OldChunk.Offset, SeekOrigin.Begin); var adler = Util.AdlerHash(fsOld, (int)match.OldChunk.UncompressedLength); - if (!adler.SequenceEqual(match.OldChunk.Checksum)) + if (!adler.SequenceEqual(BitConverter.GetBytes(match.OldChunk.Checksum))) { neededChunks.Add(match.NewChunk); } @@ -1131,7 +1103,7 @@ private static void DownloadSteam3AsyncDepotFile( fsOld.Seek((long)match.OldChunk.Offset, SeekOrigin.Begin); var tmp = new byte[match.OldChunk.UncompressedLength]; - fsOld.Read(tmp, 0, tmp.Length); + fsOld.ReadExactly(tmp); fs.Seek((long)match.NewChunk.Offset, SeekOrigin.Begin); fs.Write(tmp, 0, tmp.Length); @@ -1218,9 +1190,9 @@ private static async Task DownloadSteam3AsyncDepotFileChunk( CancellationTokenSource cts, GlobalDownloadCounter downloadCounter, DepotFilesData depotFilesData, - ProtoManifest.FileData file, + DepotManifest.FileData file, FileStreamData fileStreamData, - ProtoManifest.ChunkData chunk) + DepotManifest.ChunkData chunk) { cts.Token.ThrowIfCancellationRequested(); @@ -1229,17 +1201,8 @@ private static async Task DownloadSteam3AsyncDepotFileChunk( var chunkID = Convert.ToHexString(chunk.ChunkID).ToLowerInvariant(); - var data = new DepotManifest.ChunkData - { - ChunkID = chunk.ChunkID, - Checksum = BitConverter.ToUInt32(chunk.Checksum), - Offset = chunk.Offset, - CompressedLength = chunk.CompressedLength, - UncompressedLength = chunk.UncompressedLength - }; - var written = 0; - var chunkBuffer = ArrayPool.Shared.Rent((int)data.UncompressedLength); + var chunkBuffer = ArrayPool.Shared.Rent((int)chunk.UncompressedLength); try { @@ -1263,7 +1226,7 @@ private static async Task DownloadSteam3AsyncDepotFileChunk( DebugLog.WriteLine("ContentDownloader", "Downloading chunk {0} from {1} with {2}", chunkID, connection, cdnPool.ProxyServer != null ? cdnPool.ProxyServer : "no proxy"); written = await cdnPool.CDNClient.DownloadDepotChunkAsync( depot.DepotId, - data, + chunk, connection, chunkBuffer, depot.DepotKey, @@ -1332,7 +1295,7 @@ private static async Task DownloadSteam3AsyncDepotFileChunk( fileStreamData.fileStream = File.Open(fileFinalPath, FileMode.Open); } - fileStreamData.fileStream.Seek((long)data.Offset, SeekOrigin.Begin); + fileStreamData.fileStream.Seek((long)chunk.Offset, SeekOrigin.Begin); await fileStreamData.fileStream.WriteAsync(chunkBuffer.AsMemory(0, written), cts.Token); } finally @@ -1376,44 +1339,55 @@ private static async Task DownloadSteam3AsyncDepotFileChunk( } } - static void DumpManifestToTextFile(DepotDownloadInfo depot, ProtoManifest manifest) + class ChunkIdComparer : IEqualityComparer + { + public bool Equals(byte[] x, byte[] y) + { + if (ReferenceEquals(x, y)) return true; + if (x == null || y == null) return false; + return x.SequenceEqual(y); + } + + public int GetHashCode(byte[] obj) + { + ArgumentNullException.ThrowIfNull(obj); + + // ChunkID is SHA-1, so we can just use the first 4 bytes + return BitConverter.ToInt32(obj, 0); + } + } + + static void DumpManifestToTextFile(DepotDownloadInfo depot, DepotManifest manifest) { var txtManifest = Path.Combine(depot.InstallDir, $"manifest_{depot.DepotId}_{depot.ManifestId}.txt"); using var sw = new StreamWriter(txtManifest); - sw.WriteLine($"Content Manifest for Depot {depot.DepotId}"); + sw.WriteLine($"Content Manifest for Depot {depot.DepotId} "); sw.WriteLine(); - sw.WriteLine($"Manifest ID / date : {depot.ManifestId} / {manifest.CreationTime}"); + sw.WriteLine($"Manifest ID / date : {depot.ManifestId} / {manifest.CreationTime} "); - int numFiles = 0, numChunks = 0; - ulong uncompressedSize = 0, compressedSize = 0; + var uniqueChunks = new HashSet(new ChunkIdComparer()); foreach (var file in manifest.Files) { - if (file.Flags.HasFlag(EDepotFileFlag.Directory)) - continue; - - numFiles++; - numChunks += file.Chunks.Count; - foreach (var chunk in file.Chunks) { - uncompressedSize += chunk.UncompressedLength; - compressedSize += chunk.CompressedLength; + uniqueChunks.Add(chunk.ChunkID); } } - sw.WriteLine($"Total number of files : {numFiles}"); - sw.WriteLine($"Total number of chunks : {numChunks}"); - sw.WriteLine($"Total bytes on disk : {uncompressedSize}"); - sw.WriteLine($"Total bytes compressed : {compressedSize}"); + sw.WriteLine($"Total number of files : {manifest.Files.Count} "); + sw.WriteLine($"Total number of chunks : {uniqueChunks.Count} "); + sw.WriteLine($"Total bytes on disk : {manifest.TotalUncompressedSize} "); + sw.WriteLine($"Total bytes compressed : {manifest.TotalCompressedSize} "); + sw.WriteLine(); sw.WriteLine(); sw.WriteLine(" Size Chunks File SHA Flags Name"); foreach (var file in manifest.Files) { - var sha1Hash = BitConverter.ToString(file.FileHash).Replace("-", ""); - sw.WriteLine($"{file.TotalSize,14} {file.Chunks.Count,6} {sha1Hash} {file.Flags,5:D} {file.FileName}"); + var sha1Hash = Convert.ToHexString(file.FileHash).ToLower(); + sw.WriteLine($"{file.TotalSize,14:d} {file.Chunks.Count,6:d} {sha1Hash} {(int)file.Flags,5:x} {file.FileName}"); } } } diff --git a/DepotDownloader/DepotDownloaderMod.csproj b/DepotDownloader/DepotDownloaderMod.csproj index e2f561655..1c3b4e030 100644 --- a/DepotDownloader/DepotDownloaderMod.csproj +++ b/DepotDownloader/DepotDownloaderMod.csproj @@ -1,16 +1,17 @@ Exe - net8.0 + net9.0 true LatestMajor - 2.7.3 + 2.7.4 Steam Downloading Utility SteamRE Team Copyright © SteamRE Team 2024 ..\Icon\DepotDownloader.ico true true + true @@ -24,8 +25,8 @@ all runtime; build; native; contentfiles; analyzers; buildtransitive - + - + diff --git a/DepotDownloader/DownloadConfig.cs b/DepotDownloader/DownloadConfig.cs index ba151c21a..85d142c0c 100644 --- a/DepotDownloader/DownloadConfig.cs +++ b/DepotDownloader/DownloadConfig.cs @@ -10,6 +10,7 @@ class DownloadConfig { public int CellID { get; set; } public bool DownloadAllPlatforms { get; set; } + public bool DownloadAllArchs { get; set; } public bool DownloadAllLanguages { get; set; } public bool DownloadManifestOnly { get; set; } public string InstallDirectory { get; set; } diff --git a/DepotDownloader/NativeMethods.txt b/DepotDownloader/NativeMethods.txt index 8f2bb821f..571dcba11 100644 --- a/DepotDownloader/NativeMethods.txt +++ b/DepotDownloader/NativeMethods.txt @@ -1,2 +1,5 @@ +GetConsoleMode GetConsoleProcessList +GetStdHandle MessageBox +SetConsoleMode diff --git a/DepotDownloader/PlatformUtilities.cs b/DepotDownloader/PlatformUtilities.cs index a4f8fd77a..819ec411a 100644 --- a/DepotDownloader/PlatformUtilities.cs +++ b/DepotDownloader/PlatformUtilities.cs @@ -1,7 +1,6 @@ // This file is subject to the terms and conditions defined // in file 'LICENSE', which is part of this source code package. -using System; using System.IO; using System.Runtime.InteropServices; using System.Runtime.Versioning; diff --git a/DepotDownloader/Program.cs b/DepotDownloader/Program.cs index fc0c64d38..62cf21c52 100644 --- a/DepotDownloader/Program.cs +++ b/DepotDownloader/Program.cs @@ -246,6 +246,7 @@ ex is ContentDownloaderException ContentDownloader.Config.BetaPassword = GetParameter(args, "-betapassword"); ContentDownloader.Config.DownloadAllPlatforms = HasParameter(args, "-all-platforms"); + var os = GetParameter(args, "-os"); if (ContentDownloader.Config.DownloadAllPlatforms && !string.IsNullOrEmpty(os)) @@ -254,8 +255,16 @@ ex is ContentDownloaderException return 1; } + ContentDownloader.Config.DownloadAllArchs = HasParameter(args, "-all-archs"); + var arch = GetParameter(args, "-osarch"); + if (ContentDownloader.Config.DownloadAllArchs && !string.IsNullOrEmpty(arch)) + { + Console.WriteLine("Error: Cannot specify -osarch when -all-archs is specified."); + return 1; + } + ContentDownloader.Config.DownloadAllLanguages = HasParameter(args, "-all-languages"); var language = GetParameter(args, "-language"); @@ -436,6 +445,7 @@ static void PrintUsage() Console.WriteLine($" -beta - download from specified branch if available (default: {ContentDownloader.DEFAULT_BRANCH})."); Console.WriteLine(" -betapassword - branch password if applicable."); Console.WriteLine(" -all-platforms - downloads all platform-specific depots when -app is used."); + Console.WriteLine(" -all-archs - download all architecture-specific depots when -app is used."); Console.WriteLine(" -os - the operating system for which to download the game (windows, macos or linux, default: OS the program is currently running on)"); Console.WriteLine(" -osarch - the architecture for which to download the game (32 or 64, default: the host's architecture)"); Console.WriteLine(" -all-languages - download all language-specific depots when -app is used."); @@ -447,12 +457,14 @@ static void PrintUsage() Console.WriteLine(); Console.WriteLine(" -username - the username of the account to login to for restricted content."); Console.WriteLine(" -password - the password of the account to login to for restricted content."); - Console.WriteLine(" -remember-password - if set, remember the password for subsequent logins of this user. (Use -username -remember-password as login credentials)"); + Console.WriteLine(" -remember-password - if set, remember the password for subsequent logins of this user."); + Console.WriteLine(" use -username -remember-password as login credentials."); Console.WriteLine(); Console.WriteLine(" -dir - the directory in which to place downloaded files."); - Console.WriteLine(" -filelist - a list of files to download (from the manifest). Prefix file path with 'regex:' if you want to match with regex."); - Console.WriteLine(" -validate - Include checksum verification of files already downloaded"); + Console.WriteLine(" -filelist - the name of a local file that contains a list of files to download (from the manifest)."); + Console.WriteLine(" prefix file path with `regex:` if you want to match with regex. each file path should be on their own line."); Console.WriteLine(); + Console.WriteLine(" -validate - include checksum verification of files already downloaded"); Console.WriteLine(" -manifest-only - downloads a human readable manifest for any depots that would be downloaded."); Console.WriteLine(" -cellid <#> - the overridden CellID of the content server to download from."); Console.WriteLine(" -max-servers <#> - maximum number of content servers to use. (default: 20)."); diff --git a/DepotDownloader/ProtoManifest.cs b/DepotDownloader/ProtoManifest.cs index b6eb01001..de9046ae6 100644 --- a/DepotDownloader/ProtoManifest.cs +++ b/DepotDownloader/ProtoManifest.cs @@ -6,6 +6,7 @@ using System.IO; using System.IO.Compression; using System.Security.Cryptography; +using System.Text; using ProtoBuf; using SteamKit2; @@ -157,5 +158,39 @@ public void SaveToFile(string filename, out byte[] checksum) using var ds = new DeflateStream(fs, CompressionMode.Compress); ms.CopyTo(ds); } + + public DepotManifest ConvertToSteamManifest(uint depotId) + { + ulong uncompressedSize = 0, compressedSize = 0; + var newManifest = new DepotManifest(); + newManifest.Files = new List(Files.Count); + + foreach (var file in Files) + { + var fileNameHash = SHA1.HashData(Encoding.UTF8.GetBytes(file.FileName.Replace('/', '\\').ToLowerInvariant())); + var newFile = new DepotManifest.FileData(file.FileName, fileNameHash, file.Flags, file.TotalSize, file.FileHash, null, false, file.Chunks.Count); + + foreach (var chunk in file.Chunks) + { + var newChunk = new DepotManifest.ChunkData(chunk.ChunkID, BitConverter.ToUInt32(chunk.Checksum, 0), chunk.Offset, chunk.CompressedLength, chunk.UncompressedLength); + newFile.Chunks.Add(newChunk); + + uncompressedSize += chunk.UncompressedLength; + compressedSize += chunk.CompressedLength; + } + + newManifest.Files.Add(newFile); + } + + newManifest.FilenamesEncrypted = false; + newManifest.DepotID = depotId; + newManifest.ManifestGID = ID; + newManifest.CreationTime = CreationTime; + newManifest.TotalUncompressedSize = uncompressedSize; + newManifest.TotalCompressedSize = compressedSize; + newManifest.EncryptedCRC = 0; + + return newManifest; + } } } diff --git a/DepotDownloader/Steam3Session.cs b/DepotDownloader/Steam3Session.cs index 167b7fc84..9ebfe536c 100644 --- a/DepotDownloader/Steam3Session.cs +++ b/DepotDownloader/Steam3Session.cs @@ -39,12 +39,11 @@ public ReadOnlyCollection Licenses public SteamContent steamContent; readonly SteamApps steamApps; readonly SteamCloud steamCloud; - readonly SteamUnifiedMessages.UnifiedService steamPublishedFile; + readonly PublishedFile steamPublishedFile; readonly CallbackManager callbacks; readonly bool authenticatedUser; - bool bConnected; bool bConnecting; bool bAborted; bool bExpectingDisconnectRemote; @@ -52,15 +51,12 @@ public ReadOnlyCollection Licenses bool bIsConnectionRecovery; int connectionBackoff; int seq; // more hack fixes - DateTime connectTime; AuthSession authSession; + readonly CancellationTokenSource abortedToken = new(); // input readonly SteamUser.LogOnDetails logonDetails; - static readonly TimeSpan STEAM3_TIMEOUT = TimeSpan.FromSeconds(30); - - public Steam3Session(SteamUser.LogOnDetails details) { this.logonDetails = details; @@ -77,7 +73,7 @@ public Steam3Session(SteamUser.LogOnDetails details) this.steamApps = this.steamClient.GetHandler(); this.steamCloud = this.steamClient.GetHandler(); var steamUnifiedMessages = this.steamClient.GetHandler(); - this.steamPublishedFile = steamUnifiedMessages.CreateService(); + this.steamPublishedFile = steamUnifiedMessages.CreateService(); this.steamContent = this.steamClient.GetHandler(); this.callbacks = new CallbackManager(this.steamClient); @@ -93,7 +89,7 @@ public Steam3Session(SteamUser.LogOnDetails details) public delegate bool WaitCondition(); - private readonly object steamLock = new(); + private readonly Lock steamLock = new(); public bool WaitUntilCallback(Action submitter, WaitCondition waiter) { @@ -109,7 +105,7 @@ public bool WaitUntilCallback(Action submitter, WaitCondition waiter) { lock (steamLock) { - WaitForCallbacks(); + callbacks.RunWaitCallbacks(TimeSpan.FromSeconds(1)); } } while (!bAborted && this.seq == seq && !waiter()); } @@ -127,44 +123,51 @@ public bool WaitForCredentials() return IsLoggedOn; } - public void RequestAppInfo(uint appId, bool bForce = false) + public async Task TickCallbacks() + { + var token = abortedToken.Token; + + try + { + while (!token.IsCancellationRequested) + { + await callbacks.RunWaitCallbackAsync(token); + } + } + catch (OperationCanceledException) + { + // + } + } + + public async Task RequestAppInfo(uint appId, bool bForce = false) { if ((AppInfo.ContainsKey(appId) && !bForce) || bAborted) return; - var completed = false; + var appTokens = await steamApps.PICSGetAccessTokens([appId], []); - if (TokenCFG.useAppToken) + if (appTokens.AppTokensDenied.Contains(appId)) { - Console.WriteLine("Use App Token {0}", TokenCFG.appToken); + Console.WriteLine("Insufficient privileges to get access token for app {0}", appId); } - - if (!TokenCFG.useAppToken) - { - Action cbMethodTokens = appTokens => - { - completed = true; - if (appTokens.AppTokensDenied.Contains(appId)) - { - Console.WriteLine("Insufficient privileges to get access token for app {0}", appId); - } - foreach (var token_dict in appTokens.AppTokens) - { - this.AppTokens[token_dict.Key] = token_dict.Value; - } - }; - WaitUntilCallback(() => + foreach (var token_dict in appTokens.AppTokens) { - callbacks.Subscribe(steamApps.PICSGetAccessTokens(new List { appId }, new List()), cbMethodTokens); - }, () => { return completed; }); + this.AppTokens[token_dict.Key] = token_dict.Value; } - completed = false; - Action cbMethod = appInfo => + var request = new SteamApps.PICSRequest(appId); + + if (AppTokens.TryGetValue(appId, out var token)) { - completed = !appInfo.ResponsePending; + request.AccessToken = token; + } + var appInfoMultiple = await steamApps.PICSGetProductInfo([request], []); + + foreach (var appInfo in appInfoMultiple.Results) + { foreach (var app_value in appInfo.Apps) { var app = app_value.Value; @@ -177,132 +180,78 @@ public void RequestAppInfo(uint appId, bool bForce = false) { AppInfo[app] = null; } - }; - - if (TokenCFG.useAppToken) - { - var request = new SteamApps.PICSRequest(appId); - request.AccessToken = TokenCFG.appToken; - WaitUntilCallback(() => - { - callbacks.Subscribe(steamApps.PICSGetProductInfo(new List { request }, new List()), cbMethod); - }, () => { return completed; }); - } - else - { - var request = new SteamApps.PICSRequest(appId); - if (AppTokens.TryGetValue(appId, out var token)) - { - request.AccessToken = token; } - - WaitUntilCallback(() => - { - callbacks.Subscribe(steamApps.PICSGetProductInfo(new List { request }, new List()), cbMethod); - }, () => { return completed; }); - } } - public void RequestPackageInfo(IEnumerable packageIds) + public async Task RequestPackageInfo(IEnumerable packageIds) { var packages = packageIds.ToList(); - packages.RemoveAll(pid => PackageInfo.ContainsKey(pid)); + packages.RemoveAll(PackageInfo.ContainsKey); if (packages.Count == 0 || bAborted) return; - var completed = false; - Action cbMethod = packageInfo => - { - completed = !packageInfo.ResponsePending; - - foreach (var package_value in packageInfo.Packages) - { - var package = package_value.Value; - PackageInfo[package.ID] = package; - } - - foreach (var package in packageInfo.UnknownPackages) - { - PackageInfo[package] = null; - } - }; - var packageRequests = new List(); foreach (var package in packages) { var request = new SteamApps.PICSRequest(package); - if (TokenCFG.usePackageToken) - { - Console.WriteLine("Use App Token {0}", TokenCFG.packageToken); - } - - if (TokenCFG.usePackageToken) - { - request.AccessToken = TokenCFG.packageToken; - } - else - { if (PackageTokens.TryGetValue(package, out var token)) { request.AccessToken = token; } - } packageRequests.Add(request); } - WaitUntilCallback(() => + var packageInfoMultiple = await steamApps.PICSGetProductInfo([], packageRequests); + + foreach (var packageInfo in packageInfoMultiple.Results) { - callbacks.Subscribe(steamApps.PICSGetProductInfo(new List(), packageRequests), cbMethod); - }, () => { return completed; }); + foreach (var package_value in packageInfo.Packages) + { + var package = package_value.Value; + PackageInfo[package.ID] = package; + } + + foreach (var package in packageInfo.UnknownPackages) + { + PackageInfo[package] = null; + } + } } - public bool RequestFreeAppLicense(uint appId) + public async Task RequestFreeAppLicense(uint appId) { - var success = false; - var completed = false; - Action cbMethod = resultInfo => + try { - completed = true; - success = resultInfo.GrantedApps.Contains(appId); - }; + var resultInfo = await steamApps.RequestFreeLicense(appId); - WaitUntilCallback(() => + return resultInfo.GrantedApps.Contains(appId); + } + catch (Exception ex) { - callbacks.Subscribe(steamApps.RequestFreeLicense(appId), cbMethod); - }, () => { return completed; }); - - return success; + Console.WriteLine($"Failed to request FreeOnDemand license for app {appId}: {ex.Message}"); + return false; + } } - public void RequestDepotKey(uint depotId, uint appid = 0) + public async Task RequestDepotKey(uint depotId, uint appid = 0) { if (DepotKeys.ContainsKey(depotId) || bAborted) return; - var completed = false; - - Action cbMethod = depotKey => - { - completed = true; - Console.WriteLine("Got depot key for {0} result: {1}", depotKey.DepotID, depotKey.Result); - - if (depotKey.Result != EResult.OK) - { - Abort(); - return; - } + var depotKey = await steamApps.GetDepotDecryptionKey(depotId, appid); - DepotKeys[depotKey.DepotID] = depotKey.DepotKey; - }; + Console.WriteLine("Got depot key for {0} result: {1}", depotKey.DepotID, depotKey.Result); - WaitUntilCallback(() => + if (depotKey.Result != EResult.OK) { - callbacks.Subscribe(steamApps.GetDepotDecryptionKey(depotId, appid), cbMethod); - }, () => { return completed; }); + return; + } + + DepotKeys[depotKey.DepotID] = depotKey.DepotKey; } @@ -313,9 +262,14 @@ public async Task GetDepotManifestRequestCodeAsync(uint depotId, uint app var requestCode = await steamContent.GetManifestRequestCode(depotId, appId, manifestId, branch); - Console.WriteLine("Got manifest request code for {0} {1} result: {2}", - depotId, manifestId, - requestCode); + if (requestCode == 0) + { + Console.WriteLine($"No manifest request code was returned for depot {depotId} from app {appId}, manifest {manifestId}"); + } + else + { + Console.WriteLine($"Got manifest request code for depot {depotId} from app {appId}, manifest {manifestId}, result: {requestCode}"); + } return requestCode; } @@ -344,86 +298,48 @@ public async Task RequestCDNAuthToken(uint appid, uint depotid, Server server) completion.TrySetResult(cdnAuth); } - public void CheckAppBetaPassword(uint appid, string password) + public async Task CheckAppBetaPassword(uint appid, string password) { - var completed = false; - Action cbMethod = appPassword => - { - completed = true; - - Console.WriteLine("Retrieved {0} beta keys with result: {1}", appPassword.BetaPasswords.Count, appPassword.Result); + var appPassword = await steamApps.CheckAppBetaPassword(appid, password); - foreach (var entry in appPassword.BetaPasswords) - { - AppBetaPasswords[entry.Key] = entry.Value; - } - }; + Console.WriteLine("Retrieved {0} beta keys with result: {1}", appPassword.BetaPasswords.Count, appPassword.Result); - WaitUntilCallback(() => + foreach (var entry in appPassword.BetaPasswords) { - callbacks.Subscribe(steamApps.CheckAppBetaPassword(appid, password), cbMethod); - }, () => { return completed; }); + AppBetaPasswords[entry.Key] = entry.Value; + } } - public PublishedFileDetails GetPublishedFileDetails(uint appId, PublishedFileID pubFile) + public async Task GetPublishedFileDetails(uint appId, PublishedFileID pubFile) { var pubFileRequest = new CPublishedFile_GetDetails_Request { appid = appId }; pubFileRequest.publishedfileids.Add(pubFile); - var completed = false; - PublishedFileDetails details = null; + var details = await steamPublishedFile.GetDetails(pubFileRequest); - Action cbMethod = callback => + if (details.Result == EResult.OK) { - completed = true; - if (callback.Result == EResult.OK) - { - var response = callback.GetDeserializedResponse(); - details = response.publishedfiledetails.FirstOrDefault(); - } - else - { - throw new Exception($"EResult {(int)callback.Result} ({callback.Result}) while retrieving file details for pubfile {pubFile}."); - } - }; - - WaitUntilCallback(() => - { - callbacks.Subscribe(steamPublishedFile.SendMessage(api => api.GetDetails(pubFileRequest)), cbMethod); - }, () => { return completed; }); + return details.Body.publishedfiledetails.FirstOrDefault(); + } - return details; + throw new Exception($"EResult {(int)details.Result} ({details.Result}) while retrieving file details for pubfile {pubFile}."); } - public SteamCloud.UGCDetailsCallback GetUGCDetails(UGCHandle ugcHandle) + public async Task GetUGCDetails(UGCHandle ugcHandle) { - var completed = false; - SteamCloud.UGCDetailsCallback details = null; + var callback = await steamCloud.RequestUGCDetails(ugcHandle); - Action cbMethod = callback => + if (callback.Result == EResult.OK) { - completed = true; - if (callback.Result == EResult.OK) - { - details = callback; - } - else if (callback.Result == EResult.FileNotFound) - { - details = null; - } - else - { - throw new Exception($"EResult {(int)callback.Result} ({callback.Result}) while retrieving UGC details for {ugcHandle}."); - } - }; - - WaitUntilCallback(() => + return callback; + } + else if (callback.Result == EResult.FileNotFound) { - callbacks.Subscribe(steamCloud.RequestUGCDetails(ugcHandle), cbMethod); - }, () => { return completed; }); + return null; + } - return details; + throw new Exception($"EResult {(int)callback.Result} ({callback.Result}) while retrieving UGC details for {ugcHandle}."); } private void ResetConnectionFlags() @@ -436,14 +352,11 @@ private void ResetConnectionFlags() void Connect() { bAborted = false; - bConnected = false; bConnecting = true; connectionBackoff = 0; authSession = null; ResetConnectionFlags(); - - this.connectTime = DateTime.Now; this.steamClient.Connect(); } @@ -460,9 +373,9 @@ public void Disconnect(bool sendLogOff = true) } bAborted = true; - bConnected = false; bConnecting = false; bIsConnectionRecovery = false; + abortedToken.Cancel(); steamClient.Disconnect(); Ansi.Progress(Ansi.ProgressState.Hidden); @@ -480,28 +393,13 @@ private void Reconnect() steamClient.Disconnect(); } - private void WaitForCallbacks() - { - callbacks.RunWaitCallbacks(TimeSpan.FromSeconds(1)); - - var diff = DateTime.Now - connectTime; - - if (diff > STEAM3_TIMEOUT && !bConnected) - { - Console.WriteLine("Timeout connecting to Steam3."); - Abort(); - } - } - private async void ConnectedCallback(SteamClient.ConnectedCallback connected) { Console.WriteLine(" Done!"); bConnecting = false; - bConnected = true; // Update our tracking so that we don't time out, even if we need to reconnect multiple times, // e.g. if the authentication phase takes a while and therefore multiple connections. - connectTime = DateTime.Now; connectionBackoff = 0; if (!authenticatedUser) @@ -642,16 +540,18 @@ private void DisconnectedCallback(SteamClient.DisconnectedCallback disconnected) } else if (!bAborted) { + connectionBackoff += 1; + if (bConnecting) { - Console.WriteLine("Connection to Steam failed. Trying again"); + Console.WriteLine($"Connection to Steam failed. Trying again (#{connectionBackoff})..."); } else { Console.WriteLine("Lost connection to Steam. Reconnecting"); } - Thread.Sleep(1000 * ++connectionBackoff); + Thread.Sleep(1000 * connectionBackoff); // Any connection related flags need to be reset here to match the state after Connect ResetConnectionFlags(); diff --git a/DepotDownloader/Util.cs b/DepotDownloader/Util.cs index 46045ae46..b23eb8569 100644 --- a/DepotDownloader/Util.cs +++ b/DepotDownloader/Util.cs @@ -9,6 +9,7 @@ using System.Security.Cryptography; using System.Text; using System.Threading.Tasks; +using SteamKit2; namespace DepotDownloader { @@ -78,16 +79,16 @@ public static string ReadPassword() } // Validate a file against Steam3 Chunk data - public static List ValidateSteam3FileChecksums(FileStream fs, ProtoManifest.ChunkData[] chunkdata) + public static List ValidateSteam3FileChecksums(FileStream fs, DepotManifest.ChunkData[] chunkdata) { - var neededChunks = new List(); + var neededChunks = new List(); foreach (var data in chunkdata) { fs.Seek((long)data.Offset, SeekOrigin.Begin); var adler = AdlerHash(fs, (int)data.UncompressedLength); - if (!adler.SequenceEqual(data.Checksum)) + if (!adler.SequenceEqual(BitConverter.GetBytes(data.Checksum))) { neededChunks.Add(data); } @@ -110,6 +111,100 @@ public static byte[] AdlerHash(Stream stream, int length) return BitConverter.GetBytes(a | (b << 16)); } + public static byte[] FileSHAHash(string filename) + { + using (var fs = File.Open(filename, FileMode.Open)) + using (var sha = SHA1.Create()) + { + var output = sha.ComputeHash(fs); + + return output; + } + } + + public static DepotManifest LoadManifestFromFile(string directory, uint depotId, ulong manifestId, bool badHashWarning) + { + // Try loading Steam format manifest first. + var filename = Path.Combine(directory, string.Format("{0}_{1}.manifest", depotId, manifestId)); + + if (File.Exists(filename)) + { + byte[] expectedChecksum; + + try + { + expectedChecksum = File.ReadAllBytes(filename + ".sha"); + } + catch (IOException) + { + expectedChecksum = null; + } + + var currentChecksum = FileSHAHash(filename); + + if (expectedChecksum != null && expectedChecksum.SequenceEqual(currentChecksum)) + { + return DepotManifest.LoadFromFile(filename); + } + else if (badHashWarning) + { + Console.WriteLine("Manifest {0} on disk did not match the expected checksum.", manifestId); + } + } + + // Try converting legacy manifest format. + filename = Path.Combine(directory, string.Format("{0}_{1}.bin", depotId, manifestId)); + + if (File.Exists(filename)) + { + byte[] expectedChecksum; + + try + { + expectedChecksum = File.ReadAllBytes(filename + ".sha"); + } + catch (IOException) + { + expectedChecksum = null; + } + + byte[] currentChecksum; + var oldManifest = ProtoManifest.LoadFromFile(filename, out currentChecksum); + + if (oldManifest != null && (expectedChecksum == null || !expectedChecksum.SequenceEqual(currentChecksum))) + { + oldManifest = null; + + if (badHashWarning) + { + Console.WriteLine("Manifest {0} on disk did not match the expected checksum.", manifestId); + } + } + + if (oldManifest != null) + { + return oldManifest.ConvertToSteamManifest(depotId); + } + } + + return null; + } + + public static bool SaveManifestToFile(string directory, DepotManifest manifest) + { + try + { + var filename = Path.Combine(directory, string.Format("{0}_{1}.manifest", manifest.DepotID, manifest.ManifestGID)); + manifest.SaveToFile(filename); + File.WriteAllBytes(filename + ".sha", FileSHAHash(filename)); + return true; // If serialization completes without throwing an exception, return true + } + catch (Exception) + { + return false; // Return false if an error occurs + } + } + public static byte[] DecodeHexString(string hex) { if (hex == null) diff --git a/README.md b/README.md index abe9a5290..659fba8a1 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ DepotDownloaderMod =============== IMPROTANT: This Tool require a manifest file to work Due to GetManifestRequestCode Verification. -Steam depot downloader utilizing the SteamKit2 library with depot keys support and many other features. Supports .NET 8.0 +Steam depot downloader utilizing the SteamKit2 library with depot keys support and many other features. Supports .NET 9.0 Works with keys from SteamTools: @@ -20,52 +20,67 @@ dotnet DepotDownloader.dll -app -depotkeys [-depot [-m For example: `dotnet DepotDownloader.dll -app 730 -depot 731 -manifest 7617088375292372759 -depotkeys steam.keys -apptoken 1234567890123456789 -manifestfile 730_7617088375292372759.manifest` -### Downloading a workshop item using pubfile id +By default it will use anonymous account ([view which apps are available on it here](https://steamdb.info/sub/17906/)). -```(text) -dotnet DepotDownloader.dll -app -pubfile [-username [-password ]] +To use your account, specify the `-username ` parameter. Password will be asked interactively if you do +not use specify the `-password` parameter. + +### Downloading a workshop item using pubfile id +```powershell +./DepotDownloader -app -pubfile [-username [-password ]] ``` -For example: `dotnet DepotDownloader.dll -app 730 -pubfile 1885082371` +For example: `./DepotDownloadermod -app 730 -pubfile 1885082371` ### Downloading a workshop item using ugc id - -```(text) -dotnet DepotDownloader.dll -app -ugc [-username [-password ]] +```powershell +./DepotDownloader -app -ugc [-username [-password ]] ``` -For example: `dotnet DepotDownloader.dll -app 730 -ugc 770604181014286929` +For example: `./DepotDownloadermod -app 730 -ugc 770604181014286929` ## Parameters -Parameter | Description ---------- | ----------- --app \<#> | the AppID to download. --depot \<#> | the DepotID to download. --manifest \ | manifest id of content to download (requires -depot, default: current for branch). --ugc \<#> | the UGC ID to download. --beta \ | download from specified branch if available (default: Public). --betapassword \ | branch password if applicable. --all-platforms | downloads all platform-specific depots when -app is used. --os \ | the operating system for which to download the game (windows, macos or linux, default: OS the program is currently running on) --osarch \ | the architecture for which to download the game (32 or 64, default: the host's architecture) --all-languages | download all language-specific depots when -app is used. --language \ | the language for which to download the game (default: english) --lowviolence | download low violence depots when -app is used. --pubfile \<#> | the PublishedFileId to download. (Will automatically resolve to UGC id) --username \ | the username of the account to login to for restricted content. --password \ | the password of the account to login to for restricted content. --remember-password | if set, remember the password for subsequent logins of this user. (Use -username -remember-password as login credentials) +Parameter | Description +----------------------- | ----------- +`-app <#>` | the AppID to download. +`-depot <#>` | the DepotID to download. +`-manifest ` | manifest id of content to download (requires `-depot`, default: current for branch). +`-ugc <#>` | the UGC ID to download. +`-beta ` | download from specified branch if available (default: Public). +`-betapassword ` | branch password if applicable. +`-all-platforms` | downloads all platform-specific depots when `-app` is used. +`-os ` | the operating system for which to download the game (windows, macos or linux, default: OS the program is currently running on) +`-osarch ` | the architecture for which to download the game (32 or 64, default: the host's architecture) +`-all-archs` | download all architecture-specific depots when `-app` is used. +`-all-languages` | download all language-specific depots when `-app` is used. +`-language ` | the language for which to download the game (default: english) +`-lowviolence` | download low violence depots when `-app` is used. +`-pubfile <#>` | the PublishedFileId to download. (Will automatically resolve to UGC id) +`-username ` | the username of the account to login to for restricted content. +`-password ` | the password of the account to login to for restricted content. +`-remember-password` | if set, remember the password for subsequent logins of this user. (Use `-username -remember-password` as login credentials) -depotkeys | a list of depot keys to use ('depotID;hexKey' per line) -manifestfile | Use Specified Manifest file from Steam. -apptoken | Use Specified App Access Token -packagetoken | Use Specified Package Access Token --dir \ | the directory in which to place downloaded files. --filelist \ | a list of files to download (from the manifest). Prefix file path with `regex:` if you want to match with regex. --validate | Include checksum verification of files already downloaded --manifest-only | downloads a human readable manifest for any depots that would be downloaded. --cellid \<#> | the overridden CellID of the content server to download from. --max-servers \<#> | maximum number of content servers to use. (default: 20). --max-downloads \<#> | maximum number of chunks to download concurrently. (default: 8). --loginid \<#> | a unique 32-bit integer Steam LogonID in decimal, required if running multiple instances of DepotDownloader concurrently. - +`-dir ` | the directory in which to place downloaded files. +`-filelist ` | the name of a local file that contains a list of files to download (from the manifest). prefix file path with `regex:` if you want to match with regex. each file path should be on their own line. +`-validate` | Include checksum verification of files already downloaded +`-manifest-only` | downloads a human readable manifest for any depots that would be downloaded. +`-cellid <#>` | the overridden CellID of the content server to download from. +`-max-servers <#>` | maximum number of content servers to use. (default: 20). +`-max-downloads <#>` | maximum number of chunks to download concurrently. (default: 8). +`-loginid <#>` | a unique 32-bit integer Steam LogonID in decimal, required if running multiple instances of DepotDownloader concurrently. +`-V` or `--version` | print version and runtime + +## Frequently Asked Questions + +### Why am I prompted to enter a 2-factor code every time I run the app? +Your 2-factor code authenticates a Steam session. You need to "remember" your session with `-remember-password` which persists the login key for your Steam session. + +### Can I run DepotDownloader while an account is already connected to Steam? +Any connection to Steam will be closed if they share a LoginID. You can specify a different LoginID with `-loginid`. + +### Why doesn't my password containing special characters work? Do I have to specify the password on the command line? +If you pass the `-password` parameter with a password that contains special characters, you will need to escape the command appropriately for the shell you are using. You do not have to include the `-password` parameter on the command line as long as you include a `-username`. You will be prompted to enter your password interactively. diff --git a/global.json b/global.json index c19a2e057..2bc13e80a 100644 --- a/global.json +++ b/global.json @@ -1,6 +1,6 @@ { "sdk": { - "version": "8.0.100", + "version": "9.0.100", "rollForward": "latestMinor" } }