diff --git a/PuppeteerExtraSharp/.dockerignore b/PuppeteerExtraSharp/.dockerignore new file mode 100644 index 0000000..cd967fc --- /dev/null +++ b/PuppeteerExtraSharp/.dockerignore @@ -0,0 +1,25 @@ +**/.dockerignore +**/.env +**/.git +**/.gitignore +**/.project +**/.settings +**/.toolstarget +**/.vs +**/.vscode +**/.idea +**/*.*proj.user +**/*.dbmdl +**/*.jfm +**/azds.yaml +**/bin +**/charts +**/docker-compose* +**/Dockerfile* +**/node_modules +**/npm-debug.log +**/obj +**/secrets.dev.yaml +**/values.dev.yaml +LICENSE +README.md \ No newline at end of file diff --git a/PuppeteerExtraSharp/Plugins/AnonymizeUa/AnonymizeUaPlugin.cs b/PuppeteerExtraSharp/Plugins/AnonymizeUa/AnonymizeUaPlugin.cs index 32afbbf..5a104dc 100644 --- a/PuppeteerExtraSharp/Plugins/AnonymizeUa/AnonymizeUaPlugin.cs +++ b/PuppeteerExtraSharp/Plugins/AnonymizeUa/AnonymizeUaPlugin.cs @@ -17,7 +17,7 @@ public void CustomizeUa(Func uaAction) _customAction = uaAction; } - public override async Task OnPageCreated(Page page) + public override async Task OnPageCreated(IPage page) { var ua = await page.Browser.GetUserAgentAsync(); ua = ua.Replace("HeadlessChrome", "Chrome"); diff --git a/PuppeteerExtraSharp/Plugins/BlockResources/BlockResourcesPlugin.cs b/PuppeteerExtraSharp/Plugins/BlockResources/BlockResourcesPlugin.cs index 6ce7169..8a3b4c2 100644 --- a/PuppeteerExtraSharp/Plugins/BlockResources/BlockResourcesPlugin.cs +++ b/PuppeteerExtraSharp/Plugins/BlockResources/BlockResourcesPlugin.cs @@ -34,7 +34,7 @@ public BlockResourcesPlugin RemoveRule(BlockRule rule) } - public override async Task OnPageCreated(Page page) + public override async Task OnPageCreated(IPage page) { await page.SetRequestInterceptionAsync(true); page.Request += (sender, args) => OnPageRequest(page, args); @@ -42,7 +42,7 @@ public override async Task OnPageCreated(Page page) } - private async void OnPageRequest(Page sender, RequestEventArgs e) + private async void OnPageRequest(IPage sender, RequestEventArgs e) { if (BlockResources.Any(rule => rule.IsRequestBlocked(sender, e.Request))) { diff --git a/PuppeteerExtraSharp/Plugins/BlockResources/BlockRule.cs b/PuppeteerExtraSharp/Plugins/BlockResources/BlockRule.cs index cbc4879..16d5ace 100644 --- a/PuppeteerExtraSharp/Plugins/BlockResources/BlockRule.cs +++ b/PuppeteerExtraSharp/Plugins/BlockResources/BlockRule.cs @@ -7,7 +7,7 @@ namespace PuppeteerExtraSharp.Plugins.BlockResources public class BlockRule { public string SitePattern; - public Page Page; + public IPage IPage; public HashSet ResourceType = new HashSet(); internal BlockRule() @@ -15,7 +15,7 @@ internal BlockRule() } - public bool IsRequestBlocked(Page fromPage, Request request) + public bool IsRequestBlocked(IPage fromPage, IRequest request) { if (!IsResourcesBlocked(request.ResourceType)) return false; @@ -24,9 +24,9 @@ public bool IsRequestBlocked(Page fromPage, Request request) } - public bool IsPageBlocked(Page page) + public bool IsPageBlocked(IPage page) { - return Page != null && page.Equals(Page); + return IPage != null && page.Equals(IPage); } public bool IsSiteBlocked(string siteUrl) diff --git a/PuppeteerExtraSharp/Plugins/BlockResources/ResourcesBlockBuilder.cs b/PuppeteerExtraSharp/Plugins/BlockResources/ResourcesBlockBuilder.cs index 7b626f7..2e4b66e 100644 --- a/PuppeteerExtraSharp/Plugins/BlockResources/ResourcesBlockBuilder.cs +++ b/PuppeteerExtraSharp/Plugins/BlockResources/ResourcesBlockBuilder.cs @@ -16,9 +16,9 @@ public ResourcesBlockBuilder BlockedResources(params ResourceType[] resources) return this; } - public ResourcesBlockBuilder OnlyForPage(Page page) + public ResourcesBlockBuilder OnlyForPage(IPage page) { - Rule.Page = page; + Rule.IPage = page; return this; } diff --git a/PuppeteerExtraSharp/Plugins/ExtraStealth/Evasions/ChromeApp.cs b/PuppeteerExtraSharp/Plugins/ExtraStealth/Evasions/ChromeApp.cs index 7e90938..bc0e300 100644 --- a/PuppeteerExtraSharp/Plugins/ExtraStealth/Evasions/ChromeApp.cs +++ b/PuppeteerExtraSharp/Plugins/ExtraStealth/Evasions/ChromeApp.cs @@ -4,14 +4,14 @@ [assembly: InternalsVisibleTo("Extra.Tests")] namespace PuppeteerExtraSharp.Plugins.ExtraStealth.Evasions { - internal class ChromeApp : PuppeteerExtraPlugin + public class ChromeApp : PuppeteerExtraPlugin { public ChromeApp(): base("stealth-chromeApp") { } - public override Task OnPageCreated(Page page) + public override Task OnPageCreated(IPage page) { var script = Utils.GetScript("ChromeApp.js"); - return Utils.EvaluateOnNewPageWithUtilsScript(page, script); + return Utils.EvaluateOnNewPage(page, script); } } } diff --git a/PuppeteerExtraSharp/Plugins/ExtraStealth/Evasions/ChromeRuntime.cs b/PuppeteerExtraSharp/Plugins/ExtraStealth/Evasions/ChromeRuntime.cs index 13fbf38..b7b066c 100644 --- a/PuppeteerExtraSharp/Plugins/ExtraStealth/Evasions/ChromeRuntime.cs +++ b/PuppeteerExtraSharp/Plugins/ExtraStealth/Evasions/ChromeRuntime.cs @@ -3,14 +3,14 @@ namespace PuppeteerExtraSharp.Plugins.ExtraStealth.Evasions { - internal class ChromeRuntime: PuppeteerExtraPlugin + public class ChromeRuntime: PuppeteerExtraPlugin { public ChromeRuntime(): base("stealth-runtime") { } - public override Task OnPageCreated(Page page) + public override Task OnPageCreated(IPage page) { var script = Utils.GetScript("Runtime.js"); - return Utils.EvaluateOnNewPageWithUtilsScript(page, script); + return Utils.EvaluateOnNewPage(page, script); } } } diff --git a/PuppeteerExtraSharp/Plugins/ExtraStealth/Evasions/ChromeSci.cs b/PuppeteerExtraSharp/Plugins/ExtraStealth/Evasions/ChromeSci.cs index 84b3d20..dbeab80 100644 --- a/PuppeteerExtraSharp/Plugins/ExtraStealth/Evasions/ChromeSci.cs +++ b/PuppeteerExtraSharp/Plugins/ExtraStealth/Evasions/ChromeSci.cs @@ -3,14 +3,14 @@ namespace PuppeteerExtraSharp.Plugins.ExtraStealth.Evasions { - internal class ChromeSci: PuppeteerExtraPlugin + public class ChromeSci: PuppeteerExtraPlugin { public ChromeSci(): base("stealth_sci") { } - public override Task OnPageCreated(Page page) + public override Task OnPageCreated(IPage page) { var script = Utils.GetScript("SCI.js"); - return Utils.EvaluateOnNewPageWithUtilsScript(page, script); + return Utils.EvaluateOnNewPage(page, script); } } } diff --git a/PuppeteerExtraSharp/Plugins/ExtraStealth/Evasions/Codec.cs b/PuppeteerExtraSharp/Plugins/ExtraStealth/Evasions/Codec.cs index 51763d0..8257f55 100644 --- a/PuppeteerExtraSharp/Plugins/ExtraStealth/Evasions/Codec.cs +++ b/PuppeteerExtraSharp/Plugins/ExtraStealth/Evasions/Codec.cs @@ -3,14 +3,14 @@ namespace PuppeteerExtraSharp.Plugins.ExtraStealth.Evasions { - internal class Codec : PuppeteerExtraPlugin + public class Codec : PuppeteerExtraPlugin { public Codec() : base("stealth-codec") { } - public override Task OnPageCreated(Page page) + public override Task OnPageCreated(IPage page) { var script = Utils.GetScript("Codec.js"); - return Utils.EvaluateOnNewPageWithUtilsScript(page, script); + return Utils.EvaluateOnNewPage(page, script); } } } diff --git a/PuppeteerExtraSharp/Plugins/ExtraStealth/Evasions/ContentWindow.cs b/PuppeteerExtraSharp/Plugins/ExtraStealth/Evasions/ContentWindow.cs new file mode 100644 index 0000000..adb2a9c --- /dev/null +++ b/PuppeteerExtraSharp/Plugins/ExtraStealth/Evasions/ContentWindow.cs @@ -0,0 +1,22 @@ +using System.Collections.Generic; +using System.Threading.Tasks; +using PuppeteerSharp; + +namespace PuppeteerExtraSharp.Plugins.ExtraStealth.Evasions +{ + public class ContentWindow : PuppeteerExtraPlugin + { + public ContentWindow() : base("Iframe.ContentWindow") { } + + public override List Requirements { get; set; } = new() + { + PluginRequirements.RunLast + }; + + public override Task OnPageCreated(IPage page) + { + var script = Utils.GetScript("ContentWindow.js"); + return Utils.EvaluateOnNewPage(page, script); + } + } +} \ No newline at end of file diff --git a/PuppeteerExtraSharp/Plugins/ExtraStealth/Evasions/Frame.cs b/PuppeteerExtraSharp/Plugins/ExtraStealth/Evasions/Frame.cs deleted file mode 100644 index f06f4de..0000000 --- a/PuppeteerExtraSharp/Plugins/ExtraStealth/Evasions/Frame.cs +++ /dev/null @@ -1,22 +0,0 @@ -using System.Collections.Generic; -using System.Threading.Tasks; -using PuppeteerSharp; - -namespace PuppeteerExtraSharp.Plugins.ExtraStealth.Evasions -{ - internal class Frame: PuppeteerExtraPlugin - { - public Frame(): base("stealth-iframe") { } - - public override async Task OnPageCreated(Page page) - { - var script = Utils.GetScript("Frame.js"); - await page.EvaluateFunctionOnNewDocumentAsync(script); - } - - public override List Requirements { get; set; } = new List() - { - PluginRequirements.RunLast - }; - } -} diff --git a/PuppeteerExtraSharp/Plugins/ExtraStealth/Evasions/HardwareConcurrency.cs b/PuppeteerExtraSharp/Plugins/ExtraStealth/Evasions/HardwareConcurrency.cs index 6b93e89..066a3d0 100644 --- a/PuppeteerExtraSharp/Plugins/ExtraStealth/Evasions/HardwareConcurrency.cs +++ b/PuppeteerExtraSharp/Plugins/ExtraStealth/Evasions/HardwareConcurrency.cs @@ -3,7 +3,7 @@ namespace PuppeteerExtraSharp.Plugins.ExtraStealth.Evasions { - internal class HardwareConcurrency : PuppeteerExtraPlugin + public class HardwareConcurrency : PuppeteerExtraPlugin { public StealthHardwareConcurrencyOptions Options { get; } @@ -12,10 +12,10 @@ public HardwareConcurrency(StealthHardwareConcurrencyOptions options = null) : b Options = options ?? new StealthHardwareConcurrencyOptions(4); } - public override Task OnPageCreated(Page page) + public override Task OnPageCreated(IPage page) { var script = Utils.GetScript("HardwareConcurrency.js"); - return Utils.EvaluateOnNewPageWithUtilsScript(page, script, Options.Concurrency); + return Utils.EvaluateOnNewPage(page, script, Options.Concurrency); } } diff --git a/PuppeteerExtraSharp/Plugins/ExtraStealth/Evasions/Languages.cs b/PuppeteerExtraSharp/Plugins/ExtraStealth/Evasions/Languages.cs index f888bfa..2aa7d84 100644 --- a/PuppeteerExtraSharp/Plugins/ExtraStealth/Evasions/Languages.cs +++ b/PuppeteerExtraSharp/Plugins/ExtraStealth/Evasions/Languages.cs @@ -4,7 +4,7 @@ namespace PuppeteerExtraSharp.Plugins.ExtraStealth.Evasions { - internal class Languages : PuppeteerExtraPlugin + public class Languages : PuppeteerExtraPlugin { public StealthLanguagesOptions Options { get; } @@ -13,10 +13,10 @@ public Languages(StealthLanguagesOptions options = null) : base("stealth-languag Options = options ?? new StealthLanguagesOptions("en-US", "en"); } - public override Task OnPageCreated(Page page) + public override Task OnPageCreated(IPage page) { var script = Utils.GetScript("Language.js"); - return Utils.EvaluateOnNewPageWithUtilsScript(page,script, Options.Languages); + return Utils.EvaluateOnNewPage(page,script, Options.Languages); } } diff --git a/PuppeteerExtraSharp/Plugins/ExtraStealth/Evasions/LoadTimes.cs b/PuppeteerExtraSharp/Plugins/ExtraStealth/Evasions/LoadTimes.cs index 418cf88..44de829 100644 --- a/PuppeteerExtraSharp/Plugins/ExtraStealth/Evasions/LoadTimes.cs +++ b/PuppeteerExtraSharp/Plugins/ExtraStealth/Evasions/LoadTimes.cs @@ -7,10 +7,10 @@ public class LoadTimes : PuppeteerExtraPlugin { public LoadTimes() : base("stealth-loadTimes") { } - public override Task OnPageCreated(Page page) + public override Task OnPageCreated(IPage page) { var script = Utils.GetScript("LoadTimes.js"); - return Utils.EvaluateOnNewPageWithUtilsScript(page, script); + return Utils.EvaluateOnNewPage(page, script); } } } diff --git a/PuppeteerExtraSharp/Plugins/ExtraStealth/Evasions/OutDimensions.cs b/PuppeteerExtraSharp/Plugins/ExtraStealth/Evasions/OutDimensions.cs index d9d6532..de56e10 100644 --- a/PuppeteerExtraSharp/Plugins/ExtraStealth/Evasions/OutDimensions.cs +++ b/PuppeteerExtraSharp/Plugins/ExtraStealth/Evasions/OutDimensions.cs @@ -1,21 +1,17 @@ -using System.Threading.Tasks; +using System; +using System.Threading.Tasks; using PuppeteerSharp; namespace PuppeteerExtraSharp.Plugins.ExtraStealth.Evasions { - internal class OutDimensions : PuppeteerExtraPlugin + public class OutDimensions : PuppeteerExtraPlugin { public OutDimensions() : base("stealth-dimensions") { } - public override async Task OnPageCreated(Page page) + public override async Task OnPageCreated(IPage page) { var script = Utils.GetScript("Outdimensions.js"); await page.EvaluateFunctionOnNewDocumentAsync(script); } - - public override void BeforeLaunch(LaunchOptions options) - { - options.DefaultViewport = null; - } } } diff --git a/PuppeteerExtraSharp/Plugins/ExtraStealth/Evasions/Permissions.cs b/PuppeteerExtraSharp/Plugins/ExtraStealth/Evasions/Permissions.cs index 09b28fe..16b7313 100644 --- a/PuppeteerExtraSharp/Plugins/ExtraStealth/Evasions/Permissions.cs +++ b/PuppeteerExtraSharp/Plugins/ExtraStealth/Evasions/Permissions.cs @@ -3,14 +3,14 @@ namespace PuppeteerExtraSharp.Plugins.ExtraStealth.Evasions { - internal class Permissions: PuppeteerExtraPlugin + public class Permissions: PuppeteerExtraPlugin { public Permissions() : base("stealth-permissions") { } - public override Task OnPageCreated(Page page) + public override Task OnPageCreated(IPage page) { var script = Utils.GetScript("Permissions.js"); - return Utils.EvaluateOnNewPageWithUtilsScript(page, script); + return Utils.EvaluateOnNewPage(page, script); } } } diff --git a/PuppeteerExtraSharp/Plugins/ExtraStealth/Evasions/PluginEvasion.cs b/PuppeteerExtraSharp/Plugins/ExtraStealth/Evasions/PluginEvasion.cs index 2743ad9..5002615 100644 --- a/PuppeteerExtraSharp/Plugins/ExtraStealth/Evasions/PluginEvasion.cs +++ b/PuppeteerExtraSharp/Plugins/ExtraStealth/Evasions/PluginEvasion.cs @@ -3,14 +3,16 @@ namespace PuppeteerExtraSharp.Plugins.ExtraStealth.Evasions { - internal class PluginEvasion: PuppeteerExtraPlugin + public class PluginEvasion : PuppeteerExtraPlugin { - public PluginEvasion():base("stealth-pluginEvasion") { } + public PluginEvasion() : base("stealth-pluginEvasion") + { + } - public override async Task OnPageCreated(Page page) + public override async Task OnPageCreated(IPage page) { var scipt = Utils.GetScript("Plugin.js"); - await page.EvaluateFunctionOnNewDocumentAsync(scipt); + await Utils.EvaluateOnNewPage(page, scipt); } } -} +} \ No newline at end of file diff --git a/PuppeteerExtraSharp/Plugins/ExtraStealth/Evasions/SourceUrl.cs b/PuppeteerExtraSharp/Plugins/ExtraStealth/Evasions/SourceUrl.cs new file mode 100644 index 0000000..bbb2e5c --- /dev/null +++ b/PuppeteerExtraSharp/Plugins/ExtraStealth/Evasions/SourceUrl.cs @@ -0,0 +1,36 @@ +using System.Reflection; +using System.Threading.Tasks; +using PuppeteerSharp; + +namespace PuppeteerExtraSharp.Plugins.ExtraStealth.Evasions +{ + public class SourceUrl : PuppeteerExtraPlugin + { + public SourceUrl() : base("SourceUrl") + { + } + + public override async Task OnPageCreated(IPage page) + { + var mainWordProperty = + page.MainFrame.GetType().GetProperty("MainWorld", BindingFlags.NonPublic + | BindingFlags.Public | BindingFlags.Instance); + var mainWordGetters = mainWordProperty.GetGetMethod(true); + + page.Load += async (_, _) => + { + var mainWord = mainWordGetters.Invoke(page.MainFrame, null); + var contextField = mainWord.GetType() + .GetField("_contextResolveTaskWrapper", BindingFlags.NonPublic | BindingFlags.Instance); + if (contextField is not null) + { + var context = (TaskCompletionSource) contextField.GetValue(mainWord); + var execution = await context.Task; + var suffixField = execution.GetType() + .GetField("_evaluationScriptSuffix", BindingFlags.NonPublic | BindingFlags.Instance); + suffixField?.SetValue(execution, "//# sourceURL=''"); + } + }; + } + } +} \ No newline at end of file diff --git a/PuppeteerExtraSharp/Plugins/ExtraStealth/Evasions/StackTrace.cs b/PuppeteerExtraSharp/Plugins/ExtraStealth/Evasions/StackTrace.cs index b26a71d..a714d32 100644 --- a/PuppeteerExtraSharp/Plugins/ExtraStealth/Evasions/StackTrace.cs +++ b/PuppeteerExtraSharp/Plugins/ExtraStealth/Evasions/StackTrace.cs @@ -3,11 +3,11 @@ namespace PuppeteerExtraSharp.Plugins.ExtraStealth.Evasions { - internal class StackTrace : PuppeteerExtraPlugin + public class StackTrace : PuppeteerExtraPlugin { public StackTrace() : base("stealth-stackTrace") { } - public override async Task OnPageCreated(Page page) + public override async Task OnPageCreated(IPage page) { var script = Utils.GetScript("Stacktrace.js"); await page.EvaluateFunctionOnNewDocumentAsync(script); diff --git a/PuppeteerExtraSharp/Plugins/ExtraStealth/Evasions/UserAgent.cs b/PuppeteerExtraSharp/Plugins/ExtraStealth/Evasions/UserAgent.cs index 5809979..12300c4 100644 --- a/PuppeteerExtraSharp/Plugins/ExtraStealth/Evasions/UserAgent.cs +++ b/PuppeteerExtraSharp/Plugins/ExtraStealth/Evasions/UserAgent.cs @@ -1,18 +1,207 @@ -using System.Threading.Tasks; +using System.Collections.Generic; +using System.Linq; +using System.Text.RegularExpressions; +using System.Threading.Tasks; using PuppeteerSharp; namespace PuppeteerExtraSharp.Plugins.ExtraStealth.Evasions { - internal class UserAgent : PuppeteerExtraPlugin + public class UserAgent : PuppeteerExtraPlugin { - public UserAgent(): base("stealth-userAgent") { } + private bool _isHeadless = false; - public override async Task OnPageCreated(Page page) + public UserAgent() : base("stealth-userAgent") + { + } + + public override void BeforeLaunch(LaunchOptions options) + { + this._isHeadless = options.Headless; + } + + public override async Task OnPageCreated(IPage page) { var ua = await page.Browser.GetUserAgentAsync(); ua = ua.Replace("HeadlessChrome/", "Chrome/"); - await page.SetUserAgentAsync(ua); - + var uaVersion = ua.Contains("Chrome/") + ? Regex.Match(ua, @"Chrome\/([\d|.]+)").Groups[1].Value + : Regex.Match(await page.Browser.GetVersionAsync(), @"\/([\d|.]+)").Groups[1].Value; + + var platform = GetPlatform(ua); + var brand = GetBrands(uaVersion); + + var isMobile = GetIsMobile(ua); + var platformVersion = GetPlatformVersion(ua); + var platformArch = GetPlatformArch(isMobile); + var platformModel = GetPlatformModel(isMobile, ua); + + var overrideObject = new OverrideUserAgent() + { + UserAgent = ua, + Platform = platform, + AcceptLanguage = "en-US, en", + UserAgentMetadata = new UserAgentMetadata() + { + Brands = brand, + FullVersion = uaVersion, + Platform = platform, + PlatformVersion = platformVersion, + Architecture = platformArch, + Model = platformModel, + Mobile = isMobile + } + }; + // + // if (this._isHeadless) + // { + // var dynamicObject = overrideObject as dynamic; + // dynamicObject.AcceptLanguage = "en-US, en"; + // overrideObject = dynamicObject; + // } + + await page.Client.SendAsync("Network.setUserAgentOverride", overrideObject); + } + + private string GetPlatform(string ua) + { + if (ua.Contains("Mac OS X")) + { + return "Mac OS X"; + } + + if (ua.Contains("Android")) + { + return "Android"; + } + + if (ua.Contains("Linux")) + { + return "Linux"; + } + + return "Windows"; + } + + public string GetPlatformVersion(string ua) + { + if (ua.Contains("Mac OS X ")) + { + return Regex.Match(ua, "Mac OS X ([^)]+)").Groups[1].Value; + } + + if (ua.Contains("Android ")) + { + return Regex.Match(ua, "Android ([^;]+)").Groups[1].Value; + } + + if (ua.Contains("Windows ")) + { + return Regex.Match(ua, @"Windows .*?([\d|.]+);").Groups[1].Value; + } + + return string.Empty; + } + + public string GetPlatformArch(bool isMobile) + { + return isMobile ? string.Empty : "x86"; + } + + public string GetPlatformModel(bool isMobile, string ua) + { + return isMobile ? Regex.Match(ua, @"Android.*?;\s([^)]+)").Groups[1].Value : string.Empty; + } + + public bool GetIsMobile(string ua) + { + return ua.Contains("Android"); + } + + private List GetBrands(string uaVersion) + { + var seed = int.Parse(uaVersion.Split('.')[0]); + + var order = new List> + { + new List() + { + 0, 1, 2 + }, + new List() + { + 0, 2, 1 + }, + new List() + { + 1, 0, 2 + }, + new List() + { + 1, 2, 0 + }, + new List() + { + 2, 0, 1 + }, + new List() + { + 2, 1, 0 + }, + }[seed % 6]; + + var escapedChars = new List() + { + " ", + " ", + ";" + }; + + var greaseyBrand = $"{escapedChars[order[0]]}Not{escapedChars[order[1]]}A{escapedChars[order[2]]}Brand"; + var greasedBrandVersionList = new Dictionary(); + + greasedBrandVersionList.Add(order[0], new UserAgentBrand() + { + Brand = greaseyBrand, + Version = "99" + }); + greasedBrandVersionList.Add(order[1], new UserAgentBrand() + { + Brand = "Chromium", + Version = seed.ToString() + }); + greasedBrandVersionList.Add(order[2], new UserAgentBrand() + { + Brand = "Google Chrome", + Version = seed.ToString() + }); + + return greasedBrandVersionList.OrderBy(e=>e.Key).Select(e=>e.Value).ToList(); + } + + private class OverrideUserAgent + { + public string UserAgent { get; set; } + public string Platform { get; set; } + public string AcceptLanguage { get; set; } + public UserAgentMetadata UserAgentMetadata { get; set; } + } + + private class UserAgentMetadata + { + public List Brands { get; set; } + public string FullVersion { get; set; } + public string Platform { get; set; } + public string PlatformVersion { get; set; } + public string Architecture { get; set; } + public string Model { get; set; } + public bool Mobile { get; set; } + } + + private class UserAgentBrand + { + public string Brand { get; set; } + public string Version { get; set; } } } -} + +} \ No newline at end of file diff --git a/PuppeteerExtraSharp/Plugins/ExtraStealth/Evasions/Vendor.cs b/PuppeteerExtraSharp/Plugins/ExtraStealth/Evasions/Vendor.cs index af11b2c..f978896 100644 --- a/PuppeteerExtraSharp/Plugins/ExtraStealth/Evasions/Vendor.cs +++ b/PuppeteerExtraSharp/Plugins/ExtraStealth/Evasions/Vendor.cs @@ -4,15 +4,15 @@ namespace PuppeteerExtraSharp.Plugins.ExtraStealth.Evasions { - internal class Vendor : PuppeteerExtraPlugin + public class Vendor : PuppeteerExtraPlugin { private readonly StealthVendorSettings _settings; - public Vendor(StealthVendorSettings settings) : base("stealth-vendor") + public Vendor(StealthVendorSettings settings = null) : base("stealth-vendor") { _settings = settings ?? new StealthVendorSettings("Google Inc."); } - public override async Task OnPageCreated(Page page) + public override async Task OnPageCreated(IPage page) { var script = Utils.GetScript("Vendor.js"); await page.EvaluateFunctionOnNewDocumentAsync(script, _settings.Vendor); diff --git a/PuppeteerExtraSharp/Plugins/ExtraStealth/Evasions/WebDriver.cs b/PuppeteerExtraSharp/Plugins/ExtraStealth/Evasions/WebDriver.cs index eb525e0..6105b19 100644 --- a/PuppeteerExtraSharp/Plugins/ExtraStealth/Evasions/WebDriver.cs +++ b/PuppeteerExtraSharp/Plugins/ExtraStealth/Evasions/WebDriver.cs @@ -4,11 +4,11 @@ namespace PuppeteerExtraSharp.Plugins.ExtraStealth.Evasions { - internal class WebDriver : PuppeteerExtraPlugin + public class WebDriver : PuppeteerExtraPlugin { public WebDriver() : base("stealth-webDriver") { } - public override async Task OnPageCreated(Page page) + public override async Task OnPageCreated(IPage page) { var script = Utils.GetScript("WebDriver.js"); await page.EvaluateFunctionOnNewDocumentAsync(script); @@ -20,7 +20,8 @@ public override void BeforeLaunch(LaunchOptions options) var idx = args.FindIndex(e => e.StartsWith("--disable-blink-features=")); if (idx != -1) { - args[idx] = $"{idx}, AutomationControlled"; + var arg = args[idx]; + args[idx] = $"{arg}, AutomationControlled"; return; } diff --git a/PuppeteerExtraSharp/Plugins/ExtraStealth/Evasions/WebGl.cs b/PuppeteerExtraSharp/Plugins/ExtraStealth/Evasions/WebGl.cs index b46beed..34e184f 100644 --- a/PuppeteerExtraSharp/Plugins/ExtraStealth/Evasions/WebGl.cs +++ b/PuppeteerExtraSharp/Plugins/ExtraStealth/Evasions/WebGl.cs @@ -3,7 +3,7 @@ namespace PuppeteerExtraSharp.Plugins.ExtraStealth.Evasions { - internal class WebGl : PuppeteerExtraPlugin + public class WebGl : PuppeteerExtraPlugin { private readonly StealthWebGLOptions _options; public WebGl(StealthWebGLOptions options) : base("stealth-webGl") @@ -11,7 +11,7 @@ public WebGl(StealthWebGLOptions options) : base("stealth-webGl") _options = options ?? new StealthWebGLOptions("Intel Inc.", "Intel Iris OpenGL Engine"); } - public override async Task OnPageCreated(Page page) + public override async Task OnPageCreated(IPage page) { var script = Utils.GetScript("WebGL.js"); await page.EvaluateFunctionOnNewDocumentAsync(script, _options.Vendor, _options.Renderer); diff --git a/PuppeteerExtraSharp/Plugins/ExtraStealth/Scripts/ChromeApp.js b/PuppeteerExtraSharp/Plugins/ExtraStealth/Scripts/ChromeApp.js index b99207f..7028952 100644 --- a/PuppeteerExtraSharp/Plugins/ExtraStealth/Scripts/ChromeApp.js +++ b/PuppeteerExtraSharp/Plugins/ExtraStealth/Scripts/ChromeApp.js @@ -1,5 +1,4 @@ () => { - console.warn('testtt'); if (!window.chrome) { // Use the exact property descriptor found in headful Chrome // fetch it via `Object.getOwnPropertyDescriptor(window, 'chrome')` diff --git a/PuppeteerExtraSharp/Plugins/ExtraStealth/Scripts/ContentWindow.js b/PuppeteerExtraSharp/Plugins/ExtraStealth/Scripts/ContentWindow.js new file mode 100644 index 0000000..f3abe77 --- /dev/null +++ b/PuppeteerExtraSharp/Plugins/ExtraStealth/Scripts/ContentWindow.js @@ -0,0 +1,99 @@ +() => { + try { + // Adds a contentWindow proxy to the provided iframe element + const addContentWindowProxy = iframe => { + const contentWindowProxy = { + get(target, key) { + // Now to the interesting part: + // We actually make this thing behave like a regular iframe window, + // by intercepting calls to e.g. `.self` and redirect it to the correct thing. :) + // That makes it possible for these assertions to be correct: + // iframe.contentWindow.self === window.top // must be false + if (key === 'self') { + return this + } + // iframe.contentWindow.frameElement === iframe // must be true + if (key === 'frameElement') { + return iframe + } + return Reflect.get(target, key) + } + } + + if (!iframe.contentWindow) { + const proxy = new Proxy(window, contentWindowProxy) + Object.defineProperty(iframe, 'contentWindow', { + get() { + return proxy + }, + set(newValue) { + return newValue // contentWindow is immutable + }, + enumerable: true, + configurable: false + }) + } + } + + // Handles iframe element creation, augments `srcdoc` property so we can intercept further + const handleIframeCreation = (target, thisArg, args) => { + const iframe = target.apply(thisArg, args) + + // We need to keep the originals around + const _iframe = iframe + const _srcdoc = _iframe.srcdoc + + // Add hook for the srcdoc property + // We need to be very surgical here to not break other iframes by accident + Object.defineProperty(iframe, 'srcdoc', { + configurable: true, // Important, so we can reset this later + get: function () { + return _iframe.srcdoc + }, + set: function (newValue) { + addContentWindowProxy(this) + // Reset property, the hook is only needed once + Object.defineProperty(iframe, 'srcdoc', { + configurable: false, + writable: false, + value: _srcdoc + }) + _iframe.srcdoc = newValue + } + }) + return iframe + } + + // Adds a hook to intercept iframe creation events + const addIframeCreationSniffer = () => { + /* global document */ + const createElementHandler = { + // Make toString() native + get(target, key) { + return Reflect.get(target, key) + }, + apply: function (target, thisArg, args) { + const isIframe = + args && args.length && `${args[0]}`.toLowerCase() === 'iframe' + if (!isIframe) { + // Everything as usual + return target.apply(thisArg, args) + } else { + return handleIframeCreation(target, thisArg, args) + } + } + } + // All this just due to iframes with srcdoc bug + utils.replaceWithProxy( + document, + 'createElement', + createElementHandler + ) + } + + // Let's go + addIframeCreationSniffer() + } catch (err) { + // console.warn(err) + } +} \ No newline at end of file diff --git a/PuppeteerExtraSharp/Plugins/ExtraStealth/Scripts/Frame.js b/PuppeteerExtraSharp/Plugins/ExtraStealth/Scripts/Frame.js deleted file mode 100644 index b00663f..0000000 --- a/PuppeteerExtraSharp/Plugins/ExtraStealth/Scripts/Frame.js +++ /dev/null @@ -1,98 +0,0 @@ -() => { - try { - // Adds a contentWindow proxy to the provided iframe element - const addContentWindowProxy = iframe => { - const contentWindowProxy = { - get(target, key) { - // Now to the interesting part: - // We actually make this thing behave like a regular iframe window, - // by intercepting calls to e.g. `.self` and redirect it to the correct thing. :) - // That makes it possible for these assertions to be correct: - // iframe.contentWindow.self === window.top // must be false - if (key === 'self') { - return this - } - // iframe.contentWindow.frameElement === iframe // must be true - if (key === 'frameElement') { - return iframe - } - return Reflect.get(target, key) - } - } - - if (!iframe.contentWindow) { - const proxy = new Proxy(window, contentWindowProxy) - Object.defineProperty(iframe, 'contentWindow', { - get() { - return proxy - }, - set(newValue) { - return newValue // contentWindow is immutable - }, - enumerable: true, - configurable: false - }) - } - } - - // Handles iframe element creation, augments `srcdoc` property so we can intercept further - const handleIframeCreation = (target, thisArg, args) => { - const iframe = target.apply(thisArg, args) - - // We need to keep the originals around - const _iframe = iframe - const _srcdoc = _iframe.srcdoc - - // Add hook for the srcdoc property - // We need to be very surgical here to not break other iframes by accident - Object.defineProperty(iframe, 'srcdoc', { - configurable: true, // Important, so we can reset this later - get: function() { - return _iframe.srcdoc - }, - set: function(newValue) { - addContentWindowProxy(this) - // Reset property, the hook is only needed once - Object.defineProperty(iframe, 'srcdoc', { - configurable: false, - writable: false, - value: _srcdoc - }) - _iframe.srcdoc = newValue - } - }) - return iframe - } - - // Adds a hook to intercept iframe creation events - const addIframeCreationSniffer = () => { - /* global document */ - const createElement = { - // Make toString() native - get(target, key) { - return Reflect.get(target, key) - }, - apply: function(target, thisArg, args) { - const isIframe = - args && args.length && `${args[0]}`.toLowerCase() === 'iframe' - if (!isIframe) { - // Everything as usual - return target.apply(thisArg, args) - } else { - return handleIframeCreation(target, thisArg, args) - } - } - } - // All this just due to iframes with srcdoc bug - document.createElement = new Proxy( - document.createElement, - createElement - ) - } - - // Let's go - addIframeCreationSniffer() - } catch (err) { - // console.warn(err) - } - } \ No newline at end of file diff --git a/PuppeteerExtraSharp/Plugins/ExtraStealth/Scripts/Plugin.js b/PuppeteerExtraSharp/Plugins/ExtraStealth/Scripts/Plugin.js index 756c84f..8c5d570 100644 --- a/PuppeteerExtraSharp/Plugins/ExtraStealth/Scripts/Plugin.js +++ b/PuppeteerExtraSharp/Plugins/ExtraStealth/Scripts/Plugin.js @@ -1,195 +1,308 @@ () => { - function mockPluginsAndMimeTypes() { - /* global MimeType MimeTypeArray PluginArray */ - - // Disguise custom functions as being native - const makeFnsNative = (fns = []) => { - const oldCall = Function.prototype.call - function call() { - return oldCall.apply(this, arguments) - } - // eslint-disable-next-line - Function.prototype.call = call + function generateFunctionMocks() { + return ( + proto, + itemMainProp, + dataArray + ) => ({ + /** Returns the MimeType object with the specified index. */ + item: utils.createProxy(proto.item, { + apply(target, ctx, args) { + if (!args.length) { + throw new TypeError( + `Failed to execute 'item' on '${ + proto[Symbol.toStringTag] + }': 1 argument required, but only 0 present.` + ) + } + // Special behavior alert: + // - Vanilla tries to cast strings to Numbers (only integers!) and use them as property index lookup + // - If anything else than an integer (including as string) is provided it will return the first entry + const isInteger = args[0] && Number.isInteger(Number(args[0])) // Cast potential string to number first, then check for integer + // Note: Vanilla never returns `undefined` + return (isInteger ? dataArray[Number(args[0])] : dataArray[0]) || null + } + }), + /** Returns the MimeType object with the specified name. */ + namedItem: utils.createProxy(proto.namedItem, { + apply(target, ctx, args) { + if (!args.length) { + throw new TypeError( + `Failed to execute 'namedItem' on '${ + proto[Symbol.toStringTag] + }': 1 argument required, but only 0 present.` + ) + } + return dataArray.find(mt => mt[itemMainProp] === args[0]) || null // Not `undefined`! + } + }), + /** Does nothing and shall return nothing */ + refresh: proto.refresh + ? utils.createProxy(proto.refresh, { + apply(target, ctx, args) { + return undefined + } + }) + : undefined + }) + } - const nativeToStringFunctionString = Error.toString().replace( - /Error/g, - 'toString' - ) - const oldToString = Function.prototype.toString + function generateMagicArray() { + return ( + dataArray = [], + proto = MimeTypeArray.prototype, + itemProto = MimeType.prototype, + itemMainProp = 'type' + ) => { + // Quick helper to set props with the same descriptors vanilla is using + const defineProp = (obj, prop, value) => + Object.defineProperty(obj, prop, { + value, + writable: false, + enumerable: false, // Important for mimeTypes & plugins: `JSON.stringify(navigator.mimeTypes)` + configurable: true + }) - function functionToString() { - for (const fn of fns) { - if (this === fn.ref) { - return `function ${fn.name}() { [native code] }` + // Loop over our fake data and construct items + const makeItem = data => { + const item = {} + for (const prop of Object.keys(data)) { + if (prop.startsWith('__')) { + continue } + defineProp(item, prop, data[prop]) } + return patchItem(item, data) + } - if (this === functionToString) { - return nativeToStringFunctionString + const patchItem = (item, data) => { + let descriptor = Object.getOwnPropertyDescriptors(item) + + // Special case: Plugins have a magic length property which is not enumerable + // e.g. `navigator.plugins[i].length` should always be the length of the assigned mimeTypes + if (itemProto === Plugin.prototype) { + descriptor = { + ...descriptor, + length: { + value: data.__mimeTypes.length, + writable: false, + enumerable: false, + configurable: true // Important to be able to use the ownKeys trap in a Proxy to strip `length` + } + } } - return oldCall.call(oldToString, this) + + // We need to spoof a specific `MimeType` or `Plugin` object + const obj = Object.create(itemProto, descriptor) + + // Virtually all property keys are not enumerable in vanilla + const blacklist = [...Object.keys(data), 'length', 'enabledPlugin'] + return new Proxy(obj, { + ownKeys(target) { + return Reflect.ownKeys(target).filter(k => !blacklist.includes(k)) + }, + getOwnPropertyDescriptor(target, prop) { + if (blacklist.includes(prop)) { + return undefined + } + return Reflect.getOwnPropertyDescriptor(target, prop) + } + }) } - // eslint-disable-next-line - Function.prototype.toString = functionToString + + const magicArray = [] + + // Loop through our fake data and use that to create convincing entities + dataArray.forEach(data => { + magicArray.push(makeItem(data)) + }) + + // Add direct property access based on types (e.g. `obj['application/pdf']`) afterwards + magicArray.forEach(entry => { + defineProp(magicArray, entry[itemMainProp], entry) + }) + + // This is the best way to fake the type to make sure this is false: `Array.isArray(navigator.mimeTypes)` + const magicArrayObj = Object.create(proto, { + ...Object.getOwnPropertyDescriptors(magicArray), + + // There's one ugly quirk we unfortunately need to take care of: + // The `MimeTypeArray` prototype has an enumerable `length` property, + // but headful Chrome will still skip it when running `Object.getOwnPropertyNames(navigator.mimeTypes)`. + // To strip it we need to make it first `configurable` and can then overlay a Proxy with an `ownKeys` trap. + length: { + value: magicArray.length, + writable: false, + enumerable: false, + configurable: true // Important to be able to use the ownKeys trap in a Proxy to strip `length` + } + }) + + // Generate our functional function mocks :-) + const functionMocks = generateFunctionMocks()( + proto, + itemMainProp, + magicArray + ) + + // We need to overlay our custom object with a JS Proxy + const magicArrayObjProxy = new Proxy(magicArrayObj, { + get(target, key = '') { + // Redirect function calls to our custom proxied versions mocking the vanilla behavior + if (key === 'item') { + return functionMocks.item + } + if (key === 'namedItem') { + return functionMocks.namedItem + } + if (proto === PluginArray.prototype && key === 'refresh') { + return functionMocks.refresh + } + // Everything else can pass through as normal + return utils.cache.Reflect.get(...arguments) + }, + ownKeys(target) { + // There are a couple of quirks where the original property demonstrates "magical" behavior that makes no sense + // This can be witnessed when calling `Object.getOwnPropertyNames(navigator.mimeTypes)` and the absense of `length` + // My guess is that it has to do with the recent change of not allowing data enumeration and this being implemented weirdly + // For that reason we just completely fake the available property names based on our data to match what regular Chrome is doing + // Specific issues when not patching this: `length` property is available, direct `types` props (e.g. `obj['application/pdf']`) are missing + const keys = [] + const typeProps = magicArray.map(mt => mt[itemMainProp]) + typeProps.forEach((_, i) => keys.push(`${i}`)) + typeProps.forEach(propName => keys.push(propName)) + return keys + }, + getOwnPropertyDescriptor(target, prop) { + if (prop === 'length') { + return undefined + } + return Reflect.getOwnPropertyDescriptor(target, prop) + } + }) + + return magicArrayObjProxy + } + } + + function generateMimeTypeArray() { + return mimeTypesData => { + return generateMagicArray()( + mimeTypesData, + MimeTypeArray.prototype, + MimeType.prototype, + 'type' + ) + } + } + + function generatePluginArray() { + return pluginsData => { + return generateMagicArray()( + pluginsData, + PluginArray.prototype, + Plugin.prototype, + 'name' + ) } + } + - const mockedFns = [] + { + // That means we're running headful + const hasPlugins = 'plugins' in navigator && navigator.plugins.length + if (hasPlugins) { + return // nothing to do here + } - const fakeData = { - mimeTypes: [ + const dataMimeTypes = + [ { - type: 'application/pdf', - suffixes: 'pdf', - description: '', - __pluginName: 'Chrome PDF Viewer' + "type": "application/pdf", + "suffixes": "pdf", + "description": "", + "__pluginName": "Chrome PDF Viewer" }, { - type: 'application/x-google-chrome-pdf', - suffixes: 'pdf', - description: 'Portable Document Format', - __pluginName: 'Chrome PDF Plugin' + "type": "application/x-google-chrome-pdf", + "suffixes": "pdf", + "description": "Portable Document Format", + "__pluginName": "Chrome PDF Plugin" }, { - type: 'application/x-nacl', - suffixes: '', - description: 'Native Client Executable', - enabledPlugin: Plugin, - __pluginName: 'Native Client' + "type": "application/x-nacl", + "suffixes": "", + "description": "Native Client Executable", + "__pluginName": "Native Client" }, { - type: 'application/x-pnacl', - suffixes: '', - description: 'Portable Native Client Executable', - __pluginName: 'Native Client' + "type": "application/x-pnacl", + "suffixes": "", + "description": "Portable Native Client Executable", + "__pluginName": "Native Client" } - ], - plugins: [ + ]; + const dataPlugins = + [ { - name: 'Chrome PDF Plugin', - filename: 'internal-pdf-viewer', - description: 'Portable Document Format' + "name": "Chrome PDF Plugin", + "filename": "internal-pdf-viewer", + "description": "Portable Document Format", + "__mimeTypes": ["application/x-google-chrome-pdf"] }, { - name: 'Chrome PDF Viewer', - filename: 'mhjfbmdgcfjbbpaeojofohoefgiehjai', - description: '' + "name": "Chrome PDF Viewer", + "filename": "mhjfbmdgcfjbbpaeojofohoefgiehjai", + "description": "", + "__mimeTypes": ["application/pdf"] }, { - name: 'Native Client', - filename: 'internal-nacl-plugin', - description: '' - } - ], - fns: { - namedItem: instanceName => { - // Returns the Plugin/MimeType with the specified name. - const fn = function (name) { - if (!arguments.length) { - throw new TypeError( - `Failed to execute 'namedItem' on '${instanceName}': 1 argument required, but only 0 present.` - ) - } - return this[name] || null - } - mockedFns.push({ ref: fn, name: 'namedItem' }) - return fn - }, - item: instanceName => { - // Returns the Plugin/MimeType at the specified index into the array. - const fn = function (index) { - if (!arguments.length) { - throw new TypeError( - `Failed to execute 'namedItem' on '${instanceName}': 1 argument required, but only 0 present.` - ) - } - return this[index] || null - } - mockedFns.push({ ref: fn, name: 'item' }) - return fn - }, - refresh: instanceName => { - // Refreshes all plugins on the current page, optionally reloading documents. - const fn = function () { - return undefined - } - mockedFns.push({ ref: fn, name: 'refresh' }) - return fn + "name": "Native Client", + "filename": "internal-nacl-plugin", + "description": "", + "__mimeTypes": ["application/x-nacl", "application/x-pnacl"] } - } - } - // Poor mans _.pluck - const getSubset = (keys, obj) => - keys.reduce((a, c) => ({ ...a, [c]: obj[c] }), {}) - - function generateMimeTypeArray() { - const arr = fakeData.mimeTypes - .map(obj => getSubset(['type', 'suffixes', 'description'], obj)) - .map(obj => Object.setPrototypeOf(obj, MimeType.prototype)) - arr.forEach(obj => { - arr[obj.type] = obj - }) + ] - // Mock functions - arr.namedItem = fakeData.fns.namedItem('MimeTypeArray') - arr.item = fakeData.fns.item('MimeTypeArray') + const mimeTypes = generateMimeTypeArray()(dataMimeTypes) + const plugins = generatePluginArray()(dataPlugins) - return Object.setPrototypeOf(arr, MimeTypeArray.prototype) - } - - const mimeTypeArray = generateMimeTypeArray() - Object.defineProperty(navigator, 'mimeTypes', { - get: () => mimeTypeArray - }) + // Plugin and MimeType cross-reference each other, let's do that now + // Note: We're looping through `data.plugins` here, not the generated `plugins` + for (const pluginData of dataPlugins) { + pluginData.__mimeTypes.forEach((type, index) => { + plugins[pluginData.name][index] = mimeTypes[type] - function generatePluginArray() { - const arr = fakeData.plugins - .map(obj => getSubset(['name', 'filename', 'description'], obj)) - .map(obj => { - const mimes = fakeData.mimeTypes.filter( - m => m.__pluginName === obj.name - ) - // Add mimetypes - mimes.forEach((mime, index) => { - navigator.mimeTypes[mime.type].enabledPlugin = obj - obj[mime.type] = navigator.mimeTypes[mime.type] - obj[index] = navigator.mimeTypes[mime.type] - }) - obj.length = mimes.length - return obj + Object.defineProperty(plugins[pluginData.name], type, { + value: mimeTypes[type], + writable: false, + enumerable: false, // Not enumerable + configurable: true }) - .map(obj => { - // Mock functions - obj.namedItem = fakeData.fns.namedItem('Plugin') - obj.item = fakeData.fns.item('Plugin') - return obj + Object.defineProperty(mimeTypes[type], 'enabledPlugin', { + value: + type === 'application/x-pnacl' + ? mimeTypes['application/x-nacl'].enabledPlugin // these reference the same plugin, so we need to re-use the Proxy in order to avoid leaks + : new Proxy(plugins[pluginData.name], {}), // Prevent circular references + writable: false, + enumerable: false, // Important: `JSON.stringify(navigator.plugins)` + configurable: true }) - .map(obj => Object.setPrototypeOf(obj, Plugin.prototype)) - arr.forEach(obj => { - arr[obj.name] = obj }) - - // Mock functions - arr.namedItem = fakeData.fns.namedItem('PluginArray') - arr.item = fakeData.fns.item('PluginArray') - arr.refresh = fakeData.fns.refresh('PluginArray') - - return Object.setPrototypeOf(arr, PluginArray.prototype) } - const pluginArray = generatePluginArray() - Object.defineProperty(navigator, 'plugins', { - get: () => pluginArray - }) + const patchNavigator = (name, value) => + utils.replaceProperty(Object.getPrototypeOf(navigator), name, { + get() { + return value + } + }) - // Make mockedFns toString() representation resemble a native function - makeFnsNative(mockedFns) + patchNavigator('mimeTypes', mimeTypes) + patchNavigator('plugins', plugins) + + // All done } - try { - const isPluginArray = navigator.plugins instanceof PluginArray - const hasPlugins = isPluginArray && navigator.plugins.length > 0 - if (isPluginArray && hasPlugins) { - return // nothing to do here - } - mockPluginsAndMimeTypes() - } catch (err) { } + + } \ No newline at end of file diff --git a/PuppeteerExtraSharp/Plugins/ExtraStealth/Scripts/Runtime.js b/PuppeteerExtraSharp/Plugins/ExtraStealth/Scripts/Runtime.js index 1854ec2..bea3886 100644 --- a/PuppeteerExtraSharp/Plugins/ExtraStealth/Scripts/Runtime.js +++ b/PuppeteerExtraSharp/Plugins/ExtraStealth/Scripts/Runtime.js @@ -17,10 +17,48 @@ if (existsAlready || isNotSecure) { return // Nothing to do here } - window.chrome.runtime = { // There's a bunch of static data in that property which doesn't seem to change, // we should periodically check for updates: `JSON.stringify(window.chrome.runtime, null, 2)` + "OnInstalledReason": { + "CHROME_UPDATE": "chrome_update", + "INSTALL": "install", + "SHARED_MODULE_UPDATE": "shared_module_update", + "UPDATE": "update" + }, + "OnRestartRequiredReason": { + "APP_UPDATE": "app_update", + "OS_UPDATE": "os_update", + "PERIODIC": "periodic" + }, + "PlatformArch": { + "ARM": "arm", + "ARM64": "arm64", + "MIPS": "mips", + "MIPS64": "mips64", + "X86_32": "x86-32", + "X86_64": "x86-64" + }, + "PlatformNaclArch": { + "ARM": "arm", + "MIPS": "mips", + "MIPS64": "mips64", + "X86_32": "x86-32", + "X86_64": "x86-64" + }, + "PlatformOs": { + "ANDROID": "android", + "CROS": "cros", + "LINUX": "linux", + "MAC": "mac", + "OPENBSD": "openbsd", + "WIN": "win" + }, + "RequestUpdateCheckStatus": { + "NO_UPDATE": "no_update", + "THROTTLED": "throttled", + "UPDATE_AVAILABLE": "update_available" + }, // `chrome.runtime.id` is extension related and returns undefined in Chrome get id() { return undefined @@ -96,7 +134,8 @@ utils.mockWithProxy( window.chrome.runtime, 'sendMessage', - function sendMessage() { }, + function sendMessage() { + }, sendMessageHandler ) @@ -183,25 +222,31 @@ utils.mockWithProxy( window.chrome.runtime, 'connect', - function connect() { }, + function connect() { + }, connectHandler ) function makeConnectResponse() { const onSomething = () => ({ - addListener: function addListener() { }, - dispatch: function dispatch() { }, - hasListener: function hasListener() { }, + addListener: function addListener() { + }, + dispatch: function dispatch() { + }, + hasListener: function hasListener() { + }, hasListeners: function hasListeners() { return false }, - removeListener: function removeListener() { } + removeListener: function removeListener() { + } }) const response = { name: '', sender: undefined, - disconnect: function disconnect() { }, + disconnect: function disconnect() { + }, onDisconnect: onSomething(), onMessage: onSomething(), postMessage: function postMessage() { diff --git a/PuppeteerExtraSharp/Plugins/ExtraStealth/Scripts/Utils.js b/PuppeteerExtraSharp/Plugins/ExtraStealth/Scripts/Utils.js index 6f8d761..57fffd9 100644 --- a/PuppeteerExtraSharp/Plugins/ExtraStealth/Scripts/Utils.js +++ b/PuppeteerExtraSharp/Plugins/ExtraStealth/Scripts/Utils.js @@ -9,8 +9,12 @@ * Alternatively take a look at the `extract-stealth-evasions` package to create a finished bundle which includes these utilities. * */ +const utils = {} + +utils.init = () => { + utils.preloadCache() +} -const utils = {}; /** * Wraps a JS Proxy Handler and strips it's presence from error stacks, in case the traps throw. * @@ -19,18 +23,18 @@ const utils = {}; * @param {object} handler - The JS Proxy handler to wrap */ utils.stripProxyFromErrors = (handler = {}) => { - const newHandler = {}; + const newHandler = {} // We wrap each trap in the handler in a try/catch and modify the error stack if they throw - const traps = Object.getOwnPropertyNames(handler); + const traps = Object.getOwnPropertyNames(handler) traps.forEach(trap => { newHandler[trap] = function () { try { // Forward the call to the defined proxy handler - return handler[trap].apply(this, arguments || []); + return handler[trap].apply(this, arguments || []) } catch (err) { // Stack traces differ per browser, we only support chromium based ones currently if (!err || !err.stack || !err.stack.includes(`at `)) { - throw err; + throw err } // When something throws within one of our traps the Proxy will show up in error stacks @@ -39,47 +43,57 @@ utils.stripProxyFromErrors = (handler = {}) => { // We try to use a known "anchor" line for that and strip it with everything above it. // If the anchor line cannot be found for some reason we fall back to our blacklist approach. - const stripWithBlacklist = stack => { + const stripWithBlacklist = (stack, stripFirstLine = true) => { const blacklist = [ `at Reflect.${trap} `, // e.g. Reflect.get or Reflect.apply `at Object.${trap} `, // e.g. Object.get or Object.apply `at Object.newHandler. [as ${trap}] ` // caused by this very wrapper :-) - ]; + ] return ( err.stack .split('\n') // Always remove the first (file) line in the stack (guaranteed to be our proxy) - .filter((line, index) => index !== 1) + .filter((line, index) => !(index === 1 && stripFirstLine)) // Check if the line starts with one of our blacklisted strings .filter(line => !blacklist.some(bl => line.trim().startsWith(bl))) .join('\n') - ); - }; + ) + } - const stripWithAnchor = stack => { - const stackArr = stack.split('\n'); - const anchor = `at Object.newHandler. [as ${trap}] `; // Known first Proxy line in chromium + const stripWithAnchor = (stack, anchor) => { + const stackArr = stack.split('\n') + anchor = anchor || `at Object.newHandler. [as ${trap}] ` // Known first Proxy line in chromium const anchorIndex = stackArr.findIndex(line => line.trim().startsWith(anchor) - ); + ) if (anchorIndex === -1) { - return false; // 404, anchor not found + return false // 404, anchor not found } // Strip everything from the top until we reach the anchor line // Note: We're keeping the 1st line (zero index) as it's unrelated (e.g. `TypeError`) - stackArr.splice(1, anchorIndex); - return stackArr.join('\n'); - }; + stackArr.splice(1, anchorIndex) + return stackArr.join('\n') + } + + // Special cases due to our nested toString proxies + err.stack = err.stack.replace( + 'at Object.toString (', + 'at Function.toString (' + ) + if ((err.stack || '').includes('at Function.toString (')) { + err.stack = stripWithBlacklist(err.stack, false) + throw err + } // Try using the anchor method, fallback to blacklist if necessary - err.stack = stripWithAnchor(err.stack) || stripWithBlacklist(err.stack); + err.stack = stripWithAnchor(err.stack) || stripWithBlacklist(err.stack) - throw err; // Re-throw our now sanitized error + throw err // Re-throw our now sanitized error } - }; - }); - return newHandler; -}; + } + }) + return newHandler +} /** * Strip error lines from stack traces until (and including) a known line the stack. @@ -88,17 +102,17 @@ utils.stripProxyFromErrors = (handler = {}) => { * @param {string} anchor - The string the anchor line starts with */ utils.stripErrorWithAnchor = (err, anchor) => { - const stackArr = err.stack.split('\n'); - const anchorIndex = stackArr.findIndex(line => line.trim().startsWith(anchor)); + const stackArr = err.stack.split('\n') + const anchorIndex = stackArr.findIndex(line => line.trim().startsWith(anchor)) if (anchorIndex === -1) { - return err; // 404, anchor not found + return err // 404, anchor not found } // Strip everything from the top until we reach the anchor line (remove anchor line as well) // Note: We're keeping the 1st line (zero index) as it's unrelated (e.g. `TypeError`) - stackArr.splice(1, anchorIndex); - err.stack = stackArr.join('\n'); - return err; -}; + stackArr.splice(1, anchorIndex) + err.stack = stackArr.join('\n') + return err +} /** * Replace the property of an object in a stealthy way. @@ -123,8 +137,8 @@ utils.replaceProperty = (obj, propName, descriptorOverrides = {}) => { ...(Object.getOwnPropertyDescriptor(obj, propName) || {}), // Add our overrides (e.g. value, get()) ...descriptorOverrides - }); -}; + }) +} /** * Preload a cache of function copies and data. @@ -132,11 +146,14 @@ utils.replaceProperty = (obj, propName, descriptorOverrides = {}) => { * For a determined enough observer it would be possible to overwrite and sniff usage of functions * we use in our internal Proxies, to combat that we use a cached copy of those functions. * + * Note: Whenever we add a `Function.prototype.toString` proxy we should preload the cache before, + * by executing `utils.preloadCache()` before the proxy is applied (so we don't cause recursive lookups). + * * This is evaluated once per execution context (e.g. window) */ utils.preloadCache = () => { if (utils.cache) { - return; + return } utils.cache = { // Used in our proxies @@ -146,8 +163,8 @@ utils.preloadCache = () => { }, // Used in `makeNativeString` nativeToStringStr: Function.toString + '' // => `function toString() { [native code] }` - }; -}; + } +} /** * Utility function to generate a cross-browser `toString` result representing native code. @@ -158,19 +175,14 @@ utils.preloadCache = () => { * The only advantage we have over the other team is that our JS runs first, hence we cache the result * of the native toString result once, so they cannot spoof it afterwards and reveal that we're using it. * - * Note: Whenever we add a `Function.prototype.toString` proxy we should preload the cache before, - * by executing `utils.preloadCache()` before the proxy is applied (so we don't cause recursive lookups). - * * @example * makeNativeString('foobar') // => `function foobar() { [native code] }` * * @param {string} [name] - Optional function name */ utils.makeNativeString = (name = '') => { - // Cache (per-window) the original native toString or use that if available - utils.preloadCache(); - return utils.cache.nativeToStringStr.replace('toString', name || ''); -}; + return utils.cache.nativeToStringStr.replace('toString', name || '') +} /** * Helper function to modify the `toString()` result of the provided object. @@ -187,35 +199,38 @@ utils.makeNativeString = (name = '') => { * @param {string} str - Optional string used as a return value */ utils.patchToString = (obj, str = '') => { - utils.preloadCache(); - - const toStringProxy = new Proxy(Function.prototype.toString, { + const handler = { apply: function (target, ctx) { // This fixes e.g. `HTMLMediaElement.prototype.canPlayType.toString + ""` if (ctx === Function.prototype.toString) { - return utils.makeNativeString('toString'); + return utils.makeNativeString('toString') } // `toString` targeted at our proxied Object detected if (ctx === obj) { // We either return the optional string verbatim or derive the most desired result automatically - return str || utils.makeNativeString(obj.name); + return str || utils.makeNativeString(obj.name) } // Check if the toString protype of the context is the same as the global prototype, // if not indicates that we are doing a check across different windows., e.g. the iframeWithdirect` test case const hasSameProto = Object.getPrototypeOf( Function.prototype.toString - ).isPrototypeOf(ctx.toString); // eslint-disable-line no-prototype-builtins + ).isPrototypeOf(ctx.toString) // eslint-disable-line no-prototype-builtins if (!hasSameProto) { // Pass the call on to the local Function.prototype.toString instead - return ctx.toString(); + return ctx.toString() } - return target.call(ctx); + return target.call(ctx) } - }); + } + + const toStringProxy = new Proxy( + Function.prototype.toString, + utils.stripProxyFromErrors(handler) + ) utils.replaceProperty(Function.prototype, 'toString', { value: toStringProxy - }); -}; + }) +} /** * Make all nested functions of an object native. @@ -223,8 +238,8 @@ utils.patchToString = (obj, str = '') => { * @param {object} obj */ utils.patchToStringNested = (obj = {}) => { - return utils.execRecursively(obj, ['function'], utils.patchToString); -}; + return utils.execRecursively(obj, ['function'], utils.patchToString) +} /** * Redirect toString requests from one object to another. @@ -233,13 +248,11 @@ utils.patchToStringNested = (obj = {}) => { * @param {object} originalObj - The object which toString result we wan to return */ utils.redirectToString = (proxyObj, originalObj) => { - utils.preloadCache(); - - const toStringProxy = new Proxy(Function.prototype.toString, { + const handler = { apply: function (target, ctx) { // This fixes e.g. `HTMLMediaElement.prototype.canPlayType.toString + ""` if (ctx === Function.prototype.toString) { - return utils.makeNativeString('toString'); + return utils.makeNativeString('toString') } // `toString` targeted at our proxied Object detected @@ -247,29 +260,34 @@ utils.redirectToString = (proxyObj, originalObj) => { const fallback = () => originalObj && originalObj.name ? utils.makeNativeString(originalObj.name) - : utils.makeNativeString(proxyObj.name); + : utils.makeNativeString(proxyObj.name) // Return the toString representation of our original object if possible - return originalObj + '' || fallback(); + return originalObj + '' || fallback() } // Check if the toString protype of the context is the same as the global prototype, // if not indicates that we are doing a check across different windows., e.g. the iframeWithdirect` test case const hasSameProto = Object.getPrototypeOf( Function.prototype.toString - ).isPrototypeOf(ctx.toString); // eslint-disable-line no-prototype-builtins + ).isPrototypeOf(ctx.toString) // eslint-disable-line no-prototype-builtins if (!hasSameProto) { // Pass the call on to the local Function.prototype.toString instead - return ctx.toString(); + return ctx.toString() } - return target.call(ctx); + return target.call(ctx) } - }); + } + + const toStringProxy = new Proxy( + Function.prototype.toString, + utils.stripProxyFromErrors(handler) + ) utils.replaceProperty(Function.prototype, 'toString', { value: toStringProxy - }); -}; + }) +} /** * All-in-one method to replace a property with a JS Proxy using the provided Proxy handler with traps. @@ -285,15 +303,34 @@ utils.redirectToString = (proxyObj, originalObj) => { * @param {object} handler - The JS Proxy handler to use */ utils.replaceWithProxy = (obj, propName, handler) => { - utils.preloadCache(); - const originalObj = obj[propName]; - const proxyObj = new Proxy(obj[propName], utils.stripProxyFromErrors(handler)); + const originalObj = obj[propName] + const proxyObj = new Proxy(obj[propName], utils.stripProxyFromErrors(handler)) - utils.replaceProperty(obj, propName, { value: proxyObj }); - utils.redirectToString(proxyObj, originalObj); + utils.replaceProperty(obj, propName, { value: proxyObj }) + utils.redirectToString(proxyObj, originalObj) - return true; -}; + return true +} +/** + * All-in-one method to replace a getter with a JS Proxy using the provided Proxy handler with traps. + * + * @example + * replaceGetterWithProxy(Object.getPrototypeOf(navigator), 'vendor', proxyHandler) + * + * @param {object} obj - The object which has the property to replace + * @param {string} propName - The name of the property to replace + * @param {object} handler - The JS Proxy handler to use + */ +utils.replaceGetterWithProxy = (obj, propName, handler) => { + const fn = Object.getOwnPropertyDescriptor(obj, propName).get + const fnStr = fn.toString() // special getter function string + const proxyObj = new Proxy(fn, utils.stripProxyFromErrors(handler)) + + utils.replaceProperty(obj, propName, { get: proxyObj }) + utils.patchToString(proxyObj, fnStr) + + return true +} /** * All-in-one method to mock a non-existing property with a JS Proxy using the provided Proxy handler with traps. @@ -309,14 +346,13 @@ utils.replaceWithProxy = (obj, propName, handler) => { * @param {object} handler - The JS Proxy handler to use */ utils.mockWithProxy = (obj, propName, pseudoTarget, handler) => { - utils.preloadCache(); - const proxyObj = new Proxy(pseudoTarget, utils.stripProxyFromErrors(handler)); + const proxyObj = new Proxy(pseudoTarget, utils.stripProxyFromErrors(handler)) - utils.replaceProperty(obj, propName, { value: proxyObj }); - utils.patchToString(proxyObj); + utils.replaceProperty(obj, propName, { value: proxyObj }) + utils.patchToString(proxyObj) - return true; -}; + return true +} /** * All-in-one method to create a new JS Proxy with stealth tweaks. @@ -332,12 +368,11 @@ utils.mockWithProxy = (obj, propName, pseudoTarget, handler) => { * @param {object} handler - The JS Proxy handler to use */ utils.createProxy = (pseudoTarget, handler) => { - utils.preloadCache(); - const proxyObj = new Proxy(pseudoTarget, utils.stripProxyFromErrors(handler)); - utils.patchToString(proxyObj); + const proxyObj = new Proxy(pseudoTarget, utils.stripProxyFromErrors(handler)) + utils.patchToString(proxyObj) - return proxyObj; -}; + return proxyObj +} /** * Helper function to split a full path to an Object into the first part and property. @@ -350,13 +385,10 @@ utils.createProxy = (pseudoTarget, handler) => { */ utils.splitObjPath = objPath => ({ // Remove last dot entry (property) ==> `HTMLMediaElement.prototype` - objName: objPath - .split('.') - .slice(0, -1) - .join('.'), + objName: objPath.split('.').slice(0, -1).join('.'), // Extract last dot entry ==> `canPlayType` propName: objPath.split('.').slice(-1)[0] -}); +}) /** * Convenience method to replace a property with a JS Proxy using the provided objPath. @@ -370,10 +402,10 @@ utils.splitObjPath = objPath => ({ * @param {object} handler - The JS Proxy handler to use */ utils.replaceObjPathWithProxy = (objPath, handler) => { - const { objName, propName } = utils.splitObjPath(objPath); - const obj = eval(objName); // eslint-disable-line no-eval - return utils.replaceWithProxy(obj, propName, handler); -}; + const { objName, propName } = utils.splitObjPath(objPath) + const obj = eval(objName) // eslint-disable-line no-eval + return utils.replaceWithProxy(obj, propName, handler) +} /** * Traverse nested properties of an object recursively and apply the given function on a whitelist of value types. @@ -386,20 +418,20 @@ utils.execRecursively = (obj = {}, typeFilter = [], fn) => { function recurse(obj) { for (const key in obj) { if (obj[key] === undefined) { - continue; + continue } if (obj[key] && typeof obj[key] === 'object') { - recurse(obj[key]); + recurse(obj[key]) } else { if (obj[key] && typeFilter.includes(typeof obj[key])) { - fn.call(this, obj[key]); + fn.call(this, obj[key]) } } } } - recurse(obj); - return obj; -}; + recurse(obj) + return obj +} /** * Everything we run through e.g. `page.evaluate` runs in the browser context, not the NodeJS one. @@ -419,16 +451,16 @@ utils.stringifyFns = (fnObj = { hello: () => 'world' }) => { // https://github.com/feross/fromentries function fromEntries(iterable) { return [...iterable].reduce((obj, [key, val]) => { - obj[key] = val; - return obj; - }, {}); + obj[key] = val + return obj + }, {}) } return (Object.fromEntries || fromEntries)( Object.entries(fnObj) .filter(([key, value]) => typeof value === 'function') .map(([key, value]) => [key, value.toString()]) // eslint-disable-line no-eval - ); -}; + ) +} /** * Utility function to reverse the process of `utils.stringifyFns`. @@ -441,35 +473,14 @@ utils.materializeFns = (fnStrObj = { hello: "() => 'world'" }) => { Object.entries(fnStrObj).map(([key, value]) => { if (value.startsWith('function')) { // some trickery is needed to make oldschool functions work :-) - return [key, eval(`() => ${value}`)()]; // eslint-disable-line no-eval + return [key, eval(`() => ${value}`)()] // eslint-disable-line no-eval } else { // arrow functions just work - return [key, eval(value)]; // eslint-disable-line no-eval + return [key, eval(value)] // eslint-disable-line no-eval } }) - ); -}; - -/** - * All-in-one method to replace a getter with a JS Proxy using the provided Proxy handler with traps. - * - * @example - * replaceGetterWithProxy(Object.getPrototypeOf(navigator), 'vendor', proxyHandler) - * - * @param {object} obj - The object which has the property to replace - * @param {string} propName - The name of the property to replace - * @param {object} handler - The JS Proxy handler to use - */ -utils.replaceGetterWithProxy = (obj, propName, handler) => { - const fn = Object.getOwnPropertyDescriptor(obj, propName).get; - const fnStr = fn.toString(); // special getter function string - const proxyObj = new Proxy(fn, utils.stripProxyFromErrors(handler)); - - utils.replaceProperty(obj, propName, { get: proxyObj }); - utils.patchToString(proxyObj, fnStr); - - return true; -}; + ) +} // Proxy handler templates for re-usability utils.makeHandler = () => ({ @@ -478,84 +489,13 @@ utils.makeHandler = () => ({ apply(target, ctx, args) { // Let's fetch the value first, to trigger and escalate potential errors // Illegal invocations like `navigator.__proto__.vendor` will throw here - const ret = utils.cache.Reflect.apply(...arguments); + const ret = utils.cache.Reflect.apply(...arguments) if (args && args.length === 0) { - return value; + return value } - return ret; + return ret } }) -}); - -if (!window.chrome) { - // Use the exact property descriptor found in headful Chrome - // fetch it via `Object.getOwnPropertyDescriptor(window, 'chrome')` - Object.defineProperty(window, 'chrome', { - writable: true, - enumerable: true, - configurable: false, // note! - value: {} // We'll extend that later - }); -} - -// That means we're running headful and don't need to mock anything -if ('app' in window.chrome) { -} +}) -const makeError = { - ErrorInInvocation: fn => { - const err = new TypeError(`Error in invocation of app.${fn}()`); - return utils.stripErrorWithAnchor( - err, - `at ${fn} (eval at ` - ); - } -}; - -// There's a some static data in that property which doesn't seem to change, -// we should periodically check for updates: `JSON.stringify(window.app, null, 2)` -const STATIC_DATA = JSON.parse( - ` -{ - "isInstalled": false, - "InstallState": { - "DISABLED": "disabled", - "INSTALLED": "installed", - "NOT_INSTALLED": "not_installed" - }, - "RunningState": { - "CANNOT_RUN": "cannot_run", - "READY_TO_RUN": "ready_to_run", - "RUNNING": "running" - } -} - `.trim() -); - -window.chrome.app = { - ...STATIC_DATA, - - get isInstalled() { - return false; - }, - - getDetails: function getDetails() { - if (arguments.length) { - throw makeError.ErrorInInvocation(`getDetails`); - } - return null; - }, - getIsInstalled: function getDetails() { - if (arguments.length) { - throw makeError.ErrorInInvocation(`getIsInstalled`); - } - return false; - }, - runningState: function getDetails() { - if (arguments.length) { - throw makeError.ErrorInInvocation(`runningState`); - } - return 'cannot_run'; - } -}; -utils.patchToStringNested(window.chrome.app); \ No newline at end of file +utils.preloadCache(); \ No newline at end of file diff --git a/PuppeteerExtraSharp/Plugins/ExtraStealth/Scripts/WebDriver.js b/PuppeteerExtraSharp/Plugins/ExtraStealth/Scripts/WebDriver.js index 20f0020..412977f 100644 --- a/PuppeteerExtraSharp/Plugins/ExtraStealth/Scripts/WebDriver.js +++ b/PuppeteerExtraSharp/Plugins/ExtraStealth/Scripts/WebDriver.js @@ -1,3 +1,10 @@ () => { - delete Object.getPrototypeOf(navigator).webdriver; + if (navigator.webdriver === false) { + // Post Chrome 89.0.4339.0 and already good + } else if (navigator.webdriver === undefined) { + // Pre Chrome 89.0.4339.0 and already good + } else { + // Pre Chrome 88.0.4291.0 and needs patching + delete Object.getPrototypeOf(navigator).webdriver + } } \ No newline at end of file diff --git a/PuppeteerExtraSharp/Plugins/ExtraStealth/StealthPlugin.cs b/PuppeteerExtraSharp/Plugins/ExtraStealth/StealthPlugin.cs index 28cd082..f59dedf 100644 --- a/PuppeteerExtraSharp/Plugins/ExtraStealth/StealthPlugin.cs +++ b/PuppeteerExtraSharp/Plugins/ExtraStealth/StealthPlugin.cs @@ -1,25 +1,30 @@ using System.Collections.Generic; using System.Linq; +using System.Threading.Tasks; using PuppeteerExtraSharp.Plugins.ExtraStealth.Evasions; +using PuppeteerSharp; namespace PuppeteerExtraSharp.Plugins.ExtraStealth { public class StealthPlugin : PuppeteerExtraPlugin { private readonly IPuppeteerExtraPluginOptions[] _options; + private readonly List _standardEvasions; public StealthPlugin(params IPuppeteerExtraPluginOptions[] options) : base("stealth") { _options = options; + _standardEvasions = GetStandardEvasions(); } - public override ICollection GetDependencies() => new List() + private List GetStandardEvasions() + { + return new List() { new WebDriver(), - new ChromeApp(), + // new ChromeApp(), new ChromeSci(), new ChromeRuntime(), - new Frame(), new Codec(), new Languages(GetOptionByType()), new OutDimensions(), @@ -29,12 +34,28 @@ public StealthPlugin(params IPuppeteerExtraPluginOptions[] options) : base("stea new WebGl(GetOptionByType()), new PluginEvasion(), new StackTrace(), - new HardwareConcurrency(GetOptionByType()) + new HardwareConcurrency(GetOptionByType()), + new ContentWindow(), + new SourceUrl() }; + } + + public override ICollection GetDependencies() => _standardEvasions; + + public override async Task OnPageCreated(IPage page) + { + var utilsScript = Utils.GetScript("Utils.js"); + await page.EvaluateExpressionOnNewDocumentAsync(utilsScript); + } private T GetOptionByType() where T : IPuppeteerExtraPluginOptions { return _options.OfType().FirstOrDefault(); } + + public void RemoveEvasionByType() where T : PuppeteerExtraPlugin + { + _standardEvasions.RemoveAll(ev => ev is T); + } } -} +} \ No newline at end of file diff --git a/PuppeteerExtraSharp/Plugins/ExtraStealth/Utils.cs b/PuppeteerExtraSharp/Plugins/ExtraStealth/Utils.cs index 4028a99..896ea38 100644 --- a/PuppeteerExtraSharp/Plugins/ExtraStealth/Utils.cs +++ b/PuppeteerExtraSharp/Plugins/ExtraStealth/Utils.cs @@ -1,5 +1,6 @@ using System.Collections.Generic; using System.Text; +using System.Threading; using System.Threading.Tasks; using PuppeteerExtraSharp.Utils; using PuppeteerSharp; @@ -8,25 +9,12 @@ namespace PuppeteerExtraSharp.Plugins.ExtraStealth { internal static class Utils { - private static readonly HashSet PreloadedPage = new HashSet(); - private static readonly object Locker = new object(); - public static Task EvaluateOnNewPageWithUtilsScript(Page page, string script, params object[] args) + public static Task EvaluateOnNewPage(IPage page, string script, params object[] args) { - lock (Locker) - { - var tasks = new List(); - - if (!PreloadedPage.Contains(page.GetHashCode())) - { - var utilsScript = Utils.GetScript("Utils.js"); - PreloadedPage.Add(page.GetHashCode()); - tasks.Add(page.EvaluateExpressionOnNewDocumentAsync(utilsScript)); - } - - tasks.Add(page.EvaluateFunctionOnNewDocumentAsync(script, args)); - - return Task.WhenAll(tasks); - } + if (!page.IsClosed) + return page.EvaluateFunctionOnNewDocumentAsync(script, args); + + return Task.CompletedTask; } @@ -40,4 +28,4 @@ public static string GetScript(string name) return file; } } -} +} \ No newline at end of file diff --git a/PuppeteerExtraSharp/Plugins/ExtraStealth/readme.md b/PuppeteerExtraSharp/Plugins/ExtraStealth/readme.md index 8b13789..e665f20 100644 --- a/PuppeteerExtraSharp/Plugins/ExtraStealth/readme.md +++ b/PuppeteerExtraSharp/Plugins/ExtraStealth/readme.md @@ -1 +1,60 @@ +# Quickstart +```c# +var extra = new PuppeteerExtra(); +// initialize stealth plugin +var stealth = new StealthPlugin(); +var browser = await extra.Use(stealth).LaunchAsync(new LaunchOptions()); + +var page = await browser.NewPageAsync(); + +await page.GoToAsync("https://bot.sannysoft.com/"); +``` + +# Plugin options + +```c# +var extra = new PuppeteerExtra(); +// Initialize hardware concurrency plugin options +var stealthHardwareConcurrencyOptions = new StealthHardwareConcurrencyOptions(12); +// Put it on stealth plugin constructor +var stealth = new StealthPlugin(stealthHardwareConcurrencyOptions); +var browser = await extra.Use(stealth).LaunchAsync(new LaunchOptions()); +var page = await browser.NewPageAsync(); +await page.GoToAsync("https://bot.sannysoft.com/"); +``` + +### Available options: +#### Hardware concurrency +see (https://arh.antoinevastel.com/reports/stats/osName_hardwareConcurrency_report.html) +```c# +var concurrency = 12; // your number +var stealthHardwareConcurrencyOptions = new StealthHardwareConcurrencyOptions(concurrency); +``` +#### Vendor +```c# +var vendor = "Google Inc."; // your custom navigator.vendor +var stealthVendorSettings = new StealthVendorSettings(vendor); +``` +### Languages +```c# +var languages = "en-US"; // your custom languages array +var languagesSettings = new StealthLanguagesOptions(languages); +``` + +### WebGL +```c# +var webGLVendor = "Intel Inc."; // your custom webGL vendor +var render = "Intel Iris OpenGL Engine"; // your custom webGL renderer +var languagesSettings = new StealthWebGLOptions(webGLVendor, render); +``` + +# Removing evasions: +You can remove an evasion from the plugin by using the RemoveEvasionByType +```c# +var extra = new PuppeteerExtra(); +// initialize stealth plugin +var stealth = new StealthPlugin(); +stealthPlugin.RemoveEvasionByType(); +var browser = await extra.Use(stealth).LaunchAsync(new LaunchOptions()); +``` \ No newline at end of file diff --git a/PuppeteerExtraSharp/Plugins/PuppeteerExtraPlugin.cs b/PuppeteerExtraSharp/Plugins/PuppeteerExtraPlugin.cs index 4115a06..cb2b46a 100644 --- a/PuppeteerExtraSharp/Plugins/PuppeteerExtraPlugin.cs +++ b/PuppeteerExtraSharp/Plugins/PuppeteerExtraPlugin.cs @@ -21,12 +21,12 @@ public virtual ICollection GetDependencies() } public virtual void BeforeLaunch(LaunchOptions options) { } - public virtual void AfterLaunch(Browser browser) { } + public virtual void AfterLaunch(IBrowser browser) { } public virtual void BeforeConnect(ConnectOptions options) { } - public virtual void AfterConnect(Browser browser) { } - public virtual void OnBrowser(Browser browser) { } + public virtual void AfterConnect(IBrowser browser) { } + public virtual void OnBrowser(IBrowser browser) { } public virtual void OnTargetCreated(Target target) { } - public virtual Task OnPageCreated(Page page) { return Task.CompletedTask; } + public virtual Task OnPageCreated(IPage page) { return Task.CompletedTask; } public virtual void OnTargetChanged(Target target) { } public virtual void OnTargetDestroyed(Target target) { } public virtual void OnDisconnected() { } diff --git a/PuppeteerExtraSharp/Plugins/Recaptcha/Provider/2Captcha/TwoCaptchaApi.cs b/PuppeteerExtraSharp/Plugins/Recaptcha/Provider/2Captcha/TwoCaptchaApi.cs index dcd182c..cdc3a5a 100644 --- a/PuppeteerExtraSharp/Plugins/Recaptcha/Provider/2Captcha/TwoCaptchaApi.cs +++ b/PuppeteerExtraSharp/Plugins/Recaptcha/Provider/2Captcha/TwoCaptchaApi.cs @@ -34,17 +34,14 @@ public async Task CreateTaskAsync(string key, string pageUrl } - public async Task> GetSolution(string id) + public async Task> GetSolution(string id) { - var request = new RestRequest("res.php") {Method = Method.POST}; - - request.AddQueryParameters(new Dictionary() - { - ["id"] = id, - ["key"] = _userKey, - ["action"] = "get", - ["json"] = "1" - }); + var request = new RestRequest("res.php") {Method = Method.Post}; + + request.AddQueryParameter("id", id); + request.AddQueryParameter("key", _userKey); + request.AddQueryParameter("action", "get"); + request.AddQueryParameter("json", "1"); var result = await _client.CreatePollingBuilder(request).TriesLimit(_options.PendingCount).ActivatePollingAsync( response => response.Data.request == "CAPCHA_NOT_READY" ? PollingAction.ContinuePolling : PollingAction.Break); diff --git a/PuppeteerExtraSharp/Plugins/Recaptcha/Provider/AntiCaptcha/AntiCaptchaApi.cs b/PuppeteerExtraSharp/Plugins/Recaptcha/Provider/AntiCaptcha/AntiCaptchaApi.cs index 0f0f540..52eaeb5 100644 --- a/PuppeteerExtraSharp/Plugins/Recaptcha/Provider/AntiCaptcha/AntiCaptchaApi.cs +++ b/PuppeteerExtraSharp/Plugins/Recaptcha/Provider/AntiCaptcha/AntiCaptchaApi.cs @@ -48,7 +48,7 @@ public async Task PendingForResult(int taskId, CancellationToke var request = new RestRequest("getTaskResult"); request.AddJsonBody(content); - request.Method = Method.POST; + request.Method = Method.Post; var result = await _client.CreatePollingBuilder(request).TriesLimit(_options.PendingCount) .WithTimeoutSeconds(5).ActivatePollingAsync( diff --git a/PuppeteerExtraSharp/Plugins/Recaptcha/Provider/AntiCaptcha/Models/TaskResultModel.cs b/PuppeteerExtraSharp/Plugins/Recaptcha/Provider/AntiCaptcha/Models/TaskResultModel.cs index afa43c2..9e2f908 100644 --- a/PuppeteerExtraSharp/Plugins/Recaptcha/Provider/AntiCaptcha/Models/TaskResultModel.cs +++ b/PuppeteerExtraSharp/Plugins/Recaptcha/Provider/AntiCaptcha/Models/TaskResultModel.cs @@ -1,18 +1,26 @@ namespace PuppeteerExtraSharp.Plugins.Recaptcha.Provider.AntiCaptcha.Models { - public class Solution { + public class TaskResultModel + { + public int errorId { get; set; } + public string status { get; set; } + public Solution solution { get; set; } + public string cost { get; set; } + public string ip { get; set; } + public int createTime { get; set; } + public int endTime { get; set; } + public int solveCount { get; set; } + } + + public class Solution + { public string gRecaptchaResponse { get; set; } + public Cookies cookies { get; set; } } - public class TaskResultModel { - public int errorId { get; set; } - public string status { get; set; } - public Solution solution { get; set; } - public string cost { get; set; } - public string ip { get; set; } - public int createTime { get; set; } - public int endTime { get; set; } - public string solveCount { get; set; } + public class Cookies + { + public string empty { get; set; } } } diff --git a/PuppeteerExtraSharp/Plugins/Recaptcha/RecapchaPlugin.cs b/PuppeteerExtraSharp/Plugins/Recaptcha/RecapchaPlugin.cs index c5609e4..3cf09ee 100644 --- a/PuppeteerExtraSharp/Plugins/Recaptcha/RecapchaPlugin.cs +++ b/PuppeteerExtraSharp/Plugins/Recaptcha/RecapchaPlugin.cs @@ -13,12 +13,12 @@ public RecaptchaPlugin(IRecaptchaProvider provider, CaptchaOptions opt = null) : _recaptcha = new Recaptcha(provider, opt ?? new CaptchaOptions()); } - public async Task SolveCaptchaAsync(Page page) + public async Task SolveCaptchaAsync(IPage page) { return await _recaptcha.Solve(page); } - public override async Task OnPageCreated(Page page) + public override async Task OnPageCreated(IPage page) { await page.SetBypassCSPAsync(true); } diff --git a/PuppeteerExtraSharp/Plugins/Recaptcha/Recaptcha.cs b/PuppeteerExtraSharp/Plugins/Recaptcha/Recaptcha.cs index 66bf80d..f23fbff 100644 --- a/PuppeteerExtraSharp/Plugins/Recaptcha/Recaptcha.cs +++ b/PuppeteerExtraSharp/Plugins/Recaptcha/Recaptcha.cs @@ -18,7 +18,7 @@ public Recaptcha(IRecaptchaProvider provider, CaptchaOptions options) _options = options; } - public async Task Solve(Page page) + public async Task Solve(IPage page) { try { @@ -28,6 +28,7 @@ public async Task Solve(Page page) return new RecaptchaResult() { + result = solution, IsSuccess = true }; } @@ -42,7 +43,7 @@ public async Task Solve(Page page) } - public async Task GetKeyAsync(Page page) + public async Task GetKeyAsync(IPage page) { var element = await page.QuerySelectorAsync("iframe[src^='https://www.google.com/recaptcha/api2/anchor'][name^=\"a-\"]"); @@ -64,22 +65,24 @@ public async Task GetSolutionAsync(string key, string urlPage) return await _provider.GetSolution(key, urlPage); } - public async Task WriteToInput(Page page, string value) + public async Task WriteToInput(IPage page, string value) { await page.EvaluateFunctionAsync( $"() => {{document.getElementById('g-recaptcha-response').innerHTML='{value}'}}"); - - - var script = ResourcesReader.ReadFile(this.GetType().Namespace + ".Scripts.EnterRecaptchaCallBackScript.js"); - - try - { - await page.EvaluateFunctionAsync($@"(value) => {{{script}}}", value); - } - catch(Exception ex) - { - // ignored - } + + await page.EvaluateFunctionAsync($@" + () => {{ + const reduceObjectToArray = (obj) => Object.keys(obj).reduce((r, k) => r.concat(k, obj[k]), []); + const client = ___grecaptcha_cfg.clients[0]; + let result = []; + result = reduceObjectToArray(client).filter(c => Object.prototype.toString.call(c) === '[object Object]'); + result = result.flatMap(r => reduceObjectToArray(r)); + result = result.filter(c => Object.prototype.toString.call(c) === '[object Object]'); + const reqObj = result.find(r => r.callback); + reqObj.callback('{value}') + }}"); + + } } } diff --git a/PuppeteerExtraSharp/Plugins/Recaptcha/RecaptchaResult.cs b/PuppeteerExtraSharp/Plugins/Recaptcha/RecaptchaResult.cs index e14ed62..8631bc0 100644 --- a/PuppeteerExtraSharp/Plugins/Recaptcha/RecaptchaResult.cs +++ b/PuppeteerExtraSharp/Plugins/Recaptcha/RecaptchaResult.cs @@ -2,6 +2,7 @@ { public class RecaptchaResult { + public string result { get; set; } = ""; public bool IsSuccess { get; set; } = true; public CaptchaException Exception { get; set; } } diff --git a/PuppeteerExtraSharp/Plugins/Recaptcha/RestClient/PollingBuilder.cs b/PuppeteerExtraSharp/Plugins/Recaptcha/RestClient/PollingBuilder.cs index f25c3b9..c080a5e 100644 --- a/PuppeteerExtraSharp/Plugins/Recaptcha/RestClient/PollingBuilder.cs +++ b/PuppeteerExtraSharp/Plugins/Recaptcha/RestClient/PollingBuilder.cs @@ -6,11 +6,11 @@ namespace PuppeteerExtraSharp.Plugins.Recaptcha.RestClient { public class PollingBuilder { - private readonly IRestClient _client; - private readonly IRestRequest _request; + private readonly RestSharp.RestClient _client; + private readonly RestSharp.RestRequest _request; private int _timeout = 5; private int _limit = 5; - public PollingBuilder(IRestClient client, IRestRequest request) + public PollingBuilder(RestSharp.RestClient client, RestRequest request) { _client = client; _request = request; @@ -28,7 +28,7 @@ public PollingBuilder TriesLimit(int limit) return this; } - public async Task> ActivatePollingAsync(Func, PollingAction> resultDelegate) + public async Task> ActivatePollingAsync(Func, PollingAction> resultDelegate) { var response = await _client.ExecuteAsync(_request); diff --git a/PuppeteerExtraSharp/Plugins/Recaptcha/RestClient/RestClient.cs b/PuppeteerExtraSharp/Plugins/Recaptcha/RestClient/RestClient.cs index 94530d4..af007b6 100644 --- a/PuppeteerExtraSharp/Plugins/Recaptcha/RestClient/RestClient.cs +++ b/PuppeteerExtraSharp/Plugins/Recaptcha/RestClient/RestClient.cs @@ -15,7 +15,7 @@ public RestClient(string url = null) _client = string.IsNullOrWhiteSpace(url) ? new RestSharp.RestClient() : new RestSharp.RestClient(url); } - public PollingBuilder CreatePollingBuilder(IRestRequest request) + public PollingBuilder CreatePollingBuilder(RestRequest request) { return new PollingBuilder(_client, request); } @@ -25,18 +25,18 @@ public async Task PostWithJsonAsync(string url, object content, Cancellati var request = new RestRequest(url); request.AddHeader("Content-type", "application/json"); request.AddJsonBody(content); - request.Method = Method.POST; + request.Method = Method.Post; return await _client.PostAsync(request, token); } public async Task PostWithQueryAsync(string url, Dictionary query, CancellationToken token = default) { - var request = new RestRequest(url) { Method = Method.POST }; + var request = new RestRequest(url) { Method = Method.Post }; request.AddQueryParameters(query); return await _client.PostAsync(request, token); } - private async Task> ExecuteAsync(RestRequest request, CancellationToken token) + private async Task> ExecuteAsync(RestRequest request, CancellationToken token) { return await _client.ExecuteAsync(request, token); } diff --git a/PuppeteerExtraSharp/Plugins/Recaptcha/Scripts/EnterRecaptchaCallBackScript.js b/PuppeteerExtraSharp/Plugins/Recaptcha/Scripts/EnterRecaptchaCallBackScript.js index c24dd68..b7bf96d 100644 --- a/PuppeteerExtraSharp/Plugins/Recaptcha/Scripts/EnterRecaptchaCallBackScript.js +++ b/PuppeteerExtraSharp/Plugins/Recaptcha/Scripts/EnterRecaptchaCallBackScript.js @@ -37,10 +37,10 @@ } })() -if (typeof (result.function) == 'function') { - result.function(value) -} - -else { - eval(result.function).call(window, value); -} +// if (typeof (result.function) == 'function') { +// result.function(value) +// } +// +// else { +// eval(result.function).call(window, value); +// } diff --git a/PuppeteerExtraSharp/Plugins/Recaptcha/Scripts/InjectScript.js b/PuppeteerExtraSharp/Plugins/Recaptcha/Scripts/InjectScript.js new file mode 100644 index 0000000..22a2b43 --- /dev/null +++ b/PuppeteerExtraSharp/Plugins/Recaptcha/Scripts/InjectScript.js @@ -0,0 +1,61 @@ +var p = Object.defineProperty; +var d = (s,a,t)=>a in s ? p(s, a, { + enumerable: !0, + configurable: !0, + writable: !0, + value: t +}) : s[a] = t; +var e = (s,a,t)=>(d(s, typeof a != "symbol" ? a + "" : a, t), + t); +import {a as n} from "./axios.262c1ca8.js"; +import {M as m} from "./MapeadorDeCamposDeResponse.01121813.js"; +import {d as u} from "./dayjs.min.b82cae86.js"; +class r { + constructor(a) { + e(this, "cpf_cnpj"); + e(this, "nome_receita"); + e(this, "nome_mae_receita"); + e(this, "nome_fantasia_receita"); + e(this, "data_nascimento_receita"); + e(this, "situacao"); + e(this, "tipo_pessoa"); + e(this, "ano_obito"); + m.mapear(this, a) + } + dataNascimentoFormatada() { + return this.data_nascimento_receita != null ? u(this.data_nascimento_receita).format("DD/MM/YYYY") : void 0 + } + isFalecida() { + return this.ano_obito != null && this.ano_obito > 0 + } + temSituacaoDefinida() { + return this.situacao != null + } +} +class i extends Error { +} +class w { + static async buscarCpfCnpj(a, t) { + return t != null ? await this.buscaExternaCpfCnpj(a, t) : { + resposta: await this.buscaInternaCpfCnpj(a) + } + } + static async buscaExternaCpfCnpj(a, t) { + try { + const {data: o, headers: c} = await n.get(`/pessoas/${a}/externo?captcha=${t}`); + return { + codigo_token: c["x-cnc-tkse"], + resposta: new r(o) + } + } catch (o) { + throw o.response.data != null && o.response.data.erro == "ErroNaConsultaReceita" ? new i(o.response.data.mensagem) : o + } + } + static async buscaInternaCpfCnpj(a) { + const {data: t} = await n.get(`/pessoas/${a}`); + if (t.cpf_cnpj != null) + return new r(t); + throw new i("N\xE3o foram encontrados dados para este CPF/CNPJ") + } +} +export {w as C, i as E, r as P}; diff --git a/PuppeteerExtraSharp/PuppeteerExtra.cs b/PuppeteerExtraSharp/PuppeteerExtra.cs index dba7d0e..1803fbf 100644 --- a/PuppeteerExtraSharp/PuppeteerExtra.cs +++ b/PuppeteerExtraSharp/PuppeteerExtra.cs @@ -14,17 +14,17 @@ public class PuppeteerExtra public PuppeteerExtra Use(PuppeteerExtraPlugin plugin) { - ResolveDependencies(plugin); _plugins.Add(plugin); + ResolveDependencies(plugin); plugin.OnPluginRegistered(); return this; } - public async Task LaunchAsync(LaunchOptions options) + public async Task LaunchAsync(LaunchOptions options) { - _plugins.ForEach(e=>e.BeforeLaunch(options)); + _plugins.ForEach(e => e.BeforeLaunch(options)); var browser = await Puppeteer.LaunchAsync(options); - _plugins.ForEach(e=>e.AfterLaunch(browser)); + _plugins.ForEach(e => e.AfterLaunch(browser)); await OnStart(new BrowserStartContext() { StartType = StartType.Launch, @@ -33,24 +33,24 @@ await OnStart(new BrowserStartContext() return browser; } - public async Task ConnectAsync(ConnectOptions options) + public async Task ConnectAsync(ConnectOptions options) { - _plugins.ForEach(e=>e.BeforeConnect(options)); + _plugins.ForEach(e => e.BeforeConnect(options)); var browser = await Puppeteer.ConnectAsync(options); - _plugins.ForEach(e=>e.AfterConnect(browser)); + _plugins.ForEach(e => e.AfterConnect(browser)); await OnStart(new BrowserStartContext() { - StartType = StartType.Connect + StartType = StartType.Connect }, browser); return browser; } - public T GetPlugin() where T: PuppeteerExtraPlugin + public T GetPlugin() where T : PuppeteerExtraPlugin { - return (T)_plugins.FirstOrDefault(e => e.GetType() == typeof(T)); + return (T) _plugins.FirstOrDefault(e => e.GetType() == typeof(T)); } - private async Task OnStart(BrowserStartContext context, Browser browser) + private async Task OnStart(BrowserStartContext context, IBrowser browser) { OrderPlugins(); CheckPluginRequirements(context); @@ -62,14 +62,14 @@ private void ResolveDependencies(PuppeteerExtraPlugin plugin) var dependencies = plugin.GetDependencies()?.ToList(); if (dependencies is null || !dependencies.Any()) return; - + foreach (var puppeteerExtraPlugin in dependencies) { Use(puppeteerExtraPlugin); var plugDependencies = puppeteerExtraPlugin.GetDependencies()?.ToList(); - if(plugDependencies != null && plugDependencies.Any()) + if (plugDependencies != null && plugDependencies.Any()) plugDependencies.ForEach(ResolveDependencies); } } @@ -83,40 +83,43 @@ private void CheckPluginRequirements(BrowserStartContext context) { foreach (var puppeteerExtraPlugin in _plugins) { - if(puppeteerExtraPlugin.Requirements is null) + if (puppeteerExtraPlugin.Requirements is null) continue; foreach (var requirement in puppeteerExtraPlugin.Requirements) { switch (context.StartType) { case StartType.Launch when requirement == PluginRequirements.HeadFul && context.IsHeadless: - throw new NotSupportedException($"Plugin - {puppeteerExtraPlugin.Name} is not supported in headless mode"); + throw new NotSupportedException( + $"Plugin - {puppeteerExtraPlugin.Name} is not supported in headless mode"); case StartType.Connect when requirement == PluginRequirements.Launch: - throw new NotSupportedException($"Plugin - {puppeteerExtraPlugin.Name} doesn't support connect"); + throw new NotSupportedException( + $"Plugin - {puppeteerExtraPlugin.Name} doesn't support connect"); } } } } - private async Task Register(Browser browser) + private async Task Register(IBrowser browser) { var pages = await browser.PagesAsync(); + + browser.TargetCreated += async (sender, args) => + { + _plugins.ForEach(e => e.OnTargetCreated(args.Target)); + if (args.Target.Type == TargetType.Page) + { + var page = await args.Target.PageAsync(); + _plugins.ForEach(async e => await e.OnPageCreated(page)); + } + }; + foreach (var puppeteerExtraPlugin in _plugins) { browser.TargetChanged += (sender, args) => puppeteerExtraPlugin.OnTargetChanged(args.Target); - browser.TargetCreated += async (sender, args) => - { - puppeteerExtraPlugin.OnTargetCreated(args.Target); - if (args.Target.Type == TargetType.Page) - { - var page = await args.Target.PageAsync(); - await puppeteerExtraPlugin.OnPageCreated(page); - } - }; browser.TargetDestroyed += (sender, args) => puppeteerExtraPlugin.OnTargetDestroyed(args.Target); browser.Disconnected += (sender, args) => puppeteerExtraPlugin.OnDisconnected(); browser.Closed += (sender, args) => puppeteerExtraPlugin.OnClose(); - foreach (var page in pages) { await puppeteerExtraPlugin.OnPageCreated(page); @@ -124,4 +127,4 @@ private async Task Register(Browser browser) } } } -} +} \ No newline at end of file diff --git a/PuppeteerExtraSharp/PuppeteerExtraSharp.csproj b/PuppeteerExtraSharp/PuppeteerExtraSharp.csproj index 20d937d..4880023 100644 --- a/PuppeteerExtraSharp/PuppeteerExtraSharp.csproj +++ b/PuppeteerExtraSharp/PuppeteerExtraSharp.csproj @@ -1,9 +1,9 @@ - + netstandard2.0 latest - 1.2.0 + 1.3.1 https://github.com/Overmiind/Puppeteer-sharp-extra git puppeteer-extra recaptcha browser-automation browser-extension browser puppeteer netcore netcore31 stealth-client browser-testing c# @@ -30,12 +30,14 @@ + + Always + - @@ -49,12 +51,15 @@ - + + + Always + - - + + diff --git a/PuppeteerExtraSharp/PuppeteerExtraSharp.sln b/PuppeteerExtraSharp/PuppeteerExtraSharp.sln index f6d2bab..7e80cce 100644 --- a/PuppeteerExtraSharp/PuppeteerExtraSharp.sln +++ b/PuppeteerExtraSharp/PuppeteerExtraSharp.sln @@ -7,6 +7,13 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "PuppeteerExtraSharp", "Pupp EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Extra.Tests", "..\tests\Extra.Tests.csproj", "{FB432A25-A844-4C78-9194-E46D806B46E5}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Trexx.SocialNetworks.Helper.API", "..\..\Trexx.SocialNetworks>helpr\Trexx.SocialNetworks.Helper.API\Trexx.SocialNetworks.Helper.API.csproj", "{9D2354C3-B901-42C9-915F-017530F243AB}" +EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution Items", "{53247AA4-0A47-4B51-A066-560638078898}" + ProjectSection(SolutionItems) = preProject + docker-compose.yml = docker-compose.yml + EndProjectSection +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -21,6 +28,10 @@ Global {FB432A25-A844-4C78-9194-E46D806B46E5}.Debug|Any CPU.Build.0 = Debug|Any CPU {FB432A25-A844-4C78-9194-E46D806B46E5}.Release|Any CPU.ActiveCfg = Release|Any CPU {FB432A25-A844-4C78-9194-E46D806B46E5}.Release|Any CPU.Build.0 = Release|Any CPU + {9D2354C3-B901-42C9-915F-017530F243AB}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {9D2354C3-B901-42C9-915F-017530F243AB}.Debug|Any CPU.Build.0 = Debug|Any CPU + {9D2354C3-B901-42C9-915F-017530F243AB}.Release|Any CPU.ActiveCfg = Release|Any CPU + {9D2354C3-B901-42C9-915F-017530F243AB}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE diff --git a/PuppeteerExtraSharp/Utils/BrowserHelper.cs b/PuppeteerExtraSharp/Utils/BrowserHelper.cs new file mode 100644 index 0000000..c8f8921 --- /dev/null +++ b/PuppeteerExtraSharp/Utils/BrowserHelper.cs @@ -0,0 +1,91 @@ +using System; +using System.Collections.Generic; + +namespace PuppeteerExtraSharp.Utils; + +public static class BrowserHelper +{ + private static readonly List UserAgents = new List + { + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.93 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:88.0) Gecko/20100101 Firefox/88.0", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.93 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.85 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.212 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.128 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_6) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.0.3 Safari/605.1.15", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.85 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; rv:78.0) Gecko/20100101 Firefox/78.0", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.1 Safari/605.1.15", + "Mozilla/5.0 (X11; Linux x86_64; rv:88.0) Gecko/20100101 Firefox/88.0", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:88.0) Gecko/20100101 Firefox/88.0", + "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:88.0) Gecko/20100101 Firefox/88.0", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:87.0) Gecko/20100101 Firefox/87.0", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.72 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.114 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.212 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.93 Safari/537.36 Edg/90.0.818.56", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.93 Safari/537.36 Edg/90.0.818.51", + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.93 Safari/537.36", + "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:87.0) Gecko/20100101 Firefox/87.0", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 11_2_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.128 Safari/537.36", + "Mozilla/5.0 (X11; Linux x86_64; rv:78.0) Gecko/20100101 Firefox/78.0", + "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.93 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.128 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.0.3 Safari/605.1.15", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.90 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:87.0) Gecko/20100101 Firefox/87.0", + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.72 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.85 Safari/537.36 Edg/90.0.818.49", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.72 Safari/537.36", + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.85 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.85 Safari/537.36 Edg/90.0.818.46", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.72 Safari/537.36 Edg/90.0.818.42", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.128 Safari/537.36 Edg/89.0.774.77", + "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:88.0) Gecko/20100101 Firefox/88.0", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.128 Safari/537.36 OPR/75.0.3969.243", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.93 Safari/537.36 OPR/76.0.4017.107", + "Mozilla/5.0 (X11; Linux x86_64; rv:87.0) Gecko/20100101 Firefox/87.0", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.93 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.100 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:78.0) Gecko/20100101 Firefox/78.0", + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.212 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 11_2_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.114 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.1.2 Safari/605.1.15", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.212 Safari/537.36 Edg/90.0.818.62", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.105 YaBrowser/21.3.3.230 Yowser/2.5 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.128 Safari/537.36 OPR/75.0.3969.218", + "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.85 Safari/537.36", + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.114 Safari/537.36", + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.90 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.114 Safari/537.36", + "Mozilla/5.0 (X11; Fedora; Linux x86_64; rv:88.0) Gecko/20100101 Firefox/88.0", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:86.0) Gecko/20100101 Firefox/86.0", + "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.212 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.14; rv:88.0) Gecko/20100101 Firefox/88.0", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:89.0) Gecko/20100101 Firefox/89.0", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.93 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.7113.93 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.85 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_6) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.0.2 Safari/605.1.15", + "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.128 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.72 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.3; Win64; x64; rv:88.0) Gecko/20100101 Firefox/88.0", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.90 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:90.0) Gecko/20100101 Firefox/90.0", + "Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.93 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.114 Safari/537.36 OPR/75.0.3969.171", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.82 Safari/537.36", + "Mozilla/5.0 (X11; Linux x86_64; rv:89.0) Gecko/20100101 Firefox/89.0", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.0.3 Safari/605.1.15", + "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.93 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.85 Safari/537.36" + + + }; + + public static string GetUserAgent() + { + return UserAgents[new Random().Next(UserAgents.Count)]; + } +} \ No newline at end of file diff --git a/PuppeteerExtraSharp/Utils/RestHelper.cs b/PuppeteerExtraSharp/Utils/RestHelper.cs index 2f3210a..3c285d7 100644 --- a/PuppeteerExtraSharp/Utils/RestHelper.cs +++ b/PuppeteerExtraSharp/Utils/RestHelper.cs @@ -5,7 +5,7 @@ namespace PuppeteerExtraSharp.Utils { public static class RestHelper { - public static IRestRequest AddQueryParameters(this IRestRequest request, Dictionary parameters) + public static RestRequest AddQueryParameters(this RestRequest request, Dictionary parameters) { foreach (var parameter in parameters) { diff --git a/Tests/BrowserDefault.cs b/Tests/BrowserDefault.cs index 2685e64..94182e6 100644 --- a/Tests/BrowserDefault.cs +++ b/Tests/BrowserDefault.cs @@ -13,12 +13,12 @@ namespace Extra.Tests { public abstract class BrowserDefault : IDisposable { - private readonly List _launchedBrowsers = new List(); + private readonly List _launchedBrowsers = new List(); protected BrowserDefault() { } - protected async Task LaunchAsync(LaunchOptions options = null) + protected async Task LaunchAsync(LaunchOptions options = null) { //DownloadChromeIfNotExists(); options ??= CreateDefaultOptions(); @@ -28,7 +28,7 @@ protected async Task LaunchAsync(LaunchOptions options = null) return browser; } - protected async Task LaunchWithPluginAsync(PuppeteerExtraPlugin plugin, LaunchOptions options = null) + protected async Task LaunchWithPluginAsync(PuppeteerExtraPlugin plugin, LaunchOptions options = null) { var extra = new PuppeteerExtra().Use(plugin); //DownloadChromeIfNotExists(); @@ -39,9 +39,9 @@ protected async Task LaunchWithPluginAsync(PuppeteerExtraPlugin plugin, return browser; } - protected async Task LaunchAndGetPage(PuppeteerExtraPlugin plugin = null) + protected async Task LaunchAndGetPage(PuppeteerExtraPlugin plugin = null) { - Browser browser = null; + IBrowser browser = null; if (plugin != null) browser = await LaunchWithPluginAsync(plugin); else @@ -61,7 +61,7 @@ private async void DownloadChromeIfNotExists() await new BrowserFetcher(new BrowserFetcherOptions() { Path = Constants.PathToChrome - }).DownloadAsync(BrowserFetcher.DefaultRevision); + }).DownloadAsync(BrowserFetcher.DefaultChromiumRevision); } protected LaunchOptions CreateDefaultOptions() diff --git a/Tests/Constants.cs b/Tests/Constants.cs index 5fbb7c8..191da1a 100644 --- a/Tests/Constants.cs +++ b/Tests/Constants.cs @@ -6,7 +6,7 @@ namespace Extra.Tests { public static class Constants { - public static readonly string PathToChrome = @"C:\chrome\chrome.exe"; + public static readonly string PathToChrome = @"C:\Program Files\Google\Chrome\Application\chrome.exe"; public static readonly bool Headless = true; } } diff --git a/Tests/Extra.Tests.csproj b/Tests/Extra.Tests.csproj index 28d49f6..879898d 100644 --- a/Tests/Extra.Tests.csproj +++ b/Tests/Extra.Tests.csproj @@ -1,13 +1,16 @@  - netcoreapp3.1 + net7.0 false + + Always + @@ -15,15 +18,15 @@ - - - - - + + + + + all runtime; build; native; contentfiles; analyzers; buildtransitive - + all runtime; build; native; contentfiles; analyzers; buildtransitive @@ -34,6 +37,11 @@ + + True + True + Resources.resx + True True @@ -42,6 +50,10 @@ + + ResXFileCodeGenerator + Resources.Designer.cs + ResXFileCodeGenerator Resources.Designer.cs diff --git a/Tests/Properties/Resources.Designer.cs b/Tests/Properties/Resources.Designer.cs new file mode 100644 index 0000000..bb385da --- /dev/null +++ b/Tests/Properties/Resources.Designer.cs @@ -0,0 +1,81 @@ +//------------------------------------------------------------------------------ +// +// This code was generated by a tool. +// Runtime Version:4.0.30319.42000 +// +// Changes to this file may cause incorrect behavior and will be lost if +// the code is regenerated. +// +//------------------------------------------------------------------------------ + +namespace Extra.Tests.Properties { + using System; + + + /// + /// A strongly-typed resource class, for looking up localized strings, etc. + /// + // This class was auto-generated by the StronglyTypedResourceBuilder + // class via a tool like ResGen or Visual Studio. + // To add or remove a member, edit your .ResX file then rerun ResGen + // with the /str option, or rebuild your VS project. + [global::System.CodeDom.Compiler.GeneratedCodeAttribute("System.Resources.Tools.StronglyTypedResourceBuilder", "17.0.0.0")] + [global::System.Diagnostics.DebuggerNonUserCodeAttribute()] + [global::System.Runtime.CompilerServices.CompilerGeneratedAttribute()] + internal class Resources { + + private static global::System.Resources.ResourceManager resourceMan; + + private static global::System.Globalization.CultureInfo resourceCulture; + + [global::System.Diagnostics.CodeAnalysis.SuppressMessageAttribute("Microsoft.Performance", "CA1811:AvoidUncalledPrivateCode")] + internal Resources() { + } + + /// + /// Returns the cached ResourceManager instance used by this class. + /// + [global::System.ComponentModel.EditorBrowsableAttribute(global::System.ComponentModel.EditorBrowsableState.Advanced)] + internal static global::System.Resources.ResourceManager ResourceManager { + get { + if (object.ReferenceEquals(resourceMan, null)) { + global::System.Resources.ResourceManager temp = new global::System.Resources.ResourceManager("Extra.Tests.Properties.Resources", typeof(Resources).Assembly); + resourceMan = temp; + } + return resourceMan; + } + } + + /// + /// Overrides the current thread's CurrentUICulture property for all + /// resource lookups using this strongly typed resource class. + /// + [global::System.ComponentModel.EditorBrowsableAttribute(global::System.ComponentModel.EditorBrowsableState.Advanced)] + internal static global::System.Globalization.CultureInfo Culture { + get { + return resourceCulture; + } + set { + resourceCulture = value; + } + } + + /// + /// Looks up a localized string similar to "". + /// + internal static string AntiCaptchaKey { + get { + return ResourceManager.GetString("AntiCaptchaKey", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to "". + /// + internal static string TwoCaptchaKey { + get { + return ResourceManager.GetString("TwoCaptchaKey", resourceCulture); + } + } + } +} diff --git a/Tests/Properties/Resources.resx b/Tests/Properties/Resources.resx new file mode 100644 index 0000000..4953991 --- /dev/null +++ b/Tests/Properties/Resources.resx @@ -0,0 +1,126 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + text/microsoft-resx + + + 2.0 + + + System.Resources.ResXResourceReader, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + System.Resources.ResXResourceWriter, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + "" + + + "" + + \ No newline at end of file diff --git a/Tests/Recaptcha/AntiCaptcha/AntiCaptchaTests.cs b/Tests/Recaptcha/AntiCaptcha/AntiCaptchaTests.cs index e8a65c5..6cd3b46 100644 --- a/Tests/Recaptcha/AntiCaptcha/AntiCaptchaTests.cs +++ b/Tests/Recaptcha/AntiCaptcha/AntiCaptchaTests.cs @@ -1,4 +1,5 @@ -using PuppeteerExtraSharp.Plugins.Recaptcha; +using Extra.Tests.Properties; +using PuppeteerExtraSharp.Plugins.Recaptcha; using PuppeteerSharp; using Xunit; using Xunit.Abstractions; @@ -65,7 +66,7 @@ public async void ShouldSolveCaptchaWithCallback() await CheckSuccessVerify(page); } - private async Task CheckSuccessVerify(Page page) + private async Task CheckSuccessVerify(IPage page) { var successElement = await page.QuerySelectorAsync("div[id='main'] div[class='description'] h2"); var elementValue = await (await successElement.GetPropertyAsync("textContent")).JsonValueAsync(); diff --git a/Tests/Recaptcha/TwoCaptcha/TwoCaptchaProviderTest.cs b/Tests/Recaptcha/TwoCaptcha/TwoCaptchaProviderTest.cs index 91a76c1..b8b4ae6 100644 --- a/Tests/Recaptcha/TwoCaptcha/TwoCaptchaProviderTest.cs +++ b/Tests/Recaptcha/TwoCaptcha/TwoCaptchaProviderTest.cs @@ -3,6 +3,7 @@ using System.Net.Http; using System.Text; using System.Threading.Tasks; +using Extra.Tests.Properties; using PuppeteerExtraSharp.Plugins.Recaptcha; using PuppeteerExtraSharp.Plugins.Recaptcha.Provider; using PuppeteerSharp; diff --git a/Tests/StealthPluginTests/EvasionsTests/ChromeAppTest.cs b/Tests/StealthPluginTests/EvasionsTests/ChromeAppTest.cs index 1133981..d568221 100644 --- a/Tests/StealthPluginTests/EvasionsTests/ChromeAppTest.cs +++ b/Tests/StealthPluginTests/EvasionsTests/ChromeAppTest.cs @@ -6,7 +6,7 @@ namespace Extra.Tests.StealthPluginTests.EvasionsTests { - public class ChromeAppTest: BrowserDefault + public class ChromeAppTest : BrowserDefault { [Fact] public async Task ShouldWork() @@ -27,15 +27,15 @@ public async Task ShouldWork() var installState = await page.EvaluateExpressionAsync("chrome.app.InstallState"); Assert.NotNull(installState); - Assert.Equal("disabled",installState["DISABLED"]); - Assert.Equal("installed",installState["INSTALLED"]); - Assert.Equal("not_installed",installState["NOT_INSTALLED"]); + Assert.Equal("disabled", installState["DISABLED"]); + Assert.Equal("installed", installState["INSTALLED"]); + Assert.Equal("not_installed", installState["NOT_INSTALLED"]); var runningState = await page.EvaluateExpressionAsync("chrome.app.RunningState"); Assert.NotNull(runningState); - Assert.Equal("cannot_run",runningState["CANNOT_RUN"]); - Assert.Equal("ready_to_run",runningState["READY_TO_RUN"]); - Assert.Equal("running",runningState["RUNNING"]); + Assert.Equal("cannot_run", runningState["CANNOT_RUN"]); + Assert.Equal("ready_to_run", runningState["READY_TO_RUN"]); + Assert.Equal("running", runningState["RUNNING"]); var details = await page.EvaluateExpressionAsync("chrome.app.getDetails()"); Assert.Null(details); diff --git a/Tests/StealthPluginTests/EvasionsTests/ChromeSciTest.cs b/Tests/StealthPluginTests/EvasionsTests/ChromeSciTest.cs index c26047a..1efd87b 100644 --- a/Tests/StealthPluginTests/EvasionsTests/ChromeSciTest.cs +++ b/Tests/StealthPluginTests/EvasionsTests/ChromeSciTest.cs @@ -12,26 +12,29 @@ public async Task ShouldWork() var plugin = new ChromeSci(); var page = await LaunchAndGetPage(plugin); await page.GoToAsync("https://google.com"); - var sci = await page.EvaluateExpressionAsync("window.chrome.csi()"); - Assert.NotNull(sci); + var sci = await page.EvaluateFunctionAsync(@"() => { + const { timing } = window.performance + const csi = window.chrome.csi() + return { + csi: { + exists: window.chrome && 'csi' in window.chrome, + toString: chrome.csi.toString() + }, + dataOK: { + onloadT: csi.onloadT === timing.domContentLoadedEventEnd, + startE: csi.startE === timing.navigationStart, + pageT: Number.isInteger(csi.pageT), + tran: Number.isInteger(csi.tran) + } + } + }"); - var onLoad = - await page.EvaluateExpressionAsync( - "window.chrome.csi().onLoadT === window.performance.domContentLoadedEventEnd"); - Assert.True(onLoad); - - //var startE = - // await page.EvaluateExpressionAsync( - // "window.chrome.csi().startE === window.performance.navigationStart"); - - //Assert.True(startE); - - var pageT = await page.EvaluateExpressionAsync("Number.isInteger(window.chrome.csi().pageT)"); - Assert.True(pageT); - - - var tran = await page.EvaluateExpressionAsync("Number.isInteger(window.chrome.csi().tran)"); - Assert.True(tran); + Assert.True(sci["csi"].Value("exists")); + Assert.Equal("function () { [native code] }", sci["csi"]["toString"]); + Assert.True(sci["dataOK"].Value("onloadT")); + Assert.True(sci["dataOK"].Value("pageT")); + Assert.True(sci["dataOK"].Value("startE")); + Assert.True(sci["dataOK"].Value("tran")); } } -} +} \ No newline at end of file diff --git a/Tests/StealthPluginTests/EvasionsTests/ContentWindowTest.cs b/Tests/StealthPluginTests/EvasionsTests/ContentWindowTest.cs new file mode 100644 index 0000000..bcb0021 --- /dev/null +++ b/Tests/StealthPluginTests/EvasionsTests/ContentWindowTest.cs @@ -0,0 +1,183 @@ +using System.Linq; +using System.Threading.Tasks; +using Extra.Tests.Utils; +using PuppeteerExtraSharp.Plugins.ExtraStealth; +using Xunit; + +namespace Extra.Tests.StealthPluginTests.EvasionsTests +{ + public class ContentWindowTest : BrowserDefault + { + [Fact] + public async Task IFrameShouldBeObject() + { + var plugin = new StealthPlugin(); + + var page = await LaunchAndGetPage(plugin); + await page.GoToAsync("https://google.com"); + + var finger = await new FingerPrint().GetFingerPrint(page); + + Assert.Equal("object", finger["iframeChrome"]); + } + + [Fact] + public async Task ShouldNotBreakIFrames() + { + var plugin = new StealthPlugin(); + + var page = await LaunchAndGetPage(plugin); + await page.GoToAsync("https://google.com"); + + const string testFuncReturnValue = "TESTSTRING"; + + await page.EvaluateFunctionAsync(@"(testFuncReturnValue) => { + const { document } = window // eslint-disable-line + const body = document.querySelector('body') + const iframe = document.createElement('iframe') + iframe.srcdoc = 'foobar' + iframe.contentWindow.mySuperFunction = () => testFuncReturnValue + body.appendChild(iframe) + }", testFuncReturnValue); + + var result = + await page.EvaluateExpressionAsync( + "document.querySelector('iframe').contentWindow.mySuperFunction()"); + + Assert.Equal(testFuncReturnValue, result); + } + + [Fact] + public async Task ShouldCoverAllFrames() + { + var plugin = new StealthPlugin(); + + var page = await LaunchAndGetPage(plugin); + await page.GoToAsync("https://google.com"); + + + var basicFrame = await page.EvaluateFunctionAsync(@"() => { + const el = document.createElement('iframe') + document.body.appendChild(el) + return typeof(el.contentWindow.chrome) + }"); + + var sandboxSOIFrame = await page.EvaluateFunctionAsync(@"() => { + const el = document.createElement('iframe') + el.setAttribute('sandbox', 'allow-same-origin') + document.body.appendChild(el) + return typeof(el.contentWindow.chrome) + }"); + + var sandboxSOASIFrame = await page.EvaluateFunctionAsync(@"() => { + const el = document.createElement('iframe') + el.setAttribute('sandbox', 'allow-same-origin allow-scripts') + document.body.appendChild(el) + return typeof(el.contentWindow.chrome) + }"); + + var srcdocIFrame = await page.EvaluateFunctionAsync(@"() => { + const el = document.createElement('iframe') + el.srcdoc = 'blank page, boys.' + document.body.appendChild(el) + return typeof(el.contentWindow.chrome) + }"); + + Assert.Equal("object", basicFrame); + Assert.Equal("object", sandboxSOIFrame); + Assert.Equal("object", sandboxSOASIFrame); + Assert.Equal("object", srcdocIFrame); + } + + + [Fact] + public async Task ShouldEmulateFeatures() + { + var plugin = new StealthPlugin(); + + var page = await LaunchAndGetPage(plugin); + await page.GoToAsync("https://google.com"); + + var results = await page.EvaluateFunctionAsync(@"() => { + const results = {} + + const iframe = document.createElement('iframe') + iframe.srcdoc = 'page intentionally left blank' // Note: srcdoc + document.body.appendChild(iframe) + + const basicIframe = document.createElement('iframe') + basicIframe.src = 'data:text/plain;charset=utf-8,foobar' + document.body.appendChild(iframe) + + results.descriptorsOK = (() => { + // Verify iframe prototype isn't touched + const descriptors = Object.getOwnPropertyDescriptors( + HTMLIFrameElement.prototype + ) + const str = descriptors.contentWindow.get.toString() + return str === `function get contentWindow() { [native code] }` + })() + + results.noProxySignature = (() => { + return iframe.srcdoc.toString.hasOwnProperty('[[IsRevoked]]') // eslint-disable-line + })() + + results.doesExist = (() => { + // Verify iframe isn't remapped to main window + return !!iframe.contentWindow + })() + + results.isNotAClone = (() => { + // Verify iframe isn't remapped to main window + return iframe.contentWindow !== window + })() + + results.hasSameNumberOfPlugins = (() => { + return ( + window.navigator.plugins.length === + iframe.contentWindow.navigator.plugins.length + ) + })() + + results.SelfIsNotWindow = (() => { + return iframe.contentWindow.self !== window + })() + + results.SelfIsNotWindowTop = (() => { + return iframe.contentWindow.self !== window.top + })() + + results.TopIsNotSame = (() => { + return iframe.contentWindow.top !== iframe.contentWindow + })() + + results.FrameElementMatches = (() => { + return iframe.contentWindow.frameElement === iframe + })() + + results.StackTraces = (() => { + try { + // eslint-disable-next-line + document['createElement'](0) + } catch (e) { + return e.stack + } + return false + })() + + return results + }"); + + + Assert.True(results.Value("descriptorsOK")); + Assert.True(results.Value("doesExist")); + Assert.True(results.Value("isNotAClone")); + Assert.True(results.Value("hasSameNumberOfPlugins")); + Assert.True(results.Value("SelfIsNotWindow")); + Assert.True(results.Value("SelfIsNotWindowTop")); + Assert.True(results.Value("TopIsNotSame")); + + Assert.DoesNotContain("at Object.apply", results["StackTraces"]); + } + } +} \ No newline at end of file diff --git a/Tests/StealthPluginTests/EvasionsTests/FrameTest.cs b/Tests/StealthPluginTests/EvasionsTests/FrameTest.cs deleted file mode 100644 index 3514bca..0000000 --- a/Tests/StealthPluginTests/EvasionsTests/FrameTest.cs +++ /dev/null @@ -1,7 +0,0 @@ -namespace Extra.Tests.StealthPluginTests.EvasionsTests -{ - public class FrameTest: BrowserDefault - { - - } -} diff --git a/Tests/StealthPluginTests/EvasionsTests/LanguagesTest.cs b/Tests/StealthPluginTests/EvasionsTests/LanguagesTest.cs index 8f8aabc..b2ace62 100644 --- a/Tests/StealthPluginTests/EvasionsTests/LanguagesTest.cs +++ b/Tests/StealthPluginTests/EvasionsTests/LanguagesTest.cs @@ -20,12 +20,20 @@ public async Task ShouldWork() var fingerPrint = await new FingerPrint().GetFingerPrint(page); Assert.Contains("en-US", fingerPrint["languages"].Select(e => e.Value())); + } + + + [Fact] + public async Task ShouldWorkWithCustomSettings() + { + var plugin = new Languages(new StealthLanguagesOptions("fr-FR")); + var page = await LaunchAndGetPage(plugin); - var property = await page.EvaluateExpressionAsync("Object.getOwnPropertyDescriptor(navigator, 'languages')"); - Assert.Null(property); + await page.GoToAsync("https://google.com"); + + var fingerPrint = await new FingerPrint().GetFingerPrint(page); - var navigator = await page.EvaluateExpressionAsync("navigator"); - Assert.Empty(navigator); + Assert.Contains("fr-FR", fingerPrint["languages"].Select(e => e.Value())); } } } diff --git a/Tests/StealthPluginTests/EvasionsTests/PermissionsTest.cs b/Tests/StealthPluginTests/EvasionsTests/PermissionsTest.cs index 877174c..19fe479 100644 --- a/Tests/StealthPluginTests/EvasionsTests/PermissionsTest.cs +++ b/Tests/StealthPluginTests/EvasionsTests/PermissionsTest.cs @@ -8,15 +8,16 @@ namespace Extra.Tests.StealthPluginTests.EvasionsTests public class PermissionsTest: BrowserDefault { [Fact] - public async Task ShouldBeDenied() + public async Task ShouldBeDeniedInHttpSite() { var plugin = new Permissions(); var page = await LaunchAndGetPage(plugin); - await page.GoToAsync("https://google.com"); + await page.GoToAsync("http://info.cern.ch/"); var finger = await new FingerPrint().GetFingerPrint(page); Assert.Equal("denied", finger["permissions"]["state"]); + Assert.Equal("denied", finger["permissions"]["permission"]); } } } diff --git a/Tests/StealthPluginTests/EvasionsTests/PluginEvasionTest.cs b/Tests/StealthPluginTests/EvasionsTests/PluginEvasionTest.cs index c3e273d..396d47c 100644 --- a/Tests/StealthPluginTests/EvasionsTests/PluginEvasionTest.cs +++ b/Tests/StealthPluginTests/EvasionsTests/PluginEvasionTest.cs @@ -1,8 +1,26 @@ -namespace Extra.Tests.StealthPluginTests.EvasionsTests +using System.Linq; +using System.Threading.Tasks; +using Extra.Tests.Utils; +using PuppeteerExtraSharp.Plugins.ExtraStealth.Evasions; +using Xunit; + +namespace Extra.Tests.StealthPluginTests.EvasionsTests { public class PluginEvasionTest: BrowserDefault { - //[Fact] - //public async Task S + [Fact] + public async Task ShouldNotHaveModifications() + { + var stealthPlugin = new PluginEvasion(); + var page = await LaunchAndGetPage(stealthPlugin); + + await page.GoToAsync("https://google.com"); + + + var fingerPrint = await new FingerPrint().GetFingerPrint(page); + + Assert.Equal(3, fingerPrint["plugins"].Count()); + Assert.Equal(4, fingerPrint["mimeTypes"].Count()); + } } } diff --git a/Tests/StealthPluginTests/EvasionsTests/RuntimeTest.cs b/Tests/StealthPluginTests/EvasionsTests/RuntimeTest.cs index 8342e00..10bda7e 100644 --- a/Tests/StealthPluginTests/EvasionsTests/RuntimeTest.cs +++ b/Tests/StealthPluginTests/EvasionsTests/RuntimeTest.cs @@ -48,7 +48,7 @@ public async Task ShouldAddConnectToChrome() } - private async Task AssertThrowsConnect(Page page, string error, params object[] args) + private async Task AssertThrowsConnect(IPage page, string error, params object[] args) { var start = "Evaluation failed: TypeError: Error in invocation of runtime.connect(optional string extensionId, optional object connectInfo): "; diff --git a/Tests/StealthPluginTests/EvasionsTests/SourceUrl/SourceUrlTest.cs b/Tests/StealthPluginTests/EvasionsTests/SourceUrl/SourceUrlTest.cs new file mode 100644 index 0000000..952c6eb --- /dev/null +++ b/Tests/StealthPluginTests/EvasionsTests/SourceUrl/SourceUrlTest.cs @@ -0,0 +1,36 @@ +using System; +using System.IO; +using System.Threading.Tasks; +using PuppeteerExtraSharp.Plugins.ExtraStealth.Evasions; +using PuppeteerSharp; +using Xunit; + +namespace Extra.Tests.StealthPluginTests.EvasionsTests.SourceUrl +{ + public class SourceUrlTest : BrowserDefault + { + private readonly string _pageUrl = Environment.CurrentDirectory + "\\StealthPluginTests\\EvasionsTests\\SourceUrl\\fixtures\\Test.html"; + + [Fact] + public async Task ShouldWork() + { + var plugin = new PuppeteerExtraSharp.Plugins.ExtraStealth.Evasions.SourceUrl(); + + var page = await LaunchAndGetPage(plugin); + await page.GoToAsync(_pageUrl, WaitUntilNavigation.Load); + + await page.EvaluateExpressionAsync("document.querySelector('title')"); + var result = await page.EvaluateExpressionAsync("document.querySelector('#result').innerText"); + + var result2 = await page.EvaluateFunctionAsync(@"() => { + try { + Function.prototype.toString.apply({}) + } catch (err) { + return err.stack + }}"); + + Assert.Equal("PASS", result); + Assert.DoesNotContain("__puppeteer_evaluation_script", result2); + } + } +} \ No newline at end of file diff --git a/Tests/StealthPluginTests/EvasionsTests/SourceUrl/fixtures/Test.html b/Tests/StealthPluginTests/EvasionsTests/SourceUrl/fixtures/Test.html new file mode 100644 index 0000000..311847f --- /dev/null +++ b/Tests/StealthPluginTests/EvasionsTests/SourceUrl/fixtures/Test.html @@ -0,0 +1,36 @@ + + + + + IPage Title + + + +

Please use `document.querySelector`..

+ + + + \ No newline at end of file diff --git a/Tests/StealthPluginTests/EvasionsTests/VendorTest.cs b/Tests/StealthPluginTests/EvasionsTests/VendorTest.cs index 6eeb8c8..37a0657 100644 --- a/Tests/StealthPluginTests/EvasionsTests/VendorTest.cs +++ b/Tests/StealthPluginTests/EvasionsTests/VendorTest.cs @@ -14,7 +14,7 @@ public async Task ShouldWork() await page.GoToAsync("https://google.com"); var vendor = await page.EvaluateExpressionAsync("navigator.vendor"); - Assert.Equal("Intel inc.", vendor); + Assert.Equal("Google Inc.", vendor); } } } diff --git a/Tests/StealthPluginTests/EvasionsTests/WebDriverTest.cs b/Tests/StealthPluginTests/EvasionsTests/WebDriverTest.cs index 79faa8a..63584a5 100644 --- a/Tests/StealthPluginTests/EvasionsTests/WebDriverTest.cs +++ b/Tests/StealthPluginTests/EvasionsTests/WebDriverTest.cs @@ -4,7 +4,7 @@ namespace Extra.Tests.StealthPluginTests.EvasionsTests { - public class WebDriverTest: BrowserDefault + public class WebDriverTest : BrowserDefault { [Fact] public async Task ShouldWork() @@ -13,9 +13,19 @@ public async Task ShouldWork() var page = await LaunchAndGetPage(plugin); await page.GoToAsync("https://google.com"); - var driver = await page.EvaluateExpressionAsync("navigator.webdriver"); + var driver = await page.EvaluateExpressionAsync("navigator.webdriver"); + Assert.False(driver); + } - Assert.Null(driver); + [Fact] + public async Task WontKillOtherMethods() + { + var plugin = new WebDriver(); + var page = await LaunchAndGetPage(plugin); + await page.GoToAsync("https://google.com"); + + var data = await page.EvaluateExpressionAsync("navigator.javaEnabled()"); + Assert.False(data); } } -} +} \ No newline at end of file diff --git a/Tests/StealthPluginTests/StealthPluginTests.cs b/Tests/StealthPluginTests/StealthPluginTests.cs index e36f43c..1925059 100644 --- a/Tests/StealthPluginTests/StealthPluginTests.cs +++ b/Tests/StealthPluginTests/StealthPluginTests.cs @@ -16,8 +16,8 @@ public async Task ShouldBeNotDetected() var page = await LaunchAndGetPage(plugin); await page.GoToAsync("https://google.com"); - var webdriver = await page.EvaluateExpressionAsync("navigator.webdriver"); - Assert.Null(webdriver); + var webdriver = await page.EvaluateExpressionAsync("navigator.webdriver"); + Assert.False(webdriver); var headlessUserAgent = await page.EvaluateExpressionAsync("window.navigator.userAgent"); Assert.DoesNotContain("Headless", headlessUserAgent); @@ -25,7 +25,7 @@ public async Task ShouldBeNotDetected() var webDriverOverriden = await page.EvaluateExpressionAsync( "Object.getOwnPropertyDescriptor(navigator.__proto__, 'webdriver') !== undefined"); - Assert.False(webDriverOverriden); + Assert.True(webDriverOverriden); var plugins = await page.EvaluateExpressionAsync("navigator.plugins.length"); Assert.NotEqual(0, plugins); diff --git a/Tests/Utils/FingerPrint.cs b/Tests/Utils/FingerPrint.cs index 2296ead..2aa1bec 100644 --- a/Tests/Utils/FingerPrint.cs +++ b/Tests/Utils/FingerPrint.cs @@ -13,7 +13,7 @@ public class FingerPrint /// /// /// - public async Task GetFingerPrint(Page page) + public async Task GetFingerPrint(IPage page) { var script = ResourcesReader.ReadFile("Extra.Tests.StealthPluginTests.Script.fpCollect.js", Assembly.GetExecutingAssembly()); await page.EvaluateExpressionAsync(script);