From 4e7b87f696eaac6df5a9a16c9c83d399cbdc845e Mon Sep 17 00:00:00 2001 From: Michael Mintz Date: Mon, 11 Sep 2023 12:03:20 -0400 Subject: [PATCH 1/5] Fix issue with authenticated proxy and proxy_bypass_list --- seleniumbase/core/browser_launcher.py | 20 ++++++++++++++++---- seleniumbase/core/proxy_helper.py | 17 ++++++++++++----- 2 files changed, 28 insertions(+), 9 deletions(-) diff --git a/seleniumbase/core/browser_launcher.py b/seleniumbase/core/browser_launcher.py index e057316c08f..9a6408792b2 100644 --- a/seleniumbase/core/browser_launcher.py +++ b/seleniumbase/core/browser_launcher.py @@ -501,6 +501,7 @@ def _add_chrome_proxy_extension( proxy_string, proxy_user, proxy_pass, + proxy_bypass_list=None, zip_it=True, multi_proxy=False, ): @@ -508,18 +509,21 @@ def _add_chrome_proxy_extension( https://stackoverflow.com/questions/12848327/ (Run Selenium on a proxy server that requires authentication.)""" args = " ".join(sys.argv) + bypass_list = proxy_bypass_list if ( not ("-n" in sys.argv or " -n=" in args or args == "-c") and not multi_proxy ): # Single-threaded if zip_it: - proxy_helper.create_proxy_ext(proxy_string, proxy_user, proxy_pass) + proxy_helper.create_proxy_ext( + proxy_string, proxy_user, proxy_pass, bypass_list + ) proxy_zip = proxy_helper.PROXY_ZIP_PATH chrome_options.add_extension(proxy_zip) else: proxy_helper.create_proxy_ext( - proxy_string, proxy_user, proxy_pass, zip_it=False + proxy_string, proxy_user, proxy_pass, bypass_list, zip_it=False ) proxy_dir_path = proxy_helper.PROXY_DIR_PATH chrome_options = add_chrome_ext_dir(chrome_options, proxy_dir_path) @@ -532,7 +536,7 @@ def _add_chrome_proxy_extension( _set_proxy_filenames() if not os.path.exists(proxy_helper.PROXY_ZIP_PATH): proxy_helper.create_proxy_ext( - proxy_string, proxy_user, proxy_pass + proxy_string, proxy_user, proxy_pass, bypass_list ) proxy_zip = proxy_helper.PROXY_ZIP_PATH chrome_options.add_extension(proxy_zip) @@ -543,7 +547,11 @@ def _add_chrome_proxy_extension( _set_proxy_filenames() if not os.path.exists(proxy_helper.PROXY_DIR_PATH): proxy_helper.create_proxy_ext( - proxy_string, proxy_user, proxy_pass, False + proxy_string, + proxy_user, + proxy_pass, + bypass_list, + False, ) chrome_options = add_chrome_ext_dir( chrome_options, proxy_helper.PROXY_DIR_PATH @@ -843,6 +851,7 @@ def _set_chrome_options( proxy_string, proxy_user, proxy_pass, + proxy_bypass_list, zip_it, multi_proxy, ) @@ -861,6 +870,7 @@ def _set_chrome_options( None, proxy_user, proxy_pass, + proxy_bypass_list, zip_it, multi_proxy, ) @@ -2402,6 +2412,7 @@ def get_local_driver( proxy_string, proxy_user, proxy_pass, + proxy_bypass_list, zip_it=True, multi_proxy=multi_proxy, ) @@ -2417,6 +2428,7 @@ def get_local_driver( None, proxy_user, proxy_pass, + proxy_bypass_list, zip_it=True, multi_proxy=multi_proxy, ) diff --git a/seleniumbase/core/proxy_helper.py b/seleniumbase/core/proxy_helper.py index 875b9e42b70..a97e4a829d0 100644 --- a/seleniumbase/core/proxy_helper.py +++ b/seleniumbase/core/proxy_helper.py @@ -14,7 +14,9 @@ PROXY_DIR_LOCK = os.path.join(DOWNLOADS_DIR, "proxy_dir.lock") -def create_proxy_ext(proxy_string, proxy_user, proxy_pass, zip_it=True): +def create_proxy_ext( + proxy_string, proxy_user, proxy_pass, bypass_list=None, zip_it=True +): """Implementation of https://stackoverflow.com/a/35293284 for https://stackoverflow.com/questions/12848327/ (Run Selenium on a proxy server that requires authentication.) @@ -22,6 +24,8 @@ def create_proxy_ext(proxy_string, proxy_user, proxy_pass, zip_it=True): CHROMIUM-ONLY! *** Only Chrome and Edge browsers are supported. *** """ background_js = None + if not bypass_list: + bypass_list = "" if proxy_string: proxy_host = proxy_string.split(":")[0] proxy_port = proxy_string.split(":")[1] @@ -31,10 +35,10 @@ def create_proxy_ext(proxy_string, proxy_user, proxy_pass, zip_it=True): """ rules: {\n""" """ singleProxy: {\n""" """ scheme: "http",\n""" - """ bypassList: [],\n""" """ host: "%s",\n""" """ port: parseInt("%s")\n""" """ },\n""" + """ bypassList: ["%s"]\n""" """ }\n""" """ };\n""" """chrome.proxy.settings.set(""" @@ -52,14 +56,17 @@ def create_proxy_ext(proxy_string, proxy_user, proxy_pass, zip_it=True): """ callbackFn,\n""" """ {urls: [""]},\n""" """ ['blocking']\n""" - """);""" % (proxy_host, proxy_port, proxy_user, proxy_pass) + """);""" % ( + proxy_host, proxy_port, bypass_list, proxy_user, proxy_pass + ) ) else: background_js = ( """var config = {\n""" """ mode: "fixed_servers",\n""" """ rules: {\n""" - """ }\n""" + """ },\n""" + """ bypassList: ["%s"]\n""" """ };\n""" """chrome.proxy.settings.set(""" """{value: config, scope: "regular"}, function() {""" @@ -76,7 +83,7 @@ def create_proxy_ext(proxy_string, proxy_user, proxy_pass, zip_it=True): """ callbackFn,\n""" """ {urls: [""]},\n""" """ ['blocking']\n""" - """);""" % (proxy_user, proxy_pass) + """);""" % (bypass_list, proxy_user, proxy_pass) ) manifest_json = ( """{\n""" From c9b895e6aaf00d45ca6fa123d95fcb4f492fb62c Mon Sep 17 00:00:00 2001 From: Michael Mintz Date: Mon, 11 Sep 2023 12:04:27 -0400 Subject: [PATCH 2/5] Update default Edge preferences --- seleniumbase/core/browser_launcher.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/seleniumbase/core/browser_launcher.py b/seleniumbase/core/browser_launcher.py index 9a6408792b2..7af6301e15d 100644 --- a/seleniumbase/core/browser_launcher.py +++ b/seleniumbase/core/browser_launcher.py @@ -2268,6 +2268,8 @@ def get_local_driver( prefs["enable_do_not_track"] = True if external_pdf: prefs["plugins.always_open_pdf_externally"] = True + pdce = "user_experience_metrics.personalization_data_consent_enabled" + prefs[pdce] = True # Remove "Personalize your web experience" prompt edge_options.add_experimental_option("prefs", prefs) edge_options.add_argument( "--disable-blink-features=AutomationControlled" From aa751f6f763b4bac7b1246d03b039bd8137920e6 Mon Sep 17 00:00:00 2001 From: Michael Mintz Date: Mon, 11 Sep 2023 12:05:18 -0400 Subject: [PATCH 3/5] Refresh mkdocs dependencies --- mkdocs_build/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mkdocs_build/requirements.txt b/mkdocs_build/requirements.txt index 776180366c2..b8d8559acb4 100644 --- a/mkdocs_build/requirements.txt +++ b/mkdocs_build/requirements.txt @@ -20,7 +20,7 @@ paginate==0.5.6 pyquery==2.0.0 readtime==3.0.0 mkdocs==1.5.2 -mkdocs-material==9.2.8 +mkdocs-material==9.3.0 mkdocs-exclude-search==0.6.5 mkdocs-simple-hooks==0.1.5 mkdocs-material-extensions==1.1.1 From a5520521638097329f7554adcdcbd6988cd4eec3 Mon Sep 17 00:00:00 2001 From: Michael Mintz Date: Mon, 11 Sep 2023 12:33:37 -0400 Subject: [PATCH 4/5] Refresh mkdocs dependencies --- mkdocs_build/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mkdocs_build/requirements.txt b/mkdocs_build/requirements.txt index b8d8559acb4..e324caf8777 100644 --- a/mkdocs_build/requirements.txt +++ b/mkdocs_build/requirements.txt @@ -20,7 +20,7 @@ paginate==0.5.6 pyquery==2.0.0 readtime==3.0.0 mkdocs==1.5.2 -mkdocs-material==9.3.0 +mkdocs-material==9.3.1 mkdocs-exclude-search==0.6.5 mkdocs-simple-hooks==0.1.5 mkdocs-material-extensions==1.1.1 From efc902eabe17b8f02d57dc9dfec34e03c2e25a6a Mon Sep 17 00:00:00 2001 From: Michael Mintz Date: Mon, 11 Sep 2023 12:36:49 -0400 Subject: [PATCH 5/5] Version 4.18.5 --- seleniumbase/__version__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/seleniumbase/__version__.py b/seleniumbase/__version__.py index dd499105475..e64a576b395 100755 --- a/seleniumbase/__version__.py +++ b/seleniumbase/__version__.py @@ -1,2 +1,2 @@ # seleniumbase package -__version__ = "4.18.4" +__version__ = "4.18.5"