Merge pull request #158 from seleniumbase/ad-blocking-functionality

Add ad-blocking functionality

Author: mdmintz

help_docs/method_summary.md

@@ -114,6 +114,8 @@ self.remove_element(selector, by=By.CSS_SELECTOR)
 
 self.remove_elements(selector, by=By.CSS_SELECTOR)
 
+self.ad_block()
+
 self.get_domain_url(url)
 
 self.safe_execute_script(script)

seleniumbase/config/ad_block_list.py

@@ -0,0 +1,53 @@
+"""
+For use with SeleniumBase ad_block functionality.
+
+Usage:
+    On the command line:
+    "pytest SOME_TEST.py --ad_block"
+
+    From inside a test:
+    self.ad_block()
+
+If using the command line version, the ad_block functionality gets
+activated after "self.wait_for_ready_state_complete()" is called,
+which is always run after page loads, unless changed in "settings.py".
+Using ad_block will slow down test runs a little. (Use only if necessary.)
+
+Format: A CSS Selector that's ready for JavaScript's querySelectorAll()
+"""
+
+AD_BLOCK_LIST = [
+    '[aria-label="Ad"]',
+    '[class^="sponsored-content"]',
+    '[data-ad-details*="Advertisement"]',
+    '[data-native_ad*="placement"]',
+    '[data-provider="dianomi"]',
+    '[data-type="ad"]',
+    '[data-track-event-label*="-taboola-"]',
+    '[href*="doubleclick.net/"]',
+    '[id*="-ad-"]',
+    '[id*="_ads_"]',
+    '[id*="AdFrame"]',
+    '[id^="ad-"]',
+    '[id^="outbrain_widget"]',
+    '[id^="taboola-"]',
+    '[id="dianomiRightRail"]',
+    '[src*="smartads."]',
+    '[src*="ad_nexus"]',
+    '[src*="/ads/"]',
+    '[data-dcm-click-tracker*="/adclick."]',
+    '[data-google-query-id^="C"]',
+    'div.ad-container',
+    'div.ad_module',
+    'div.ad-subnav-container',
+    'div.ad-wrapper',
+    'div.data-ad-container',
+    'div.l-ad',
+    'div.right-ad',
+    'div.wx-adWrapper',
+    'img.img_ad',
+    'link[href*="/adservice."]',
+    'script[src*="/adservice."]',
+    'script[src*="/pagead/"]',
+    'section.dianomi-ad',
+]

seleniumbase/config/proxy_list.py

@@ -8,7 +8,9 @@
 "pytest SOME_TEST.py --proxy=proxy1"
 
 Format of PROXY_LIST server entries:
-* "ip_address:port"
+* "ip_address:port" OR
+* "server:port"
+(Do NOT include the http:// or https:// in your proxy string!)
 
 Example proxies in PROXY_LIST below are not guaranteed to be active or secure.
 If you don't already have a proxy server to connect to,

seleniumbase/core/log_helper.py

@@ -10,7 +10,10 @@
 def log_screenshot(test_logpath, driver):
     screenshot_name = settings.SCREENSHOT_NAME
     screenshot_path = "%s/%s" % (test_logpath, screenshot_name)
-    driver.get_screenshot_as_file(screenshot_path)
+    try:
+        driver.get_screenshot_as_file(screenshot_path)
+    except Exception:
+        print("WARNING: Unable to get screenshot for failure logs!")
 
 
 def log_test_failure_data(test, test_logpath, driver, browser):

seleniumbase/fixtures/base_case.py

@@ -36,6 +36,7 @@ def test_anything(self):
 from bs4 import BeautifulSoup
 from pyvirtualdisplay import Display
 from seleniumbase.common import decorators
+from seleniumbase.config import ad_block_list
 from seleniumbase.config import settings
 from seleniumbase.core.application_manager import ApplicationManager
 from seleniumbase.core.s3_manager import S3LoggingBucket
@@ -75,13 +76,15 @@ def __init__(self, *args, **kwargs):
         self.driver = None
         self.environment = None
         self._last_url_of_delayed_assert = "data:,"
+        self._last_page_load_url = "data:,"
         self._page_check_count = 0
         self._page_check_failures = []
         self._html_report_extra = []
         self._default_driver = None
         self._drivers_list = []
 
     def open(self, url):
+        self._last_page_load_url = None
         self.driver.get(url)
         if settings.WAIT_FOR_RSC_ON_PAGE_LOADS:
             self.wait_for_ready_state_complete()
@@ -411,11 +414,13 @@ def get_attribute(self, selector, attribute, by=By.CSS_SELECTOR,
                 selector, attribute))
 
     def refresh_page(self):
+        self._last_page_load_url = None
         self.driver.refresh()
+        self.wait_for_ready_state_complete()
 
     def refresh(self):
         """ The shorter version of self.refresh_page() """
-        self.driver.refresh()
+        self.refresh_page()
 
     def get_current_url(self):
         return self.driver.current_url
@@ -431,10 +436,14 @@ def get_title(self):
         return self.driver.title
 
     def go_back(self):
+        self._last_page_load_url = None
         self.driver.back()
+        self.wait_for_ready_state_complete()
 
     def go_forward(self):
+        self._last_page_load_url = None
         self.driver.forward()
+        self.wait_for_ready_state_complete()
 
     def get_image_url(self, selector, by=By.CSS_SELECTOR,
                       timeout=settings.SMALL_TIMEOUT):
@@ -978,6 +987,19 @@ def remove_elements(self, selector, by=By.CSS_SELECTOR):
         remove_script = """jQuery('%s').remove()""" % selector
         self.safe_execute_script(remove_script)
 
+    def ad_block(self):
+        for css_selector in ad_block_list.AD_BLOCK_LIST:
+            css_selector = re.escape(css_selector)
+            script = ("""var $elements = document.querySelectorAll('%s');
+                      var index = 0, length = $elements.length;
+                      for(; index < length; index++){
+                      $elements[index].remove();}"""
+                      % css_selector)
+            try:
+                self.execute_script(script)
+            except Exception:
+                pass  # Don't fail test if ad_blocking fails
+
     def jq_format(self, code):
         # DEPRECATED - Use re.escape() instead, which does the action you want.
         return page_utils._jq_format(code)
@@ -1077,6 +1099,8 @@ def set_value(self, selector, new_value, by=By.CSS_SELECTOR,
             element = self.wait_for_element_present(
                 orginal_selector, by=by, timeout=timeout)
             element.send_keys(Keys.RETURN)
+            if settings.WAIT_FOR_RSC_ON_PAGE_LOADS:
+                self.wait_for_ready_state_complete()
         self._demo_mode_pause_if_active()
 
     def jquery_update_text_value(self, selector, new_value, by=By.CSS_SELECTOR,
@@ -1441,6 +1465,17 @@ def wait_for_ready_state_complete(self, timeout=settings.EXTREME_TIMEOUT):
         is_ready = page_actions.wait_for_ready_state_complete(self.driver,
                                                               timeout)
         self.wait_for_angularjs(timeout=settings.MINI_TIMEOUT)
+        if self.ad_block_on:
+            # If the ad_block feature is enabled, then block ads for new URLs
+            current_url = self.get_current_url()
+            if not current_url == self._last_page_load_url:
+                time.sleep(0.02)
+                self.ad_block()
+                time.sleep(0.01)
+                if self.is_element_present("iframe"):
+                    time.sleep(0.07)  # iframe ads take slightly longer to load
+                    self.ad_block()  # Do ad_block on slower-loading iframes
+                self._last_page_load_url = current_url
         return is_ready
 
     def wait_for_angularjs(self, timeout=settings.LARGE_TIMEOUT, **kwargs):
@@ -1896,6 +1931,7 @@ def setUp(self):
             self.demo_mode = pytest.config.option.demo_mode
             self.demo_sleep = pytest.config.option.demo_sleep
             self.highlights = pytest.config.option.highlights
+            self.ad_block_on = pytest.config.option.ad_block_on
             self.verify_delay = pytest.config.option.verify_delay
             self.timeout_multiplier = pytest.config.option.timeout_multiplier
             self.use_grid = False

seleniumbase/plugins/pytest_plugin.py

@@ -88,7 +88,7 @@ def pytest_addoption(parser):
                      dest='headless',
                      default=False,
                      help="""Using this makes Webdriver run headlessly,
-                          which is useful inside a Linux Docker.""")
+                          which is required on headless machines.""")
     parser.addoption('--is_pytest', action="store_true",
                      dest='is_pytest',
                      default=True,
@@ -107,6 +107,11 @@ def pytest_addoption(parser):
                      default=None,
                      help="""Setting this overrides the default number of
                           highlight animation loops to have per call.""")
+    parser.addoption('--ad_block', action="store_true",
+                     dest='ad_block_on',
+                     default=False,
+                     help="""Using this makes WebDriver block display ads
+                          that are defined in ad_block_list.AD_BLOCK_LIST.""")
     parser.addoption('--verify_delay', action='store', dest='verify_delay',
                      default=None,
                      help="""Setting this overrides the default wait time

seleniumbase/plugins/selenium_plugin.py

@@ -17,10 +17,12 @@ class SeleniumBrowser(Plugin):
     self.options.browser -- the browser to use (--browser)
     self.options.server -- the server used by the test (--server)
     self.options.port -- the port used by the test (--port)
+    self.options.proxy -- designates the proxy server:port to use. (--proxy)
     self.options.headless -- the option to run headlessly (--headless)
     self.options.demo_mode -- the option to slow down Selenium (--demo_mode)
     self.options.demo_sleep -- Selenium action delay in DemoMode (--demo_sleep)
     self.options.highlights -- # of highlight animations shown (--highlights)
+    self.options.ad_block -- the option to block some display ads (--ad_block)
     self.options.verify_delay -- delay before MasterQA checks (--verify_delay)
     self.options.timeout_multiplier -- increase defaults (--timeout_multiplier)
     """
@@ -66,7 +68,7 @@ def options(self, parser, env):
             dest='headless',
             default=False,
             help="""Using this makes Webdriver run headlessly,
-                    which is useful inside a Linux Docker.""")
+                    which is required on headless machines.""")
         parser.add_option(
             '--demo_mode', action="store_true",
             dest='demo_mode',
@@ -83,6 +85,12 @@ def options(self, parser, env):
             dest='highlights', default=None,
             help="""Setting this overrides the default number of
                     highlight animation loops to have per call.""")
+        parser.add_option(
+            '--ad_block', action="store_true",
+            dest='ad_block_on',
+            default=False,
+            help="""Using this makes WebDriver block display ads
+                    that are defined in ad_block_list.AD_BLOCK_LIST.""")
         parser.add_option(
             '--verify_delay', action='store',
             dest='verify_delay', default=None,
@@ -103,9 +111,6 @@ def configure(self, options, conf):
         self.headless_active = False  # Default setting
 
     def beforeTest(self, test):
-        """ Running Selenium locally will be handled differently
-            from how Selenium is run remotely, such as from Jenkins. """
-
         test.test.browser = self.options.browser
         test.test.headless = self.options.headless
         test.test.servername = self.options.servername
@@ -114,6 +119,7 @@ def beforeTest(self, test):
         test.test.demo_mode = self.options.demo_mode
         test.test.demo_sleep = self.options.demo_sleep
         test.test.highlights = self.options.highlights
+        test.test.ad_block_on = self.options.ad_block_on
         test.test.verify_delay = self.options.verify_delay  # MasterQA
         test.test.timeout_multiplier = self.options.timeout_multiplier
         test.test.use_grid = False

setup.py

@@ -7,7 +7,7 @@
 
 setup(
     name='seleniumbase',
-    version='1.8.4',
+    version='1.8.5',
     description='Web Automation & Testing Framework - http://seleniumbase.com',
     long_description='Web Automation and Testing Framework - seleniumbase.com',
     platforms='Mac * Windows * Linux * Docker',