Skip to content

Commit

Permalink
Merge pull request #3268 from seleniumbase/cdp-mode-patch-10
Browse files Browse the repository at this point in the history
CDP Mode - Patch 10
  • Loading branch information
mdmintz authored Nov 14, 2024
2 parents ffdd6c9 + f472788 commit d4cde11
Show file tree
Hide file tree
Showing 22 changed files with 746 additions and 145 deletions.
24 changes: 20 additions & 4 deletions examples/cdp_mode/ReadMe.md
Original file line number Diff line number Diff line change
Expand Up @@ -323,19 +323,22 @@ sb.cdp.get_active_element_css()
sb.cdp.click(selector)
sb.cdp.click_active_element()
sb.cdp.click_if_visible(selector)
sb.cdp.click_visible_elements(selector)
sb.cdp.mouse_click(selector)
sb.cdp.nested_click(parent_selector, selector)
sb.cdp.get_nested_element(parent_selector, selector)
sb.cdp.flash(selector)
sb.cdp.select_option_by_text(dropdown_selector, option)
sb.cdp.flash(selector, duration=1, color="44CC88", pause=0)
sb.cdp.highlight(selector)
sb.cdp.focus(selector)
sb.cdp.highlight_overlay(selector)
sb.cdp.remove_element(selector)
sb.cdp.remove_from_dom(selector)
sb.cdp.remove_elements(selector)
sb.cdp.scroll_into_view(selector)
sb.cdp.send_keys(selector, text)
sb.cdp.press_keys(selector, text)
sb.cdp.type(selector, text)
sb.cdp.set_value(selector, text)
sb.cdp.evaluate(expression)
sb.cdp.js_dumps(obj_name)
sb.cdp.maximize()
Expand Down Expand Up @@ -373,6 +376,11 @@ sb.cdp.gui_press_keys(keys)
sb.cdp.gui_write(text)
sb.cdp.gui_click_x_y(x, y)
sb.cdp.gui_click_element(selector)
sb.cdp.gui_drag_drop_points(x1, y1, x2, y2)
sb.cdp.gui_drag_and_drop(drag_selector, drop_selector)
sb.cdp.gui_hover_x_y(x, y)
sb.cdp.gui_hover_element(selector)
sb.cdp.gui_hover_and_click(hover_selector, click_selector)
sb.cdp.internalize_links()
sb.cdp.is_checked(selector)
sb.cdp.is_selected(selector)
Expand All @@ -382,12 +390,20 @@ sb.cdp.uncheck_if_checked(selector)
sb.cdp.unselect_if_selected(selector)
sb.cdp.is_element_present(selector)
sb.cdp.is_element_visible(selector)
sb.cdp.assert_element(selector)
sb.cdp.assert_element_present(selector)
sb.cdp.assert_element_absent(selector)
sb.cdp.assert_element(selector)
sb.cdp.assert_element_visible(selector)
sb.cdp.assert_element_not_visible(selector)
sb.cdp.assert_title(title)
sb.cdp.assert_text(text, selector="html")
sb.cdp.assert_exact_text(text, selector="html")
sb.cdp.scroll_down(amount=25)
sb.cdp.scroll_into_view(selector)
sb.cdp.scroll_to_y(y)
sb.cdp.scroll_to_top()
sb.cdp.scroll_to_bottom()
sb.cdp.scroll_up(amount=25)
sb.cdp.scroll_down(amount=25)
sb.cdp.save_screenshot(name, folder=None, selector=None)
```

Expand Down
2 changes: 1 addition & 1 deletion examples/cdp_mode/raw_albertsons.py
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@
item.scroll_into_view()
sb.sleep(0.025)
if required_text in item.text:
item.flash()
item.flash(color="44CC88")
sb.sleep(0.025)
if item.text not in unique_item_text:
unique_item_text.append(item.text)
Expand Down
10 changes: 10 additions & 0 deletions examples/cdp_mode/raw_browserscan.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
from seleniumbase import SB

with SB(uc=True, test=True, ad_block=True) as sb:
url = "https://www.browserscan.net/bot-detection"
sb.activate_cdp_mode(url)
sb.sleep(1)
sb.cdp.flash("Test Results", duration=4)
sb.sleep(1)
sb.cdp.assert_element('strong:contains("Normal")')
sb.cdp.flash('strong:contains("Normal")', duration=4, pause=4)
2 changes: 1 addition & 1 deletion examples/cdp_mode/raw_cdp_with_sb.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
with SB(uc=True, test=True, locale_code="en") as sb:
url = "https://www.priceline.com/"
sb.activate_cdp_mode(url)
sb.sleep(3)
sb.sleep(2.5)
sb.internalize_links() # Don't open links in a new tab
sb.click("#link_header_nav_experiences")
sb.sleep(2.5)
Expand Down
2 changes: 1 addition & 1 deletion examples/cdp_mode/raw_footlocker.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
with SB(uc=True, test=True, locale_code="en") as sb:
url = "https://www.footlocker.com/"
sb.activate_cdp_mode(url)
sb.sleep(3)
sb.sleep(2.5)
sb.cdp.click_if_visible('button[id*="Agree"]')
sb.sleep(1.5)
sb.cdp.mouse_click('input[aria-label="Search"]')
Expand Down
2 changes: 1 addition & 1 deletion examples/cdp_mode/raw_nike.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
with SB(uc=True, test=True, locale_code="en") as sb:
url = "https://www.nike.com/"
sb.activate_cdp_mode(url)
sb.sleep(3)
sb.sleep(2.5)
sb.cdp.gui_click_element('div[data-testid="user-tools-container"]')
sb.sleep(1.5)
search = "Nike Air Force 1"
Expand Down
4 changes: 2 additions & 2 deletions examples/cdp_mode/raw_planetmc.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,6 @@
with SB(uc=True, test=True, locale_code="en") as sb:
url = "www.planetminecraft.com/account/sign_in/"
sb.activate_cdp_mode(url)
sb.sleep(1)
sb.cdp.gui_click_element("#turnstile-widget")
sb.sleep(2)
sb.cdp.gui_click_element("#turnstile-widget div")
sb.sleep(2)
16 changes: 9 additions & 7 deletions examples/cdp_mode/raw_pokemon.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,22 +3,24 @@
with SB(uc=True, test=True, locale_code="en") as sb:
url = "https://www.pokemon.com/us"
sb.activate_cdp_mode(url)
sb.sleep(3)
sb.sleep(2.5)
sb.cdp.click_if_visible("button#onetrust-reject-all-handler")
sb.sleep(1)
sb.sleep(1.2)
sb.cdp.click('a[href="https://www.pokemon.com/us/pokedex/"]')
sb.sleep(1)
sb.sleep(1.2)
sb.cdp.click('b:contains("Show Advanced Search")')
sb.sleep(1)
sb.sleep(1.2)
sb.cdp.click('span[data-type="type"][data-value="electric"]')
sb.sleep(1)
sb.sleep(0.5)
sb.scroll_into_view("a#advSearch")
sb.sleep(0.5)
sb.cdp.click("a#advSearch")
sb.sleep(1)
sb.sleep(1.2)
sb.cdp.click('img[src*="img/pokedex/detail/025.png"]')
sb.cdp.assert_text("Pikachu", 'div[class*="title"]')
sb.cdp.assert_element('img[alt="Pikachu"]')
sb.cdp.scroll_into_view("div.pokemon-ability-info")
sb.sleep(1)
sb.sleep(1.2)
sb.cdp.flash('div[class*="title"]')
sb.cdp.flash('img[alt="Pikachu"]')
sb.cdp.flash("div.pokemon-ability-info")
Expand Down
10 changes: 7 additions & 3 deletions examples/cdp_mode/raw_priceline.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
window_handle = sb.driver.current_window_handle
url = "https://www.priceline.com"
sb.activate_cdp_mode(url)
sb.sleep(3)
sb.sleep(2.5)
sb.cdp.click('input[name="endLocation"]')
sb.sleep(1)
location = "Portland, OR, USA"
Expand All @@ -16,14 +16,17 @@
sb.cdp.click(selection)
sb.sleep(1.5)
sb.cdp.click('button[aria-label="Dismiss calendar"]')
sb.sleep(5)
sb.sleep(4.5)
sb.connect()
if len(sb.driver.window_handles) > 1:
sb.switch_to_window(window_handle)
sb.driver.close()
sb.sleep(0.2)
sb.switch_to_newest_window()
sb.sleep(0.6)
for y in range(1, 9):
sb.scroll_to_y(y * 400)
sb.sleep(1.25)
hotel_names = sb.find_elements('a[data-autobot-element-id*="HOTEL_NAME"]')
hotel_prices = sb.find_elements('span[font-size="4,,,5"]')
print("Priceline Hotels in %s:" % location)
Expand All @@ -34,4 +37,5 @@
for i, hotel in enumerate(hotel_names):
if hotel_prices[i] and hotel_prices[i].text:
count += 1
print("* %s: %s => %s" % (count, hotel.text, hotel_prices[i].text))
hotel_price = "$" + hotel_prices[i].text
print("* %s: %s => %s" % (count, hotel.text, hotel_price))
92 changes: 92 additions & 0 deletions examples/cdp_mode/raw_xhr_sb.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,92 @@
"""CDP.network.ResponseReceived with CDP.network.ResourceType.XHR."""
import ast
import asyncio
import colorama
import mycdp
import sys
import time
from seleniumbase.undetected import cdp_driver

xhr_requests = []
last_xhr_request = None
c1 = colorama.Fore.BLUE + colorama.Back.LIGHTYELLOW_EX
c2 = colorama.Fore.BLUE + colorama.Back.LIGHTGREEN_EX
cr = colorama.Style.RESET_ALL
if "linux" in sys.platform:
c1 = c2 = cr = ""


def listenXHR(page):
async def handler(evt):
# Get AJAX requests
if evt.type_ is mycdp.network.ResourceType.XHR:
xhr_requests.append([evt.response.url, evt.request_id])
global last_xhr_request
last_xhr_request = time.time()
page.add_handler(mycdp.network.ResponseReceived, handler)


async def receiveXHR(page, requests):
responses = []
retries = 0
max_retries = 5
# Wait at least 2 seconds after last XHR request for more
while True:
if last_xhr_request is None or retries > max_retries:
break
if time.time() - last_xhr_request <= 2:
retries = retries + 1
time.sleep(2)
continue
else:
break
await page
# Loop through gathered requests and get response body
for request in requests:
try:
res = await page.send(mycdp.network.get_response_body(request[1]))
if res is None:
continue
responses.append({
"url": request[0],
"body": res[0],
"is_base64": res[1],
})
except Exception as e:
print("Error getting response:", e)
return responses


async def crawl():
driver = await cdp_driver.cdp_util.start_async()
tab = await driver.get("about:blank")
listenXHR(tab)

# Change url to something that makes ajax requests
tab = await driver.get("https://resttesttest.com/")
time.sleep(1)
# Click AJAX button on https://resttesttest.com/
element = await tab.select("button#submitajax")
await element.click_async()
time.sleep(2)

xhr_responses = await receiveXHR(tab, xhr_requests)
for response in xhr_responses:
print(c1 + "*** ==> XHR Request URL <== ***" + cr)
print(f'{response["url"]}')
is_base64 = response["is_base64"]
b64_data = "Base64 encoded data"
try:
headers = ast.literal_eval(response["body"])["headers"]
print(c2 + "*** ==> XHR Response Headers <== ***" + cr)
print(headers if not is_base64 else b64_data)
except Exception:
response_body = response["body"]
print(c2 + "*** ==> XHR Response Body <== ***" + cr)
print(response_body if not is_base64 else b64_data)


if __name__ == "__main__":
print("================= Starting =================")
loop = asyncio.new_event_loop()
loop.run_until_complete(crawl())
8 changes: 4 additions & 4 deletions requirements.txt
Original file line number Diff line number Diff line change
@@ -1,16 +1,16 @@
pip>=24.2
packaging>=24.2
setuptools~=70.2;python_version<"3.10"
setuptools>=73.0.1;python_version>="3.10"
setuptools>=75.5.0;python_version>="3.10"
wheel>=0.45.0
attrs>=24.2.0
certifi>=2024.8.30
exceptiongroup>=1.2.2
websockets~=13.1;python_version<"3.9"
websockets>=14.0;python_version>="3.9"
websockets>=14.1;python_version>="3.9"
filelock>=3.16.1
fasteners>=0.19
mycdp>=1.0.1
mycdp>=1.1.0
pynose>=1.5.3
platformdirs>=4.3.6
typing-extensions>=4.12.2
Expand Down Expand Up @@ -64,7 +64,7 @@ rich==13.9.4
# ("pip install -r requirements.txt" also installs this, but "pip install -e ." won't.)

coverage>=7.6.1;python_version<"3.9"
coverage>=7.6.4;python_version>="3.9"
coverage>=7.6.5;python_version>="3.9"
pytest-cov>=5.0.0;python_version<"3.9"
pytest-cov>=6.0.0;python_version>="3.9"
flake8==5.0.4;python_version<"3.9"
Expand Down
2 changes: 1 addition & 1 deletion seleniumbase/__version__.py
Original file line number Diff line number Diff line change
@@ -1,2 +1,2 @@
# seleniumbase package
__version__ = "4.32.9"
__version__ = "4.32.10"
Loading

0 comments on commit d4cde11

Please sign in to comment.