world athletics react year dropdown - pure cdp possible?

[shel1kest0learn]

This script is successful. Yet if sb.connect() is commented out, the click to change year does not complete. (see between the two blue arrows ⬇️ )

I've tested multiple times with different sb methods and no success. When I watch headed, I see the date change in the input field yet is not accepted and the table does not refresh.

Thanks for your help folks!

import argparse
import asyncio
from datetime import datetime
from concurrent.futures import ThreadPoolExecutor
from seleniumbase import SB
import time
from bs4 import BeautifulSoup
import json

print("Starting ...")
start_time = time.perf_counter()

ATHLETE_URLS = [
"https://worldathletics.org/athletes/united-states/cole-hocker-14905834"
]

def scrape_athlete(url, year):
print(datetime.now().strftime('%Y-%m-%d %H:%M:%S') + f" Enter: {url}")
with SB(uc=True, headless2=True) as sb:
sb.activate_cdp_mode(url)
# above activate affected by ...
# ... connect/disconnect later
print(datetime.now().strftime('%Y-%m-%d %H:%M:%S') + " URL opened")
sb.sleep(3)
print(datetime.now().strftime('%Y-%m-%d %H:%M:%S') + " connected to chromedriver: ",sb.is_connected())
print()
try:
sb.click('button:contains("Allow all cookies")')
print(datetime.now().strftime('%Y-%m-%d %H:%M:%S') + " Cookies allowed")
sb.sleep(2)
except Exception:
pass

    try:
        sb.click('button:contains("STATISTICS")')
        print(datetime.now().strftime('%Y-%m-%d %H:%M:%S') + " Click STATISTICS")
        sb.sleep(5)
        sb.click('button:contains("Results")')
        print(datetime.now().strftime('%Y-%m-%d %H:%M:%S') + " Click Results")
        sb.sleep(3)
    except Exception:
        print(f"Could not find sections for {url}")
        return None

    # Select year
    try:
        sb.click("div.athletesSelectInput__control") 
        sb.sleep(15)
        print(datetime.now().strftime('%Y-%m-%d %H:%M:%S') + " connected to chromedriver: ",sb.is_connected())
        # ⬇️
        sb.connect();print(datetime.now().strftime('%Y-%m-%d %H:%M:%S') + " sb.connect() : connected to chromedriver: ",sb.is_connected())
        sb.send_keys("input[id^='react-select-']", f"{year}\n") 
        sb.sleep(10)
        print(datetime.now().strftime('%Y-%m-%d %H:%M:%S') + " connected to chromedriver: ",sb.is_connected())
        print(datetime.now().strftime('%Y-%m-%d %H:%M:%S') + f" Selected year {year}")
        # ⬇️
        sb.disconnect();print(datetime.now().strftime('%Y-%m-%d %H:%M:%S') + " sb.disconnect() : connected to chromedriver: ",sb.is_connected())
        sb.sleep(5)  
    except Exception as e:
        print(f"Failed to select year {year}: {e}")


    page_source = sb.get_page_source()
    soup = BeautifulSoup(page_source, 'html.parser')
    
    table = soup.select_one('table[class*="profileStatistics_results__"]')
    if not table:
        print(f"No results table found for {url} in year {year}")
        return None

    rows_data = []
    rows = table.find_all('tr')

    if not rows:
        print("No <tr> rows found in table")
        return None

    # Extract headers
    header_row = rows[0]
    headers = [th.get_text(strip=True).lower() for th in header_row.find_all(['th', 'td'])]
    
    # Extract data rows
    for row in rows[1:]:
        cells = [cell.get_text(strip=True) for cell in row.find_all('td')]
        if len(cells) != len(headers):
            print(f"Warning: row has {len(cells)} cells, expected {len(headers)}. Skipping.")
            continue

        # Optional: filter by year in date column (first column usually)
        date_str = cells[0] if cells else ""
        if str(year) not in date_str:
            print(f"Skipping row (wrong year): {date_str}")
            continue

        row_dict = dict(zip(headers, cells))
        rows_data.append(row_dict)

    athlete_name = sb.get_title().split("|")[0].strip()

    print(f"Successfully parsed {len(rows_data)} results for {athlete_name} in {year}")
    print()
    for dictionary_item in rows_data:
         print(dictionary_item,"\n")
    print()
    
    return {
        "name": athlete_name,
        "url": url,
        "year": year,
        "results": rows_data
    }

async def launch_with_delay(loop, executor, delay, url, year):
await asyncio.sleep(delay)
print(f"Launching scrape for {url} after {delay:.1f}s delay")
return await loop.run_in_executor(executor, scrape_athlete, url, year)

async def scrape_all_async(urls, year):
loop = asyncio.get_running_loop()
results = []

with ThreadPoolExecutor(max_workers=min(len(urls), 5)) as executor:
    tasks = []
    for i, url in enumerate(urls):
        delay = i * 15  # stagger by 15 seconds per athlete
        task = asyncio.create_task(launch_with_delay(loop, executor, delay, url, year))
        tasks.append(task)

    for coro in asyncio.as_completed(tasks):
        res = await coro
        if res:
            results.append(res)
return results

async def main():
parser = argparse.ArgumentParser()
parser.add_argument("--year", type=int, default=datetime.now().year,
help="Year to filter athlete results")
args = parser.parse_args()
results = await scrape_all_async(ATHLETE_URLS, args.year)

if name == "main":
asyncio.run(main())
print("Operation finished.")
end_time = time.perf_counter()
elapsed_seconds = end_time - start_time
minutes = int(elapsed_seconds // 60)
seconds = elapsed_seconds % 60
print(f"\nElapsed time: {minutes}min {seconds:.2f}sec")

[mdmintz]

Here are two different scripts that can do that.

First one is with regular CDP Mode:

from seleniumbase import SB

with SB(uc=True, test=True) as sb:
    url = "https://worldathletics.org/athletes/united-states/cole-hocker-14905834"
    sb.activate_cdp_mode(url)
    sb.sleep(2)
    sb.open("https://worldathletics.org/athletes/united-states/cole-hocker-14905834")
    sb.click("button#CybotCookiebotDialogBodyLevelButtonLevelOptinAllowAll")
    sb.click('button[value="STATISTICS"]')
    sb.click('button[value="Results"]')
    sb.sleep(1)
    sb.click('[class*="athletesSelectInput_athletesSelectContainer"]')
    sb.sleep(1)
    sb.click('[class*="athletesSelectInput__menu-list"] div:contains("2025")')
    sb.sleep(2)
    print("Results:")
    rows = sb.find_elements('tbody[class*="profileStatistics_tableBody"] tr')
    for row in rows:
        print("* " + row.text)

And here's a script that does the same thing using the Pure CDP Mode format:

from seleniumbase import sb_cdp

url = "https://worldathletics.org/athletes/united-states/cole-hocker-14905834"
sb = sb_cdp.Chrome(url)
sb.sleep(2)
sb.open("https://worldathletics.org/athletes/united-states/cole-hocker-14905834")
sb.click("button#CybotCookiebotDialogBodyLevelButtonLevelOptinAllowAll")
sb.click('button[value="STATISTICS"]')
sb.click('button[value="Results"]')
sb.sleep(1)
sb.click('[class*="athletesSelectInput_athletesSelectContainer"]')
sb.sleep(1)
sb.click('[class*="athletesSelectInput__menu-list"] div:contains("2025")')
sb.sleep(2)
print("Results:")
rows = sb.find_elements('tbody[class*="profileStatistics_tableBody"] tr')
for row in rows:
    print("* " + row.text)

[shel1kest0learn]

@mdmintz Works... thank you.