fix(PlaywrightFetcher): Use more dependable response data

D4Vinci · Dec 25, 2024 · f9b85cf · f9b85cf
1 parent 3cca3fd
commit f9b85cf
Showing 1 changed file with 10 additions and 12 deletions.
diff --git a/scrapling/engines/pw.py b/scrapling/engines/pw.py
@@ -206,7 +206,7 @@ def fetch(self, url: str) -> Response:
 
         def handle_response(finished_response: PlaywrightResponse):
             nonlocal final_response
-            if finished_response.request.resource_type == "document":
+            if finished_response.request.resource_type == "document" and finished_response.request.is_navigation_request():
                 final_response = finished_response
 
         with sync_playwright() as p:
@@ -252,7 +252,6 @@ def handle_response(finished_response: PlaywrightResponse):
                 if self.network_idle:
                     page.wait_for_load_state('networkidle')
 
-            response_bytes = final_response.body() if final_response else page.content().encode('utf-8')
             # In case we didn't catch a document type somehow
             final_response = final_response if final_response else first_response
             # This will be parsed inside `Response`
@@ -261,15 +260,15 @@ def handle_response(finished_response: PlaywrightResponse):
             status_text = final_response.status_text or StatusText.get(final_response.status)
 
             response = Response(
-                url=final_response.url,
+                url=page.url,
                 text=page.content(),
-                body=response_bytes,
+                body=page.content().encode('utf-8'),
                 status=final_response.status,
                 reason=status_text,
                 encoding=encoding,
                 cookies={cookie['name']: cookie['value'] for cookie in page.context.cookies()},
-                headers=final_response.all_headers(),
-                request_headers=final_response.request.all_headers(),
+                headers=first_response.all_headers(),
+                request_headers=first_response.request.all_headers(),
                 **self.adaptor_arguments
             )
             page.close()
@@ -293,7 +292,7 @@ async def async_fetch(self, url: str) -> Response:
 
         async def handle_response(finished_response: PlaywrightResponse):
             nonlocal final_response
-            if finished_response.request.resource_type == "document":
+            if finished_response.request.resource_type == "document" and finished_response.request.is_navigation_request():
                 final_response = finished_response
 
         async with async_playwright() as p:
@@ -339,7 +338,6 @@ async def handle_response(finished_response: PlaywrightResponse):
                 if self.network_idle:
                     await page.wait_for_load_state('networkidle')
 
-            response_bytes = await final_response.body() if final_response else (await page.content()).encode('utf-8')
             # In case we didn't catch a document type somehow
             final_response = final_response if final_response else first_response
             # This will be parsed inside `Response`
@@ -348,15 +346,15 @@ async def handle_response(finished_response: PlaywrightResponse):
             status_text = final_response.status_text or StatusText.get(final_response.status)
 
             response = Response(
-                url=final_response.url,
+                url=page.url,
                 text=await page.content(),
-                body=response_bytes,
+                body=(await page.content()).encode('utf-8'),
                 status=final_response.status,
                 reason=status_text,
                 encoding=encoding,
                 cookies={cookie['name']: cookie['value'] for cookie in await page.context.cookies()},
-                headers=await final_response.all_headers(),
-                request_headers=await final_response.request.all_headers(),
+                headers=await first_response.all_headers(),
+                request_headers=await first_response.request.all_headers(),
                 **self.adaptor_arguments
             )
             await page.close()