chore(python): add ijson streaming jsonl

cli/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "spider-cloud-cli"
-version = "0.1.5"
+version = "0.1.22"
 edition = "2021"
 authors = [ "j-mendez <[email protected]>"]
 description = "The Spider Cloud CLI for web crawling and scraping"

javascript/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@spider-cloud/spider-client",
-  "version": "0.0.72",
+  "version": "0.1.22",
   "description": "Isomorphic Javascript SDK for Spider Cloud services",
   "scripts": {
     "test": "node --import tsx  --test __tests__/*test.ts",

python/example_streaming.py

@@ -0,0 +1,23 @@
+
+from spider import Spider
+
+# Initialize the Spider with your API key using the env key SPIDER_API_KEY
+app = Spider()
+
+crawler_params = {
+    'limit': 1000,
+    'proxy_enabled': False,
+    'store_data': False,
+    'metadata': False,
+    'request': 'http'
+}
+
+count = [0]
+
+def process_json(data: dict) -> None:
+    print(f"Processing: {count[0]}")
+    count[0] += 1
+    for key, value in data.items():
+        print(f"{key}: {value}")
+
+app.crawl_url('https://spider.cloud', params=crawler_params, stream=True, callback=process_json)

python/requirements.txt

@@ -2,4 +2,5 @@ pytest
 pytest-asyncio
 python-dotenv
 aiohttp
-python-dotenv
+python-dotenv
+ijson

python/setup.py

@@ -8,7 +8,7 @@ def read_file(fname):
 
 setup(
     name="spider-client",
-    version="0.0.72",
+    version="0.1.22",
     url="https://github.com/spider-rs/spider-clients/tree/main/python",
     author="Spider",
     author_email="[email protected]",

python/spider/async_spider.py

@@ -430,7 +430,7 @@ def _prepare_headers(
         return {
             "Content-Type": content_type,
             "Authorization": f"Bearer {self.api_key}",
-            "User-Agent": "AsyncSpider-Client/0.0.72",
+            "User-Agent": "AsyncSpider-Client/0.1.22",
         }
 
     async def _handle_error(self, response: ClientResponse, action: str) -> None:

python/spider/spider.py

@@ -1,7 +1,8 @@
-import os, requests, json, logging
+import os, requests, logging, ijson
 from typing import Optional, Dict
 from spider.spider_types import RequestParamsDict, JsonCallback, QueryRequest
 
+
 class Spider:
     def __init__(self, api_key: Optional[str] = None):
         """
@@ -402,18 +403,19 @@ def data_delete(
 
     def stream_reader(self, response, callback):
         response.raise_for_status()
-        for chunk in response.iter_lines(chunk_size=None, decode_unicode=True):
-            try:
-                json_obj = json.loads(chunk)
+
+        try:
+            for json_obj in ijson.items(response.raw, "", multiple_values=True):
                 callback(json_obj)
-            except json.JSONDecodeError:
-                logging.error("Failed to parse chunk: %s", chunk)
+
+        except Exception as e:
+            logging.error(f"An error occurred while parsing JSON: {e}")
 
     def _prepare_headers(self, content_type: str = "application/json"):
         return {
             "Content-Type": content_type,
             "Authorization": f"Bearer {self.api_key}",
-            "User-Agent": f"Spider-Client/0.0.72",
+            "User-Agent": f"Spider-Client/0.1.22",
         }
 
     def _post_request(self, url: str, data, headers, stream=False):

python/tests/test_spider.py

@@ -266,13 +266,16 @@ def test_create_signed_url(mock_get, spider):
 def test_stream_reader():
     spider = Spider(api_key="test_api_key")
     mock_response = MagicMock()
-    mock_response.iter_lines.return_value = [b'{"key": "value"}\n', b'{"key2": "value2"}\n']
+    raw_data = b'{"key": "value"}\n{"key2": "value2"}\n'
+    mock_response.raw = MagicMock()
+    mock_response.raw.read = MagicMock(return_value=raw_data)
 
     callback_data = []
     def callback(json_obj):
         callback_data.append(json_obj)
 
     spider.stream_reader(mock_response, callback)
+
     assert len(callback_data) == 2
     assert callback_data[0] == {"key": "value"}
     assert callback_data[1] == {"key2": "value2"}