release: 0.1.23

cli/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "spider-cloud-cli"
-version = "0.1.22"
+version = "0.1.23"
 edition = "2021"
 authors = [ "j-mendez <[email protected]>"]
 description = "The Spider Cloud CLI for web crawling and scraping"

javascript/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@spider-cloud/spider-client",
-  "version": "0.1.22",
+  "version": "0.1.23",
   "description": "Isomorphic Javascript SDK for Spider Cloud services",
   "scripts": {
     "test": "node --import tsx  --test __tests__/*test.ts",

python/spider/async_spider.py

@@ -430,7 +430,7 @@ def _prepare_headers(
         return {
             "Content-Type": content_type,
             "Authorization": f"Bearer {self.api_key}",
-            "User-Agent": "AsyncSpider-Client/0.1.22",
+            "User-Agent": "AsyncSpider-Client/0.1.23",
         }
 
     async def _handle_error(self, response: ClientResponse, action: str) -> None:

python/spider/spider.py

@@ -415,7 +415,7 @@ def _prepare_headers(self, content_type: str = "application/json"):
         return {
             "Content-Type": content_type,
             "Authorization": f"Bearer {self.api_key}",
-            "User-Agent": f"Spider-Client/0.1.22",
+            "User-Agent": f"Spider-Client/0.1.23",
         }
 
     def _post_request(self, url: str, data, headers, stream=False):

rust/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "spider-client"
-version = "0.1.22"
+version = "0.1.23"
 edition = "2021"
 authors = [ "j-mendez <[email protected]>"]
 description = "Spider Cloud client"

rust/src/lib.rs

@@ -65,7 +65,6 @@ use reqwest::Client;
 use reqwest::{Error, Response};
 use serde::{Deserialize, Serialize};
 use std::collections::HashMap;
-use std::default;
 use tokio_stream::StreamExt;
 
 /// Structure representing the Chunking algorithm dictionary.
@@ -104,15 +103,15 @@ pub enum WebAutomation {
     ScrollX { pixels: i32 },
     ScrollY { pixels: i32 },
     Fill { selector: String, value: String },
-    InfiniteScroll { times: u32 }
+    InfiniteScroll { times: u32 },
 }
 
 #[derive(Default, Serialize, Deserialize, Debug, Clone)]
 #[serde(tag = "type", rename_all = "PascalCase")]
 pub enum RedirectPolicy {
     Loose,
     #[default]
-    Strict
+    Strict,
 }
 
 pub type WebAutomationMap = std::collections::HashMap<String, Vec<WebAutomation>>;
@@ -379,7 +378,7 @@ pub struct RequestParams {
     /// Perform web automated tasks on a url or url path. You need to make your `request` `chrome` or `smart`
     pub automation_scripts: Option<WebAutomationMap>,
     /// The redirect policy for HTTP request. Set the value to Loose to allow all.
-    pub redirect_policy: Option<RedirectPolicy>
+    pub redirect_policy: Option<RedirectPolicy>,
 }
 
 /// The structure representing request parameters for a search request.
@@ -1160,8 +1159,6 @@ impl Spider {
 
 #[cfg(test)]
 mod tests {
-    use std::time::Duration;
-
     use super::*;
     use dotenv::dotenv;
     use lazy_static::lazy_static;
@@ -1171,24 +1168,22 @@ mod tests {
         static ref SPIDER_CLIENT: Spider = {
             dotenv().ok();
             let client = ClientBuilder::new();
-            let client = client
-                .tcp_keepalive(Some(Duration::from_secs(5)))
-                .build()
-                .unwrap();
+            let client = client.user_agent("SpiderBot").build().unwrap();
 
             Spider::new_with_client(None, client).expect("client to build")
         };
     }
 
-    #[tokio::test(flavor = "multi_thread")]
+    #[tokio::test]
+    #[ignore]
     async fn test_scrape_url() {
         let response = SPIDER_CLIENT
             .scrape_url("https://example.com", None, "application/json")
             .await;
         assert!(response.is_ok());
     }
 
-    #[tokio::test(flavor = "multi_thread")]
+    #[tokio::test]
     async fn test_crawl_url() {
         let response = SPIDER_CLIENT
             .crawl_url(
@@ -1202,15 +1197,17 @@ mod tests {
         assert!(response.is_ok());
     }
 
-    #[tokio::test(flavor = "multi_thread")]
+    #[tokio::test]
+    #[ignore]
     async fn test_links() {
         let response: Result<serde_json::Value, Error> = SPIDER_CLIENT
             .links("https://example.com", None, false, "application/json")
             .await;
         assert!(response.is_ok());
     }
 
-    #[tokio::test(flavor = "multi_thread")]
+    #[tokio::test]
+    #[ignore]
     async fn test_screenshot() {
         let mut params = RequestParams::default();
         params.limit = Some(1);
@@ -1241,7 +1238,8 @@ mod tests {
     //     assert!(response.is_ok());
     // }
 
-    #[tokio::test(flavor = "multi_thread")]
+    #[tokio::test]
+    #[ignore]
     async fn test_transform() {
         let data = vec![HashMap::from([(
             "<html><body><h1>Transformation</h1></body></html>".into(),
@@ -1253,39 +1251,41 @@ mod tests {
         assert!(response.is_ok());
     }
 
-    #[tokio::test(flavor = "multi_thread")]
+    #[tokio::test]
+    #[ignore]
     async fn test_extract_contacts() {
         let response = SPIDER_CLIENT
             .extract_contacts("https://example.com", None, false, "application/json")
             .await;
         assert!(response.is_ok());
     }
 
-    #[tokio::test(flavor = "multi_thread")]
+    #[tokio::test]
+    #[ignore]
     async fn test_label() {
         let response = SPIDER_CLIENT
             .label("https://example.com", None, false, "application/json")
             .await;
         assert!(response.is_ok());
     }
 
-    #[tokio::test(flavor = "multi_thread")]
+    #[tokio::test]
     async fn test_create_signed_url() {
         let response = SPIDER_CLIENT
             .create_signed_url(Some("example.com"), None)
             .await;
         assert!(response.is_ok());
     }
 
-    #[tokio::test(flavor = "multi_thread")]
+    #[tokio::test]
     async fn test_get_crawl_state() {
         let response = SPIDER_CLIENT
             .get_crawl_state("https://example.com", None, "application/json")
             .await;
         assert!(response.is_ok());
     }
 
-    #[tokio::test(flavor = "multi_thread")]
+    #[tokio::test]
     async fn test_query() {
         let mut query = QueryRequest::default();
 
@@ -1295,7 +1295,7 @@ mod tests {
         assert!(response.is_ok());
     }
 
-    #[tokio::test(flavor = "multi_thread")]
+    #[tokio::test]
     async fn test_get_credits() {
         let response = SPIDER_CLIENT.get_credits().await;
         assert!(response.is_ok());