From c5d0ecbc5de58b45cf17351911dc28e74257ceb0 Mon Sep 17 00:00:00 2001 From: Karanjot Singh Date: Wed, 25 Jan 2023 22:30:55 +0530 Subject: [PATCH 1/4] Added http stdlib module Signed-off-by: Karanjot Singh --- tremor-script/lib/http/content_types.tremor | 13 +++++++++++++ tremor-script/lib/http/headers.tremor | 18 ++++++++++++++++++ tremor-script/lib/http/methods.tremor | 9 +++++++++ tremor-script/lib/http/status.tremor | 11 +++++++++++ 4 files changed, 51 insertions(+) create mode 100644 tremor-script/lib/http/content_types.tremor create mode 100644 tremor-script/lib/http/headers.tremor create mode 100644 tremor-script/lib/http/methods.tremor create mode 100644 tremor-script/lib/http/status.tremor diff --git a/tremor-script/lib/http/content_types.tremor b/tremor-script/lib/http/content_types.tremor new file mode 100644 index 0000000000..961594c7be --- /dev/null +++ b/tremor-script/lib/http/content_types.tremor @@ -0,0 +1,13 @@ +const JSON = "application/json"; +const HTML = "text/html"; +const CSS = "text/css"; +const TEXT = "text/plain"; +const XML = "application/xml"; +const JS = "application/javascript"; +const PNG = "image/png"; +const JPG = "image/jpeg"; +const GIF = "image/gif"; +const SVG = "image/svg+xml"; +const CSV = "text/csv"; +const PDF = "application/pdf"; +const ZIP = "application/zip"; diff --git a/tremor-script/lib/http/headers.tremor b/tremor-script/lib/http/headers.tremor new file mode 100644 index 0000000000..f67e899372 --- /dev/null +++ b/tremor-script/lib/http/headers.tremor @@ -0,0 +1,18 @@ +const CONTENT_TYPE = "Content-Type"; +const ACCEPT = "Accept"; +const AUTHORIZATION = "Authorization"; +const CACHE_CONTROL = "Cache-Control"; +const CONTENT_LENGTH = "Content-Length"; +const CONTENT_DISPOSITION = "Content-Disposition"; +const DATE = "Date"; +const ETAG = "ETag"; +const EXPIRES = "Expires"; +const HOST = "Host"; +const IF_MODIFIED_SINCE = "If-Modified-Since"; +const IF_NONE_MATCH = "If-None-Match"; +const LAST_MODIFIED = "Last-Modified"; +const LOCATION = "Location"; +const USER_AGENT = "User-Agent"; +const X_FORWARDED_FOR = "X-Forwarded-For"; +const X_FORWARDED_HOST = "X-Forwarded-Host"; +const X_FORWARDED_PROTO = "X-Forwarded-Proto"; diff --git a/tremor-script/lib/http/methods.tremor b/tremor-script/lib/http/methods.tremor new file mode 100644 index 0000000000..785339dc23 --- /dev/null +++ b/tremor-script/lib/http/methods.tremor @@ -0,0 +1,9 @@ +const GET = "GET"; +const POST = "POST"; +const PUT = "PUT"; +const DELETE = "DELETE"; +const HEAD = "HEAD"; +const OPTIONS = "OPTIONS"; +const CONNECT = "CONNECT"; +const TRACE = "TRACE"; +const PATCH = "PATCH"; diff --git a/tremor-script/lib/http/status.tremor b/tremor-script/lib/http/status.tremor new file mode 100644 index 0000000000..c00a91252e --- /dev/null +++ b/tremor-script/lib/http/status.tremor @@ -0,0 +1,11 @@ +const OK = 200; +const CREATED = 201; +const NO_CONTENT = 204; +const BAD_REQUEST = 400; +const UNAUTHORIZED = 401; +const FORBIDDEN = 403; +const NOT_FOUND = 404; +const INTERNAL_SERVER_ERROR = 500; +const BAD_GATEWAY = 502; +const SERVICE_UNAVAILABLE = 503; +const GATEWAY_TIMEOUT = 504; From 6b70c6d676de5b2582cabcd7c1e7c51eeac0274e Mon Sep 17 00:00:00 2001 From: Karanjot Singh Date: Thu, 26 Jan 2023 00:00:53 +0530 Subject: [PATCH 2/4] Added comments for docs Signed-off-by: Karanjot Singh --- tremor-script/lib/http/content_types.tremor | 23 ++++++++++--- tremor-script/lib/http/headers.tremor | 37 ++++++++++++++++++++- tremor-script/lib/http/methods.tremor | 19 ++++++++++- tremor-script/lib/http/status.tremor | 23 ++++++++++++- 4 files changed, 94 insertions(+), 8 deletions(-) diff --git a/tremor-script/lib/http/content_types.tremor b/tremor-script/lib/http/content_types.tremor index 961594c7be..8666b17490 100644 --- a/tremor-script/lib/http/content_types.tremor +++ b/tremor-script/lib/http/content_types.tremor @@ -1,13 +1,26 @@ +## The JSON content type. const JSON = "application/json"; + +## The HTML content type. const HTML = "text/html"; + +## The CSS content type. const CSS = "text/css"; + +## The Text content type. const TEXT = "text/plain"; + +## The XML content type. const XML = "application/xml"; + +## The JS content type. const JS = "application/javascript"; + +## The PNG content type. const PNG = "image/png"; + +## The JPG content type. const JPG = "image/jpeg"; -const GIF = "image/gif"; -const SVG = "image/svg+xml"; -const CSV = "text/csv"; -const PDF = "application/pdf"; -const ZIP = "application/zip"; + +## The GIF content type. +const GIF = "image/gif"; \ No newline at end of file diff --git a/tremor-script/lib/http/headers.tremor b/tremor-script/lib/http/headers.tremor index f67e899372..41b707dad5 100644 --- a/tremor-script/lib/http/headers.tremor +++ b/tremor-script/lib/http/headers.tremor @@ -1,18 +1,53 @@ +## The Content-Type header. const CONTENT_TYPE = "Content-Type"; + +## The Accept header. const ACCEPT = "Accept"; + +## The Authorization header. const AUTHORIZATION = "Authorization"; + +## The Cache-Control header. const CACHE_CONTROL = "Cache-Control"; + +## The Content-Length header. const CONTENT_LENGTH = "Content-Length"; + +## The Content-Disposition header. const CONTENT_DISPOSITION = "Content-Disposition"; + +## The Date header. const DATE = "Date"; + +## The ETag header. const ETAG = "ETag"; + +## The Expires header. const EXPIRES = "Expires"; + +## The Host header. const HOST = "Host"; + +## The If-Modified-Since header. const IF_MODIFIED_SINCE = "If-Modified-Since"; + +## The If-None-Match header. const IF_NONE_MATCH = "If-None-Match"; + +## The Last-Modified header. const LAST_MODIFIED = "Last-Modified"; + +## The Location header. const LOCATION = "Location"; + +## The User-Agent header. const USER_AGENT = "User-Agent"; + +## The X-Forwarded-For header. const X_FORWARDED_FOR = "X-Forwarded-For"; + +## The X-Forwarded-Host header. const X_FORWARDED_HOST = "X-Forwarded-Host"; -const X_FORWARDED_PROTO = "X-Forwarded-Proto"; + +## The X-Forwarded-Proto header. +const X_FORWARDED_PROTO = "X-Forwarded-Proto"; \ No newline at end of file diff --git a/tremor-script/lib/http/methods.tremor b/tremor-script/lib/http/methods.tremor index 785339dc23..762a2a1d4c 100644 --- a/tremor-script/lib/http/methods.tremor +++ b/tremor-script/lib/http/methods.tremor @@ -1,9 +1,26 @@ +## The GET HTTP method. const GET = "GET"; + +## The POST HTTP method. const POST = "POST"; + +## The PUT HTTP method. const PUT = "PUT"; + +## The DELETE HTTP method. const DELETE = "DELETE"; + +## The HEAD HTTP method. const HEAD = "HEAD"; + +## The OPTIONS HTTP method. const OPTIONS = "OPTIONS"; + +## The CONNECT HTTP method. const CONNECT = "CONNECT"; + +## The TRACE HTTP method. const TRACE = "TRACE"; -const PATCH = "PATCH"; + +## The PATCH HTTP method. +const PATCH = "PATCH";ß \ No newline at end of file diff --git a/tremor-script/lib/http/status.tremor b/tremor-script/lib/http/status.tremor index c00a91252e..a0e858c59f 100644 --- a/tremor-script/lib/http/status.tremor +++ b/tremor-script/lib/http/status.tremor @@ -1,11 +1,32 @@ +## The request was successful. const OK = 200; + +## The request was successful and a resource was created. const CREATED = 201; + +## The request was successful but there's no content to return. const NO_CONTENT = 204; + +## The request was invalid or malformed. const BAD_REQUEST = 400; + +## The request requires user authentication. const UNAUTHORIZED = 401; + +## The user is not authorized to perform the requested action. const FORBIDDEN = 403; + +## The requested resource could not be found. const NOT_FOUND = 404; + +## An internal server error occurred. const INTERNAL_SERVER_ERROR = 500; + +## The server received an invalid response from an upstream server. const BAD_GATEWAY = 502; + +## The server is currently unavailable. const SERVICE_UNAVAILABLE = 503; -const GATEWAY_TIMEOUT = 504; + +## The server timed out waiting for a response from an upstream server. +const GATEWAY_TIMEOUT = 504; \ No newline at end of file From b7336b6eee86e132a02937110304779ed6460352 Mon Sep 17 00:00:00 2001 From: Karanjot Singh Date: Mon, 30 Jan 2023 02:38:49 +0530 Subject: [PATCH 3/4] added url stlib module for parsing url struct Signed-off-by: Karanjot Singh --- tremor-cli/tests/stdlib/std/all.tremor | 43 ++++++++++ tremor-script/lib/std/url.tremor | 56 +++++++++++++ tremor-script/src/std_lib/url.rs | 106 ++++++++++++++++++++++++- 3 files changed, 201 insertions(+), 4 deletions(-) diff --git a/tremor-cli/tests/stdlib/std/all.tremor b/tremor-cli/tests/stdlib/std/all.tremor index 1283170253..8e97a31709 100644 --- a/tremor-cli/tests/stdlib/std/all.tremor +++ b/tremor-cli/tests/stdlib/std/all.tremor @@ -865,4 +865,47 @@ test::suite({ ] }); +test::suite({ + "name": "URL Construct and Modification tests", + "tags": [ "url", "construct", "modify" ], + "tests": [ + test::test({ + "name": "Test construct function", + "test": test::assert("url::construct", + url::construct("http", "www.example.com", "/path", "query=test", "fragment"), + "http://www.example.com/path?query=test#fragment"), + }), + test::test({ + "name": "Test set_scheme function", + "test": test::assert("url::set_scheme", + url::set_scheme("http://www.example.com", "https"), + "https://www.example.com"), + }), + test::test({ + "name": "Test set_host function", + "test": test::assert("url::set_host", + url::set_host("http://www.example.com", "www.example2.com"), + "http://www.example2.com"), + }), + test::test({ + "name": "Test set_path function", + "test": test::assert("url::set_path", + url::set_path("http://www.example.com", "/newpath"), + "http://www.example.com/newpath"), + }), + test::test({ + "name": "Test set_query function", + "test": test::assert("url::set_query", + url::set_query("http://www.example.com?query=test", "newquery=test2"), + "http://www.example.com?newquery=test2"), + }), + test::test({ + "name": "Test set_fragment function", + "test": test::assert("url::set_fragment", + url::set_fragment("http://www.example.com#fragment", "newfragment"), + "http://www.example.com#newfragment"), + }), + ], +}); + "snot badger"; diff --git a/tremor-script/lib/std/url.tremor b/tremor-script/lib/std/url.tremor index d384c6eaa9..38ea3150fc 100644 --- a/tremor-script/lib/std/url.tremor +++ b/tremor-script/lib/std/url.tremor @@ -19,3 +19,59 @@ intrinsic fn encode(str) as url::encode; ## ## Returns a `string` intrinsic fn decode(str) as url::decode; + +## Constructs a url from the provided scheme, host, path, query and fragment +## +## > ```tremor +## > use std::url; +## > +## > "https://example.com/path?query=1#fragment" == url::construct("https","example.com","/path","query=1","fragment") +## > ``` +## +## Returns a `string` +intrinsic fn construct(scheme:str,host:str,path:str,query:str,fragment:str) as url::construct; + +## Returns a new URL string with the specified scheme +## +## > ```tremor +## > "https://example.com" == url::set_scheme("http://example.com", "https") +## > ``` +## +## Returns a `string` +intrinsic fn set_scheme(url:str, scheme:str) as url::set_scheme; + +## Returns a new URL string with the specified host +## +## > ```tremor +## > "http://newhost.com" == url::set_host("http://example.com", "newhost.com") +## > ``` +## +## Returns a `string` +intrinsic fn set_host(url:str, host:str) as url::set_host; + +## Returns a new URL string with the specified path +## +## > ```tremor +## > "http://example.com/newpath" == url::set_path("http://example.com/oldpath", "/newpath") +## > ``` +## +## Returns a `string` +intrinsic fn set_path(url:str, path:str) as url::set_path; + +## Returns a new URL string with the specified query +## +## > ```tremor +## > "http://example.com?newquery=value" == url::set_query("http://example.com", "newquery=value") +## > ``` +## +## Returns a `string` +intrinsic fn set_query(url:str, query:str) as url::set_query; + +## Returns a new URL string with the specified fragment +## +## > ```tremor +## > "http://example.com#newfragment" == url::set_fragment("http://example.com", "newfragment") +## > ``` +## +## Returns a `string` +intrinsic fn set_fragment(url:str, fragment:str) as url::set_fragment; \ No newline at end of file diff --git a/tremor-script/src/std_lib/url.rs b/tremor-script/src/std_lib/url.rs index 7f7bae8210..e00fa6c8f8 100644 --- a/tremor-script/src/std_lib/url.rs +++ b/tremor-script/src/std_lib/url.rs @@ -15,6 +15,7 @@ use crate::registry::Registry; use crate::tremor_fn; use percent_encoding::{percent_decode_str, utf8_percent_encode, NON_ALPHANUMERIC}; +use url::Url; pub fn load(registry: &mut Registry) { registry @@ -26,9 +27,69 @@ pub fn load(registry: &mut Registry) { Err(to_runtime_error(format!("Could not urldecode value: {s}"))) } })) - .insert(tremor_fn! (url|encode(ctx, s: String) { - Ok(Value::from(utf8_percent_encode(s, NON_ALPHANUMERIC).to_string())) - })); + .insert(tremor_fn!(url|construct(ctx, scheme: String, host: String, path: String, query: String, fragment: String) { + let url = Url::build(|b| { + b.scheme(scheme) + .host(host) + .path(path) + .query(query) + .fragment(fragment) + }); + match url { + Ok(url) => Ok(Value::from(url.as_str().to_owned())), + Err(e) => Err(to_runtime_error(format!("Error building URL: {}", e))), + } + })) + .insert(tremor_fn!(url|set_scheme(ctx, url: String, scheme: String) { + let parsed_url = Url::parse(&url); + match parsed_url { + Ok(mut parsed_url) => { + parsed_url.set_scheme(scheme).unwrap(); + Ok(Value::from(parsed_url.as_str().to_owned())) + } + Err(e) => Err(to_runtime_error(format!("Error parsing URL: {}", e))), + } + })) + .insert(tremor_fn!(url|set_host(ctx, url: String, host: String) { + let parsed_url = Url::parse(&url); + match parsed_url { + Ok(mut parsed_url) => { + parsed_url.set_host(Some(host)).unwrap(); + Ok(Value::from(parsed_url.as_str().to_owned())) + } + Err(e) => Err(to_runtime_error(format!("Error parsing URL: {}", e))), + } + })) + .insert(tremor_fn!(url|set_path(ctx, url: String, path: String) { + let parsed_url = Url::parse(&url); + match parsed_url { + Ok(mut parsed_url) => { + parsed_url.set_path(path); + Ok(Value::from(parsed_url.as_str().to_owned())) + } + Err(e) => Err(to_runtime_error(format!("Error parsing URL: {}", e))), + } + })) + .insert(tremor_fn!(url|set_query(ctx, url: String, query: String) { + let parsed_url = Url::parse(&url); + match parsed_url { + Ok(mut parsed_url) => { + parsed_url.set_query(Some(query)); + Ok(Value::from(parsed_url.as_str().to_owned())) + } + Err(e) => Err(to_runtime_error(format!("Error parsing URL: {}", e))), + } + })) + .insert(tremor_fn! (url|set_fragment(url: String, fragment: String) { + let parsed_url = Url::parse(&url); + match parsed_url { + Ok(mut parsed_url) => { + parsed_url.set_fragment(Some(&fragment)); + Ok(Value::from(parsed_url.into_string())) + } + Err(e) => Err(to_runtime_error(format!("Error parsing URL: {}", e))) + } + })) } #[cfg(test)] @@ -53,4 +114,41 @@ mod test { let v = Value::from("%22snot%20badger%22"); assert_val!(d(&[&v]), r#""snot badger""#); } -} + + #[test] + fn url_construct_smoke_test() { + let c = fun("url", "construct"); + assert_val!(c(&[Value::from("http"), Value::from("example.com"), Value::from("/path"), Value::from("query"), Value::from("fragment")]), "http://example.com/path?query#fragment"); + } + + #[test] + fn url_set_scheme_smoke_test() { + let c = fun("url", "set_scheme"); + assert_val!(c(&[Value::from("http://example.com"), Value::from("https")]), "https://example.com"); + } + + #[test] + fn url_set_host_smoke_test() { + let c = fun("url", "set_host"); + assert_val!(c(&[Value::from("http://example.com"), Value::from("newexample.com")]), "http://newexample.com"); + } + + #[test] + fn url_set_path_smoke_test() { + let c = fun("url", "set_path"); + assert_val!(c(&[Value::from("http://example.com"), Value::from("/newpath")]), "http://example.com/newpath"); + } + + #[test] + fn url_set_query_smoke_test() { + let c = fun("url", "set_query"); + assert_val!(c(&[Value::from("http://example.com"), Value::from("newquery")]), "http://example.com?newquery"); + } + + #[test] + fn url_set_fragment_smoke_test() { + let c = fun("url", "set_fragment"); + assert_val!(c(&[Value::from("http://example.com"), Value::from("newfragment")]), "http://example.com#newfragment"); + } + +} \ No newline at end of file From ea06b3ca78015f97c2abc7ff514d418731651709 Mon Sep 17 00:00:00 2001 From: Karanjot Singh Date: Mon, 30 Jan 2023 02:42:21 +0530 Subject: [PATCH 4/4] Fixed url encode ( accidently removed it ) Signed-off-by: Karanjot Singh --- tremor-script/src/std_lib/url.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tremor-script/src/std_lib/url.rs b/tremor-script/src/std_lib/url.rs index e00fa6c8f8..ef05f2c4cd 100644 --- a/tremor-script/src/std_lib/url.rs +++ b/tremor-script/src/std_lib/url.rs @@ -27,6 +27,9 @@ pub fn load(registry: &mut Registry) { Err(to_runtime_error(format!("Could not urldecode value: {s}"))) } })) + .insert(tremor_fn! (url|encode(ctx, s: String) { + Ok(Value::from(utf8_percent_encode(s, NON_ALPHANUMERIC).to_string())) + })); .insert(tremor_fn!(url|construct(ctx, scheme: String, host: String, path: String, query: String, fragment: String) { let url = Url::build(|b| { b.scheme(scheme)