Merge pull request umccr#230 from umccr/test/concat-responses

test: concatenate byte range responses

Author: mmalenic

Cargo.lock

@@ -26,7 +26,7 @@ futures-util = { version = "0.3" }
 htsget-http = { version = "0.4.12", path = "../htsget-http", default-features = false }
 htsget-search = { version = "0.6.6", path = "../htsget-search", default-features = false }
 htsget-config = { version = "0.8.1", path = "../htsget-config", default-features = false }
-htsget-test = { version = "0.5.4", path = "../htsget-test", features = ["server-tests", "cors-tests"], default-features = false }
+htsget-test = { version = "0.5.4", path = "../htsget-test", features = ["http"], default-features = false }
 futures = { version = "0.3" }
 tokio = { version = "1.28", features = ["macros", "rt-multi-thread"] }
 

data/cram/htsnexus_test_NA12878.cram

@@ -12,7 +12,7 @@ use serde::{Deserialize, Serialize};
 
 use htsget_config::types::{Headers, JsonResponse};
 use htsget_http::{PostRequest, Region};
-use htsget_test::http_tests::{default_config_fixed_port, default_dir, default_dir_data};
+use htsget_test::http::{default_config_fixed_port, default_dir, default_dir_data};
 
 const REFSERVER_DOCKER_IMAGE: &str = "ga4gh/htsget-refserver:1.5.0";
 const BENCHMARK_DURATION_SECONDS: u64 = 30;

data/cram/htsnexus_test_NA12878.cram.crai

@@ -151,12 +151,12 @@ mod tests {
 
   use htsget_config::types::JsonResponse;
   use htsget_search::storage::data_server::BindDataServer;
-  use htsget_test::http_tests::{config_with_tls, default_test_config};
-  use htsget_test::http_tests::{
+  use htsget_test::http::server::expected_url_path;
+  use htsget_test::http::{config_with_tls, default_test_config};
+  use htsget_test::http::{cors, server};
+  use htsget_test::http::{
     Header as TestHeader, Response as TestResponse, TestRequest, TestServer,
   };
-  use htsget_test::server_tests::expected_url_path;
-  use htsget_test::{cors_tests, server_tests};
 
   use crate::Config;
 
@@ -272,55 +272,51 @@ mod tests {
 
   #[actix_web::test]
   async fn get_http_tickets() {
-    server_tests::test_get::<JsonResponse, _>(&ActixTestServer::default()).await;
+    server::test_get::<JsonResponse, _>(&ActixTestServer::default()).await;
   }
 
   #[actix_web::test]
   async fn post_http_tickets() {
-    server_tests::test_post::<JsonResponse, _>(&ActixTestServer::default()).await;
+    server::test_post::<JsonResponse, _>(&ActixTestServer::default()).await;
   }
 
   #[actix_web::test]
   async fn parameterized_get_http_tickets() {
-    server_tests::test_parameterized_get::<JsonResponse, _>(&ActixTestServer::default()).await;
+    server::test_parameterized_get::<JsonResponse, _>(&ActixTestServer::default()).await;
   }
 
   #[actix_web::test]
   async fn parameterized_post_http_tickets() {
-    server_tests::test_parameterized_post::<JsonResponse, _>(&ActixTestServer::default()).await;
+    server::test_parameterized_post::<JsonResponse, _>(&ActixTestServer::default()).await;
   }
 
   #[actix_web::test]
   async fn parameterized_post_class_header_http_tickets() {
-    server_tests::test_parameterized_post_class_header::<JsonResponse, _>(
-      &ActixTestServer::default(),
-    )
-    .await;
+    server::test_parameterized_post_class_header::<JsonResponse, _>(&ActixTestServer::default())
+      .await;
   }
 
   #[actix_web::test]
   async fn service_info() {
-    server_tests::test_service_info(&ActixTestServer::default()).await;
+    server::test_service_info(&ActixTestServer::default()).await;
   }
 
   #[actix_web::test]
   async fn get_https_tickets() {
     let base_path = TempDir::new().unwrap();
-    server_tests::test_get::<JsonResponse, _>(&ActixTestServer::new_with_tls(base_path.path()))
-      .await;
+    server::test_get::<JsonResponse, _>(&ActixTestServer::new_with_tls(base_path.path())).await;
   }
 
   #[actix_web::test]
   async fn post_https_tickets() {
     let base_path = TempDir::new().unwrap();
-    server_tests::test_post::<JsonResponse, _>(&ActixTestServer::new_with_tls(base_path.path()))
-      .await;
+    server::test_post::<JsonResponse, _>(&ActixTestServer::new_with_tls(base_path.path())).await;
   }
 
   #[actix_web::test]
   async fn parameterized_get_https_tickets() {
     let base_path = TempDir::new().unwrap();
-    server_tests::test_parameterized_get::<JsonResponse, _>(&ActixTestServer::new_with_tls(
+    server::test_parameterized_get::<JsonResponse, _>(&ActixTestServer::new_with_tls(
       base_path.path(),
     ))
     .await;
@@ -329,7 +325,7 @@ mod tests {
   #[actix_web::test]
   async fn parameterized_post_https_tickets() {
     let base_path = TempDir::new().unwrap();
-    server_tests::test_parameterized_post::<JsonResponse, _>(&ActixTestServer::new_with_tls(
+    server::test_parameterized_post::<JsonResponse, _>(&ActixTestServer::new_with_tls(
       base_path.path(),
     ))
     .await;
@@ -338,19 +334,19 @@ mod tests {
   #[actix_web::test]
   async fn parameterized_post_class_header_https_tickets() {
     let base_path = TempDir::new().unwrap();
-    server_tests::test_parameterized_post_class_header::<JsonResponse, _>(
+    server::test_parameterized_post_class_header::<JsonResponse, _>(
       &ActixTestServer::new_with_tls(base_path.path()),
     )
     .await;
   }
 
   #[actix_web::test]
   async fn cors_simple_request() {
-    cors_tests::test_cors_simple_request(&ActixTestServer::default()).await;
+    cors::test_cors_simple_request(&ActixTestServer::default()).await;
   }
 
   #[actix_web::test]
   async fn cors_preflight_request() {
-    cors_tests::test_cors_preflight_request(&ActixTestServer::default()).await;
+    cors::test_cors_preflight_request(&ActixTestServer::default()).await;
   }
 }

htsget-actix/Cargo.toml

@@ -23,7 +23,7 @@ lambda_runtime = { version = "0.8" }
 htsget-config = { version = "0.8.1", path = "../htsget-config", default-features = false }
 htsget-search = { version = "0.6.6", path = "../htsget-search", default-features = false }
 htsget-http = { version = "0.4.12", path = "../htsget-http", default-features = false }
-htsget-test = { version = "0.5.4", path = "../htsget-test", features = ["server-tests", "cors-tests"], default-features = false }
+htsget-test = { version = "0.5.4", path = "../htsget-test", features = ["http"], default-features = false }
 serde = { version = "1.0" }
 serde_json = "1.0"
 mime = "0.3"

htsget-actix/benches/request_benchmarks.rs

@@ -281,10 +281,10 @@ mod tests {
   use htsget_http::Endpoint;
   use htsget_search::storage::configure_cors;
   use htsget_search::storage::data_server::BindDataServer;
-  use htsget_test::http_tests::{config_with_tls, default_test_config, get_test_file};
-  use htsget_test::http_tests::{Header, Response as TestResponse, TestRequest, TestServer};
-  use htsget_test::server_tests::{expected_url_path, test_response, test_response_service_info};
-  use htsget_test::{cors_tests, server_tests};
+  use htsget_test::http::server::{expected_url_path, test_response, test_response_service_info};
+  use htsget_test::http::{config_with_tls, default_test_config, get_test_file};
+  use htsget_test::http::{cors, server};
+  use htsget_test::http::{Header, Response as TestResponse, TestRequest, TestServer};
 
   use super::*;
 
@@ -383,60 +383,56 @@ mod tests {
 
   #[tokio::test]
   async fn get_http_tickets() {
-    server_tests::test_get::<JsonResponse, _>(&LambdaTestServer::default()).await;
+    server::test_get::<JsonResponse, _>(&LambdaTestServer::default()).await;
   }
 
   #[tokio::test]
   async fn post_http_tickets() {
-    server_tests::test_post::<JsonResponse, _>(&LambdaTestServer::default()).await;
+    server::test_post::<JsonResponse, _>(&LambdaTestServer::default()).await;
   }
 
   #[tokio::test]
   async fn parameterized_get_http_tickets() {
-    server_tests::test_parameterized_get::<JsonResponse, _>(&LambdaTestServer::default()).await;
+    server::test_parameterized_get::<JsonResponse, _>(&LambdaTestServer::default()).await;
   }
 
   #[tokio::test]
   async fn parameterized_post_http_tickets() {
-    server_tests::test_parameterized_post::<JsonResponse, _>(&LambdaTestServer::default()).await;
+    server::test_parameterized_post::<JsonResponse, _>(&LambdaTestServer::default()).await;
   }
 
   #[tokio::test]
   async fn parameterized_post_class_header_http_tickets() {
-    server_tests::test_parameterized_post_class_header::<JsonResponse, _>(
-      &LambdaTestServer::default(),
-    )
-    .await;
+    server::test_parameterized_post_class_header::<JsonResponse, _>(&LambdaTestServer::default())
+      .await;
   }
 
   #[tokio::test]
   async fn cors_simple_request() {
-    cors_tests::test_cors_simple_request(&LambdaTestServer::default()).await;
+    cors::test_cors_simple_request(&LambdaTestServer::default()).await;
   }
 
   #[tokio::test]
   async fn cors_preflight_request() {
-    cors_tests::test_cors_preflight_request(&LambdaTestServer::default()).await;
+    cors::test_cors_preflight_request(&LambdaTestServer::default()).await;
   }
 
   #[tokio::test]
   async fn get_https_tickets() {
     let base_path = TempDir::new().unwrap();
-    server_tests::test_get::<JsonResponse, _>(&LambdaTestServer::new_with_tls(base_path.path()))
-      .await;
+    server::test_get::<JsonResponse, _>(&LambdaTestServer::new_with_tls(base_path.path())).await;
   }
 
   #[tokio::test]
   async fn post_https_tickets() {
     let base_path = TempDir::new().unwrap();
-    server_tests::test_post::<JsonResponse, _>(&LambdaTestServer::new_with_tls(base_path.path()))
-      .await;
+    server::test_post::<JsonResponse, _>(&LambdaTestServer::new_with_tls(base_path.path())).await;
   }
 
   #[tokio::test]
   async fn parameterized_get_https_tickets() {
     let base_path = TempDir::new().unwrap();
-    server_tests::test_parameterized_get::<JsonResponse, _>(&LambdaTestServer::new_with_tls(
+    server::test_parameterized_get::<JsonResponse, _>(&LambdaTestServer::new_with_tls(
       base_path.path(),
     ))
     .await;
@@ -445,7 +441,7 @@ mod tests {
   #[tokio::test]
   async fn parameterized_post_https_tickets() {
     let base_path = TempDir::new().unwrap();
-    server_tests::test_parameterized_post::<JsonResponse, _>(&LambdaTestServer::new_with_tls(
+    server::test_parameterized_post::<JsonResponse, _>(&LambdaTestServer::new_with_tls(
       base_path.path(),
     ))
     .await;
@@ -454,15 +450,15 @@ mod tests {
   #[tokio::test]
   async fn parameterized_post_class_header_https_tickets() {
     let base_path = TempDir::new().unwrap();
-    server_tests::test_parameterized_post_class_header::<JsonResponse, _>(
+    server::test_parameterized_post_class_header::<JsonResponse, _>(
       &LambdaTestServer::new_with_tls(base_path.path()),
     )
     .await;
   }
 
   #[tokio::test]
   async fn service_info() {
-    server_tests::test_service_info(&LambdaTestServer::default()).await;
+    server::test_service_info(&LambdaTestServer::default()).await;
   }
 
   #[tokio::test]

htsget-actix/src/lib.rs

@@ -60,7 +60,7 @@ hyper-rustls = { version = "0.24", features = ["rustls-native-certs", "http2", "
 # Error control, tracing, config
 thiserror = "1.0"
 htsget-config = { version = "0.8.1", path = "../htsget-config", default-features = false }
-htsget-test = { version = "0.5.4", path = "../htsget-test", features = ["cors-tests"], default-features = false }
+htsget-test = { version = "0.5.4", path = "../htsget-test", features = ["http"], default-features = false }
 tracing = "0.1"
 base64 = "0.21"
 serde = "1.0"

htsget-lambda/Cargo.toml

@@ -169,6 +169,7 @@ pub(crate) mod tests {
   use std::future::Future;
 
   use htsget_config::storage::local::LocalStorage as ConfigLocalStorage;
+  use htsget_test::http::concat::ConcatResponse;
   use htsget_test::util::expected_bgzf_eof_data_url;
 
   #[cfg(feature = "s3-storage")]
@@ -181,6 +182,7 @@ pub(crate) mod tests {
 
   const DATA_LOCATION: &str = "data/bam";
   const INDEX_FILE_LOCATION: &str = "htsnexus_test_NA12878.bam.bai";
+  pub(crate) const BAM_FILE_NAME: &str = "htsnexus_test_NA12878.bam";
 
   #[tokio::test]
   async fn search_all_reads() {
@@ -198,7 +200,9 @@ pub(crate) mod tests {
           Url::new(expected_bgzf_eof_data_url()),
         ],
       ));
-      assert_eq!(response, expected_response)
+      assert_eq!(response, expected_response);
+
+      Some((BAM_FILE_NAME.to_string(), (response.unwrap(), Body).into()))
     })
     .await;
   }
@@ -224,13 +228,39 @@ pub(crate) mod tests {
           Url::new(expected_bgzf_eof_data_url()).with_class(Body),
         ],
       ));
-      assert_eq!(response, expected_response)
+      assert_eq!(response, expected_response);
+
+      Some((BAM_FILE_NAME.to_string(), (response.unwrap(), Body).into()))
+    })
+    .await;
+  }
+
+  #[tokio::test]
+  async fn search_reference_name_without_seq_range_chr11() {
+    with_local_storage(|storage| async move {
+      let search = BamSearch::new(storage.clone());
+      let query = Query::new_with_default_request("htsnexus_test_NA12878", Format::Bam)
+        .with_reference_name("11");
+      let response = search.search(query).await;
+      println!("{response:#?}");
+
+      let expected_response = Ok(Response::new(
+        Format::Bam,
+        vec![
+          Url::new(expected_url())
+            .with_headers(Headers::default().with_header("Range", "bytes=0-996014")),
+          Url::new(expected_bgzf_eof_data_url()),
+        ],
+      ));
+      assert_eq!(response, expected_response);
+
+      Some((BAM_FILE_NAME.to_string(), (response.unwrap(), Body).into()))
     })
     .await;
   }
 
   #[tokio::test]
-  async fn search_reference_name_without_seq_range() {
+  async fn search_reference_name_without_seq_range_chr20() {
     with_local_storage(|storage| async move {
       let search = BamSearch::new(storage.clone());
       let query = Query::new_with_default_request("htsnexus_test_NA12878", Format::Bam)
@@ -250,7 +280,9 @@ pub(crate) mod tests {
           Url::new(expected_bgzf_eof_data_url()).with_class(Body),
         ],
       ));
-      assert_eq!(response, expected_response)
+      assert_eq!(response, expected_response);
+
+      Some((BAM_FILE_NAME.to_string(), (response.unwrap(), Body).into()))
     })
     .await;
   }
@@ -284,7 +316,9 @@ pub(crate) mod tests {
           Url::new(expected_bgzf_eof_data_url()).with_class(Body),
         ],
       ));
-      assert_eq!(response, expected_response)
+      assert_eq!(response, expected_response);
+
+      Some((BAM_FILE_NAME.to_string(), (response.unwrap(), Body).into()))
     })
     .await;
   }
@@ -311,7 +345,9 @@ pub(crate) mod tests {
           Url::new(expected_bgzf_eof_data_url()).with_class(Body),
         ],
       ));
-      assert_eq!(response, expected_response)
+      assert_eq!(response, expected_response);
+
+      Some((BAM_FILE_NAME.to_string(), (response.unwrap(), Body).into()))
     })
     .await;
   }
@@ -343,7 +379,9 @@ pub(crate) mod tests {
           Url::new(expected_bgzf_eof_data_url()),
         ],
       ));
-      assert_eq!(response, expected_response)
+      assert_eq!(response, expected_response);
+
+      Some((BAM_FILE_NAME.to_string(), (response.unwrap(), Body).into()))
     })
     .await
   }
@@ -372,10 +410,12 @@ pub(crate) mod tests {
             Url::new(expected_bgzf_eof_data_url()).with_class(Body),
           ],
         ));
-        assert_eq!(response, expected_response)
+        assert_eq!(response, expected_response);
+
+        Some((BAM_FILE_NAME.to_string(), (response.unwrap(), Body).into()))
       },
       DATA_LOCATION,
-      &["htsnexus_test_NA12878.bam", INDEX_FILE_LOCATION],
+      &[BAM_FILE_NAME, INDEX_FILE_LOCATION],
     )
     .await
   }
@@ -395,7 +435,12 @@ pub(crate) mod tests {
           .with_headers(Headers::default().with_header("Range", "bytes=0-4667"))
           .with_class(Header)],
       ));
-      assert_eq!(response, expected_response)
+      assert_eq!(response, expected_response);
+
+      Some((
+        BAM_FILE_NAME.to_string(),
+        (response.unwrap(), Header).into(),
+      ))
     })
     .await;
   }
@@ -419,6 +464,8 @@ pub(crate) mod tests {
         ],
       ));
       assert_eq!(response, expected_response);
+
+      Some((BAM_FILE_NAME.to_string(), (response.unwrap(), Body).into()))
     })
     .await;
   }
@@ -433,6 +480,8 @@ pub(crate) mod tests {
       println!("{response:#?}");
 
       assert!(matches!(response, Err(NotFound(_))));
+
+      None
     })
     .await;
   }
@@ -445,6 +494,8 @@ pub(crate) mod tests {
         let query = Query::new_with_default_request("htsnexus_test_NA12878", Format::Bam);
         let response = search.search(query).await;
         assert!(matches!(response, Err(NotFound(_))));
+
+        None
       },
       DATA_LOCATION,
       &[INDEX_FILE_LOCATION],
@@ -461,6 +512,8 @@ pub(crate) mod tests {
           .with_reference_name("20");
         let response = search.search(query).await;
         assert!(matches!(response, Err(NotFound(_))));
+
+        None
       },
       DATA_LOCATION,
       &[INDEX_FILE_LOCATION],
@@ -477,6 +530,8 @@ pub(crate) mod tests {
           Query::new_with_default_request("htsnexus_test_NA12878", Format::Bam).with_class(Header);
         let response = search.search(query).await;
         assert!(matches!(response, Err(NotFound(_))));
+
+        None
       },
       DATA_LOCATION,
       &[INDEX_FILE_LOCATION],
@@ -496,6 +551,8 @@ pub(crate) mod tests {
         let response = search.get_header_end_offset(&index).await;
 
         assert_eq!(response, Ok(70204));
+
+        None
       },
       DATA_LOCATION,
       &[INDEX_FILE_LOCATION],
@@ -512,6 +569,8 @@ pub(crate) mod tests {
         let query = Query::new_with_default_request("htsnexus_test_NA12878", Format::Bam);
         let response = search.search(query).await;
         assert!(response.is_err());
+
+        None
       },
       DATA_LOCATION,
       &[INDEX_FILE_LOCATION],
@@ -529,6 +588,8 @@ pub(crate) mod tests {
           .with_reference_name("20");
         let response = search.search(query).await;
         assert!(response.is_err());
+
+        None
       },
       DATA_LOCATION,
       &[INDEX_FILE_LOCATION],
@@ -546,6 +607,8 @@ pub(crate) mod tests {
           Query::new_with_default_request("htsnexus_test_NA12878", Format::Bam).with_class(Header);
         let response = search.search(query).await;
         assert!(response.is_err());
+
+        None
       },
       DATA_LOCATION,
       &[INDEX_FILE_LOCATION],
@@ -556,7 +619,7 @@ pub(crate) mod tests {
   pub(crate) async fn with_local_storage<F, Fut>(test: F)
   where
     F: FnOnce(Arc<LocalStorage<ConfigLocalStorage>>) -> Fut,
-    Fut: Future<Output = ()>,
+    Fut: Future<Output = Option<(String, ConcatResponse)>>,
   {
     with_local_storage_fn(test, DATA_LOCATION, &[]).await
   }

htsget-lambda/src/lib.rs

@@ -132,6 +132,8 @@ mod tests {
   use std::future::Future;
 
   use htsget_config::storage::local::LocalStorage as ConfigLocalStorage;
+  use htsget_config::types::Class::Body;
+  use htsget_test::http::concat::ConcatResponse;
   use htsget_test::util::expected_bgzf_eof_data_url;
 
   #[cfg(feature = "s3-storage")]
@@ -145,6 +147,8 @@ mod tests {
 
   const DATA_LOCATION: &str = "data/bcf";
   const INDEX_FILE_LOCATION: &str = "vcf-spec-v4.3.bcf.csi";
+  const BCF_FILE_NAME_SPEC: &str = "vcf-spec-v4.3.bcf";
+  const BCF_FILE_NAME_SAMPLE: &str = "sample1-bcbio-cancer.bcf";
 
   #[tokio::test]
   async fn search_all_variants() {
@@ -156,7 +160,12 @@ mod tests {
       println!("{response:#?}");
 
       let expected_response = Ok(expected_bcf_response(filename));
-      assert_eq!(response, expected_response)
+      assert_eq!(response, expected_response);
+
+      Some((
+        BCF_FILE_NAME_SAMPLE.to_string(),
+        (response.unwrap(), Body).into(),
+      ))
     })
     .await
   }
@@ -178,7 +187,12 @@ mod tests {
           Url::new(expected_bgzf_eof_data_url()),
         ],
       ));
-      assert_eq!(response, expected_response)
+      assert_eq!(response, expected_response);
+
+      Some((
+        BCF_FILE_NAME_SPEC.to_string(),
+        (response.unwrap(), Body).into(),
+      ))
     })
     .await
   }
@@ -203,7 +217,12 @@ mod tests {
       println!("{response:#?}");
 
       let expected_response = Ok(expected_bcf_response(filename));
-      assert_eq!(response, expected_response)
+      assert_eq!(response, expected_response);
+
+      Some((
+        BCF_FILE_NAME_SAMPLE.to_string(),
+        (response.unwrap(), Body).into(),
+      ))
     })
     .await
   }
@@ -233,7 +252,12 @@ mod tests {
           .with_headers(Headers::default().with_header("Range", "bytes=0-949"))
           .with_class(Header)],
       ));
-      assert_eq!(response, expected_response)
+      assert_eq!(response, expected_response);
+
+      Some((
+        BCF_FILE_NAME_SPEC.to_string(),
+        (response.unwrap(), Header).into(),
+      ))
     })
     .await
   }
@@ -246,6 +270,8 @@ mod tests {
         let query = Query::new_with_default_request("vcf-spec-v4.3", Format::Bcf);
         let response = search.search(query).await;
         assert!(matches!(response, Err(NotFound(_))));
+
+        None
       },
       DATA_LOCATION,
       &[INDEX_FILE_LOCATION],
@@ -262,6 +288,8 @@ mod tests {
           Query::new_with_default_request("vcf-spec-v4.3", Format::Bcf).with_reference_name("chrM");
         let response = search.search(query).await;
         assert!(matches!(response, Err(NotFound(_))));
+
+        None
       },
       DATA_LOCATION,
       &[INDEX_FILE_LOCATION],
@@ -278,6 +306,8 @@ mod tests {
           Query::new_with_default_request("vcf-spec-v4.3", Format::Bcf).with_class(Header);
         let response = search.search(query).await;
         assert!(matches!(response, Err(NotFound(_))));
+
+        None
       },
       DATA_LOCATION,
       &[INDEX_FILE_LOCATION],
@@ -295,6 +325,8 @@ mod tests {
       println!("{response:#?}");
 
       assert!(matches!(response, Err(NotFound(_))));
+
+      None
     })
     .await;
   }
@@ -311,6 +343,8 @@ mod tests {
         let response = search.get_header_end_offset(&index).await;
 
         assert_eq!(response, Ok(65536));
+
+        None
       },
       DATA_LOCATION,
       &[INDEX_FILE_LOCATION],
@@ -327,6 +361,8 @@ mod tests {
         let query = Query::new_with_default_request("vcf-spec-v4.3", Format::Bcf);
         let response = search.search(query).await;
         assert!(response.is_err());
+
+        None
       },
       DATA_LOCATION,
       &[INDEX_FILE_LOCATION],
@@ -344,6 +380,8 @@ mod tests {
           Query::new_with_default_request("vcf-spec-v4.3", Format::Bcf).with_reference_name("chrM");
         let response = search.search(query).await;
         assert!(response.is_err());
+
+        None
       },
       DATA_LOCATION,
       &[INDEX_FILE_LOCATION],
@@ -361,14 +399,18 @@ mod tests {
           Query::new_with_default_request("vcf-spec-v4.3", Format::Bcf).with_class(Header);
         let response = search.search(query).await;
         assert!(response.is_err());
+
+        None
       },
       DATA_LOCATION,
       &[INDEX_FILE_LOCATION],
     )
     .await
   }
 
-  async fn test_reference_sequence_with_seq_range(storage: Arc<LocalStorage<ConfigLocalStorage>>) {
+  async fn test_reference_sequence_with_seq_range(
+    storage: Arc<LocalStorage<ConfigLocalStorage>>,
+  ) -> Option<(String, ConcatResponse)> {
     let search = BcfSearch::new(storage.clone());
     let filename = "sample1-bcbio-cancer";
     let query = Query::new_with_default_request(filename, Format::Bcf)
@@ -379,7 +421,12 @@ mod tests {
     println!("{response:#?}");
 
     let expected_response = Ok(expected_bcf_response(filename));
-    assert_eq!(response, expected_response)
+    assert_eq!(response, expected_response);
+
+    Some((
+      BCF_FILE_NAME_SAMPLE.to_string(),
+      (response.unwrap(), Body).into(),
+    ))
   }
 
   fn expected_bcf_response(filename: &str) -> Response {
@@ -396,7 +443,7 @@ mod tests {
   async fn with_local_storage<F, Fut>(test: F)
   where
     F: FnOnce(Arc<LocalStorage<ConfigLocalStorage>>) -> Fut,
-    Fut: Future<Output = ()>,
+    Fut: Future<Output = Option<(String, ConcatResponse)>>,
   {
     with_local_storage_fn(test, "data/bcf", &[]).await
   }

htsget-search/Cargo.toml

@@ -292,6 +292,7 @@ mod tests {
   use std::future::Future;
 
   use htsget_config::storage::local::LocalStorage as ConfigLocalStorage;
+  use htsget_test::http::concat::ConcatResponse;
   use htsget_test::util::expected_cram_eof_data_url;
 
   #[cfg(feature = "s3-storage")]
@@ -304,6 +305,7 @@ mod tests {
 
   const DATA_LOCATION: &str = "data/cram";
   const INDEX_FILE_LOCATION: &str = "htsnexus_test_NA12878.cram.crai";
+  const CRAM_FILE_NAME: &str = "htsnexus_test_NA12878.cram";
 
   #[tokio::test]
   async fn search_all_reads() {
@@ -317,11 +319,13 @@ mod tests {
         Format::Cram,
         vec![
           Url::new(expected_url())
-            .with_headers(Headers::default().with_header("Range", "bytes=0-1627755")),
+            .with_headers(Headers::default().with_header("Range", "bytes=0-1672409")),
           Url::new(expected_cram_eof_data_url()),
         ],
       ));
-      assert_eq!(response, expected_response)
+      assert_eq!(response, expected_response);
+
+      Some((CRAM_FILE_NAME.to_string(), (response.unwrap(), Body).into()))
     })
     .await;
   }
@@ -339,21 +343,47 @@ mod tests {
         Format::Cram,
         vec![
           Url::new(expected_url())
-            .with_headers(Headers::default().with_header("Range", "bytes=0-6086"))
+            .with_headers(Headers::default().with_header("Range", "bytes=0-6133"))
             .with_class(Header),
           Url::new(expected_url())
-            .with_headers(Headers::default().with_header("Range", "bytes=1280106-1627755"))
+            .with_headers(Headers::default().with_header("Range", "bytes=1324614-1672409"))
             .with_class(Body),
           Url::new(expected_cram_eof_data_url()).with_class(Body),
         ],
       ));
-      assert_eq!(response, expected_response)
+      assert_eq!(response, expected_response);
+
+      Some((CRAM_FILE_NAME.to_string(), (response.unwrap(), Body).into()))
     })
     .await;
   }
 
   #[tokio::test]
-  async fn search_reference_name_without_seq_range() {
+  async fn search_reference_name_without_seq_range_chr11() {
+    with_local_storage(|storage| async move {
+      let search = CramSearch::new(storage.clone());
+      let query = Query::new_with_default_request("htsnexus_test_NA12878", Format::Cram)
+        .with_reference_name("11");
+      let response = search.search(query).await;
+      println!("{response:#?}");
+
+      let expected_response = Ok(Response::new(
+        Format::Cram,
+        vec![
+          Url::new(expected_url())
+            .with_headers(Headers::default().with_header("Range", "bytes=0-625727")),
+          Url::new(expected_cram_eof_data_url()),
+        ],
+      ));
+      assert_eq!(response, expected_response);
+
+      Some((CRAM_FILE_NAME.to_string(), (response.unwrap(), Body).into()))
+    })
+    .await;
+  }
+
+  #[tokio::test]
+  async fn search_reference_name_without_seq_range_chr20() {
     with_local_storage(|storage| async move {
       let search = CramSearch::new(storage.clone());
       let query = Query::new_with_default_request("htsnexus_test_NA12878", Format::Cram)
@@ -365,15 +395,17 @@ mod tests {
         Format::Cram,
         vec![
           Url::new(expected_url())
-            .with_headers(Headers::default().with_header("Range", "bytes=0-6086"))
+            .with_headers(Headers::default().with_header("Range", "bytes=0-6133"))
             .with_class(Header),
           Url::new(expected_url())
-            .with_headers(Headers::default().with_header("Range", "bytes=604231-1280105"))
+            .with_headers(Headers::default().with_header("Range", "bytes=625728-1324613"))
             .with_class(Body),
           Url::new(expected_cram_eof_data_url()).with_class(Body),
         ],
       ));
-      assert_eq!(response, expected_response)
+      assert_eq!(response, expected_response);
+
+      Some((CRAM_FILE_NAME.to_string(), (response.unwrap(), Body).into()))
     })
     .await;
   }
@@ -393,11 +425,13 @@ mod tests {
         Format::Cram,
         vec![
           Url::new(expected_url())
-            .with_headers(Headers::default().with_header("Range", "bytes=0-465708")),
+            .with_headers(Headers::default().with_header("Range", "bytes=0-480537")),
           Url::new(expected_cram_eof_data_url()),
         ],
       ));
-      assert_eq!(response, expected_response)
+      assert_eq!(response, expected_response);
+
+      Some((CRAM_FILE_NAME.to_string(), (response.unwrap(), Body).into()))
     })
     .await;
   }
@@ -414,7 +448,9 @@ mod tests {
       println!("{response:#?}");
 
       let expected_response = Ok(expected_response_with_start());
-      assert_eq!(response, expected_response)
+      assert_eq!(response, expected_response);
+
+      Some((CRAM_FILE_NAME.to_string(), (response.unwrap(), Body).into()))
     })
     .await;
   }
@@ -430,7 +466,9 @@ mod tests {
       println!("{response:#?}");
 
       let expected_response = Ok(expected_response_with_start());
-      assert_eq!(response, expected_response)
+      assert_eq!(response, expected_response);
+
+      Some((CRAM_FILE_NAME.to_string(), (response.unwrap(), Body).into()))
     })
     .await;
   }
@@ -440,7 +478,7 @@ mod tests {
       Format::Cram,
       vec![
         Url::new(expected_url())
-          .with_headers(Headers::default().with_header("Range", "bytes=0-604230")),
+          .with_headers(Headers::default().with_header("Range", "bytes=0-625727")),
         Url::new(expected_cram_eof_data_url()),
       ],
     )
@@ -458,10 +496,15 @@ mod tests {
       let expected_response = Ok(Response::new(
         Format::Cram,
         vec![Url::new(expected_url())
-          .with_headers(Headers::default().with_header("Range", "bytes=0-6086"))
+          .with_headers(Headers::default().with_header("Range", "bytes=0-6133"))
           .with_class(Header)],
       ));
-      assert_eq!(response, expected_response)
+      assert_eq!(response, expected_response);
+
+      Some((
+        CRAM_FILE_NAME.to_string(),
+        (response.unwrap(), Header).into(),
+      ))
     })
     .await;
   }
@@ -474,6 +517,8 @@ mod tests {
         let query = Query::new_with_default_request("htsnexus_test_NA12878", Format::Cram);
         let response = search.search(query).await;
         assert!(matches!(response, Err(NotFound(_))));
+
+        None
       },
       DATA_LOCATION,
       &[INDEX_FILE_LOCATION],
@@ -490,6 +535,8 @@ mod tests {
           .with_reference_name("20");
         let response = search.search(query).await;
         assert!(matches!(response, Err(NotFound(_))));
+
+        None
       },
       DATA_LOCATION,
       &[INDEX_FILE_LOCATION],
@@ -506,6 +553,8 @@ mod tests {
           Query::new_with_default_request("htsnexus_test_NA12878", Format::Cram).with_class(Header);
         let response = search.search(query).await;
         assert!(matches!(response, Err(NotFound(_))));
+
+        None
       },
       DATA_LOCATION,
       &[INDEX_FILE_LOCATION],
@@ -522,6 +571,8 @@ mod tests {
         let query = Query::new_with_default_request("htsnexus_test_NA12878", Format::Cram);
         let response = search.search(query).await;
         assert!(response.is_err());
+
+        None
       },
       DATA_LOCATION,
       &[INDEX_FILE_LOCATION],
@@ -539,6 +590,8 @@ mod tests {
           .with_reference_name("20");
         let response = search.search(query).await;
         assert!(response.is_err());
+
+        None
       },
       DATA_LOCATION,
       &[INDEX_FILE_LOCATION],
@@ -556,6 +609,8 @@ mod tests {
           Query::new_with_default_request("htsnexus_test_NA12878", Format::Cram).with_class(Header);
         let response = search.search(query).await;
         assert!(response.is_err());
+
+        None
       },
       DATA_LOCATION,
       &[INDEX_FILE_LOCATION],
@@ -566,7 +621,7 @@ mod tests {
   async fn with_local_storage<F, Fut>(test: F)
   where
     F: FnOnce(Arc<LocalStorage<ConfigLocalStorage>>) -> Fut,
-    Fut: Future<Output = ()>,
+    Fut: Future<Output = Option<(String, ConcatResponse)>>,
   {
     with_local_storage_fn(test, "data/cram", &[]).await
   }

htsget-search/src/htsget/bam_search.rs

@@ -133,14 +133,17 @@ pub(crate) mod tests {
   use tempfile::TempDir;
 
   use htsget_config::storage;
+  use htsget_config::types::Class::Body;
   use htsget_config::types::Scheme::Http;
+  use htsget_test::http::concat::ConcatResponse;
   use htsget_test::util::expected_bgzf_eof_data_url;
 
   use crate::htsget::bam_search::tests::{
-    expected_url as bam_expected_url, with_local_storage as with_bam_local_storage,
+    expected_url as bam_expected_url, with_local_storage as with_bam_local_storage, BAM_FILE_NAME,
   };
   use crate::htsget::vcf_search::tests::{
     expected_url as vcf_expected_url, with_local_storage as with_vcf_local_storage,
+    VCF_FILE_NAME_SPEC,
   };
   #[cfg(feature = "s3-storage")]
   use crate::storage::s3::tests::with_aws_s3_storage_fn;
@@ -164,7 +167,9 @@ pub(crate) mod tests {
           Url::new(expected_bgzf_eof_data_url()),
         ],
       ));
-      assert_eq!(response, expected_response)
+      assert_eq!(response, expected_response);
+
+      Some((BAM_FILE_NAME.to_string(), (response.unwrap(), Body).into()))
     })
     .await;
   }
@@ -179,6 +184,11 @@ pub(crate) mod tests {
       println!("{response:#?}");
 
       assert_eq!(response, expected_vcf_response(filename));
+
+      Some((
+        VCF_FILE_NAME_SPEC.to_string(),
+        (response.unwrap(), Body).into(),
+      ))
     })
     .await;
   }
@@ -192,6 +202,11 @@ pub(crate) mod tests {
         let response = HtsGetFromStorage::<()>::from_local(&local_storage, &query).await;
 
         assert_eq!(response, expected_vcf_response(filename));
+
+        Some((
+          VCF_FILE_NAME_SPEC.to_string(),
+          (response.unwrap(), Body).into(),
+        ))
       },
       "data/vcf",
       &[],
@@ -216,6 +231,11 @@ pub(crate) mod tests {
         let response = resolvers.search(query).await;
 
         assert_eq!(response, expected_vcf_response(filename));
+
+        Some((
+          VCF_FILE_NAME_SPEC.to_string(),
+          (response.unwrap(), Body).into(),
+        ))
       },
       "data/vcf",
       &[],
@@ -234,33 +254,33 @@ pub(crate) mod tests {
     ))
   }
 
-  async fn copy_files(from_path: &str, to_path: &Path, file_names: &[&str]) -> PathBuf {
+  async fn copy_files_from(from_path: &str, to_path: &Path, copy_files: &[&str]) -> PathBuf {
     let mut base_path = std::env::current_dir()
       .unwrap()
       .parent()
       .unwrap()
       .join(from_path);
 
-    for file_name in file_names {
+    for file_name in copy_files {
       fs::copy(base_path.join(file_name), to_path.join(file_name)).unwrap();
     }
-    if !file_names.is_empty() {
+    if !copy_files.is_empty() {
       base_path = PathBuf::from(to_path);
     }
 
     base_path
   }
 
-  async fn with_config_local_storage<F, Fut>(test: F, path: &str, file_names: &[&str])
+  async fn with_config_local_storage<F, Fut>(test: F, path: &str, copy_files: &[&str])
   where
     F: FnOnce(PathBuf, LocalStorageConfig) -> Fut,
-    Fut: Future<Output = ()>,
+    Fut: Future<Output = Option<(String, ConcatResponse)>>,
   {
     let tmp_dir = TempDir::new().unwrap();
-    let base_path = copy_files(path, tmp_dir.path(), file_names).await;
+    let base_path = copy_files_from(path, tmp_dir.path(), copy_files).await;
 
     println!("{:#?}", base_path);
-    test(
+    let response = test(
       base_path.clone(),
       LocalStorageConfig::new(
         Http,
@@ -269,13 +289,27 @@ pub(crate) mod tests {
         "/data".to_string(),
       ),
     )
-    .await
+    .await;
+
+    read_records(response, &base_path).await;
   }
 
-  pub(crate) async fn with_local_storage_fn<F, Fut>(test: F, path: &str, file_names: &[&str])
+  async fn read_records(response: Option<(String, ConcatResponse)>, base_path: &Path) {
+    if let Some((target_file, response)) = response {
+      response
+        .concat_from_file_path(&base_path.join(target_file))
+        .await
+        .unwrap()
+        .read_records()
+        .await
+        .unwrap();
+    }
+  }
+
+  pub(crate) async fn with_local_storage_fn<F, Fut>(test: F, path: &str, copy_files: &[&str])
   where
     F: FnOnce(Arc<LocalStorage<LocalStorageConfig>>) -> Fut,
-    Fut: Future<Output = ()>,
+    Fut: Future<Output = Option<(String, ConcatResponse)>>,
   {
     with_config_local_storage(
       |base_path, local_storage| async {
@@ -285,23 +319,31 @@ pub(crate) mod tests {
         .await
       },
       path,
-      file_names,
+      copy_files,
     )
     .await;
   }
 
   #[cfg(feature = "s3-storage")]
-  pub(crate) async fn with_aws_storage_fn<F, Fut>(test: F, path: &str, file_names: &[&str])
+  pub(crate) async fn with_aws_storage_fn<F, Fut>(test: F, path: &str, copy_files: &[&str])
   where
     F: FnOnce(Arc<S3Storage>) -> Fut,
-    Fut: Future<Output = ()>,
+    Fut: Future<Output = Option<(String, ConcatResponse)>>,
   {
     let tmp_dir = TempDir::new().unwrap();
     let to_path = tmp_dir.into_path().join("folder");
     create_dir(&to_path).unwrap();
 
-    let base_path = copy_files(path, &to_path, file_names).await;
+    let base_path = copy_files_from(path, &to_path, copy_files).await;
 
-    with_aws_s3_storage_fn(test, "folder".to_string(), base_path.parent().unwrap()).await;
+    with_aws_s3_storage_fn(
+      |storage| async {
+        let response = test(storage).await;
+        read_records(response, &base_path).await;
+      },
+      "folder".to_string(),
+      base_path.parent().unwrap(),
+    )
+    .await;
   }
 }

htsget-search/src/htsget/bcf_search.rs

@@ -137,6 +137,8 @@ pub(crate) mod tests {
   use std::future::Future;
 
   use htsget_config::storage::local::LocalStorage as ConfigLocalStorage;
+  use htsget_config::types::Class::Body;
+  use htsget_test::http::concat::ConcatResponse;
   use htsget_test::util::expected_bgzf_eof_data_url;
 
   #[cfg(feature = "s3-storage")]
@@ -150,6 +152,8 @@ pub(crate) mod tests {
 
   const VCF_LOCATION: &str = "data/vcf";
   const INDEX_FILE_LOCATION: &str = "spec-v4.3.vcf.gz.tbi";
+  pub(crate) const VCF_FILE_NAME_SPEC: &str = "spec-v4.3.vcf.gz";
+  const VCF_FILE_NAME_SAMPLE: &str = "sample1-bcbio-cancer.vcf.gz";
 
   #[tokio::test]
   async fn search_all_variants() {
@@ -161,7 +165,12 @@ pub(crate) mod tests {
       println!("{response:#?}");
 
       let expected_response = Ok(expected_vcf_response(filename));
-      assert_eq!(response, expected_response)
+      assert_eq!(response, expected_response);
+
+      Some((
+        VCF_FILE_NAME_SAMPLE.to_string(),
+        (response.unwrap(), Body).into(),
+      ))
     })
     .await;
   }
@@ -183,7 +192,12 @@ pub(crate) mod tests {
           Url::new(expected_bgzf_eof_data_url()),
         ],
       ));
-      assert_eq!(response, expected_response)
+      assert_eq!(response, expected_response);
+
+      Some((
+        VCF_FILE_NAME_SPEC.to_string(),
+        (response.unwrap(), Body).into(),
+      ))
     })
     .await;
   }
@@ -208,6 +222,11 @@ pub(crate) mod tests {
 
       let expected_response = Ok(expected_vcf_response(filename));
       assert_eq!(response, expected_response);
+
+      Some((
+        VCF_FILE_NAME_SAMPLE.to_string(),
+        (response.unwrap(), Body).into(),
+      ))
     })
     .await;
   }
@@ -240,7 +259,12 @@ pub(crate) mod tests {
           .with_headers(Headers::default().with_header("Range", "bytes=0-822"))
           .with_class(Header)],
       ));
-      assert_eq!(response, expected_response)
+      assert_eq!(response, expected_response);
+
+      Some((
+        VCF_FILE_NAME_SPEC.to_string(),
+        (response.unwrap(), Header).into(),
+      ))
     })
     .await;
   }
@@ -253,6 +277,8 @@ pub(crate) mod tests {
         let query = Query::new_with_default_request("spec-v4.3", Format::Vcf);
         let response = search.search(query).await;
         assert!(matches!(response, Err(NotFound(_))));
+
+        None
       },
       VCF_LOCATION,
       &[INDEX_FILE_LOCATION],
@@ -269,6 +295,8 @@ pub(crate) mod tests {
           Query::new_with_default_request("spec-v4.3", Format::Vcf).with_reference_name("chrM");
         let response = search.search(query).await;
         assert!(matches!(response, Err(NotFound(_))));
+
+        None
       },
       VCF_LOCATION,
       &[INDEX_FILE_LOCATION],
@@ -284,6 +312,8 @@ pub(crate) mod tests {
         let query = Query::new_with_default_request("spec-v4.3", Format::Vcf).with_class(Header);
         let response = search.search(query).await;
         assert!(matches!(response, Err(NotFound(_))));
+
+        None
       },
       VCF_LOCATION,
       &[INDEX_FILE_LOCATION],
@@ -301,6 +331,8 @@ pub(crate) mod tests {
       println!("{response:#?}");
 
       assert!(matches!(response, Err(NotFound(_))));
+
+      None
     })
     .await;
   }
@@ -316,6 +348,8 @@ pub(crate) mod tests {
         let response = search.get_header_end_offset(&index).await;
 
         assert_eq!(response, Ok(65536));
+
+        None
       },
       VCF_LOCATION,
       &[INDEX_FILE_LOCATION],
@@ -332,6 +366,8 @@ pub(crate) mod tests {
         let query = Query::new_with_default_request("spec-v4.3", Format::Vcf);
         let response = search.search(query).await;
         assert!(response.is_err());
+
+        None
       },
       VCF_LOCATION,
       &[INDEX_FILE_LOCATION],
@@ -349,6 +385,8 @@ pub(crate) mod tests {
           Query::new_with_default_request("spec-v4.3", Format::Vcf).with_reference_name("chrM");
         let response = search.search(query).await;
         assert!(response.is_err());
+
+        None
       },
       VCF_LOCATION,
       &[INDEX_FILE_LOCATION],
@@ -365,14 +403,18 @@ pub(crate) mod tests {
         let query = Query::new_with_default_request("spec-v4.3", Format::Vcf).with_class(Header);
         let response = search.search(query).await;
         assert!(response.is_err());
+
+        None
       },
       VCF_LOCATION,
       &[INDEX_FILE_LOCATION],
     )
     .await
   }
 
-  async fn test_reference_name_with_seq_range(storage: Arc<LocalStorage<ConfigLocalStorage>>) {
+  async fn test_reference_name_with_seq_range(
+    storage: Arc<LocalStorage<ConfigLocalStorage>>,
+  ) -> Option<(String, ConcatResponse)> {
     let search = VcfSearch::new(storage.clone());
     let filename = "sample1-bcbio-cancer";
     let query = Query::new_with_default_request(filename, Format::Vcf)
@@ -384,6 +426,11 @@ pub(crate) mod tests {
 
     let expected_response = Ok(expected_vcf_response(filename));
     assert_eq!(response, expected_response);
+
+    Some((
+      VCF_FILE_NAME_SAMPLE.to_string(),
+      (response.unwrap(), Body).into(),
+    ))
   }
 
   fn expected_vcf_response(filename: &str) -> Response {
@@ -400,7 +447,7 @@ pub(crate) mod tests {
   pub(crate) async fn with_local_storage<F, Fut>(test: F)
   where
     F: FnOnce(Arc<LocalStorage<ConfigLocalStorage>>) -> Fut,
-    Fut: Future<Output = ()>,
+    Fut: Future<Output = Option<(String, ConcatResponse)>>,
   {
     with_local_storage_fn(test, "data/vcf", &[]).await
   }

htsget-search/src/htsget/cram_search.rs

@@ -211,8 +211,8 @@ mod tests {
   use reqwest::{Client, ClientBuilder, RequestBuilder};
   use tempfile::tempdir;
 
-  use htsget_test::cors_tests::{test_cors_preflight_request_uri, test_cors_simple_request_uri};
-  use htsget_test::http_tests::{
+  use htsget_test::http::cors::{test_cors_preflight_request_uri, test_cors_simple_request_uri};
+  use htsget_test::http::{
     config_with_tls, default_cors_config, default_test_config, Header, Response as TestResponse,
     TestRequest, TestServer,
   };

htsget-search/src/htsget/from_storage.rs

@@ -11,17 +11,12 @@ homepage = "https://github.com/umccr/htsget-rs/blob/main/htsget-test/README.md"
 repository = "https://github.com/umccr/htsget-rs"
 
 [features]
-http-tests = [
+http = [
     "dep:async-trait",
     "dep:http",
     "dep:serde_json",
     "dep:serde",
     "dep:htsget-config",
-]
-cors-tests = ["http-tests", "dep:htsget-config"]
-server-tests = [
-    "http-tests",
-    "dep:htsget-config",
     "dep:noodles",
     "dep:reqwest",
     "dep:tokio",
@@ -47,10 +42,10 @@ default = []
 # Server tests dependencies
 htsget-config = { version = "0.8.1", path = "../htsget-config", default-features = false, optional = true }
 
-noodles = { version = "0.65", optional = true, features = ["async", "bgzf", "vcf"] }
+noodles = { version = "0.65", optional = true, features = ["async", "bgzf", "vcf", "cram", "bcf", "bam", "fasta"] }
 
 reqwest = { version = "0.11", default-features = false, features = ["json", "rustls-tls"], optional = true }
-tokio = { version = "1", features = ["rt-multi-thread"], optional = true }
+tokio = { version = "1", features = ["rt-multi-thread", "fs"], optional = true }
 futures = { version = "0.3", optional = true }
 async-trait = { version = "0.1", optional = true }
 http = { version = "0.2", optional = true }
@@ -69,3 +64,4 @@ s3s-aws = { version = "0.8", optional = true }
 
 # Default dependencies
 rcgen = "0.12"
+thiserror = "1.0"

htsget-search/src/htsget/vcf_search.rs

@@ -0,0 +1,45 @@
+//! Errors defined by the htsget-test crate.
+//!
+
+use std::fmt::Display;
+use std::io::Error;
+use std::{io, result};
+use thiserror::Error;
+
+/// Result type for this crate.
+pub type Result<T> = result::Result<T, TestError>;
+
+/// The error that this crate can make.
+#[derive(Error, Debug)]
+pub enum TestError {
+  #[error("{0}")]
+  Io(io::Error),
+  #[error("reading records: {0}")]
+  ReadRecord(String),
+  #[error("concatenating response: {0}")]
+  ConcatResponse(String),
+}
+
+impl TestError {
+  /// Create a read record error.
+  pub fn read_record<E: Display>(error: E) -> Self {
+    Self::ReadRecord(error.to_string())
+  }
+
+  /// Create a concat response error.
+  pub fn concat_response<E: Display>(error: E) -> Self {
+    Self::ConcatResponse(error.to_string())
+  }
+}
+
+impl From<io::Error> for TestError {
+  fn from(error: Error) -> Self {
+    Self::Io(error)
+  }
+}
+
+impl From<TestError> for io::Error {
+  fn from(error: TestError) -> Self {
+    Error::new(io::ErrorKind::Other, error)
+  }
+}

htsget-search/src/storage/data_server.rs

@@ -0,0 +1,258 @@
+use crate::error::TestError::ConcatResponse as ConcatResponseError;
+use crate::error::{Result, TestError};
+use base64::engine::general_purpose;
+use base64::Engine;
+use futures::future::join_all;
+use futures::{Stream, TryStreamExt};
+use htsget_config::types::{Class, Format, Response, Url};
+use http::{HeaderMap, HeaderName, HeaderValue};
+use noodles::{bam, bcf, bgzf, cram, fasta, vcf};
+use reqwest::Client;
+use std::future::Future;
+use std::io;
+use std::path::Path;
+use std::str::FromStr;
+use tokio::fs::File;
+use tokio::io::AsyncReadExt;
+
+/// A response concatenator which concatenates url tickets.
+#[derive(Debug)]
+pub struct ConcatResponse {
+  response: Response,
+  class: Class,
+}
+
+impl ConcatResponse {
+  /// Create a new response concatenator.
+  pub fn new(response: Response, class: Class) -> Self {
+    Self { response, class }
+  }
+
+  /// Get the inner response.
+  pub fn into_inner(self) -> Response {
+    self.response
+  }
+
+  /// Get the inner response.
+  pub fn response(&self) -> &Response {
+    &self.response
+  }
+
+  /// Concatenate a response into the bytes represented by the url ticket with a file path
+  pub async fn concat_from_file_path(self, path: impl AsRef<Path>) -> Result<ReadRecords> {
+    let file = File::open(path).await?;
+    self.concat_from_file(file).await
+  }
+
+  /// Concatenate a response into the bytes represented by the url ticket with file data.
+  pub async fn concat_from_file(self, mut file: File) -> Result<ReadRecords> {
+    let mut bytes = vec![];
+    file.read_to_end(&mut bytes).await?;
+
+    self.concat_from_bytes(bytes.as_slice()).await
+  }
+
+  /// Concatentate a response into bytes using a reqwest client.
+  pub async fn concat_from_client(self, client: &Client) -> Result<ReadRecords> {
+    let merged_bytes = join_all(self.response.urls.into_iter().map(|url| {
+      Self::url_to_bytes(url, |url| async move {
+        Ok(
+          client
+            .get(url.url.as_str())
+            .headers(HeaderMap::from_iter(
+              url
+                .headers
+                .unwrap_or_default()
+                .into_inner()
+                .into_iter()
+                .map(|(key, value)| {
+                  Ok((
+                    HeaderName::from_str(&key).map_err(TestError::concat_response)?,
+                    HeaderValue::from_str(&value).map_err(TestError::concat_response)?,
+                  ))
+                })
+                .collect::<Result<Vec<(HeaderName, HeaderValue)>>>()?
+                .into_iter(),
+            ))
+            .send()
+            .await
+            .map_err(TestError::concat_response)?
+            .bytes()
+            .await
+            .map_err(TestError::concat_response)?
+            .to_vec(),
+        )
+      })
+    }))
+    .await
+    .into_iter()
+    .collect::<Result<Vec<Vec<u8>>>>()?
+    .concat();
+
+    Ok(ReadRecords::new(
+      self.response.format,
+      self.class,
+      merged_bytes,
+    ))
+  }
+
+  /// Concatenate a response into the bytes represented by the url ticket with bytes data.
+  pub async fn concat_from_bytes(self, bytes: &[u8]) -> Result<ReadRecords> {
+    let merged_bytes = join_all(self.response.urls.into_iter().map(|url| {
+      Self::url_to_bytes(url, |url| async move {
+        let headers = url
+          .headers
+          .ok_or_else(|| ConcatResponseError("missing url headers".to_string()))?
+          .into_inner();
+        let range = headers
+          .get("Range")
+          .ok_or_else(|| ConcatResponseError("missing header range".to_string()))?;
+        let range = range
+          .strip_prefix("bytes=")
+          .ok_or_else(|| ConcatResponseError("failed to parse header range bytes".to_string()))?;
+
+        let split: Vec<&str> = range.splitn(2, '-').collect();
+
+        Ok(
+          bytes[split[0].parse().map_err(TestError::read_record)?
+            ..split[1].parse::<usize>().map_err(TestError::read_record)? + 1]
+            .to_vec(),
+        )
+      })
+    }))
+    .await
+    .into_iter()
+    .collect::<Result<Vec<Vec<u8>>>>()?
+    .concat();
+
+    Ok(ReadRecords::new(
+      self.response.format,
+      self.class,
+      merged_bytes,
+    ))
+  }
+
+  /// Convert the url to bytes with a transform function for the range urls.
+  pub async fn url_to_bytes<F, Fut>(url: Url, for_range_url: F) -> Result<Vec<u8>>
+  where
+    F: FnOnce(Url) -> Fut,
+    Fut: Future<Output = Result<Vec<u8>>>,
+  {
+    if let Some(data_uri) = url.url.strip_prefix("data:;base64,") {
+      general_purpose::STANDARD
+        .decode(data_uri)
+        .map_err(TestError::concat_response)
+    } else {
+      for_range_url(url).await
+    }
+  }
+}
+
+impl From<(Response, Class)> for ConcatResponse {
+  fn from((response, class): (Response, Class)) -> Self {
+    Self::new(response, class)
+  }
+}
+
+/// A record reader.
+#[derive(Debug)]
+pub struct ReadRecords {
+  format: Format,
+  class: Class,
+  merged_bytes: Vec<u8>,
+}
+
+impl ReadRecords {
+  /// Create a new record reader.
+  pub fn new(format: Format, class: Class, merged_bytes: Vec<u8>) -> Self {
+    Self {
+      format,
+      class,
+      merged_bytes,
+    }
+  }
+
+  /// Get the format.
+  pub fn format(&self) -> &Format {
+    &self.format
+  }
+
+  /// Get the format.
+  pub fn merged_bytes(&self) -> &[u8] {
+    self.merged_bytes.as_slice()
+  }
+
+  /// Read records to confirm they are valid.
+  pub async fn read_records(self) -> Result<()> {
+    match self.format {
+      Format::Bam => {
+        let mut reader = bam::AsyncReader::new(self.merged_bytes.as_slice());
+        let header = reader.read_header().await.map_err(TestError::read_record)?;
+        println!("{:#?}", header);
+
+        self.iterate_records(reader.records()).await
+      }
+      Format::Cram => {
+        let mut reader = cram::AsyncReader::new(self.merged_bytes.as_slice());
+
+        reader
+          .read_file_definition()
+          .await
+          .map_err(TestError::read_record)?;
+        let repository = fasta::Repository::default();
+        let header = reader
+          .read_file_header()
+          .await
+          .map_err(TestError::read_record)?
+          .parse()
+          .map_err(TestError::read_record)?;
+        println!("{:#?}", header);
+
+        self
+          .iterate_records(reader.records(&repository, &header))
+          .await
+      }
+      Format::Vcf => {
+        let mut reader =
+          vcf::AsyncReader::new(bgzf::AsyncReader::new(self.merged_bytes.as_slice()));
+        let header = reader.read_header().await.map_err(TestError::read_record)?;
+        println!("{header}");
+
+        self.iterate_records(reader.records(&header)).await
+      }
+      Format::Bcf => {
+        let mut reader = bcf::AsyncReader::new(self.merged_bytes.as_slice());
+        reader
+          .read_file_format()
+          .await
+          .map_err(TestError::read_record)?;
+        reader.read_header().await.map_err(TestError::read_record)?;
+
+        self.iterate_records(reader.lazy_records()).await
+      }
+    }
+  }
+
+  async fn iterate_records<T>(
+    &self,
+    mut records: impl Stream<Item = io::Result<T>> + Unpin,
+  ) -> Result<()> {
+    if let Class::Body = self.class {
+      let mut total_records = 0;
+
+      while records
+        .try_next()
+        .await
+        .map_err(TestError::read_record)?
+        .is_some()
+      {
+        total_records += 1;
+        continue;
+      }
+
+      println!("total records read: {}", total_records);
+    }
+
+    Ok(())
+  }
+}

htsget-test/Cargo.toml

@@ -4,7 +4,7 @@ use http::header::{
 };
 use http::Method;
 
-use crate::http_tests::{Header, TestRequest, TestServer};
+use crate::http::{Header, TestRequest, TestServer};
 
 /// A simple cors request test.
 pub async fn test_cors_simple_request<T: TestRequest>(tester: &impl TestServer<T>) {

htsget-test/src/error.rs

@@ -1,3 +1,10 @@
+//! Testing functionality related to http and url tickets.
+//!
+
+pub mod concat;
+pub mod cors;
+pub mod server;
+
 use std::fs;
 use std::net::{SocketAddr, TcpListener};
 use std::path::{Path, PathBuf};

htsget-test/src/http/concat.rs

@@ -1,23 +1,16 @@
 use std::fmt::Debug;
 use std::net::SocketAddr;
-use std::str::FromStr;
 
-use base64::engine::general_purpose;
-use base64::Engine;
-use futures::future::join_all;
-use futures::TryStreamExt;
-use http::header::HeaderName;
-use http::{HeaderMap, HeaderValue, Method};
-use noodles::bgzf;
-use noodles::vcf;
+use http::Method;
 use reqwest::ClientBuilder;
 use serde::Deserialize;
 use serde_json::{json, Value};
 
+use crate::http::concat::ConcatResponse;
 use htsget_config::types::Class;
 use htsget_config::types::Format;
 
-use crate::http_tests::{Header, Response, TestRequest, TestServer};
+use crate::http::{Header, Response, TestRequest, TestServer};
 use crate::util::expected_bgzf_eof_data_url;
 use crate::Config;
 
@@ -45,65 +38,16 @@ where
     .build()
     .unwrap();
 
-  let merged_response = join_all(
-    expected_response
-      .get("htsget")
-      .unwrap()
-      .get("urls")
-      .unwrap()
-      .as_array()
-      .unwrap()
-      .iter()
-      .map(|url| async {
-        if let Some(data_uri) = url
-          .get("url")
-          .unwrap()
-          .as_str()
-          .unwrap()
-          .strip_prefix("data:;base64,")
-        {
-          general_purpose::STANDARD.decode(data_uri).unwrap()
-        } else {
-          client
-            .get(url.get("url").unwrap().as_str().unwrap())
-            .headers(HeaderMap::from_iter(
-              url
-                .get("headers")
-                .unwrap()
-                .as_object()
-                .unwrap_or(&serde_json::Map::new())
-                .into_iter()
-                .map(|(key, value)| {
-                  (
-                    HeaderName::from_str(key).unwrap(),
-                    HeaderValue::from_str(value.as_str().unwrap()).unwrap(),
-                  )
-                }),
-            ))
-            .send()
-            .await
-            .unwrap()
-            .bytes()
-            .await
-            .unwrap()
-            .to_vec()
-        }
-      }),
+  ConcatResponse::new(
+    serde_json::from_value(expected_response.get("htsget").unwrap().clone()).unwrap(),
+    class,
   )
+  .concat_from_client(&client)
+  .await
+  .unwrap()
+  .read_records()
   .await
-  .into_iter()
-  .reduce(|acc, x| [acc, x].concat())
   .unwrap();
-
-  let mut reader = vcf::AsyncReader::new(bgzf::AsyncReader::new(merged_response.as_slice()));
-  let header = reader.read_header().await.unwrap();
-  println!("{header}");
-
-  let mut records = reader.records(&header);
-  while let Some(record) = records.try_next().await.unwrap() {
-    println!("{record}");
-    continue;
-  }
 }
 
 /// Get the expected url path from the formatter.

htsget-test/src/cors_tests.rs htsget-test/src/http/cors.rs

@@ -1,19 +1,12 @@
-#[cfg(any(
-  feature = "cors-tests",
-  feature = "server-tests",
-  feature = "http-tests"
-))]
+#[cfg(any(feature = "server-tests", feature = "http"))]
 pub use htsget_config::{
   config::{Config, DataServerConfig, ServiceInfo, TicketServerConfig},
   storage::Storage,
 };
 
 #[cfg(feature = "aws-mocks")]
 pub mod aws_mocks;
-#[cfg(feature = "cors-tests")]
-pub mod cors_tests;
-#[cfg(feature = "http-tests")]
-pub mod http_tests;
-#[cfg(feature = "server-tests")]
-pub mod server_tests;
+pub mod error;
+#[cfg(feature = "http")]
+pub mod http;
 pub mod util;