From 8c320439d1ff04a147b356afa48ba2e2b58cb162 Mon Sep 17 00:00:00 2001 From: Yuri Astrakhan Date: Fri, 2 Feb 2024 18:42:18 -0500 Subject: [PATCH 1/3] v0.6.1: minor cleanups * only add `async-compression/tokio` dependency feature with `mmap-async-tokio` * use `just fmt` to clean up some formatting * slightly shorter vars in a few places * sort mods first, uses second in the lib.rs --- Cargo.toml | 11 +++++------ src/backend/http.rs | 16 ++++++++-------- src/backend/s3.rs | 17 ++++++----------- src/lib.rs | 44 +++++++++++++++++--------------------------- 4 files changed, 36 insertions(+), 52 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 4215e1f..58b9aee 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pmtiles" -version = "0.6.0" +version = "0.6.1" edition = "2021" authors = ["Luke Seelenbinder "] license = "MIT OR Apache-2.0" @@ -13,17 +13,16 @@ categories = ["science::geo"] [features] default = [] http-async = ["dep:tokio", "dep:reqwest"] -s3-async-native = ["dep:tokio", "dep:rust-s3", "rust-s3/tokio-native-tls"] -s3-async-rustls = ["dep:tokio", "dep:rust-s3", "rust-s3/tokio-rustls-tls"] -mmap-async-tokio = ["dep:tokio", "dep:fmmap", "fmmap?/tokio-async"] +s3-async-native = ["dep:tokio", "dep:rust-s3", "rust-s3?/tokio-native-tls"] +s3-async-rustls = ["dep:tokio", "dep:rust-s3", "rust-s3?/tokio-rustls-tls"] +mmap-async-tokio = ["dep:tokio", "dep:fmmap", "fmmap?/tokio-async", "async-compression?/tokio"] tilejson = ["dep:tilejson", "dep:serde", "dep:serde_json"] # TODO: support other async libraries [dependencies] # TODO: determine how we want to handle compression in async & sync environments -# TODO: tokio is always requested here, but the tokio dependency is optional below - maybe make it required? -async-compression = { version = "0.4", features = ["gzip", "zstd", "brotli", "tokio"] } +async-compression = { version = "0.4", features = ["gzip", "zstd", "brotli"] } async-recursion = "1" async-trait = "0.1" bytes = "1" diff --git a/src/backend/http.rs b/src/backend/http.rs index 917567e..4d9291c 100644 --- a/src/backend/http.rs +++ b/src/backend/http.rs @@ -1,22 +1,22 @@ use async_trait::async_trait; use bytes::Bytes; -use reqwest::{ - header::{HeaderValue, RANGE}, - Client, IntoUrl, Method, Request, StatusCode, Url, -}; +use reqwest::header::{HeaderValue, RANGE}; +use reqwest::{Client, IntoUrl, Method, Request, StatusCode, Url}; -use crate::{async_reader::AsyncBackend, error::PmtResult, PmtError}; +use crate::async_reader::AsyncBackend; +use crate::error::PmtResult; +use crate::PmtError; pub struct HttpBackend { client: Client, - pmtiles_url: Url, + url: Url, } impl HttpBackend { pub fn try_from(client: Client, url: U) -> PmtResult { Ok(HttpBackend { client, - pmtiles_url: url.into_url()?, + url: url.into_url()?, }) } } @@ -41,7 +41,7 @@ impl AsyncBackend for HttpBackend { let range = format!("bytes={offset}-{end}"); let range = HeaderValue::try_from(range)?; - let mut req = Request::new(Method::GET, self.pmtiles_url.clone()); + let mut req = Request::new(Method::GET, self.url.clone()); req.headers_mut().insert(RANGE, range); let response = self.client.execute(req).await?.error_for_status()?; diff --git a/src/backend/s3.rs b/src/backend/s3.rs index f13ba38..17d3580 100644 --- a/src/backend/s3.rs +++ b/src/backend/s3.rs @@ -2,23 +2,18 @@ use async_trait::async_trait; use bytes::Bytes; use s3::Bucket; -use crate::{ - async_reader::AsyncBackend, - error::PmtError::{ResponseBodyTooLong, UnexpectedNumberOfBytesReturned}, -}; +use crate::async_reader::AsyncBackend; +use crate::error::PmtError::{ResponseBodyTooLong, UnexpectedNumberOfBytesReturned}; pub struct S3Backend { bucket: Bucket, - pmtiles_path: String, + path: String, } impl S3Backend { #[must_use] - pub fn from(bucket: Bucket, pmtiles_path: String) -> S3Backend { - Self { - bucket, - pmtiles_path, - } + pub fn from(bucket: Bucket, path: String) -> S3Backend { + Self { bucket, path } } } @@ -38,7 +33,7 @@ impl AsyncBackend for S3Backend { let response = self .bucket .get_object_range( - self.pmtiles_path.as_str(), + self.path.as_str(), offset as _, Some((offset + length - 1) as _), ) diff --git a/src/lib.rs b/src/lib.rs index 31bebe9..49722d3 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,34 +1,10 @@ #![forbid(unsafe_code)] -pub use directory::{DirEntry, Directory}; -pub use error::{PmtError, PmtResult}; - -pub use header::{Compression, Header, TileType}; - -#[cfg(any(feature = "s3-async-rustls", feature = "s3-async-native"))] -pub use backend::S3Backend; - -#[cfg(any(feature = "s3-async-rustls", feature = "s3-async-native"))] -pub use s3; - -#[cfg(feature = "http-async")] -pub use backend::HttpBackend; - -#[cfg(feature = "http-async")] -pub use reqwest; - -#[cfg(feature = "mmap-async-tokio")] -pub use backend::MmapBackend; - -mod tile; - -mod header; - +mod backend; mod directory; - mod error; - -mod backend; +mod header; +mod tile; #[cfg(any( feature = "http-async", @@ -46,6 +22,20 @@ pub mod async_reader; ))] pub mod cache; +#[cfg(feature = "http-async")] +pub use backend::HttpBackend; +#[cfg(feature = "mmap-async-tokio")] +pub use backend::MmapBackend; +#[cfg(any(feature = "s3-async-rustls", feature = "s3-async-native"))] +pub use backend::S3Backend; +pub use directory::{DirEntry, Directory}; +pub use error::{PmtError, PmtResult}; +pub use header::{Compression, Header, TileType}; +#[cfg(feature = "http-async")] +pub use reqwest; +#[cfg(any(feature = "s3-async-rustls", feature = "s3-async-native"))] +pub use s3; + #[cfg(test)] mod tests { pub const RASTER_FILE: &str = "fixtures/stamen_toner(raster)CC-BY+ODbL_z3.pmtiles"; From 1e221171819e779522343de4e7964ab2bd48a6ba Mon Sep 17 00:00:00 2001 From: Yuri Astrakhan Date: Fri, 2 Feb 2024 22:17:17 -0500 Subject: [PATCH 2/3] Consolidate backends with their invokers --- Cargo.toml | 20 +++-- src/async_reader.rs | 101 +---------------------- src/backend/mod.rs | 17 ---- src/backend/s3.rs | 50 ----------- src/{backend/http.rs => backend_http.rs} | 27 +++++- src/{backend/mmap.rs => backend_mmap.rs} | 23 +++++- src/backend_s3.rs | 76 +++++++++++++++++ src/error.rs | 14 +--- src/header.rs | 2 + src/lib.rs | 42 +++++----- src/tile.rs | 46 ++++++++++- 11 files changed, 210 insertions(+), 208 deletions(-) delete mode 100644 src/backend/mod.rs delete mode 100644 src/backend/s3.rs rename src/{backend/http.rs => backend_http.rs} (67%) rename src/{backend/mmap.rs => backend_mmap.rs} (60%) create mode 100644 src/backend_s3.rs diff --git a/Cargo.toml b/Cargo.toml index 58b9aee..e619f4c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -12,13 +12,21 @@ categories = ["science::geo"] [features] default = [] -http-async = ["dep:tokio", "dep:reqwest"] -s3-async-native = ["dep:tokio", "dep:rust-s3", "rust-s3?/tokio-native-tls"] -s3-async-rustls = ["dep:tokio", "dep:rust-s3", "rust-s3?/tokio-rustls-tls"] -mmap-async-tokio = ["dep:tokio", "dep:fmmap", "fmmap?/tokio-async", "async-compression?/tokio"] +http-async = ["__async", "dep:reqwest"] +mmap-async-tokio = ["__async", "dep:fmmap", "fmmap?/tokio-async"] +s3-async-native = ["__async-s3"] +s3-async-rustls = ["__async-s3"] tilejson = ["dep:tilejson", "dep:serde", "dep:serde_json"] -# TODO: support other async libraries +# Forward some of the common features to reqwest dependency +reqwest-native-tls = ["reqwest?/native-tls"] +reqwest-rustls-tls = ["reqwest?/rustls-tls"] +reqwest-rustls-tls-webpki-roots = ["reqwest?/rustls-tls-webpki-roots"] +reqwest-rustls-tls-native-roots = ["reqwest?/rustls-tls-native-roots"] + +# Internal features, do not use +__async = ["dep:tokio", "async-compression/tokio"] +__async-s3 = ["__async", "dep:rust-s3", "rust-s3?/tokio-native-tls"] [dependencies] # TODO: determine how we want to handle compression in async & sync environments @@ -29,13 +37,13 @@ bytes = "1" fmmap = { version = "0.3", default-features = false, optional = true } hilbert_2d = "1" reqwest = { version = "0.11", default-features = false, optional = true } +rust-s3 = { version = "0.33.0", optional = true, default-features = false, features = ["fail-on-err"] } serde = { version = "1", optional = true } serde_json = { version = "1", optional = true } thiserror = "1" tilejson = { version = "0.4", optional = true } tokio = { version = "1", default-features = false, features = ["io-util"], optional = true } varint-rs = "2" -rust-s3 = { version = "0.33.0", optional = true, default-features = false, features = ["fail-on-err"] } [dev-dependencies] flate2 = "1" diff --git a/src/async_reader.rs b/src/async_reader.rs index 1126b46..a8ff6d3 100644 --- a/src/async_reader.rs +++ b/src/async_reader.rs @@ -2,35 +2,14 @@ // so any file larger than 4GB, or an untrusted file with bad data may crash. #![allow(clippy::cast_possible_truncation)] -#[cfg(feature = "mmap-async-tokio")] -use std::path::Path; - use async_recursion::async_recursion; use async_trait::async_trait; use bytes::Bytes; -#[cfg(feature = "http-async")] -use reqwest::{Client, IntoUrl}; -#[cfg(any( - feature = "http-async", - feature = "mmap-async-tokio", - feature = "s3-async-rustls", - feature = "s3-async-native" -))] +#[cfg(feature = "__async")] use tokio::io::AsyncReadExt; -#[cfg(feature = "http-async")] -use crate::backend::HttpBackend; -#[cfg(feature = "mmap-async-tokio")] -use crate::backend::MmapBackend; -#[cfg(any(feature = "s3-async-rustls", feature = "s3-async-native"))] -use crate::backend::S3Backend; use crate::cache::DirCacheResult; -#[cfg(any( - feature = "http-async", - feature = "mmap-async-tokio", - feature = "s3-async-native", - feature = "s3-async-rustls" -))] +#[cfg(feature = "__async")] use crate::cache::{DirectoryCache, NoCache}; use crate::directory::{DirEntry, Directory}; use crate::error::{PmtError, PmtResult}; @@ -227,80 +206,6 @@ impl AsyncPmTile } } -#[cfg(feature = "http-async")] -impl AsyncPmTilesReader { - /// Creates a new `PMTiles` reader from a URL using the Reqwest backend. - /// - /// Fails if [url] does not exist or is an invalid archive. (Note: HTTP requests are made to validate it.) - pub async fn new_with_url(client: Client, url: U) -> PmtResult { - Self::new_with_cached_url(NoCache, client, url).await - } -} - -#[cfg(feature = "http-async")] -impl AsyncPmTilesReader { - /// Creates a new `PMTiles` reader with cache from a URL using the Reqwest backend. - /// - /// Fails if [url] does not exist or is an invalid archive. (Note: HTTP requests are made to validate it.) - pub async fn new_with_cached_url( - cache: C, - client: Client, - url: U, - ) -> PmtResult { - let backend = HttpBackend::try_from(client, url)?; - - Self::try_from_cached_source(backend, cache).await - } -} - -#[cfg(feature = "mmap-async-tokio")] -impl AsyncPmTilesReader { - /// Creates a new `PMTiles` reader from a file path using the async mmap backend. - /// - /// Fails if [p] does not exist or is an invalid archive. - pub async fn new_with_path>(path: P) -> PmtResult { - Self::new_with_cached_path(NoCache, path).await - } -} - -#[cfg(feature = "mmap-async-tokio")] -impl AsyncPmTilesReader { - /// Creates a new cached `PMTiles` reader from a file path using the async mmap backend. - /// - /// Fails if [p] does not exist or is an invalid archive. - pub async fn new_with_cached_path>(cache: C, path: P) -> PmtResult { - let backend = MmapBackend::try_from(path).await?; - - Self::try_from_cached_source(backend, cache).await - } -} - -#[cfg(any(feature = "s3-async-native", feature = "s3-async-rustls"))] -impl AsyncPmTilesReader { - /// Creates a new `PMTiles` reader from a URL using the Reqwest backend. - /// - /// Fails if [url] does not exist or is an invalid archive. (Note: HTTP requests are made to validate it.) - pub async fn new_with_bucket_path(bucket: s3::Bucket, path: String) -> PmtResult { - Self::new_with_cached_bucket_path(NoCache, bucket, path).await - } -} - -#[cfg(any(feature = "s3-async-native", feature = "s3-async-rustls"))] -impl AsyncPmTilesReader { - /// Creates a new `PMTiles` reader with cache from a URL using the Reqwest backend. - /// - /// Fails if [url] does not exist or is an invalid archive. (Note: HTTP requests are made to validate it.) - pub async fn new_with_cached_bucket_path( - cache: C, - bucket: s3::Bucket, - path: String, - ) -> PmtResult { - let backend = S3Backend::from(bucket, path); - - Self::try_from_cached_source(backend, cache).await - } -} - #[async_trait] pub trait AsyncBackend { /// Reads exactly `length` bytes starting at `offset` @@ -314,8 +219,8 @@ pub trait AsyncBackend { #[cfg(feature = "mmap-async-tokio")] mod tests { use super::AsyncPmTilesReader; - use crate::backend::MmapBackend; use crate::tests::{RASTER_FILE, VECTOR_FILE}; + use crate::MmapBackend; #[tokio::test] async fn open_sanity_check() { diff --git a/src/backend/mod.rs b/src/backend/mod.rs deleted file mode 100644 index c6341f2..0000000 --- a/src/backend/mod.rs +++ /dev/null @@ -1,17 +0,0 @@ -#[cfg(any(feature = "s3-async-native", feature = "s3-async-rustls"))] -mod s3; - -#[cfg(any(feature = "s3-async-native", feature = "s3-async-rustls"))] -pub use s3::S3Backend; - -#[cfg(feature = "http-async")] -mod http; - -#[cfg(feature = "http-async")] -pub use http::HttpBackend; - -#[cfg(feature = "mmap-async-tokio")] -mod mmap; - -#[cfg(feature = "mmap-async-tokio")] -pub use mmap::MmapBackend; diff --git a/src/backend/s3.rs b/src/backend/s3.rs deleted file mode 100644 index 17d3580..0000000 --- a/src/backend/s3.rs +++ /dev/null @@ -1,50 +0,0 @@ -use async_trait::async_trait; -use bytes::Bytes; -use s3::Bucket; - -use crate::async_reader::AsyncBackend; -use crate::error::PmtError::{ResponseBodyTooLong, UnexpectedNumberOfBytesReturned}; - -pub struct S3Backend { - bucket: Bucket, - path: String, -} - -impl S3Backend { - #[must_use] - pub fn from(bucket: Bucket, path: String) -> S3Backend { - Self { bucket, path } - } -} - -#[async_trait] -impl AsyncBackend for S3Backend { - async fn read_exact(&self, offset: usize, length: usize) -> crate::error::PmtResult { - let data = self.read(offset, length).await?; - - if data.len() == length { - Ok(data) - } else { - Err(UnexpectedNumberOfBytesReturned(length, data.len())) - } - } - - async fn read(&self, offset: usize, length: usize) -> crate::error::PmtResult { - let response = self - .bucket - .get_object_range( - self.path.as_str(), - offset as _, - Some((offset + length - 1) as _), - ) - .await?; - - let response_bytes = response.bytes(); - - if response_bytes.len() > length { - Err(ResponseBodyTooLong(response_bytes.len(), length)) - } else { - Ok(response_bytes.clone()) - } - } -} diff --git a/src/backend/http.rs b/src/backend_http.rs similarity index 67% rename from src/backend/http.rs rename to src/backend_http.rs index 4d9291c..56866ab 100644 --- a/src/backend/http.rs +++ b/src/backend_http.rs @@ -3,10 +3,35 @@ use bytes::Bytes; use reqwest::header::{HeaderValue, RANGE}; use reqwest::{Client, IntoUrl, Method, Request, StatusCode, Url}; -use crate::async_reader::AsyncBackend; +use crate::async_reader::{AsyncBackend, AsyncPmTilesReader}; +use crate::cache::{DirectoryCache, NoCache}; use crate::error::PmtResult; use crate::PmtError; +impl AsyncPmTilesReader { + /// Creates a new `PMTiles` reader from a URL using the Reqwest backend. + /// + /// Fails if [url] does not exist or is an invalid archive. (Note: HTTP requests are made to validate it.) + pub async fn new_with_url(client: Client, url: U) -> PmtResult { + Self::new_with_cached_url(NoCache, client, url).await + } +} + +impl AsyncPmTilesReader { + /// Creates a new `PMTiles` reader with cache from a URL using the Reqwest backend. + /// + /// Fails if [url] does not exist or is an invalid archive. (Note: HTTP requests are made to validate it.) + pub async fn new_with_cached_url( + cache: C, + client: Client, + url: U, + ) -> PmtResult { + let backend = HttpBackend::try_from(client, url)?; + + Self::try_from_cached_source(backend, cache).await + } +} + pub struct HttpBackend { client: Client, url: Url, diff --git a/src/backend/mmap.rs b/src/backend_mmap.rs similarity index 60% rename from src/backend/mmap.rs rename to src/backend_mmap.rs index deb6230..deef24a 100644 --- a/src/backend/mmap.rs +++ b/src/backend_mmap.rs @@ -5,9 +5,30 @@ use async_trait::async_trait; use bytes::{Buf, Bytes}; use fmmap::tokio::{AsyncMmapFile, AsyncMmapFileExt as _, AsyncOptions}; -use crate::async_reader::AsyncBackend; +use crate::async_reader::{AsyncBackend, AsyncPmTilesReader}; +use crate::cache::{DirectoryCache, NoCache}; use crate::error::{PmtError, PmtResult}; +impl AsyncPmTilesReader { + /// Creates a new `PMTiles` reader from a file path using the async mmap backend. + /// + /// Fails if [p] does not exist or is an invalid archive. + pub async fn new_with_path>(path: P) -> PmtResult { + Self::new_with_cached_path(NoCache, path).await + } +} + +impl AsyncPmTilesReader { + /// Creates a new cached `PMTiles` reader from a file path using the async mmap backend. + /// + /// Fails if [p] does not exist or is an invalid archive. + pub async fn new_with_cached_path>(cache: C, path: P) -> PmtResult { + let backend = MmapBackend::try_from(path).await?; + + Self::try_from_cached_source(backend, cache).await + } +} + pub struct MmapBackend { file: AsyncMmapFile, } diff --git a/src/backend_s3.rs b/src/backend_s3.rs new file mode 100644 index 0000000..32fba59 --- /dev/null +++ b/src/backend_s3.rs @@ -0,0 +1,76 @@ +use async_trait::async_trait; +use bytes::Bytes; +use s3::Bucket; + +use crate::async_reader::{AsyncBackend, AsyncPmTilesReader}; +use crate::cache::{DirectoryCache, NoCache}; +use crate::error::PmtError::{ResponseBodyTooLong, UnexpectedNumberOfBytesReturned}; +use crate::PmtResult; + +impl AsyncPmTilesReader { + /// Creates a new `PMTiles` reader from a URL using the Reqwest backend. + /// + /// Fails if [url] does not exist or is an invalid archive. (Note: HTTP requests are made to validate it.) + pub async fn new_with_bucket_path(bucket: Bucket, path: String) -> PmtResult { + Self::new_with_cached_bucket_path(NoCache, bucket, path).await + } +} + +impl AsyncPmTilesReader { + /// Creates a new `PMTiles` reader with cache from a URL using the Reqwest backend. + /// + /// Fails if [url] does not exist or is an invalid archive. (Note: HTTP requests are made to validate it.) + pub async fn new_with_cached_bucket_path( + cache: C, + bucket: Bucket, + path: String, + ) -> PmtResult { + let backend = S3Backend::from(bucket, path); + + Self::try_from_cached_source(backend, cache).await + } +} + +pub struct S3Backend { + bucket: Bucket, + path: String, +} + +impl S3Backend { + #[must_use] + pub fn from(bucket: Bucket, path: String) -> S3Backend { + Self { bucket, path } + } +} + +#[async_trait] +impl AsyncBackend for S3Backend { + async fn read_exact(&self, offset: usize, length: usize) -> PmtResult { + let data = self.read(offset, length).await?; + + if data.len() == length { + Ok(data) + } else { + Err(UnexpectedNumberOfBytesReturned(length, data.len())) + } + } + + async fn read(&self, offset: usize, length: usize) -> PmtResult { + let response = self + .bucket + .get_object_range( + self.path.as_str(), + offset as _, + Some((offset + length - 1) as _), + ) + .await?; + + let response_bytes = response.bytes(); + + if response_bytes.len() > length { + Err(ResponseBodyTooLong(response_bytes.len(), length)) + } else { + Ok(response_bytes.clone()) + } + } +} diff --git a/src/error.rs b/src/error.rs index 3c022b6..0cfab96 100644 --- a/src/error.rs +++ b/src/error.rs @@ -29,21 +29,13 @@ pub enum PmtError { #[cfg(feature = "mmap-async-tokio")] #[error("Unable to open mmap file")] UnableToOpenMmapFile, - #[cfg(any( - feature = "http-async", - feature = "s3-async-native", - feature = "s3-async-rustls" - ))] + #[cfg(any(feature = "http-async", feature = "__async-s3"))] #[error("Unexpected number of bytes returned [expected: {0}, received: {1}].")] UnexpectedNumberOfBytesReturned(usize, usize), #[cfg(feature = "http-async")] #[error("Range requests unsupported")] RangeRequestsUnsupported, - #[cfg(any( - feature = "http-async", - feature = "s3-async-native", - feature = "s3-async-rustls" - ))] + #[cfg(any(feature = "http-async", feature = "__async-s3"))] #[error("HTTP response body is too long, Response {0}B > requested {1}B")] ResponseBodyTooLong(usize, usize), #[cfg(feature = "http-async")] @@ -52,7 +44,7 @@ pub enum PmtError { #[cfg(feature = "http-async")] #[error(transparent)] InvalidHeaderValue(#[from] reqwest::header::InvalidHeaderValue), - #[cfg(any(feature = "s3-async-rustls", feature = "s3-async-native"))] + #[cfg(feature = "__async-s3")] #[error(transparent)] S3(#[from] s3::error::S3Error), } diff --git a/src/header.rs b/src/header.rs index 89b6c94..f51aed3 100644 --- a/src/header.rs +++ b/src/header.rs @@ -5,7 +5,9 @@ use bytes::{Buf, Bytes}; use crate::error::{PmtError, PmtResult}; +#[cfg(feature = "__async")] pub(crate) const MAX_INITIAL_BYTES: usize = 16_384; +#[cfg(any(test, feature = "__async"))] pub(crate) const HEADER_SIZE: usize = 127; #[allow(dead_code)] diff --git a/src/lib.rs b/src/lib.rs index 49722d3..a8fe6f3 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,40 +1,38 @@ #![forbid(unsafe_code)] -mod backend; +#[cfg(feature = "__async")] +pub mod async_reader; +#[cfg(feature = "http-async")] +mod backend_http; +#[cfg(feature = "mmap-async-tokio")] +mod backend_mmap; +#[cfg(feature = "__async-s3")] +mod backend_s3; +#[cfg(feature = "__async")] +pub mod cache; mod directory; mod error; mod header; +#[cfg(feature = "__async")] mod tile; -#[cfg(any( - feature = "http-async", - feature = "mmap-async-tokio", - feature = "s3-async-rustls", - feature = "s3-async-native" -))] -pub mod async_reader; - -#[cfg(any( - feature = "http-async", - feature = "mmap-async-tokio", - feature = "s3-async-native", - feature = "s3-async-rustls" -))] -pub mod cache; - #[cfg(feature = "http-async")] -pub use backend::HttpBackend; +pub use backend_http::HttpBackend; #[cfg(feature = "mmap-async-tokio")] -pub use backend::MmapBackend; -#[cfg(any(feature = "s3-async-rustls", feature = "s3-async-native"))] -pub use backend::S3Backend; +pub use backend_mmap::MmapBackend; +#[cfg(feature = "__async-s3")] +pub use backend_s3::S3Backend; pub use directory::{DirEntry, Directory}; pub use error::{PmtError, PmtResult}; pub use header::{Compression, Header, TileType}; +// +// Re-export crates exposed in our API to simplify dependency management #[cfg(feature = "http-async")] pub use reqwest; -#[cfg(any(feature = "s3-async-rustls", feature = "s3-async-native"))] +#[cfg(feature = "__async-s3")] pub use s3; +#[cfg(feature = "tilejson")] +pub use tilejson; #[cfg(test)] mod tests { diff --git a/src/tile.rs b/src/tile.rs index 1a3cf0a..f2fd86f 100644 --- a/src/tile.rs +++ b/src/tile.rs @@ -1,10 +1,42 @@ +#![allow(clippy::unreadable_literal)] + +const PYRAMID_SIZE_BY_ZOOM: [u64; 21] = [ + /* 0 */ 0, + /* 1 */ 1, + /* 2 */ 5, + /* 3 */ 21, + /* 4 */ 85, + /* 5 */ 341, + /* 6 */ 1365, + /* 7 */ 5461, + /* 8 */ 21845, + /* 9 */ 87381, + /* 10 */ 349525, + /* 11 */ 1398101, + /* 12 */ 5592405, + /* 13 */ 22369621, + /* 14 */ 89478485, + /* 15 */ 357913941, + /* 16 */ 1431655765, + /* 17 */ 5726623061, + /* 18 */ 22906492245, + /* 19 */ 91625968981, + /* 20 */ 366503875925, +]; + pub(crate) fn tile_id(z: u8, x: u64, y: u64) -> u64 { + // The 0/0/0 case is not needed for the base id computation, but it will fail hilbert_2d::u64::xy2h_discrete if z == 0 { return 0; } - // TODO: minor optimization with bit shifting - let base_id: u64 = 1 + (1..z).map(|i| 4u64.pow(u32::from(i))).sum::(); + let z_ind = usize::from(z); + let base_id = if z_ind < PYRAMID_SIZE_BY_ZOOM.len() { + PYRAMID_SIZE_BY_ZOOM[z_ind] + } else { + let last_ind = PYRAMID_SIZE_BY_ZOOM.len() - 1; + PYRAMID_SIZE_BY_ZOOM[last_ind] + (last_ind..z_ind).map(|i| 1_u64 << (i << 1)).sum::() + }; let tile_id = hilbert_2d::u64::xy2h_discrete(x, y, z.into(), hilbert_2d::Variant::Hilbert); @@ -21,5 +53,15 @@ mod test { assert_eq!(tile_id(1, 1, 0), 4); assert_eq!(tile_id(2, 1, 3), 11); assert_eq!(tile_id(3, 3, 0), 26); + assert_eq!(tile_id(20, 0, 0), 366503875925); + assert_eq!(tile_id(21, 0, 0), 1466015503701); + assert_eq!(tile_id(22, 0, 0), 5864062014805); + assert_eq!(tile_id(22, 0, 0), 5864062014805); + assert_eq!(tile_id(23, 0, 0), 23456248059221); + assert_eq!(tile_id(24, 0, 0), 93824992236885); + assert_eq!(tile_id(25, 0, 0), 375299968947541); + assert_eq!(tile_id(26, 0, 0), 1501199875790165); + assert_eq!(tile_id(27, 0, 0), 6004799503160661); + assert_eq!(tile_id(28, 0, 0), 24019198012642645); } } From 58ead54cd964d7f3fd0f308fd1c13705cbb9b6f9 Mon Sep 17 00:00:00 2001 From: Yuri Astrakhan Date: Mon, 5 Feb 2024 13:52:33 -0500 Subject: [PATCH 3/3] Set to v0.7.0 --- Cargo.toml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index e619f4c..f69e500 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pmtiles" -version = "0.6.1" +version = "0.7.0" edition = "2021" authors = ["Luke Seelenbinder "] license = "MIT OR Apache-2.0" @@ -19,6 +19,7 @@ s3-async-rustls = ["__async-s3"] tilejson = ["dep:tilejson", "dep:serde", "dep:serde_json"] # Forward some of the common features to reqwest dependency +reqwest-default = ["reqwest?/default"] reqwest-native-tls = ["reqwest?/native-tls"] reqwest-rustls-tls = ["reqwest?/rustls-tls"] reqwest-rustls-tls-webpki-roots = ["reqwest?/rustls-tls-webpki-roots"]