Skip to content

Commit

Permalink
Handle unknown and rm unused compressions (#39)
Browse files Browse the repository at this point in the history
* we only need gzip for now, so remove brotli and zstd
* add a new error case when dir compression is unsupported
* bump reqwest version dep
* add clippy warnings on unwrap and todo
  • Loading branch information
nyurik authored May 2, 2024
1 parent 1f1b92a commit 3012fc4
Show file tree
Hide file tree
Showing 5 changed files with 18 additions and 6 deletions.
12 changes: 7 additions & 5 deletions Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -17,14 +17,13 @@ mmap-async-tokio = ["__async", "dep:fmmap", "fmmap?/tokio-async"]
s3-async-native = ["__async-s3", "__async-s3-nativetls"]
s3-async-rustls = ["__async-s3", "__async-s3-rustls"]
tilejson = ["dep:tilejson", "dep:serde", "dep:serde_json"]
zstd = [ "async-compression/zstd" ]

# Forward some of the common features to reqwest dependency
reqwest-default = ["reqwest?/default"]
reqwest-native-tls = ["reqwest?/native-tls"]
reqwest-rustls-tls = ["reqwest?/rustls-tls"]
reqwest-rustls-tls-webpki-roots = ["reqwest?/rustls-tls-webpki-roots"]
reqwest-rustls-tls-native-roots = ["reqwest?/rustls-tls-native-roots"]
reqwest-rustls-tls-webpki-roots = ["reqwest?/rustls-tls-webpki-roots"]

# Internal features, do not use
__async = ["dep:tokio", "async-compression/tokio"]
Expand All @@ -34,11 +33,11 @@ __async-s3-rustls = ["rust-s3?/tokio-rustls-tls"]

[dependencies]
# TODO: determine how we want to handle compression in async & sync environments
async-compression = { version = "0.4", features = ["gzip", "brotli"] }
async-compression = { version = "0.4", features = ["gzip"] }
bytes = "1"
fmmap = { version = "0.3", default-features = false, optional = true }
hilbert_2d = "1"
reqwest = { version = "0.12.3", default-features = false, optional = true }
reqwest = { version = "0.12.4", default-features = false, optional = true }
rust-s3 = { version = "0.33.0", optional = true, default-features = false, features = ["fail-on-err"] }
serde = { version = "1", optional = true }
serde_json = { version = "1", optional = true }
Expand All @@ -50,7 +49,7 @@ varint-rs = "2"
[dev-dependencies]
flate2 = "1"
fmmap = { version = "0.3", features = ["tokio-async"] }
reqwest = { version = "0.12.3", features = ["rustls-tls-webpki-roots"] }
reqwest = { version = "0.12.4", features = ["rustls-tls-webpki-roots"] }
tokio = { version = "1", features = ["test-util", "macros", "rt"] }

[package.metadata.docs.rs]
Expand All @@ -64,4 +63,7 @@ unused_qualifications = "warn"
pedantic = { level = "warn", priority = -1 }
missing_errors_doc = "allow"
module_name_repetitions = "allow"
panic_in_result_fn = "warn"
similar_names = "allow"
todo = "warn"
unwrap_used = "warn"
1 change: 1 addition & 0 deletions clippy.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
allow-unwrap-in-tests = true
3 changes: 2 additions & 1 deletion src/async_reader.rs
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ use crate::directory::{DirEntry, Directory};
use crate::error::{PmtError, PmtResult};
use crate::header::{HEADER_SIZE, MAX_INITIAL_BYTES};
use crate::tile::tile_id;
use crate::PmtError::UnsupportedCompression;
use crate::{Compression, Header};

pub struct AsyncPmTilesReader<B, C = NoCache> {
Expand Down Expand Up @@ -205,7 +206,7 @@ impl<B: AsyncBackend + Sync + Send, C: DirectoryCache + Sync + Send> AsyncPmTile
.read_to_end(&mut decompressed_bytes)
.await?;
}
_ => todo!("Support other forms of internal compression."),
v => Err(UnsupportedCompression(v))?,
}

Ok(Bytes::from(decompressed_bytes))
Expand Down
4 changes: 4 additions & 0 deletions src/cache.rs
Original file line number Diff line number Diff line change
Expand Up @@ -56,13 +56,17 @@ pub struct HashMapCache {

impl DirectoryCache for HashMapCache {
async fn get_dir_entry(&self, offset: usize, tile_id: u64) -> DirCacheResult {
// Panic if the lock is poisoned is not something the user can handle
#[allow(clippy::unwrap_used)]
if let Some(dir) = self.cache.read().unwrap().get(&offset) {
return dir.find_tile_id(tile_id).into();
}
DirCacheResult::NotCached
}

async fn insert_dir(&self, offset: usize, directory: Directory) {
// Panic if the lock is poisoned is not something the user can handle
#[allow(clippy::unwrap_used)]
self.cache.write().unwrap().insert(offset, directory);
}
}
4 changes: 4 additions & 0 deletions src/error.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,8 @@ use std::string::FromUtf8Error;

use thiserror::Error;

use crate::Compression;

/// A specialized [`Result`] type for `PMTiles` operations.
pub type PmtResult<T> = Result<T, PmtError>;

Expand All @@ -14,6 +16,8 @@ pub enum PmtError {
UnsupportedPmTilesVersion,
#[error("Invalid compression")]
InvalidCompression,
#[error("Unsupported compression {0:?}")]
UnsupportedCompression(Compression),
#[error("Invalid PMTiles entry")]
InvalidEntry,
#[error("Invalid header")]
Expand Down

0 comments on commit 3012fc4

Please sign in to comment.