diff --git a/Cargo.toml b/Cargo.toml index baf432f..e0c04d8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -16,14 +16,43 @@ categories = ["multimedia::images", "multimedia::encoding"] exclude = ["tests/images/*", "tests/fuzz_images/*"] +[features] +async_decoder = ["dep:futures", "dep:async-trait"] +multithread = ["async_decoder"] +# only for async example reading a COG +ehttp = ["async_decoder", "dep:ehttp"] + [dependencies] weezl = "0.1.0" jpeg = { package = "jpeg-decoder", version = "0.3.0", default-features = false } flate2 = "1.0.20" +futures = {version = "0.3.30", optional = true } +async-trait = {version ="^0.1", optional = true } +ehttp = { version = "0.5.0", features=["native-async"], optional = true } [dev-dependencies] criterion = "0.3.1" +# use rt since most is io-bound +tokio = {version = "1.29.1", features = ["macros", "fs", "rt-multi-thread"]} [[bench]] name = "lzw" harness = false + +[package.metadata.example.async_http] +name = "Async http" +description = "Example showing use of async features using async http requests" + +[[example]] +name = "async_http" +path="examples/async_http.rs" +required-features=["ehttp", "async_decoder"] + +[package.metadata.example.multithread] +name="multithread_http" +description = "example showing multithreaded reading of COG" + +[[example]] +name = "multithread_http" +path="examples/multithread_http.rs" +required-features=["ehttp", "multithread"] \ No newline at end of file diff --git a/examples/async_http.rs b/examples/async_http.rs new file mode 100644 index 0000000..a396230 --- /dev/null +++ b/examples/async_http.rs @@ -0,0 +1,148 @@ +// Special thanks to Alice for the help: https://users.rust-lang.org/t/63019/6 +use std::io::{Error, Result, SeekFrom}; +use std::pin::Pin; + +use futures::{ + future::BoxFuture, + io::{AsyncRead, AsyncSeek}, + Future, +}; +use tiff::decoder::ifd::Value; +use tiff::decoder::DecodingResult; +use tiff::decoder::Decoder; + +// extern crate ehttp; + + +pub struct RangedStreamer { + state: State, + range_get: F, + min_request_size: usize, // requests have at least this size +} + +enum State { + HasChunk(SeekOutput), + Seeking(BoxFuture<'static, std::io::Result>), +} + +pub struct SeekOutput { + pub start: u64, + pub data: Vec, +} + +pub type F = Box< + dyn Fn(u64, u64) -> BoxFuture<'static, std::io::Result> + Send + Sync, +>; + +impl RangedStreamer { + pub fn new(length: usize, min_request_size: usize, range_get: F) -> Self { + let length = length as u64; + Self { + pos: 0, + length, + state: State::HasChunk(SeekOutput { + start: 0, + data: vec![], + }), + range_get, + min_request_size, + } + } +} + +// whether `test_interval` is inside `a` (start, length). +fn range_includes(a: (usize, usize), test_interval: (usize, usize)) -> bool { + if test_interval.0 < a.0 { + return false; + } + let test_end = test_interval.0 + test_interval.1; + let a_end = a.0 + a.1; + if test_end > a_end { + return false; + } + true +} + +impl AsyncRead for RangedStreamer { + fn poll_read( + mut self: std::pin::Pin<&mut Self>, + cx: &mut std::task::Context<'_>, + buf: &mut [u8], + ) -> std::task::Poll> { + let requested_range = (self.pos as usize, buf.len()); + let min_request_size = self.min_request_size; + match &mut self.state { + State::HasChunk(output) => { + let existing_range = (output.start as usize, output.data.len()); + if range_includes(existing_range, requested_range) { + let offset = requested_range.0 - existing_range.0; + buf.copy_from_slice(&output.data[offset..offset + buf.len()]); + self.pos += buf.len() as u64; + std::task::Poll::Ready(Ok(buf.len())) + } else { + let start = requested_range.0 as u64; + let length = std::cmp::max(min_request_size, requested_range.1); + let future = (self.range_get)(start, length.try_into().unwrap()); + self.state = State::Seeking(Box::pin(future)); + self.poll_read(cx, buf) + } + } + State::Seeking(ref mut future) => match Pin::new(future).poll(cx) { + std::task::Poll::Ready(v) => { + match v { + Ok(output) => self.state = State::HasChunk(output), + Err(e) => return std::task::Poll::Ready(Err(e)), + }; + self.poll_read(cx, buf) + } + std::task::Poll::Pending => std::task::Poll::Pending, + }, + } + } +} + +impl AsyncSeek for RangedStreamer { + fn poll_seek( + mut self: std::pin::Pin<&mut Self>, + _: &mut std::task::Context<'_>, + pos: std::io::SeekFrom, + ) -> std::task::Poll> { + match pos { + SeekFrom::Start(pos) => self.pos = pos, + SeekFrom::End(pos) => self.pos = (self.length as i64 + pos) as u64, + SeekFrom::Current(pos) => self.pos = (self.pos as i64 + pos) as u64, + }; + std::task::Poll::Ready(Ok(self.pos)) + } +} + + + +#[tokio::main] +async fn main() { + let url = "https://isdasoil.s3.amazonaws.com/covariates/dem_30m/dem_30m.tif"; + let Ok(url_head) = ehttp::fetch_async(ehttp::Request::head(url)).await else {println!("EPIC FAIL!"); return;}; + let length = usize::from_str_radix(url_head.headers.get("content-length").unwrap(), 10).expect("Could not parse content length"); + println!("head: {:?}", url_head); + let range_get = Box::new(move |start: u64, length: u64| { + // let bucket = bucket.clone(); + let url = url; + Box::pin(async move { + println!("requested: {} kb", length / 1024); + let mut request = ehttp::Request::get(url); + request.headers.insert("Range".to_string(), format!("bytes={:?}-{:?}",start,start+length)); + let resp = ehttp::fetch_async(request).await.map_err(|e| std::io::Error::other(e))?; + if !resp.ok { + Err(std::io::Error::other(format!("Received invalid response: {:?}", resp.status))) + } else { + println!("received: {} kb", resp.bytes.len() / 1024); + Ok(SeekOutput {start, data: resp.bytes}) + } + }) as BoxFuture<'static, std::io::Result> + }); + let reader = RangedStreamer::new(length, 30*1024, range_get); + match Decoder::new_overview_async(reader, 5).await { + Ok(mut d) => println!("{:?}", d.read_chunk_async(42).await.unwrap()), + Err(e) => println!("err: {:?}", e), + } +} diff --git a/examples/multithread_http.rs b/examples/multithread_http.rs new file mode 100644 index 0000000..e295b01 --- /dev/null +++ b/examples/multithread_http.rs @@ -0,0 +1,194 @@ +// Special thanks to Alice for the help: https://users.rust-lang.org/t/63019/6 +use std::io::{Result, SeekFrom}; +use std::pin::Pin; +use std::sync::Arc; +use futures::{ + future::BoxFuture, + io::{AsyncRead, AsyncSeek}, + Future, +}; +use tiff::decoder::Decoder; + +// extern crate ehttp; + +// Arc for sharing, see https://users.rust-lang.org/t/how-to-clone-a-boxed-closure/31035/9 +// or https://stackoverflow.com/a/27883612/14681457 +pub type F = Arc< + dyn Fn(u64, u64) -> BoxFuture<'static, std::io::Result> + Send + Sync, +>; +pub struct RangedStreamer { + pos: u64, + length: u64, // total size + state: State, + range_get: F, + min_request_size: usize, // requests have at least this size +} + +/// This is a fake clone, that doesn't clone the currently pending task, but everything else +impl Clone for RangedStreamer { + fn clone(&self) -> Self { + RangedStreamer { + pos: self.pos, + length: self.length, + state: State::HasChunk(SeekOutput { + start: 0, + data: vec![], + }), + range_get: self.range_get.clone(), + min_request_size: self.min_request_size, + } + } +} + +enum State { + HasChunk(SeekOutput), + Seeking(BoxFuture<'static, std::io::Result>), +} + +#[derive(Debug, Clone)] +pub struct SeekOutput { + pub start: u64, + pub data: Vec, +} + + + +impl RangedStreamer { + pub fn new(length: usize, min_request_size: usize, range_get: F) -> Self { + let length = length as u64; + Self { + pos: 0, + length, + state: State::HasChunk(SeekOutput { + start: 0, + data: vec![], + }), + range_get, + min_request_size, + } + } +} + +// whether `test_interval` is inside `a` (start, length). +fn range_includes(a: (usize, usize), test_interval: (usize, usize)) -> bool { + if test_interval.0 < a.0 { + return false; + } + let test_end = test_interval.0 + test_interval.1; + let a_end = a.0 + a.1; + if test_end > a_end { + return false; + } + true +} + +impl AsyncRead for RangedStreamer { + fn poll_read( + mut self: std::pin::Pin<&mut Self>, + cx: &mut std::task::Context<'_>, + buf: &mut [u8], + ) -> std::task::Poll> { + let requested_range = (self.pos as usize, buf.len()); + let min_request_size = self.min_request_size; + match &mut self.state { + State::HasChunk(output) => { + let existing_range = (output.start as usize, output.data.len()); + if range_includes(existing_range, requested_range) { + let offset = requested_range.0 - existing_range.0; + buf.copy_from_slice(&output.data[offset..offset + buf.len()]); + self.pos += buf.len() as u64; + std::task::Poll::Ready(Ok(buf.len())) + } else { + let start = requested_range.0 as u64; + let length = std::cmp::max(min_request_size, requested_range.1); + let future = (self.range_get)(start, length.try_into().unwrap()); + self.state = State::Seeking(Box::pin(future)); + self.poll_read(cx, buf) + } + } + State::Seeking(ref mut future) => match Pin::new(future).poll(cx) { + std::task::Poll::Ready(v) => { + match v { + Ok(output) => self.state = State::HasChunk(output), + Err(e) => return std::task::Poll::Ready(Err(e)), + }; + self.poll_read(cx, buf) + } + std::task::Poll::Pending => std::task::Poll::Pending, + }, + } + } +} + +impl AsyncSeek for RangedStreamer { + fn poll_seek( + mut self: std::pin::Pin<&mut Self>, + _: &mut std::task::Context<'_>, + pos: std::io::SeekFrom, + ) -> std::task::Poll> { + match pos { + SeekFrom::Start(pos) => self.pos = pos, + SeekFrom::End(pos) => self.pos = (self.length as i64 + pos) as u64, + SeekFrom::Current(pos) => self.pos = (self.pos as i64 + pos) as u64, + }; + std::task::Poll::Ready(Ok(self.pos)) + } +} + + + +#[tokio::main] +async fn main() { + let url = "https://isdasoil.s3.amazonaws.com/covariates/dem_30m/dem_30m.tif"; + let Ok(url_head) = ehttp::fetch_async(ehttp::Request::head(url)).await else {println!("EPIC FAIL!"); return;}; + let length = usize::from_str_radix(url_head.headers.get("content-length").unwrap(), 10).expect("Could not parse content length"); + println!("head: {:?}", url_head); + let range_get = Arc::new(move |start: u64, length: u64| { + // let bucket = bucket.clone(); + let url = url; + Box::pin(async move { + println!("requested: {} kb", length / 1024); + let mut request = ehttp::Request::get(url); + request.headers.insert("Range".to_string(), format!("bytes={:?}-{:?}",start,start+length)); + let resp = ehttp::fetch_async(request).await.map_err(|e| std::io::Error::other(e))?; + if !resp.ok { + Err(std::io::Error::other(format!("Received invalid response: {:?}", resp.status))) + } else { + println!("received: {} kb", resp.bytes.len() / 1024); + Ok(SeekOutput {start, data: resp.bytes}) + } + }) as BoxFuture<'static, std::io::Result> + }); + let reader = RangedStreamer::new(length, 30*1024, range_get); + + // this decoder will read all necessary tags + let decoder = Decoder::new_overview_async(reader, 0).await.expect("oh noes!"); + println!("initialized decoder"); + let cloneable_decoder = tiff::decoder::ChunkDecoder::from_decoder(decoder); + + let mut handles = Vec::new(); + for chunk in 42..69 { + let mut cloned_decoder = cloneable_decoder.clone(); + + let handle = tokio::spawn(async move { + + let result = cloned_decoder.read_chunk_async(chunk).await; + match result { + Ok(data) => { + println!("Successfully read chunk {}", chunk); + Ok(data) // Return the data for collection + } + Err(e) => { + eprintln!("Error reading chunk {}: {:?}", chunk, e); + Err(e) // Return the error for handling + } + } + }); + handles.push(handle); + } + + let results = futures::future::join_all(handles).await; + for r in results { + println!("result: {:?}", r.expect("idk").expect("idk²").len()) + } +} diff --git a/src/decoder/async_decoder/ifd.rs b/src/decoder/async_decoder/ifd.rs new file mode 100644 index 0000000..654fac2 --- /dev/null +++ b/src/decoder/async_decoder/ifd.rs @@ -0,0 +1,36 @@ +use futures::{AsyncRead, AsyncReadExt, AsyncSeek}; + +use crate::decoder::{ + Limits, stream::SmartReader, + ifd::Value +}; +use crate::{TiffResult, TiffError}; +pub use crate::decoder::ifd::Entry; + +use super::stream::AsyncEndianReader; + +impl Entry { + pub async fn async_val( + &self, + limits: &Limits, + bigtiff: bool, + reader: &mut SmartReader, + ) -> TiffResult { + let bo = reader.byte_order(); + if let Some(res) = self.val_if_in_offset(bigtiff, bo)? { + return Ok(res); + } + + // check if we exceed the limits and read required bytes into a buffer if everything is ok + // This allows us to also create a cursor in async code + let v_bytes = usize::try_from(self.value_bytes()?)?; + if v_bytes > limits.decoding_buffer_size { + return Err(TiffError::LimitsExceeded); + } + let mut buf = vec![0; v_bytes]; + reader.goto_offset_async(self.offset(bigtiff, bo)?).await?; + reader.read_exact(&mut buf).await?; + let mut r = SmartReader::wrap(std::io::Cursor::new(buf), bo); + self.val_from_cursor(&mut r) + } +} \ No newline at end of file diff --git a/src/decoder/async_decoder/image.rs b/src/decoder/async_decoder/image.rs new file mode 100644 index 0000000..dde4670 --- /dev/null +++ b/src/decoder/async_decoder/image.rs @@ -0,0 +1,262 @@ +use super::{tag_reader::AsyncTagReader}; +use crate::decoder::{ + ifd::{Value, Directory}, + image::{StripDecodeState, TileAttributes}, + stream::SmartReader, + Limits, ChunkType, Image, +}; +use crate::tags::{ + CompressionMethod, PhotometricInterpretation, PlanarConfiguration, Predictor, SampleFormat, Tag, +}; +use crate::{TiffError, TiffFormatError, TiffResult, TiffUnsupportedError}; + +use futures::{AsyncRead, AsyncSeek}; + +use std::sync::Arc; + +impl Image { + /// Creates this image from a reader. Will not read in chunk tags + /// Rather, this + pub async fn from_async_reader( + reader: &mut SmartReader, + ifd: Directory, + limits: &Limits, + bigtiff: bool, + ) -> TiffResult { + let mut tag_reader = AsyncTagReader { + reader, + limits, + ifd: &ifd, + bigtiff, + }; + + let width = tag_reader.require_tag(Tag::ImageWidth).await?.into_u32()?; + let height = tag_reader.require_tag(Tag::ImageLength).await?.into_u32()?; + if width == 0 || height == 0 { + return Err(TiffError::FormatError(TiffFormatError::InvalidDimensions( + width, height, + ))); + } + + let photometric_interpretation = tag_reader + .find_tag(Tag::PhotometricInterpretation) + .await? + .map(Value::into_u16) + .transpose()? + .and_then(PhotometricInterpretation::from_u16) + .ok_or(TiffUnsupportedError::UnknownInterpretation)?; + + // Try to parse both the compression method and the number, format, and bits of the included samples. + // If they are not explicitly specified, those tags are reset to their default values and not carried from previous images. + let compression_method = match tag_reader.find_tag(Tag::Compression).await? { + Some(val) => CompressionMethod::from_u16_exhaustive(val.into_u16()?), + None => CompressionMethod::None, + }; + + let jpeg_tables = if compression_method == CompressionMethod::ModernJPEG + && ifd.contains_key(&Tag::JPEGTables) + { + let vec = tag_reader + .find_tag(Tag::JPEGTables) + .await? + .unwrap() + .into_u8_vec()?; + if vec.len() < 2 { + return Err(TiffError::FormatError( + TiffFormatError::InvalidTagValueType(Tag::JPEGTables), + )); + } + + Some(Arc::new(vec)) + } else { + None + }; + + let samples: u16 = tag_reader + .find_tag(Tag::SamplesPerPixel) + .await? + .map(Value::into_u16) + .transpose()? + .unwrap_or(1); + if samples == 0 { + return Err(TiffFormatError::SamplesPerPixelIsZero.into()); + } + + let sample_format = match tag_reader.find_tag_uint_vec(Tag::SampleFormat).await? { + Some(vals) => { + let sample_format: Vec<_> = vals + .into_iter() + .map(SampleFormat::from_u16_exhaustive) + .collect(); + + // TODO: for now, only homogenous formats across samples are supported. + if !sample_format.windows(2).all(|s| s[0] == s[1]) { + return Err(TiffUnsupportedError::UnsupportedSampleFormat(sample_format).into()); + } + + sample_format[0] + } + None => SampleFormat::Uint, + }; + + let bits_per_sample: Vec = tag_reader + .find_tag_uint_vec(Tag::BitsPerSample) + .await? + .unwrap_or_else(|| vec![1]); + + // Technically bits_per_sample.len() should be *equal* to samples, but libtiff also allows + // it to be a single value that applies to all samples. + if bits_per_sample.len() != samples as usize && bits_per_sample.len() != 1 { + return Err(TiffError::FormatError( + TiffFormatError::InconsistentSizesEncountered, + )); + } + + // This library (and libtiff) do not support mixed sample formats and zero bits per sample + // doesn't make sense. + if bits_per_sample.iter().any(|&b| b != bits_per_sample[0]) || bits_per_sample[0] == 0 { + return Err(TiffUnsupportedError::InconsistentBitsPerSample(bits_per_sample).into()); + } + + let predictor = tag_reader + .find_tag(Tag::Predictor) + .await? + .map(Value::into_u16) + .transpose()? + .map(|p| { + Predictor::from_u16(p) + .ok_or(TiffError::FormatError(TiffFormatError::UnknownPredictor(p))) + }) + .transpose()? + .unwrap_or(Predictor::None); + + let planar_config = tag_reader + .find_tag(Tag::PlanarConfiguration) + .await? + .map(Value::into_u16) + .transpose()? + .map(|p| { + PlanarConfiguration::from_u16(p).ok_or(TiffError::FormatError( + TiffFormatError::UnknownPlanarConfiguration(p), + )) + }) + .transpose()? + .unwrap_or(PlanarConfiguration::Chunky); + + let planes = match planar_config { + PlanarConfiguration::Chunky => 1, + PlanarConfiguration::Planar => samples, + }; + + let chunk_type; + let chunk_offsets; + let chunk_bytes; + let strip_decoder; + let tile_attributes; + match ( + ifd.contains_key(&Tag::StripByteCounts), + ifd.contains_key(&Tag::StripOffsets), + ifd.contains_key(&Tag::TileByteCounts), + ifd.contains_key(&Tag::TileOffsets), + ) { + (true, true, false, false) => { + chunk_type = ChunkType::Strip; + + chunk_offsets = //ifd[&Tag::StripOffsets]; + tag_reader + .find_tag(Tag::StripOffsets).await? + .unwrap() + .into_u64_vec()?; + chunk_bytes = //ifd[&Tag::StripByteCounts]; + tag_reader + .find_tag(Tag::StripByteCounts).await? + .unwrap() + .into_u64_vec()?; + let rows_per_strip = tag_reader + .find_tag(Tag::RowsPerStrip) + .await? + .map(Value::into_u32) + .transpose()? + .unwrap_or(height); + strip_decoder = Some(StripDecodeState { rows_per_strip }); + tile_attributes = None; + + if chunk_offsets.len() != chunk_bytes.len() + || rows_per_strip == 0 + || u32::try_from(chunk_offsets.len())? + != (height.saturating_sub(1) / rows_per_strip + 1) * planes as u32 + { + return Err(TiffError::FormatError( + TiffFormatError::InconsistentSizesEncountered, + )); + } + } + (false, false, true, true) => { + chunk_type = ChunkType::Tile; + + let tile_width = + usize::try_from(tag_reader.require_tag(Tag::TileWidth).await?.into_u32()?)?; + let tile_length = + usize::try_from(tag_reader.require_tag(Tag::TileLength).await?.into_u32()?)?; + + if tile_width == 0 { + return Err(TiffFormatError::InvalidTagValueType(Tag::TileWidth).into()); + } else if tile_length == 0 { + return Err(TiffFormatError::InvalidTagValueType(Tag::TileLength).into()); + } + + strip_decoder = None; + tile_attributes = Some(TileAttributes { + image_width: usize::try_from(width)?, + image_height: usize::try_from(height)?, + tile_width, + tile_length, + }); + chunk_offsets = //ifd[&Tag::TileOffsets]; + tag_reader + .find_tag(Tag::TileOffsets).await? + .unwrap() + .into_u64_vec()?; + chunk_bytes = //ifd[&Tag::TileByteCounts]; + tag_reader + .find_tag(Tag::TileByteCounts).await? + .unwrap() + .into_u64_vec()?; + + let tile = tile_attributes.as_ref().unwrap(); + if chunk_offsets.len() != chunk_bytes.len() + || chunk_offsets.len() as usize + != tile.tiles_down() * tile.tiles_across() * planes as usize + { + return Err(TiffError::FormatError( + TiffFormatError::InconsistentSizesEncountered, + )); + } + } + (_, _, _, _) => { + return Err(TiffError::FormatError( + TiffFormatError::StripTileTagConflict, + )) + } + }; + + Ok(Image { + ifd: Some(ifd), + width, + height, + bits_per_sample: bits_per_sample[0], + samples, + sample_format, + photometric_interpretation, + compression_method, + jpeg_tables, + predictor, + chunk_type, + planar_config, + strip_decoder, + tile_attributes, + chunk_offsets, + chunk_bytes, + }) + } +} diff --git a/src/decoder/async_decoder/mod.rs b/src/decoder/async_decoder/mod.rs new file mode 100644 index 0000000..6fcfe8e --- /dev/null +++ b/src/decoder/async_decoder/mod.rs @@ -0,0 +1,541 @@ +use futures::{ + // future::BoxFuture, + io::{AsyncRead, AsyncReadExt, AsyncSeek, SeekFrom}, + AsyncSeekExt, +}; +use std::collections::{HashMap, HashSet}; + +use crate::{TiffError, TiffFormatError, TiffResult, TiffUnsupportedError}; + +// use self::ifd::Directory; +// use self::image::Image; +use crate::tags::{ + CompressionMethod, PhotometricInterpretation, PlanarConfiguration, Predictor, SampleFormat, + Tag, Type, +}; + +use crate::decoder::{ + Decoder, + ifd::{Value, Directory}, Image, stream::{ + ByteOrder, SmartReader, + }, ChunkType, DecodingBuffer, DecodingResult, +}; + +use stream::AsyncEndianReader; + +extern crate async_trait; + +pub mod ifd; +pub mod image; +pub(self) mod stream; +// pub mod stream; +pub mod tag_reader; + +#[async_trait::async_trait] +pub trait RangeReader { + async fn read_range( + &mut self, + bytes_start: u64, + bytes_end: u64, + ) -> futures::io::Result>; +} + +#[async_trait::async_trait] +impl RangeReader for R { + async fn read_range( + &mut self, + bytes_start: u64, + bytes_end: u64, + ) -> futures::io::Result> { + let length = bytes_end - bytes_start; + let mut buffer = vec![0; length.try_into().map_err(|e| std::io::Error::other(e))?]; + + // Seek to the start position + self.seek(SeekFrom::Start(bytes_start as u64)).await?; + + // Read exactly the number of bytes we need + self.read_exact(&mut buffer).await?; + + Ok(buffer) + } +} + +impl Decoder { + + pub async fn new_async(r: R) -> TiffResult> { + Self::new_overview_async(r, 0).await + } + + pub async fn new_overview_async(mut r: R, overview: u32) -> TiffResult> { + let mut endianess = [0; 2]; + r.read_exact(&mut endianess).await?; + let byte_order = match &endianess { + b"II" => ByteOrder::LittleEndian, + b"MM" => ByteOrder::BigEndian, + _ => { + return Err(TiffError::FormatError( + TiffFormatError::TiffSignatureNotFound, + )); + } + }; + + let mut reader = SmartReader::wrap(r, byte_order); + + let bigtiff = match reader.read_u16().await? { + 42 => false, + 43 => { + if reader.read_u16().await? != 8 { + return Err(TiffError::FormatError( + TiffFormatError::TiffSignatureNotFound, + )); + } + if reader.read_u16().await? != 0 { + return Err(TiffError::FormatError( + TiffFormatError::TiffSignatureNotFound, + )); + } + true + } + _ => { + return Err(TiffError::FormatError( + TiffFormatError::TiffSignatureInvalid, + )); + } + }; + + let next_ifd = if bigtiff { + Some(reader.read_u64().await?) + } else { + Some(u64::from(reader.read_u32().await?)) + }; + + let mut seen_ifds = HashSet::new(); + seen_ifds.insert(next_ifd.unwrap()); + + let mut decoder = Decoder { + reader, + bigtiff, + limits: Default::default(), // Replace with actual initialization + next_ifd, + ifd_offsets: vec![next_ifd.unwrap()], + seen_ifds, + image: Image { + ifd: None, + width: 0, + height: 0, + bits_per_sample: 1, + samples: 1, + sample_format: SampleFormat::Uint, + photometric_interpretation: PhotometricInterpretation::BlackIsZero, + compression_method: CompressionMethod::None, + jpeg_tables: None, + predictor: Predictor::None, + chunk_type: ChunkType::Tile, + planar_config: PlanarConfiguration::Chunky, + strip_decoder: None, + tile_attributes: None, + chunk_offsets: Vec::new(), + chunk_bytes: Vec::new(), + }, + }; + + decoder.seek_to_image_async(overview.try_into()?).await?; + decoder.next_image_async().await?; + Ok(decoder) + } + + /// Loads the IFD at the specified index in the list, if one exists + pub async fn seek_to_image_async(&mut self, ifd_index: usize) -> TiffResult<()> { + // Check whether we have seen this IFD before, if so then the index will be less than the length of the list of ifd offsets + if ifd_index >= self.ifd_offsets.len() { + // We possibly need to load in the next IFD + if self.next_ifd.is_none() { + return Err(TiffError::FormatError( + TiffFormatError::ImageFileDirectoryNotFound, + )); + } + + loop { + // Follow the list until we find the one we want, or we reach the end, whichever happens first + let (_ifd, next_ifd) = self.next_ifd_async().await?; + + if next_ifd.is_none() { + break; + } + + if ifd_index < self.ifd_offsets.len() { + break; + } + } + } + + // If the index is within the list of ifds then we can load the selected image/IFD + if let Some(ifd_offset) = self.ifd_offsets.get(ifd_index) { + let (ifd, _next_ifd) = + Self::read_ifd_async(&mut self.reader, self.bigtiff, *ifd_offset).await?; + self.image = + Image::from_async_reader(&mut self.reader, ifd, &self.limits, self.bigtiff).await?; + + Ok(()) + } else { + Err(TiffError::FormatError( + TiffFormatError::ImageFileDirectoryNotFound, + )) + } + } + + async fn next_ifd_async(&mut self) -> TiffResult<(Directory, Option)> { + if self.next_ifd.is_none() { + return Err(TiffError::FormatError( + TiffFormatError::ImageFileDirectoryNotFound, + )); + } + + let (ifd, next_ifd) = Self::read_ifd_async( + &mut self.reader, + self.bigtiff, + self.next_ifd.take().unwrap(), + ) + .await?; + + if let Some(next) = next_ifd { + if !self.seen_ifds.insert(next) { + return Err(TiffError::FormatError(TiffFormatError::CycleInOffsets)); + } + self.next_ifd = Some(next); + self.ifd_offsets.push(next); + } + + Ok((ifd, next_ifd)) + } + + /// Reads in the next image. + /// If there is no further image in the TIFF file a format error is returned. + /// To determine whether there are more images call `TIFFDecoder::more_images` instead. + pub async fn next_image_async(&mut self) -> TiffResult<()> { + let (ifd, _next_ifd) = self.next_ifd_async().await?; + + self.image = Image::from_async_reader(&mut self.reader, ifd, &self.limits, self.bigtiff).await?; + Ok(()) + } + + // Reads the IFD starting at the indicated location. + async fn read_ifd_async( + reader: &mut SmartReader, + bigtiff: bool, + ifd_location: u64, + ) -> TiffResult<(Directory, Option)> { + reader.goto_offset_async(ifd_location).await?; + + let mut dir: Directory = HashMap::new(); + + let num_tags = if bigtiff { + reader.read_u64().await? + } else { + reader.read_u16().await?.into() + }; + + // const TAG_SIZE: i64 = 2 + 2 + 4 + 4; + // reader + // .seek(std::io::SeekFrom::Current( + // TAG_SIZE * i64::try_from(num_tags).unwrap(), + // )) + // .await?; + for _ in 0..num_tags { + let (tag, entry) = match Self::read_entry_async(reader, bigtiff).await? { + Some(val) => val, + None => { + continue; + } // Unknown data type in tag, skip + }; + dir.insert(tag, entry); + } + + let next_ifd = if bigtiff { + reader.read_u64().await? + } else { + reader.read_u32().await?.into() + }; + + let next_ifd = match next_ifd { + 0 => None, + _ => Some(next_ifd), + }; + + Ok((dir, next_ifd)) + } + + /// Reads a IFD entry. + // An IFD entry has four fields: + // + // Tag 2 bytes + // Type 2 bytes + // Count 4 bytes + // Value 4 bytes either a pointer the value itself + async fn read_entry_async( + reader: &mut SmartReader, + bigtiff: bool, + ) -> TiffResult> { + let tag = Tag::from_u16_exhaustive(reader.read_u16().await?); + let type_ = match Type::from_u16(reader.read_u16().await?) { + Some(t) => t, + None => { + // Unknown type. Skip this entry according to spec. + reader.read_u32().await?; + reader.read_u32().await?; + return Ok(None); + } + }; + let entry = if bigtiff { + let mut offset = [0; 8]; + + let count = reader.read_u64().await?; + reader.read_exact(&mut offset).await?; + ifd::Entry::new_u64(type_, count, offset) + } else { + let mut offset = [0; 4]; + + let count = reader.read_u32().await?; + reader.read_exact(&mut offset).await?; + ifd::Entry::new(type_, count, offset) + }; + Ok(Some((tag, entry))) + } + + /// Tries to retrieve a tag. + /// Return `Ok(None)` if the tag is not present. + pub async fn find_tag_async(&mut self, tag: Tag) -> TiffResult> { + let entry = match self.image().ifd.as_ref().unwrap().get(&tag) { + None => return Ok(None), + Some(entry) => entry.clone(), + }; + + Ok(Some( + entry + .async_val(&self.limits, self.bigtiff, &mut self.reader) + .await?, + )) + } + + /// Tries to retrieve a tag and convert it to the desired unsigned type. + pub async fn find_tag_unsigned_async>(&mut self, tag: Tag) -> TiffResult> { + self.find_tag_async(tag) + .await? + .map(|v| v.into_u64()) + .transpose()? + .map(|value| { + T::try_from(value).map_err(|_| TiffFormatError::InvalidTagValueType(tag).into()) + }) + .transpose() + } + + /// Tries to retrieve a vector of all a tag's values and convert them to + /// the desired unsigned type. + pub async fn find_tag_unsigned_vec_async>( + &mut self, + tag: Tag, + ) -> TiffResult>> { + self.find_tag_async(tag) + .await? + .map(|v| v.into_u64_vec()) + .transpose()? + .map(|v| { + v.into_iter() + .map(|u| { + T::try_from(u).map_err(|_| TiffFormatError::InvalidTagValueType(tag).into()) + }) + .collect() + }) + .transpose() + } + + /// Tries to retrieve a tag and convert it to the desired unsigned type. + /// Returns an error if the tag is not present. + pub async fn get_tag_unsigned_async>(&mut self, tag: Tag) -> TiffResult { + self.find_tag_unsigned_async(tag) + .await? + .ok_or_else(|| TiffFormatError::RequiredTagNotFound(tag).into()) + } + + /// Tries to retrieve a tag. + /// Returns an error if the tag is not present + pub async fn get_tag_async(&mut self, tag: Tag) -> TiffResult { + match self.find_tag_async(tag).await? { + Some(val) => Ok(val), + None => Err(TiffError::FormatError( + TiffFormatError::RequiredTagNotFound(tag), + )), + } + } + + // /// Tries to retrieve a tag and convert it to the desired type. + // pub async fn get_tag_u32(&mut self, tag: Tag) -> TiffResult { + // self.get_tag(tag).await?.into_u32() + // } + // pub async fn get_tag_u64(&mut self, tag: Tag) -> TiffResult { + // self.get_tag(tag).await?.into_u64() + // } + + // /// Tries to retrieve a tag and convert it to the desired type. + // pub async fn get_tag_f32(&mut self, tag: Tag) -> TiffResult { + // self.get_tag(tag).await?.into_f32() + // } + + // /// Tries to retrieve a tag and convert it to the desired type. + // pub async fn get_tag_f64(&mut self, tag: Tag) -> TiffResult { + // self.get_tag(tag).await?.into_f64() + // } + + // /// Tries to retrieve a tag and convert it to the desired type. + // pub async fn get_tag_u32_vec(&mut self, tag: Tag) -> TiffResult> { + // self.get_tag(tag).await?.into_u32_vec() + // } + + // pub async fn get_tag_u16_vec(&mut self, tag: Tag) -> TiffResult> { + // self.get_tag(tag).await?.into_u16_vec() + // } + // pub async fn get_tag_u64_vec(&mut self, tag: Tag) -> TiffResult> { + // self.get_tag(tag).await?.into_u64_vec() + // } + + // /// Tries to retrieve a tag and convert it to the desired type. + // pub async fn get_tag_f32_vec(&mut self, tag: Tag) -> TiffResult> { + // self.get_tag(tag).await?.into_f32_vec() + // } + + // /// Tries to retrieve a tag and convert it to the desired type. + // pub async fn get_tag_f64_vec(&mut self, tag: Tag) -> TiffResult> { + // self.get_tag(tag).await?.into_f64_vec() + // } + + // /// Tries to retrieve a tag and convert it to a 8bit vector. + // pub async fn get_tag_u8_vec(&mut self, tag: Tag) -> TiffResult> { + // self.get_tag(tag).await?.into_u8_vec() + // } + + // /// Tries to retrieve a tag and convert it to a ascii vector. + // pub async fn get_tag_ascii_string(&mut self, tag: Tag) -> TiffResult { + // self.get_tag(tag).await?.into_string() + // } + + pub async fn read_chunk_to_buffer_async( + &mut self, + mut buffer: DecodingBuffer<'_>, + chunk_index: u32, + output_width: usize, + ) -> TiffResult<()> { + let (offset, length) = self.image.chunk_file_range(chunk_index)?; + let v = self + .reader + .read_range(offset, offset + length) + .await?; + + let byte_order = self.reader.byte_order; + + let output_row_stride = (output_width as u64) + .saturating_mul(self.image.samples_per_pixel() as u64) + .saturating_mul(self.image.bits_per_sample as u64) + / 8; + + self.image.expand_chunk( + &mut std::io::Cursor::new(v), + buffer.as_bytes_mut(), + output_row_stride.try_into()?, + byte_order, + chunk_index, + &self.limits, + )?; + + Ok(()) + } + + /// Read the specified chunk (at index `chunk_index`) and return the binary data as a Vector. + pub async fn read_chunk_async(&mut self, chunk_index: u32) -> TiffResult { + let data_dims = self.image().chunk_data_dimensions(chunk_index)?; + + let mut result = Self::result_buffer(data_dims.0 as usize, data_dims.1 as usize, self.image(), &self.limits)?; + + self.read_chunk_to_buffer_async(result.as_buffer(0), chunk_index, data_dims.0 as usize) + .await?; + + Ok(result) + } + + /// Decodes the entire image and return it as a Vector + pub async fn read_image_async(&mut self) -> TiffResult { + let width = self.image().width; + let height = self.image().height; + let mut result = Self::result_buffer(usize::try_from(width)?, usize::try_from(height)?, self.image(), &self.limits )?; + if width == 0 || height == 0 { + return Ok(result); + } + + let chunk_dimensions = self.image().chunk_dimensions()?; + let chunk_dimensions = ( + chunk_dimensions.0.min(width), + chunk_dimensions.1.min(height), + ); + if chunk_dimensions.0 == 0 || chunk_dimensions.1 == 0 { + return Err(TiffError::FormatError( + TiffFormatError::InconsistentSizesEncountered, + )); + } + + let samples = self.image().samples_per_pixel(); + if samples == 0 { + return Err(TiffError::FormatError( + TiffFormatError::InconsistentSizesEncountered, + )); + } + + let output_row_bits = (width as u64 * self.image.bits_per_sample as u64) + .checked_mul(samples as u64) + .ok_or(TiffError::LimitsExceeded)?; + let output_row_stride: usize = ((output_row_bits + 7) / 8).try_into()?; + + let chunk_row_bits = (chunk_dimensions.0 as u64 * self.image.bits_per_sample as u64) + .checked_mul(samples as u64) + .ok_or(TiffError::LimitsExceeded)?; + let chunk_row_bytes: usize = ((chunk_row_bits + 7) / 8).try_into()?; + + let chunks_across = ((width - 1) / chunk_dimensions.0 + 1) as usize; + + if chunks_across > 1 && chunk_row_bits % 8 != 0 { + return Err(TiffError::UnsupportedError( + TiffUnsupportedError::MisalignedTileBoundaries, + )); + } + + // in planar config, an image has chunks/n_bands chunks + let image_chunks = self.image().chunk_offsets.len() / self.image().strips_per_pixel(); + // For multi-band images, only the first band is read. + // Possible improvements: + // * pass requested band as parameter + // * collect bands to a RGB encoding result in case of RGB bands + for chunk in 0..image_chunks { + let (offset, length) = self.image.chunk_file_range(chunk.try_into().unwrap())?; + let v = self + .reader + .read_range(offset, offset + length) + .await?; + let mut reader = std::io::Cursor::new(v); + // self.goto_offset_u64(self.image().chunk_offsets[chunk]).await?; + + let x = chunk % chunks_across; + let y = chunk / chunks_across; + let buffer_offset = + y * output_row_stride * chunk_dimensions.1 as usize + x * chunk_row_bytes; + let byte_order = self.reader.byte_order; + self.image.expand_chunk( + &mut reader, + &mut result.as_buffer(0).as_bytes_mut()[buffer_offset..], + output_row_stride, + byte_order, + chunk as u32, + &self.limits, + )?; + } + + Ok(result) + } +} diff --git a/src/decoder/async_decoder/readme.md b/src/decoder/async_decoder/readme.md new file mode 100644 index 0000000..a0cd452 --- /dev/null +++ b/src/decoder/async_decoder/readme.md @@ -0,0 +1,32 @@ +# Making Tiff crate async for Cogs + +This folder is a bunch of copied code, made async with futures crate. This makes the reader runtime agnostic, as well as backend agnostic. This allows it to be used anywhere (tokio, smol, bevy), from any datasource that provides an AsyncReader. For using this on Sync stuff, such as files, use the following snippet: + +```rust +use futures::io::AllowStdIo; +use std::fs::File; +use tiff::decoder_async::Decoder; + +#[tokio::main] +fn main() { + let f = AllowStdIo(File("/path/to/file.tiff")); + let mut decoder = Decoder::new(f).await.expect("Could not open file"); + let result = decoder.read_image(); +} +``` + +For more fine-grained control over what the tiff does, another method, `new_raw` is provided, that does not read all tags by default, since this caused excessive requests when testing on isdasoil data. + +The plan is: + +1. Read image and check header +2. Scan all IFD tags to see how many overviews there are + - may actually read into values that don't hold pointers, since we're ideally buffering anyways. ChatGPT says all metadata (excluding tile offsets) should be in the first ~16kB for COGs, which is a - 16 * 1024 byte buffer, which is not that big. + - The call chain that slows execution is: `decoder.next_image()->tag_reader.find_tag(tag)->entry.clone().val()->case 4 entry.decode_offset()` where `decode_offset` possibly goes out of the currently read buffer. Of course, this could be circumvented by having a grow-only buffer, but then the reader and decoder would have to be more tightly coupled + - another alternative (which I kind of like) is to (optionally) store offset data in our own buffer that is the right size. However, for the largest overview, even this may be kind of big? + - ideally, we would be able to pass as arguments to our reader: read_bytes(a, b), because that would directly translate to a range request. <- this could be a homebrew trait with a blanket implementation? +3. Read only (the first?) IFD tag +4. Load tiles on-demand + + +I think the current implementation is mainly inefficient because tags that hold more data than what fits in an IFD entries' Value field, the Value is a pointer that gets followed through (because a file was assumed). diff --git a/src/decoder/async_decoder/stream.rs b/src/decoder/async_decoder/stream.rs new file mode 100644 index 0000000..ef0bbaa --- /dev/null +++ b/src/decoder/async_decoder/stream.rs @@ -0,0 +1,72 @@ +use futures::{AsyncRead, AsyncReadExt, AsyncSeek, AsyncSeekExt}; +use std::io; +use crate::decoder::stream::{SmartReader, ByteOrder}; + +macro_rules! read_async_fn { + ($name:ident, $type:ty) => { + /// reads an $type + #[inline(always)] + async fn $name(&mut self) -> Result<$type, io::Error> { + let mut n = [0u8; std::mem::size_of::<$type>()]; + self.read_exact(&mut n).await?; + Ok(match self.byte_order() { + ByteOrder::LittleEndian => <$type>::from_le_bytes(n), + ByteOrder::BigEndian => <$type>::from_be_bytes(n), + }) + } + }; +} + +#[async_trait::async_trait] +/// Reader that is aware of the byte order. +pub trait AsyncEndianReader: AsyncRead + Unpin { + /// Byte order that should be adhered to + fn byte_order(&self) -> ByteOrder; + + read_async_fn!(read_u16, u16); + // read_async_fn!(read_i8, i8); + // read_async_fn!(read_i16, i16); + read_async_fn!(read_u32, u32); + // read_async_fn!(read_i32, i32); + read_async_fn!(read_u64, u64); + // read_async_fn!(read_i64, i64); + // read_async_fn!(read_f32, f32); + // read_async_fn!(read_f64, f64); +} + + + +impl AsyncEndianReader for SmartReader { + #[inline(always)] + fn byte_order(&self) -> ByteOrder { + self.byte_order + } +} + +impl AsyncRead for SmartReader { + fn poll_read( + self: std::pin::Pin<&mut Self>, + cx: &mut std::task::Context<'_>, + buf: &mut [u8], + ) -> std::task::Poll> { + let pinned = std::pin::Pin::new(&mut self.get_mut().reader); + pinned.poll_read(cx, buf) + } +} + +impl AsyncSeek for SmartReader { + fn poll_seek( + self: std::pin::Pin<&mut Self>, + cx: &mut std::task::Context<'_>, + pos: io::SeekFrom, + ) -> std::task::Poll> { + let pinned = std::pin::Pin::new(&mut self.get_mut().reader); + pinned.poll_seek(cx, pos) + } +} + +impl SmartReader { + pub async fn goto_offset_async(&mut self, offset: u64) -> io::Result<()> { + self.seek(io::SeekFrom::Start(offset)).await.map(|_| ()) + } +} \ No newline at end of file diff --git a/src/decoder/async_decoder/tag_reader.rs b/src/decoder/async_decoder/tag_reader.rs new file mode 100644 index 0000000..cc09f25 --- /dev/null +++ b/src/decoder/async_decoder/tag_reader.rs @@ -0,0 +1,52 @@ +use crate::tags::Tag; +use crate::{TiffError, TiffFormatError, TiffResult}; + +use super::{Directory}; +use crate::decoder::{ifd::Value, Limits, stream::SmartReader}; + +use futures::{AsyncRead, AsyncSeek}; + +pub struct AsyncTagReader<'a, R: AsyncRead + AsyncSeek + Unpin + Send> { + pub reader: &'a mut SmartReader, + pub ifd: &'a Directory, + pub limits: &'a Limits, + pub bigtiff: bool, +} +impl<'a, R: AsyncRead + AsyncSeek + Unpin + Send> AsyncTagReader<'a, R> { + pub(crate) async fn find_tag(&mut self, tag: Tag) -> TiffResult> { + Ok(match self.ifd.get(&tag) { + Some(entry) => Some( + entry + .clone() + .async_val(self.limits, self.bigtiff, self.reader) + .await?, + ), + None => None, + }) // dinsdagmiddag 14-16 2e verdieping + } + pub(crate) async fn require_tag(&mut self, tag: Tag) -> TiffResult { + match self.find_tag(tag).await? { + Some(val) => Ok(val), + None => Err(TiffError::FormatError( + TiffFormatError::RequiredTagNotFound(tag), + )), + } + } + pub(crate) async fn find_tag_uint_vec>( + &mut self, + tag: Tag, + ) -> TiffResult>> { + self.find_tag(tag) + .await? + .map(|v| v.into_u64_vec()) + .transpose()? + .map(|v| { + v.into_iter() + .map(|u| { + T::try_from(u).map_err(|_| TiffFormatError::InvalidTagValueType(tag).into()) + }) + .collect() + }) + .transpose() + } +} diff --git a/src/decoder/ifd.rs b/src/decoder/ifd.rs index 9be0d35..df6d9a9 100644 --- a/src/decoder/ifd.rs +++ b/src/decoder/ifd.rs @@ -2,7 +2,6 @@ use std::collections::HashMap; use std::io::{self, Read, Seek}; -use std::mem; use std::str; use super::stream::{ByteOrder, EndianReader, SmartReader}; @@ -362,24 +361,13 @@ impl Entry { } /// Returns a mem_reader for the offset/value field - fn r(&self, byte_order: ByteOrder) -> SmartReader>> { + pub(crate) fn r(&self, byte_order: ByteOrder) -> SmartReader>> { SmartReader::wrap(io::Cursor::new(self.offset.to_vec()), byte_order) } - pub fn val( - &self, - limits: &super::Limits, - bigtiff: bool, - reader: &mut SmartReader, - ) -> TiffResult { - // Case 1: there are no values so we can return immediately. - if self.count == 0 { - return Ok(List(Vec::new())); - } - - let bo = reader.byte_order(); - - let tag_size = match self.type_ { + #[inline(always)] + pub(crate) fn tag_size(&self) -> u64 { + match self.type_ { Type::BYTE | Type::SBYTE | Type::ASCII | Type::UNDEFINED => 1, Type::SHORT | Type::SSHORT => 2, Type::LONG | Type::SLONG | Type::FLOAT | Type::IFD => 4, @@ -389,20 +377,27 @@ impl Entry { | Type::RATIONAL | Type::SRATIONAL | Type::IFD8 => 8, - }; + } + } - let value_bytes = match self.count.checked_mul(tag_size) { - Some(n) => n, + #[inline(always)] + pub(crate) fn value_bytes(&self) -> TiffResult { + match self.count.checked_mul(self.tag_size()) { + Some(n) => Ok(n), None => { return Err(TiffError::LimitsExceeded); } - }; + } + } + pub(crate) fn val_if_in_offset(&self, bigtiff: bool, bo: ByteOrder) -> TiffResult> { + let value_bytes = self.value_bytes()?; + // Case 2: there is one value. if self.count == 1 { // 2a: the value is 5-8 bytes and we're in BigTiff mode. if bigtiff && value_bytes > 4 && value_bytes <= 8 { - return Ok(match self.type_ { + return Ok(Some(match self.type_ { Type::LONG8 => UnsignedBig(self.r(bo).read_u64()?), Type::SLONG8 => SignedBig(self.r(bo).read_i64()?), Type::DOUBLE => Double(self.r(bo).read_f64()?), @@ -425,11 +420,11 @@ impl Entry { | Type::SLONG | Type::FLOAT | Type::IFD => unreachable!(), - }); + })); } // 2b: the value is at most 4 bytes or doesn't fit in the offset field. - return Ok(match self.type_ { + return Ok(Some(match self.type_ { Type::BYTE => Unsigned(u32::from(self.offset[0])), Type::SBYTE => Signed(i32::from(self.offset[0] as i8)), Type::UNDEFINED => Byte(self.offset[0]), @@ -438,6 +433,7 @@ impl Entry { Type::LONG => Unsigned(self.r(bo).read_u32()?), Type::SLONG => Signed(self.r(bo).read_i32()?), Type::FLOAT => Float(self.r(bo).read_f32()?), + Type::IFD => Ifd(self.r(bo).read_u32()?), Type::ASCII => { if self.offset[0] == 0 { Ascii("".to_string()) @@ -445,57 +441,34 @@ impl Entry { return Err(TiffError::FormatError(TiffFormatError::InvalidTag)); } } - Type::LONG8 => { - reader.goto_offset(self.r(bo).read_u32()?.into())?; - UnsignedBig(reader.read_u64()?) - } - Type::SLONG8 => { - reader.goto_offset(self.r(bo).read_u32()?.into())?; - SignedBig(reader.read_i64()?) - } - Type::DOUBLE => { - reader.goto_offset(self.r(bo).read_u32()?.into())?; - Double(reader.read_f64()?) - } - Type::RATIONAL => { - reader.goto_offset(self.r(bo).read_u32()?.into())?; - Rational(reader.read_u32()?, reader.read_u32()?) - } - Type::SRATIONAL => { - reader.goto_offset(self.r(bo).read_u32()?.into())?; - SRational(reader.read_i32()?, reader.read_i32()?) - } - Type::IFD => Ifd(self.r(bo).read_u32()?), - Type::IFD8 => { - reader.goto_offset(self.r(bo).read_u32()?.into())?; - IfdBig(reader.read_u64()?) - } - }); + _ => return Ok(None) + })); } + // Case 3: There is more than one value, but it fits in the offset field. if value_bytes <= 4 || bigtiff && value_bytes <= 8 { match self.type_ { - Type::BYTE => return offset_to_bytes(self.count as usize, self), - Type::SBYTE => return offset_to_sbytes(self.count as usize, self), + Type::BYTE => return Ok(Some(offset_to_bytes(self.count as usize, self)?)), + Type::SBYTE => return Ok(Some(offset_to_sbytes(self.count as usize, self)?)), Type::ASCII => { let mut buf = vec![0; self.count as usize]; self.r(bo).read_exact(&mut buf)?; if buf.is_ascii() && buf.ends_with(&[0]) { let v = str::from_utf8(&buf)?; let v = v.trim_matches(char::from(0)); - return Ok(Ascii(v.into())); + return Ok(Some(Ascii(v.into()))); } else { return Err(TiffError::FormatError(TiffFormatError::InvalidTag)); } } Type::UNDEFINED => { - return Ok(List( + return Ok(Some(List( self.offset[0..self.count as usize] .iter() .map(|&b| Byte(b)) .collect(), - )); + ))); } Type::SHORT => { let mut r = self.r(bo); @@ -503,7 +476,7 @@ impl Entry { for _ in 0..self.count { v.push(Short(r.read_u16()?)); } - return Ok(List(v)); + return Ok(Some(List(v))); } Type::SSHORT => { let mut r = self.r(bo); @@ -511,7 +484,7 @@ impl Entry { for _ in 0..self.count { v.push(Signed(i32::from(r.read_i16()?))); } - return Ok(List(v)); + return Ok(Some(List(v))); } Type::LONG => { let mut r = self.r(bo); @@ -519,7 +492,7 @@ impl Entry { for _ in 0..self.count { v.push(Unsigned(r.read_u32()?)); } - return Ok(List(v)); + return Ok(Some(List(v))); } Type::SLONG => { let mut r = self.r(bo); @@ -527,7 +500,7 @@ impl Entry { for _ in 0..self.count { v.push(Signed(r.read_i32()?)); } - return Ok(List(v)); + return Ok(Some(List(v))); } Type::FLOAT => { let mut r = self.r(bo); @@ -535,7 +508,7 @@ impl Entry { for _ in 0..self.count { v.push(Float(r.read_f32()?)); } - return Ok(List(v)); + return Ok(Some(List(v))); } Type::IFD => { let mut r = self.r(bo); @@ -543,7 +516,7 @@ impl Entry { for _ in 0..self.count { v.push(Ifd(r.read_u32()?)); } - return Ok(List(v)); + return Ok(Some(List(v))); } Type::LONG8 | Type::SLONG8 @@ -556,78 +529,132 @@ impl Entry { } } + Ok(None) + } + + #[inline(always)] + pub(crate) fn offset(&self, bigtiff: bool, bo: ByteOrder) -> TiffResult { + if bigtiff { + Ok(self.r(bo).read_u64()?) + } else { + Ok(self.r(bo).read_u32()?.into()) + } + } + + pub fn val( + &self, + limits: &super::Limits, + bigtiff: bool, + reader: &mut SmartReader, + ) -> TiffResult { + let bo = reader.byte_order(); + if let Some(res) = self.val_if_in_offset(bigtiff, bo)? { + return Ok(res); + } + + // check if we exceed the limits and read required bytes into a buffer if everything is ok + // This allows us to also create a cursor in async code + let v_bytes = usize::try_from(self.value_bytes()?)?; + if v_bytes > limits.decoding_buffer_size { + return Err(TiffError::LimitsExceeded); + } + let mut buf = vec![0; v_bytes]; + reader.goto_offset(self.offset(bigtiff, bo)?)?; + reader.read_exact(&mut buf)?; + let mut r = SmartReader::wrap(std::io::Cursor::new(buf), bo); + self.val_from_cursor(&mut r) + } + + pub(crate) fn val_from_cursor( + &self, + reader: &mut SmartReader, + ) -> TiffResult { + if self.count == 1 { + // 2b: the value is at most 4 bytes or doesn't fit in the offset field. + return Ok(match self.type_ { + Type::LONG8 => { + UnsignedBig(reader.read_u64()?) + } + Type::SLONG8 => { + SignedBig(reader.read_i64()?) + } + Type::DOUBLE => { + Double(reader.read_f64()?) + } + Type::RATIONAL => { + Rational(reader.read_u32()?, reader.read_u32()?) + } + Type::SRATIONAL => { + SRational(reader.read_i32()?, reader.read_i32()?) + } + Type::IFD8 => { + IfdBig(reader.read_u64()?) + } + Type::IFD | Type::BYTE | Type::SBYTE | Type::UNDEFINED | Type::SHORT | Type::SSHORT | Type::LONG | Type::SLONG | Type::FLOAT | Type::ASCII => unreachable!() + }); + } + // Case 4: there is more than one value, and it doesn't fit in the offset field. match self.type_ { // TODO check if this could give wrong results // at a different endianess of file/computer. - Type::BYTE => self.decode_offset(self.count, bo, bigtiff, limits, reader, |reader| { + Type::BYTE => self.decode_from_cursor(self.count, reader, |reader| { let mut buf = [0; 1]; reader.read_exact(&mut buf)?; Ok(UnsignedBig(u64::from(buf[0]))) }), - Type::SBYTE => self.decode_offset(self.count, bo, bigtiff, limits, reader, |reader| { + Type::SBYTE => self.decode_from_cursor(self.count, reader, |reader| { Ok(SignedBig(i64::from(reader.read_i8()?))) }), - Type::SHORT => self.decode_offset(self.count, bo, bigtiff, limits, reader, |reader| { + Type::SHORT => self.decode_from_cursor(self.count, reader, |reader| { Ok(UnsignedBig(u64::from(reader.read_u16()?))) }), - Type::SSHORT => self.decode_offset(self.count, bo, bigtiff, limits, reader, |reader| { + Type::SSHORT => self.decode_from_cursor(self.count, reader, |reader| { Ok(SignedBig(i64::from(reader.read_i16()?))) }), - Type::LONG => self.decode_offset(self.count, bo, bigtiff, limits, reader, |reader| { + Type::LONG => self.decode_from_cursor(self.count, reader, |reader| { Ok(Unsigned(reader.read_u32()?)) }), - Type::SLONG => self.decode_offset(self.count, bo, bigtiff, limits, reader, |reader| { + Type::SLONG => self.decode_from_cursor(self.count, reader, |reader| { Ok(Signed(reader.read_i32()?)) }), - Type::FLOAT => self.decode_offset(self.count, bo, bigtiff, limits, reader, |reader| { + Type::FLOAT => self.decode_from_cursor(self.count, reader, |reader| { Ok(Float(reader.read_f32()?)) }), - Type::DOUBLE => self.decode_offset(self.count, bo, bigtiff, limits, reader, |reader| { + Type::DOUBLE => self.decode_from_cursor(self.count, reader, |reader| { Ok(Double(reader.read_f64()?)) }), Type::RATIONAL => { - self.decode_offset(self.count, bo, bigtiff, limits, reader, |reader| { + self.decode_from_cursor(self.count, reader, |reader| { Ok(Rational(reader.read_u32()?, reader.read_u32()?)) }) } Type::SRATIONAL => { - self.decode_offset(self.count, bo, bigtiff, limits, reader, |reader| { + self.decode_from_cursor(self.count, reader, |reader| { Ok(SRational(reader.read_i32()?, reader.read_i32()?)) }) } - Type::LONG8 => self.decode_offset(self.count, bo, bigtiff, limits, reader, |reader| { + Type::LONG8 => self.decode_from_cursor(self.count, reader, |reader| { Ok(UnsignedBig(reader.read_u64()?)) }), - Type::SLONG8 => self.decode_offset(self.count, bo, bigtiff, limits, reader, |reader| { + Type::SLONG8 => self.decode_from_cursor(self.count, reader, |reader| { Ok(SignedBig(reader.read_i64()?)) }), - Type::IFD => self.decode_offset(self.count, bo, bigtiff, limits, reader, |reader| { + Type::IFD => self.decode_from_cursor(self.count, reader, |reader| { Ok(Ifd(reader.read_u32()?)) }), - Type::IFD8 => self.decode_offset(self.count, bo, bigtiff, limits, reader, |reader| { + Type::IFD8 => self.decode_from_cursor(self.count, reader, |reader| { Ok(IfdBig(reader.read_u64()?)) }), Type::UNDEFINED => { - self.decode_offset(self.count, bo, bigtiff, limits, reader, |reader| { + self.decode_from_cursor(self.count, reader, |reader| { let mut buf = [0; 1]; reader.read_exact(&mut buf)?; Ok(Byte(buf[0])) }) } Type::ASCII => { - let n = usize::try_from(self.count)?; - if n > limits.decoding_buffer_size { - return Err(TiffError::LimitsExceeded); - } - - if bigtiff { - reader.goto_offset(self.r(bo).read_u64()?)? - } else { - reader.goto_offset(self.r(bo).read_u32()?.into())? - } - - let mut out = vec![0; n]; + let mut out = vec![0; self.count.try_into()?]; reader.read_exact(&mut out)?; // Strings may be null-terminated, so we trim anything downstream of the null byte if let Some(first) = out.iter().position(|&b| b == 0) { @@ -638,36 +665,16 @@ impl Entry { } } - #[inline] - fn decode_offset( + #[inline(always)] + pub(crate) fn decode_from_cursor) -> TiffResult>( &self, value_count: u64, - bo: ByteOrder, - bigtiff: bool, - limits: &super::Limits, reader: &mut SmartReader, - decode_fn: F, - ) -> TiffResult - where - R: Read + Seek, - F: Fn(&mut SmartReader) -> TiffResult, - { - let value_count = usize::try_from(value_count)?; - if value_count > limits.decoding_buffer_size / mem::size_of::() { - return Err(TiffError::LimitsExceeded); - } - - let mut v = Vec::with_capacity(value_count); - - let offset = if bigtiff { - self.r(bo).read_u64()? - } else { - self.r(bo).read_u32()?.into() - }; - reader.goto_offset(offset)?; - + decode_fn: F + ) -> TiffResult{ + let mut v = Vec::with_capacity(usize::try_from(value_count)?); for _ in 0..value_count { - v.push(decode_fn(reader)?) + v.push(decode_fn(reader)?); } Ok(List(v)) } diff --git a/src/decoder/image.rs b/src/decoder/image.rs index 97658f4..f0604c5 100644 --- a/src/decoder/image.rs +++ b/src/decoder/image.rs @@ -10,14 +10,14 @@ use crate::{ColorType, TiffError, TiffFormatError, TiffResult, TiffUnsupportedEr use std::io::{self, Cursor, Read, Seek}; use std::sync::Arc; -#[derive(Debug)] -pub(crate) struct StripDecodeState { +#[derive(Debug, Clone)] +pub struct StripDecodeState { pub rows_per_strip: u32, } -#[derive(Debug)] +#[derive(Debug, Clone)] /// Computed values useful for tile decoding -pub(crate) struct TileAttributes { +pub struct TileAttributes { pub image_width: usize, pub image_height: usize, @@ -58,8 +58,8 @@ impl TileAttributes { } } -#[derive(Debug)] -pub(crate) struct Image { +#[derive(Debug, Clone)] +pub struct Image { pub ifd: Option, pub width: u32, pub height: u32, @@ -164,7 +164,7 @@ impl Image { // Technically bits_per_sample.len() should be *equal* to samples, but libtiff also allows // it to be a single value that applies to all samples. - if bits_per_sample.len() != samples.into() && bits_per_sample.len() != 1 { + if bits_per_sample.len() != usize::from(samples) && bits_per_sample.len() != 1 { return Err(TiffError::FormatError( TiffFormatError::InconsistentSizesEncountered, )); @@ -679,7 +679,7 @@ impl Image { } } } else { - for (i, row) in buf + for (_, row) in buf .chunks_mut(output_row_stride) .take(data_dims.1 as usize) .enumerate() @@ -687,8 +687,6 @@ impl Image { let row = &mut row[..data_row_bytes]; reader.read_exact(row)?; - println!("chunk={chunk_index}, index={i}"); - // Skip horizontal padding if chunk_row_bytes > data_row_bytes { let len = u64::try_from(chunk_row_bytes - data_row_bytes)?; diff --git a/src/decoder/mod.rs b/src/decoder/mod.rs index 09d530b..6d86c51 100644 --- a/src/decoder/mod.rs +++ b/src/decoder/mod.rs @@ -15,10 +15,17 @@ use crate::tags::{ use self::stream::{ByteOrder, EndianReader, SmartReader}; pub mod ifd; -mod image; -mod stream; +pub mod image; +pub mod stream; mod tag_reader; +#[cfg(feature = "async_decoder")] +pub mod async_decoder; +#[cfg(feature = "multithread")] +mod multithread_decoder; +#[cfg(feature = "multithread")] +pub use multithread_decoder::ChunkDecoder; + /// Result of a decoding process #[derive(Debug)] pub enum DecodingResult { @@ -139,6 +146,21 @@ impl DecodingResult { DecodingResult::I64(ref mut buf) => DecodingBuffer::I64(&mut buf[start..]), } } + + pub fn len(&self) -> usize { + match self { + DecodingResult::U8(v) => v.len(), + DecodingResult::U16(v) => v.len(), + DecodingResult::U32(v) => v.len(), + DecodingResult::U64(v) => v.len(), + DecodingResult::F32(v) => v.len(), + DecodingResult::F64(v) => v.len(), + DecodingResult::I8(v) => v.len(), + DecodingResult::I16(v) => v.len(), + DecodingResult::I32(v) => v.len(), + DecodingResult::I64(v) => v.len(), + } + } } // A buffer for image decoding @@ -243,8 +265,6 @@ impl Default for Limits { /// Currently does not support decoding of interlaced images #[derive(Debug)] pub struct Decoder -where - R: Read + Seek, { reader: SmartReader, bigtiff: bool, @@ -420,6 +440,132 @@ fn fix_endianness(buf: &mut [u8], byte_order: ByteOrder, bit_depth: u8) { }; } +impl Decoder { + pub fn with_limits(mut self, limits: Limits) -> Decoder { + self.limits = limits; + self + } + + pub fn dimensions(&mut self) -> TiffResult<(u32, u32)> { + Ok((self.image().width, self.image().height)) + } + + pub fn colortype(&mut self) -> TiffResult { + self.image().colortype() + } + + pub fn image(&self) -> &Image { + &self.image + } + + /// Returns `true` if there is at least one more image available. + pub fn more_images(&self) -> bool { + self.next_ifd.is_some() + } + + + fn check_chunk_type(&self, expected: ChunkType) -> TiffResult<()> { + if expected != self.image().chunk_type { + return Err(TiffError::UsageError(UsageError::InvalidChunkType( + expected, + self.image().chunk_type, + ))); + } + + Ok(()) + } + + /// The chunk type (Strips / Tiles) of the image + pub fn get_chunk_type(&self) -> ChunkType { + self.image().chunk_type + } + + /// Number of strips in image + pub fn strip_count(&mut self) -> TiffResult { + self.check_chunk_type(ChunkType::Strip)?; + let rows_per_strip = self.image().strip_decoder.as_ref().unwrap().rows_per_strip; + + if rows_per_strip == 0 { + return Ok(0); + } + + // rows_per_strip - 1 can never fail since we know it's at least 1 + let height = match self.image().height.checked_add(rows_per_strip - 1) { + Some(h) => h, + None => return Err(TiffError::IntSizeError), + }; + + let strips = match self.image().planar_config { + PlanarConfiguration::Chunky => height / rows_per_strip, + PlanarConfiguration::Planar => height / rows_per_strip * self.image().samples as u32, + }; + + Ok(strips) + } + + /// Number of tiles in image + pub fn tile_count(&mut self) -> TiffResult { + self.check_chunk_type(ChunkType::Tile)?; + Ok(u32::try_from(self.image().chunk_offsets.len())?) + } + + + fn result_buffer(width: usize, height: usize, image: &Image, limits: &Limits) -> TiffResult { + let buffer_size = match width + .checked_mul(height) + .and_then(|x| x.checked_mul(image.samples_per_pixel())) + { + Some(s) => s, + None => return Err(TiffError::LimitsExceeded), + }; + + let max_sample_bits = image.bits_per_sample; + match image.sample_format { + SampleFormat::Uint => match max_sample_bits { + n if n <= 8 => DecodingResult::new_u8(buffer_size, &limits), + n if n <= 16 => DecodingResult::new_u16(buffer_size, &limits), + n if n <= 32 => DecodingResult::new_u32(buffer_size, &limits), + n if n <= 64 => DecodingResult::new_u64(buffer_size, &limits), + n => Err(TiffError::UnsupportedError( + TiffUnsupportedError::UnsupportedBitsPerChannel(n), + )), + }, + SampleFormat::IEEEFP => match max_sample_bits { + 32 => DecodingResult::new_f32(buffer_size, &limits), + 64 => DecodingResult::new_f64(buffer_size, &limits), + n => Err(TiffError::UnsupportedError( + TiffUnsupportedError::UnsupportedBitsPerChannel(n), + )), + }, + SampleFormat::Int => match max_sample_bits { + n if n <= 8 => DecodingResult::new_i8(buffer_size, &limits), + n if n <= 16 => DecodingResult::new_i16(buffer_size, &limits), + n if n <= 32 => DecodingResult::new_i32(buffer_size, &limits), + n if n <= 64 => DecodingResult::new_i64(buffer_size, &limits), + n => Err(TiffError::UnsupportedError( + TiffUnsupportedError::UnsupportedBitsPerChannel(n), + )), + }, + format => Err(TiffUnsupportedError::UnsupportedSampleFormat(vec![format]).into()), + } + } + + + /// Returns the default chunk size for the current image. Any given chunk in the image is at most as large as + /// the value returned here. For the size of the data (chunk minus padding), use `chunk_data_dimensions`. + pub fn chunk_dimensions(&self) -> (u32, u32) { + self.image().chunk_dimensions().unwrap() + } + + /// Returns the size of the data in the chunk with the specified index. This is the default size of the chunk, + /// minus any padding. + pub fn chunk_data_dimensions(&self, chunk_index: u32) -> (u32, u32) { + self.image() + .chunk_data_dimensions(chunk_index) + .expect("invalid chunk_index") + } +} + impl Decoder { /// Create a new decoder that decodes from the stream ```r``` pub fn new(mut r: R) -> TiffResult> { @@ -499,22 +645,6 @@ impl Decoder { Ok(decoder) } - pub fn with_limits(mut self, limits: Limits) -> Decoder { - self.limits = limits; - self - } - - pub fn dimensions(&mut self) -> TiffResult<(u32, u32)> { - Ok((self.image().width, self.image().height)) - } - - pub fn colortype(&mut self) -> TiffResult { - self.image().colortype() - } - - fn image(&self) -> &Image { - &self.image - } /// Loads the IFD at the specified index in the list, if one exists pub fn seek_to_image(&mut self, ifd_index: usize) -> TiffResult<()> { @@ -589,10 +719,6 @@ impl Decoder { Ok(()) } - /// Returns `true` if there is at least one more image available. - pub fn more_images(&self) -> bool { - self.next_ifd.is_some() - } /// Returns the byte_order pub fn byte_order(&self) -> ByteOrder { @@ -895,50 +1021,6 @@ impl Decoder { self.get_tag(tag)?.into_string() } - fn check_chunk_type(&self, expected: ChunkType) -> TiffResult<()> { - if expected != self.image().chunk_type { - return Err(TiffError::UsageError(UsageError::InvalidChunkType( - expected, - self.image().chunk_type, - ))); - } - - Ok(()) - } - - /// The chunk type (Strips / Tiles) of the image - pub fn get_chunk_type(&self) -> ChunkType { - self.image().chunk_type - } - - /// Number of strips in image - pub fn strip_count(&mut self) -> TiffResult { - self.check_chunk_type(ChunkType::Strip)?; - let rows_per_strip = self.image().strip_decoder.as_ref().unwrap().rows_per_strip; - - if rows_per_strip == 0 { - return Ok(0); - } - - // rows_per_strip - 1 can never fail since we know it's at least 1 - let height = match self.image().height.checked_add(rows_per_strip - 1) { - Some(h) => h, - None => return Err(TiffError::IntSizeError), - }; - - let strips = match self.image().planar_config { - PlanarConfiguration::Chunky => height / rows_per_strip, - PlanarConfiguration::Planar => height / rows_per_strip * self.image().samples as u32, - }; - - Ok(strips) - } - - /// Number of tiles in image - pub fn tile_count(&mut self) -> TiffResult { - self.check_chunk_type(ChunkType::Tile)?; - Ok(u32::try_from(self.image().chunk_offsets.len())?) - } pub fn read_chunk_to_buffer( &mut self, @@ -968,76 +1050,23 @@ impl Decoder { Ok(()) } - fn result_buffer(&self, width: usize, height: usize) -> TiffResult { - let buffer_size = match width - .checked_mul(height) - .and_then(|x| x.checked_mul(self.image().samples_per_pixel())) - { - Some(s) => s, - None => return Err(TiffError::LimitsExceeded), - }; - - let max_sample_bits = self.image().bits_per_sample; - match self.image().sample_format { - SampleFormat::Uint => match max_sample_bits { - n if n <= 8 => DecodingResult::new_u8(buffer_size, &self.limits), - n if n <= 16 => DecodingResult::new_u16(buffer_size, &self.limits), - n if n <= 32 => DecodingResult::new_u32(buffer_size, &self.limits), - n if n <= 64 => DecodingResult::new_u64(buffer_size, &self.limits), - n => Err(TiffError::UnsupportedError( - TiffUnsupportedError::UnsupportedBitsPerChannel(n), - )), - }, - SampleFormat::IEEEFP => match max_sample_bits { - 32 => DecodingResult::new_f32(buffer_size, &self.limits), - 64 => DecodingResult::new_f64(buffer_size, &self.limits), - n => Err(TiffError::UnsupportedError( - TiffUnsupportedError::UnsupportedBitsPerChannel(n), - )), - }, - SampleFormat::Int => match max_sample_bits { - n if n <= 8 => DecodingResult::new_i8(buffer_size, &self.limits), - n if n <= 16 => DecodingResult::new_i16(buffer_size, &self.limits), - n if n <= 32 => DecodingResult::new_i32(buffer_size, &self.limits), - n if n <= 64 => DecodingResult::new_i64(buffer_size, &self.limits), - n => Err(TiffError::UnsupportedError( - TiffUnsupportedError::UnsupportedBitsPerChannel(n), - )), - }, - format => Err(TiffUnsupportedError::UnsupportedSampleFormat(vec![format]).into()), - } - } /// Read the specified chunk (at index `chunk_index`) and return the binary data as a Vector. pub fn read_chunk(&mut self, chunk_index: u32) -> TiffResult { let data_dims = self.image().chunk_data_dimensions(chunk_index)?; - let mut result = self.result_buffer(data_dims.0 as usize, data_dims.1 as usize)?; + let mut result = Self::result_buffer(data_dims.0 as usize, data_dims.1 as usize, self.image(), &self.limits)?; self.read_chunk_to_buffer(result.as_buffer(0), chunk_index, data_dims.0 as usize)?; Ok(result) } - /// Returns the default chunk size for the current image. Any given chunk in the image is at most as large as - /// the value returned here. For the size of the data (chunk minus padding), use `chunk_data_dimensions`. - pub fn chunk_dimensions(&self) -> (u32, u32) { - self.image().chunk_dimensions().unwrap() - } - - /// Returns the size of the data in the chunk with the specified index. This is the default size of the chunk, - /// minus any padding. - pub fn chunk_data_dimensions(&self, chunk_index: u32) -> (u32, u32) { - self.image() - .chunk_data_dimensions(chunk_index) - .expect("invalid chunk_index") - } - /// Decodes the entire image and return it as a Vector pub fn read_image(&mut self) -> TiffResult { let width = self.image().width; let height = self.image().height; - let mut result = self.result_buffer(width as usize, height as usize)?; + let mut result = Self::result_buffer(width as usize, height as usize, self.image(), &self.limits)?; if width == 0 || height == 0 { return Ok(result); } diff --git a/src/decoder/multithread_decoder/mod.rs b/src/decoder/multithread_decoder/mod.rs new file mode 100644 index 0000000..ae91a60 --- /dev/null +++ b/src/decoder/multithread_decoder/mod.rs @@ -0,0 +1,68 @@ +//! Decoder that can be Cloned, sharing the [`Image`] data between threads +//! Therefore, it holds an `Arc` +//! Loading in the image meatadata should be done using another decoder +//! Also shows how terrificly easy and ergonomic the api for the folks over at geotiff would be :P +use std::sync::Arc; + +use futures::{AsyncRead, AsyncSeek}; + +use crate::decoder::{ + image::Image, + Decoder, Limits, DecodingResult, + stream::SmartReader, + async_decoder::RangeReader, +}; +use crate::TiffResult; +/// Decoder that can be Cloned, sharing the [`Image`] data between threads +#[derive(Clone, Debug)] +pub struct ChunkDecoder { + reader: SmartReader, + // bigtiff: bool, + limits: Limits, + image: Arc, +} + +impl Clone for SmartReader { + fn clone(&self) -> Self { + Self { + reader: self.reader.clone(), + byte_order: self.byte_order, + } + } +} + +impl ChunkDecoder{ + pub fn from_decoder(decoder: Decoder) -> Self { + ChunkDecoder { + reader: decoder.reader.clone(), + // bigtiff: decoder.bigtiff, + limits: decoder.limits.clone(), + image: Arc::new(decoder.image().clone()), + } + } + + /// Get a reference to the image (in read mode) + pub fn image(&self) -> &Image { + // this is really bad + &self.image//.read().expect("Could not obtain lock") + } + + pub async fn read_chunk_async(&mut self, chunk_index: u32) -> TiffResult{ + // read_chunk code + let (width, height) = self.image().chunk_data_dimensions(chunk_index)?; + let mut result = Decoder::::result_buffer(usize::try_from(width)?, usize::try_from(height)?, self.image(), &self.limits)?; + // read_chunk_to_buffer code + let (offset, length) = self.image().chunk_file_range(chunk_index)?; + let v = self.reader.read_range(offset, offset + length).await?; + let output_row_stride = (width as u64).saturating_mul(self.image().samples_per_pixel() as u64).saturating_mul(self.image.bits_per_sample as u64) / 8; + self.image().expand_chunk( + &mut std::io::Cursor::new(v), + result.as_buffer(0).as_bytes_mut(), + output_row_stride.try_into()?, + self.reader.byte_order, + chunk_index, + &self.limits, + )?; + Ok(result) + } +} diff --git a/src/decoder/stream.rs b/src/decoder/stream.rs index 8a995b0..0fe161d 100644 --- a/src/decoder/stream.rs +++ b/src/decoder/stream.rs @@ -2,6 +2,8 @@ use std::io::{self, BufRead, BufReader, Read, Seek, Take}; + + /// Byte order of the TIFF file. #[derive(Clone, Copy, Debug)] pub enum ByteOrder { @@ -11,109 +13,36 @@ pub enum ByteOrder { BigEndian, } +macro_rules! read_fn { + ($name:ident, $type:ty) => { + /// reads an $type + #[inline(always)] + fn $name(&mut self) -> Result<$type, io::Error> { + let mut n = [0u8; std::mem::size_of::<$type>()]; + self.read_exact(&mut n)?; + Ok(match self.byte_order() { + ByteOrder::LittleEndian => <$type>::from_le_bytes(n), + ByteOrder::BigEndian => <$type>::from_be_bytes(n), + }) + } + }; +} + + /// Reader that is aware of the byte order. pub trait EndianReader: Read { /// Byte order that should be adhered to fn byte_order(&self) -> ByteOrder; - - /// Reads an u16 - #[inline(always)] - fn read_u16(&mut self) -> Result { - let mut n = [0u8; 2]; - self.read_exact(&mut n)?; - Ok(match self.byte_order() { - ByteOrder::LittleEndian => u16::from_le_bytes(n), - ByteOrder::BigEndian => u16::from_be_bytes(n), - }) - } - - /// Reads an i8 - #[inline(always)] - fn read_i8(&mut self) -> Result { - let mut n = [0u8; 1]; - self.read_exact(&mut n)?; - Ok(match self.byte_order() { - ByteOrder::LittleEndian => i8::from_le_bytes(n), - ByteOrder::BigEndian => i8::from_be_bytes(n), - }) - } - - /// Reads an i16 - #[inline(always)] - fn read_i16(&mut self) -> Result { - let mut n = [0u8; 2]; - self.read_exact(&mut n)?; - Ok(match self.byte_order() { - ByteOrder::LittleEndian => i16::from_le_bytes(n), - ByteOrder::BigEndian => i16::from_be_bytes(n), - }) - } - - /// Reads an u32 - #[inline(always)] - fn read_u32(&mut self) -> Result { - let mut n = [0u8; 4]; - self.read_exact(&mut n)?; - Ok(match self.byte_order() { - ByteOrder::LittleEndian => u32::from_le_bytes(n), - ByteOrder::BigEndian => u32::from_be_bytes(n), - }) - } - - /// Reads an i32 - #[inline(always)] - fn read_i32(&mut self) -> Result { - let mut n = [0u8; 4]; - self.read_exact(&mut n)?; - Ok(match self.byte_order() { - ByteOrder::LittleEndian => i32::from_le_bytes(n), - ByteOrder::BigEndian => i32::from_be_bytes(n), - }) - } - - /// Reads an u64 - #[inline(always)] - fn read_u64(&mut self) -> Result { - let mut n = [0u8; 8]; - self.read_exact(&mut n)?; - Ok(match self.byte_order() { - ByteOrder::LittleEndian => u64::from_le_bytes(n), - ByteOrder::BigEndian => u64::from_be_bytes(n), - }) - } - - /// Reads an i64 - #[inline(always)] - fn read_i64(&mut self) -> Result { - let mut n = [0u8; 8]; - self.read_exact(&mut n)?; - Ok(match self.byte_order() { - ByteOrder::LittleEndian => i64::from_le_bytes(n), - ByteOrder::BigEndian => i64::from_be_bytes(n), - }) - } - - /// Reads an f32 - #[inline(always)] - fn read_f32(&mut self) -> Result { - let mut n = [0u8; 4]; - self.read_exact(&mut n)?; - Ok(f32::from_bits(match self.byte_order() { - ByteOrder::LittleEndian => u32::from_le_bytes(n), - ByteOrder::BigEndian => u32::from_be_bytes(n), - })) - } - - /// Reads an f64 - #[inline(always)] - fn read_f64(&mut self) -> Result { - let mut n = [0u8; 8]; - self.read_exact(&mut n)?; - Ok(f64::from_bits(match self.byte_order() { - ByteOrder::LittleEndian => u64::from_le_bytes(n), - ByteOrder::BigEndian => u64::from_be_bytes(n), - })) - } + + read_fn!(read_u16, u16); + read_fn!(read_i8, i8); + read_fn!(read_i16, i16); + read_fn!(read_u32, u32); + read_fn!(read_i32, i32); + read_fn!(read_u64, u64); + read_fn!(read_i64, i64); + read_fn!(read_f32, f32); + read_fn!(read_f64, f64); } /// @@ -259,16 +188,12 @@ impl Read for PackBitsReader { /// Reader that is aware of the byte order. #[derive(Debug)] pub struct SmartReader -where - R: Read, { - reader: R, + pub(super) reader: R, pub byte_order: ByteOrder, } impl SmartReader -where - R: Read, { /// Wraps a reader pub fn wrap(reader: R, byte_order: ByteOrder) -> SmartReader { @@ -284,9 +209,7 @@ impl SmartReader { } } -impl EndianReader for SmartReader -where - R: Read, +impl EndianReader for SmartReader { #[inline(always)] fn byte_order(&self) -> ByteOrder { @@ -294,6 +217,7 @@ where } } + impl Read for SmartReader { #[inline] fn read(&mut self, buf: &mut [u8]) -> io::Result { @@ -308,6 +232,7 @@ impl Seek for SmartReader { } } + #[cfg(test)] mod test { use super::*; diff --git a/src/tags.rs b/src/tags.rs index 6c18fa5..f130571 100644 --- a/src/tags.rs +++ b/src/tags.rs @@ -208,6 +208,10 @@ pub enum PlanarConfiguration(u16) { } tags! { +/// Additional compression methods: +/// - *None*: No predictor is used. This is the default mode, meaning the pixel values are stored without modification. +/// - *Horizontal Differencing*: For each pixel, the predictor calculates the difference between the current pixel and the previous pixel in the same row. +/// - *Floating Point Predictor*: Based on a similar concept as Predictor 2, but it is specific to floating-point pixel values. pub enum Predictor(u16) { /// No changes were made to the data None = 1, diff --git a/tests/decode_images_async.rs b/tests/decode_images_async.rs new file mode 100644 index 0000000..d971a25 --- /dev/null +++ b/tests/decode_images_async.rs @@ -0,0 +1,554 @@ +#[cfg(feature="async_decoder")] +mod test_async +{ +extern crate tiff; + +use tiff::decoder::DecodingResult; +use tiff::decoder::Decoder; +use tiff::ColorType; + +use futures::io::AllowStdIo; + +use std::fs::File; +use std::io::Cursor; +use std::path::PathBuf; + +const TEST_IMAGE_DIR: &str = "./tests/images/"; + +macro_rules! test_image_sum { + ($name:ident, $buffer:ident, $sum_ty:ty) => { + async fn $name(file: &str, expected_type: ColorType, expected_sum: $sum_ty) { + let path = PathBuf::from(TEST_IMAGE_DIR).join(file); + let img_file = AllowStdIo::new(File::open(path).expect("Cannot find test image!")); + let mut decoder = Decoder::new_async(img_file) + .await + .expect("Cannot create decoder"); + assert_eq!(decoder.colortype().unwrap(), expected_type); + let img_res = decoder.read_image_async().await.unwrap(); + + match img_res { + DecodingResult::$buffer(res) => { + let sum: $sum_ty = res.into_iter().map(<$sum_ty>::from).sum(); + assert_eq!(sum, expected_sum); + } + _ => panic!("Wrong bit depth"), + } + } + }; +} + +test_image_sum!(test_image_sum_u8, U8, u64); +test_image_sum!(test_image_sum_i8, I8, i64); +test_image_sum!(test_image_sum_u16, U16, u64); +test_image_sum!(test_image_sum_i16, I16, i64); +test_image_sum!(test_image_sum_u32, U32, u64); +test_image_sum!(test_image_sum_u64, U64, u64); +test_image_sum!(test_image_sum_f32, F32, f32); +test_image_sum!(test_image_sum_f64, F64, f64); + +/// Tests that a decoder can be constructed for an image and the color type +/// read from the IFD and is of the appropriate type, but the type is +/// unsupported. +async fn test_image_color_type_unsupported(file: &str, expected_type: ColorType) { + let path = PathBuf::from(TEST_IMAGE_DIR).join(file); + let img_file = AllowStdIo::new(File::open(path).expect("Cannot find test image!")); + let mut decoder = Decoder::new_async(img_file) + .await + .expect("Cannot create decoder"); + assert_eq!(decoder.colortype().unwrap(), expected_type); + assert!(match decoder.read_image_async().await { + Err(tiff::TiffError::UnsupportedError( + tiff::TiffUnsupportedError::UnsupportedColorType(_), + )) => true, + _ => false, + }); +} + +#[tokio::test] +async fn test_cmyk_u8() { + test_image_sum_u8("cmyk-3c-8b.tiff", ColorType::CMYK(8), 8522658).await; +} + +#[tokio::test] +async fn test_cmyk_u16() { + test_image_sum_u16("cmyk-3c-16b.tiff", ColorType::CMYK(16), 2181426827).await; +} + +#[tokio::test] +async fn test_cmyk_f32() { + test_image_sum_f32("cmyk-3c-32b-float.tiff", ColorType::CMYK(32), 496.0405).await; +} + +#[tokio::test] +async fn test_gray_u8() { + test_image_sum_u8("minisblack-1c-8b.tiff", ColorType::Gray(8), 2840893).await; +} + +#[tokio::test] +async fn test_gray_u12() { + test_image_color_type_unsupported("12bit.cropped.tiff", ColorType::Gray(12)).await; +} + +#[tokio::test] +async fn test_gray_u16() { + test_image_sum_u16("minisblack-1c-16b.tiff", ColorType::Gray(16), 733126239).await; +} + +#[tokio::test] +async fn test_gray_u32() { + test_image_sum_u32("gradient-1c-32b.tiff", ColorType::Gray(32), 549892913787).await; +} + +#[tokio::test] +async fn test_gray_u64() { + test_image_sum_u64("gradient-1c-64b.tiff", ColorType::Gray(64), 549892913787).await; +} + +#[tokio::test] +async fn test_gray_f32() { + test_image_sum_f32("gradient-1c-32b-float.tiff", ColorType::Gray(32), 128.03194).await; +} + +#[tokio::test] +async fn test_gray_f64() { + test_image_sum_f64( + "gradient-1c-64b-float.tiff", + ColorType::Gray(64), + 128.0319210877642, + ) + .await; +} + +#[tokio::test] +async fn test_rgb_u8() { + test_image_sum_u8("rgb-3c-8b.tiff", ColorType::RGB(8), 7842108).await; +} + +#[tokio::test] +async fn test_rgb_u12() { + test_image_color_type_unsupported("12bit.cropped.rgb.tiff", ColorType::RGB(12)).await; +} + +#[tokio::test] +async fn test_rgb_u16() { + test_image_sum_u16("rgb-3c-16b.tiff", ColorType::RGB(16), 2024349944).await; +} + +#[tokio::test] +async fn test_rgb_u32() { + test_image_sum_u32("gradient-3c-32b.tiff", ColorType::RGB(32), 2030834111716).await; +} + +#[tokio::test] +async fn test_rgb_u64() { + test_image_sum_u64("gradient-3c-64b.tiff", ColorType::RGB(64), 2030834111716).await; +} + +#[tokio::test] +async fn test_rgb_f32() { + test_image_sum_f32("gradient-3c-32b-float.tiff", ColorType::RGB(32), 472.8405).await; +} + +#[tokio::test] +async fn test_int8() { + test_image_sum_i8("int8.tif", ColorType::Gray(8), 3111).await; +} + +#[tokio::test] +async fn test_int8_rgb() { + test_image_sum_i8("int8_rgb.tif", ColorType::RGB(8), -10344).await; +} + +#[tokio::test] +async fn test_int16() { + test_image_sum_i16("int16.tif", ColorType::Gray(16), 354396).await; +} + +#[tokio::test] +async fn test_int16_rgb() { + test_image_sum_i16("int16_rgb.tif", ColorType::RGB(16), 1063188).await; +} + +#[tokio::test] +async fn test_string_tags() { + // these files have null-terminated strings for their Software tag. One has extra bytes after + // the null byte, so we check both to ensure that we're truncating properly + let filenames = ["minisblack-1c-16b.tiff", "rgb-3c-16b.tiff"]; + for filename in filenames.iter() { + let path = PathBuf::from(TEST_IMAGE_DIR).join(filename); + let img_file = AllowStdIo::new(File::open(path).expect("Cannot find test image!")); + let mut decoder = Decoder::new_async(img_file) + .await + .expect("Cannot create decoder"); + let software = decoder.get_tag_async(tiff::tags::Tag::Software).await.unwrap(); + match software { + tiff::decoder::ifd::Value::Ascii(s) => assert_eq!( + &s, + "GraphicsMagick 1.2 unreleased Q16 http://www.GraphicsMagick.org/" + ), + _ => assert!(false), + }; + } +} + +#[tokio::test] +async fn test_decode_data() { + let mut image_data = Vec::new(); + for x in 0..100 { + for y in 0..100u8 { + let val = x + y; + image_data.push(val); + image_data.push(val); + image_data.push(val); + } + } + let file = AllowStdIo::new(File::open("./tests/decodedata-rgb-3c-8b.tiff").unwrap()); + let mut decoder = Decoder::new_async(file).await.unwrap(); + assert_eq!(decoder.colortype().unwrap(), ColorType::RGB(8)); + assert_eq!(decoder.dimensions().unwrap(), (100, 100)); + if let DecodingResult::U8(img_res) = decoder.read_image_async().await.unwrap() { + assert_eq!(image_data, img_res); + } else { + panic!("Wrong data type"); + } +} + +#[tokio::test] +async fn issue_69() { + test_image_sum_u16("issue_69_lzw.tiff", ColorType::Gray(16), 1015486).await; + // test_image_sum_u16("issue_69_packbits.tiff", ColorType::Gray(16), 1015486).await; +} + +// TODO: GrayA support +//#[tokio::test] +//fn test_gray_alpha_u8() +//{ +//let img_file = File::open("./tests/images/minisblack-2c-8b-alpha.tiff").expect("Cannot find test image!"); +//let mut decoder = Decoder::new_async(img_file).expect("Cannot create decoder"); +//assert_eq!(decoder.colortype().unwrap(), ColorType::GrayA(8)); +//let img_res = decoder.read_image_async(); +//assert!(img_res.is_ok()); +//} + +#[tokio::test] +async fn test_tiled_rgb_u8() { + test_image_sum_u8("tiled-rgb-u8.tif", ColorType::RGB(8), 39528948).await; +} + +#[tokio::test] +async fn test_tiled_rect_rgb_u8() { + test_image_sum_u8("tiled-rect-rgb-u8.tif", ColorType::RGB(8), 62081032).await; +} + +/* #[tokio::test] +async fn test_tiled_jpeg_rgb_u8() { + test_image_sum_u8("tiled-jpeg-rgb-u8.tif", ColorType::RGB(8), 93031606); +} */ +#[tokio::test] +async fn test_tiled_oversize_gray_i8() { + test_image_sum_i8("tiled-oversize-gray-i8.tif", ColorType::Gray(8), 1214996).await; +} + +#[tokio::test] +async fn test_tiled_cmyk_i8() { + test_image_sum_i8("tiled-cmyk-i8.tif", ColorType::CMYK(8), 1759101).await +} + +#[tokio::test] +async fn test_tiled_incremental() { + let file = "tiled-rgb-u8.tif"; + let expected_type = ColorType::RGB(8); + let sums = [ + 188760, 195639, 108148, 81986, 665088, 366140, 705317, 423366, 172033, 324455, 244102, + 81853, 181258, 247971, 129486, 55600, 565625, 422102, 730888, 379271, 232142, 292549, + 244045, 86866, 188141, 115036, 150785, 84389, 353170, 459325, 719619, 329594, 278663, + 220474, 243048, 113563, 189152, 109684, 179391, 122188, 279651, 622093, 724682, 302459, + 268428, 204499, 224255, 124674, 170668, 121868, 192768, 183367, 378029, 585651, 657712, + 296790, 241444, 197083, 198429, 134869, 182318, 86034, 203655, 182338, 297255, 601284, + 633813, 242531, 228578, 206441, 193552, 125412, 181527, 165439, 202531, 159538, 268388, + 565790, 611382, 272967, 236497, 215154, 158881, 90806, 106114, 182342, 191824, 186138, + 215174, 393193, 701228, 198866, 227944, 193830, 166330, 49008, 55719, 122820, 197316, + 161969, 203152, 170986, 624427, 188605, 186187, 111064, 115192, 39538, 48626, 163929, + 144682, 135796, 194141, 154198, 584125, 180255, 153524, 121433, 132641, 35743, 47798, + 152343, 162874, 167664, 160175, 133038, 659882, 138339, 166470, 124173, 118929, 51317, + 45267, 155776, 161331, 161006, 130052, 137618, 337291, 106481, 161999, 127343, 87724, + 59540, 63907, 155677, 140668, 141523, 108061, 168657, 186482, 98599, 147614, 139963, 90444, + 56602, 92547, 125644, 134212, 126569, 144153, 179800, 174516, 133969, 129399, 117681, + 83305, 55075, 110737, 115108, 128572, 128911, 130922, 179986, 143288, 145884, 155856, + 96683, 94057, 56238, 79649, 71651, 70182, 75010, 77009, 98855, 78979, 74341, 83482, 53403, + 59842, 30305, + ]; + + let path = PathBuf::from(TEST_IMAGE_DIR).join(file); + let img_file = AllowStdIo::new(File::open(path).expect("Cannot find test image!")); + let mut decoder = Decoder::new_async(img_file) + .await + .expect("Cannot create decoder"); + assert_eq!(decoder.colortype().unwrap(), expected_type); + + let tiles = decoder.tile_count().unwrap(); + assert_eq!(tiles as usize, sums.len()); + + for tile in 0..tiles { + match decoder.read_chunk_async(tile).await.unwrap() { + DecodingResult::U8(res) => { + let sum: u64 = res.into_iter().map(::from).sum(); + assert_eq!(sum, sums[tile as usize]); + } + _ => panic!("Wrong bit depth"), + } + } +} + +#[tokio::test] +async fn test_planar_rgb_u8() { + // gdal_translate tiled-rgb-u8.tif planar-rgb-u8.tif -co INTERLEAVE=BAND -co COMPRESS=LZW -co PROFILE=BASELINE + let file = "planar-rgb-u8.tif"; + let expected_type = ColorType::RGB(8); + + let path = PathBuf::from(TEST_IMAGE_DIR).join(file); + let img_file = AllowStdIo::new(File::open(path).expect("Cannot find test image!")); + let mut decoder = Decoder::new_async(img_file) + .await + .expect("Cannot create decoder"); + assert_eq!(decoder.colortype().unwrap(), expected_type); + + let chunks = decoder.strip_count().unwrap(); + assert_eq!(chunks as usize, 72); + + // convert -quiet planar-rgb-u8.tif[0] -crop 1x1+0+0 txt: + // 0,0: (73,51,30) #49331E srgb(73,51,30) + + // 1st band (red) + match decoder.read_chunk_async(0).await.unwrap() { + DecodingResult::U8(chunk) => { + assert_eq!(chunk[0], 73); + } + _ => panic!("Wrong bit depth"), + } + // 2nd band (green) + match decoder.read_chunk_async(chunks / 3).await.unwrap() { + DecodingResult::U8(chunk) => { + assert_eq!(chunk[0], 51); + } + _ => panic!("Wrong bit depth"), + } + // 3rd band (blue) + match decoder.read_chunk_async(chunks / 3 * 2).await.unwrap() { + DecodingResult::U8(chunk) => { + assert_eq!(chunk[0], 30); + } + _ => panic!("Wrong bit depth"), + } + + test_image_sum_u8(file, ColorType::RGB(8), 15417630).await; +} + +#[tokio::test] +async fn test_div_zero() { + use tiff::{TiffError, TiffFormatError}; + + let image = [ + 73, 73, 42, 0, 8, 0, 0, 0, 8, 0, 0, 1, 4, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 40, 1, 0, 0, + 0, 158, 0, 0, 251, 3, 1, 3, 0, 1, 0, 0, 0, 1, 0, 0, 39, 6, 1, 3, 0, 1, 0, 0, 0, 0, 0, 0, 0, + 17, 1, 4, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 1, 3, 0, 1, 0, 0, 0, 158, 0, 0, 251, 67, 1, 3, 0, + 1, 0, 0, 0, 40, 0, 0, 0, 66, 1, 4, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 178, 178, 178, 178, + 178, 178, 178, + ]; + + let err = Decoder::new_async(AllowStdIo::new(Cursor::new(&image))) + .await + .unwrap_err(); + + match err { + TiffError::FormatError(TiffFormatError::StripTileTagConflict) => {} + unexpected => panic!("Unexpected error {}", unexpected), + } +} + +#[tokio::test] +async fn test_too_many_value_bytes() { + let image = [ + 73, 73, 43, 0, 8, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 255, 0, 8, 0, 0, 0, + 23, 0, 12, 0, 0, 65, 4, 0, 1, 6, 0, 0, 1, 16, 0, 1, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, + 0, 0, 3, 0, 1, 0, 0, 0, 1, 0, 0, 0, 59, 73, 84, 186, 202, 83, 240, 66, 1, 53, 22, 56, 47, + 0, 0, 0, 0, 0, 0, 1, 222, 4, 0, 58, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 1, 1, 4, 0, 0, 100, 0, + 0, 89, 89, 89, 89, 89, 89, 89, 89, 96, 1, 20, 89, 89, 89, 89, 18, + ]; + + let error = Decoder::new_async(AllowStdIo::new(std::io::Cursor::new(&image))) + .await + .unwrap_err(); + + match error { + tiff::TiffError::LimitsExceeded => {} + unexpected => panic!("Unexpected error {}", unexpected), + } +} + +#[tokio::test] +async fn fuzzer_testcase5() { + let image = [ + 73, 73, 42, 0, 8, 0, 0, 0, 8, 0, 0, 1, 4, 0, 1, 0, 0, 0, 100, 0, 0, 0, 1, 1, 4, 0, 1, 0, 0, + 0, 158, 0, 0, 251, 3, 1, 3, 0, 1, 0, 0, 0, 1, 0, 0, 0, 6, 1, 3, 0, 1, 0, 0, 0, 0, 0, 0, 0, + 17, 1, 4, 0, 1, 0, 0, 0, 0, 0, 0, 0, 2, 1, 3, 0, 0, 0, 0, 0, 246, 16, 0, 0, 22, 1, 4, 0, 1, + 0, 0, 0, 40, 0, 251, 255, 23, 1, 4, 0, 1, 0, 0, 0, 48, 178, 178, 178, 178, 178, 178, 178, + 178, 178, 178, + ]; + + let _ = Decoder::new_async(AllowStdIo::new(Cursor::new(&image))) + .await + .unwrap_err(); +} + +#[tokio::test] +async fn fuzzer_testcase1() { + let image = [ + 73, 73, 42, 0, 8, 0, 0, 0, 8, 0, 0, 1, 4, 0, 1, 0, 0, 0, 99, 255, 255, 254, 1, 1, 4, 0, 1, + 0, 0, 0, 158, 0, 0, 251, 3, 1, 3, 255, 254, 255, 255, 0, 1, 0, 0, 0, 6, 1, 3, 0, 1, 0, 0, + 0, 0, 0, 0, 0, 17, 1, 4, 0, 9, 0, 0, 0, 0, 0, 0, 0, 2, 1, 3, 0, 2, 0, 0, 0, 63, 0, 0, 0, + 22, 1, 4, 0, 1, 0, 0, 0, 44, 0, 0, 0, 23, 1, 4, 0, 0, 0, 0, 0, 0, 0, 2, 1, 3, 1, 0, 178, + 178, + ]; + + let _ = Decoder::new_async(AllowStdIo::new(Cursor::new(&image))) + .await + .unwrap_err(); +} + +#[tokio::test] +async fn fuzzer_testcase6() { + let image = [ + 73, 73, 42, 0, 8, 0, 0, 0, 8, 0, 0, 1, 4, 0, 1, 0, 0, 0, 100, 0, 0, 148, 1, 1, 4, 0, 1, 0, + 0, 0, 158, 0, 0, 251, 3, 1, 3, 255, 254, 255, 255, 0, 1, 0, 0, 0, 6, 1, 3, 0, 1, 0, 0, 0, + 0, 0, 0, 0, 17, 1, 4, 0, 1, 0, 0, 0, 0, 0, 0, 0, 2, 1, 3, 0, 2, 0, 0, 0, 63, 0, 0, 0, 22, + 1, 4, 0, 1, 0, 0, 0, 44, 0, 248, 255, 23, 1, 4, 0, 1, 0, 0, 0, 178, 178, 178, 0, 1, 178, + 178, 178, + ]; + + let _ = Decoder::new_async(AllowStdIo::new(Cursor::new(&image))) + .await + .unwrap_err(); +} + +#[tokio::test] +async fn oom() { + let image = [ + 73, 73, 42, 0, 8, 0, 0, 0, 8, 0, 0, 1, 4, 0, 1, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 40, 1, 0, 0, + 0, 158, 0, 0, 251, 3, 1, 3, 0, 1, 0, 0, 0, 7, 0, 0, 0, 6, 1, 3, 0, 1, 0, 0, 0, 2, 0, 0, 0, + 17, 1, 4, 0, 1, 0, 0, 0, 3, 77, 0, 0, 1, 1, 3, 0, 1, 0, 0, 0, 3, 128, 0, 0, 22, 1, 4, 0, 1, + 0, 0, 0, 40, 0, 0, 0, 23, 1, 4, 0, 1, 0, 0, 0, 178, 48, 178, 178, 178, 178, 162, 178, + ]; + + let _ = Decoder::new_async(AllowStdIo::new(Cursor::new(&image))) + .await + .unwrap_err(); +} + +#[tokio::test] +async fn fuzzer_testcase4() { + let image = [ + 73, 73, 42, 0, 8, 0, 0, 0, 8, 0, 0, 1, 4, 0, 1, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 40, 1, 0, 0, + 0, 158, 0, 0, 251, 3, 1, 3, 0, 1, 0, 0, 0, 5, 0, 0, 0, 6, 1, 3, 0, 1, 0, 0, 0, 0, 0, 0, 0, + 17, 1, 4, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 1, 3, 0, 1, 0, 0, 0, 3, 128, 0, 0, 22, 1, 4, 0, 1, + 0, 0, 0, 40, 0, 0, 0, 23, 1, 4, 0, 1, 0, 0, 0, 48, 178, 178, 178, 0, 1, 0, 13, 13, + ]; + + let _ = Decoder::new_async(AllowStdIo::new(Cursor::new(&image))) + .await + .unwrap_err(); +} + +#[tokio::test] +async fn fuzzer_testcase2() { + let image = [ + 73, 73, 42, 0, 8, 0, 0, 0, 15, 0, 0, 254, 44, 1, 0, 0, 0, 0, 0, 32, 0, 0, 0, 1, 4, 0, 1, 0, + 0, 0, 0, 1, 0, 0, 91, 1, 1, 0, 0, 0, 0, 0, 242, 4, 0, 0, 0, 22, 0, 56, 77, 0, 77, 1, 0, 0, + 73, 42, 0, 1, 4, 0, 1, 0, 0, 0, 4, 0, 8, 0, 0, 1, 4, 0, 1, 0, 0, 0, 158, 0, 0, 251, 3, 1, + 3, 0, 1, 0, 0, 0, 7, 0, 0, 0, 6, 1, 3, 0, 1, 0, 0, 0, 2, 0, 0, 0, 17, 1, 4, 0, 1, 0, 0, 0, + 0, 0, 0, 0, 1, 1, 3, 0, 1, 0, 0, 0, 0, 0, 0, 4, 61, 1, 18, 0, 1, 0, 0, 0, 202, 0, 0, 0, 17, + 1, 100, 0, 129, 0, 0, 0, 0, 0, 0, 0, 232, 254, 252, 255, 254, 255, 255, 255, 1, 29, 0, 0, + 22, 1, 3, 0, 1, 0, 0, 0, 16, 0, 0, 0, 23, 1, 1, 0, 1, 0, 0, 0, 0, 0, 0, 123, 73, 254, 0, + 73, + ]; + + let _ = Decoder::new_async(AllowStdIo::new(Cursor::new(&image))) + .await + .unwrap_err(); +} + +#[tokio::test] +async fn invalid_jpeg_tag_2() { + let image = [ + 73, 73, 42, 0, 8, 0, 0, 0, 16, 0, 254, 0, 4, 0, 1, 0, 0, 0, 0, 0, 0, 242, 0, 1, 4, 0, 1, 0, + 0, 0, 0, 129, 16, 0, 1, 1, 4, 0, 1, 0, 0, 0, 214, 0, 0, 248, 253, 1, 3, 0, 1, 0, 0, 0, 64, + 0, 0, 0, 3, 1, 3, 0, 1, 0, 0, 0, 7, 0, 0, 0, 6, 1, 3, 0, 1, 0, 0, 0, 1, 0, 0, 64, 14, 1, 0, + 2, 0, 0, 148, 0, 206, 0, 0, 0, 17, 1, 4, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, + 1, 0, 0, 0, 22, 1, 4, 0, 17, 0, 0, 201, 1, 0, 0, 0, 23, 1, 2, 0, 20, 0, 0, 0, 194, 0, 0, 0, + 91, 1, 7, 0, 5, 0, 0, 0, 64, 0, 0, 0, 237, 254, 65, 255, 255, 255, 255, 255, 1, 0, 0, 0, + 22, 1, 4, 0, 1, 0, 0, 0, 42, 0, 0, 0, 23, 1, 255, 255, 255, 255, 255, 36, 36, 0, 0, 0, 0, + 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 36, 73, 73, 0, 42, 36, 36, 36, 36, 0, 0, 8, 0, + ]; + + let _ = Decoder::new_async(AllowStdIo::new(Cursor::new(&image))) + .await + .unwrap_err(); +} + +#[tokio::test] +async fn fuzzer_testcase3() { + let image = [ + 73, 73, 42, 0, 8, 0, 0, 0, 8, 0, 0, 1, 4, 0, 1, 0, 0, 0, 2, 0, 0, 0, 61, 1, 9, 0, 46, 22, + 128, 0, 0, 0, 0, 1, 6, 1, 3, 0, 1, 0, 0, 0, 0, 0, 0, 0, 17, 1, 4, 0, 27, 0, 0, 0, 0, 0, 0, + 0, 1, 1, 3, 0, 1, 0, 0, 0, 17, 1, 0, 231, 22, 1, 1, 0, 1, 0, 0, 0, 130, 0, 0, 0, 23, 1, 4, + 0, 14, 0, 0, 0, 0, 0, 0, 0, 133, 133, 133, 77, 77, 77, 0, 0, 22, 128, 0, 255, 255, 255, + 255, 255, + ]; + + let _ = Decoder::new_async(AllowStdIo::new(Cursor::new(&image))) + .await + .unwrap_err(); +} + +#[tokio::test] +async fn timeout() { + use tiff::{TiffError, TiffFormatError}; + + let image = [ + 73, 73, 42, 0, 8, 0, 0, 0, 16, 0, 254, 0, 4, 0, 1, 68, 0, 0, 0, 2, 0, 32, 254, 252, 0, 109, + 0, 129, 0, 0, 0, 32, 0, 58, 0, 1, 4, 0, 1, 0, 6, 0, 0, 0, 8, 0, 0, 1, 73, 73, 42, 0, 8, 0, + 0, 0, 8, 0, 0, 1, 4, 0, 1, 0, 0, 0, 21, 0, 0, 0, 61, 1, 255, 128, 9, 0, 0, 8, 0, 1, 113, 2, + 3, 1, 3, 0, 1, 0, 0, 0, 5, 0, 65, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 112, 0, 0, 36, 0, 0, + 0, 112, 56, 200, 0, 5, 0, 0, 64, 0, 0, 1, 0, 4, 0, 0, 0, 2, 0, 6, 1, 3, 0, 1, 0, 0, 0, 0, + 0, 0, 4, 17, 1, 1, 0, 93, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 1, 0, 0, 0, 3, 6, 0, 231, 22, 1, + 1, 0, 1, 0, 0, 0, 2, 64, 118, 36, 23, 1, 1, 0, 43, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4, 0, 8, + 0, 0, 73, 73, 42, 0, 8, 0, 0, 0, 0, 0, 32, + ]; + + // hacked the test a bit: seek until we are sure that we loop, + // otherwise it will read the faulty image before finding out it is circular + let error = Decoder::new_overview_async(AllowStdIo::new(Cursor::new(&image)), 5) + .await + .unwrap_err(); + + match error { + TiffError::FormatError(TiffFormatError::CycleInOffsets) => {} + e => panic!("Unexpected error {:?}", e), + } +} + +#[tokio::test] +async fn test_no_rows_per_strip() { + test_image_sum_u8("no_rows_per_strip.tiff", ColorType::RGB(8), 99448840).await; +} + +#[tokio::test] +async fn test_predictor_3_rgb_f32() { + test_image_sum_f32("predictor-3-rgb-f32.tif", ColorType::RGB(32), 54004.33).await; +} + +#[tokio::test] +async fn test_predictor_3_gray_f32() { + test_image_sum_f32("predictor-3-gray-f32.tif", ColorType::Gray(32), 20008.275).await; +} +} \ No newline at end of file