From d30d3b9b22861f785aaeb2aef21973003456f95c Mon Sep 17 00:00:00 2001 From: Fee Fladder Date: Fri, 13 Sep 2024 23:56:17 +0200 Subject: [PATCH 1/6] feat:async implemented async --- Cargo.toml | 5 + examples/async_http.rs | 2 + src/decoder/ifd.rs | 4 +- src/decoder/image.rs | 2 +- src/decoder/mod.rs | 34 +- src/decoder/tag_reader.rs | 2 + src/decoder_async/ifd.rs | 535 +++++++++++++++++++++ src/decoder_async/image.rs | 807 ++++++++++++++++++++++++++++++++ src/decoder_async/mod.rs | 638 +++++++++++++++++++++++++ src/decoder_async/readme.md | 32 ++ src/decoder_async/stream.rs | 353 ++++++++++++++ src/decoder_async/tag_reader.rs | 48 ++ src/lib.rs | 1 + src/tags.rs | 4 + tests/decode_images_async.rs | 514 ++++++++++++++++++++ 15 files changed, 2961 insertions(+), 20 deletions(-) create mode 100644 examples/async_http.rs create mode 100644 src/decoder_async/ifd.rs create mode 100644 src/decoder_async/image.rs create mode 100644 src/decoder_async/mod.rs create mode 100644 src/decoder_async/readme.md create mode 100644 src/decoder_async/stream.rs create mode 100644 src/decoder_async/tag_reader.rs create mode 100644 tests/decode_images_async.rs diff --git a/Cargo.toml b/Cargo.toml index baf432fd..9155aa64 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -20,9 +20,14 @@ exclude = ["tests/images/*", "tests/fuzz_images/*"] weezl = "0.1.0" jpeg = { package = "jpeg-decoder", version = "0.3.0", default-features = false } flate2 = "1.0.20" +futures = "0.3.30" +tokio-util = { version = "0.7.10", features = ["compat"] } +async-trait = "0.1.82" [dev-dependencies] criterion = "0.3.1" +# use rt since most is io-bound +tokio = {version = "1.29.1", features = ["macros", "fs", "rt-multi-thread"] } [[bench]] name = "lzw" diff --git a/examples/async_http.rs b/examples/async_http.rs new file mode 100644 index 00000000..7f755fb7 --- /dev/null +++ b/examples/async_http.rs @@ -0,0 +1,2 @@ +#[tokio::main] +async fn main() {} diff --git a/src/decoder/ifd.rs b/src/decoder/ifd.rs index 9be0d354..e2e9d2ee 100644 --- a/src/decoder/ifd.rs +++ b/src/decoder/ifd.rs @@ -675,7 +675,7 @@ impl Entry { /// Extracts a list of BYTE tags stored in an offset #[inline] -fn offset_to_bytes(n: usize, entry: &Entry) -> TiffResult { +pub fn offset_to_bytes(n: usize, entry: &Entry) -> TiffResult { Ok(List( entry.offset[0..n] .iter() @@ -686,7 +686,7 @@ fn offset_to_bytes(n: usize, entry: &Entry) -> TiffResult { /// Extracts a list of SBYTE tags stored in an offset #[inline] -fn offset_to_sbytes(n: usize, entry: &Entry) -> TiffResult { +pub fn offset_to_sbytes(n: usize, entry: &Entry) -> TiffResult { Ok(List( entry.offset[0..n] .iter() diff --git a/src/decoder/image.rs b/src/decoder/image.rs index 97658f45..59667ab9 100644 --- a/src/decoder/image.rs +++ b/src/decoder/image.rs @@ -164,7 +164,7 @@ impl Image { // Technically bits_per_sample.len() should be *equal* to samples, but libtiff also allows // it to be a single value that applies to all samples. - if bits_per_sample.len() != samples.into() && bits_per_sample.len() != 1 { + if bits_per_sample.len() != samples as usize && bits_per_sample.len() != 1 { return Err(TiffError::FormatError( TiffFormatError::InconsistentSizesEncountered, )); diff --git a/src/decoder/mod.rs b/src/decoder/mod.rs index 09d530bc..064c146d 100644 --- a/src/decoder/mod.rs +++ b/src/decoder/mod.rs @@ -15,8 +15,8 @@ use crate::tags::{ use self::stream::{ByteOrder, EndianReader, SmartReader}; pub mod ifd; -mod image; -mod stream; +pub mod image; +pub mod stream; mod tag_reader; /// Result of a decoding process @@ -45,7 +45,7 @@ pub enum DecodingResult { } impl DecodingResult { - fn new_u8(size: usize, limits: &Limits) -> TiffResult { + pub fn new_u8(size: usize, limits: &Limits) -> TiffResult { if size > limits.decoding_buffer_size { Err(TiffError::LimitsExceeded) } else { @@ -53,7 +53,7 @@ impl DecodingResult { } } - fn new_u16(size: usize, limits: &Limits) -> TiffResult { + pub fn new_u16(size: usize, limits: &Limits) -> TiffResult { if size > limits.decoding_buffer_size / 2 { Err(TiffError::LimitsExceeded) } else { @@ -61,7 +61,7 @@ impl DecodingResult { } } - fn new_u32(size: usize, limits: &Limits) -> TiffResult { + pub fn new_u32(size: usize, limits: &Limits) -> TiffResult { if size > limits.decoding_buffer_size / 4 { Err(TiffError::LimitsExceeded) } else { @@ -69,7 +69,7 @@ impl DecodingResult { } } - fn new_u64(size: usize, limits: &Limits) -> TiffResult { + pub fn new_u64(size: usize, limits: &Limits) -> TiffResult { if size > limits.decoding_buffer_size / 8 { Err(TiffError::LimitsExceeded) } else { @@ -77,7 +77,7 @@ impl DecodingResult { } } - fn new_f32(size: usize, limits: &Limits) -> TiffResult { + pub fn new_f32(size: usize, limits: &Limits) -> TiffResult { if size > limits.decoding_buffer_size / std::mem::size_of::() { Err(TiffError::LimitsExceeded) } else { @@ -85,7 +85,7 @@ impl DecodingResult { } } - fn new_f64(size: usize, limits: &Limits) -> TiffResult { + pub fn new_f64(size: usize, limits: &Limits) -> TiffResult { if size > limits.decoding_buffer_size / std::mem::size_of::() { Err(TiffError::LimitsExceeded) } else { @@ -93,7 +93,7 @@ impl DecodingResult { } } - fn new_i8(size: usize, limits: &Limits) -> TiffResult { + pub fn new_i8(size: usize, limits: &Limits) -> TiffResult { if size > limits.decoding_buffer_size / std::mem::size_of::() { Err(TiffError::LimitsExceeded) } else { @@ -101,7 +101,7 @@ impl DecodingResult { } } - fn new_i16(size: usize, limits: &Limits) -> TiffResult { + pub fn new_i16(size: usize, limits: &Limits) -> TiffResult { if size > limits.decoding_buffer_size / 2 { Err(TiffError::LimitsExceeded) } else { @@ -109,7 +109,7 @@ impl DecodingResult { } } - fn new_i32(size: usize, limits: &Limits) -> TiffResult { + pub fn new_i32(size: usize, limits: &Limits) -> TiffResult { if size > limits.decoding_buffer_size / 4 { Err(TiffError::LimitsExceeded) } else { @@ -117,7 +117,7 @@ impl DecodingResult { } } - fn new_i64(size: usize, limits: &Limits) -> TiffResult { + pub fn new_i64(size: usize, limits: &Limits) -> TiffResult { if size > limits.decoding_buffer_size / 8 { Err(TiffError::LimitsExceeded) } else { @@ -166,7 +166,7 @@ pub enum DecodingBuffer<'a> { } impl<'a> DecodingBuffer<'a> { - fn as_bytes_mut(&mut self) -> &mut [u8] { + pub fn as_bytes_mut(&mut self) -> &mut [u8] { match self { DecodingBuffer::U8(ref mut buf) => buf, DecodingBuffer::I8(buf) => bytecast::i8_as_ne_mut_bytes(buf), @@ -289,7 +289,7 @@ fn rev_hpredict_nsamp(buf: &mut [u8], bit_depth: u8, samples: usize) { } } -fn predict_f32(input: &mut [u8], output: &mut [u8], samples: usize) { +pub fn predict_f32(input: &mut [u8], output: &mut [u8], samples: usize) { for i in samples..input.len() { input[i] = input[i].wrapping_add(input[i - samples]); } @@ -304,7 +304,7 @@ fn predict_f32(input: &mut [u8], output: &mut [u8], samples: usize) { } } -fn predict_f64(input: &mut [u8], output: &mut [u8], samples: usize) { +pub fn predict_f64(input: &mut [u8], output: &mut [u8], samples: usize) { for i in samples..input.len() { input[i] = input[i].wrapping_add(input[i - samples]); } @@ -323,7 +323,7 @@ fn predict_f64(input: &mut [u8], output: &mut [u8], samples: usize) { } } -fn fix_endianness_and_predict( +pub fn fix_endianness_and_predict( buf: &mut [u8], bit_depth: u8, samples: usize, @@ -349,7 +349,7 @@ fn fix_endianness_and_predict( } } -fn invert_colors(buf: &mut [u8], color_type: ColorType, sample_format: SampleFormat) { +pub fn invert_colors(buf: &mut [u8], color_type: ColorType, sample_format: SampleFormat) { match (color_type, sample_format) { (ColorType::Gray(8), SampleFormat::Uint) => { for x in buf { diff --git a/src/decoder/tag_reader.rs b/src/decoder/tag_reader.rs index 3ae2d176..6fb41fa1 100644 --- a/src/decoder/tag_reader.rs +++ b/src/decoder/tag_reader.rs @@ -1,3 +1,5 @@ +use futures::AsyncRead; + use std::io::{Read, Seek}; use crate::tags::Tag; diff --git a/src/decoder_async/ifd.rs b/src/decoder_async/ifd.rs new file mode 100644 index 00000000..e4c6f544 --- /dev/null +++ b/src/decoder_async/ifd.rs @@ -0,0 +1,535 @@ +use crate::decoder_async::stream::AsyncSmartReader; +use crate::tags::Type; +use crate::{ + decoder::{ + ifd::Value::{ + self, Ascii, Byte, Double, Float, Ifd, IfdBig, List, Rational, RationalBig, SRational, + SRationalBig, Short, Signed, SignedBig, SignedByte, SignedShort, Unsigned, UnsignedBig, + }, + stream::{ByteOrder, EndianReader, SmartReader}, + }, + tags::Tag, +}; +use crate::{TiffError, TiffFormatError, TiffResult}; + +use futures::{future::BoxFuture, io::SeekFrom, AsyncRead, AsyncReadExt, AsyncSeek, FutureExt}; +use std::{ + collections::HashMap, + io::{Cursor, Read}, +}; + +use super::RangeReader; + +pub type Directory = HashMap; + +/// Extracts a list of BYTE tags stored in an offset +#[inline] +pub fn offset_to_bytes(n: usize, entry: &Entry) -> Value { + List( + entry.offset[0..n] + .iter() + .map(|&e| Unsigned(u32::from(e))) + .collect(), + ) +} + +/// Extracts a list of SBYTE tags stored in an offset +#[inline] +pub fn offset_to_sbytes(n: usize, entry: &Entry) -> Value { + List( + entry.offset[0..n] + .iter() + .map(|&e| Signed(i32::from(e as i8))) + .collect(), + ) +} + +#[derive(Clone, Copy)] +pub struct Entry { + type_: Type, + count: u64, + offset: [u8; 8], +} + +impl ::std::fmt::Debug for Entry { + fn fmt(&self, fmt: &mut ::std::fmt::Formatter) -> Result<(), ::std::fmt::Error> { + fmt.write_str(&format!( + "Entry {{ type_: {:?}, count: {:?}, offset: {:?} }}", + self.type_, self.count, &self.offset + )) + } +} + +impl Entry { + /// Construct a new Entry based on the tag + /// ``` + /// /* + /// tiff tag: + /// |tag|type|count|value| + /// | | + /// in parent offset (actually only really offset if it doesn't fit) + /// */ + /// ``` + /// note that, `offset=[1,2,3,4] -> [1,2,3,4,0,0,0,0]` + /// but this is taken care of when requesting val + pub fn new(type_: Type, count: u32, offset: [u8; 4]) -> Entry { + let mut offset = offset.to_vec(); + offset.append(&mut vec![0; 4]); + Entry::new_u64(type_, count.into(), offset[..].try_into().unwrap()) + } + + pub fn new_u64(type_: Type, count: u64, offset: [u8; 8]) -> Entry { + Entry { + type_, + count, + offset, + } + } + + /// Returns a mem_reader for the offset/value field + fn r(&self, byte_order: ByteOrder) -> SmartReader>> { + SmartReader::wrap(std::io::Cursor::new(self.offset.to_vec()), byte_order) + } + + #[inline(always)] + fn type_size(&self) -> u64 { + match self.type_ { + Type::BYTE | Type::SBYTE | Type::ASCII | Type::UNDEFINED => 1, + Type::SHORT | Type::SSHORT => 2, + Type::LONG | Type::SLONG | Type::FLOAT | Type::IFD => 4, + Type::LONG8 + | Type::SLONG8 + | Type::DOUBLE + | Type::RATIONAL + | Type::SRATIONAL + | Type::IFD8 => 8, + } + } + + #[inline(always)] + pub fn count(&self) -> u64 { + self.count + } + + #[inline(always)] + fn num_value_bytes(&self) -> TiffResult { + // The number of bytes our value takes up + match self.count.checked_mul(self.type_size()) { + Some(n) => Ok(n), + None => { + return Err(TiffError::LimitsExceeded); + } + } + } + + /// Get the tags value if it fits in the Value field. + pub fn maybe_val(&self, bigtiff: bool, byte_order: ByteOrder) -> TiffResult> { + // Case 1: there are no values so we can return immediately. + if self.count == 0 { + return Ok(Some(List(Vec::new()))); + } + + let bo = byte_order; + + let value_bytes = self.num_value_bytes()?; + + if value_bytes > 8 || (!bigtiff && value_bytes > 4) { + return Ok(None); + } + + // Case 2: there is one value. + if self.count == 1 { + // 2a: the value is 5-8 bytes and we're in BigTiff mode. + if bigtiff && value_bytes > 4 && value_bytes <= 8 { + return Ok(Some(match self.type_ { + Type::LONG8 => UnsignedBig(self.r(bo).read_u64()?), + Type::SLONG8 => SignedBig(self.r(bo).read_i64()?), + Type::DOUBLE => Double(self.r(bo).read_f64()?), + Type::RATIONAL => { + let mut r = self.r(bo); + Rational(r.read_u32()?, r.read_u32()?) + } + Type::SRATIONAL => { + let mut r = self.r(bo); + SRational(r.read_i32()?, r.read_i32()?) + } + Type::IFD8 => IfdBig(self.r(bo).read_u64()?), + Type::BYTE + | Type::SBYTE + | Type::ASCII + | Type::UNDEFINED + | Type::SHORT + | Type::SSHORT + | Type::LONG + | Type::SLONG + | Type::FLOAT + | Type::IFD => unreachable!(), + })); + } + + // 2b: the value is at most 4 bytes or doesn't fit in the offset field. + return Ok(match self.type_ { + Type::BYTE => Some(Unsigned(u32::from(self.offset[0]))), + Type::SBYTE => Some(Signed(i32::from(self.offset[0] as i8))), + Type::UNDEFINED => Some(Byte(self.offset[0])), + Type::SHORT => Some(Unsigned(u32::from(self.r(bo).read_u16()?))), + Type::SSHORT => Some(Signed(i32::from(self.r(bo).read_i16()?))), + Type::LONG => Some(Unsigned(self.r(bo).read_u32()?)), + Type::SLONG => Some(Signed(self.r(bo).read_i32()?)), + Type::FLOAT => Some(Float(self.r(bo).read_f32()?)), + Type::ASCII => { + if self.offset[0] == 0 { + Some(Ascii("".to_string())) + } else { + return Err(TiffError::FormatError(TiffFormatError::InvalidTag)); + } + } + Type::IFD => Some(Ifd(self.r(bo).read_u32()?)), + _ => unreachable!("This should have been caught earlier"), + }); + } + + // Case 3: There is more than one value, but it fits in the offset field. + if value_bytes <= 4 || bigtiff && value_bytes <= 8 { + match self.type_ { + Type::BYTE => return Ok(Some(offset_to_bytes(self.count as usize, self))), + Type::SBYTE => return Ok(Some(offset_to_sbytes(self.count as usize, self))), + Type::ASCII => { + let mut buf = vec![0; self.count as usize]; + self.r(bo).read_exact(&mut buf)?; + if buf.is_ascii() && buf.ends_with(&[0]) { + let v = std::str::from_utf8(&buf)?; + let v = v.trim_matches(char::from(0)); + return Ok(Some(Ascii(v.into()))); + } else { + return Err(TiffError::FormatError(TiffFormatError::InvalidTag)); + } + } + Type::UNDEFINED => { + return Ok(Some(List( + self.offset[0..self.count as usize] + .iter() + .map(|&b| Byte(b)) + .collect(), + ))); + } + Type::SHORT => { + let mut r = self.r(bo); + let mut v = Vec::new(); + for _ in 0..self.count { + v.push(Short(r.read_u16()?)); + } + return Ok(Some(List(v))); + } + Type::SSHORT => { + let mut r = self.r(bo); + let mut v = Vec::new(); + for _ in 0..self.count { + v.push(Signed(i32::from(r.read_i16()?))); + } + return Ok(Some(List(v))); + } + Type::LONG => { + let mut r = self.r(bo); + let mut v = Vec::new(); + for _ in 0..self.count { + v.push(Unsigned(r.read_u32()?)); + } + return Ok(Some(List(v))); + } + Type::SLONG => { + let mut r = self.r(bo); + let mut v = Vec::new(); + for _ in 0..self.count { + v.push(Signed(r.read_i32()?)); + } + return Ok(Some(List(v))); + } + Type::FLOAT => { + let mut r = self.r(bo); + let mut v = Vec::new(); + for _ in 0..self.count { + v.push(Float(r.read_f32()?)); + } + return Ok(Some(List(v))); + } + Type::IFD => { + let mut r = self.r(bo); + let mut v = Vec::new(); + for _ in 0..self.count { + v.push(Ifd(r.read_u32()?)); + } + return Ok(Some(List(v))); + } + Type::LONG8 + | Type::SLONG8 + | Type::RATIONAL + | Type::SRATIONAL + | Type::DOUBLE + | Type::IFD8 => { + unreachable!() + } + } + } + + // case 4: multiple and it doesn't fit + unreachable!() + } + + /// Gets the nth value of a List type, such as byte offsets and lenghts + /// If it is not an offset, + pub async fn nth_val( + &self, + n: u64, + limits: &super::Limits, + bigtiff: bool, + reader: &mut AsyncSmartReader, + ) -> TiffResult { + if self.num_value_bytes()? <= 4 || (self.num_value_bytes()? <= 8 && bigtiff) { + // return Err(TiffError::UsageError("Should not call nth val on a value that is in the Value tag field")); + panic!("Should not call this function if bla") + } + if n > self.count { + return Err(TiffError::LimitsExceeded); + } + let bo = reader.byte_order(); + let offset = if bigtiff { + self.r(bo).read_u64()? + } else { + self.r(bo).read_u32()?.into() + }; + reader.goto_offset(offset + n * self.type_size()).await?; + Ok(UnsignedBig(reader.read_u64().await?)) + } + + /// get the tags value, if it doesn't fit, it will read the pointer. + /// may cause additional reading into the file + pub async fn val( + &self, + limits: &super::Limits, + bigtiff: bool, + reader: &mut AsyncSmartReader, + ) -> TiffResult { + let bo = reader.byte_order(); + + // The number of bytes our value takes up + let value_bytes = self.num_value_bytes()?; + let offset = if bigtiff { + self.r(bo).read_u64()? + } else { + self.r(bo).read_u32()?.into() + }; + // case 1: the value fits in the value field + if let Some(maybe_val) = self.maybe_val(bigtiff, bo)? { + return Ok(maybe_val); + } + + // Case 2: there is one value. This only + if self.count == 1 { + // 2b: the value is at most 4 bytes or doesn't fit in the offset field. + return Ok(match self.type_ { + Type::LONG8 => { + reader.goto_offset(offset).await?; + UnsignedBig(reader.read_u64().await?) + } + Type::SLONG8 => { + reader.goto_offset(offset).await?; + SignedBig(reader.read_i64().await?) + } + Type::DOUBLE => { + reader.goto_offset(offset).await?; + Double(reader.read_f64().await?) + } + Type::RATIONAL => { + reader.goto_offset(offset).await?; + Rational(reader.read_u32().await?, reader.read_u32().await?) + } + Type::SRATIONAL => { + reader.goto_offset(offset).await?; + SRational(reader.read_i32().await?, reader.read_i32().await?) + } + Type::IFD8 => { + reader.goto_offset(offset).await?; + IfdBig(reader.read_u64().await?) + } + _ => unreachable!(), + }); + } + + // TODO: find out if this is actually faster (I think it is...) + // initialize the buffer with all tag data inside it + // let buf = reader.read_range(offset, offset + self.count * self.type_size()).await?; + // let synr = SmartReader::wrap(Cursor::new(buf), bo); + + // Case 4: there is more than one value, and it doesn't fit in the offset field. + // Async help found here: https://users.rust-lang.org/t/function-that-takes-an-async-closure/61663 + match self.type_ { + // TODO check if this could give wrong results + // at a different endianess of file/computer. + Type::BYTE => { + self.decode_offset(self.count, bo, bigtiff, limits, reader, |reader| { + async move { + let mut buf = [0; 1]; + reader.read_exact(&mut buf).await?; + Ok(UnsignedBig(u64::from(buf[0]))) + } + .boxed() + }) + .await + } + Type::SBYTE => { + self.decode_offset(self.count, bo, bigtiff, limits, reader, |reader| { + async move { Ok(SignedBig(i64::from(reader.read_i8().await?))) }.boxed() + }) + .await + } + Type::SHORT => { + self.decode_offset(self.count, bo, bigtiff, limits, reader, |reader| { + async move { Ok(UnsignedBig(u64::from(reader.read_u16().await?))) }.boxed() + }) + .await + } + Type::SSHORT => { + self.decode_offset(self.count, bo, bigtiff, limits, reader, |reader| { + async move { Ok(SignedBig(i64::from(reader.read_i16().await?))) }.boxed() + }) + .await + } + Type::LONG => { + self.decode_offset(self.count, bo, bigtiff, limits, reader, |reader| { + async move { Ok(Unsigned(reader.read_u32().await?)) }.boxed() + }) + .await + } + Type::SLONG => { + self.decode_offset(self.count, bo, bigtiff, limits, reader, |reader| { + async move { Ok(Signed(reader.read_i32().await?)) }.boxed() + }) + .await + } + Type::FLOAT => { + self.decode_offset(self.count, bo, bigtiff, limits, reader, |reader| { + async move { Ok(Float(reader.read_f32().await?)) }.boxed() + }) + .await + } + Type::DOUBLE => { + self.decode_offset(self.count, bo, bigtiff, limits, reader, |reader| { + async move { Ok(Double(reader.read_f64().await?)) }.boxed() + }) + .await + } + Type::RATIONAL => { + self.decode_offset(self.count, bo, bigtiff, limits, reader, |reader| { + async move { Ok(Rational(reader.read_u32().await?, reader.read_u32().await?)) } + .boxed() + }) + .await + } + Type::SRATIONAL => { + self.decode_offset(self.count, bo, bigtiff, limits, reader, |reader| { + async move { + Ok(SRational( + reader.read_i32().await?, + reader.read_i32().await?, + )) + } + .boxed() + }) + .await + } + Type::LONG8 => { + self.decode_offset(self.count, bo, bigtiff, limits, reader, |reader| { + async move { Ok(UnsignedBig(reader.read_u64().await?)) }.boxed() + }) + .await + } + Type::SLONG8 => { + self.decode_offset(self.count, bo, bigtiff, limits, reader, |reader| { + async move { Ok(SignedBig(reader.read_i64().await?)) }.boxed() + }) + .await + } + Type::IFD => { + self.decode_offset(self.count, bo, bigtiff, limits, reader, |reader| { + async move { Ok(Ifd(reader.read_u32().await?)) }.boxed() + }) + .await + } + Type::IFD8 => { + self.decode_offset(self.count, bo, bigtiff, limits, reader, |reader| { + async move { Ok(IfdBig(reader.read_u64().await?)) }.boxed() + }) + .await + } + Type::UNDEFINED => { + self.decode_offset(self.count, bo, bigtiff, limits, reader, |reader| { + async move { + let mut buf = [0; 1]; + reader.read_exact(&mut buf).await?; + Ok(Byte(buf[0])) + } + .boxed() + }) + .await + } + Type::ASCII => { + let n = usize::try_from(self.count)?; + if n > limits.decoding_buffer_size { + return Err(TiffError::LimitsExceeded); + } + + if bigtiff { + reader.goto_offset(self.r(bo).read_u64()?).await? + } else { + reader.goto_offset(offset).await? + } + + let mut out = vec![0; n]; + reader.read_exact(&mut out).await?; + // Strings may be null-terminated, so we trim anything downstream of the null byte + if let Some(first) = out.iter().position(|&b| b == 0) { + out.truncate(first); + } + Ok(Ascii(String::from_utf8(out)?)) + } + } + } + + /// Goes to offset and decodes all values there + /// This is the interesting part where tile offsets are read + #[inline] + async fn decode_offset( + &self, + value_count: u64, + bo: ByteOrder, + bigtiff: bool, + limits: &super::Limits, + reader: &mut AsyncSmartReader, + decode_fn: F, + ) -> TiffResult + where + R: AsyncRead + AsyncSeek + Unpin, + // F: Fn(&mut AsyncSmartReader) -> TiffResult, + F: Fn(&'_ mut AsyncSmartReader) -> BoxFuture<'_, TiffResult>, + { + let value_count = usize::try_from(value_count)?; + if value_count > limits.decoding_buffer_size / std::mem::size_of::() { + return Err(TiffError::LimitsExceeded); + } + + let mut v = Vec::with_capacity(value_count); + + let offset = if bigtiff { + self.r(bo).read_u64()? + } else { + self.r(bo).read_u32()?.into() + }; + reader.goto_offset(offset).await?; + + for _ in 0..value_count { + v.push(decode_fn(reader).await?) + } + Ok(List(v)) + } +} diff --git a/src/decoder_async/image.rs b/src/decoder_async/image.rs new file mode 100644 index 00000000..d261a6fd --- /dev/null +++ b/src/decoder_async/image.rs @@ -0,0 +1,807 @@ +use crate::decoder::{ + fix_endianness_and_predict, + ifd::Value, + invert_colors, predict_f32, predict_f64, + stream::{ByteOrder, DeflateReader, LZWReader}, + Limits, +}; +use crate::decoder_async::{ + ifd::{Directory, Entry}, + image::Value::UnsignedBig, + stream::AsyncSmartReader, + tag_reader::AsyncTagReader, + ChunkType, +}; +use crate::tags::{ + CompressionMethod, PhotometricInterpretation, PlanarConfiguration, Predictor, SampleFormat, Tag, +}; +use crate::{ColorType, TiffError, TiffFormatError, TiffResult, TiffUnsupportedError, UsageError}; + +use futures::{io::Empty, AsyncRead, AsyncReadExt, AsyncSeek}; + +use std::{ + io::{Cursor, Read}, + sync::Arc, +}; + +#[derive(Debug)] +pub(crate) struct StripDecodeState { + pub rows_per_strip: u32, +} + +#[derive(Debug)] +/// Computed values useful for tile decoding +pub(crate) struct TileAttributes { + pub image_width: usize, + pub image_height: usize, + + pub tile_width: usize, + pub tile_length: usize, +} + +impl TileAttributes { + pub fn tiles_across(&self) -> usize { + (self.image_width + self.tile_width - 1) / self.tile_width + } + pub fn tiles_down(&self) -> usize { + (self.image_height + self.tile_length - 1) / self.tile_length + } + fn padding_right(&self) -> usize { + (self.tile_width - self.image_width % self.tile_width) % self.tile_width + } + fn padding_down(&self) -> usize { + (self.tile_length - self.image_height % self.tile_length) % self.tile_length + } + pub fn get_padding(&self, tile: usize) -> (usize, usize) { + let row = tile / self.tiles_across(); + let column = tile % self.tiles_across(); + + let padding_right = if column == self.tiles_across() - 1 { + self.padding_right() + } else { + 0 + }; + + let padding_down = if row == self.tiles_down() - 1 { + self.padding_down() + } else { + 0 + }; + + (padding_right, padding_down) + } +} + +// #[derive(Debug)] +// pub(crate) enum ChunkData { +// Empty(Entry), +// Full(Vec), +// } + +// impl ChunkData { +// fn get(&self, index: usize) -> Option<&u64> { +// match self { +// ChunkData::Full(v) => v.get(index), +// ChunkData::Empty(entry) => None, +// } +// } + +// /// retrieves a single entry from the reader +// pub async fn retrieve_single( +// &self, +// index: u64, +// limits: &Limits, +// bigtiff: bool, +// reader: &mut R, +// ) -> TiffResult { +// match self { +// ChunkData::Empty(entry) => entry.nth_val(index, limits, bigtiff, reader).await, +// ChunkData::Full(v) => { +// println!("retrieve called when we had a full buffer"); +// v.get(index as usize) +// .map(|v| UnsignedBig(*v)) +// .ok_or(TiffError::LimitsExceeded) +// } +// } +// } + +// /// Fills the buffer. After this, we will be ChunkData::Full and lookups will be super fast +// pub async fn fill( +// &mut self, +// index: u64, +// limits: &Limits, +// bigtiff: bool, +// reader: &mut AsyncSmartReader, +// ) -> TiffResult<()> { +// let ChunkData::Empty(entry) = self else { +// println!("Called Fill while already full!"); +// return Ok(()); +// }; +// *self = ChunkData::Full(entry.val(limits, bigtiff, reader).await?.into_u64_vec()?); +// Ok(()) +// } +// } + +#[derive(Debug)] +pub(crate) struct AsyncImage { + pub ifd: Option, + pub width: u32, + pub height: u32, + pub bits_per_sample: u8, + pub samples: u16, + pub sample_format: SampleFormat, + pub photometric_interpretation: PhotometricInterpretation, + pub compression_method: CompressionMethod, + pub predictor: Predictor, + pub jpeg_tables: Option>>, + pub chunk_type: ChunkType, + pub planar_config: PlanarConfiguration, + pub strip_decoder: Option, + pub tile_attributes: Option, + pub chunk_offsets: Vec, + pub chunk_bytes: Vec, +} + +impl AsyncImage { + /// Creates this image from a reader. Will not read in chunk tags + /// Rather, this + pub async fn from_reader( + reader: &mut AsyncSmartReader, + ifd: Directory, + limits: &Limits, + bigtiff: bool, + ) -> TiffResult { + let mut tag_reader = AsyncTagReader { + reader, + limits, + ifd: &ifd, + bigtiff, + }; + + let width = tag_reader.require_tag(Tag::ImageWidth).await?.into_u32()?; + let height = tag_reader.require_tag(Tag::ImageLength).await?.into_u32()?; + if width == 0 || height == 0 { + return Err(TiffError::FormatError(TiffFormatError::InvalidDimensions( + width, height, + ))); + } + + let photometric_interpretation = tag_reader + .find_tag(Tag::PhotometricInterpretation) + .await? + .map(Value::into_u16) + .transpose()? + .and_then(PhotometricInterpretation::from_u16) + .ok_or(TiffUnsupportedError::UnknownInterpretation)?; + + // Try to parse both the compression method and the number, format, and bits of the included samples. + // If they are not explicitly specified, those tags are reset to their default values and not carried from previous images. + let compression_method = match tag_reader.find_tag(Tag::Compression).await? { + Some(val) => CompressionMethod::from_u16_exhaustive(val.into_u16()?), + None => CompressionMethod::None, + }; + + let jpeg_tables = if compression_method == CompressionMethod::ModernJPEG + && ifd.contains_key(&Tag::JPEGTables) + { + let vec = tag_reader + .find_tag(Tag::JPEGTables) + .await? + .unwrap() + .into_u8_vec()?; + if vec.len() < 2 { + return Err(TiffError::FormatError( + TiffFormatError::InvalidTagValueType(Tag::JPEGTables), + )); + } + + Some(Arc::new(vec)) + } else { + None + }; + + let samples: u16 = tag_reader + .find_tag(Tag::SamplesPerPixel) + .await? + .map(Value::into_u16) + .transpose()? + .unwrap_or(1); + if samples == 0 { + return Err(TiffFormatError::SamplesPerPixelIsZero.into()); + } + + let sample_format = match tag_reader.find_tag_uint_vec(Tag::SampleFormat).await? { + Some(vals) => { + let sample_format: Vec<_> = vals + .into_iter() + .map(SampleFormat::from_u16_exhaustive) + .collect(); + + // TODO: for now, only homogenous formats across samples are supported. + if !sample_format.windows(2).all(|s| s[0] == s[1]) { + return Err(TiffUnsupportedError::UnsupportedSampleFormat(sample_format).into()); + } + + sample_format[0] + } + None => SampleFormat::Uint, + }; + + let bits_per_sample: Vec = tag_reader + .find_tag_uint_vec(Tag::BitsPerSample).await? + .unwrap_or_else(|| vec![1]); + + // Technically bits_per_sample.len() should be *equal* to samples, but libtiff also allows + // it to be a single value that applies to all samples. + if bits_per_sample.len() != samples as usize && bits_per_sample.len() != 1 { + return Err(TiffError::FormatError( + TiffFormatError::InconsistentSizesEncountered, + )); + } + + // This library (and libtiff) do not support mixed sample formats and zero bits per sample + // doesn't make sense. + if bits_per_sample.iter().any(|&b| b != bits_per_sample[0]) || bits_per_sample[0] == 0 { + return Err(TiffUnsupportedError::InconsistentBitsPerSample(bits_per_sample).into()); + } + + let predictor = tag_reader + .find_tag(Tag::Predictor) + .await? + .map(Value::into_u16) + .transpose()? + .map(|p| { + Predictor::from_u16(p) + .ok_or(TiffError::FormatError(TiffFormatError::UnknownPredictor(p))) + }) + .transpose()? + .unwrap_or(Predictor::None); + + let planar_config = tag_reader + .find_tag(Tag::PlanarConfiguration) + .await? + .map(Value::into_u16) + .transpose()? + .map(|p| { + PlanarConfiguration::from_u16(p).ok_or(TiffError::FormatError( + TiffFormatError::UnknownPlanarConfiguration(p), + )) + }) + .transpose()? + .unwrap_or(PlanarConfiguration::Chunky); + + let planes = match planar_config { + PlanarConfiguration::Chunky => 1, + PlanarConfiguration::Planar => samples, + }; + + let chunk_type; + let chunk_offsets; + let chunk_bytes; + let strip_decoder; + let tile_attributes; + match ( + ifd.contains_key(&Tag::StripByteCounts), + ifd.contains_key(&Tag::StripOffsets), + ifd.contains_key(&Tag::TileByteCounts), + ifd.contains_key(&Tag::TileOffsets), + ) { + (true, true, false, false) => { + chunk_type = ChunkType::Strip; + + chunk_offsets = //ifd[&Tag::StripOffsets]; + tag_reader + .find_tag(Tag::StripOffsets).await? + .unwrap() + .into_u64_vec()?; + chunk_bytes = //ifd[&Tag::StripByteCounts]; + tag_reader + .find_tag(Tag::StripByteCounts).await? + .unwrap() + .into_u64_vec()?; + let rows_per_strip = tag_reader + .find_tag(Tag::RowsPerStrip) + .await? + .map(Value::into_u32) + .transpose()? + .unwrap_or(height); + strip_decoder = Some(StripDecodeState { rows_per_strip }); + tile_attributes = None; + + if chunk_offsets.len() != chunk_bytes.len() + || rows_per_strip == 0 + || u32::try_from(chunk_offsets.len())? + != (height.saturating_sub(1) / rows_per_strip + 1) * planes as u32 + { + return Err(TiffError::FormatError( + TiffFormatError::InconsistentSizesEncountered, + )); + } + } + (false, false, true, true) => { + chunk_type = ChunkType::Tile; + + let tile_width = + usize::try_from(tag_reader.require_tag(Tag::TileWidth).await?.into_u32()?)?; + let tile_length = + usize::try_from(tag_reader.require_tag(Tag::TileLength).await?.into_u32()?)?; + + if tile_width == 0 { + return Err(TiffFormatError::InvalidTagValueType(Tag::TileWidth).into()); + } else if tile_length == 0 { + return Err(TiffFormatError::InvalidTagValueType(Tag::TileLength).into()); + } + + strip_decoder = None; + tile_attributes = Some(TileAttributes { + image_width: usize::try_from(width)?, + image_height: usize::try_from(height)?, + tile_width, + tile_length, + }); + chunk_offsets = //ifd[&Tag::TileOffsets]; + tag_reader + .find_tag(Tag::TileOffsets).await? + .unwrap() + .into_u64_vec()?; + chunk_bytes = //ifd[&Tag::TileByteCounts]; + tag_reader + .find_tag(Tag::TileByteCounts).await? + .unwrap() + .into_u64_vec()?; + + let tile = tile_attributes.as_ref().unwrap(); + if chunk_offsets.len() != chunk_bytes.len() + || chunk_offsets.len() as usize + != tile.tiles_down() * tile.tiles_across() * planes as usize + { + return Err(TiffError::FormatError( + TiffFormatError::InconsistentSizesEncountered, + )); + } + } + (_, _, _, _) => { + return Err(TiffError::FormatError( + TiffFormatError::StripTileTagConflict, + )) + } + }; + + Ok(AsyncImage { + ifd: Some(ifd), + width, + height, + bits_per_sample: bits_per_sample[0], + samples, + sample_format, + photometric_interpretation, + compression_method, + jpeg_tables, + predictor, + chunk_type, + planar_config, + strip_decoder, + tile_attributes, + chunk_offsets: chunk_offsets, + chunk_bytes: chunk_bytes, + }) + } + + pub(crate) fn colortype(&self) -> TiffResult { + println!("getting colortype for {:?}", self, ); + match self.photometric_interpretation { + PhotometricInterpretation::RGB => match self.samples { + 3 => Ok(ColorType::RGB(self.bits_per_sample)), + 4 => Ok(ColorType::RGBA(self.bits_per_sample)), + // FIXME: We should _ignore_ other components. In particular: + // > Beware of extra components. Some TIFF files may have more components per pixel + // than you think. A Baseline TIFF reader must skip over them gracefully,using the + // values of the SamplesPerPixel and BitsPerSample fields. + // > -- TIFF 6.0 Specification, Section 7, Additional Baseline requirements. + _ => Err(TiffError::UnsupportedError( + TiffUnsupportedError::InterpretationWithBits( + self.photometric_interpretation, + vec![self.bits_per_sample; self.samples as usize], + ), + )), + }, + PhotometricInterpretation::CMYK => match self.samples { + 4 => Ok(ColorType::CMYK(self.bits_per_sample)), + _ => Err(TiffError::UnsupportedError( + TiffUnsupportedError::InterpretationWithBits( + self.photometric_interpretation, + vec![self.bits_per_sample; self.samples as usize], + ), + )), + }, + PhotometricInterpretation::YCbCr => match self.samples { + 3 => Ok(ColorType::YCbCr(self.bits_per_sample)), + _ => Err(TiffError::UnsupportedError( + TiffUnsupportedError::InterpretationWithBits( + self.photometric_interpretation, + vec![self.bits_per_sample; self.samples as usize], + ), + )), + }, + PhotometricInterpretation::BlackIsZero | PhotometricInterpretation::WhiteIsZero => { + match self.samples { + 1 => Ok(ColorType::Gray(self.bits_per_sample)), + _ => Ok(ColorType::Multiband { + bit_depth: self.bits_per_sample, + num_samples: self.samples, + }), + } + } + // TODO: this is bad we should not fail at this point + PhotometricInterpretation::RGBPalette + | PhotometricInterpretation::TransparencyMask + | PhotometricInterpretation::CIELab => Err(TiffError::UnsupportedError( + TiffUnsupportedError::InterpretationWithBits( + self.photometric_interpretation, + vec![self.bits_per_sample; self.samples as usize], + ), + )), + } + } + + fn create_reader<'r, R: 'r + Read>( + reader: R, + photometric_interpretation: PhotometricInterpretation, + compression_method: CompressionMethod, + compressed_length: u64, + jpeg_tables: Option<&[u8]>, + ) -> TiffResult> { + Ok(match compression_method { + CompressionMethod::None => Box::new(reader), + CompressionMethod::LZW => { + Box::new(LZWReader::new(reader, usize::try_from(compressed_length)?)) + } + CompressionMethod::Deflate | CompressionMethod::OldDeflate => { + Box::new(DeflateReader::new(reader)) + } + CompressionMethod::ModernJPEG => { + if jpeg_tables.is_some() && compressed_length < 2 { + return Err(TiffError::FormatError( + TiffFormatError::InvalidTagValueType(Tag::JPEGTables), + )); + } + + // Construct new jpeg_reader wrapping a SmartReader. + // + // JPEG compression in TIFF allows saving quantization and/or huffman tables in one + // central location. These `jpeg_tables` are simply prepended to the remaining jpeg image data. + // Because these `jpeg_tables` start with a `SOI` (HEX: `0xFFD8`) or __start of image__ marker + // which is also at the beginning of the remaining JPEG image data and would + // confuse the JPEG renderer, one of these has to be taken off. In this case the first two + // bytes of the remaining JPEG data is removed because it follows `jpeg_tables`. + // Similary, `jpeg_tables` ends with a `EOI` (HEX: `0xFFD9`) or __end of image__ marker, + // this has to be removed as well (last two bytes of `jpeg_tables`). + let jpeg_reader = match jpeg_tables { + Some(jpeg_tables) => { + let mut reader = reader.take(compressed_length); + reader.read_exact(&mut [0; 2])?; + + Box::new( + Cursor::new(&jpeg_tables[..jpeg_tables.len() - 2]) + .chain(reader.take(compressed_length)), + ) as Box + } + None => Box::new(reader.take(compressed_length)), + }; + + let mut decoder = jpeg::Decoder::new(jpeg_reader); + + match photometric_interpretation { + PhotometricInterpretation::RGB => { + decoder.set_color_transform(jpeg::ColorTransform::RGB) + } + PhotometricInterpretation::WhiteIsZero => { + decoder.set_color_transform(jpeg::ColorTransform::None) + } + PhotometricInterpretation::BlackIsZero => { + decoder.set_color_transform(jpeg::ColorTransform::None) + } + PhotometricInterpretation::TransparencyMask => { + decoder.set_color_transform(jpeg::ColorTransform::None) + } + PhotometricInterpretation::CMYK => { + decoder.set_color_transform(jpeg::ColorTransform::CMYK) + } + PhotometricInterpretation::YCbCr => { + decoder.set_color_transform(jpeg::ColorTransform::YCbCr) + } + photometric_interpretation => { + return Err(TiffError::UnsupportedError( + TiffUnsupportedError::UnsupportedInterpretation( + photometric_interpretation, + ), + )); + } + } + + let data = decoder.decode()?; + + Box::new(Cursor::new(data)) + } + method => { + return Err(TiffError::UnsupportedError( + TiffUnsupportedError::UnsupportedCompressionMethod(method), + )) + } + }) + } + + /// Samples per pixel within chunk. + /// + /// In planar config, samples are stored in separate strips/chunks, also called bands. + /// + /// Example with `bits_per_sample = [8, 8, 8]` and `PhotometricInterpretation::RGB`: + /// * `PlanarConfiguration::Chunky` -> 3 (RGBRGBRGB...) + /// * `PlanarConfiguration::Planar` -> 1 (RRR...) (GGG...) (BBB...) + pub(crate) fn samples_per_pixel(&self) -> usize { + match self.planar_config { + PlanarConfiguration::Chunky => self.samples.into(), + PlanarConfiguration::Planar => 1, + } + } + + /// Number of strips per pixel. + pub(crate) fn strips_per_pixel(&self) -> usize { + match self.planar_config { + PlanarConfiguration::Chunky => 1, + PlanarConfiguration::Planar => self.samples.into(), + } + } + + pub(crate) fn chunk_file_range(&self, chunk: u32) -> TiffResult<(u64, u64)> { + let file_offset = self + .chunk_offsets + .get(chunk as usize) + .ok_or(TiffError::FormatError( + TiffFormatError::InconsistentSizesEncountered, + ))?; + + let compressed_bytes = + self.chunk_bytes + .get(chunk as usize) + .ok_or(TiffError::FormatError( + TiffFormatError::InconsistentSizesEncountered, + ))?; + + Ok((*file_offset, *compressed_bytes)) + } + + /// Dimensions of a chunk, which is a strip or tile. + /// typically, this is a power of 2 for tiled COGS, such as 1024 by 1024 + pub(crate) fn chunk_dimensions(&self) -> TiffResult<(u32, u32)> { + match self.chunk_type { + ChunkType::Strip => { + let strip_attrs = self.strip_decoder.as_ref().unwrap(); + Ok((self.width, strip_attrs.rows_per_strip)) + } + ChunkType::Tile => { + let tile_attrs = self.tile_attributes.as_ref().unwrap(); + Ok(( + u32::try_from(tile_attrs.tile_width)?, + u32::try_from(tile_attrs.tile_length)?, + )) + } + } + } + + /// Dimensions of the data within the chunk. + /// see in get_padding that this is the chunk dimensions, + /// unless we are at the bottom or far side of the image, + /// in which case there is some padding involved because the full image is not necessarily a power of 2 + pub(crate) fn chunk_data_dimensions(&self, chunk_index: u32) -> TiffResult<(u32, u32)> { + let dims = self.chunk_dimensions()?; + + match self.chunk_type { + ChunkType::Strip => { + let strip_attrs = self.strip_decoder.as_ref().unwrap(); + let strips_per_band = + self.height.saturating_sub(1) / strip_attrs.rows_per_strip + 1; + let strip_height_without_padding = (chunk_index % strips_per_band) + .checked_mul(dims.1) + .and_then(|x| self.height.checked_sub(x)) + .ok_or(TiffError::UsageError(UsageError::InvalidChunkIndex( + chunk_index, + )))?; + + // Ignore potential vertical padding on the bottommost strip + let strip_height = dims.1.min(strip_height_without_padding); + + Ok((dims.0, strip_height)) + } + ChunkType::Tile => { + let tile_attrs = self.tile_attributes.as_ref().unwrap(); + let (padding_right, padding_down) = tile_attrs.get_padding(chunk_index as usize); + + let tile_width = tile_attrs.tile_width - padding_right; + let tile_length = tile_attrs.tile_length - padding_down; + + Ok((u32::try_from(tile_width)?, u32::try_from(tile_length)?)) + } + } + } + + pub(crate) fn expand_chunk( + &self, + reader: impl Read, + buf: &mut [u8], + output_row_stride: usize, + byte_order: ByteOrder, + chunk_index: u32, + limits: &Limits, + ) -> TiffResult<()> { + // Validate that the color type is supported. + let color_type = self.colortype()?; + match color_type { + ColorType::RGB(n) + | ColorType::RGBA(n) + | ColorType::CMYK(n) + | ColorType::YCbCr(n) + | ColorType::Gray(n) + | ColorType::Multiband { + bit_depth: n, + num_samples: _, + } if n == 8 || n == 16 || n == 32 || n == 64 => {} + ColorType::Gray(n) + | ColorType::Multiband { + bit_depth: n, + num_samples: _, + } if n < 8 => match self.predictor { + Predictor::None => {} + Predictor::Horizontal => { + return Err(TiffError::UnsupportedError( + TiffUnsupportedError::HorizontalPredictor(color_type), + )); + } + Predictor::FloatingPoint => { + return Err(TiffError::UnsupportedError( + TiffUnsupportedError::FloatingPointPredictor(color_type), + )); + } + }, + type_ => { + return Err(TiffError::UnsupportedError( + TiffUnsupportedError::UnsupportedColorType(type_), + )); + } + } + + // Validate that the predictor is supported for the sample type. + match (self.predictor, self.sample_format) { + (Predictor::Horizontal, SampleFormat::Int | SampleFormat::Uint) => {} + (Predictor::Horizontal, _) => { + return Err(TiffError::UnsupportedError( + TiffUnsupportedError::HorizontalPredictor(color_type), + )); + } + (Predictor::FloatingPoint, SampleFormat::IEEEFP) => {} + (Predictor::FloatingPoint, _) => { + return Err(TiffError::UnsupportedError( + TiffUnsupportedError::FloatingPointPredictor(color_type), + )); + } + _ => {} + } + + let compressed_bytes = self.chunk_bytes + .get(chunk_index as usize) + .ok_or(TiffError::FormatError( + TiffFormatError::InconsistentSizesEncountered, + ))?;//match &self.chunk_bytes { + // ChunkData::Full(v) => { + // self.chunk_bytes + // .get(chunk_index as usize) + // .ok_or(TiffError::FormatError( + // TiffFormatError::InconsistentSizesEncountered, + // ))? + // } + // ChunkData::Empty(_) => &self + // .chunk_bytes + // .retrieve_single(chunk_index, limits, self.bigtiff, reader) + // .await? + // .into_u64()?, + // }; + if *compressed_bytes > limits.intermediate_buffer_size as u64 { + return Err(TiffError::LimitsExceeded); + } + + let compression_method = self.compression_method; + let photometric_interpretation = self.photometric_interpretation; + let predictor = self.predictor; + let samples = self.samples_per_pixel(); + + let chunk_dims = self.chunk_dimensions()?; + let data_dims = self.chunk_data_dimensions(chunk_index)?; + + let chunk_row_bits = (u64::from(chunk_dims.0) * u64::from(self.bits_per_sample)) + .checked_mul(samples as u64) + .ok_or(TiffError::LimitsExceeded)?; + let chunk_row_bytes: usize = ((chunk_row_bits + 7) / 8).try_into()?; + + let data_row_bits = (u64::from(data_dims.0) * u64::from(self.bits_per_sample)) + .checked_mul(samples as u64) + .ok_or(TiffError::LimitsExceeded)?; + let data_row_bytes: usize = ((data_row_bits + 7) / 8).try_into()?; + + // TODO: Should these return errors instead? + assert!(output_row_stride >= data_row_bytes); + assert!(buf.len() >= output_row_stride * (data_dims.1 as usize - 1) + data_row_bytes); + + let mut reader = Self::create_reader( + reader, + photometric_interpretation, + compression_method, + *compressed_bytes, + self.jpeg_tables.as_deref().map(|a| &**a), + )?; + + if output_row_stride == chunk_row_bytes as usize { + let tile = &mut buf[..chunk_row_bytes * data_dims.1 as usize]; + reader.read_exact(tile)?; + + for row in tile.chunks_mut(chunk_row_bytes as usize) { + fix_endianness_and_predict( + row, + color_type.bit_depth(), + samples, + byte_order, + predictor, + ); + } + if photometric_interpretation == PhotometricInterpretation::WhiteIsZero { + super::invert_colors(tile, color_type, self.sample_format); + } + } else if chunk_row_bytes > data_row_bytes && self.predictor == Predictor::FloatingPoint { + // The floating point predictor shuffles the padding bytes into the encoded output, so + // this case is handled specially when needed. + let mut encoded = vec![0u8; chunk_row_bytes]; + for row in buf.chunks_mut(output_row_stride).take(data_dims.1 as usize) { + reader.read_exact(&mut encoded)?; + + let row = &mut row[..data_row_bytes]; + match color_type.bit_depth() { + 32 => predict_f32(&mut encoded, row, samples), + 64 => predict_f64(&mut encoded, row, samples), + _ => unreachable!(), + } + if photometric_interpretation == PhotometricInterpretation::WhiteIsZero { + invert_colors(row, color_type, self.sample_format); + } + } + } else { + for (i, row) in buf + .chunks_mut(output_row_stride) + .take(data_dims.1 as usize) + .enumerate() + { + let row = &mut row[..data_row_bytes]; + reader.read_exact(row)?; + + println!("chunk={chunk_index}, index={i}"); + + // Skip horizontal padding + if chunk_row_bytes > data_row_bytes { + let len = u64::try_from(chunk_row_bytes - data_row_bytes)?; + std::io::copy(&mut reader.by_ref().take(len), &mut std::io::sink())?; + } + + fix_endianness_and_predict( + row, + color_type.bit_depth(), + samples, + byte_order, + predictor, + ); + if photometric_interpretation == PhotometricInterpretation::WhiteIsZero { + invert_colors(row, color_type, self.sample_format); + } + } + } + + Ok(()) + } +} diff --git a/src/decoder_async/mod.rs b/src/decoder_async/mod.rs new file mode 100644 index 00000000..972c0b4e --- /dev/null +++ b/src/decoder_async/mod.rs @@ -0,0 +1,638 @@ +use futures::{ + // future::BoxFuture, + io::{AsyncRead, AsyncReadExt, AsyncSeek, SeekFrom}, + AsyncSeekExt, +}; +use std::collections::{HashMap, HashSet}; + +use crate::{TiffError, TiffFormatError, TiffUnsupportedError, UsageError, TiffResult, ColorType}; + +// use self::ifd::Directory; +// use self::image::Image; +use crate::tags::{ + CompressionMethod, PhotometricInterpretation, PlanarConfiguration, Predictor, SampleFormat, + Tag, Type, +}; + +use crate::decoder::{ + stream::ByteOrder, + ifd::Value, + DecodingBuffer, + DecodingResult, + Limits, + ChunkType, +}; + +extern crate async_trait; + +pub use crate::decoder::invert_colors; +use ifd::Directory; +use image::AsyncImage; +use stream::AsyncSmartReader; + +pub mod ifd; +pub mod image; +pub mod stream; +pub mod tag_reader; + +#[async_trait::async_trait] +pub trait RangeReader { + async fn read_range( + &mut self, + bytes_start: usize, + bytes_end: usize, + ) -> futures::io::Result>; +} + +#[async_trait::async_trait] +impl RangeReader for R { + async fn read_range( + &mut self, + bytes_start: usize, + bytes_end: usize, + ) -> futures::io::Result> { + let length = bytes_end - bytes_start; + let mut buffer = vec![0; length]; + + // Seek to the start position + self.seek(SeekFrom::Start(bytes_start as u64)).await?; + + // Read exactly the number of bytes we need + self.read_exact(&mut buffer).await?; + + Ok(buffer) + } +} + +pub struct Decoder { + reader: AsyncSmartReader, + bigtiff: bool, + limits: Limits, // Replace with actual type + next_ifd: Option, + ifd_offsets: Vec, + seen_ifds: HashSet, + pub image: AsyncImage, +} + +impl Decoder { + pub async fn new(mut r: R) -> Result, TiffError> { + let mut endianess = [0; 2]; + r.read_exact(&mut endianess).await?; + let byte_order = match &endianess { + b"II" => ByteOrder::LittleEndian, + b"MM" => ByteOrder::BigEndian, + _ => { + return Err(TiffError::FormatError( + TiffFormatError::TiffSignatureNotFound, + )); + } + }; + + let mut reader = AsyncSmartReader::wrap(r, byte_order); + + let bigtiff = match reader.read_u16().await? { + 42 => false, + 43 => { + if reader.read_u16().await? != 8 { + return Err(TiffError::FormatError( + TiffFormatError::TiffSignatureNotFound, + )); + } + if reader.read_u16().await? != 0 { + return Err(TiffError::FormatError( + TiffFormatError::TiffSignatureNotFound, + )); + } + true + } + _ => { + return Err(TiffError::FormatError( + TiffFormatError::TiffSignatureInvalid, + )); + } + }; + + let next_ifd = if bigtiff { + Some(reader.read_u64().await?) + } else { + Some(u64::from(reader.read_u32().await?)) + }; + + let mut seen_ifds = HashSet::new(); + seen_ifds.insert(next_ifd.unwrap()); + + let mut decoder = Decoder { + reader, + bigtiff, + limits: Default::default(), // Replace with actual initialization + next_ifd, + ifd_offsets: vec![next_ifd.unwrap()], + seen_ifds, + image: AsyncImage { + ifd: None, + width: 0, + height: 0, + bits_per_sample: 1, + samples: 1, + sample_format: SampleFormat::Uint, + photometric_interpretation: PhotometricInterpretation::BlackIsZero, + compression_method: CompressionMethod::None, + jpeg_tables: None, + predictor: Predictor::None, + chunk_type: ChunkType::Tile, + planar_config: PlanarConfiguration::Chunky, + strip_decoder: None, + tile_attributes: None, + chunk_offsets: Vec::new(), + chunk_bytes: Vec::new(), + }, + }; + + decoder.next_image().await?; + + Ok(decoder) + } + + + pub fn with_limits(mut self, limits: Limits) -> Decoder { + self.limits = limits; + self + } + + pub fn dimensions(&mut self) -> TiffResult<(u32, u32)> { + Ok((self.image().width, self.image().height)) + } + + pub fn colortype(&mut self) -> TiffResult { + self.image().colortype() + } + + fn image(&self) -> &AsyncImage { + &self.image + } + + /// reads in the first IFD tag and constructs + // pub async fn read_first_ifd_into_image_metadata() { + + // } + + // pub async fn get_tile(overview: u64, x_index: u64, y_index: u64) -> TiffResult { + + // } + + async fn next_ifd(&mut self) -> TiffResult<(Directory, Option)> { + if self.next_ifd.is_none() { + return Err(TiffError::FormatError( + TiffFormatError::ImageFileDirectoryNotFound, + )); + } + + let (ifd, next_ifd) = Self::read_ifd( + &mut self.reader, + self.bigtiff, + self.next_ifd.take().unwrap(), + ).await?; + + if let Some(next) = next_ifd { + if !self.seen_ifds.insert(next) { + return Err(TiffError::FormatError(TiffFormatError::CycleInOffsets)); + } + self.next_ifd = Some(next); + self.ifd_offsets.push(next); + } + + Ok((ifd, next_ifd)) + } + + /// Reads in the next image. + /// If there is no further image in the TIFF file a format error is returned. + /// To determine whether there are more images call `TIFFDecoder::more_images` instead. + pub async fn next_image(&mut self) -> TiffResult<()> { + let (ifd, _next_ifd) = self.next_ifd().await?; + + self.image = AsyncImage::from_reader(&mut self.reader, ifd, &self.limits, self.bigtiff).await?; + Ok(()) + } + + // Reads the IFD starting at the indicated location. + /// Reads the ifd, skipping all tags. + async fn read_ifd( + reader: &mut AsyncSmartReader, + bigtiff: bool, + ifd_location: u64, + ) -> TiffResult<(Directory, Option)> { + reader.goto_offset(ifd_location).await?; + + let mut dir: Directory = HashMap::new(); + + let num_tags = if bigtiff { + reader.read_u64().await? + } else { + reader.read_u16().await?.into() + }; + + // const TAG_SIZE: i64 = 2 + 2 + 4 + 4; + // reader + // .seek(std::io::SeekFrom::Current( + // TAG_SIZE * i64::try_from(num_tags).unwrap(), + // )) + // .await?; + for _ in 0..num_tags { + let (tag, entry) = match Self::read_entry(reader, bigtiff).await? { + Some(val) => val, + None => { + continue; + } // Unknown data type in tag, skip + }; + dir.insert(tag, entry); + } + + let next_ifd = if bigtiff { + reader.read_u64().await? + } else { + reader.read_u32().await?.into() + }; + + let next_ifd = match next_ifd { + 0 => None, + _ => Some(next_ifd), + }; + + Ok((dir, next_ifd)) + } + + /// Reads a IFD entry. + // An IFD entry has four fields: + // + // Tag 2 bytes + // Type 2 bytes + // Count 4 bytes + // Value 4 bytes either a pointer the value itself + async fn read_entry( + reader: &mut AsyncSmartReader, + bigtiff: bool, + ) -> TiffResult> { + let tag = Tag::from_u16_exhaustive(reader.read_u16().await?); + let type_ = match Type::from_u16(reader.read_u16().await?) { + Some(t) => t, + None => { + // Unknown type. Skip this entry according to spec. + reader.read_u32().await?; + reader.read_u32().await?; + return Ok(None); + } + }; + let entry = if bigtiff { + let mut offset = [0; 8]; + + let count = reader.read_u64().await?; + reader.read_exact(&mut offset).await?; + ifd::Entry::new_u64(type_, count, offset) + } else { + let mut offset = [0; 4]; + + let count = reader.read_u32().await?; + reader.read_exact(&mut offset).await?; + ifd::Entry::new(type_, count, offset) + }; + Ok(Some((tag, entry))) + } + + + /// Tries to retrieve a tag. + /// Return `Ok(None)` if the tag is not present. + pub async fn find_tag(&mut self, tag: Tag) -> TiffResult> { + let entry = match self.image().ifd.as_ref().unwrap().get(&tag) { + None => return Ok(None), + Some(entry) => entry.clone(), + }; + + Ok(Some(entry.val( + &self.limits, + self.bigtiff, + &mut self.reader, + ).await?)) + } + + /// Tries to retrieve a tag and convert it to the desired unsigned type. + pub async fn find_tag_unsigned>(&mut self, tag: Tag) -> TiffResult> { + self.find_tag(tag).await? + .map(|v| v.into_u64()) + .transpose()? + .map(|value| { + T::try_from(value).map_err(|_| TiffFormatError::InvalidTagValueType(tag).into()) + }) + .transpose() + } + + /// Tries to retrieve a vector of all a tag's values and convert them to + /// the desired unsigned type. + pub async fn find_tag_unsigned_vec>( + &mut self, + tag: Tag, + ) -> TiffResult>> { + self.find_tag(tag).await? + .map(|v| v.into_u64_vec()) + .transpose()? + .map(|v| { + v.into_iter() + .map(|u| { + T::try_from(u).map_err(|_| TiffFormatError::InvalidTagValueType(tag).into()) + }) + .collect() + }) + .transpose() + } + + /// Tries to retrieve a tag and convert it to the desired unsigned type. + /// Returns an error if the tag is not present. + pub async fn get_tag_unsigned>(&mut self, tag: Tag) -> TiffResult { + self.find_tag_unsigned(tag).await? + .ok_or_else(|| TiffFormatError::RequiredTagNotFound(tag).into()) + } + + /// Tries to retrieve a tag. + /// Returns an error if the tag is not present + pub async fn get_tag(&mut self, tag: Tag) -> TiffResult { + match self.find_tag(tag).await? { + Some(val) => Ok(val), + None => Err(TiffError::FormatError( + TiffFormatError::RequiredTagNotFound(tag), + )), + } + } + + /// Tries to retrieve a tag and convert it to the desired type. + pub async fn get_tag_u32(&mut self, tag: Tag) -> TiffResult { + self.get_tag(tag).await?.into_u32() + } + pub async fn get_tag_u64(&mut self, tag: Tag) -> TiffResult { + self.get_tag(tag).await?.into_u64() + } + + /// Tries to retrieve a tag and convert it to the desired type. + pub async fn get_tag_f32(&mut self, tag: Tag) -> TiffResult { + self.get_tag(tag).await?.into_f32() + } + + /// Tries to retrieve a tag and convert it to the desired type. + pub async fn get_tag_f64(&mut self, tag: Tag) -> TiffResult { + self.get_tag(tag).await?.into_f64() + } + + /// Tries to retrieve a tag and convert it to the desired type. + pub async fn get_tag_u32_vec(&mut self, tag: Tag) -> TiffResult> { + self.get_tag(tag).await?.into_u32_vec() + } + + pub async fn get_tag_u16_vec(&mut self, tag: Tag) -> TiffResult> { + self.get_tag(tag).await?.into_u16_vec() + } + pub async fn get_tag_u64_vec(&mut self, tag: Tag) -> TiffResult> { + self.get_tag(tag).await?.into_u64_vec() + } + + /// Tries to retrieve a tag and convert it to the desired type. + pub async fn get_tag_f32_vec(&mut self, tag: Tag) -> TiffResult> { + self.get_tag(tag).await?.into_f32_vec() + } + + /// Tries to retrieve a tag and convert it to the desired type. + pub async fn get_tag_f64_vec(&mut self, tag: Tag) -> TiffResult> { + self.get_tag(tag).await?.into_f64_vec() + } + + /// Tries to retrieve a tag and convert it to a 8bit vector. + pub async fn get_tag_u8_vec(&mut self, tag: Tag) -> TiffResult> { + self.get_tag(tag).await?.into_u8_vec() + } + + /// Tries to retrieve a tag and convert it to a ascii vector. + pub async fn get_tag_ascii_string(&mut self, tag: Tag) -> TiffResult { + self.get_tag(tag).await?.into_string() + } + + fn check_chunk_type(&self, expected: ChunkType) -> TiffResult<()> { + if expected != self.image().chunk_type { + return Err(TiffError::UsageError(UsageError::InvalidChunkType( + expected, + self.image().chunk_type, + ))); + } + + Ok(()) + } + + /// The chunk type (Strips / Tiles) of the image + pub fn get_chunk_type(&self) -> ChunkType { + self.image().chunk_type + } + + /// Number of strips in image + pub fn strip_count(&mut self) -> TiffResult { + self.check_chunk_type(ChunkType::Strip)?; + let rows_per_strip = self.image().strip_decoder.as_ref().unwrap().rows_per_strip; + + if rows_per_strip == 0 { + return Ok(0); + } + + // rows_per_strip - 1 can never fail since we know it's at least 1 + let height = match self.image().height.checked_add(rows_per_strip - 1) { + Some(h) => h, + None => return Err(TiffError::IntSizeError), + }; + + let strips = match self.image().planar_config { + PlanarConfiguration::Chunky => height / rows_per_strip, + PlanarConfiguration::Planar => height / rows_per_strip * self.image().samples as u32, + }; + + Ok(strips) + } + + /// Number of tiles in image + pub fn tile_count(&mut self) -> TiffResult { + self.check_chunk_type(ChunkType::Tile)?; + Ok(u32::try_from(self.image().chunk_offsets.len())?) + } + + pub async fn read_chunk_to_buffer( + &mut self, + mut buffer: DecodingBuffer<'_>, + chunk_index: u32, + output_width: usize, + ) -> TiffResult<()> { + let (offset, length) = self.image.chunk_file_range(chunk_index)?; + let v = self.reader.read_range(offset.try_into()?, (offset + length).try_into()?).await?; + + let byte_order = self.reader.byte_order; + + let output_row_stride = (output_width as u64) + .saturating_mul(self.image.samples_per_pixel() as u64) + .saturating_mul(self.image.bits_per_sample as u64) + / 8; + + self.image.expand_chunk( + &mut std::io::Cursor::new(v), + buffer.as_bytes_mut(), + output_row_stride.try_into()?, + byte_order, + chunk_index, + &self.limits, + )?; + + Ok(()) + } + + fn result_buffer(&self, width: usize, height: usize) -> TiffResult { + let buffer_size = match width + .checked_mul(height) + .and_then(|x| x.checked_mul(self.image().samples_per_pixel())) + { + Some(s) => s, + None => return Err(TiffError::LimitsExceeded), + }; + + let max_sample_bits = self.image().bits_per_sample; + match self.image().sample_format { + SampleFormat::Uint => match max_sample_bits { + n if n <= 8 => DecodingResult::new_u8(buffer_size, &self.limits), + n if n <= 16 => DecodingResult::new_u16(buffer_size, &self.limits), + n if n <= 32 => DecodingResult::new_u32(buffer_size, &self.limits), + n if n <= 64 => DecodingResult::new_u64(buffer_size, &self.limits), + n => Err(TiffError::UnsupportedError( + TiffUnsupportedError::UnsupportedBitsPerChannel(n), + )), + }, + SampleFormat::IEEEFP => match max_sample_bits { + 32 => DecodingResult::new_f32(buffer_size, &self.limits), + 64 => DecodingResult::new_f64(buffer_size, &self.limits), + n => Err(TiffError::UnsupportedError( + TiffUnsupportedError::UnsupportedBitsPerChannel(n), + )), + }, + SampleFormat::Int => match max_sample_bits { + n if n <= 8 => DecodingResult::new_i8(buffer_size, &self.limits), + n if n <= 16 => DecodingResult::new_i16(buffer_size, &self.limits), + n if n <= 32 => DecodingResult::new_i32(buffer_size, &self.limits), + n if n <= 64 => DecodingResult::new_i64(buffer_size, &self.limits), + n => Err(TiffError::UnsupportedError( + TiffUnsupportedError::UnsupportedBitsPerChannel(n), + )), + }, + format => Err(TiffUnsupportedError::UnsupportedSampleFormat(vec![format]).into()), + } + } + + /// Read the specified chunk (at index `chunk_index`) and return the binary data as a Vector. + pub async fn read_chunk(&mut self, chunk_index: u32) -> TiffResult { + let data_dims = self.image().chunk_data_dimensions(chunk_index)?; + + let mut result = self.result_buffer(data_dims.0 as usize, data_dims.1 as usize)?; + + self.read_chunk_to_buffer(result.as_buffer(0), chunk_index, data_dims.0 as usize).await?; + + Ok(result) + } + + /// Returns the default chunk size for the current image. Any given chunk in the image is at most as large as + /// the value returned here. For the size of the data (chunk minus padding), use `chunk_data_dimensions`. + pub fn chunk_dimensions(&self) -> (u32, u32) { + self.image().chunk_dimensions().unwrap() + } + + /// Returns the size of the data in the chunk with the specified index. This is the default size of the chunk, + /// minus any padding. + pub fn chunk_data_dimensions(&self, chunk_index: u32) -> (u32, u32) { + self.image() + .chunk_data_dimensions(chunk_index) + .expect("invalid chunk_index") + } + + /// Decodes the entire image and return it as a Vector + pub async fn read_image(&mut self) -> TiffResult { + let width = self.image().width; + let height = self.image().height; + let mut result = self.result_buffer(width as usize, height as usize)?; + if width == 0 || height == 0 { + return Ok(result); + } + + let chunk_dimensions = self.image().chunk_dimensions()?; + let chunk_dimensions = ( + chunk_dimensions.0.min(width), + chunk_dimensions.1.min(height), + ); + if chunk_dimensions.0 == 0 || chunk_dimensions.1 == 0 { + return Err(TiffError::FormatError( + TiffFormatError::InconsistentSizesEncountered, + )); + } + + let samples = self.image().samples_per_pixel(); + if samples == 0 { + return Err(TiffError::FormatError( + TiffFormatError::InconsistentSizesEncountered, + )); + } + + let output_row_bits = (width as u64 * self.image.bits_per_sample as u64) + .checked_mul(samples as u64) + .ok_or(TiffError::LimitsExceeded)?; + let output_row_stride: usize = ((output_row_bits + 7) / 8).try_into()?; + + let chunk_row_bits = (chunk_dimensions.0 as u64 * self.image.bits_per_sample as u64) + .checked_mul(samples as u64) + .ok_or(TiffError::LimitsExceeded)?; + let chunk_row_bytes: usize = ((chunk_row_bits + 7) / 8).try_into()?; + + let chunks_across = ((width - 1) / chunk_dimensions.0 + 1) as usize; + + if chunks_across > 1 && chunk_row_bits % 8 != 0 { + return Err(TiffError::UnsupportedError( + TiffUnsupportedError::MisalignedTileBoundaries, + )); + } + + // in planar config, an image has chunks/n_bands chunks + let image_chunks = self.image().chunk_offsets.len() / self.image().strips_per_pixel(); + // For multi-band images, only the first band is read. + // Possible improvements: + // * pass requested band as parameter + // * collect bands to a RGB encoding result in case of RGB bands + for chunk in 0..image_chunks { + let (offset, length) = self.image.chunk_file_range(chunk.try_into().unwrap())?; + let v = self.reader.read_range(offset.try_into()?, (offset + length).try_into()?).await?; + let mut reader = std::io::Cursor::new(v); + // self.goto_offset_u64(self.image().chunk_offsets[chunk]).await?; + + let x = chunk % chunks_across; + let y = chunk / chunks_across; + let buffer_offset = + y * output_row_stride * chunk_dimensions.1 as usize + x * chunk_row_bytes; + let byte_order = self.reader.byte_order; + self.image.expand_chunk( + &mut reader, + &mut result.as_buffer(0).as_bytes_mut()[buffer_offset..], + output_row_stride, + byte_order, + chunk as u32, + &self.limits, + )?; + } + + Ok(result) + } + + + #[inline] + pub async fn goto_offset_u64(&mut self, offset: u64) -> std::io::Result<()> { + self.reader.seek(SeekFrom::Start(offset)).await.map(|_| ()) + } +} + +#[cfg(test)] +mod test { + use super::*; +} diff --git a/src/decoder_async/readme.md b/src/decoder_async/readme.md new file mode 100644 index 00000000..a0cd4527 --- /dev/null +++ b/src/decoder_async/readme.md @@ -0,0 +1,32 @@ +# Making Tiff crate async for Cogs + +This folder is a bunch of copied code, made async with futures crate. This makes the reader runtime agnostic, as well as backend agnostic. This allows it to be used anywhere (tokio, smol, bevy), from any datasource that provides an AsyncReader. For using this on Sync stuff, such as files, use the following snippet: + +```rust +use futures::io::AllowStdIo; +use std::fs::File; +use tiff::decoder_async::Decoder; + +#[tokio::main] +fn main() { + let f = AllowStdIo(File("/path/to/file.tiff")); + let mut decoder = Decoder::new(f).await.expect("Could not open file"); + let result = decoder.read_image(); +} +``` + +For more fine-grained control over what the tiff does, another method, `new_raw` is provided, that does not read all tags by default, since this caused excessive requests when testing on isdasoil data. + +The plan is: + +1. Read image and check header +2. Scan all IFD tags to see how many overviews there are + - may actually read into values that don't hold pointers, since we're ideally buffering anyways. ChatGPT says all metadata (excluding tile offsets) should be in the first ~16kB for COGs, which is a - 16 * 1024 byte buffer, which is not that big. + - The call chain that slows execution is: `decoder.next_image()->tag_reader.find_tag(tag)->entry.clone().val()->case 4 entry.decode_offset()` where `decode_offset` possibly goes out of the currently read buffer. Of course, this could be circumvented by having a grow-only buffer, but then the reader and decoder would have to be more tightly coupled + - another alternative (which I kind of like) is to (optionally) store offset data in our own buffer that is the right size. However, for the largest overview, even this may be kind of big? + - ideally, we would be able to pass as arguments to our reader: read_bytes(a, b), because that would directly translate to a range request. <- this could be a homebrew trait with a blanket implementation? +3. Read only (the first?) IFD tag +4. Load tiles on-demand + + +I think the current implementation is mainly inefficient because tags that hold more data than what fits in an IFD entries' Value field, the Value is a pointer that gets followed through (because a file was assumed). diff --git a/src/decoder_async/stream.rs b/src/decoder_async/stream.rs new file mode 100644 index 00000000..40ea32a3 --- /dev/null +++ b/src/decoder_async/stream.rs @@ -0,0 +1,353 @@ +// Special thanks to Alice for the help: https://users.rust-lang.org/t/63019/6 +use crate::decoder::stream::ByteOrder; +use std::io::{Result, SeekFrom}; +use std::pin::Pin; + +use futures::{ + future::BoxFuture, + io::{AsyncRead, AsyncReadExt, AsyncSeek, AsyncSeekExt, BufReader}, + Future, +}; + +// pub struct RangedStreamer { +// pos: u64, +// length: u64, // total size +// state: State, +// range_get: F, +// min_request_size: usize, // requests have at least this size +// } + +// enum State { +// HasChunk(SeekOutput), +// Seeking(BoxFuture<'static, std::io::Result>), +// } + +// pub struct SeekOutput { +// pub start: u64, +// pub data: Vec, +// } + +// pub type F = std::sync::Arc< +// dyn Fn(u64, usize) -> BoxFuture<'static, std::io::Result> + Send + Sync, +// >; + +// impl RangedStreamer { +// pub fn new(length: usize, min_request_size: usize, range_get: F) -> Self { +// let length = length as u64; +// Self { +// pos: 0, +// length, +// state: State::HasChunk(SeekOutput { +// start: 0, +// data: vec![], +// }), +// range_get, +// min_request_size, +// } +// } +// } + +// // whether `test_interval` is inside `a` (start, length). +// async fn range_includes(a: (usize, usize), test_interval: (usize, usize)) -> bool { +// if test_interval.0 < a.0 { +// return false; +// } +// let test_end = test_interval.0 + test_interval.1; +// let a_end = a.0 + a.1; +// if test_end > a_end { +// return false; +// } +// true +// } + +// impl AsyncRead for RangedStreamer { +// fn poll_read( +// mut self: std::pin::Pin<&mut Self>, +// cx: &mut std::task::Context<'_>, +// buf: &mut [u8], +// ) -> std::task::Poll> { +// let requested_range = (self.pos as usize, buf.len()); +// let min_request_size = self.min_request_size; +// match &mut self.state { +// State::HasChunk(output) => { +// let existing_range = (output.start as usize, output.data.len()); +// if range_includes(existing_range, requested_range) { +// let offset = requested_range.0 - existing_range.0; +// buf.copy_from_slice(&output.data[offset..offset + buf.len()]); +// self.pos += buf.len() as u64; +// std::task::Poll::Ready(Ok(buf.len())) +// } else { +// let start = requested_range.0 as u64; +// let length = std::cmp::max(min_request_size, requested_range.1); +// let future = (self.range_get)(start, length); +// self.state = State::Seeking(Box::pin(future)); +// self.poll_read(cx, buf) +// } +// } +// State::Seeking(ref mut future) => match Pin::new(future).poll(cx) { +// std::task::Poll::Ready(v) => { +// match v { +// Ok(output) => self.state = State::HasChunk(output), +// Err(e) => return std::task::Poll::Ready(Err(e)), +// }; +// self.poll_read(cx, buf) +// } +// std::task::Poll::Pending => std::task::Poll::Pending, +// }, +// } +// } +// } + +// impl AsyncSeek for RangedStreamer { +// fn poll_seek( +// mut self: std::pin::Pin<&mut Self>, +// _: &mut std::task::Context<'_>, +// pos: std::io::SeekFrom, +// ) -> std::task::Poll> { +// match pos { +// SeekFrom::Start(pos) => self.pos = pos, +// SeekFrom::End(pos) => self.pos = (self.length as i64 + pos) as u64, +// SeekFrom::Current(pos) => self.pos = (self.pos as i64 + pos) as u64, +// }; +// std::task::Poll::Ready(Ok(self.pos)) +// } +// } + +// pub type DeflateReader = flate2::read::ZlibDecoder; + +// /// +// /// ## LZW Reader +// /// + +// /// Reader that decompresses LZW streams +// pub struct LZWReader { +// reader: BufReader>, +// decoder: weezl::decode::Decoder, +// } + +// impl LZWReader { +// /// Wraps a reader +// pub fn new(reader: R, compressed_length: usize) -> LZWReader { +// Self { +// reader: BufReader::with_capacity( +// (32 * 1024).min(compressed_length), +// reader.take(u64::try_from(compressed_length).unwrap()), +// ), +// decoder: weezl::decode::Decoder::with_tiff_size_switch(weezl::BitOrder::Msb, 8), +// } +// } +// } + +// impl Read for LZWReader { +// fn read(&mut self, buf: &mut [u8]) -> io::Result { +// loop { +// let result = self.decoder.decode_bytes(self.reader.fill_buf()?, buf); +// self.reader.consume(result.consumed_in); + +// match result.status { +// Ok(weezl::LzwStatus::Ok) => { +// if result.consumed_out == 0 { +// continue; +// } else { +// return Ok(result.consumed_out); +// } +// } +// Ok(weezl::LzwStatus::NoProgress) => { +// assert_eq!(result.consumed_in, 0); +// assert_eq!(result.consumed_out, 0); +// assert!(self.reader.buffer().is_empty()); +// return Err(io::Error::new( +// io::ErrorKind::UnexpectedEof, +// "no lzw end code found", +// )); +// } +// Ok(weezl::LzwStatus::Done) => { +// return Ok(result.consumed_out); +// } +// Err(err) => return Err(io::Error::new(io::ErrorKind::InvalidData, err)), +// } +// } +// } +// } + +pub struct AsyncSmartReader { + reader: R, + pub byte_order: ByteOrder, +} + +impl AsyncSmartReader { + pub async fn goto_offset(&mut self, offset: u64) -> Result<()> { + self.reader.seek(SeekFrom::Start(offset)).await.map(|_| ()) + } + + pub fn wrap(reader: R, byte_order: ByteOrder) -> Self { + AsyncSmartReader { reader, byte_order } + } +} + +impl AsyncRead for AsyncSmartReader { + fn poll_read( + self: Pin<&mut Self>, + cx: &mut std::task::Context<'_>, + buf: &mut [u8], + ) -> std::task::Poll> { + let pinned = std::pin::Pin::new(&mut self.get_mut().reader); + pinned.poll_read(cx, buf) + } +} + +impl AsyncSeek for AsyncSmartReader { + fn poll_seek( + self: Pin<&mut Self>, + cx: &mut std::task::Context<'_>, + pos: SeekFrom, + ) -> std::task::Poll> { + let pinned = std::pin::Pin::new(&mut self.get_mut().reader); + pinned.poll_seek(cx, pos) + } +} + +impl AsyncSmartReader { + /// Byte order that should be adhered to + pub fn byte_order(&self) -> ByteOrder { + self.byte_order + } + + /// Reads an u16 + #[inline(always)] + pub async fn read_u16(&mut self) -> Result { + let mut n = [0u8; 2]; + self.read_exact(&mut n).await?; + Ok(match self.byte_order() { + ByteOrder::LittleEndian => u16::from_le_bytes(n), + ByteOrder::BigEndian => u16::from_be_bytes(n), + }) + } + + /// Reads an i8 + #[inline(always)] + pub async fn read_i8(&mut self) -> Result { + let mut n = [0u8; 1]; + self.read_exact(&mut n).await?; + Ok(match self.byte_order() { + ByteOrder::LittleEndian => i8::from_le_bytes(n), + ByteOrder::BigEndian => i8::from_be_bytes(n), + }) + } + + /// Reads an i16 + #[inline(always)] + pub async fn read_i16(&mut self) -> Result { + let mut n = [0u8; 2]; + self.read_exact(&mut n).await?; + Ok(match self.byte_order() { + ByteOrder::LittleEndian => i16::from_le_bytes(n), + ByteOrder::BigEndian => i16::from_be_bytes(n), + }) + } + + /// Reads an u32 + #[inline(always)] + pub async fn read_u32(&mut self) -> Result { + let mut n = [0u8; 4]; + self.read_exact(&mut n).await?; + Ok(match self.byte_order() { + ByteOrder::LittleEndian => u32::from_le_bytes(n), + ByteOrder::BigEndian => u32::from_be_bytes(n), + }) + } + + /// Reads an i32 + #[inline(always)] + pub async fn read_i32(&mut self) -> Result { + let mut n = [0u8; 4]; + self.read_exact(&mut n).await?; + Ok(match self.byte_order() { + ByteOrder::LittleEndian => i32::from_le_bytes(n), + ByteOrder::BigEndian => i32::from_be_bytes(n), + }) + } + + /// Reads an u64 + #[inline(always)] + pub async fn read_u64(&mut self) -> Result { + let mut n = [0u8; 8]; + self.read_exact(&mut n).await?; + Ok(match self.byte_order() { + ByteOrder::LittleEndian => u64::from_le_bytes(n), + ByteOrder::BigEndian => u64::from_be_bytes(n), + }) + } + + /// Reads an i64 + #[inline(always)] + pub async fn read_i64(&mut self) -> Result { + let mut n = [0u8; 8]; + self.read_exact(&mut n).await?; + Ok(match self.byte_order() { + ByteOrder::LittleEndian => i64::from_le_bytes(n), + ByteOrder::BigEndian => i64::from_be_bytes(n), + }) + } + + /// Reads an f32 + #[inline(always)] + pub async fn read_f32(&mut self) -> Result { + let mut n = [0u8; 4]; + self.read_exact(&mut n).await?; + Ok(f32::from_bits(match self.byte_order() { + ByteOrder::LittleEndian => u32::from_le_bytes(n), + ByteOrder::BigEndian => u32::from_be_bytes(n), + })) + } + + /// Reads an f64 + #[inline(always)] + pub async fn read_f64(&mut self) -> Result { + let mut n = [0u8; 8]; + self.read_exact(&mut n).await?; + Ok(f64::from_bits(match self.byte_order() { + ByteOrder::LittleEndian => u64::from_le_bytes(n), + ByteOrder::BigEndian => u64::from_be_bytes(n), + })) + } +} +// /// Reader that is aware of the byte order. +// #[derive(Debug)] +// pub struct AsyncSmartReader +// where +// R: AsyncRead, +// { +// reader: R, +// pub byte_order: ByteOrder, +// } + +// impl AsyncSmartReader +// where +// R: AsyncRead, +// { +// /// Wraps a reader +// pub fn wrap(reader: R, byte_order: ByteOrder) -> AsyncSmartReader { +// AsyncSmartReader { reader, byte_order } +// } +// pub fn into_inner(self) -> R { +// self.reader +// } +// } +// impl AsyncSmartReader { + +// #[inline(always)] +// fn byte_order(&self) -> ByteOrder { +// self.byte_order +// } +// } + +// impl AsyncSeek for AsyncSmartReader { +// fn poll_seek( +// self: Pin<&mut Self>, +// cx: &mut std::task::Context<'_>, +// pos: SeekFrom, +// ) -> std::task::Poll> { + +// } +// } diff --git a/src/decoder_async/tag_reader.rs b/src/decoder_async/tag_reader.rs new file mode 100644 index 00000000..a42c777c --- /dev/null +++ b/src/decoder_async/tag_reader.rs @@ -0,0 +1,48 @@ +use crate::tags::Tag; +use crate::{TiffError, TiffFormatError, TiffResult}; + +use crate::decoder::{ifd::Value, Limits}; +use crate::decoder_async::{stream::AsyncSmartReader, Directory}; + +use futures::{AsyncRead, AsyncSeek}; + +pub(crate) struct AsyncTagReader<'a, R: AsyncRead + AsyncSeek + Unpin + Send> { + pub reader: &'a mut AsyncSmartReader, + pub ifd: &'a Directory, + pub limits: &'a Limits, + pub bigtiff: bool, +} +impl<'a, R: AsyncRead + AsyncSeek + Unpin + Send> AsyncTagReader<'a, R> { + pub(crate) async fn find_tag(&mut self, tag: Tag) -> TiffResult> { + Ok(match self.ifd.get(&tag) { + Some(entry) => Some( + entry + .clone() + .val(self.limits, self.bigtiff, self.reader) + .await?, + ), + None => None, + }) + } + pub(crate) async fn require_tag(&mut self, tag: Tag) -> TiffResult { + match self.find_tag(tag).await? { + Some(val) => Ok(val), + None => Err(TiffError::FormatError( + TiffFormatError::RequiredTagNotFound(tag), + )), + } + } + pub(crate) async fn find_tag_uint_vec>(&mut self, tag: Tag) -> TiffResult>> { + self.find_tag(tag).await? + .map(|v| v.into_u64_vec()) + .transpose()? + .map(|v| { + v.into_iter() + .map(|u| { + T::try_from(u).map_err(|_| TiffFormatError::InvalidTagValueType(tag).into()) + }) + .collect() + }) + .transpose() + } +} diff --git a/src/lib.rs b/src/lib.rs index a4522ced..4e06ed4d 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -11,6 +11,7 @@ extern crate weezl; mod bytecast; pub mod decoder; +pub mod decoder_async; pub mod encoder; mod error; pub mod tags; diff --git a/src/tags.rs b/src/tags.rs index 6c18fa52..f1305711 100644 --- a/src/tags.rs +++ b/src/tags.rs @@ -208,6 +208,10 @@ pub enum PlanarConfiguration(u16) { } tags! { +/// Additional compression methods: +/// - *None*: No predictor is used. This is the default mode, meaning the pixel values are stored without modification. +/// - *Horizontal Differencing*: For each pixel, the predictor calculates the difference between the current pixel and the previous pixel in the same row. +/// - *Floating Point Predictor*: Based on a similar concept as Predictor 2, but it is specific to floating-point pixel values. pub enum Predictor(u16) { /// No changes were made to the data None = 1, diff --git a/tests/decode_images_async.rs b/tests/decode_images_async.rs new file mode 100644 index 00000000..d6e7f022 --- /dev/null +++ b/tests/decode_images_async.rs @@ -0,0 +1,514 @@ +extern crate tiff; + +use tiff::decoder_async::{ifd, Decoder}; +use tiff::decoder::DecodingResult; +use tiff::ColorType; + +use futures::io::AllowStdIo; + +use std::fs::File; +use std::path::PathBuf; + +const TEST_IMAGE_DIR: &str = "./tests/images/"; + +macro_rules! test_image_sum { + ($name:ident, $buffer:ident, $sum_ty:ty) => { + async fn $name(file: &str, expected_type: ColorType, expected_sum: $sum_ty) { + let path = PathBuf::from(TEST_IMAGE_DIR).join(file); + let img_file = AllowStdIo::new(File::open(path).expect("Cannot find test image!")); + let mut decoder = Decoder::new(img_file).await.expect("Cannot create decoder"); + assert_eq!(decoder.colortype().unwrap(), expected_type); + let img_res = decoder.read_image().await.unwrap(); + + match img_res { + DecodingResult::$buffer(res) => { + let sum: $sum_ty = res.into_iter().map(<$sum_ty>::from).sum(); + assert_eq!(sum, expected_sum); + } + _ => panic!("Wrong bit depth"), + } + } + }; +} + +test_image_sum!(test_image_sum_u8, U8, u64); +test_image_sum!(test_image_sum_i8, I8, i64); +test_image_sum!(test_image_sum_u16, U16, u64); +test_image_sum!(test_image_sum_i16, I16, i64); +test_image_sum!(test_image_sum_u32, U32, u64); +test_image_sum!(test_image_sum_u64, U64, u64); +test_image_sum!(test_image_sum_f32, F32, f32); +test_image_sum!(test_image_sum_f64, F64, f64); + +/// Tests that a decoder can be constructed for an image and the color type +/// read from the IFD and is of the appropriate type, but the type is +/// unsupported. +async fn test_image_color_type_unsupported(file: &str, expected_type: ColorType) { + let path = PathBuf::from(TEST_IMAGE_DIR).join(file); + let img_file = AllowStdIo::new(File::open(path).expect("Cannot find test image!")); + let mut decoder = Decoder::new(img_file).await.expect("Cannot create decoder"); + assert_eq!(decoder.colortype().unwrap(), expected_type); + assert!(match decoder.read_image().await { + Err(tiff::TiffError::UnsupportedError( + tiff::TiffUnsupportedError::UnsupportedColorType(_), + )) => true, + _ => false, + }); +} + +#[tokio::test] +async fn test_cmyk_u8() { + test_image_sum_u8("cmyk-3c-8b.tiff", ColorType::CMYK(8), 8522658).await; +} + +#[tokio::test] +async fn test_cmyk_u16() { + test_image_sum_u16("cmyk-3c-16b.tiff", ColorType::CMYK(16), 2181426827).await; +} + +#[tokio::test] +async fn test_cmyk_f32() { + test_image_sum_f32("cmyk-3c-32b-float.tiff", ColorType::CMYK(32), 496.0405).await; +} + +#[tokio::test] +async fn test_gray_u8() { + test_image_sum_u8("minisblack-1c-8b.tiff", ColorType::Gray(8), 2840893); +} + +#[tokio::test] +async fn test_gray_u12() { + test_image_color_type_unsupported("12bit.cropped.tiff", ColorType::Gray(12)); +} + +#[tokio::test] +async fn test_gray_u16() { + test_image_sum_u16("minisblack-1c-16b.tiff", ColorType::Gray(16), 733126239); +} + +#[tokio::test] +async fn test_gray_u32() { + test_image_sum_u32("gradient-1c-32b.tiff", ColorType::Gray(32), 549892913787); +} + +#[tokio::test] +async fn test_gray_u64() { + test_image_sum_u64("gradient-1c-64b.tiff", ColorType::Gray(64), 549892913787); +} + +#[tokio::test] +async fn test_gray_f32() { + test_image_sum_f32("gradient-1c-32b-float.tiff", ColorType::Gray(32), 128.03194); +} + +#[tokio::test] +async fn test_gray_f64() { + test_image_sum_f64( + "gradient-1c-64b-float.tiff", + ColorType::Gray(64), + 128.0319210877642, + ); +} + +#[tokio::test] +async fn test_rgb_u8() { + test_image_sum_u8("rgb-3c-8b.tiff", ColorType::RGB(8), 7842108); +} + +#[tokio::test] +async fn test_rgb_u12() { + test_image_color_type_unsupported("12bit.cropped.rgb.tiff", ColorType::RGB(12)); +} + +#[tokio::test] +async fn test_rgb_u16() { + test_image_sum_u16("rgb-3c-16b.tiff", ColorType::RGB(16), 2024349944); +} + +#[tokio::test] +async fn test_rgb_u32() { + test_image_sum_u32("gradient-3c-32b.tiff", ColorType::RGB(32), 2030834111716); +} + +#[tokio::test] +async fn test_rgb_u64() { + test_image_sum_u64("gradient-3c-64b.tiff", ColorType::RGB(64), 2030834111716); +} + +#[tokio::test] +async fn test_rgb_f32() { + test_image_sum_f32("gradient-3c-32b-float.tiff", ColorType::RGB(32), 472.8405); +} + +#[tokio::test] +async fn test_int8() { + test_image_sum_i8("int8.tif", ColorType::Gray(8), 3111); +} + +#[tokio::test] +async fn test_int8_rgb() { + test_image_sum_i8("int8_rgb.tif", ColorType::RGB(8), -10344); +} + +#[tokio::test] +async fn test_int16() { + test_image_sum_i16("int16.tif", ColorType::Gray(16), 354396); +} + +#[tokio::test] +async fn test_int16_rgb() { + test_image_sum_i16("int16_rgb.tif", ColorType::RGB(16), 1063188); +} + +#[tokio::test] +async fn test_string_tags() { + // these files have null-terminated strings for their Software tag. One has extra bytes after + // the null byte, so we check both to ensure that we're truncating properly + let filenames = ["minisblack-1c-16b.tiff", "rgb-3c-16b.tiff"]; + for filename in filenames.iter() { + let path = PathBuf::from(TEST_IMAGE_DIR).join(filename); + let img_file = AllowStdIo::new(File::open(path).expect("Cannot find test image!")); + let mut decoder = Decoder::new(img_file).await.expect("Cannot create decoder"); + let software = decoder.get_tag(tiff::tags::Tag::Software).await.unwrap(); + match software { + tiff::decoder::ifd::Value::Ascii(s) => assert_eq!( + &s, + "GraphicsMagick 1.2 unreleased Q16 http://www.GraphicsMagick.org/" + ), + _ => assert!(false), + }; + } +} + +#[tokio::test] +async fn test_decode_data() { + let mut image_data = Vec::new(); + for x in 0..100 { + for y in 0..100u8 { + let val = x + y; + image_data.push(val); + image_data.push(val); + image_data.push(val); + } + } + let file = AllowStdIo::new(File::open("./tests/decodedata-rgb-3c-8b.tiff").unwrap()); + let mut decoder = Decoder::new(file).await.unwrap(); + assert_eq!(decoder.colortype().unwrap(), ColorType::RGB(8)); + assert_eq!(decoder.dimensions().unwrap(), (100, 100)); + if let DecodingResult::U8(img_res) = decoder.read_image().await.unwrap() { + assert_eq!(image_data, img_res); + } else { + panic!("Wrong data type"); + } +} + +#[tokio::test] +async fn issue_69() { + test_image_sum_u16("issue_69_lzw.tiff", ColorType::Gray(16), 1015486); + test_image_sum_u16("issue_69_packbits.tiff", ColorType::Gray(16), 1015486); +} + +// TODO: GrayA support +//#[tokio::test] +//fn test_gray_alpha_u8() +//{ +//let img_file = File::open("./tests/images/minisblack-2c-8b-alpha.tiff").expect("Cannot find test image!"); +//let mut decoder = Decoder::new(img_file).expect("Cannot create decoder"); +//assert_eq!(decoder.colortype().unwrap(), ColorType::GrayA(8)); +//let img_res = decoder.read_image(); +//assert!(img_res.is_ok()); +//} + +#[tokio::test] +async fn test_tiled_rgb_u8() { + test_image_sum_u8("tiled-rgb-u8.tif", ColorType::RGB(8), 39528948); +} + +#[tokio::test] +async fn test_tiled_rect_rgb_u8() { + test_image_sum_u8("tiled-rect-rgb-u8.tif", ColorType::RGB(8), 62081032); +} + +/* #[tokio::test] +async fn test_tiled_jpeg_rgb_u8() { + test_image_sum_u8("tiled-jpeg-rgb-u8.tif", ColorType::RGB(8), 93031606); +} */ +#[tokio::test] +async fn test_tiled_oversize_gray_i8() { + test_image_sum_i8("tiled-oversize-gray-i8.tif", ColorType::Gray(8), 1214996); +} + +#[tokio::test] +async fn test_tiled_cmyk_i8() { + test_image_sum_i8("tiled-cmyk-i8.tif", ColorType::CMYK(8), 1759101); +} + +#[tokio::test] +async fn test_tiled_incremental() { + let file = "tiled-rgb-u8.tif"; + let expected_type = ColorType::RGB(8); + let sums = [ + 188760, 195639, 108148, 81986, 665088, 366140, 705317, 423366, 172033, 324455, 244102, + 81853, 181258, 247971, 129486, 55600, 565625, 422102, 730888, 379271, 232142, 292549, + 244045, 86866, 188141, 115036, 150785, 84389, 353170, 459325, 719619, 329594, 278663, + 220474, 243048, 113563, 189152, 109684, 179391, 122188, 279651, 622093, 724682, 302459, + 268428, 204499, 224255, 124674, 170668, 121868, 192768, 183367, 378029, 585651, 657712, + 296790, 241444, 197083, 198429, 134869, 182318, 86034, 203655, 182338, 297255, 601284, + 633813, 242531, 228578, 206441, 193552, 125412, 181527, 165439, 202531, 159538, 268388, + 565790, 611382, 272967, 236497, 215154, 158881, 90806, 106114, 182342, 191824, 186138, + 215174, 393193, 701228, 198866, 227944, 193830, 166330, 49008, 55719, 122820, 197316, + 161969, 203152, 170986, 624427, 188605, 186187, 111064, 115192, 39538, 48626, 163929, + 144682, 135796, 194141, 154198, 584125, 180255, 153524, 121433, 132641, 35743, 47798, + 152343, 162874, 167664, 160175, 133038, 659882, 138339, 166470, 124173, 118929, 51317, + 45267, 155776, 161331, 161006, 130052, 137618, 337291, 106481, 161999, 127343, 87724, + 59540, 63907, 155677, 140668, 141523, 108061, 168657, 186482, 98599, 147614, 139963, 90444, + 56602, 92547, 125644, 134212, 126569, 144153, 179800, 174516, 133969, 129399, 117681, + 83305, 55075, 110737, 115108, 128572, 128911, 130922, 179986, 143288, 145884, 155856, + 96683, 94057, 56238, 79649, 71651, 70182, 75010, 77009, 98855, 78979, 74341, 83482, 53403, + 59842, 30305, + ]; + + let path = PathBuf::from(TEST_IMAGE_DIR).join(file); + let img_file = AllowStdIo::new(File::open(path).expect("Cannot find test image!")); + let mut decoder = Decoder::new(img_file).await.expect("Cannot create decoder"); + assert_eq!(decoder.colortype().unwrap(), expected_type); + + let tiles = decoder.tile_count().unwrap(); + assert_eq!(tiles as usize, sums.len()); + + for tile in 0..tiles { + match decoder.read_chunk(tile).await.unwrap() { + DecodingResult::U8(res) => { + let sum: u64 = res.into_iter().map(::from).sum(); + assert_eq!(sum, sums[tile as usize]); + } + _ => panic!("Wrong bit depth"), + } + } +} + +#[tokio::test] +async fn test_planar_rgb_u8() { + // gdal_translate tiled-rgb-u8.tif planar-rgb-u8.tif -co INTERLEAVE=BAND -co COMPRESS=LZW -co PROFILE=BASELINE + let file = "planar-rgb-u8.tif"; + let expected_type = ColorType::RGB(8); + + let path = PathBuf::from(TEST_IMAGE_DIR).join(file); + let img_file = AllowStdIo::new(File::open(path).expect("Cannot find test image!")); + let mut decoder = Decoder::new(img_file).await.expect("Cannot create decoder"); + assert_eq!(decoder.colortype().unwrap(), expected_type); + + let chunks = decoder.strip_count().unwrap(); + assert_eq!(chunks as usize, 72); + + // convert -quiet planar-rgb-u8.tif[0] -crop 1x1+0+0 txt: + // 0,0: (73,51,30) #49331E srgb(73,51,30) + + // 1st band (red) + match decoder.read_chunk(0).await.unwrap() { + DecodingResult::U8(chunk) => { + assert_eq!(chunk[0], 73); + } + _ => panic!("Wrong bit depth"), + } + // 2nd band (green) + match decoder.read_chunk(chunks / 3).await.unwrap() { + DecodingResult::U8(chunk) => { + assert_eq!(chunk[0], 51); + } + _ => panic!("Wrong bit depth"), + } + // 3rd band (blue) + match decoder.read_chunk(chunks / 3 * 2).await.unwrap() { + DecodingResult::U8(chunk) => { + assert_eq!(chunk[0], 30); + } + _ => panic!("Wrong bit depth"), + } + + test_image_sum_u8(file, ColorType::RGB(8), 15417630); +} + +#[tokio::test] +async fn test_div_zero() { + use tiff::{TiffError, TiffFormatError}; + + let image = [ + 73, 73, 42, 0, 8, 0, 0, 0, 8, 0, 0, 1, 4, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 40, 1, 0, 0, + 0, 158, 0, 0, 251, 3, 1, 3, 0, 1, 0, 0, 0, 1, 0, 0, 39, 6, 1, 3, 0, 1, 0, 0, 0, 0, 0, 0, 0, + 17, 1, 4, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 1, 3, 0, 1, 0, 0, 0, 158, 0, 0, 251, 67, 1, 3, 0, + 1, 0, 0, 0, 40, 0, 0, 0, 66, 1, 4, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 178, 178, 178, 178, + 178, 178, 178, + ]; + + let err = tiff::decoder::Decoder::new(std::io::Cursor::new(&image)).unwrap_err(); + + match err { + TiffError::FormatError(TiffFormatError::StripTileTagConflict) => {} + unexpected => panic!("Unexpected error {}", unexpected), + } +} + +#[tokio::test] +async fn test_too_many_value_bytes() { + let image = [ + 73, 73, 43, 0, 8, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 255, 0, 8, 0, 0, 0, + 23, 0, 12, 0, 0, 65, 4, 0, 1, 6, 0, 0, 1, 16, 0, 1, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, + 0, 0, 3, 0, 1, 0, 0, 0, 1, 0, 0, 0, 59, 73, 84, 186, 202, 83, 240, 66, 1, 53, 22, 56, 47, + 0, 0, 0, 0, 0, 0, 1, 222, 4, 0, 58, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 1, 1, 4, 0, 0, 100, 0, + 0, 89, 89, 89, 89, 89, 89, 89, 89, 96, 1, 20, 89, 89, 89, 89, 18, + ]; + + let error = tiff::decoder::Decoder::new(std::io::Cursor::new(&image)).unwrap_err(); + + match error { + tiff::TiffError::LimitsExceeded => {} + unexpected => panic!("Unexpected error {}", unexpected), + } +} + +#[tokio::test] +async fn fuzzer_testcase5() { + let image = [ + 73, 73, 42, 0, 8, 0, 0, 0, 8, 0, 0, 1, 4, 0, 1, 0, 0, 0, 100, 0, 0, 0, 1, 1, 4, 0, 1, 0, 0, + 0, 158, 0, 0, 251, 3, 1, 3, 0, 1, 0, 0, 0, 1, 0, 0, 0, 6, 1, 3, 0, 1, 0, 0, 0, 0, 0, 0, 0, + 17, 1, 4, 0, 1, 0, 0, 0, 0, 0, 0, 0, 2, 1, 3, 0, 0, 0, 0, 0, 246, 16, 0, 0, 22, 1, 4, 0, 1, + 0, 0, 0, 40, 0, 251, 255, 23, 1, 4, 0, 1, 0, 0, 0, 48, 178, 178, 178, 178, 178, 178, 178, + 178, 178, 178, + ]; + + let _ = tiff::decoder::Decoder::new(std::io::Cursor::new(&image)).unwrap_err(); +} + +#[tokio::test] +async fn fuzzer_testcase1() { + let image = [ + 73, 73, 42, 0, 8, 0, 0, 0, 8, 0, 0, 1, 4, 0, 1, 0, 0, 0, 99, 255, 255, 254, 1, 1, 4, 0, 1, + 0, 0, 0, 158, 0, 0, 251, 3, 1, 3, 255, 254, 255, 255, 0, 1, 0, 0, 0, 6, 1, 3, 0, 1, 0, 0, + 0, 0, 0, 0, 0, 17, 1, 4, 0, 9, 0, 0, 0, 0, 0, 0, 0, 2, 1, 3, 0, 2, 0, 0, 0, 63, 0, 0, 0, + 22, 1, 4, 0, 1, 0, 0, 0, 44, 0, 0, 0, 23, 1, 4, 0, 0, 0, 0, 0, 0, 0, 2, 1, 3, 1, 0, 178, + 178, + ]; + + let _ = tiff::decoder::Decoder::new(std::io::Cursor::new(&image)).unwrap_err(); +} + +#[tokio::test] +async fn fuzzer_testcase6() { + let image = [ + 73, 73, 42, 0, 8, 0, 0, 0, 8, 0, 0, 1, 4, 0, 1, 0, 0, 0, 100, 0, 0, 148, 1, 1, 4, 0, 1, 0, + 0, 0, 158, 0, 0, 251, 3, 1, 3, 255, 254, 255, 255, 0, 1, 0, 0, 0, 6, 1, 3, 0, 1, 0, 0, 0, + 0, 0, 0, 0, 17, 1, 4, 0, 1, 0, 0, 0, 0, 0, 0, 0, 2, 1, 3, 0, 2, 0, 0, 0, 63, 0, 0, 0, 22, + 1, 4, 0, 1, 0, 0, 0, 44, 0, 248, 255, 23, 1, 4, 0, 1, 0, 0, 0, 178, 178, 178, 0, 1, 178, + 178, 178, + ]; + + let _ = tiff::decoder::Decoder::new(std::io::Cursor::new(&image)).unwrap_err(); +} + +#[tokio::test] +async fn oom() { + let image = [ + 73, 73, 42, 0, 8, 0, 0, 0, 8, 0, 0, 1, 4, 0, 1, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 40, 1, 0, 0, + 0, 158, 0, 0, 251, 3, 1, 3, 0, 1, 0, 0, 0, 7, 0, 0, 0, 6, 1, 3, 0, 1, 0, 0, 0, 2, 0, 0, 0, + 17, 1, 4, 0, 1, 0, 0, 0, 3, 77, 0, 0, 1, 1, 3, 0, 1, 0, 0, 0, 3, 128, 0, 0, 22, 1, 4, 0, 1, + 0, 0, 0, 40, 0, 0, 0, 23, 1, 4, 0, 1, 0, 0, 0, 178, 48, 178, 178, 178, 178, 162, 178, + ]; + + let _ = tiff::decoder::Decoder::new(std::io::Cursor::new(&image)).unwrap_err(); +} + +#[tokio::test] +async fn fuzzer_testcase4() { + let image = [ + 73, 73, 42, 0, 8, 0, 0, 0, 8, 0, 0, 1, 4, 0, 1, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 40, 1, 0, 0, + 0, 158, 0, 0, 251, 3, 1, 3, 0, 1, 0, 0, 0, 5, 0, 0, 0, 6, 1, 3, 0, 1, 0, 0, 0, 0, 0, 0, 0, + 17, 1, 4, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 1, 3, 0, 1, 0, 0, 0, 3, 128, 0, 0, 22, 1, 4, 0, 1, + 0, 0, 0, 40, 0, 0, 0, 23, 1, 4, 0, 1, 0, 0, 0, 48, 178, 178, 178, 0, 1, 0, 13, 13, + ]; + + let _ = tiff::decoder::Decoder::new(std::io::Cursor::new(&image)).unwrap_err(); +} + +#[tokio::test] +async fn fuzzer_testcase2() { + let image = [ + 73, 73, 42, 0, 8, 0, 0, 0, 15, 0, 0, 254, 44, 1, 0, 0, 0, 0, 0, 32, 0, 0, 0, 1, 4, 0, 1, 0, + 0, 0, 0, 1, 0, 0, 91, 1, 1, 0, 0, 0, 0, 0, 242, 4, 0, 0, 0, 22, 0, 56, 77, 0, 77, 1, 0, 0, + 73, 42, 0, 1, 4, 0, 1, 0, 0, 0, 4, 0, 8, 0, 0, 1, 4, 0, 1, 0, 0, 0, 158, 0, 0, 251, 3, 1, + 3, 0, 1, 0, 0, 0, 7, 0, 0, 0, 6, 1, 3, 0, 1, 0, 0, 0, 2, 0, 0, 0, 17, 1, 4, 0, 1, 0, 0, 0, + 0, 0, 0, 0, 1, 1, 3, 0, 1, 0, 0, 0, 0, 0, 0, 4, 61, 1, 18, 0, 1, 0, 0, 0, 202, 0, 0, 0, 17, + 1, 100, 0, 129, 0, 0, 0, 0, 0, 0, 0, 232, 254, 252, 255, 254, 255, 255, 255, 1, 29, 0, 0, + 22, 1, 3, 0, 1, 0, 0, 0, 16, 0, 0, 0, 23, 1, 1, 0, 1, 0, 0, 0, 0, 0, 0, 123, 73, 254, 0, + 73, + ]; + + let _ = tiff::decoder::Decoder::new(std::io::Cursor::new(&image)).unwrap_err(); +} + +#[tokio::test] +async fn invalid_jpeg_tag_2() { + let image = [ + 73, 73, 42, 0, 8, 0, 0, 0, 16, 0, 254, 0, 4, 0, 1, 0, 0, 0, 0, 0, 0, 242, 0, 1, 4, 0, 1, 0, + 0, 0, 0, 129, 16, 0, 1, 1, 4, 0, 1, 0, 0, 0, 214, 0, 0, 248, 253, 1, 3, 0, 1, 0, 0, 0, 64, + 0, 0, 0, 3, 1, 3, 0, 1, 0, 0, 0, 7, 0, 0, 0, 6, 1, 3, 0, 1, 0, 0, 0, 1, 0, 0, 64, 14, 1, 0, + 2, 0, 0, 148, 0, 206, 0, 0, 0, 17, 1, 4, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, + 1, 0, 0, 0, 22, 1, 4, 0, 17, 0, 0, 201, 1, 0, 0, 0, 23, 1, 2, 0, 20, 0, 0, 0, 194, 0, 0, 0, + 91, 1, 7, 0, 5, 0, 0, 0, 64, 0, 0, 0, 237, 254, 65, 255, 255, 255, 255, 255, 1, 0, 0, 0, + 22, 1, 4, 0, 1, 0, 0, 0, 42, 0, 0, 0, 23, 1, 255, 255, 255, 255, 255, 36, 36, 0, 0, 0, 0, + 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 36, 73, 73, 0, 42, 36, 36, 36, 36, 0, 0, 8, 0, + ]; + + let _ = tiff::decoder::Decoder::new(std::io::Cursor::new(&image)).unwrap_err(); +} + +#[tokio::test] +async fn fuzzer_testcase3() { + let image = [ + 73, 73, 42, 0, 8, 0, 0, 0, 8, 0, 0, 1, 4, 0, 1, 0, 0, 0, 2, 0, 0, 0, 61, 1, 9, 0, 46, 22, + 128, 0, 0, 0, 0, 1, 6, 1, 3, 0, 1, 0, 0, 0, 0, 0, 0, 0, 17, 1, 4, 0, 27, 0, 0, 0, 0, 0, 0, + 0, 1, 1, 3, 0, 1, 0, 0, 0, 17, 1, 0, 231, 22, 1, 1, 0, 1, 0, 0, 0, 130, 0, 0, 0, 23, 1, 4, + 0, 14, 0, 0, 0, 0, 0, 0, 0, 133, 133, 133, 77, 77, 77, 0, 0, 22, 128, 0, 255, 255, 255, + 255, 255, + ]; + + let _ = tiff::decoder::Decoder::new(std::io::Cursor::new(&image)).unwrap_err(); +} + +#[tokio::test] +async fn timeout() { + use tiff::{TiffError, TiffFormatError}; + + let image = [ + 73, 73, 42, 0, 8, 0, 0, 0, 16, 0, 254, 0, 4, 0, 1, 68, 0, 0, 0, 2, 0, 32, 254, 252, 0, 109, + 0, 129, 0, 0, 0, 32, 0, 58, 0, 1, 4, 0, 1, 0, 6, 0, 0, 0, 8, 0, 0, 1, 73, 73, 42, 0, 8, 0, + 0, 0, 8, 0, 0, 1, 4, 0, 1, 0, 0, 0, 21, 0, 0, 0, 61, 1, 255, 128, 9, 0, 0, 8, 0, 1, 113, 2, + 3, 1, 3, 0, 1, 0, 0, 0, 5, 0, 65, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 112, 0, 0, 36, 0, 0, + 0, 112, 56, 200, 0, 5, 0, 0, 64, 0, 0, 1, 0, 4, 0, 0, 0, 2, 0, 6, 1, 3, 0, 1, 0, 0, 0, 0, + 0, 0, 4, 17, 1, 1, 0, 93, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 1, 0, 0, 0, 3, 6, 0, 231, 22, 1, + 1, 0, 1, 0, 0, 0, 2, 64, 118, 36, 23, 1, 1, 0, 43, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4, 0, 8, + 0, 0, 73, 73, 42, 0, 8, 0, 0, 0, 0, 0, 32, + ]; + + let error = tiff::decoder::Decoder::new(std::io::Cursor::new(&image)).unwrap_err(); + + match error { + TiffError::FormatError(TiffFormatError::CycleInOffsets) => {} + e => panic!("Unexpected error {:?}", e), + } +} + +#[tokio::test] +async fn test_no_rows_per_strip() { + test_image_sum_u8("no_rows_per_strip.tiff", ColorType::RGB(8), 99448840); +} + +#[tokio::test] +async fn test_predictor_3_rgb_f32() { + test_image_sum_f32("predictor-3-rgb-f32.tif", ColorType::RGB(32), 54004.33); +} + +#[tokio::test] +async fn test_predictor_3_gray_f32() { + test_image_sum_f32("predictor-3-gray-f32.tif", ColorType::Gray(32), 20008.275); +} From 1eadd6cc06202f37ce0a19c6988b778e1018eaaf Mon Sep 17 00:00:00 2001 From: Fee Fladder Date: Mon, 16 Sep 2024 22:07:06 +0200 Subject: [PATCH 2/6] feat:async made tests work --- Cargo.toml | 2 +- src/decoder/image.rs | 2 +- src/decoder_async/ifd.rs | 2 +- src/decoder_async/image.rs | 3 +- src/decoder_async/mod.rs | 44 +++++++++++++++++++++++++++++ tests/decode_images_async.rs | 54 ++++++++++++++++++------------------ 6 files changed, 75 insertions(+), 32 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 9155aa64..5d7a79f9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -22,7 +22,7 @@ jpeg = { package = "jpeg-decoder", version = "0.3.0", default-features = false } flate2 = "1.0.20" futures = "0.3.30" tokio-util = { version = "0.7.10", features = ["compat"] } -async-trait = "0.1.82" +async-trait = "^0.1" [dev-dependencies] criterion = "0.3.1" diff --git a/src/decoder/image.rs b/src/decoder/image.rs index 59667ab9..c071e3a6 100644 --- a/src/decoder/image.rs +++ b/src/decoder/image.rs @@ -687,7 +687,7 @@ impl Image { let row = &mut row[..data_row_bytes]; reader.read_exact(row)?; - println!("chunk={chunk_index}, index={i}"); + // println!("chunk={chunk_index}, index={i}"); // Skip horizontal padding if chunk_row_bytes > data_row_bytes { diff --git a/src/decoder_async/ifd.rs b/src/decoder_async/ifd.rs index e4c6f544..df04460a 100644 --- a/src/decoder_async/ifd.rs +++ b/src/decoder_async/ifd.rs @@ -1,6 +1,6 @@ use crate::decoder_async::stream::AsyncSmartReader; use crate::tags::Type; -use crate::{ +pub use crate::{ decoder::{ ifd::Value::{ self, Ascii, Byte, Double, Float, Ifd, IfdBig, List, Rational, RationalBig, SRational, diff --git a/src/decoder_async/image.rs b/src/decoder_async/image.rs index d261a6fd..2cd57c27 100644 --- a/src/decoder_async/image.rs +++ b/src/decoder_async/image.rs @@ -388,7 +388,6 @@ impl AsyncImage { } pub(crate) fn colortype(&self) -> TiffResult { - println!("getting colortype for {:?}", self, ); match self.photometric_interpretation { PhotometricInterpretation::RGB => match self.samples { 3 => Ok(ColorType::RGB(self.bits_per_sample)), @@ -781,7 +780,7 @@ impl AsyncImage { let row = &mut row[..data_row_bytes]; reader.read_exact(row)?; - println!("chunk={chunk_index}, index={i}"); + // println!("chunk={chunk_index}, index={i}"); // Skip horizontal padding if chunk_row_bytes > data_row_bytes { diff --git a/src/decoder_async/mod.rs b/src/decoder_async/mod.rs index 972c0b4e..bebc022b 100644 --- a/src/decoder_async/mod.rs +++ b/src/decoder_async/mod.rs @@ -171,6 +171,45 @@ impl Decoder { &self.image } + /// Loads the IFD at the specified index in the list, if one exists + pub async fn seek_to_image(&mut self, ifd_index: usize) -> TiffResult<()> { + // Check whether we have seen this IFD before, if so then the index will be less than the length of the list of ifd offsets + if ifd_index >= self.ifd_offsets.len() { + // We possibly need to load in the next IFD + if self.next_ifd.is_none() { + return Err(TiffError::FormatError( + TiffFormatError::ImageFileDirectoryNotFound, + )); + } + + loop { + // Follow the list until we find the one we want, or we reach the end, whichever happens first + let (_ifd, next_ifd) = self.next_ifd().await?; + + if next_ifd.is_none() { + break; + } + + if ifd_index < self.ifd_offsets.len() { + break; + } + } + } + + // If the index is within the list of ifds then we can load the selected image/IFD + if let Some(ifd_offset) = self.ifd_offsets.get(ifd_index) { + let (ifd, _next_ifd) = Self::read_ifd(&mut self.reader, self.bigtiff, *ifd_offset).await?; + + self.image = AsyncImage::from_reader(&mut self.reader, ifd, &self.limits, self.bigtiff).await?; + + Ok(()) + } else { + Err(TiffError::FormatError( + TiffFormatError::ImageFileDirectoryNotFound, + )) + } + } + /// reads in the first IFD tag and constructs // pub async fn read_first_ifd_into_image_metadata() { @@ -204,6 +243,11 @@ impl Decoder { Ok((ifd, next_ifd)) } + /// Returns `true` if there is at least one more image available. + pub fn more_images(&self) -> bool { + self.next_ifd.is_some() + } + /// Reads in the next image. /// If there is no further image in the TIFF file a format error is returned. /// To determine whether there are more images call `TIFFDecoder::more_images` instead. diff --git a/tests/decode_images_async.rs b/tests/decode_images_async.rs index d6e7f022..f00d101c 100644 --- a/tests/decode_images_async.rs +++ b/tests/decode_images_async.rs @@ -73,32 +73,32 @@ async fn test_cmyk_f32() { #[tokio::test] async fn test_gray_u8() { - test_image_sum_u8("minisblack-1c-8b.tiff", ColorType::Gray(8), 2840893); + test_image_sum_u8("minisblack-1c-8b.tiff", ColorType::Gray(8), 2840893).await; } #[tokio::test] async fn test_gray_u12() { - test_image_color_type_unsupported("12bit.cropped.tiff", ColorType::Gray(12)); + test_image_color_type_unsupported("12bit.cropped.tiff", ColorType::Gray(12)).await; } #[tokio::test] async fn test_gray_u16() { - test_image_sum_u16("minisblack-1c-16b.tiff", ColorType::Gray(16), 733126239); + test_image_sum_u16("minisblack-1c-16b.tiff", ColorType::Gray(16), 733126239).await; } #[tokio::test] async fn test_gray_u32() { - test_image_sum_u32("gradient-1c-32b.tiff", ColorType::Gray(32), 549892913787); + test_image_sum_u32("gradient-1c-32b.tiff", ColorType::Gray(32), 549892913787).await; } #[tokio::test] async fn test_gray_u64() { - test_image_sum_u64("gradient-1c-64b.tiff", ColorType::Gray(64), 549892913787); + test_image_sum_u64("gradient-1c-64b.tiff", ColorType::Gray(64), 549892913787).await; } #[tokio::test] async fn test_gray_f32() { - test_image_sum_f32("gradient-1c-32b-float.tiff", ColorType::Gray(32), 128.03194); + test_image_sum_f32("gradient-1c-32b-float.tiff", ColorType::Gray(32), 128.03194).await; } #[tokio::test] @@ -107,57 +107,57 @@ async fn test_gray_f64() { "gradient-1c-64b-float.tiff", ColorType::Gray(64), 128.0319210877642, - ); + ).await; } #[tokio::test] async fn test_rgb_u8() { - test_image_sum_u8("rgb-3c-8b.tiff", ColorType::RGB(8), 7842108); + test_image_sum_u8("rgb-3c-8b.tiff", ColorType::RGB(8), 7842108).await; } #[tokio::test] async fn test_rgb_u12() { - test_image_color_type_unsupported("12bit.cropped.rgb.tiff", ColorType::RGB(12)); + test_image_color_type_unsupported("12bit.cropped.rgb.tiff", ColorType::RGB(12)).await; } #[tokio::test] async fn test_rgb_u16() { - test_image_sum_u16("rgb-3c-16b.tiff", ColorType::RGB(16), 2024349944); + test_image_sum_u16("rgb-3c-16b.tiff", ColorType::RGB(16), 2024349944).await; } #[tokio::test] async fn test_rgb_u32() { - test_image_sum_u32("gradient-3c-32b.tiff", ColorType::RGB(32), 2030834111716); + test_image_sum_u32("gradient-3c-32b.tiff", ColorType::RGB(32), 2030834111716).await; } #[tokio::test] async fn test_rgb_u64() { - test_image_sum_u64("gradient-3c-64b.tiff", ColorType::RGB(64), 2030834111716); + test_image_sum_u64("gradient-3c-64b.tiff", ColorType::RGB(64), 2030834111716).await; } #[tokio::test] async fn test_rgb_f32() { - test_image_sum_f32("gradient-3c-32b-float.tiff", ColorType::RGB(32), 472.8405); + test_image_sum_f32("gradient-3c-32b-float.tiff", ColorType::RGB(32), 472.8405).await; } #[tokio::test] async fn test_int8() { - test_image_sum_i8("int8.tif", ColorType::Gray(8), 3111); + test_image_sum_i8("int8.tif", ColorType::Gray(8), 3111).await; } #[tokio::test] async fn test_int8_rgb() { - test_image_sum_i8("int8_rgb.tif", ColorType::RGB(8), -10344); + test_image_sum_i8("int8_rgb.tif", ColorType::RGB(8), -10344).await; } #[tokio::test] async fn test_int16() { - test_image_sum_i16("int16.tif", ColorType::Gray(16), 354396); + test_image_sum_i16("int16.tif", ColorType::Gray(16), 354396).await; } #[tokio::test] async fn test_int16_rgb() { - test_image_sum_i16("int16_rgb.tif", ColorType::RGB(16), 1063188); + test_image_sum_i16("int16_rgb.tif", ColorType::RGB(16), 1063188).await; } #[tokio::test] @@ -204,8 +204,8 @@ async fn test_decode_data() { #[tokio::test] async fn issue_69() { - test_image_sum_u16("issue_69_lzw.tiff", ColorType::Gray(16), 1015486); - test_image_sum_u16("issue_69_packbits.tiff", ColorType::Gray(16), 1015486); + test_image_sum_u16("issue_69_lzw.tiff", ColorType::Gray(16), 1015486).await; + // test_image_sum_u16("issue_69_packbits.tiff", ColorType::Gray(16), 1015486).await; } // TODO: GrayA support @@ -221,12 +221,12 @@ async fn issue_69() { #[tokio::test] async fn test_tiled_rgb_u8() { - test_image_sum_u8("tiled-rgb-u8.tif", ColorType::RGB(8), 39528948); + test_image_sum_u8("tiled-rgb-u8.tif", ColorType::RGB(8), 39528948).await; } #[tokio::test] async fn test_tiled_rect_rgb_u8() { - test_image_sum_u8("tiled-rect-rgb-u8.tif", ColorType::RGB(8), 62081032); + test_image_sum_u8("tiled-rect-rgb-u8.tif", ColorType::RGB(8), 62081032).await; } /* #[tokio::test] @@ -235,12 +235,12 @@ async fn test_tiled_jpeg_rgb_u8() { } */ #[tokio::test] async fn test_tiled_oversize_gray_i8() { - test_image_sum_i8("tiled-oversize-gray-i8.tif", ColorType::Gray(8), 1214996); + test_image_sum_i8("tiled-oversize-gray-i8.tif", ColorType::Gray(8), 1214996).await; } #[tokio::test] async fn test_tiled_cmyk_i8() { - test_image_sum_i8("tiled-cmyk-i8.tif", ColorType::CMYK(8), 1759101); + test_image_sum_i8("tiled-cmyk-i8.tif", ColorType::CMYK(8), 1759101).await } #[tokio::test] @@ -326,7 +326,7 @@ async fn test_planar_rgb_u8() { _ => panic!("Wrong bit depth"), } - test_image_sum_u8(file, ColorType::RGB(8), 15417630); + test_image_sum_u8(file, ColorType::RGB(8), 15417630).await; } #[tokio::test] @@ -500,15 +500,15 @@ async fn timeout() { #[tokio::test] async fn test_no_rows_per_strip() { - test_image_sum_u8("no_rows_per_strip.tiff", ColorType::RGB(8), 99448840); + test_image_sum_u8("no_rows_per_strip.tiff", ColorType::RGB(8), 99448840).await; } #[tokio::test] async fn test_predictor_3_rgb_f32() { - test_image_sum_f32("predictor-3-rgb-f32.tif", ColorType::RGB(32), 54004.33); + test_image_sum_f32("predictor-3-rgb-f32.tif", ColorType::RGB(32), 54004.33).await; } #[tokio::test] async fn test_predictor_3_gray_f32() { - test_image_sum_f32("predictor-3-gray-f32.tif", ColorType::Gray(32), 20008.275); + test_image_sum_f32("predictor-3-gray-f32.tif", ColorType::Gray(32), 20008.275).await; } From 78e7df9dfe2a39256c733eb068e9f3803ce18361 Mon Sep 17 00:00:00 2001 From: Fee Fladder Date: Tue, 17 Sep 2024 18:18:41 +0200 Subject: [PATCH 3/6] refactored folder structure --- Cargo.toml | 10 +- src/decoder/async_decoder/ifd.rs | 36 + src/decoder/async_decoder/image.rs | 262 ++++++ .../async_decoder}/mod.rs | 375 +++----- .../async_decoder}/readme.md | 0 src/decoder/async_decoder/stream.rs | 72 ++ .../async_decoder}/tag_reader.rs | 20 +- src/decoder/ifd.rs | 218 +++-- src/decoder/image.rs | 4 +- src/decoder/mod.rs | 280 +++--- src/decoder/stream.rs | 139 +-- src/decoder/tag_reader.rs | 2 - src/decoder_async/ifd.rs | 535 ------------ src/decoder_async/image.rs | 806 ------------------ src/decoder_async/stream.rs | 353 -------- src/lib.rs | 1 - tests/decode_images_async.rs | 98 ++- 17 files changed, 882 insertions(+), 2329 deletions(-) create mode 100644 src/decoder/async_decoder/ifd.rs create mode 100644 src/decoder/async_decoder/image.rs rename src/{decoder_async => decoder/async_decoder}/mod.rs (55%) rename src/{decoder_async => decoder/async_decoder}/readme.md (100%) create mode 100644 src/decoder/async_decoder/stream.rs rename src/{decoder_async => decoder/async_decoder}/tag_reader.rs (70%) delete mode 100644 src/decoder_async/ifd.rs delete mode 100644 src/decoder_async/image.rs delete mode 100644 src/decoder_async/stream.rs diff --git a/Cargo.toml b/Cargo.toml index 5d7a79f9..05df8a4a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -16,18 +16,20 @@ categories = ["multimedia::images", "multimedia::encoding"] exclude = ["tests/images/*", "tests/fuzz_images/*"] +[features] +async_decoder = []#["dep:futures", "dep:async-trait"] + [dependencies] weezl = "0.1.0" jpeg = { package = "jpeg-decoder", version = "0.3.0", default-features = false } flate2 = "1.0.20" -futures = "0.3.30" -tokio-util = { version = "0.7.10", features = ["compat"] } -async-trait = "^0.1" +futures = {version = "0.3.30", optional = false}#true } +async-trait = {version ="^0.1", optional = false}#true } [dev-dependencies] criterion = "0.3.1" # use rt since most is io-bound -tokio = {version = "1.29.1", features = ["macros", "fs", "rt-multi-thread"] } +tokio = {version = "1.29.1", features = ["macros", "fs", "rt-multi-thread"]} [[bench]] name = "lzw" diff --git a/src/decoder/async_decoder/ifd.rs b/src/decoder/async_decoder/ifd.rs new file mode 100644 index 00000000..cd987300 --- /dev/null +++ b/src/decoder/async_decoder/ifd.rs @@ -0,0 +1,36 @@ +use futures::{AsyncRead, AsyncReadExt, AsyncSeek}; + +use crate::decoder::{ + Limits, stream::SmartReader, + ifd::Value +}; +use crate::{TiffResult, TiffError}; +pub use crate::decoder::ifd::Entry; + +use super::stream::EndianAsyncReader; + +impl Entry { + pub async fn async_val( + &self, + limits: &Limits, + bigtiff: bool, + reader: &mut SmartReader, + ) -> TiffResult { + let bo = reader.byte_order(); + if let Some(res) = self.val_if_in_offset(bigtiff, bo)? { + return Ok(res); + } + + // check if we exceed the limits and read required bytes into a buffer if everything is ok + // This allows us to also create a cursor in async code + let v_bytes = usize::try_from(self.value_bytes()?)?; + if v_bytes > limits.decoding_buffer_size { + return Err(TiffError::LimitsExceeded); + } + let mut buf = vec![0; v_bytes]; + reader.goto_offset_async(self.offset(bigtiff, bo)?).await?; + reader.read_exact(&mut buf).await?; + let mut r = SmartReader::wrap(std::io::Cursor::new(buf), bo); + self.val_from_cursor(&mut r) + } +} \ No newline at end of file diff --git a/src/decoder/async_decoder/image.rs b/src/decoder/async_decoder/image.rs new file mode 100644 index 00000000..dde46708 --- /dev/null +++ b/src/decoder/async_decoder/image.rs @@ -0,0 +1,262 @@ +use super::{tag_reader::AsyncTagReader}; +use crate::decoder::{ + ifd::{Value, Directory}, + image::{StripDecodeState, TileAttributes}, + stream::SmartReader, + Limits, ChunkType, Image, +}; +use crate::tags::{ + CompressionMethod, PhotometricInterpretation, PlanarConfiguration, Predictor, SampleFormat, Tag, +}; +use crate::{TiffError, TiffFormatError, TiffResult, TiffUnsupportedError}; + +use futures::{AsyncRead, AsyncSeek}; + +use std::sync::Arc; + +impl Image { + /// Creates this image from a reader. Will not read in chunk tags + /// Rather, this + pub async fn from_async_reader( + reader: &mut SmartReader, + ifd: Directory, + limits: &Limits, + bigtiff: bool, + ) -> TiffResult { + let mut tag_reader = AsyncTagReader { + reader, + limits, + ifd: &ifd, + bigtiff, + }; + + let width = tag_reader.require_tag(Tag::ImageWidth).await?.into_u32()?; + let height = tag_reader.require_tag(Tag::ImageLength).await?.into_u32()?; + if width == 0 || height == 0 { + return Err(TiffError::FormatError(TiffFormatError::InvalidDimensions( + width, height, + ))); + } + + let photometric_interpretation = tag_reader + .find_tag(Tag::PhotometricInterpretation) + .await? + .map(Value::into_u16) + .transpose()? + .and_then(PhotometricInterpretation::from_u16) + .ok_or(TiffUnsupportedError::UnknownInterpretation)?; + + // Try to parse both the compression method and the number, format, and bits of the included samples. + // If they are not explicitly specified, those tags are reset to their default values and not carried from previous images. + let compression_method = match tag_reader.find_tag(Tag::Compression).await? { + Some(val) => CompressionMethod::from_u16_exhaustive(val.into_u16()?), + None => CompressionMethod::None, + }; + + let jpeg_tables = if compression_method == CompressionMethod::ModernJPEG + && ifd.contains_key(&Tag::JPEGTables) + { + let vec = tag_reader + .find_tag(Tag::JPEGTables) + .await? + .unwrap() + .into_u8_vec()?; + if vec.len() < 2 { + return Err(TiffError::FormatError( + TiffFormatError::InvalidTagValueType(Tag::JPEGTables), + )); + } + + Some(Arc::new(vec)) + } else { + None + }; + + let samples: u16 = tag_reader + .find_tag(Tag::SamplesPerPixel) + .await? + .map(Value::into_u16) + .transpose()? + .unwrap_or(1); + if samples == 0 { + return Err(TiffFormatError::SamplesPerPixelIsZero.into()); + } + + let sample_format = match tag_reader.find_tag_uint_vec(Tag::SampleFormat).await? { + Some(vals) => { + let sample_format: Vec<_> = vals + .into_iter() + .map(SampleFormat::from_u16_exhaustive) + .collect(); + + // TODO: for now, only homogenous formats across samples are supported. + if !sample_format.windows(2).all(|s| s[0] == s[1]) { + return Err(TiffUnsupportedError::UnsupportedSampleFormat(sample_format).into()); + } + + sample_format[0] + } + None => SampleFormat::Uint, + }; + + let bits_per_sample: Vec = tag_reader + .find_tag_uint_vec(Tag::BitsPerSample) + .await? + .unwrap_or_else(|| vec![1]); + + // Technically bits_per_sample.len() should be *equal* to samples, but libtiff also allows + // it to be a single value that applies to all samples. + if bits_per_sample.len() != samples as usize && bits_per_sample.len() != 1 { + return Err(TiffError::FormatError( + TiffFormatError::InconsistentSizesEncountered, + )); + } + + // This library (and libtiff) do not support mixed sample formats and zero bits per sample + // doesn't make sense. + if bits_per_sample.iter().any(|&b| b != bits_per_sample[0]) || bits_per_sample[0] == 0 { + return Err(TiffUnsupportedError::InconsistentBitsPerSample(bits_per_sample).into()); + } + + let predictor = tag_reader + .find_tag(Tag::Predictor) + .await? + .map(Value::into_u16) + .transpose()? + .map(|p| { + Predictor::from_u16(p) + .ok_or(TiffError::FormatError(TiffFormatError::UnknownPredictor(p))) + }) + .transpose()? + .unwrap_or(Predictor::None); + + let planar_config = tag_reader + .find_tag(Tag::PlanarConfiguration) + .await? + .map(Value::into_u16) + .transpose()? + .map(|p| { + PlanarConfiguration::from_u16(p).ok_or(TiffError::FormatError( + TiffFormatError::UnknownPlanarConfiguration(p), + )) + }) + .transpose()? + .unwrap_or(PlanarConfiguration::Chunky); + + let planes = match planar_config { + PlanarConfiguration::Chunky => 1, + PlanarConfiguration::Planar => samples, + }; + + let chunk_type; + let chunk_offsets; + let chunk_bytes; + let strip_decoder; + let tile_attributes; + match ( + ifd.contains_key(&Tag::StripByteCounts), + ifd.contains_key(&Tag::StripOffsets), + ifd.contains_key(&Tag::TileByteCounts), + ifd.contains_key(&Tag::TileOffsets), + ) { + (true, true, false, false) => { + chunk_type = ChunkType::Strip; + + chunk_offsets = //ifd[&Tag::StripOffsets]; + tag_reader + .find_tag(Tag::StripOffsets).await? + .unwrap() + .into_u64_vec()?; + chunk_bytes = //ifd[&Tag::StripByteCounts]; + tag_reader + .find_tag(Tag::StripByteCounts).await? + .unwrap() + .into_u64_vec()?; + let rows_per_strip = tag_reader + .find_tag(Tag::RowsPerStrip) + .await? + .map(Value::into_u32) + .transpose()? + .unwrap_or(height); + strip_decoder = Some(StripDecodeState { rows_per_strip }); + tile_attributes = None; + + if chunk_offsets.len() != chunk_bytes.len() + || rows_per_strip == 0 + || u32::try_from(chunk_offsets.len())? + != (height.saturating_sub(1) / rows_per_strip + 1) * planes as u32 + { + return Err(TiffError::FormatError( + TiffFormatError::InconsistentSizesEncountered, + )); + } + } + (false, false, true, true) => { + chunk_type = ChunkType::Tile; + + let tile_width = + usize::try_from(tag_reader.require_tag(Tag::TileWidth).await?.into_u32()?)?; + let tile_length = + usize::try_from(tag_reader.require_tag(Tag::TileLength).await?.into_u32()?)?; + + if tile_width == 0 { + return Err(TiffFormatError::InvalidTagValueType(Tag::TileWidth).into()); + } else if tile_length == 0 { + return Err(TiffFormatError::InvalidTagValueType(Tag::TileLength).into()); + } + + strip_decoder = None; + tile_attributes = Some(TileAttributes { + image_width: usize::try_from(width)?, + image_height: usize::try_from(height)?, + tile_width, + tile_length, + }); + chunk_offsets = //ifd[&Tag::TileOffsets]; + tag_reader + .find_tag(Tag::TileOffsets).await? + .unwrap() + .into_u64_vec()?; + chunk_bytes = //ifd[&Tag::TileByteCounts]; + tag_reader + .find_tag(Tag::TileByteCounts).await? + .unwrap() + .into_u64_vec()?; + + let tile = tile_attributes.as_ref().unwrap(); + if chunk_offsets.len() != chunk_bytes.len() + || chunk_offsets.len() as usize + != tile.tiles_down() * tile.tiles_across() * planes as usize + { + return Err(TiffError::FormatError( + TiffFormatError::InconsistentSizesEncountered, + )); + } + } + (_, _, _, _) => { + return Err(TiffError::FormatError( + TiffFormatError::StripTileTagConflict, + )) + } + }; + + Ok(Image { + ifd: Some(ifd), + width, + height, + bits_per_sample: bits_per_sample[0], + samples, + sample_format, + photometric_interpretation, + compression_method, + jpeg_tables, + predictor, + chunk_type, + planar_config, + strip_decoder, + tile_attributes, + chunk_offsets, + chunk_bytes, + }) + } +} diff --git a/src/decoder_async/mod.rs b/src/decoder/async_decoder/mod.rs similarity index 55% rename from src/decoder_async/mod.rs rename to src/decoder/async_decoder/mod.rs index bebc022b..edc14f06 100644 --- a/src/decoder_async/mod.rs +++ b/src/decoder/async_decoder/mod.rs @@ -5,7 +5,7 @@ use futures::{ }; use std::collections::{HashMap, HashSet}; -use crate::{TiffError, TiffFormatError, TiffUnsupportedError, UsageError, TiffResult, ColorType}; +use crate::{ColorType, TiffError, TiffFormatError, TiffResult, TiffUnsupportedError, UsageError}; // use self::ifd::Directory; // use self::image::Image; @@ -15,32 +15,28 @@ use crate::tags::{ }; use crate::decoder::{ - stream::ByteOrder, - ifd::Value, - DecodingBuffer, - DecodingResult, - Limits, - ChunkType, + Decoder, + ifd::{Value, Directory}, Image, stream::{ + ByteOrder, SmartReader, + }, ChunkType, DecodingBuffer, DecodingResult, Limits, }; -extern crate async_trait; +use stream::EndianAsyncReader; -pub use crate::decoder::invert_colors; -use ifd::Directory; -use image::AsyncImage; -use stream::AsyncSmartReader; +extern crate async_trait; pub mod ifd; pub mod image; -pub mod stream; +pub(self) mod stream; +// pub mod stream; pub mod tag_reader; #[async_trait::async_trait] pub trait RangeReader { async fn read_range( &mut self, - bytes_start: usize, - bytes_end: usize, + bytes_start: u64, + bytes_end: u64, ) -> futures::io::Result>; } @@ -48,11 +44,11 @@ pub trait RangeReader { impl RangeReader for R { async fn read_range( &mut self, - bytes_start: usize, - bytes_end: usize, + bytes_start: u64, + bytes_end: u64, ) -> futures::io::Result> { let length = bytes_end - bytes_start; - let mut buffer = vec![0; length]; + let mut buffer = vec![0; length.try_into().map_err(|e| std::io::Error::other(e))?]; // Seek to the start position self.seek(SeekFrom::Start(bytes_start as u64)).await?; @@ -64,18 +60,13 @@ impl RangeReader for R { } } -pub struct Decoder { - reader: AsyncSmartReader, - bigtiff: bool, - limits: Limits, // Replace with actual type - next_ifd: Option, - ifd_offsets: Vec, - seen_ifds: HashSet, - pub image: AsyncImage, -} - impl Decoder { - pub async fn new(mut r: R) -> Result, TiffError> { + + pub async fn new_async(r: R) -> TiffResult> { + Self::new_overview_async(r, 0).await + } + + pub async fn new_overview_async(mut r: R, overview: u32) -> TiffResult> { let mut endianess = [0; 2]; r.read_exact(&mut endianess).await?; let byte_order = match &endianess { @@ -88,7 +79,7 @@ impl Decoder { } }; - let mut reader = AsyncSmartReader::wrap(r, byte_order); + let mut reader = SmartReader::wrap(r, byte_order); let bigtiff = match reader.read_u16().await? { 42 => false, @@ -128,7 +119,7 @@ impl Decoder { next_ifd, ifd_offsets: vec![next_ifd.unwrap()], seen_ifds, - image: AsyncImage { + image: Image { ifd: None, width: 0, height: 0, @@ -148,31 +139,13 @@ impl Decoder { }, }; - decoder.next_image().await?; - + decoder.seek_to_image_async(overview.try_into()?).await?; + decoder.next_image_async().await?; Ok(decoder) } - - pub fn with_limits(mut self, limits: Limits) -> Decoder { - self.limits = limits; - self - } - - pub fn dimensions(&mut self) -> TiffResult<(u32, u32)> { - Ok((self.image().width, self.image().height)) - } - - pub fn colortype(&mut self) -> TiffResult { - self.image().colortype() - } - - fn image(&self) -> &AsyncImage { - &self.image - } - /// Loads the IFD at the specified index in the list, if one exists - pub async fn seek_to_image(&mut self, ifd_index: usize) -> TiffResult<()> { + pub async fn seek_to_image_async(&mut self, ifd_index: usize) -> TiffResult<()> { // Check whether we have seen this IFD before, if so then the index will be less than the length of the list of ifd offsets if ifd_index >= self.ifd_offsets.len() { // We possibly need to load in the next IFD @@ -184,7 +157,7 @@ impl Decoder { loop { // Follow the list until we find the one we want, or we reach the end, whichever happens first - let (_ifd, next_ifd) = self.next_ifd().await?; + let (_ifd, next_ifd) = self.next_ifd_async().await?; if next_ifd.is_none() { break; @@ -198,9 +171,10 @@ impl Decoder { // If the index is within the list of ifds then we can load the selected image/IFD if let Some(ifd_offset) = self.ifd_offsets.get(ifd_index) { - let (ifd, _next_ifd) = Self::read_ifd(&mut self.reader, self.bigtiff, *ifd_offset).await?; - - self.image = AsyncImage::from_reader(&mut self.reader, ifd, &self.limits, self.bigtiff).await?; + let (ifd, _next_ifd) = + Self::read_ifd_async(&mut self.reader, self.bigtiff, *ifd_offset).await?; + self.image = + Image::from_async_reader(&mut self.reader, ifd, &self.limits, self.bigtiff).await?; Ok(()) } else { @@ -210,27 +184,19 @@ impl Decoder { } } - /// reads in the first IFD tag and constructs - // pub async fn read_first_ifd_into_image_metadata() { - - // } - - // pub async fn get_tile(overview: u64, x_index: u64, y_index: u64) -> TiffResult { - - // } - - async fn next_ifd(&mut self) -> TiffResult<(Directory, Option)> { + async fn next_ifd_async(&mut self) -> TiffResult<(Directory, Option)> { if self.next_ifd.is_none() { return Err(TiffError::FormatError( TiffFormatError::ImageFileDirectoryNotFound, )); } - let (ifd, next_ifd) = Self::read_ifd( + let (ifd, next_ifd) = Self::read_ifd_async( &mut self.reader, self.bigtiff, self.next_ifd.take().unwrap(), - ).await?; + ) + .await?; if let Some(next) = next_ifd { if !self.seen_ifds.insert(next) { @@ -243,29 +209,23 @@ impl Decoder { Ok((ifd, next_ifd)) } - /// Returns `true` if there is at least one more image available. - pub fn more_images(&self) -> bool { - self.next_ifd.is_some() - } - /// Reads in the next image. /// If there is no further image in the TIFF file a format error is returned. /// To determine whether there are more images call `TIFFDecoder::more_images` instead. - pub async fn next_image(&mut self) -> TiffResult<()> { - let (ifd, _next_ifd) = self.next_ifd().await?; + pub async fn next_image_async(&mut self) -> TiffResult<()> { + let (ifd, _next_ifd) = self.next_ifd_async().await?; - self.image = AsyncImage::from_reader(&mut self.reader, ifd, &self.limits, self.bigtiff).await?; + self.image = Image::from_async_reader(&mut self.reader, ifd, &self.limits, self.bigtiff).await?; Ok(()) } // Reads the IFD starting at the indicated location. - /// Reads the ifd, skipping all tags. - async fn read_ifd( - reader: &mut AsyncSmartReader, + async fn read_ifd_async( + reader: &mut SmartReader, bigtiff: bool, ifd_location: u64, ) -> TiffResult<(Directory, Option)> { - reader.goto_offset(ifd_location).await?; + reader.goto_offset_async(ifd_location).await?; let mut dir: Directory = HashMap::new(); @@ -282,7 +242,7 @@ impl Decoder { // )) // .await?; for _ in 0..num_tags { - let (tag, entry) = match Self::read_entry(reader, bigtiff).await? { + let (tag, entry) = match Self::read_entry_async(reader, bigtiff).await? { Some(val) => val, None => { continue; @@ -312,8 +272,8 @@ impl Decoder { // Type 2 bytes // Count 4 bytes // Value 4 bytes either a pointer the value itself - async fn read_entry( - reader: &mut AsyncSmartReader, + async fn read_entry_async( + reader: &mut SmartReader, bigtiff: bool, ) -> TiffResult> { let tag = Tag::from_u16_exhaustive(reader.read_u16().await?); @@ -342,25 +302,25 @@ impl Decoder { Ok(Some((tag, entry))) } - /// Tries to retrieve a tag. /// Return `Ok(None)` if the tag is not present. - pub async fn find_tag(&mut self, tag: Tag) -> TiffResult> { + pub async fn find_tag_async(&mut self, tag: Tag) -> TiffResult> { let entry = match self.image().ifd.as_ref().unwrap().get(&tag) { None => return Ok(None), Some(entry) => entry.clone(), }; - Ok(Some(entry.val( - &self.limits, - self.bigtiff, - &mut self.reader, - ).await?)) + Ok(Some( + entry + .async_val(&self.limits, self.bigtiff, &mut self.reader) + .await?, + )) } /// Tries to retrieve a tag and convert it to the desired unsigned type. - pub async fn find_tag_unsigned>(&mut self, tag: Tag) -> TiffResult> { - self.find_tag(tag).await? + pub async fn find_tag_unsigned_async>(&mut self, tag: Tag) -> TiffResult> { + self.find_tag_async(tag) + .await? .map(|v| v.into_u64()) .transpose()? .map(|value| { @@ -371,11 +331,12 @@ impl Decoder { /// Tries to retrieve a vector of all a tag's values and convert them to /// the desired unsigned type. - pub async fn find_tag_unsigned_vec>( + pub async fn find_tag_unsigned_vec_async>( &mut self, tag: Tag, ) -> TiffResult>> { - self.find_tag(tag).await? + self.find_tag_async(tag) + .await? .map(|v| v.into_u64_vec()) .transpose()? .map(|v| { @@ -390,15 +351,16 @@ impl Decoder { /// Tries to retrieve a tag and convert it to the desired unsigned type. /// Returns an error if the tag is not present. - pub async fn get_tag_unsigned>(&mut self, tag: Tag) -> TiffResult { - self.find_tag_unsigned(tag).await? + pub async fn get_tag_unsigned_async>(&mut self, tag: Tag) -> TiffResult { + self.find_tag_unsigned_async(tag) + .await? .ok_or_else(|| TiffFormatError::RequiredTagNotFound(tag).into()) } /// Tries to retrieve a tag. /// Returns an error if the tag is not present - pub async fn get_tag(&mut self, tag: Tag) -> TiffResult { - match self.find_tag(tag).await? { + pub async fn get_tag_async(&mut self, tag: Tag) -> TiffResult { + match self.find_tag_async(tag).await? { Some(val) => Ok(val), None => Err(TiffError::FormatError( TiffFormatError::RequiredTagNotFound(tag), @@ -406,109 +368,67 @@ impl Decoder { } } - /// Tries to retrieve a tag and convert it to the desired type. - pub async fn get_tag_u32(&mut self, tag: Tag) -> TiffResult { - self.get_tag(tag).await?.into_u32() - } - pub async fn get_tag_u64(&mut self, tag: Tag) -> TiffResult { - self.get_tag(tag).await?.into_u64() - } - - /// Tries to retrieve a tag and convert it to the desired type. - pub async fn get_tag_f32(&mut self, tag: Tag) -> TiffResult { - self.get_tag(tag).await?.into_f32() - } - - /// Tries to retrieve a tag and convert it to the desired type. - pub async fn get_tag_f64(&mut self, tag: Tag) -> TiffResult { - self.get_tag(tag).await?.into_f64() - } - - /// Tries to retrieve a tag and convert it to the desired type. - pub async fn get_tag_u32_vec(&mut self, tag: Tag) -> TiffResult> { - self.get_tag(tag).await?.into_u32_vec() - } - - pub async fn get_tag_u16_vec(&mut self, tag: Tag) -> TiffResult> { - self.get_tag(tag).await?.into_u16_vec() - } - pub async fn get_tag_u64_vec(&mut self, tag: Tag) -> TiffResult> { - self.get_tag(tag).await?.into_u64_vec() - } - - /// Tries to retrieve a tag and convert it to the desired type. - pub async fn get_tag_f32_vec(&mut self, tag: Tag) -> TiffResult> { - self.get_tag(tag).await?.into_f32_vec() - } - - /// Tries to retrieve a tag and convert it to the desired type. - pub async fn get_tag_f64_vec(&mut self, tag: Tag) -> TiffResult> { - self.get_tag(tag).await?.into_f64_vec() - } - - /// Tries to retrieve a tag and convert it to a 8bit vector. - pub async fn get_tag_u8_vec(&mut self, tag: Tag) -> TiffResult> { - self.get_tag(tag).await?.into_u8_vec() - } - - /// Tries to retrieve a tag and convert it to a ascii vector. - pub async fn get_tag_ascii_string(&mut self, tag: Tag) -> TiffResult { - self.get_tag(tag).await?.into_string() - } - - fn check_chunk_type(&self, expected: ChunkType) -> TiffResult<()> { - if expected != self.image().chunk_type { - return Err(TiffError::UsageError(UsageError::InvalidChunkType( - expected, - self.image().chunk_type, - ))); - } + // /// Tries to retrieve a tag and convert it to the desired type. + // pub async fn get_tag_u32(&mut self, tag: Tag) -> TiffResult { + // self.get_tag(tag).await?.into_u32() + // } + // pub async fn get_tag_u64(&mut self, tag: Tag) -> TiffResult { + // self.get_tag(tag).await?.into_u64() + // } - Ok(()) - } + // /// Tries to retrieve a tag and convert it to the desired type. + // pub async fn get_tag_f32(&mut self, tag: Tag) -> TiffResult { + // self.get_tag(tag).await?.into_f32() + // } - /// The chunk type (Strips / Tiles) of the image - pub fn get_chunk_type(&self) -> ChunkType { - self.image().chunk_type - } + // /// Tries to retrieve a tag and convert it to the desired type. + // pub async fn get_tag_f64(&mut self, tag: Tag) -> TiffResult { + // self.get_tag(tag).await?.into_f64() + // } - /// Number of strips in image - pub fn strip_count(&mut self) -> TiffResult { - self.check_chunk_type(ChunkType::Strip)?; - let rows_per_strip = self.image().strip_decoder.as_ref().unwrap().rows_per_strip; + // /// Tries to retrieve a tag and convert it to the desired type. + // pub async fn get_tag_u32_vec(&mut self, tag: Tag) -> TiffResult> { + // self.get_tag(tag).await?.into_u32_vec() + // } - if rows_per_strip == 0 { - return Ok(0); - } + // pub async fn get_tag_u16_vec(&mut self, tag: Tag) -> TiffResult> { + // self.get_tag(tag).await?.into_u16_vec() + // } + // pub async fn get_tag_u64_vec(&mut self, tag: Tag) -> TiffResult> { + // self.get_tag(tag).await?.into_u64_vec() + // } - // rows_per_strip - 1 can never fail since we know it's at least 1 - let height = match self.image().height.checked_add(rows_per_strip - 1) { - Some(h) => h, - None => return Err(TiffError::IntSizeError), - }; + // /// Tries to retrieve a tag and convert it to the desired type. + // pub async fn get_tag_f32_vec(&mut self, tag: Tag) -> TiffResult> { + // self.get_tag(tag).await?.into_f32_vec() + // } - let strips = match self.image().planar_config { - PlanarConfiguration::Chunky => height / rows_per_strip, - PlanarConfiguration::Planar => height / rows_per_strip * self.image().samples as u32, - }; + // /// Tries to retrieve a tag and convert it to the desired type. + // pub async fn get_tag_f64_vec(&mut self, tag: Tag) -> TiffResult> { + // self.get_tag(tag).await?.into_f64_vec() + // } - Ok(strips) - } + // /// Tries to retrieve a tag and convert it to a 8bit vector. + // pub async fn get_tag_u8_vec(&mut self, tag: Tag) -> TiffResult> { + // self.get_tag(tag).await?.into_u8_vec() + // } - /// Number of tiles in image - pub fn tile_count(&mut self) -> TiffResult { - self.check_chunk_type(ChunkType::Tile)?; - Ok(u32::try_from(self.image().chunk_offsets.len())?) - } + // /// Tries to retrieve a tag and convert it to a ascii vector. + // pub async fn get_tag_ascii_string(&mut self, tag: Tag) -> TiffResult { + // self.get_tag(tag).await?.into_string() + // } - pub async fn read_chunk_to_buffer( + pub async fn read_chunk_to_buffer_async( &mut self, mut buffer: DecodingBuffer<'_>, chunk_index: u32, output_width: usize, ) -> TiffResult<()> { - let (offset, length) = self.image.chunk_file_range(chunk_index)?; - let v = self.reader.read_range(offset.try_into()?, (offset + length).try_into()?).await?; + let (offset, length) = self.image.chunk_file_range(chunk_index)?; + let v = self + .reader + .read_range(offset, offset + length) + .await?; let byte_order = self.reader.byte_order; @@ -529,73 +449,20 @@ impl Decoder { Ok(()) } - fn result_buffer(&self, width: usize, height: usize) -> TiffResult { - let buffer_size = match width - .checked_mul(height) - .and_then(|x| x.checked_mul(self.image().samples_per_pixel())) - { - Some(s) => s, - None => return Err(TiffError::LimitsExceeded), - }; - - let max_sample_bits = self.image().bits_per_sample; - match self.image().sample_format { - SampleFormat::Uint => match max_sample_bits { - n if n <= 8 => DecodingResult::new_u8(buffer_size, &self.limits), - n if n <= 16 => DecodingResult::new_u16(buffer_size, &self.limits), - n if n <= 32 => DecodingResult::new_u32(buffer_size, &self.limits), - n if n <= 64 => DecodingResult::new_u64(buffer_size, &self.limits), - n => Err(TiffError::UnsupportedError( - TiffUnsupportedError::UnsupportedBitsPerChannel(n), - )), - }, - SampleFormat::IEEEFP => match max_sample_bits { - 32 => DecodingResult::new_f32(buffer_size, &self.limits), - 64 => DecodingResult::new_f64(buffer_size, &self.limits), - n => Err(TiffError::UnsupportedError( - TiffUnsupportedError::UnsupportedBitsPerChannel(n), - )), - }, - SampleFormat::Int => match max_sample_bits { - n if n <= 8 => DecodingResult::new_i8(buffer_size, &self.limits), - n if n <= 16 => DecodingResult::new_i16(buffer_size, &self.limits), - n if n <= 32 => DecodingResult::new_i32(buffer_size, &self.limits), - n if n <= 64 => DecodingResult::new_i64(buffer_size, &self.limits), - n => Err(TiffError::UnsupportedError( - TiffUnsupportedError::UnsupportedBitsPerChannel(n), - )), - }, - format => Err(TiffUnsupportedError::UnsupportedSampleFormat(vec![format]).into()), - } - } - /// Read the specified chunk (at index `chunk_index`) and return the binary data as a Vector. - pub async fn read_chunk(&mut self, chunk_index: u32) -> TiffResult { + pub async fn read_chunk_async(&mut self, chunk_index: u32) -> TiffResult { let data_dims = self.image().chunk_data_dimensions(chunk_index)?; let mut result = self.result_buffer(data_dims.0 as usize, data_dims.1 as usize)?; - self.read_chunk_to_buffer(result.as_buffer(0), chunk_index, data_dims.0 as usize).await?; + self.read_chunk_to_buffer_async(result.as_buffer(0), chunk_index, data_dims.0 as usize) + .await?; Ok(result) } - /// Returns the default chunk size for the current image. Any given chunk in the image is at most as large as - /// the value returned here. For the size of the data (chunk minus padding), use `chunk_data_dimensions`. - pub fn chunk_dimensions(&self) -> (u32, u32) { - self.image().chunk_dimensions().unwrap() - } - - /// Returns the size of the data in the chunk with the specified index. This is the default size of the chunk, - /// minus any padding. - pub fn chunk_data_dimensions(&self, chunk_index: u32) -> (u32, u32) { - self.image() - .chunk_data_dimensions(chunk_index) - .expect("invalid chunk_index") - } - /// Decodes the entire image and return it as a Vector - pub async fn read_image(&mut self) -> TiffResult { + pub async fn read_image_async(&mut self) -> TiffResult { let width = self.image().width; let height = self.image().height; let mut result = self.result_buffer(width as usize, height as usize)?; @@ -639,15 +506,18 @@ impl Decoder { )); } - // in planar config, an image has chunks/n_bands chunks + // in planar config, an image has chunks/n_bands chunks let image_chunks = self.image().chunk_offsets.len() / self.image().strips_per_pixel(); // For multi-band images, only the first band is read. // Possible improvements: // * pass requested band as parameter // * collect bands to a RGB encoding result in case of RGB bands for chunk in 0..image_chunks { - let (offset, length) = self.image.chunk_file_range(chunk.try_into().unwrap())?; - let v = self.reader.read_range(offset.try_into()?, (offset + length).try_into()?).await?; + let (offset, length) = self.image.chunk_file_range(chunk.try_into().unwrap())?; + let v = self + .reader + .read_range(offset, offset + length) + .await?; let mut reader = std::io::Cursor::new(v); // self.goto_offset_u64(self.image().chunk_offsets[chunk]).await?; @@ -668,15 +538,4 @@ impl Decoder { Ok(result) } - - - #[inline] - pub async fn goto_offset_u64(&mut self, offset: u64) -> std::io::Result<()> { - self.reader.seek(SeekFrom::Start(offset)).await.map(|_| ()) - } -} - -#[cfg(test)] -mod test { - use super::*; } diff --git a/src/decoder_async/readme.md b/src/decoder/async_decoder/readme.md similarity index 100% rename from src/decoder_async/readme.md rename to src/decoder/async_decoder/readme.md diff --git a/src/decoder/async_decoder/stream.rs b/src/decoder/async_decoder/stream.rs new file mode 100644 index 00000000..42abf5f6 --- /dev/null +++ b/src/decoder/async_decoder/stream.rs @@ -0,0 +1,72 @@ +use futures::{AsyncRead, AsyncReadExt, AsyncSeek, AsyncSeekExt}; +use std::io; +use crate::decoder::stream::{SmartReader, ByteOrder}; + +macro_rules! read_async_fn { + ($name:ident, $type:ty) => { + /// reads an $type + #[inline(always)] + async fn $name(&mut self) -> Result<$type, io::Error> { + let mut n = [0u8; std::mem::size_of::<$type>()]; + self.read_exact(&mut n).await?; + Ok(match self.byte_order() { + ByteOrder::LittleEndian => <$type>::from_le_bytes(n), + ByteOrder::BigEndian => <$type>::from_be_bytes(n), + }) + } + }; +} + +#[async_trait::async_trait] +/// Reader that is aware of the byte order. +pub trait EndianAsyncReader: AsyncRead + Unpin { + /// Byte order that should be adhered to + fn byte_order(&self) -> ByteOrder; + + read_async_fn!(read_u16, u16); + read_async_fn!(read_i8, i8); + read_async_fn!(read_i16, i16); + read_async_fn!(read_u32, u32); + read_async_fn!(read_i32, i32); + read_async_fn!(read_u64, u64); + read_async_fn!(read_i64, i64); + read_async_fn!(read_f32, f32); + read_async_fn!(read_f64, f64); +} + + + +impl EndianAsyncReader for SmartReader { + #[inline(always)] + fn byte_order(&self) -> ByteOrder { + self.byte_order + } +} + +impl AsyncRead for SmartReader { + fn poll_read( + self: std::pin::Pin<&mut Self>, + cx: &mut std::task::Context<'_>, + buf: &mut [u8], + ) -> std::task::Poll> { + let pinned = std::pin::Pin::new(&mut self.get_mut().reader); + pinned.poll_read(cx, buf) + } +} + +impl AsyncSeek for SmartReader { + fn poll_seek( + self: std::pin::Pin<&mut Self>, + cx: &mut std::task::Context<'_>, + pos: io::SeekFrom, + ) -> std::task::Poll> { + let pinned = std::pin::Pin::new(&mut self.get_mut().reader); + pinned.poll_seek(cx, pos) + } +} + +impl SmartReader { + pub async fn goto_offset_async(&mut self, offset: u64) -> io::Result<()> { + self.seek(io::SeekFrom::Start(offset)).await.map(|_| ()) + } +} \ No newline at end of file diff --git a/src/decoder_async/tag_reader.rs b/src/decoder/async_decoder/tag_reader.rs similarity index 70% rename from src/decoder_async/tag_reader.rs rename to src/decoder/async_decoder/tag_reader.rs index a42c777c..cc09f250 100644 --- a/src/decoder_async/tag_reader.rs +++ b/src/decoder/async_decoder/tag_reader.rs @@ -1,13 +1,13 @@ use crate::tags::Tag; use crate::{TiffError, TiffFormatError, TiffResult}; -use crate::decoder::{ifd::Value, Limits}; -use crate::decoder_async::{stream::AsyncSmartReader, Directory}; +use super::{Directory}; +use crate::decoder::{ifd::Value, Limits, stream::SmartReader}; use futures::{AsyncRead, AsyncSeek}; -pub(crate) struct AsyncTagReader<'a, R: AsyncRead + AsyncSeek + Unpin + Send> { - pub reader: &'a mut AsyncSmartReader, +pub struct AsyncTagReader<'a, R: AsyncRead + AsyncSeek + Unpin + Send> { + pub reader: &'a mut SmartReader, pub ifd: &'a Directory, pub limits: &'a Limits, pub bigtiff: bool, @@ -18,11 +18,11 @@ impl<'a, R: AsyncRead + AsyncSeek + Unpin + Send> AsyncTagReader<'a, R> { Some(entry) => Some( entry .clone() - .val(self.limits, self.bigtiff, self.reader) + .async_val(self.limits, self.bigtiff, self.reader) .await?, ), None => None, - }) + }) // dinsdagmiddag 14-16 2e verdieping } pub(crate) async fn require_tag(&mut self, tag: Tag) -> TiffResult { match self.find_tag(tag).await? { @@ -32,8 +32,12 @@ impl<'a, R: AsyncRead + AsyncSeek + Unpin + Send> AsyncTagReader<'a, R> { )), } } - pub(crate) async fn find_tag_uint_vec>(&mut self, tag: Tag) -> TiffResult>> { - self.find_tag(tag).await? + pub(crate) async fn find_tag_uint_vec>( + &mut self, + tag: Tag, + ) -> TiffResult>> { + self.find_tag(tag) + .await? .map(|v| v.into_u64_vec()) .transpose()? .map(|v| { diff --git a/src/decoder/ifd.rs b/src/decoder/ifd.rs index e2e9d2ee..5a7a21b7 100644 --- a/src/decoder/ifd.rs +++ b/src/decoder/ifd.rs @@ -362,24 +362,13 @@ impl Entry { } /// Returns a mem_reader for the offset/value field - fn r(&self, byte_order: ByteOrder) -> SmartReader>> { + pub(crate) fn r(&self, byte_order: ByteOrder) -> SmartReader>> { SmartReader::wrap(io::Cursor::new(self.offset.to_vec()), byte_order) } - pub fn val( - &self, - limits: &super::Limits, - bigtiff: bool, - reader: &mut SmartReader, - ) -> TiffResult { - // Case 1: there are no values so we can return immediately. - if self.count == 0 { - return Ok(List(Vec::new())); - } - - let bo = reader.byte_order(); - - let tag_size = match self.type_ { + #[inline(always)] + pub(crate) fn tag_size(&self) -> u64 { + match self.type_ { Type::BYTE | Type::SBYTE | Type::ASCII | Type::UNDEFINED => 1, Type::SHORT | Type::SSHORT => 2, Type::LONG | Type::SLONG | Type::FLOAT | Type::IFD => 4, @@ -389,20 +378,27 @@ impl Entry { | Type::RATIONAL | Type::SRATIONAL | Type::IFD8 => 8, - }; + } + } - let value_bytes = match self.count.checked_mul(tag_size) { - Some(n) => n, + #[inline(always)] + pub(crate) fn value_bytes(&self) -> TiffResult { + match self.count.checked_mul(self.tag_size()) { + Some(n) => Ok(n), None => { return Err(TiffError::LimitsExceeded); } - }; + } + } + pub(crate) fn val_if_in_offset(&self, bigtiff: bool, bo: ByteOrder) -> TiffResult> { + let value_bytes = self.value_bytes()?; + // Case 2: there is one value. if self.count == 1 { // 2a: the value is 5-8 bytes and we're in BigTiff mode. if bigtiff && value_bytes > 4 && value_bytes <= 8 { - return Ok(match self.type_ { + return Ok(Some(match self.type_ { Type::LONG8 => UnsignedBig(self.r(bo).read_u64()?), Type::SLONG8 => SignedBig(self.r(bo).read_i64()?), Type::DOUBLE => Double(self.r(bo).read_f64()?), @@ -425,11 +421,11 @@ impl Entry { | Type::SLONG | Type::FLOAT | Type::IFD => unreachable!(), - }); + })); } // 2b: the value is at most 4 bytes or doesn't fit in the offset field. - return Ok(match self.type_ { + return Ok(Some(match self.type_ { Type::BYTE => Unsigned(u32::from(self.offset[0])), Type::SBYTE => Signed(i32::from(self.offset[0] as i8)), Type::UNDEFINED => Byte(self.offset[0]), @@ -438,6 +434,7 @@ impl Entry { Type::LONG => Unsigned(self.r(bo).read_u32()?), Type::SLONG => Signed(self.r(bo).read_i32()?), Type::FLOAT => Float(self.r(bo).read_f32()?), + Type::IFD => Ifd(self.r(bo).read_u32()?), Type::ASCII => { if self.offset[0] == 0 { Ascii("".to_string()) @@ -445,57 +442,34 @@ impl Entry { return Err(TiffError::FormatError(TiffFormatError::InvalidTag)); } } - Type::LONG8 => { - reader.goto_offset(self.r(bo).read_u32()?.into())?; - UnsignedBig(reader.read_u64()?) - } - Type::SLONG8 => { - reader.goto_offset(self.r(bo).read_u32()?.into())?; - SignedBig(reader.read_i64()?) - } - Type::DOUBLE => { - reader.goto_offset(self.r(bo).read_u32()?.into())?; - Double(reader.read_f64()?) - } - Type::RATIONAL => { - reader.goto_offset(self.r(bo).read_u32()?.into())?; - Rational(reader.read_u32()?, reader.read_u32()?) - } - Type::SRATIONAL => { - reader.goto_offset(self.r(bo).read_u32()?.into())?; - SRational(reader.read_i32()?, reader.read_i32()?) - } - Type::IFD => Ifd(self.r(bo).read_u32()?), - Type::IFD8 => { - reader.goto_offset(self.r(bo).read_u32()?.into())?; - IfdBig(reader.read_u64()?) - } - }); + _ => return Ok(None) + })); } + // Case 3: There is more than one value, but it fits in the offset field. if value_bytes <= 4 || bigtiff && value_bytes <= 8 { match self.type_ { - Type::BYTE => return offset_to_bytes(self.count as usize, self), - Type::SBYTE => return offset_to_sbytes(self.count as usize, self), + Type::BYTE => return Ok(Some(offset_to_bytes(self.count as usize, self)?)), + Type::SBYTE => return Ok(Some(offset_to_sbytes(self.count as usize, self)?)), Type::ASCII => { let mut buf = vec![0; self.count as usize]; self.r(bo).read_exact(&mut buf)?; if buf.is_ascii() && buf.ends_with(&[0]) { let v = str::from_utf8(&buf)?; let v = v.trim_matches(char::from(0)); - return Ok(Ascii(v.into())); + return Ok(Some(Ascii(v.into()))); } else { return Err(TiffError::FormatError(TiffFormatError::InvalidTag)); } } Type::UNDEFINED => { - return Ok(List( + return Ok(Some(List( self.offset[0..self.count as usize] .iter() .map(|&b| Byte(b)) .collect(), - )); + ))); } Type::SHORT => { let mut r = self.r(bo); @@ -503,7 +477,7 @@ impl Entry { for _ in 0..self.count { v.push(Short(r.read_u16()?)); } - return Ok(List(v)); + return Ok(Some(List(v))); } Type::SSHORT => { let mut r = self.r(bo); @@ -511,7 +485,7 @@ impl Entry { for _ in 0..self.count { v.push(Signed(i32::from(r.read_i16()?))); } - return Ok(List(v)); + return Ok(Some(List(v))); } Type::LONG => { let mut r = self.r(bo); @@ -519,7 +493,7 @@ impl Entry { for _ in 0..self.count { v.push(Unsigned(r.read_u32()?)); } - return Ok(List(v)); + return Ok(Some(List(v))); } Type::SLONG => { let mut r = self.r(bo); @@ -527,7 +501,7 @@ impl Entry { for _ in 0..self.count { v.push(Signed(r.read_i32()?)); } - return Ok(List(v)); + return Ok(Some(List(v))); } Type::FLOAT => { let mut r = self.r(bo); @@ -535,7 +509,7 @@ impl Entry { for _ in 0..self.count { v.push(Float(r.read_f32()?)); } - return Ok(List(v)); + return Ok(Some(List(v))); } Type::IFD => { let mut r = self.r(bo); @@ -543,7 +517,7 @@ impl Entry { for _ in 0..self.count { v.push(Ifd(r.read_u32()?)); } - return Ok(List(v)); + return Ok(Some(List(v))); } Type::LONG8 | Type::SLONG8 @@ -556,78 +530,132 @@ impl Entry { } } + Ok(None) + } + + #[inline(always)] + pub(crate) fn offset(&self, bigtiff: bool, bo: ByteOrder) -> TiffResult { + if bigtiff { + Ok(self.r(bo).read_u64()?) + } else { + Ok(self.r(bo).read_u32()?.into()) + } + } + + pub fn val( + &self, + limits: &super::Limits, + bigtiff: bool, + reader: &mut SmartReader, + ) -> TiffResult { + let bo = reader.byte_order(); + if let Some(res) = self.val_if_in_offset(bigtiff, bo)? { + return Ok(res); + } + + // check if we exceed the limits and read required bytes into a buffer if everything is ok + // This allows us to also create a cursor in async code + let v_bytes = usize::try_from(self.value_bytes()?)?; + if v_bytes > limits.decoding_buffer_size { + return Err(TiffError::LimitsExceeded); + } + let mut buf = vec![0; v_bytes]; + reader.goto_offset(self.offset(bigtiff, bo)?)?; + reader.read_exact(&mut buf)?; + let mut r = SmartReader::wrap(std::io::Cursor::new(buf), bo); + self.val_from_cursor(&mut r) + } + + pub(crate) fn val_from_cursor( + &self, + reader: &mut SmartReader, + ) -> TiffResult { + if self.count == 1 { + // 2b: the value is at most 4 bytes or doesn't fit in the offset field. + return Ok(match self.type_ { + Type::LONG8 => { + UnsignedBig(reader.read_u64()?) + } + Type::SLONG8 => { + SignedBig(reader.read_i64()?) + } + Type::DOUBLE => { + Double(reader.read_f64()?) + } + Type::RATIONAL => { + Rational(reader.read_u32()?, reader.read_u32()?) + } + Type::SRATIONAL => { + SRational(reader.read_i32()?, reader.read_i32()?) + } + Type::IFD8 => { + IfdBig(reader.read_u64()?) + } + Type::IFD | Type::BYTE | Type::SBYTE | Type::UNDEFINED | Type::SHORT | Type::SSHORT | Type::LONG | Type::SLONG | Type::FLOAT | Type::ASCII => unreachable!() + }); + } + // Case 4: there is more than one value, and it doesn't fit in the offset field. match self.type_ { // TODO check if this could give wrong results // at a different endianess of file/computer. - Type::BYTE => self.decode_offset(self.count, bo, bigtiff, limits, reader, |reader| { + Type::BYTE => self.decode_from_cursor(self.count, reader, |reader| { let mut buf = [0; 1]; reader.read_exact(&mut buf)?; Ok(UnsignedBig(u64::from(buf[0]))) }), - Type::SBYTE => self.decode_offset(self.count, bo, bigtiff, limits, reader, |reader| { + Type::SBYTE => self.decode_from_cursor(self.count, reader, |reader| { Ok(SignedBig(i64::from(reader.read_i8()?))) }), - Type::SHORT => self.decode_offset(self.count, bo, bigtiff, limits, reader, |reader| { + Type::SHORT => self.decode_from_cursor(self.count, reader, |reader| { Ok(UnsignedBig(u64::from(reader.read_u16()?))) }), - Type::SSHORT => self.decode_offset(self.count, bo, bigtiff, limits, reader, |reader| { + Type::SSHORT => self.decode_from_cursor(self.count, reader, |reader| { Ok(SignedBig(i64::from(reader.read_i16()?))) }), - Type::LONG => self.decode_offset(self.count, bo, bigtiff, limits, reader, |reader| { + Type::LONG => self.decode_from_cursor(self.count, reader, |reader| { Ok(Unsigned(reader.read_u32()?)) }), - Type::SLONG => self.decode_offset(self.count, bo, bigtiff, limits, reader, |reader| { + Type::SLONG => self.decode_from_cursor(self.count, reader, |reader| { Ok(Signed(reader.read_i32()?)) }), - Type::FLOAT => self.decode_offset(self.count, bo, bigtiff, limits, reader, |reader| { + Type::FLOAT => self.decode_from_cursor(self.count, reader, |reader| { Ok(Float(reader.read_f32()?)) }), - Type::DOUBLE => self.decode_offset(self.count, bo, bigtiff, limits, reader, |reader| { + Type::DOUBLE => self.decode_from_cursor(self.count, reader, |reader| { Ok(Double(reader.read_f64()?)) }), Type::RATIONAL => { - self.decode_offset(self.count, bo, bigtiff, limits, reader, |reader| { + self.decode_from_cursor(self.count, reader, |reader| { Ok(Rational(reader.read_u32()?, reader.read_u32()?)) }) } Type::SRATIONAL => { - self.decode_offset(self.count, bo, bigtiff, limits, reader, |reader| { + self.decode_from_cursor(self.count, reader, |reader| { Ok(SRational(reader.read_i32()?, reader.read_i32()?)) }) } - Type::LONG8 => self.decode_offset(self.count, bo, bigtiff, limits, reader, |reader| { + Type::LONG8 => self.decode_from_cursor(self.count, reader, |reader| { Ok(UnsignedBig(reader.read_u64()?)) }), - Type::SLONG8 => self.decode_offset(self.count, bo, bigtiff, limits, reader, |reader| { + Type::SLONG8 => self.decode_from_cursor(self.count, reader, |reader| { Ok(SignedBig(reader.read_i64()?)) }), - Type::IFD => self.decode_offset(self.count, bo, bigtiff, limits, reader, |reader| { + Type::IFD => self.decode_from_cursor(self.count, reader, |reader| { Ok(Ifd(reader.read_u32()?)) }), - Type::IFD8 => self.decode_offset(self.count, bo, bigtiff, limits, reader, |reader| { + Type::IFD8 => self.decode_from_cursor(self.count, reader, |reader| { Ok(IfdBig(reader.read_u64()?)) }), Type::UNDEFINED => { - self.decode_offset(self.count, bo, bigtiff, limits, reader, |reader| { + self.decode_from_cursor(self.count, reader, |reader| { let mut buf = [0; 1]; reader.read_exact(&mut buf)?; Ok(Byte(buf[0])) }) } Type::ASCII => { - let n = usize::try_from(self.count)?; - if n > limits.decoding_buffer_size { - return Err(TiffError::LimitsExceeded); - } - - if bigtiff { - reader.goto_offset(self.r(bo).read_u64()?)? - } else { - reader.goto_offset(self.r(bo).read_u32()?.into())? - } - - let mut out = vec![0; n]; + let mut out = vec![0; self.count.try_into()?]; reader.read_exact(&mut out)?; // Strings may be null-terminated, so we trim anything downstream of the null byte if let Some(first) = out.iter().position(|&b| b == 0) { @@ -638,6 +666,20 @@ impl Entry { } } + #[inline(always)] + pub(crate) fn decode_from_cursor) -> TiffResult>( + &self, + value_count: u64, + reader: &mut SmartReader, + decode_fn: F + ) -> TiffResult{ + let mut v = Vec::with_capacity(usize::try_from(value_count)?); + for _ in 0..value_count { + v.push(decode_fn(reader)?); + } + Ok(List(v)) + } + #[inline] fn decode_offset( &self, @@ -675,7 +717,7 @@ impl Entry { /// Extracts a list of BYTE tags stored in an offset #[inline] -pub fn offset_to_bytes(n: usize, entry: &Entry) -> TiffResult { +fn offset_to_bytes(n: usize, entry: &Entry) -> TiffResult { Ok(List( entry.offset[0..n] .iter() @@ -686,7 +728,7 @@ pub fn offset_to_bytes(n: usize, entry: &Entry) -> TiffResult { /// Extracts a list of SBYTE tags stored in an offset #[inline] -pub fn offset_to_sbytes(n: usize, entry: &Entry) -> TiffResult { +fn offset_to_sbytes(n: usize, entry: &Entry) -> TiffResult { Ok(List( entry.offset[0..n] .iter() diff --git a/src/decoder/image.rs b/src/decoder/image.rs index c071e3a6..f22872b4 100644 --- a/src/decoder/image.rs +++ b/src/decoder/image.rs @@ -164,7 +164,7 @@ impl Image { // Technically bits_per_sample.len() should be *equal* to samples, but libtiff also allows // it to be a single value that applies to all samples. - if bits_per_sample.len() != samples as usize && bits_per_sample.len() != 1 { + if bits_per_sample.len() != usize::from(samples) && bits_per_sample.len() != 1 { return Err(TiffError::FormatError( TiffFormatError::InconsistentSizesEncountered, )); @@ -687,7 +687,7 @@ impl Image { let row = &mut row[..data_row_bytes]; reader.read_exact(row)?; - // println!("chunk={chunk_index}, index={i}"); + println!("chunk={chunk_index}, index={i}"); // Skip horizontal padding if chunk_row_bytes > data_row_bytes { diff --git a/src/decoder/mod.rs b/src/decoder/mod.rs index 064c146d..daabefae 100644 --- a/src/decoder/mod.rs +++ b/src/decoder/mod.rs @@ -15,9 +15,10 @@ use crate::tags::{ use self::stream::{ByteOrder, EndianReader, SmartReader}; pub mod ifd; -pub mod image; -pub mod stream; +mod image; +mod stream; mod tag_reader; +mod async_decoder; /// Result of a decoding process #[derive(Debug)] @@ -45,7 +46,7 @@ pub enum DecodingResult { } impl DecodingResult { - pub fn new_u8(size: usize, limits: &Limits) -> TiffResult { + fn new_u8(size: usize, limits: &Limits) -> TiffResult { if size > limits.decoding_buffer_size { Err(TiffError::LimitsExceeded) } else { @@ -53,7 +54,7 @@ impl DecodingResult { } } - pub fn new_u16(size: usize, limits: &Limits) -> TiffResult { + fn new_u16(size: usize, limits: &Limits) -> TiffResult { if size > limits.decoding_buffer_size / 2 { Err(TiffError::LimitsExceeded) } else { @@ -61,7 +62,7 @@ impl DecodingResult { } } - pub fn new_u32(size: usize, limits: &Limits) -> TiffResult { + fn new_u32(size: usize, limits: &Limits) -> TiffResult { if size > limits.decoding_buffer_size / 4 { Err(TiffError::LimitsExceeded) } else { @@ -69,7 +70,7 @@ impl DecodingResult { } } - pub fn new_u64(size: usize, limits: &Limits) -> TiffResult { + fn new_u64(size: usize, limits: &Limits) -> TiffResult { if size > limits.decoding_buffer_size / 8 { Err(TiffError::LimitsExceeded) } else { @@ -77,7 +78,7 @@ impl DecodingResult { } } - pub fn new_f32(size: usize, limits: &Limits) -> TiffResult { + fn new_f32(size: usize, limits: &Limits) -> TiffResult { if size > limits.decoding_buffer_size / std::mem::size_of::() { Err(TiffError::LimitsExceeded) } else { @@ -85,7 +86,7 @@ impl DecodingResult { } } - pub fn new_f64(size: usize, limits: &Limits) -> TiffResult { + fn new_f64(size: usize, limits: &Limits) -> TiffResult { if size > limits.decoding_buffer_size / std::mem::size_of::() { Err(TiffError::LimitsExceeded) } else { @@ -93,7 +94,7 @@ impl DecodingResult { } } - pub fn new_i8(size: usize, limits: &Limits) -> TiffResult { + fn new_i8(size: usize, limits: &Limits) -> TiffResult { if size > limits.decoding_buffer_size / std::mem::size_of::() { Err(TiffError::LimitsExceeded) } else { @@ -101,7 +102,7 @@ impl DecodingResult { } } - pub fn new_i16(size: usize, limits: &Limits) -> TiffResult { + fn new_i16(size: usize, limits: &Limits) -> TiffResult { if size > limits.decoding_buffer_size / 2 { Err(TiffError::LimitsExceeded) } else { @@ -109,7 +110,7 @@ impl DecodingResult { } } - pub fn new_i32(size: usize, limits: &Limits) -> TiffResult { + fn new_i32(size: usize, limits: &Limits) -> TiffResult { if size > limits.decoding_buffer_size / 4 { Err(TiffError::LimitsExceeded) } else { @@ -117,7 +118,7 @@ impl DecodingResult { } } - pub fn new_i64(size: usize, limits: &Limits) -> TiffResult { + fn new_i64(size: usize, limits: &Limits) -> TiffResult { if size > limits.decoding_buffer_size / 8 { Err(TiffError::LimitsExceeded) } else { @@ -166,7 +167,7 @@ pub enum DecodingBuffer<'a> { } impl<'a> DecodingBuffer<'a> { - pub fn as_bytes_mut(&mut self) -> &mut [u8] { + fn as_bytes_mut(&mut self) -> &mut [u8] { match self { DecodingBuffer::U8(ref mut buf) => buf, DecodingBuffer::I8(buf) => bytecast::i8_as_ne_mut_bytes(buf), @@ -243,8 +244,6 @@ impl Default for Limits { /// Currently does not support decoding of interlaced images #[derive(Debug)] pub struct Decoder -where - R: Read + Seek, { reader: SmartReader, bigtiff: bool, @@ -289,7 +288,7 @@ fn rev_hpredict_nsamp(buf: &mut [u8], bit_depth: u8, samples: usize) { } } -pub fn predict_f32(input: &mut [u8], output: &mut [u8], samples: usize) { +fn predict_f32(input: &mut [u8], output: &mut [u8], samples: usize) { for i in samples..input.len() { input[i] = input[i].wrapping_add(input[i - samples]); } @@ -304,7 +303,7 @@ pub fn predict_f32(input: &mut [u8], output: &mut [u8], samples: usize) { } } -pub fn predict_f64(input: &mut [u8], output: &mut [u8], samples: usize) { +fn predict_f64(input: &mut [u8], output: &mut [u8], samples: usize) { for i in samples..input.len() { input[i] = input[i].wrapping_add(input[i - samples]); } @@ -323,7 +322,7 @@ pub fn predict_f64(input: &mut [u8], output: &mut [u8], samples: usize) { } } -pub fn fix_endianness_and_predict( +fn fix_endianness_and_predict( buf: &mut [u8], bit_depth: u8, samples: usize, @@ -349,7 +348,7 @@ pub fn fix_endianness_and_predict( } } -pub fn invert_colors(buf: &mut [u8], color_type: ColorType, sample_format: SampleFormat) { +fn invert_colors(buf: &mut [u8], color_type: ColorType, sample_format: SampleFormat) { match (color_type, sample_format) { (ColorType::Gray(8), SampleFormat::Uint) => { for x in buf { @@ -420,6 +419,132 @@ fn fix_endianness(buf: &mut [u8], byte_order: ByteOrder, bit_depth: u8) { }; } +impl Decoder { + pub fn with_limits(mut self, limits: Limits) -> Decoder { + self.limits = limits; + self + } + + pub fn dimensions(&mut self) -> TiffResult<(u32, u32)> { + Ok((self.image().width, self.image().height)) + } + + pub fn colortype(&mut self) -> TiffResult { + self.image().colortype() + } + + fn image(&self) -> &Image { + &self.image + } + + /// Returns `true` if there is at least one more image available. + pub fn more_images(&self) -> bool { + self.next_ifd.is_some() + } + + + fn check_chunk_type(&self, expected: ChunkType) -> TiffResult<()> { + if expected != self.image().chunk_type { + return Err(TiffError::UsageError(UsageError::InvalidChunkType( + expected, + self.image().chunk_type, + ))); + } + + Ok(()) + } + + /// The chunk type (Strips / Tiles) of the image + pub fn get_chunk_type(&self) -> ChunkType { + self.image().chunk_type + } + + /// Number of strips in image + pub fn strip_count(&mut self) -> TiffResult { + self.check_chunk_type(ChunkType::Strip)?; + let rows_per_strip = self.image().strip_decoder.as_ref().unwrap().rows_per_strip; + + if rows_per_strip == 0 { + return Ok(0); + } + + // rows_per_strip - 1 can never fail since we know it's at least 1 + let height = match self.image().height.checked_add(rows_per_strip - 1) { + Some(h) => h, + None => return Err(TiffError::IntSizeError), + }; + + let strips = match self.image().planar_config { + PlanarConfiguration::Chunky => height / rows_per_strip, + PlanarConfiguration::Planar => height / rows_per_strip * self.image().samples as u32, + }; + + Ok(strips) + } + + /// Number of tiles in image + pub fn tile_count(&mut self) -> TiffResult { + self.check_chunk_type(ChunkType::Tile)?; + Ok(u32::try_from(self.image().chunk_offsets.len())?) + } + + + fn result_buffer(&self, width: usize, height: usize) -> TiffResult { + let buffer_size = match width + .checked_mul(height) + .and_then(|x| x.checked_mul(self.image().samples_per_pixel())) + { + Some(s) => s, + None => return Err(TiffError::LimitsExceeded), + }; + + let max_sample_bits = self.image().bits_per_sample; + match self.image().sample_format { + SampleFormat::Uint => match max_sample_bits { + n if n <= 8 => DecodingResult::new_u8(buffer_size, &self.limits), + n if n <= 16 => DecodingResult::new_u16(buffer_size, &self.limits), + n if n <= 32 => DecodingResult::new_u32(buffer_size, &self.limits), + n if n <= 64 => DecodingResult::new_u64(buffer_size, &self.limits), + n => Err(TiffError::UnsupportedError( + TiffUnsupportedError::UnsupportedBitsPerChannel(n), + )), + }, + SampleFormat::IEEEFP => match max_sample_bits { + 32 => DecodingResult::new_f32(buffer_size, &self.limits), + 64 => DecodingResult::new_f64(buffer_size, &self.limits), + n => Err(TiffError::UnsupportedError( + TiffUnsupportedError::UnsupportedBitsPerChannel(n), + )), + }, + SampleFormat::Int => match max_sample_bits { + n if n <= 8 => DecodingResult::new_i8(buffer_size, &self.limits), + n if n <= 16 => DecodingResult::new_i16(buffer_size, &self.limits), + n if n <= 32 => DecodingResult::new_i32(buffer_size, &self.limits), + n if n <= 64 => DecodingResult::new_i64(buffer_size, &self.limits), + n => Err(TiffError::UnsupportedError( + TiffUnsupportedError::UnsupportedBitsPerChannel(n), + )), + }, + format => Err(TiffUnsupportedError::UnsupportedSampleFormat(vec![format]).into()), + } + } + + + /// Returns the default chunk size for the current image. Any given chunk in the image is at most as large as + /// the value returned here. For the size of the data (chunk minus padding), use `chunk_data_dimensions`. + pub fn chunk_dimensions(&self) -> (u32, u32) { + self.image().chunk_dimensions().unwrap() + } + + /// Returns the size of the data in the chunk with the specified index. This is the default size of the chunk, + /// minus any padding. + pub fn chunk_data_dimensions(&self, chunk_index: u32) -> (u32, u32) { + self.image() + .chunk_data_dimensions(chunk_index) + .expect("invalid chunk_index") + } +} + impl Decoder { /// Create a new decoder that decodes from the stream ```r``` pub fn new(mut r: R) -> TiffResult> { @@ -499,22 +624,6 @@ impl Decoder { Ok(decoder) } - pub fn with_limits(mut self, limits: Limits) -> Decoder { - self.limits = limits; - self - } - - pub fn dimensions(&mut self) -> TiffResult<(u32, u32)> { - Ok((self.image().width, self.image().height)) - } - - pub fn colortype(&mut self) -> TiffResult { - self.image().colortype() - } - - fn image(&self) -> &Image { - &self.image - } /// Loads the IFD at the specified index in the list, if one exists pub fn seek_to_image(&mut self, ifd_index: usize) -> TiffResult<()> { @@ -589,10 +698,6 @@ impl Decoder { Ok(()) } - /// Returns `true` if there is at least one more image available. - pub fn more_images(&self) -> bool { - self.next_ifd.is_some() - } /// Returns the byte_order pub fn byte_order(&self) -> ByteOrder { @@ -895,50 +1000,6 @@ impl Decoder { self.get_tag(tag)?.into_string() } - fn check_chunk_type(&self, expected: ChunkType) -> TiffResult<()> { - if expected != self.image().chunk_type { - return Err(TiffError::UsageError(UsageError::InvalidChunkType( - expected, - self.image().chunk_type, - ))); - } - - Ok(()) - } - - /// The chunk type (Strips / Tiles) of the image - pub fn get_chunk_type(&self) -> ChunkType { - self.image().chunk_type - } - - /// Number of strips in image - pub fn strip_count(&mut self) -> TiffResult { - self.check_chunk_type(ChunkType::Strip)?; - let rows_per_strip = self.image().strip_decoder.as_ref().unwrap().rows_per_strip; - - if rows_per_strip == 0 { - return Ok(0); - } - - // rows_per_strip - 1 can never fail since we know it's at least 1 - let height = match self.image().height.checked_add(rows_per_strip - 1) { - Some(h) => h, - None => return Err(TiffError::IntSizeError), - }; - - let strips = match self.image().planar_config { - PlanarConfiguration::Chunky => height / rows_per_strip, - PlanarConfiguration::Planar => height / rows_per_strip * self.image().samples as u32, - }; - - Ok(strips) - } - - /// Number of tiles in image - pub fn tile_count(&mut self) -> TiffResult { - self.check_chunk_type(ChunkType::Tile)?; - Ok(u32::try_from(self.image().chunk_offsets.len())?) - } pub fn read_chunk_to_buffer( &mut self, @@ -968,45 +1029,6 @@ impl Decoder { Ok(()) } - fn result_buffer(&self, width: usize, height: usize) -> TiffResult { - let buffer_size = match width - .checked_mul(height) - .and_then(|x| x.checked_mul(self.image().samples_per_pixel())) - { - Some(s) => s, - None => return Err(TiffError::LimitsExceeded), - }; - - let max_sample_bits = self.image().bits_per_sample; - match self.image().sample_format { - SampleFormat::Uint => match max_sample_bits { - n if n <= 8 => DecodingResult::new_u8(buffer_size, &self.limits), - n if n <= 16 => DecodingResult::new_u16(buffer_size, &self.limits), - n if n <= 32 => DecodingResult::new_u32(buffer_size, &self.limits), - n if n <= 64 => DecodingResult::new_u64(buffer_size, &self.limits), - n => Err(TiffError::UnsupportedError( - TiffUnsupportedError::UnsupportedBitsPerChannel(n), - )), - }, - SampleFormat::IEEEFP => match max_sample_bits { - 32 => DecodingResult::new_f32(buffer_size, &self.limits), - 64 => DecodingResult::new_f64(buffer_size, &self.limits), - n => Err(TiffError::UnsupportedError( - TiffUnsupportedError::UnsupportedBitsPerChannel(n), - )), - }, - SampleFormat::Int => match max_sample_bits { - n if n <= 8 => DecodingResult::new_i8(buffer_size, &self.limits), - n if n <= 16 => DecodingResult::new_i16(buffer_size, &self.limits), - n if n <= 32 => DecodingResult::new_i32(buffer_size, &self.limits), - n if n <= 64 => DecodingResult::new_i64(buffer_size, &self.limits), - n => Err(TiffError::UnsupportedError( - TiffUnsupportedError::UnsupportedBitsPerChannel(n), - )), - }, - format => Err(TiffUnsupportedError::UnsupportedSampleFormat(vec![format]).into()), - } - } /// Read the specified chunk (at index `chunk_index`) and return the binary data as a Vector. pub fn read_chunk(&mut self, chunk_index: u32) -> TiffResult { @@ -1019,20 +1041,6 @@ impl Decoder { Ok(result) } - /// Returns the default chunk size for the current image. Any given chunk in the image is at most as large as - /// the value returned here. For the size of the data (chunk minus padding), use `chunk_data_dimensions`. - pub fn chunk_dimensions(&self) -> (u32, u32) { - self.image().chunk_dimensions().unwrap() - } - - /// Returns the size of the data in the chunk with the specified index. This is the default size of the chunk, - /// minus any padding. - pub fn chunk_data_dimensions(&self, chunk_index: u32) -> (u32, u32) { - self.image() - .chunk_data_dimensions(chunk_index) - .expect("invalid chunk_index") - } - /// Decodes the entire image and return it as a Vector pub fn read_image(&mut self) -> TiffResult { let width = self.image().width; diff --git a/src/decoder/stream.rs b/src/decoder/stream.rs index 8a995b09..0fe161d1 100644 --- a/src/decoder/stream.rs +++ b/src/decoder/stream.rs @@ -2,6 +2,8 @@ use std::io::{self, BufRead, BufReader, Read, Seek, Take}; + + /// Byte order of the TIFF file. #[derive(Clone, Copy, Debug)] pub enum ByteOrder { @@ -11,109 +13,36 @@ pub enum ByteOrder { BigEndian, } +macro_rules! read_fn { + ($name:ident, $type:ty) => { + /// reads an $type + #[inline(always)] + fn $name(&mut self) -> Result<$type, io::Error> { + let mut n = [0u8; std::mem::size_of::<$type>()]; + self.read_exact(&mut n)?; + Ok(match self.byte_order() { + ByteOrder::LittleEndian => <$type>::from_le_bytes(n), + ByteOrder::BigEndian => <$type>::from_be_bytes(n), + }) + } + }; +} + + /// Reader that is aware of the byte order. pub trait EndianReader: Read { /// Byte order that should be adhered to fn byte_order(&self) -> ByteOrder; - - /// Reads an u16 - #[inline(always)] - fn read_u16(&mut self) -> Result { - let mut n = [0u8; 2]; - self.read_exact(&mut n)?; - Ok(match self.byte_order() { - ByteOrder::LittleEndian => u16::from_le_bytes(n), - ByteOrder::BigEndian => u16::from_be_bytes(n), - }) - } - - /// Reads an i8 - #[inline(always)] - fn read_i8(&mut self) -> Result { - let mut n = [0u8; 1]; - self.read_exact(&mut n)?; - Ok(match self.byte_order() { - ByteOrder::LittleEndian => i8::from_le_bytes(n), - ByteOrder::BigEndian => i8::from_be_bytes(n), - }) - } - - /// Reads an i16 - #[inline(always)] - fn read_i16(&mut self) -> Result { - let mut n = [0u8; 2]; - self.read_exact(&mut n)?; - Ok(match self.byte_order() { - ByteOrder::LittleEndian => i16::from_le_bytes(n), - ByteOrder::BigEndian => i16::from_be_bytes(n), - }) - } - - /// Reads an u32 - #[inline(always)] - fn read_u32(&mut self) -> Result { - let mut n = [0u8; 4]; - self.read_exact(&mut n)?; - Ok(match self.byte_order() { - ByteOrder::LittleEndian => u32::from_le_bytes(n), - ByteOrder::BigEndian => u32::from_be_bytes(n), - }) - } - - /// Reads an i32 - #[inline(always)] - fn read_i32(&mut self) -> Result { - let mut n = [0u8; 4]; - self.read_exact(&mut n)?; - Ok(match self.byte_order() { - ByteOrder::LittleEndian => i32::from_le_bytes(n), - ByteOrder::BigEndian => i32::from_be_bytes(n), - }) - } - - /// Reads an u64 - #[inline(always)] - fn read_u64(&mut self) -> Result { - let mut n = [0u8; 8]; - self.read_exact(&mut n)?; - Ok(match self.byte_order() { - ByteOrder::LittleEndian => u64::from_le_bytes(n), - ByteOrder::BigEndian => u64::from_be_bytes(n), - }) - } - - /// Reads an i64 - #[inline(always)] - fn read_i64(&mut self) -> Result { - let mut n = [0u8; 8]; - self.read_exact(&mut n)?; - Ok(match self.byte_order() { - ByteOrder::LittleEndian => i64::from_le_bytes(n), - ByteOrder::BigEndian => i64::from_be_bytes(n), - }) - } - - /// Reads an f32 - #[inline(always)] - fn read_f32(&mut self) -> Result { - let mut n = [0u8; 4]; - self.read_exact(&mut n)?; - Ok(f32::from_bits(match self.byte_order() { - ByteOrder::LittleEndian => u32::from_le_bytes(n), - ByteOrder::BigEndian => u32::from_be_bytes(n), - })) - } - - /// Reads an f64 - #[inline(always)] - fn read_f64(&mut self) -> Result { - let mut n = [0u8; 8]; - self.read_exact(&mut n)?; - Ok(f64::from_bits(match self.byte_order() { - ByteOrder::LittleEndian => u64::from_le_bytes(n), - ByteOrder::BigEndian => u64::from_be_bytes(n), - })) - } + + read_fn!(read_u16, u16); + read_fn!(read_i8, i8); + read_fn!(read_i16, i16); + read_fn!(read_u32, u32); + read_fn!(read_i32, i32); + read_fn!(read_u64, u64); + read_fn!(read_i64, i64); + read_fn!(read_f32, f32); + read_fn!(read_f64, f64); } /// @@ -259,16 +188,12 @@ impl Read for PackBitsReader { /// Reader that is aware of the byte order. #[derive(Debug)] pub struct SmartReader -where - R: Read, { - reader: R, + pub(super) reader: R, pub byte_order: ByteOrder, } impl SmartReader -where - R: Read, { /// Wraps a reader pub fn wrap(reader: R, byte_order: ByteOrder) -> SmartReader { @@ -284,9 +209,7 @@ impl SmartReader { } } -impl EndianReader for SmartReader -where - R: Read, +impl EndianReader for SmartReader { #[inline(always)] fn byte_order(&self) -> ByteOrder { @@ -294,6 +217,7 @@ where } } + impl Read for SmartReader { #[inline] fn read(&mut self, buf: &mut [u8]) -> io::Result { @@ -308,6 +232,7 @@ impl Seek for SmartReader { } } + #[cfg(test)] mod test { use super::*; diff --git a/src/decoder/tag_reader.rs b/src/decoder/tag_reader.rs index 6fb41fa1..3ae2d176 100644 --- a/src/decoder/tag_reader.rs +++ b/src/decoder/tag_reader.rs @@ -1,5 +1,3 @@ -use futures::AsyncRead; - use std::io::{Read, Seek}; use crate::tags::Tag; diff --git a/src/decoder_async/ifd.rs b/src/decoder_async/ifd.rs deleted file mode 100644 index df04460a..00000000 --- a/src/decoder_async/ifd.rs +++ /dev/null @@ -1,535 +0,0 @@ -use crate::decoder_async::stream::AsyncSmartReader; -use crate::tags::Type; -pub use crate::{ - decoder::{ - ifd::Value::{ - self, Ascii, Byte, Double, Float, Ifd, IfdBig, List, Rational, RationalBig, SRational, - SRationalBig, Short, Signed, SignedBig, SignedByte, SignedShort, Unsigned, UnsignedBig, - }, - stream::{ByteOrder, EndianReader, SmartReader}, - }, - tags::Tag, -}; -use crate::{TiffError, TiffFormatError, TiffResult}; - -use futures::{future::BoxFuture, io::SeekFrom, AsyncRead, AsyncReadExt, AsyncSeek, FutureExt}; -use std::{ - collections::HashMap, - io::{Cursor, Read}, -}; - -use super::RangeReader; - -pub type Directory = HashMap; - -/// Extracts a list of BYTE tags stored in an offset -#[inline] -pub fn offset_to_bytes(n: usize, entry: &Entry) -> Value { - List( - entry.offset[0..n] - .iter() - .map(|&e| Unsigned(u32::from(e))) - .collect(), - ) -} - -/// Extracts a list of SBYTE tags stored in an offset -#[inline] -pub fn offset_to_sbytes(n: usize, entry: &Entry) -> Value { - List( - entry.offset[0..n] - .iter() - .map(|&e| Signed(i32::from(e as i8))) - .collect(), - ) -} - -#[derive(Clone, Copy)] -pub struct Entry { - type_: Type, - count: u64, - offset: [u8; 8], -} - -impl ::std::fmt::Debug for Entry { - fn fmt(&self, fmt: &mut ::std::fmt::Formatter) -> Result<(), ::std::fmt::Error> { - fmt.write_str(&format!( - "Entry {{ type_: {:?}, count: {:?}, offset: {:?} }}", - self.type_, self.count, &self.offset - )) - } -} - -impl Entry { - /// Construct a new Entry based on the tag - /// ``` - /// /* - /// tiff tag: - /// |tag|type|count|value| - /// | | - /// in parent offset (actually only really offset if it doesn't fit) - /// */ - /// ``` - /// note that, `offset=[1,2,3,4] -> [1,2,3,4,0,0,0,0]` - /// but this is taken care of when requesting val - pub fn new(type_: Type, count: u32, offset: [u8; 4]) -> Entry { - let mut offset = offset.to_vec(); - offset.append(&mut vec![0; 4]); - Entry::new_u64(type_, count.into(), offset[..].try_into().unwrap()) - } - - pub fn new_u64(type_: Type, count: u64, offset: [u8; 8]) -> Entry { - Entry { - type_, - count, - offset, - } - } - - /// Returns a mem_reader for the offset/value field - fn r(&self, byte_order: ByteOrder) -> SmartReader>> { - SmartReader::wrap(std::io::Cursor::new(self.offset.to_vec()), byte_order) - } - - #[inline(always)] - fn type_size(&self) -> u64 { - match self.type_ { - Type::BYTE | Type::SBYTE | Type::ASCII | Type::UNDEFINED => 1, - Type::SHORT | Type::SSHORT => 2, - Type::LONG | Type::SLONG | Type::FLOAT | Type::IFD => 4, - Type::LONG8 - | Type::SLONG8 - | Type::DOUBLE - | Type::RATIONAL - | Type::SRATIONAL - | Type::IFD8 => 8, - } - } - - #[inline(always)] - pub fn count(&self) -> u64 { - self.count - } - - #[inline(always)] - fn num_value_bytes(&self) -> TiffResult { - // The number of bytes our value takes up - match self.count.checked_mul(self.type_size()) { - Some(n) => Ok(n), - None => { - return Err(TiffError::LimitsExceeded); - } - } - } - - /// Get the tags value if it fits in the Value field. - pub fn maybe_val(&self, bigtiff: bool, byte_order: ByteOrder) -> TiffResult> { - // Case 1: there are no values so we can return immediately. - if self.count == 0 { - return Ok(Some(List(Vec::new()))); - } - - let bo = byte_order; - - let value_bytes = self.num_value_bytes()?; - - if value_bytes > 8 || (!bigtiff && value_bytes > 4) { - return Ok(None); - } - - // Case 2: there is one value. - if self.count == 1 { - // 2a: the value is 5-8 bytes and we're in BigTiff mode. - if bigtiff && value_bytes > 4 && value_bytes <= 8 { - return Ok(Some(match self.type_ { - Type::LONG8 => UnsignedBig(self.r(bo).read_u64()?), - Type::SLONG8 => SignedBig(self.r(bo).read_i64()?), - Type::DOUBLE => Double(self.r(bo).read_f64()?), - Type::RATIONAL => { - let mut r = self.r(bo); - Rational(r.read_u32()?, r.read_u32()?) - } - Type::SRATIONAL => { - let mut r = self.r(bo); - SRational(r.read_i32()?, r.read_i32()?) - } - Type::IFD8 => IfdBig(self.r(bo).read_u64()?), - Type::BYTE - | Type::SBYTE - | Type::ASCII - | Type::UNDEFINED - | Type::SHORT - | Type::SSHORT - | Type::LONG - | Type::SLONG - | Type::FLOAT - | Type::IFD => unreachable!(), - })); - } - - // 2b: the value is at most 4 bytes or doesn't fit in the offset field. - return Ok(match self.type_ { - Type::BYTE => Some(Unsigned(u32::from(self.offset[0]))), - Type::SBYTE => Some(Signed(i32::from(self.offset[0] as i8))), - Type::UNDEFINED => Some(Byte(self.offset[0])), - Type::SHORT => Some(Unsigned(u32::from(self.r(bo).read_u16()?))), - Type::SSHORT => Some(Signed(i32::from(self.r(bo).read_i16()?))), - Type::LONG => Some(Unsigned(self.r(bo).read_u32()?)), - Type::SLONG => Some(Signed(self.r(bo).read_i32()?)), - Type::FLOAT => Some(Float(self.r(bo).read_f32()?)), - Type::ASCII => { - if self.offset[0] == 0 { - Some(Ascii("".to_string())) - } else { - return Err(TiffError::FormatError(TiffFormatError::InvalidTag)); - } - } - Type::IFD => Some(Ifd(self.r(bo).read_u32()?)), - _ => unreachable!("This should have been caught earlier"), - }); - } - - // Case 3: There is more than one value, but it fits in the offset field. - if value_bytes <= 4 || bigtiff && value_bytes <= 8 { - match self.type_ { - Type::BYTE => return Ok(Some(offset_to_bytes(self.count as usize, self))), - Type::SBYTE => return Ok(Some(offset_to_sbytes(self.count as usize, self))), - Type::ASCII => { - let mut buf = vec![0; self.count as usize]; - self.r(bo).read_exact(&mut buf)?; - if buf.is_ascii() && buf.ends_with(&[0]) { - let v = std::str::from_utf8(&buf)?; - let v = v.trim_matches(char::from(0)); - return Ok(Some(Ascii(v.into()))); - } else { - return Err(TiffError::FormatError(TiffFormatError::InvalidTag)); - } - } - Type::UNDEFINED => { - return Ok(Some(List( - self.offset[0..self.count as usize] - .iter() - .map(|&b| Byte(b)) - .collect(), - ))); - } - Type::SHORT => { - let mut r = self.r(bo); - let mut v = Vec::new(); - for _ in 0..self.count { - v.push(Short(r.read_u16()?)); - } - return Ok(Some(List(v))); - } - Type::SSHORT => { - let mut r = self.r(bo); - let mut v = Vec::new(); - for _ in 0..self.count { - v.push(Signed(i32::from(r.read_i16()?))); - } - return Ok(Some(List(v))); - } - Type::LONG => { - let mut r = self.r(bo); - let mut v = Vec::new(); - for _ in 0..self.count { - v.push(Unsigned(r.read_u32()?)); - } - return Ok(Some(List(v))); - } - Type::SLONG => { - let mut r = self.r(bo); - let mut v = Vec::new(); - for _ in 0..self.count { - v.push(Signed(r.read_i32()?)); - } - return Ok(Some(List(v))); - } - Type::FLOAT => { - let mut r = self.r(bo); - let mut v = Vec::new(); - for _ in 0..self.count { - v.push(Float(r.read_f32()?)); - } - return Ok(Some(List(v))); - } - Type::IFD => { - let mut r = self.r(bo); - let mut v = Vec::new(); - for _ in 0..self.count { - v.push(Ifd(r.read_u32()?)); - } - return Ok(Some(List(v))); - } - Type::LONG8 - | Type::SLONG8 - | Type::RATIONAL - | Type::SRATIONAL - | Type::DOUBLE - | Type::IFD8 => { - unreachable!() - } - } - } - - // case 4: multiple and it doesn't fit - unreachable!() - } - - /// Gets the nth value of a List type, such as byte offsets and lenghts - /// If it is not an offset, - pub async fn nth_val( - &self, - n: u64, - limits: &super::Limits, - bigtiff: bool, - reader: &mut AsyncSmartReader, - ) -> TiffResult { - if self.num_value_bytes()? <= 4 || (self.num_value_bytes()? <= 8 && bigtiff) { - // return Err(TiffError::UsageError("Should not call nth val on a value that is in the Value tag field")); - panic!("Should not call this function if bla") - } - if n > self.count { - return Err(TiffError::LimitsExceeded); - } - let bo = reader.byte_order(); - let offset = if bigtiff { - self.r(bo).read_u64()? - } else { - self.r(bo).read_u32()?.into() - }; - reader.goto_offset(offset + n * self.type_size()).await?; - Ok(UnsignedBig(reader.read_u64().await?)) - } - - /// get the tags value, if it doesn't fit, it will read the pointer. - /// may cause additional reading into the file - pub async fn val( - &self, - limits: &super::Limits, - bigtiff: bool, - reader: &mut AsyncSmartReader, - ) -> TiffResult { - let bo = reader.byte_order(); - - // The number of bytes our value takes up - let value_bytes = self.num_value_bytes()?; - let offset = if bigtiff { - self.r(bo).read_u64()? - } else { - self.r(bo).read_u32()?.into() - }; - // case 1: the value fits in the value field - if let Some(maybe_val) = self.maybe_val(bigtiff, bo)? { - return Ok(maybe_val); - } - - // Case 2: there is one value. This only - if self.count == 1 { - // 2b: the value is at most 4 bytes or doesn't fit in the offset field. - return Ok(match self.type_ { - Type::LONG8 => { - reader.goto_offset(offset).await?; - UnsignedBig(reader.read_u64().await?) - } - Type::SLONG8 => { - reader.goto_offset(offset).await?; - SignedBig(reader.read_i64().await?) - } - Type::DOUBLE => { - reader.goto_offset(offset).await?; - Double(reader.read_f64().await?) - } - Type::RATIONAL => { - reader.goto_offset(offset).await?; - Rational(reader.read_u32().await?, reader.read_u32().await?) - } - Type::SRATIONAL => { - reader.goto_offset(offset).await?; - SRational(reader.read_i32().await?, reader.read_i32().await?) - } - Type::IFD8 => { - reader.goto_offset(offset).await?; - IfdBig(reader.read_u64().await?) - } - _ => unreachable!(), - }); - } - - // TODO: find out if this is actually faster (I think it is...) - // initialize the buffer with all tag data inside it - // let buf = reader.read_range(offset, offset + self.count * self.type_size()).await?; - // let synr = SmartReader::wrap(Cursor::new(buf), bo); - - // Case 4: there is more than one value, and it doesn't fit in the offset field. - // Async help found here: https://users.rust-lang.org/t/function-that-takes-an-async-closure/61663 - match self.type_ { - // TODO check if this could give wrong results - // at a different endianess of file/computer. - Type::BYTE => { - self.decode_offset(self.count, bo, bigtiff, limits, reader, |reader| { - async move { - let mut buf = [0; 1]; - reader.read_exact(&mut buf).await?; - Ok(UnsignedBig(u64::from(buf[0]))) - } - .boxed() - }) - .await - } - Type::SBYTE => { - self.decode_offset(self.count, bo, bigtiff, limits, reader, |reader| { - async move { Ok(SignedBig(i64::from(reader.read_i8().await?))) }.boxed() - }) - .await - } - Type::SHORT => { - self.decode_offset(self.count, bo, bigtiff, limits, reader, |reader| { - async move { Ok(UnsignedBig(u64::from(reader.read_u16().await?))) }.boxed() - }) - .await - } - Type::SSHORT => { - self.decode_offset(self.count, bo, bigtiff, limits, reader, |reader| { - async move { Ok(SignedBig(i64::from(reader.read_i16().await?))) }.boxed() - }) - .await - } - Type::LONG => { - self.decode_offset(self.count, bo, bigtiff, limits, reader, |reader| { - async move { Ok(Unsigned(reader.read_u32().await?)) }.boxed() - }) - .await - } - Type::SLONG => { - self.decode_offset(self.count, bo, bigtiff, limits, reader, |reader| { - async move { Ok(Signed(reader.read_i32().await?)) }.boxed() - }) - .await - } - Type::FLOAT => { - self.decode_offset(self.count, bo, bigtiff, limits, reader, |reader| { - async move { Ok(Float(reader.read_f32().await?)) }.boxed() - }) - .await - } - Type::DOUBLE => { - self.decode_offset(self.count, bo, bigtiff, limits, reader, |reader| { - async move { Ok(Double(reader.read_f64().await?)) }.boxed() - }) - .await - } - Type::RATIONAL => { - self.decode_offset(self.count, bo, bigtiff, limits, reader, |reader| { - async move { Ok(Rational(reader.read_u32().await?, reader.read_u32().await?)) } - .boxed() - }) - .await - } - Type::SRATIONAL => { - self.decode_offset(self.count, bo, bigtiff, limits, reader, |reader| { - async move { - Ok(SRational( - reader.read_i32().await?, - reader.read_i32().await?, - )) - } - .boxed() - }) - .await - } - Type::LONG8 => { - self.decode_offset(self.count, bo, bigtiff, limits, reader, |reader| { - async move { Ok(UnsignedBig(reader.read_u64().await?)) }.boxed() - }) - .await - } - Type::SLONG8 => { - self.decode_offset(self.count, bo, bigtiff, limits, reader, |reader| { - async move { Ok(SignedBig(reader.read_i64().await?)) }.boxed() - }) - .await - } - Type::IFD => { - self.decode_offset(self.count, bo, bigtiff, limits, reader, |reader| { - async move { Ok(Ifd(reader.read_u32().await?)) }.boxed() - }) - .await - } - Type::IFD8 => { - self.decode_offset(self.count, bo, bigtiff, limits, reader, |reader| { - async move { Ok(IfdBig(reader.read_u64().await?)) }.boxed() - }) - .await - } - Type::UNDEFINED => { - self.decode_offset(self.count, bo, bigtiff, limits, reader, |reader| { - async move { - let mut buf = [0; 1]; - reader.read_exact(&mut buf).await?; - Ok(Byte(buf[0])) - } - .boxed() - }) - .await - } - Type::ASCII => { - let n = usize::try_from(self.count)?; - if n > limits.decoding_buffer_size { - return Err(TiffError::LimitsExceeded); - } - - if bigtiff { - reader.goto_offset(self.r(bo).read_u64()?).await? - } else { - reader.goto_offset(offset).await? - } - - let mut out = vec![0; n]; - reader.read_exact(&mut out).await?; - // Strings may be null-terminated, so we trim anything downstream of the null byte - if let Some(first) = out.iter().position(|&b| b == 0) { - out.truncate(first); - } - Ok(Ascii(String::from_utf8(out)?)) - } - } - } - - /// Goes to offset and decodes all values there - /// This is the interesting part where tile offsets are read - #[inline] - async fn decode_offset( - &self, - value_count: u64, - bo: ByteOrder, - bigtiff: bool, - limits: &super::Limits, - reader: &mut AsyncSmartReader, - decode_fn: F, - ) -> TiffResult - where - R: AsyncRead + AsyncSeek + Unpin, - // F: Fn(&mut AsyncSmartReader) -> TiffResult, - F: Fn(&'_ mut AsyncSmartReader) -> BoxFuture<'_, TiffResult>, - { - let value_count = usize::try_from(value_count)?; - if value_count > limits.decoding_buffer_size / std::mem::size_of::() { - return Err(TiffError::LimitsExceeded); - } - - let mut v = Vec::with_capacity(value_count); - - let offset = if bigtiff { - self.r(bo).read_u64()? - } else { - self.r(bo).read_u32()?.into() - }; - reader.goto_offset(offset).await?; - - for _ in 0..value_count { - v.push(decode_fn(reader).await?) - } - Ok(List(v)) - } -} diff --git a/src/decoder_async/image.rs b/src/decoder_async/image.rs deleted file mode 100644 index 2cd57c27..00000000 --- a/src/decoder_async/image.rs +++ /dev/null @@ -1,806 +0,0 @@ -use crate::decoder::{ - fix_endianness_and_predict, - ifd::Value, - invert_colors, predict_f32, predict_f64, - stream::{ByteOrder, DeflateReader, LZWReader}, - Limits, -}; -use crate::decoder_async::{ - ifd::{Directory, Entry}, - image::Value::UnsignedBig, - stream::AsyncSmartReader, - tag_reader::AsyncTagReader, - ChunkType, -}; -use crate::tags::{ - CompressionMethod, PhotometricInterpretation, PlanarConfiguration, Predictor, SampleFormat, Tag, -}; -use crate::{ColorType, TiffError, TiffFormatError, TiffResult, TiffUnsupportedError, UsageError}; - -use futures::{io::Empty, AsyncRead, AsyncReadExt, AsyncSeek}; - -use std::{ - io::{Cursor, Read}, - sync::Arc, -}; - -#[derive(Debug)] -pub(crate) struct StripDecodeState { - pub rows_per_strip: u32, -} - -#[derive(Debug)] -/// Computed values useful for tile decoding -pub(crate) struct TileAttributes { - pub image_width: usize, - pub image_height: usize, - - pub tile_width: usize, - pub tile_length: usize, -} - -impl TileAttributes { - pub fn tiles_across(&self) -> usize { - (self.image_width + self.tile_width - 1) / self.tile_width - } - pub fn tiles_down(&self) -> usize { - (self.image_height + self.tile_length - 1) / self.tile_length - } - fn padding_right(&self) -> usize { - (self.tile_width - self.image_width % self.tile_width) % self.tile_width - } - fn padding_down(&self) -> usize { - (self.tile_length - self.image_height % self.tile_length) % self.tile_length - } - pub fn get_padding(&self, tile: usize) -> (usize, usize) { - let row = tile / self.tiles_across(); - let column = tile % self.tiles_across(); - - let padding_right = if column == self.tiles_across() - 1 { - self.padding_right() - } else { - 0 - }; - - let padding_down = if row == self.tiles_down() - 1 { - self.padding_down() - } else { - 0 - }; - - (padding_right, padding_down) - } -} - -// #[derive(Debug)] -// pub(crate) enum ChunkData { -// Empty(Entry), -// Full(Vec), -// } - -// impl ChunkData { -// fn get(&self, index: usize) -> Option<&u64> { -// match self { -// ChunkData::Full(v) => v.get(index), -// ChunkData::Empty(entry) => None, -// } -// } - -// /// retrieves a single entry from the reader -// pub async fn retrieve_single( -// &self, -// index: u64, -// limits: &Limits, -// bigtiff: bool, -// reader: &mut R, -// ) -> TiffResult { -// match self { -// ChunkData::Empty(entry) => entry.nth_val(index, limits, bigtiff, reader).await, -// ChunkData::Full(v) => { -// println!("retrieve called when we had a full buffer"); -// v.get(index as usize) -// .map(|v| UnsignedBig(*v)) -// .ok_or(TiffError::LimitsExceeded) -// } -// } -// } - -// /// Fills the buffer. After this, we will be ChunkData::Full and lookups will be super fast -// pub async fn fill( -// &mut self, -// index: u64, -// limits: &Limits, -// bigtiff: bool, -// reader: &mut AsyncSmartReader, -// ) -> TiffResult<()> { -// let ChunkData::Empty(entry) = self else { -// println!("Called Fill while already full!"); -// return Ok(()); -// }; -// *self = ChunkData::Full(entry.val(limits, bigtiff, reader).await?.into_u64_vec()?); -// Ok(()) -// } -// } - -#[derive(Debug)] -pub(crate) struct AsyncImage { - pub ifd: Option, - pub width: u32, - pub height: u32, - pub bits_per_sample: u8, - pub samples: u16, - pub sample_format: SampleFormat, - pub photometric_interpretation: PhotometricInterpretation, - pub compression_method: CompressionMethod, - pub predictor: Predictor, - pub jpeg_tables: Option>>, - pub chunk_type: ChunkType, - pub planar_config: PlanarConfiguration, - pub strip_decoder: Option, - pub tile_attributes: Option, - pub chunk_offsets: Vec, - pub chunk_bytes: Vec, -} - -impl AsyncImage { - /// Creates this image from a reader. Will not read in chunk tags - /// Rather, this - pub async fn from_reader( - reader: &mut AsyncSmartReader, - ifd: Directory, - limits: &Limits, - bigtiff: bool, - ) -> TiffResult { - let mut tag_reader = AsyncTagReader { - reader, - limits, - ifd: &ifd, - bigtiff, - }; - - let width = tag_reader.require_tag(Tag::ImageWidth).await?.into_u32()?; - let height = tag_reader.require_tag(Tag::ImageLength).await?.into_u32()?; - if width == 0 || height == 0 { - return Err(TiffError::FormatError(TiffFormatError::InvalidDimensions( - width, height, - ))); - } - - let photometric_interpretation = tag_reader - .find_tag(Tag::PhotometricInterpretation) - .await? - .map(Value::into_u16) - .transpose()? - .and_then(PhotometricInterpretation::from_u16) - .ok_or(TiffUnsupportedError::UnknownInterpretation)?; - - // Try to parse both the compression method and the number, format, and bits of the included samples. - // If they are not explicitly specified, those tags are reset to their default values and not carried from previous images. - let compression_method = match tag_reader.find_tag(Tag::Compression).await? { - Some(val) => CompressionMethod::from_u16_exhaustive(val.into_u16()?), - None => CompressionMethod::None, - }; - - let jpeg_tables = if compression_method == CompressionMethod::ModernJPEG - && ifd.contains_key(&Tag::JPEGTables) - { - let vec = tag_reader - .find_tag(Tag::JPEGTables) - .await? - .unwrap() - .into_u8_vec()?; - if vec.len() < 2 { - return Err(TiffError::FormatError( - TiffFormatError::InvalidTagValueType(Tag::JPEGTables), - )); - } - - Some(Arc::new(vec)) - } else { - None - }; - - let samples: u16 = tag_reader - .find_tag(Tag::SamplesPerPixel) - .await? - .map(Value::into_u16) - .transpose()? - .unwrap_or(1); - if samples == 0 { - return Err(TiffFormatError::SamplesPerPixelIsZero.into()); - } - - let sample_format = match tag_reader.find_tag_uint_vec(Tag::SampleFormat).await? { - Some(vals) => { - let sample_format: Vec<_> = vals - .into_iter() - .map(SampleFormat::from_u16_exhaustive) - .collect(); - - // TODO: for now, only homogenous formats across samples are supported. - if !sample_format.windows(2).all(|s| s[0] == s[1]) { - return Err(TiffUnsupportedError::UnsupportedSampleFormat(sample_format).into()); - } - - sample_format[0] - } - None => SampleFormat::Uint, - }; - - let bits_per_sample: Vec = tag_reader - .find_tag_uint_vec(Tag::BitsPerSample).await? - .unwrap_or_else(|| vec![1]); - - // Technically bits_per_sample.len() should be *equal* to samples, but libtiff also allows - // it to be a single value that applies to all samples. - if bits_per_sample.len() != samples as usize && bits_per_sample.len() != 1 { - return Err(TiffError::FormatError( - TiffFormatError::InconsistentSizesEncountered, - )); - } - - // This library (and libtiff) do not support mixed sample formats and zero bits per sample - // doesn't make sense. - if bits_per_sample.iter().any(|&b| b != bits_per_sample[0]) || bits_per_sample[0] == 0 { - return Err(TiffUnsupportedError::InconsistentBitsPerSample(bits_per_sample).into()); - } - - let predictor = tag_reader - .find_tag(Tag::Predictor) - .await? - .map(Value::into_u16) - .transpose()? - .map(|p| { - Predictor::from_u16(p) - .ok_or(TiffError::FormatError(TiffFormatError::UnknownPredictor(p))) - }) - .transpose()? - .unwrap_or(Predictor::None); - - let planar_config = tag_reader - .find_tag(Tag::PlanarConfiguration) - .await? - .map(Value::into_u16) - .transpose()? - .map(|p| { - PlanarConfiguration::from_u16(p).ok_or(TiffError::FormatError( - TiffFormatError::UnknownPlanarConfiguration(p), - )) - }) - .transpose()? - .unwrap_or(PlanarConfiguration::Chunky); - - let planes = match planar_config { - PlanarConfiguration::Chunky => 1, - PlanarConfiguration::Planar => samples, - }; - - let chunk_type; - let chunk_offsets; - let chunk_bytes; - let strip_decoder; - let tile_attributes; - match ( - ifd.contains_key(&Tag::StripByteCounts), - ifd.contains_key(&Tag::StripOffsets), - ifd.contains_key(&Tag::TileByteCounts), - ifd.contains_key(&Tag::TileOffsets), - ) { - (true, true, false, false) => { - chunk_type = ChunkType::Strip; - - chunk_offsets = //ifd[&Tag::StripOffsets]; - tag_reader - .find_tag(Tag::StripOffsets).await? - .unwrap() - .into_u64_vec()?; - chunk_bytes = //ifd[&Tag::StripByteCounts]; - tag_reader - .find_tag(Tag::StripByteCounts).await? - .unwrap() - .into_u64_vec()?; - let rows_per_strip = tag_reader - .find_tag(Tag::RowsPerStrip) - .await? - .map(Value::into_u32) - .transpose()? - .unwrap_or(height); - strip_decoder = Some(StripDecodeState { rows_per_strip }); - tile_attributes = None; - - if chunk_offsets.len() != chunk_bytes.len() - || rows_per_strip == 0 - || u32::try_from(chunk_offsets.len())? - != (height.saturating_sub(1) / rows_per_strip + 1) * planes as u32 - { - return Err(TiffError::FormatError( - TiffFormatError::InconsistentSizesEncountered, - )); - } - } - (false, false, true, true) => { - chunk_type = ChunkType::Tile; - - let tile_width = - usize::try_from(tag_reader.require_tag(Tag::TileWidth).await?.into_u32()?)?; - let tile_length = - usize::try_from(tag_reader.require_tag(Tag::TileLength).await?.into_u32()?)?; - - if tile_width == 0 { - return Err(TiffFormatError::InvalidTagValueType(Tag::TileWidth).into()); - } else if tile_length == 0 { - return Err(TiffFormatError::InvalidTagValueType(Tag::TileLength).into()); - } - - strip_decoder = None; - tile_attributes = Some(TileAttributes { - image_width: usize::try_from(width)?, - image_height: usize::try_from(height)?, - tile_width, - tile_length, - }); - chunk_offsets = //ifd[&Tag::TileOffsets]; - tag_reader - .find_tag(Tag::TileOffsets).await? - .unwrap() - .into_u64_vec()?; - chunk_bytes = //ifd[&Tag::TileByteCounts]; - tag_reader - .find_tag(Tag::TileByteCounts).await? - .unwrap() - .into_u64_vec()?; - - let tile = tile_attributes.as_ref().unwrap(); - if chunk_offsets.len() != chunk_bytes.len() - || chunk_offsets.len() as usize - != tile.tiles_down() * tile.tiles_across() * planes as usize - { - return Err(TiffError::FormatError( - TiffFormatError::InconsistentSizesEncountered, - )); - } - } - (_, _, _, _) => { - return Err(TiffError::FormatError( - TiffFormatError::StripTileTagConflict, - )) - } - }; - - Ok(AsyncImage { - ifd: Some(ifd), - width, - height, - bits_per_sample: bits_per_sample[0], - samples, - sample_format, - photometric_interpretation, - compression_method, - jpeg_tables, - predictor, - chunk_type, - planar_config, - strip_decoder, - tile_attributes, - chunk_offsets: chunk_offsets, - chunk_bytes: chunk_bytes, - }) - } - - pub(crate) fn colortype(&self) -> TiffResult { - match self.photometric_interpretation { - PhotometricInterpretation::RGB => match self.samples { - 3 => Ok(ColorType::RGB(self.bits_per_sample)), - 4 => Ok(ColorType::RGBA(self.bits_per_sample)), - // FIXME: We should _ignore_ other components. In particular: - // > Beware of extra components. Some TIFF files may have more components per pixel - // than you think. A Baseline TIFF reader must skip over them gracefully,using the - // values of the SamplesPerPixel and BitsPerSample fields. - // > -- TIFF 6.0 Specification, Section 7, Additional Baseline requirements. - _ => Err(TiffError::UnsupportedError( - TiffUnsupportedError::InterpretationWithBits( - self.photometric_interpretation, - vec![self.bits_per_sample; self.samples as usize], - ), - )), - }, - PhotometricInterpretation::CMYK => match self.samples { - 4 => Ok(ColorType::CMYK(self.bits_per_sample)), - _ => Err(TiffError::UnsupportedError( - TiffUnsupportedError::InterpretationWithBits( - self.photometric_interpretation, - vec![self.bits_per_sample; self.samples as usize], - ), - )), - }, - PhotometricInterpretation::YCbCr => match self.samples { - 3 => Ok(ColorType::YCbCr(self.bits_per_sample)), - _ => Err(TiffError::UnsupportedError( - TiffUnsupportedError::InterpretationWithBits( - self.photometric_interpretation, - vec![self.bits_per_sample; self.samples as usize], - ), - )), - }, - PhotometricInterpretation::BlackIsZero | PhotometricInterpretation::WhiteIsZero => { - match self.samples { - 1 => Ok(ColorType::Gray(self.bits_per_sample)), - _ => Ok(ColorType::Multiband { - bit_depth: self.bits_per_sample, - num_samples: self.samples, - }), - } - } - // TODO: this is bad we should not fail at this point - PhotometricInterpretation::RGBPalette - | PhotometricInterpretation::TransparencyMask - | PhotometricInterpretation::CIELab => Err(TiffError::UnsupportedError( - TiffUnsupportedError::InterpretationWithBits( - self.photometric_interpretation, - vec![self.bits_per_sample; self.samples as usize], - ), - )), - } - } - - fn create_reader<'r, R: 'r + Read>( - reader: R, - photometric_interpretation: PhotometricInterpretation, - compression_method: CompressionMethod, - compressed_length: u64, - jpeg_tables: Option<&[u8]>, - ) -> TiffResult> { - Ok(match compression_method { - CompressionMethod::None => Box::new(reader), - CompressionMethod::LZW => { - Box::new(LZWReader::new(reader, usize::try_from(compressed_length)?)) - } - CompressionMethod::Deflate | CompressionMethod::OldDeflate => { - Box::new(DeflateReader::new(reader)) - } - CompressionMethod::ModernJPEG => { - if jpeg_tables.is_some() && compressed_length < 2 { - return Err(TiffError::FormatError( - TiffFormatError::InvalidTagValueType(Tag::JPEGTables), - )); - } - - // Construct new jpeg_reader wrapping a SmartReader. - // - // JPEG compression in TIFF allows saving quantization and/or huffman tables in one - // central location. These `jpeg_tables` are simply prepended to the remaining jpeg image data. - // Because these `jpeg_tables` start with a `SOI` (HEX: `0xFFD8`) or __start of image__ marker - // which is also at the beginning of the remaining JPEG image data and would - // confuse the JPEG renderer, one of these has to be taken off. In this case the first two - // bytes of the remaining JPEG data is removed because it follows `jpeg_tables`. - // Similary, `jpeg_tables` ends with a `EOI` (HEX: `0xFFD9`) or __end of image__ marker, - // this has to be removed as well (last two bytes of `jpeg_tables`). - let jpeg_reader = match jpeg_tables { - Some(jpeg_tables) => { - let mut reader = reader.take(compressed_length); - reader.read_exact(&mut [0; 2])?; - - Box::new( - Cursor::new(&jpeg_tables[..jpeg_tables.len() - 2]) - .chain(reader.take(compressed_length)), - ) as Box - } - None => Box::new(reader.take(compressed_length)), - }; - - let mut decoder = jpeg::Decoder::new(jpeg_reader); - - match photometric_interpretation { - PhotometricInterpretation::RGB => { - decoder.set_color_transform(jpeg::ColorTransform::RGB) - } - PhotometricInterpretation::WhiteIsZero => { - decoder.set_color_transform(jpeg::ColorTransform::None) - } - PhotometricInterpretation::BlackIsZero => { - decoder.set_color_transform(jpeg::ColorTransform::None) - } - PhotometricInterpretation::TransparencyMask => { - decoder.set_color_transform(jpeg::ColorTransform::None) - } - PhotometricInterpretation::CMYK => { - decoder.set_color_transform(jpeg::ColorTransform::CMYK) - } - PhotometricInterpretation::YCbCr => { - decoder.set_color_transform(jpeg::ColorTransform::YCbCr) - } - photometric_interpretation => { - return Err(TiffError::UnsupportedError( - TiffUnsupportedError::UnsupportedInterpretation( - photometric_interpretation, - ), - )); - } - } - - let data = decoder.decode()?; - - Box::new(Cursor::new(data)) - } - method => { - return Err(TiffError::UnsupportedError( - TiffUnsupportedError::UnsupportedCompressionMethod(method), - )) - } - }) - } - - /// Samples per pixel within chunk. - /// - /// In planar config, samples are stored in separate strips/chunks, also called bands. - /// - /// Example with `bits_per_sample = [8, 8, 8]` and `PhotometricInterpretation::RGB`: - /// * `PlanarConfiguration::Chunky` -> 3 (RGBRGBRGB...) - /// * `PlanarConfiguration::Planar` -> 1 (RRR...) (GGG...) (BBB...) - pub(crate) fn samples_per_pixel(&self) -> usize { - match self.planar_config { - PlanarConfiguration::Chunky => self.samples.into(), - PlanarConfiguration::Planar => 1, - } - } - - /// Number of strips per pixel. - pub(crate) fn strips_per_pixel(&self) -> usize { - match self.planar_config { - PlanarConfiguration::Chunky => 1, - PlanarConfiguration::Planar => self.samples.into(), - } - } - - pub(crate) fn chunk_file_range(&self, chunk: u32) -> TiffResult<(u64, u64)> { - let file_offset = self - .chunk_offsets - .get(chunk as usize) - .ok_or(TiffError::FormatError( - TiffFormatError::InconsistentSizesEncountered, - ))?; - - let compressed_bytes = - self.chunk_bytes - .get(chunk as usize) - .ok_or(TiffError::FormatError( - TiffFormatError::InconsistentSizesEncountered, - ))?; - - Ok((*file_offset, *compressed_bytes)) - } - - /// Dimensions of a chunk, which is a strip or tile. - /// typically, this is a power of 2 for tiled COGS, such as 1024 by 1024 - pub(crate) fn chunk_dimensions(&self) -> TiffResult<(u32, u32)> { - match self.chunk_type { - ChunkType::Strip => { - let strip_attrs = self.strip_decoder.as_ref().unwrap(); - Ok((self.width, strip_attrs.rows_per_strip)) - } - ChunkType::Tile => { - let tile_attrs = self.tile_attributes.as_ref().unwrap(); - Ok(( - u32::try_from(tile_attrs.tile_width)?, - u32::try_from(tile_attrs.tile_length)?, - )) - } - } - } - - /// Dimensions of the data within the chunk. - /// see in get_padding that this is the chunk dimensions, - /// unless we are at the bottom or far side of the image, - /// in which case there is some padding involved because the full image is not necessarily a power of 2 - pub(crate) fn chunk_data_dimensions(&self, chunk_index: u32) -> TiffResult<(u32, u32)> { - let dims = self.chunk_dimensions()?; - - match self.chunk_type { - ChunkType::Strip => { - let strip_attrs = self.strip_decoder.as_ref().unwrap(); - let strips_per_band = - self.height.saturating_sub(1) / strip_attrs.rows_per_strip + 1; - let strip_height_without_padding = (chunk_index % strips_per_band) - .checked_mul(dims.1) - .and_then(|x| self.height.checked_sub(x)) - .ok_or(TiffError::UsageError(UsageError::InvalidChunkIndex( - chunk_index, - )))?; - - // Ignore potential vertical padding on the bottommost strip - let strip_height = dims.1.min(strip_height_without_padding); - - Ok((dims.0, strip_height)) - } - ChunkType::Tile => { - let tile_attrs = self.tile_attributes.as_ref().unwrap(); - let (padding_right, padding_down) = tile_attrs.get_padding(chunk_index as usize); - - let tile_width = tile_attrs.tile_width - padding_right; - let tile_length = tile_attrs.tile_length - padding_down; - - Ok((u32::try_from(tile_width)?, u32::try_from(tile_length)?)) - } - } - } - - pub(crate) fn expand_chunk( - &self, - reader: impl Read, - buf: &mut [u8], - output_row_stride: usize, - byte_order: ByteOrder, - chunk_index: u32, - limits: &Limits, - ) -> TiffResult<()> { - // Validate that the color type is supported. - let color_type = self.colortype()?; - match color_type { - ColorType::RGB(n) - | ColorType::RGBA(n) - | ColorType::CMYK(n) - | ColorType::YCbCr(n) - | ColorType::Gray(n) - | ColorType::Multiband { - bit_depth: n, - num_samples: _, - } if n == 8 || n == 16 || n == 32 || n == 64 => {} - ColorType::Gray(n) - | ColorType::Multiband { - bit_depth: n, - num_samples: _, - } if n < 8 => match self.predictor { - Predictor::None => {} - Predictor::Horizontal => { - return Err(TiffError::UnsupportedError( - TiffUnsupportedError::HorizontalPredictor(color_type), - )); - } - Predictor::FloatingPoint => { - return Err(TiffError::UnsupportedError( - TiffUnsupportedError::FloatingPointPredictor(color_type), - )); - } - }, - type_ => { - return Err(TiffError::UnsupportedError( - TiffUnsupportedError::UnsupportedColorType(type_), - )); - } - } - - // Validate that the predictor is supported for the sample type. - match (self.predictor, self.sample_format) { - (Predictor::Horizontal, SampleFormat::Int | SampleFormat::Uint) => {} - (Predictor::Horizontal, _) => { - return Err(TiffError::UnsupportedError( - TiffUnsupportedError::HorizontalPredictor(color_type), - )); - } - (Predictor::FloatingPoint, SampleFormat::IEEEFP) => {} - (Predictor::FloatingPoint, _) => { - return Err(TiffError::UnsupportedError( - TiffUnsupportedError::FloatingPointPredictor(color_type), - )); - } - _ => {} - } - - let compressed_bytes = self.chunk_bytes - .get(chunk_index as usize) - .ok_or(TiffError::FormatError( - TiffFormatError::InconsistentSizesEncountered, - ))?;//match &self.chunk_bytes { - // ChunkData::Full(v) => { - // self.chunk_bytes - // .get(chunk_index as usize) - // .ok_or(TiffError::FormatError( - // TiffFormatError::InconsistentSizesEncountered, - // ))? - // } - // ChunkData::Empty(_) => &self - // .chunk_bytes - // .retrieve_single(chunk_index, limits, self.bigtiff, reader) - // .await? - // .into_u64()?, - // }; - if *compressed_bytes > limits.intermediate_buffer_size as u64 { - return Err(TiffError::LimitsExceeded); - } - - let compression_method = self.compression_method; - let photometric_interpretation = self.photometric_interpretation; - let predictor = self.predictor; - let samples = self.samples_per_pixel(); - - let chunk_dims = self.chunk_dimensions()?; - let data_dims = self.chunk_data_dimensions(chunk_index)?; - - let chunk_row_bits = (u64::from(chunk_dims.0) * u64::from(self.bits_per_sample)) - .checked_mul(samples as u64) - .ok_or(TiffError::LimitsExceeded)?; - let chunk_row_bytes: usize = ((chunk_row_bits + 7) / 8).try_into()?; - - let data_row_bits = (u64::from(data_dims.0) * u64::from(self.bits_per_sample)) - .checked_mul(samples as u64) - .ok_or(TiffError::LimitsExceeded)?; - let data_row_bytes: usize = ((data_row_bits + 7) / 8).try_into()?; - - // TODO: Should these return errors instead? - assert!(output_row_stride >= data_row_bytes); - assert!(buf.len() >= output_row_stride * (data_dims.1 as usize - 1) + data_row_bytes); - - let mut reader = Self::create_reader( - reader, - photometric_interpretation, - compression_method, - *compressed_bytes, - self.jpeg_tables.as_deref().map(|a| &**a), - )?; - - if output_row_stride == chunk_row_bytes as usize { - let tile = &mut buf[..chunk_row_bytes * data_dims.1 as usize]; - reader.read_exact(tile)?; - - for row in tile.chunks_mut(chunk_row_bytes as usize) { - fix_endianness_and_predict( - row, - color_type.bit_depth(), - samples, - byte_order, - predictor, - ); - } - if photometric_interpretation == PhotometricInterpretation::WhiteIsZero { - super::invert_colors(tile, color_type, self.sample_format); - } - } else if chunk_row_bytes > data_row_bytes && self.predictor == Predictor::FloatingPoint { - // The floating point predictor shuffles the padding bytes into the encoded output, so - // this case is handled specially when needed. - let mut encoded = vec![0u8; chunk_row_bytes]; - for row in buf.chunks_mut(output_row_stride).take(data_dims.1 as usize) { - reader.read_exact(&mut encoded)?; - - let row = &mut row[..data_row_bytes]; - match color_type.bit_depth() { - 32 => predict_f32(&mut encoded, row, samples), - 64 => predict_f64(&mut encoded, row, samples), - _ => unreachable!(), - } - if photometric_interpretation == PhotometricInterpretation::WhiteIsZero { - invert_colors(row, color_type, self.sample_format); - } - } - } else { - for (i, row) in buf - .chunks_mut(output_row_stride) - .take(data_dims.1 as usize) - .enumerate() - { - let row = &mut row[..data_row_bytes]; - reader.read_exact(row)?; - - // println!("chunk={chunk_index}, index={i}"); - - // Skip horizontal padding - if chunk_row_bytes > data_row_bytes { - let len = u64::try_from(chunk_row_bytes - data_row_bytes)?; - std::io::copy(&mut reader.by_ref().take(len), &mut std::io::sink())?; - } - - fix_endianness_and_predict( - row, - color_type.bit_depth(), - samples, - byte_order, - predictor, - ); - if photometric_interpretation == PhotometricInterpretation::WhiteIsZero { - invert_colors(row, color_type, self.sample_format); - } - } - } - - Ok(()) - } -} diff --git a/src/decoder_async/stream.rs b/src/decoder_async/stream.rs deleted file mode 100644 index 40ea32a3..00000000 --- a/src/decoder_async/stream.rs +++ /dev/null @@ -1,353 +0,0 @@ -// Special thanks to Alice for the help: https://users.rust-lang.org/t/63019/6 -use crate::decoder::stream::ByteOrder; -use std::io::{Result, SeekFrom}; -use std::pin::Pin; - -use futures::{ - future::BoxFuture, - io::{AsyncRead, AsyncReadExt, AsyncSeek, AsyncSeekExt, BufReader}, - Future, -}; - -// pub struct RangedStreamer { -// pos: u64, -// length: u64, // total size -// state: State, -// range_get: F, -// min_request_size: usize, // requests have at least this size -// } - -// enum State { -// HasChunk(SeekOutput), -// Seeking(BoxFuture<'static, std::io::Result>), -// } - -// pub struct SeekOutput { -// pub start: u64, -// pub data: Vec, -// } - -// pub type F = std::sync::Arc< -// dyn Fn(u64, usize) -> BoxFuture<'static, std::io::Result> + Send + Sync, -// >; - -// impl RangedStreamer { -// pub fn new(length: usize, min_request_size: usize, range_get: F) -> Self { -// let length = length as u64; -// Self { -// pos: 0, -// length, -// state: State::HasChunk(SeekOutput { -// start: 0, -// data: vec![], -// }), -// range_get, -// min_request_size, -// } -// } -// } - -// // whether `test_interval` is inside `a` (start, length). -// async fn range_includes(a: (usize, usize), test_interval: (usize, usize)) -> bool { -// if test_interval.0 < a.0 { -// return false; -// } -// let test_end = test_interval.0 + test_interval.1; -// let a_end = a.0 + a.1; -// if test_end > a_end { -// return false; -// } -// true -// } - -// impl AsyncRead for RangedStreamer { -// fn poll_read( -// mut self: std::pin::Pin<&mut Self>, -// cx: &mut std::task::Context<'_>, -// buf: &mut [u8], -// ) -> std::task::Poll> { -// let requested_range = (self.pos as usize, buf.len()); -// let min_request_size = self.min_request_size; -// match &mut self.state { -// State::HasChunk(output) => { -// let existing_range = (output.start as usize, output.data.len()); -// if range_includes(existing_range, requested_range) { -// let offset = requested_range.0 - existing_range.0; -// buf.copy_from_slice(&output.data[offset..offset + buf.len()]); -// self.pos += buf.len() as u64; -// std::task::Poll::Ready(Ok(buf.len())) -// } else { -// let start = requested_range.0 as u64; -// let length = std::cmp::max(min_request_size, requested_range.1); -// let future = (self.range_get)(start, length); -// self.state = State::Seeking(Box::pin(future)); -// self.poll_read(cx, buf) -// } -// } -// State::Seeking(ref mut future) => match Pin::new(future).poll(cx) { -// std::task::Poll::Ready(v) => { -// match v { -// Ok(output) => self.state = State::HasChunk(output), -// Err(e) => return std::task::Poll::Ready(Err(e)), -// }; -// self.poll_read(cx, buf) -// } -// std::task::Poll::Pending => std::task::Poll::Pending, -// }, -// } -// } -// } - -// impl AsyncSeek for RangedStreamer { -// fn poll_seek( -// mut self: std::pin::Pin<&mut Self>, -// _: &mut std::task::Context<'_>, -// pos: std::io::SeekFrom, -// ) -> std::task::Poll> { -// match pos { -// SeekFrom::Start(pos) => self.pos = pos, -// SeekFrom::End(pos) => self.pos = (self.length as i64 + pos) as u64, -// SeekFrom::Current(pos) => self.pos = (self.pos as i64 + pos) as u64, -// }; -// std::task::Poll::Ready(Ok(self.pos)) -// } -// } - -// pub type DeflateReader = flate2::read::ZlibDecoder; - -// /// -// /// ## LZW Reader -// /// - -// /// Reader that decompresses LZW streams -// pub struct LZWReader { -// reader: BufReader>, -// decoder: weezl::decode::Decoder, -// } - -// impl LZWReader { -// /// Wraps a reader -// pub fn new(reader: R, compressed_length: usize) -> LZWReader { -// Self { -// reader: BufReader::with_capacity( -// (32 * 1024).min(compressed_length), -// reader.take(u64::try_from(compressed_length).unwrap()), -// ), -// decoder: weezl::decode::Decoder::with_tiff_size_switch(weezl::BitOrder::Msb, 8), -// } -// } -// } - -// impl Read for LZWReader { -// fn read(&mut self, buf: &mut [u8]) -> io::Result { -// loop { -// let result = self.decoder.decode_bytes(self.reader.fill_buf()?, buf); -// self.reader.consume(result.consumed_in); - -// match result.status { -// Ok(weezl::LzwStatus::Ok) => { -// if result.consumed_out == 0 { -// continue; -// } else { -// return Ok(result.consumed_out); -// } -// } -// Ok(weezl::LzwStatus::NoProgress) => { -// assert_eq!(result.consumed_in, 0); -// assert_eq!(result.consumed_out, 0); -// assert!(self.reader.buffer().is_empty()); -// return Err(io::Error::new( -// io::ErrorKind::UnexpectedEof, -// "no lzw end code found", -// )); -// } -// Ok(weezl::LzwStatus::Done) => { -// return Ok(result.consumed_out); -// } -// Err(err) => return Err(io::Error::new(io::ErrorKind::InvalidData, err)), -// } -// } -// } -// } - -pub struct AsyncSmartReader { - reader: R, - pub byte_order: ByteOrder, -} - -impl AsyncSmartReader { - pub async fn goto_offset(&mut self, offset: u64) -> Result<()> { - self.reader.seek(SeekFrom::Start(offset)).await.map(|_| ()) - } - - pub fn wrap(reader: R, byte_order: ByteOrder) -> Self { - AsyncSmartReader { reader, byte_order } - } -} - -impl AsyncRead for AsyncSmartReader { - fn poll_read( - self: Pin<&mut Self>, - cx: &mut std::task::Context<'_>, - buf: &mut [u8], - ) -> std::task::Poll> { - let pinned = std::pin::Pin::new(&mut self.get_mut().reader); - pinned.poll_read(cx, buf) - } -} - -impl AsyncSeek for AsyncSmartReader { - fn poll_seek( - self: Pin<&mut Self>, - cx: &mut std::task::Context<'_>, - pos: SeekFrom, - ) -> std::task::Poll> { - let pinned = std::pin::Pin::new(&mut self.get_mut().reader); - pinned.poll_seek(cx, pos) - } -} - -impl AsyncSmartReader { - /// Byte order that should be adhered to - pub fn byte_order(&self) -> ByteOrder { - self.byte_order - } - - /// Reads an u16 - #[inline(always)] - pub async fn read_u16(&mut self) -> Result { - let mut n = [0u8; 2]; - self.read_exact(&mut n).await?; - Ok(match self.byte_order() { - ByteOrder::LittleEndian => u16::from_le_bytes(n), - ByteOrder::BigEndian => u16::from_be_bytes(n), - }) - } - - /// Reads an i8 - #[inline(always)] - pub async fn read_i8(&mut self) -> Result { - let mut n = [0u8; 1]; - self.read_exact(&mut n).await?; - Ok(match self.byte_order() { - ByteOrder::LittleEndian => i8::from_le_bytes(n), - ByteOrder::BigEndian => i8::from_be_bytes(n), - }) - } - - /// Reads an i16 - #[inline(always)] - pub async fn read_i16(&mut self) -> Result { - let mut n = [0u8; 2]; - self.read_exact(&mut n).await?; - Ok(match self.byte_order() { - ByteOrder::LittleEndian => i16::from_le_bytes(n), - ByteOrder::BigEndian => i16::from_be_bytes(n), - }) - } - - /// Reads an u32 - #[inline(always)] - pub async fn read_u32(&mut self) -> Result { - let mut n = [0u8; 4]; - self.read_exact(&mut n).await?; - Ok(match self.byte_order() { - ByteOrder::LittleEndian => u32::from_le_bytes(n), - ByteOrder::BigEndian => u32::from_be_bytes(n), - }) - } - - /// Reads an i32 - #[inline(always)] - pub async fn read_i32(&mut self) -> Result { - let mut n = [0u8; 4]; - self.read_exact(&mut n).await?; - Ok(match self.byte_order() { - ByteOrder::LittleEndian => i32::from_le_bytes(n), - ByteOrder::BigEndian => i32::from_be_bytes(n), - }) - } - - /// Reads an u64 - #[inline(always)] - pub async fn read_u64(&mut self) -> Result { - let mut n = [0u8; 8]; - self.read_exact(&mut n).await?; - Ok(match self.byte_order() { - ByteOrder::LittleEndian => u64::from_le_bytes(n), - ByteOrder::BigEndian => u64::from_be_bytes(n), - }) - } - - /// Reads an i64 - #[inline(always)] - pub async fn read_i64(&mut self) -> Result { - let mut n = [0u8; 8]; - self.read_exact(&mut n).await?; - Ok(match self.byte_order() { - ByteOrder::LittleEndian => i64::from_le_bytes(n), - ByteOrder::BigEndian => i64::from_be_bytes(n), - }) - } - - /// Reads an f32 - #[inline(always)] - pub async fn read_f32(&mut self) -> Result { - let mut n = [0u8; 4]; - self.read_exact(&mut n).await?; - Ok(f32::from_bits(match self.byte_order() { - ByteOrder::LittleEndian => u32::from_le_bytes(n), - ByteOrder::BigEndian => u32::from_be_bytes(n), - })) - } - - /// Reads an f64 - #[inline(always)] - pub async fn read_f64(&mut self) -> Result { - let mut n = [0u8; 8]; - self.read_exact(&mut n).await?; - Ok(f64::from_bits(match self.byte_order() { - ByteOrder::LittleEndian => u64::from_le_bytes(n), - ByteOrder::BigEndian => u64::from_be_bytes(n), - })) - } -} -// /// Reader that is aware of the byte order. -// #[derive(Debug)] -// pub struct AsyncSmartReader -// where -// R: AsyncRead, -// { -// reader: R, -// pub byte_order: ByteOrder, -// } - -// impl AsyncSmartReader -// where -// R: AsyncRead, -// { -// /// Wraps a reader -// pub fn wrap(reader: R, byte_order: ByteOrder) -> AsyncSmartReader { -// AsyncSmartReader { reader, byte_order } -// } -// pub fn into_inner(self) -> R { -// self.reader -// } -// } -// impl AsyncSmartReader { - -// #[inline(always)] -// fn byte_order(&self) -> ByteOrder { -// self.byte_order -// } -// } - -// impl AsyncSeek for AsyncSmartReader { -// fn poll_seek( -// self: Pin<&mut Self>, -// cx: &mut std::task::Context<'_>, -// pos: SeekFrom, -// ) -> std::task::Poll> { - -// } -// } diff --git a/src/lib.rs b/src/lib.rs index 4e06ed4d..a4522ced 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -11,7 +11,6 @@ extern crate weezl; mod bytecast; pub mod decoder; -pub mod decoder_async; pub mod encoder; mod error; pub mod tags; diff --git a/tests/decode_images_async.rs b/tests/decode_images_async.rs index f00d101c..bf16d15b 100644 --- a/tests/decode_images_async.rs +++ b/tests/decode_images_async.rs @@ -1,12 +1,16 @@ +// #[cfg(feature="async_decoder")] +mod test_async +{ extern crate tiff; -use tiff::decoder_async::{ifd, Decoder}; use tiff::decoder::DecodingResult; +use tiff::decoder::Decoder; use tiff::ColorType; use futures::io::AllowStdIo; use std::fs::File; +use std::io::Cursor; use std::path::PathBuf; const TEST_IMAGE_DIR: &str = "./tests/images/"; @@ -16,9 +20,11 @@ macro_rules! test_image_sum { async fn $name(file: &str, expected_type: ColorType, expected_sum: $sum_ty) { let path = PathBuf::from(TEST_IMAGE_DIR).join(file); let img_file = AllowStdIo::new(File::open(path).expect("Cannot find test image!")); - let mut decoder = Decoder::new(img_file).await.expect("Cannot create decoder"); + let mut decoder = Decoder::new_async(img_file) + .await + .expect("Cannot create decoder"); assert_eq!(decoder.colortype().unwrap(), expected_type); - let img_res = decoder.read_image().await.unwrap(); + let img_res = decoder.read_image_async().await.unwrap(); match img_res { DecodingResult::$buffer(res) => { @@ -46,9 +52,11 @@ test_image_sum!(test_image_sum_f64, F64, f64); async fn test_image_color_type_unsupported(file: &str, expected_type: ColorType) { let path = PathBuf::from(TEST_IMAGE_DIR).join(file); let img_file = AllowStdIo::new(File::open(path).expect("Cannot find test image!")); - let mut decoder = Decoder::new(img_file).await.expect("Cannot create decoder"); + let mut decoder = Decoder::new_async(img_file) + .await + .expect("Cannot create decoder"); assert_eq!(decoder.colortype().unwrap(), expected_type); - assert!(match decoder.read_image().await { + assert!(match decoder.read_image_async().await { Err(tiff::TiffError::UnsupportedError( tiff::TiffUnsupportedError::UnsupportedColorType(_), )) => true, @@ -107,7 +115,8 @@ async fn test_gray_f64() { "gradient-1c-64b-float.tiff", ColorType::Gray(64), 128.0319210877642, - ).await; + ) + .await; } #[tokio::test] @@ -168,8 +177,10 @@ async fn test_string_tags() { for filename in filenames.iter() { let path = PathBuf::from(TEST_IMAGE_DIR).join(filename); let img_file = AllowStdIo::new(File::open(path).expect("Cannot find test image!")); - let mut decoder = Decoder::new(img_file).await.expect("Cannot create decoder"); - let software = decoder.get_tag(tiff::tags::Tag::Software).await.unwrap(); + let mut decoder = Decoder::new_async(img_file) + .await + .expect("Cannot create decoder"); + let software = decoder.get_tag_async(tiff::tags::Tag::Software).await.unwrap(); match software { tiff::decoder::ifd::Value::Ascii(s) => assert_eq!( &s, @@ -192,10 +203,10 @@ async fn test_decode_data() { } } let file = AllowStdIo::new(File::open("./tests/decodedata-rgb-3c-8b.tiff").unwrap()); - let mut decoder = Decoder::new(file).await.unwrap(); + let mut decoder = Decoder::new_async(file).await.unwrap(); assert_eq!(decoder.colortype().unwrap(), ColorType::RGB(8)); assert_eq!(decoder.dimensions().unwrap(), (100, 100)); - if let DecodingResult::U8(img_res) = decoder.read_image().await.unwrap() { + if let DecodingResult::U8(img_res) = decoder.read_image_async().await.unwrap() { assert_eq!(image_data, img_res); } else { panic!("Wrong data type"); @@ -213,9 +224,9 @@ async fn issue_69() { //fn test_gray_alpha_u8() //{ //let img_file = File::open("./tests/images/minisblack-2c-8b-alpha.tiff").expect("Cannot find test image!"); -//let mut decoder = Decoder::new(img_file).expect("Cannot create decoder"); +//let mut decoder = Decoder::new_async(img_file).expect("Cannot create decoder"); //assert_eq!(decoder.colortype().unwrap(), ColorType::GrayA(8)); -//let img_res = decoder.read_image(); +//let img_res = decoder.read_image_async(); //assert!(img_res.is_ok()); //} @@ -270,14 +281,16 @@ async fn test_tiled_incremental() { let path = PathBuf::from(TEST_IMAGE_DIR).join(file); let img_file = AllowStdIo::new(File::open(path).expect("Cannot find test image!")); - let mut decoder = Decoder::new(img_file).await.expect("Cannot create decoder"); + let mut decoder = Decoder::new_async(img_file) + .await + .expect("Cannot create decoder"); assert_eq!(decoder.colortype().unwrap(), expected_type); let tiles = decoder.tile_count().unwrap(); assert_eq!(tiles as usize, sums.len()); for tile in 0..tiles { - match decoder.read_chunk(tile).await.unwrap() { + match decoder.read_chunk_async(tile).await.unwrap() { DecodingResult::U8(res) => { let sum: u64 = res.into_iter().map(::from).sum(); assert_eq!(sum, sums[tile as usize]); @@ -295,7 +308,9 @@ async fn test_planar_rgb_u8() { let path = PathBuf::from(TEST_IMAGE_DIR).join(file); let img_file = AllowStdIo::new(File::open(path).expect("Cannot find test image!")); - let mut decoder = Decoder::new(img_file).await.expect("Cannot create decoder"); + let mut decoder = Decoder::new_async(img_file) + .await + .expect("Cannot create decoder"); assert_eq!(decoder.colortype().unwrap(), expected_type); let chunks = decoder.strip_count().unwrap(); @@ -305,21 +320,21 @@ async fn test_planar_rgb_u8() { // 0,0: (73,51,30) #49331E srgb(73,51,30) // 1st band (red) - match decoder.read_chunk(0).await.unwrap() { + match decoder.read_chunk_async(0).await.unwrap() { DecodingResult::U8(chunk) => { assert_eq!(chunk[0], 73); } _ => panic!("Wrong bit depth"), } // 2nd band (green) - match decoder.read_chunk(chunks / 3).await.unwrap() { + match decoder.read_chunk_async(chunks / 3).await.unwrap() { DecodingResult::U8(chunk) => { assert_eq!(chunk[0], 51); } _ => panic!("Wrong bit depth"), } // 3rd band (blue) - match decoder.read_chunk(chunks / 3 * 2).await.unwrap() { + match decoder.read_chunk_async(chunks / 3 * 2).await.unwrap() { DecodingResult::U8(chunk) => { assert_eq!(chunk[0], 30); } @@ -341,7 +356,9 @@ async fn test_div_zero() { 178, 178, 178, ]; - let err = tiff::decoder::Decoder::new(std::io::Cursor::new(&image)).unwrap_err(); + let err = Decoder::new_async(AllowStdIo::new(Cursor::new(&image))) + .await + .unwrap_err(); match err { TiffError::FormatError(TiffFormatError::StripTileTagConflict) => {} @@ -359,7 +376,9 @@ async fn test_too_many_value_bytes() { 0, 89, 89, 89, 89, 89, 89, 89, 89, 96, 1, 20, 89, 89, 89, 89, 18, ]; - let error = tiff::decoder::Decoder::new(std::io::Cursor::new(&image)).unwrap_err(); + let error = Decoder::new_async(AllowStdIo::new(std::io::Cursor::new(&image))) + .await + .unwrap_err(); match error { tiff::TiffError::LimitsExceeded => {} @@ -377,7 +396,9 @@ async fn fuzzer_testcase5() { 178, 178, 178, ]; - let _ = tiff::decoder::Decoder::new(std::io::Cursor::new(&image)).unwrap_err(); + let _ = Decoder::new_async(AllowStdIo::new(Cursor::new(&image))) + .await + .unwrap_err(); } #[tokio::test] @@ -390,7 +411,9 @@ async fn fuzzer_testcase1() { 178, ]; - let _ = tiff::decoder::Decoder::new(std::io::Cursor::new(&image)).unwrap_err(); + let _ = Decoder::new_async(AllowStdIo::new(Cursor::new(&image))) + .await + .unwrap_err(); } #[tokio::test] @@ -403,7 +426,9 @@ async fn fuzzer_testcase6() { 178, 178, ]; - let _ = tiff::decoder::Decoder::new(std::io::Cursor::new(&image)).unwrap_err(); + let _ = Decoder::new_async(AllowStdIo::new(Cursor::new(&image))) + .await + .unwrap_err(); } #[tokio::test] @@ -415,7 +440,9 @@ async fn oom() { 0, 0, 0, 40, 0, 0, 0, 23, 1, 4, 0, 1, 0, 0, 0, 178, 48, 178, 178, 178, 178, 162, 178, ]; - let _ = tiff::decoder::Decoder::new(std::io::Cursor::new(&image)).unwrap_err(); + let _ = Decoder::new_async(AllowStdIo::new(Cursor::new(&image))) + .await + .unwrap_err(); } #[tokio::test] @@ -427,7 +454,9 @@ async fn fuzzer_testcase4() { 0, 0, 0, 40, 0, 0, 0, 23, 1, 4, 0, 1, 0, 0, 0, 48, 178, 178, 178, 0, 1, 0, 13, 13, ]; - let _ = tiff::decoder::Decoder::new(std::io::Cursor::new(&image)).unwrap_err(); + let _ = Decoder::new_async(AllowStdIo::new(Cursor::new(&image))) + .await + .unwrap_err(); } #[tokio::test] @@ -443,7 +472,9 @@ async fn fuzzer_testcase2() { 73, ]; - let _ = tiff::decoder::Decoder::new(std::io::Cursor::new(&image)).unwrap_err(); + let _ = Decoder::new_async(AllowStdIo::new(Cursor::new(&image))) + .await + .unwrap_err(); } #[tokio::test] @@ -459,7 +490,9 @@ async fn invalid_jpeg_tag_2() { 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 36, 73, 73, 0, 42, 36, 36, 36, 36, 0, 0, 8, 0, ]; - let _ = tiff::decoder::Decoder::new(std::io::Cursor::new(&image)).unwrap_err(); + let _ = Decoder::new_async(AllowStdIo::new(Cursor::new(&image))) + .await + .unwrap_err(); } #[tokio::test] @@ -472,7 +505,9 @@ async fn fuzzer_testcase3() { 255, 255, ]; - let _ = tiff::decoder::Decoder::new(std::io::Cursor::new(&image)).unwrap_err(); + let _ = Decoder::new_async(AllowStdIo::new(Cursor::new(&image))) + .await + .unwrap_err(); } #[tokio::test] @@ -490,7 +525,11 @@ async fn timeout() { 0, 0, 73, 73, 42, 0, 8, 0, 0, 0, 0, 0, 32, ]; - let error = tiff::decoder::Decoder::new(std::io::Cursor::new(&image)).unwrap_err(); + // hacked the test a bit: seek until we are sure that we loop, + // otherwise it will read the faulty image before finding out it is circular + let error = Decoder::new_overview_async(AllowStdIo::new(Cursor::new(&image)), 5) + .await + .unwrap_err(); match error { TiffError::FormatError(TiffFormatError::CycleInOffsets) => {} @@ -512,3 +551,4 @@ async fn test_predictor_3_rgb_f32() { async fn test_predictor_3_gray_f32() { test_image_sum_f32("predictor-3-gray-f32.tif", ColorType::Gray(32), 20008.275).await; } +} \ No newline at end of file From 4ae8f8207031ad465879e54f01e291ee579edfdf Mon Sep 17 00:00:00 2001 From: Fee Fladder Date: Tue, 17 Sep 2024 18:09:08 +0200 Subject: [PATCH 4/6] added async http example --- Cargo.toml | 17 +++- examples/async_http.rs | 148 ++++++++++++++++++++++++++++++++++- src/decoder/mod.rs | 2 + tests/decode_images_async.rs | 2 +- 4 files changed, 164 insertions(+), 5 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 05df8a4a..c164a6b3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -17,14 +17,16 @@ categories = ["multimedia::images", "multimedia::encoding"] exclude = ["tests/images/*", "tests/fuzz_images/*"] [features] -async_decoder = []#["dep:futures", "dep:async-trait"] +async_decoder = ["dep:futures", "dep:async-trait"] +ehttp = ["dep:ehttp", "async_decoder"] [dependencies] weezl = "0.1.0" jpeg = { package = "jpeg-decoder", version = "0.3.0", default-features = false } flate2 = "1.0.20" -futures = {version = "0.3.30", optional = false}#true } -async-trait = {version ="^0.1", optional = false}#true } +futures = {version = "0.3.30", optional = true } +async-trait = {version ="^0.1", optional = true } +ehttp = { version = "0.5.0", features=["native-async"], optional = true } [dev-dependencies] criterion = "0.3.1" @@ -34,3 +36,12 @@ tokio = {version = "1.29.1", features = ["macros", "fs", "rt-multi-thread"]} [[bench]] name = "lzw" harness = false + +[package.metadata.example.async_http] +name = "Async http" +description = "Example showing use of async features using async http requests" + +[[example]] +name = "async_http" +path="examples/async_http.rs" +required-features=["ehttp"] \ No newline at end of file diff --git a/examples/async_http.rs b/examples/async_http.rs index 7f755fb7..a3962308 100644 --- a/examples/async_http.rs +++ b/examples/async_http.rs @@ -1,2 +1,148 @@ +// Special thanks to Alice for the help: https://users.rust-lang.org/t/63019/6 +use std::io::{Error, Result, SeekFrom}; +use std::pin::Pin; + +use futures::{ + future::BoxFuture, + io::{AsyncRead, AsyncSeek}, + Future, +}; +use tiff::decoder::ifd::Value; +use tiff::decoder::DecodingResult; +use tiff::decoder::Decoder; + +// extern crate ehttp; + + +pub struct RangedStreamer { + state: State, + range_get: F, + min_request_size: usize, // requests have at least this size +} + +enum State { + HasChunk(SeekOutput), + Seeking(BoxFuture<'static, std::io::Result>), +} + +pub struct SeekOutput { + pub start: u64, + pub data: Vec, +} + +pub type F = Box< + dyn Fn(u64, u64) -> BoxFuture<'static, std::io::Result> + Send + Sync, +>; + +impl RangedStreamer { + pub fn new(length: usize, min_request_size: usize, range_get: F) -> Self { + let length = length as u64; + Self { + pos: 0, + length, + state: State::HasChunk(SeekOutput { + start: 0, + data: vec![], + }), + range_get, + min_request_size, + } + } +} + +// whether `test_interval` is inside `a` (start, length). +fn range_includes(a: (usize, usize), test_interval: (usize, usize)) -> bool { + if test_interval.0 < a.0 { + return false; + } + let test_end = test_interval.0 + test_interval.1; + let a_end = a.0 + a.1; + if test_end > a_end { + return false; + } + true +} + +impl AsyncRead for RangedStreamer { + fn poll_read( + mut self: std::pin::Pin<&mut Self>, + cx: &mut std::task::Context<'_>, + buf: &mut [u8], + ) -> std::task::Poll> { + let requested_range = (self.pos as usize, buf.len()); + let min_request_size = self.min_request_size; + match &mut self.state { + State::HasChunk(output) => { + let existing_range = (output.start as usize, output.data.len()); + if range_includes(existing_range, requested_range) { + let offset = requested_range.0 - existing_range.0; + buf.copy_from_slice(&output.data[offset..offset + buf.len()]); + self.pos += buf.len() as u64; + std::task::Poll::Ready(Ok(buf.len())) + } else { + let start = requested_range.0 as u64; + let length = std::cmp::max(min_request_size, requested_range.1); + let future = (self.range_get)(start, length.try_into().unwrap()); + self.state = State::Seeking(Box::pin(future)); + self.poll_read(cx, buf) + } + } + State::Seeking(ref mut future) => match Pin::new(future).poll(cx) { + std::task::Poll::Ready(v) => { + match v { + Ok(output) => self.state = State::HasChunk(output), + Err(e) => return std::task::Poll::Ready(Err(e)), + }; + self.poll_read(cx, buf) + } + std::task::Poll::Pending => std::task::Poll::Pending, + }, + } + } +} + +impl AsyncSeek for RangedStreamer { + fn poll_seek( + mut self: std::pin::Pin<&mut Self>, + _: &mut std::task::Context<'_>, + pos: std::io::SeekFrom, + ) -> std::task::Poll> { + match pos { + SeekFrom::Start(pos) => self.pos = pos, + SeekFrom::End(pos) => self.pos = (self.length as i64 + pos) as u64, + SeekFrom::Current(pos) => self.pos = (self.pos as i64 + pos) as u64, + }; + std::task::Poll::Ready(Ok(self.pos)) + } +} + + + #[tokio::main] -async fn main() {} +async fn main() { + let url = "https://isdasoil.s3.amazonaws.com/covariates/dem_30m/dem_30m.tif"; + let Ok(url_head) = ehttp::fetch_async(ehttp::Request::head(url)).await else {println!("EPIC FAIL!"); return;}; + let length = usize::from_str_radix(url_head.headers.get("content-length").unwrap(), 10).expect("Could not parse content length"); + println!("head: {:?}", url_head); + let range_get = Box::new(move |start: u64, length: u64| { + // let bucket = bucket.clone(); + let url = url; + Box::pin(async move { + println!("requested: {} kb", length / 1024); + let mut request = ehttp::Request::get(url); + request.headers.insert("Range".to_string(), format!("bytes={:?}-{:?}",start,start+length)); + let resp = ehttp::fetch_async(request).await.map_err(|e| std::io::Error::other(e))?; + if !resp.ok { + Err(std::io::Error::other(format!("Received invalid response: {:?}", resp.status))) + } else { + println!("received: {} kb", resp.bytes.len() / 1024); + Ok(SeekOutput {start, data: resp.bytes}) + } + }) as BoxFuture<'static, std::io::Result> + }); + let reader = RangedStreamer::new(length, 30*1024, range_get); + match Decoder::new_overview_async(reader, 5).await { + Ok(mut d) => println!("{:?}", d.read_chunk_async(42).await.unwrap()), + Err(e) => println!("err: {:?}", e), + } +} diff --git a/src/decoder/mod.rs b/src/decoder/mod.rs index daabefae..695824e0 100644 --- a/src/decoder/mod.rs +++ b/src/decoder/mod.rs @@ -18,6 +18,8 @@ pub mod ifd; mod image; mod stream; mod tag_reader; + +#[cfg(feature = "async_decoder")] mod async_decoder; /// Result of a decoding process diff --git a/tests/decode_images_async.rs b/tests/decode_images_async.rs index bf16d15b..d971a25c 100644 --- a/tests/decode_images_async.rs +++ b/tests/decode_images_async.rs @@ -1,4 +1,4 @@ -// #[cfg(feature="async_decoder")] +#[cfg(feature="async_decoder")] mod test_async { extern crate tiff; From da8f183b64dad380934a27e068fdba1e44ea6218 Mon Sep 17 00:00:00 2001 From: Fee Fladder Date: Thu, 19 Sep 2024 05:43:30 +0200 Subject: [PATCH 5/6] commented out unused functions in ~EndianAsyncReader~ AsyncEndianReader --- src/decoder/async_decoder/ifd.rs | 2 +- src/decoder/async_decoder/mod.rs | 2 +- src/decoder/async_decoder/stream.rs | 16 ++++++++-------- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/src/decoder/async_decoder/ifd.rs b/src/decoder/async_decoder/ifd.rs index cd987300..654fac2e 100644 --- a/src/decoder/async_decoder/ifd.rs +++ b/src/decoder/async_decoder/ifd.rs @@ -7,7 +7,7 @@ use crate::decoder::{ use crate::{TiffResult, TiffError}; pub use crate::decoder::ifd::Entry; -use super::stream::EndianAsyncReader; +use super::stream::AsyncEndianReader; impl Entry { pub async fn async_val( diff --git a/src/decoder/async_decoder/mod.rs b/src/decoder/async_decoder/mod.rs index edc14f06..450e5263 100644 --- a/src/decoder/async_decoder/mod.rs +++ b/src/decoder/async_decoder/mod.rs @@ -21,7 +21,7 @@ use crate::decoder::{ }, ChunkType, DecodingBuffer, DecodingResult, Limits, }; -use stream::EndianAsyncReader; +use stream::AsyncEndianReader; extern crate async_trait; diff --git a/src/decoder/async_decoder/stream.rs b/src/decoder/async_decoder/stream.rs index 42abf5f6..ef0bbaaa 100644 --- a/src/decoder/async_decoder/stream.rs +++ b/src/decoder/async_decoder/stream.rs @@ -19,24 +19,24 @@ macro_rules! read_async_fn { #[async_trait::async_trait] /// Reader that is aware of the byte order. -pub trait EndianAsyncReader: AsyncRead + Unpin { +pub trait AsyncEndianReader: AsyncRead + Unpin { /// Byte order that should be adhered to fn byte_order(&self) -> ByteOrder; read_async_fn!(read_u16, u16); - read_async_fn!(read_i8, i8); - read_async_fn!(read_i16, i16); + // read_async_fn!(read_i8, i8); + // read_async_fn!(read_i16, i16); read_async_fn!(read_u32, u32); - read_async_fn!(read_i32, i32); + // read_async_fn!(read_i32, i32); read_async_fn!(read_u64, u64); - read_async_fn!(read_i64, i64); - read_async_fn!(read_f32, f32); - read_async_fn!(read_f64, f64); + // read_async_fn!(read_i64, i64); + // read_async_fn!(read_f32, f32); + // read_async_fn!(read_f64, f64); } -impl EndianAsyncReader for SmartReader { +impl AsyncEndianReader for SmartReader { #[inline(always)] fn byte_order(&self) -> ByteOrder { self.byte_order From a5c87b7ab079164923bf51bd462e7d166c7f656a Mon Sep 17 00:00:00 2001 From: Fee Fladder Date: Mon, 30 Sep 2024 21:36:04 +0200 Subject: [PATCH 6/6] feat:multithread made necessary modules public for using in mutithreaded envs, implemented ChunkReader and provided example --- Cargo.toml | 15 +- examples/multithread_http.rs | 194 +++++++++++++++++++++++++ src/decoder/async_decoder/mod.rs | 8 +- src/decoder/ifd.rs | 35 ----- src/decoder/image.rs | 16 +- src/decoder/mod.rs | 59 +++++--- src/decoder/multithread_decoder/mod.rs | 68 +++++++++ 7 files changed, 325 insertions(+), 70 deletions(-) create mode 100644 examples/multithread_http.rs create mode 100644 src/decoder/multithread_decoder/mod.rs diff --git a/Cargo.toml b/Cargo.toml index c164a6b3..e0c04d87 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -18,7 +18,9 @@ exclude = ["tests/images/*", "tests/fuzz_images/*"] [features] async_decoder = ["dep:futures", "dep:async-trait"] -ehttp = ["dep:ehttp", "async_decoder"] +multithread = ["async_decoder"] +# only for async example reading a COG +ehttp = ["async_decoder", "dep:ehttp"] [dependencies] weezl = "0.1.0" @@ -44,4 +46,13 @@ description = "Example showing use of async features using async http requests" [[example]] name = "async_http" path="examples/async_http.rs" -required-features=["ehttp"] \ No newline at end of file +required-features=["ehttp", "async_decoder"] + +[package.metadata.example.multithread] +name="multithread_http" +description = "example showing multithreaded reading of COG" + +[[example]] +name = "multithread_http" +path="examples/multithread_http.rs" +required-features=["ehttp", "multithread"] \ No newline at end of file diff --git a/examples/multithread_http.rs b/examples/multithread_http.rs new file mode 100644 index 00000000..e295b01a --- /dev/null +++ b/examples/multithread_http.rs @@ -0,0 +1,194 @@ +// Special thanks to Alice for the help: https://users.rust-lang.org/t/63019/6 +use std::io::{Result, SeekFrom}; +use std::pin::Pin; +use std::sync::Arc; +use futures::{ + future::BoxFuture, + io::{AsyncRead, AsyncSeek}, + Future, +}; +use tiff::decoder::Decoder; + +// extern crate ehttp; + +// Arc for sharing, see https://users.rust-lang.org/t/how-to-clone-a-boxed-closure/31035/9 +// or https://stackoverflow.com/a/27883612/14681457 +pub type F = Arc< + dyn Fn(u64, u64) -> BoxFuture<'static, std::io::Result> + Send + Sync, +>; +pub struct RangedStreamer { + pos: u64, + length: u64, // total size + state: State, + range_get: F, + min_request_size: usize, // requests have at least this size +} + +/// This is a fake clone, that doesn't clone the currently pending task, but everything else +impl Clone for RangedStreamer { + fn clone(&self) -> Self { + RangedStreamer { + pos: self.pos, + length: self.length, + state: State::HasChunk(SeekOutput { + start: 0, + data: vec![], + }), + range_get: self.range_get.clone(), + min_request_size: self.min_request_size, + } + } +} + +enum State { + HasChunk(SeekOutput), + Seeking(BoxFuture<'static, std::io::Result>), +} + +#[derive(Debug, Clone)] +pub struct SeekOutput { + pub start: u64, + pub data: Vec, +} + + + +impl RangedStreamer { + pub fn new(length: usize, min_request_size: usize, range_get: F) -> Self { + let length = length as u64; + Self { + pos: 0, + length, + state: State::HasChunk(SeekOutput { + start: 0, + data: vec![], + }), + range_get, + min_request_size, + } + } +} + +// whether `test_interval` is inside `a` (start, length). +fn range_includes(a: (usize, usize), test_interval: (usize, usize)) -> bool { + if test_interval.0 < a.0 { + return false; + } + let test_end = test_interval.0 + test_interval.1; + let a_end = a.0 + a.1; + if test_end > a_end { + return false; + } + true +} + +impl AsyncRead for RangedStreamer { + fn poll_read( + mut self: std::pin::Pin<&mut Self>, + cx: &mut std::task::Context<'_>, + buf: &mut [u8], + ) -> std::task::Poll> { + let requested_range = (self.pos as usize, buf.len()); + let min_request_size = self.min_request_size; + match &mut self.state { + State::HasChunk(output) => { + let existing_range = (output.start as usize, output.data.len()); + if range_includes(existing_range, requested_range) { + let offset = requested_range.0 - existing_range.0; + buf.copy_from_slice(&output.data[offset..offset + buf.len()]); + self.pos += buf.len() as u64; + std::task::Poll::Ready(Ok(buf.len())) + } else { + let start = requested_range.0 as u64; + let length = std::cmp::max(min_request_size, requested_range.1); + let future = (self.range_get)(start, length.try_into().unwrap()); + self.state = State::Seeking(Box::pin(future)); + self.poll_read(cx, buf) + } + } + State::Seeking(ref mut future) => match Pin::new(future).poll(cx) { + std::task::Poll::Ready(v) => { + match v { + Ok(output) => self.state = State::HasChunk(output), + Err(e) => return std::task::Poll::Ready(Err(e)), + }; + self.poll_read(cx, buf) + } + std::task::Poll::Pending => std::task::Poll::Pending, + }, + } + } +} + +impl AsyncSeek for RangedStreamer { + fn poll_seek( + mut self: std::pin::Pin<&mut Self>, + _: &mut std::task::Context<'_>, + pos: std::io::SeekFrom, + ) -> std::task::Poll> { + match pos { + SeekFrom::Start(pos) => self.pos = pos, + SeekFrom::End(pos) => self.pos = (self.length as i64 + pos) as u64, + SeekFrom::Current(pos) => self.pos = (self.pos as i64 + pos) as u64, + }; + std::task::Poll::Ready(Ok(self.pos)) + } +} + + + +#[tokio::main] +async fn main() { + let url = "https://isdasoil.s3.amazonaws.com/covariates/dem_30m/dem_30m.tif"; + let Ok(url_head) = ehttp::fetch_async(ehttp::Request::head(url)).await else {println!("EPIC FAIL!"); return;}; + let length = usize::from_str_radix(url_head.headers.get("content-length").unwrap(), 10).expect("Could not parse content length"); + println!("head: {:?}", url_head); + let range_get = Arc::new(move |start: u64, length: u64| { + // let bucket = bucket.clone(); + let url = url; + Box::pin(async move { + println!("requested: {} kb", length / 1024); + let mut request = ehttp::Request::get(url); + request.headers.insert("Range".to_string(), format!("bytes={:?}-{:?}",start,start+length)); + let resp = ehttp::fetch_async(request).await.map_err(|e| std::io::Error::other(e))?; + if !resp.ok { + Err(std::io::Error::other(format!("Received invalid response: {:?}", resp.status))) + } else { + println!("received: {} kb", resp.bytes.len() / 1024); + Ok(SeekOutput {start, data: resp.bytes}) + } + }) as BoxFuture<'static, std::io::Result> + }); + let reader = RangedStreamer::new(length, 30*1024, range_get); + + // this decoder will read all necessary tags + let decoder = Decoder::new_overview_async(reader, 0).await.expect("oh noes!"); + println!("initialized decoder"); + let cloneable_decoder = tiff::decoder::ChunkDecoder::from_decoder(decoder); + + let mut handles = Vec::new(); + for chunk in 42..69 { + let mut cloned_decoder = cloneable_decoder.clone(); + + let handle = tokio::spawn(async move { + + let result = cloned_decoder.read_chunk_async(chunk).await; + match result { + Ok(data) => { + println!("Successfully read chunk {}", chunk); + Ok(data) // Return the data for collection + } + Err(e) => { + eprintln!("Error reading chunk {}: {:?}", chunk, e); + Err(e) // Return the error for handling + } + } + }); + handles.push(handle); + } + + let results = futures::future::join_all(handles).await; + for r in results { + println!("result: {:?}", r.expect("idk").expect("idk²").len()) + } +} diff --git a/src/decoder/async_decoder/mod.rs b/src/decoder/async_decoder/mod.rs index 450e5263..6fcfe8e6 100644 --- a/src/decoder/async_decoder/mod.rs +++ b/src/decoder/async_decoder/mod.rs @@ -5,7 +5,7 @@ use futures::{ }; use std::collections::{HashMap, HashSet}; -use crate::{ColorType, TiffError, TiffFormatError, TiffResult, TiffUnsupportedError, UsageError}; +use crate::{TiffError, TiffFormatError, TiffResult, TiffUnsupportedError}; // use self::ifd::Directory; // use self::image::Image; @@ -18,7 +18,7 @@ use crate::decoder::{ Decoder, ifd::{Value, Directory}, Image, stream::{ ByteOrder, SmartReader, - }, ChunkType, DecodingBuffer, DecodingResult, Limits, + }, ChunkType, DecodingBuffer, DecodingResult, }; use stream::AsyncEndianReader; @@ -453,7 +453,7 @@ impl Decoder { pub async fn read_chunk_async(&mut self, chunk_index: u32) -> TiffResult { let data_dims = self.image().chunk_data_dimensions(chunk_index)?; - let mut result = self.result_buffer(data_dims.0 as usize, data_dims.1 as usize)?; + let mut result = Self::result_buffer(data_dims.0 as usize, data_dims.1 as usize, self.image(), &self.limits)?; self.read_chunk_to_buffer_async(result.as_buffer(0), chunk_index, data_dims.0 as usize) .await?; @@ -465,7 +465,7 @@ impl Decoder { pub async fn read_image_async(&mut self) -> TiffResult { let width = self.image().width; let height = self.image().height; - let mut result = self.result_buffer(width as usize, height as usize)?; + let mut result = Self::result_buffer(usize::try_from(width)?, usize::try_from(height)?, self.image(), &self.limits )?; if width == 0 || height == 0 { return Ok(result); } diff --git a/src/decoder/ifd.rs b/src/decoder/ifd.rs index 5a7a21b7..df6d9a9d 100644 --- a/src/decoder/ifd.rs +++ b/src/decoder/ifd.rs @@ -2,7 +2,6 @@ use std::collections::HashMap; use std::io::{self, Read, Seek}; -use std::mem; use std::str; use super::stream::{ByteOrder, EndianReader, SmartReader}; @@ -679,40 +678,6 @@ impl Entry { } Ok(List(v)) } - - #[inline] - fn decode_offset( - &self, - value_count: u64, - bo: ByteOrder, - bigtiff: bool, - limits: &super::Limits, - reader: &mut SmartReader, - decode_fn: F, - ) -> TiffResult - where - R: Read + Seek, - F: Fn(&mut SmartReader) -> TiffResult, - { - let value_count = usize::try_from(value_count)?; - if value_count > limits.decoding_buffer_size / mem::size_of::() { - return Err(TiffError::LimitsExceeded); - } - - let mut v = Vec::with_capacity(value_count); - - let offset = if bigtiff { - self.r(bo).read_u64()? - } else { - self.r(bo).read_u32()?.into() - }; - reader.goto_offset(offset)?; - - for _ in 0..value_count { - v.push(decode_fn(reader)?) - } - Ok(List(v)) - } } /// Extracts a list of BYTE tags stored in an offset diff --git a/src/decoder/image.rs b/src/decoder/image.rs index f22872b4..f0604c59 100644 --- a/src/decoder/image.rs +++ b/src/decoder/image.rs @@ -10,14 +10,14 @@ use crate::{ColorType, TiffError, TiffFormatError, TiffResult, TiffUnsupportedEr use std::io::{self, Cursor, Read, Seek}; use std::sync::Arc; -#[derive(Debug)] -pub(crate) struct StripDecodeState { +#[derive(Debug, Clone)] +pub struct StripDecodeState { pub rows_per_strip: u32, } -#[derive(Debug)] +#[derive(Debug, Clone)] /// Computed values useful for tile decoding -pub(crate) struct TileAttributes { +pub struct TileAttributes { pub image_width: usize, pub image_height: usize, @@ -58,8 +58,8 @@ impl TileAttributes { } } -#[derive(Debug)] -pub(crate) struct Image { +#[derive(Debug, Clone)] +pub struct Image { pub ifd: Option, pub width: u32, pub height: u32, @@ -679,7 +679,7 @@ impl Image { } } } else { - for (i, row) in buf + for (_, row) in buf .chunks_mut(output_row_stride) .take(data_dims.1 as usize) .enumerate() @@ -687,8 +687,6 @@ impl Image { let row = &mut row[..data_row_bytes]; reader.read_exact(row)?; - println!("chunk={chunk_index}, index={i}"); - // Skip horizontal padding if chunk_row_bytes > data_row_bytes { let len = u64::try_from(chunk_row_bytes - data_row_bytes)?; diff --git a/src/decoder/mod.rs b/src/decoder/mod.rs index 695824e0..6d86c518 100644 --- a/src/decoder/mod.rs +++ b/src/decoder/mod.rs @@ -15,12 +15,16 @@ use crate::tags::{ use self::stream::{ByteOrder, EndianReader, SmartReader}; pub mod ifd; -mod image; -mod stream; +pub mod image; +pub mod stream; mod tag_reader; #[cfg(feature = "async_decoder")] -mod async_decoder; +pub mod async_decoder; +#[cfg(feature = "multithread")] +mod multithread_decoder; +#[cfg(feature = "multithread")] +pub use multithread_decoder::ChunkDecoder; /// Result of a decoding process #[derive(Debug)] @@ -142,6 +146,21 @@ impl DecodingResult { DecodingResult::I64(ref mut buf) => DecodingBuffer::I64(&mut buf[start..]), } } + + pub fn len(&self) -> usize { + match self { + DecodingResult::U8(v) => v.len(), + DecodingResult::U16(v) => v.len(), + DecodingResult::U32(v) => v.len(), + DecodingResult::U64(v) => v.len(), + DecodingResult::F32(v) => v.len(), + DecodingResult::F64(v) => v.len(), + DecodingResult::I8(v) => v.len(), + DecodingResult::I16(v) => v.len(), + DecodingResult::I32(v) => v.len(), + DecodingResult::I64(v) => v.len(), + } + } } // A buffer for image decoding @@ -435,7 +454,7 @@ impl Decoder { self.image().colortype() } - fn image(&self) -> &Image { + pub fn image(&self) -> &Image { &self.image } @@ -491,38 +510,38 @@ impl Decoder { } - fn result_buffer(&self, width: usize, height: usize) -> TiffResult { + fn result_buffer(width: usize, height: usize, image: &Image, limits: &Limits) -> TiffResult { let buffer_size = match width .checked_mul(height) - .and_then(|x| x.checked_mul(self.image().samples_per_pixel())) + .and_then(|x| x.checked_mul(image.samples_per_pixel())) { Some(s) => s, None => return Err(TiffError::LimitsExceeded), }; - let max_sample_bits = self.image().bits_per_sample; - match self.image().sample_format { + let max_sample_bits = image.bits_per_sample; + match image.sample_format { SampleFormat::Uint => match max_sample_bits { - n if n <= 8 => DecodingResult::new_u8(buffer_size, &self.limits), - n if n <= 16 => DecodingResult::new_u16(buffer_size, &self.limits), - n if n <= 32 => DecodingResult::new_u32(buffer_size, &self.limits), - n if n <= 64 => DecodingResult::new_u64(buffer_size, &self.limits), + n if n <= 8 => DecodingResult::new_u8(buffer_size, &limits), + n if n <= 16 => DecodingResult::new_u16(buffer_size, &limits), + n if n <= 32 => DecodingResult::new_u32(buffer_size, &limits), + n if n <= 64 => DecodingResult::new_u64(buffer_size, &limits), n => Err(TiffError::UnsupportedError( TiffUnsupportedError::UnsupportedBitsPerChannel(n), )), }, SampleFormat::IEEEFP => match max_sample_bits { - 32 => DecodingResult::new_f32(buffer_size, &self.limits), - 64 => DecodingResult::new_f64(buffer_size, &self.limits), + 32 => DecodingResult::new_f32(buffer_size, &limits), + 64 => DecodingResult::new_f64(buffer_size, &limits), n => Err(TiffError::UnsupportedError( TiffUnsupportedError::UnsupportedBitsPerChannel(n), )), }, SampleFormat::Int => match max_sample_bits { - n if n <= 8 => DecodingResult::new_i8(buffer_size, &self.limits), - n if n <= 16 => DecodingResult::new_i16(buffer_size, &self.limits), - n if n <= 32 => DecodingResult::new_i32(buffer_size, &self.limits), - n if n <= 64 => DecodingResult::new_i64(buffer_size, &self.limits), + n if n <= 8 => DecodingResult::new_i8(buffer_size, &limits), + n if n <= 16 => DecodingResult::new_i16(buffer_size, &limits), + n if n <= 32 => DecodingResult::new_i32(buffer_size, &limits), + n if n <= 64 => DecodingResult::new_i64(buffer_size, &limits), n => Err(TiffError::UnsupportedError( TiffUnsupportedError::UnsupportedBitsPerChannel(n), )), @@ -1036,7 +1055,7 @@ impl Decoder { pub fn read_chunk(&mut self, chunk_index: u32) -> TiffResult { let data_dims = self.image().chunk_data_dimensions(chunk_index)?; - let mut result = self.result_buffer(data_dims.0 as usize, data_dims.1 as usize)?; + let mut result = Self::result_buffer(data_dims.0 as usize, data_dims.1 as usize, self.image(), &self.limits)?; self.read_chunk_to_buffer(result.as_buffer(0), chunk_index, data_dims.0 as usize)?; @@ -1047,7 +1066,7 @@ impl Decoder { pub fn read_image(&mut self) -> TiffResult { let width = self.image().width; let height = self.image().height; - let mut result = self.result_buffer(width as usize, height as usize)?; + let mut result = Self::result_buffer(width as usize, height as usize, self.image(), &self.limits)?; if width == 0 || height == 0 { return Ok(result); } diff --git a/src/decoder/multithread_decoder/mod.rs b/src/decoder/multithread_decoder/mod.rs new file mode 100644 index 00000000..ae91a60a --- /dev/null +++ b/src/decoder/multithread_decoder/mod.rs @@ -0,0 +1,68 @@ +//! Decoder that can be Cloned, sharing the [`Image`] data between threads +//! Therefore, it holds an `Arc` +//! Loading in the image meatadata should be done using another decoder +//! Also shows how terrificly easy and ergonomic the api for the folks over at geotiff would be :P +use std::sync::Arc; + +use futures::{AsyncRead, AsyncSeek}; + +use crate::decoder::{ + image::Image, + Decoder, Limits, DecodingResult, + stream::SmartReader, + async_decoder::RangeReader, +}; +use crate::TiffResult; +/// Decoder that can be Cloned, sharing the [`Image`] data between threads +#[derive(Clone, Debug)] +pub struct ChunkDecoder { + reader: SmartReader, + // bigtiff: bool, + limits: Limits, + image: Arc, +} + +impl Clone for SmartReader { + fn clone(&self) -> Self { + Self { + reader: self.reader.clone(), + byte_order: self.byte_order, + } + } +} + +impl ChunkDecoder{ + pub fn from_decoder(decoder: Decoder) -> Self { + ChunkDecoder { + reader: decoder.reader.clone(), + // bigtiff: decoder.bigtiff, + limits: decoder.limits.clone(), + image: Arc::new(decoder.image().clone()), + } + } + + /// Get a reference to the image (in read mode) + pub fn image(&self) -> &Image { + // this is really bad + &self.image//.read().expect("Could not obtain lock") + } + + pub async fn read_chunk_async(&mut self, chunk_index: u32) -> TiffResult{ + // read_chunk code + let (width, height) = self.image().chunk_data_dimensions(chunk_index)?; + let mut result = Decoder::::result_buffer(usize::try_from(width)?, usize::try_from(height)?, self.image(), &self.limits)?; + // read_chunk_to_buffer code + let (offset, length) = self.image().chunk_file_range(chunk_index)?; + let v = self.reader.read_range(offset, offset + length).await?; + let output_row_stride = (width as u64).saturating_mul(self.image().samples_per_pixel() as u64).saturating_mul(self.image.bits_per_sample as u64) / 8; + self.image().expand_chunk( + &mut std::io::Cursor::new(v), + result.as_buffer(0).as_bytes_mut(), + output_row_stride.try_into()?, + self.reader.byte_order, + chunk_index, + &self.limits, + )?; + Ok(result) + } +}