Bug fixes and polishing

Author: lschuetze

.gitignore

@@ -5,5 +5,4 @@
 *tar.gz
 *.clang_complete
 *.root
-
 .idea/

malice/src/event.rs

@@ -4,10 +4,10 @@ use std::fmt::Debug;
 use failure::Error;
 use futures::prelude::*;
 use itertools::izip;
-use nom::{combinator::map, number::complete::*, sequence::tuple, IResult};
+use nom::{combinator::map, number::complete::*, sequence::tuple};
 use wasm_bindgen::prelude::*;
 
-use root_io::core::parsers::{parse_custom_mantissa, parse_tobjarray_of_tnameds, RootError};
+use root_io::core::parsers::{Span, RResult, parse_custom_mantissa, parse_tobjarray_of_tnameds, RootError};
 use root_io::stream_zip;
 use root_io::tree_reader::Tree;
 
@@ -241,9 +241,9 @@ fn string_to_mask(s: &str, run_number: i32) -> TriggerMask {
     }
 }
 
-fn parse_pid_probabilities<'s, E>(input: &'s [u8]) -> IResult<&'s [u8], PidProbabilities, E>
+fn parse_pid_probabilities<'s, E>(input: Span<'s>) -> RResult<'s, PidProbabilities, E>
 where
-    E: RootError<&'s [u8]>,
+    E: RootError<Span<'s>>,
 {
     let (input, electron) = parse_custom_mantissa(input, 8)?;
     let (input, muon) = parse_custom_mantissa(input, 8)?;

root-io/Cargo.toml

@@ -24,7 +24,9 @@ uuid = "0.8.2"
 lz4-compress = "0.1.1"
 nom = "7"
 nom-supreme = "0.6"
+nom_locate = "4.0.0"
 thiserror = "1"
+ouroboros = "0.14"
 
 [target.'cfg(target_arch = "wasm32")'.dependencies]
 wasm-bindgen-futures = "0.4"

root-io/src/core/data_source.rs

@@ -9,7 +9,7 @@ use std::io::{Read, Seek, SeekFrom};
 use std::path::Path;
 use std::path::PathBuf;
 
-use crate::tree_reader::ReadError;
+use crate::core::ReadError;
 
 /// The source from where the Root file is read. Construct it using
 /// `.into()` on a `Url` or `Path`. The latter is not availible for

root-io/src/core/file.rs

@@ -1,5 +1,4 @@
 use nom::{self,
-          IResult,
           number::complete::{be_i16, be_i32, be_u128, be_u16, be_u32, be_u64, be_u8}, Parser};
 use nom::sequence::tuple;
 use nom_supreme::{ParserExt, tag::complete::tag};
@@ -13,7 +12,7 @@ use crate::{
     core::tstreamer::streamers,
     MAP_OFFSET,
 };
-use crate::tree_reader::{ReadError, WriteError};
+use crate::core::{ReadError, WriteError};
 
 /// Size of serialized `FileHeader` in bytes
 const FILE_HEADER_SIZE: u64 = 75;
@@ -60,57 +59,60 @@ pub struct Directory {
 }
 
 /// Parse opening part of a root file
-fn file_header<'s, E: RootError<&'s [u8]>>(i: &'s [u8]) -> IResult<&'s [u8], FileHeader, E> {
-    fn version_dep_int<'s, E: RootError<&'s [u8]>>(i: &'s [u8], is_64_bit: bool) -> IResult<&'s [u8], u64, E> {
-        if is_64_bit {
-            be_u64(i)
-        } else {
-            let (i, end) = be_u32(i)?;
-            Ok((i, end as u64))
+fn file_header<'s, E: RootError<Span<'s>>>(i: Span<'s>) -> RResult<'s, FileHeader, E> {
+    let parser = |i| {
+        fn version_dep_int<'s, E: RootError<Span<'s>>>(i: Span<'s>, is_64_bit: bool) -> RResult<'s, u64, E> {
+            if is_64_bit {
+                be_u64(i)
+            } else {
+                let (i, end) = be_u32(i)?;
+                Ok((i, end as u64))
+            }
         }
-    }
-    let (i, _) = tag("root")(i)?;
-    let (i, version) = be_i32(i)?;
-    let is_64_bit = version > 1000000;
-    let (i, begin) = be_i32(i)?;
-    let (i, end) = version_dep_int(i, is_64_bit)?;
-    let (i, seek_free) = version_dep_int(i, is_64_bit)?;
-    let (i, nbytes_free) = be_i32(i)?;
-    let (i, n_entries_free) = be_i32(i)?;
-    let (i, n_bytes_name) = be_i32(i)?;
-    let (i, pointer_size) = be_u8(i)?;
-    let (i, compression) = be_i32(i)?;
-    let (i, seek_info) = version_dep_int(i, is_64_bit)?;
-    let (i, nbytes_info) = be_i32(i)?;
-    let (i, _uuid_version) = be_u16(i)?;
-    let (i, uuid) = be_u128(i)?;
-
-    let uuid = Uuid::from_u128(uuid);
-    let seek_dir = (begin + n_bytes_name) as u64;
-    Ok((
-        i,
-        FileHeader {
-            version,
-            begin,
-            end,
-            seek_free,
-            nbytes_free,
-            n_entries_free,
-            n_bytes_name,
-            pointer_size,
-            compression,
-            seek_info,
-            nbytes_info,
-            uuid,
-            seek_dir,
-        },
-    ))
+        let (i, _) = tag("root")(i)?;
+        let (i, version) = be_i32(i)?;
+        let is_64_bit = version > 1000000;
+        let (i, begin) = be_i32(i)?;
+        let (i, end) = version_dep_int(i, is_64_bit)?;
+        let (i, seek_free) = version_dep_int(i, is_64_bit)?;
+        let (i, nbytes_free) = be_i32(i)?;
+        let (i, n_entries_free) = be_i32(i)?;
+        let (i, n_bytes_name) = be_i32(i)?;
+        let (i, pointer_size) = be_u8(i)?;
+        let (i, compression) = be_i32(i)?;
+        let (i, seek_info) = version_dep_int(i, is_64_bit)?;
+        let (i, nbytes_info) = be_i32(i)?;
+        let (i, _uuid_version) = be_u16(i)?;
+        let (i, uuid) = be_u128(i)?;
+
+        let uuid = Uuid::from_u128(uuid);
+        let seek_dir = (begin + n_bytes_name) as u64;
+        Ok((
+            i,
+            FileHeader {
+                version,
+                begin,
+                end,
+                seek_free,
+                nbytes_free,
+                n_entries_free,
+                n_bytes_name,
+                pointer_size,
+                compression,
+                seek_info,
+                nbytes_info,
+                uuid,
+                seek_dir,
+            },
+        ))
+    };
+    parser.context("file header").parse(i)
 }
 
 /// Parse a file-pointer based on the version of the file
-fn versioned_pointer<'s, E>(version: i16) -> impl nom::Parser<&'s [u8], u64, E>
+fn versioned_pointer<'s, E>(version: i16) -> impl RParser<'s, u64, E>
     where
-        E: RootError<&'s [u8]>
+        E: RootError<Span<'s>>
 {
     move |i| {
         if version > 1000 {
@@ -122,9 +124,9 @@ fn versioned_pointer<'s, E>(version: i16) -> impl nom::Parser<&'s [u8], u64, E>
 }
 
 /// Directory within a root file; exists on ever file
-fn directory<'s, E>(input: &'s [u8]) -> IResult<&'s [u8], Directory, E>
+fn directory<'s, E>(input: Span<'s>) -> RResult<'s, Directory, E>
     where
-        E: RootError<&'s [u8]>
+        E: RootError<Span<'s>>
 {
     tuple((
         be_i16.context("directory version"),
@@ -158,7 +160,7 @@ impl RootFile {
     pub async fn new<S: Into<Source>>(source: S) -> Result<Self, ReadError> {
         let source = source.into();
         let hdr_buf = source.fetch(0, FILE_HEADER_SIZE).await?;
-        let hdr = wrap_parser(file_header.all_consuming())(&hdr_buf)?;
+        let hdr = wrap_parser(file_header.context("file header"))(&hdr_buf)?;
         //let hdr = _hdr?;
 
         // Jump to the TDirectory and parse it
@@ -179,13 +181,11 @@ impl RootFile {
     }
 
     pub async fn get_streamer_context(&self) -> Result<Context, ReadError> {
-        let seek_info_len = (self.hdr.nbytes_info + 4) as u64;
-        let info_key_buf = self
-            .source
+        let seek_info_len = (self.hdr.nbytes_info) as u64;
+        let info_key_buf = self.source
             .fetch(self.hdr.seek_info, seek_info_len)
             .await?;
-        let info_key = wrap_parser(tkey.all_consuming())(&info_key_buf)?;
-
+        let info_key = wrap_parser(tkey.all_consuming().context("streamer info key"))(&info_key_buf)?;
         let key_len = info_key.hdr.key_len;
         Ok(Context {
             source: self.source.clone(),
@@ -202,8 +202,7 @@ impl RootFile {
     /// Get the stream info of this file
     pub async fn streamer_infos(&self) -> Result<Vec<TStreamerInfo>, ReadError> {
         let ctx = self.get_streamer_context().await?;
-        let buf = ctx.s.as_slice();
-        let res = wrap_parser(streamers(&ctx))(buf)?;
+        let res = wrap_parser_ctx(streamers)(&ctx)?;
         Ok(res)
     }
 
@@ -286,7 +285,12 @@ mod test {
         let sources: Vec<Source> = vec![local, remote];
         for source in &sources {
             let hdr_buf = source.fetch(0, FILE_HEADER_SIZE).await?;
-            let hdr = wrap_parser(file_header.all_consuming())(&hdr_buf)?;
+            let hdr = match wrap_parser(file_header)(&hdr_buf) {
+                Ok(hdr) => hdr,
+                Err(e) => {
+                    println!("{}", e);
+                    return Err(ReadError::ParseError(e)); }
+            };
 
             let should = FileHeader {
                 version: 60600,
@@ -315,10 +319,20 @@ mod test {
         let sources: Vec<Source> = vec![local, remote];
         for source in &sources {
             let hdr_buf = source.fetch(0, FILE_HEADER_SIZE).await?;
-            let hdr = wrap_parser(file_header.all_consuming())(&hdr_buf)?;
+            let hdr = match wrap_parser(file_header)(&hdr_buf) {
+                Ok(hdr) => hdr,
+                Err(e) => {
+                    println!("{}", e);
+                    return Err(ReadError::ParseError(e)); }
+            };
 
             let dir_buf = source.fetch(hdr.seek_dir, TDIRECTORY_MAX_SIZE).await?;
-            let dir = wrap_parser(directory)(&dir_buf)?;
+            let dir = match wrap_parser(directory)(&dir_buf) {
+                Ok(dir) => dir,
+                Err(e) => {
+                    println!("{}", e);
+                    return Err(ReadError::ParseError(e)); }
+            };
 
             assert_eq!(
                 dir,
@@ -348,7 +362,12 @@ mod test {
         let sources: Vec<Source> = vec![local, remote];
         for source in &sources {
             let key_buf = source.fetch(1117, 4446).await?;
-            let key = wrap_parser(tkey.all_consuming())(&key_buf)?;
+            let key = match wrap_parser(tkey)(&key_buf) {
+                Ok(key) => key,
+                Err(e) => {
+                    println!("{}", e);
+                    return Err(ReadError::ParseError(e)); }
+            };
 
             assert_eq!(key.hdr.obj_name, "StreamerInfo");
 
@@ -360,9 +379,17 @@ mod test {
                 s: key.obj,
             };
 
-            let l = wrap_parser(length_value(checked_byte_count, |i| {
-                tlist::<VerboseError<_>>(&context).parse(i)
-            }).all_consuming())(&context.s)?;
+            let mut parser = wrap_parser_ctx(|ctx| {
+                length_value(checked_byte_count, move |i| {
+                    tlist::<VerboseError<_>>(&ctx).parse(i)
+                }).all_consuming()
+            });
+            let l = match parser(&context) {
+                Ok(l) => l,
+                Err(e) => {
+                    println!("{}", e);
+                    return Err(ReadError::ParseError(e)); }
+            };
 
             assert_eq!(l.len(), 19);
         }

root-io/src/core/file_item.rs

@@ -3,7 +3,8 @@ use nom_supreme::ParserExt;
 
 use crate::core::{checked_byte_count, Context, Source, TKeyHeader, wrap_parser};
 use crate::core::compression::decompress;
-use crate::tree_reader::{ReadError, Tree, ttree};
+use crate::core::ReadError;
+use crate::tree_reader::{Tree, ttree};
 
 /// Describes a single item within this file (e.g. a `Tree`)
 #[derive(Debug)]
@@ -61,8 +62,10 @@ impl FileItem {
         let ctx = self.get_context().await?;
         let buf = ctx.s.as_slice();
 
-        let res = wrap_parser(
-            length_value(checked_byte_count, ttree(&ctx)).all_consuming()
+        let res = wrap_parser(length_value(checked_byte_count, ttree(&ctx))
+            .complete()
+            .all_consuming()
+            .context("ttree wrapper")
         )(buf)?;
         Ok(res)
     }
@@ -73,15 +76,15 @@ mod tests {
     use std::path::Path;
 
     use crate::core::RootFile;
+    use crate::core::UnwrapPrint;
 
     #[tokio::test]
     async fn open_simple() {
         let path = Path::new("./src/test_data/simple.root");
-        let f = RootFile::new(path).await.expect("Failed to open file");
+        let f = RootFile::new(path).await.unwrap_print();
         assert_eq!(f.items().len(), 1);
         assert_eq!(f.items()[0].tkey_hdr.obj_name, "tree");
-        // Only streamers; not rules
-        assert_eq!(f.streamer_infos().await.unwrap().len(), 18);
+        assert_eq!(f.streamer_infos().await.unwrap_print().len(), 18);
     }
 
     // Skip this test on MacOs since the downloaded file is not working on Travis
@@ -91,13 +94,11 @@ mod tests {
         use alice_open_data;
         let path = alice_open_data::test_file().unwrap();
 
-        let f = RootFile::new(path.as_path())
-            .await
-            .expect("Failed to open file");
+        let f = RootFile::new(path.as_path()).await.unwrap_print();
 
         assert_eq!(f.items().len(), 2);
         assert_eq!(f.items()[0].tkey_hdr.obj_name, "esdTree");
         assert_eq!(f.items()[1].tkey_hdr.obj_name, "HLTesdTree");
-        assert_eq!(f.streamer_infos().await.unwrap().len(), 87);
+        assert_eq!(f.streamer_infos().await.unwrap_print().len(), 87);
     }
 }