More work

Author: friendlymatthew

src/classifier.rs

@@ -84,13 +84,13 @@ impl<'a> CsvClassifier<'a> {
 
         let mut temp = [0u8; 16];
         temp[..slice.len()].copy_from_slice(slice);
-      
+
         let last = self.data[self.data.len() - 1];
-      
+
         if last != 0x0A && last != 0x0D {
-            temp[slice.len()] = 0x0A; 
+            temp[slice.len()] = 0x0A;
         }
-       
+
         (u8x16::from_slice_unchecked(&temp), false)
     }
 }

src/main.rs

@@ -5,12 +5,11 @@ fn main() -> anyhow::Result<()> {
     let mut args = std::env::args().skip(1);
 
     let data = args.next().ok_or_else(|| anyhow!("No argument passed"))?;
-   
-    
+
     let mut reader = CsvReader::new(data.as_bytes());
     let rows = reader.read()?;
-   
+
     println!("{:?}", rows);
 
     Ok(())
-}
+}

src/reader.rs

@@ -45,10 +45,10 @@ impl CsvReader {
 
         let mut current_row = Vec::new();
 
-        let cursor = 0;
+        let mut cursor = 0;
 
         for i in 0..self.quotation_bitsets.len() {
-            let valid_quotations  = remove_escaped_quotations(self.quotation_bitsets[i]);
+            let valid_quotations = remove_escaped_quotations(self.quotation_bitsets[i]);
             let outside_quotations = !mark_inside_quotations(valid_quotations);
 
             let mut valid_commas = self.comma_bitsets[i] & outside_quotations;
@@ -70,12 +70,10 @@ impl CsvReader {
                     break;
                 }
 
-                current_row.push(
-                    Range {
-                        start: cursor + bitset_cursor,
-                        end: cursor + bitset_cursor + bits_traveled
-                    }
-                );
+                current_row.push(Range {
+                    start: cursor + bitset_cursor,
+                    end: cursor + bitset_cursor + bits_traveled,
+                });
 
                 if first_whitespace < first_comma {
                     rows.push(Row::from(current_row.clone()));
@@ -86,15 +84,17 @@ impl CsvReader {
                 valid_commas <<= bits_traveled + 1;
                 valid_whitespace <<= bits_traveled + 1;
             }
+
+            cursor += bitset_cursor;
         }
-        
+
         Ok(rows)
     }
 }
 
 fn remove_escaped_quotations(q: u64) -> u64 {
     let escaped = q & (q << 1);
-    let escaped = escaped | (escaped>> 1);
+    let escaped = escaped | (escaped >> 1);
 
     q & !escaped
 }
@@ -116,21 +116,50 @@ fn mark_inside_quotations(mut x: u64) -> u64 {
 
 // todo, find a quicker way to do this
 #[inline]
-fn build_u64(chunk: &[u8x16], broadcast: u8x16) -> u64 {
+fn build_u64(chunks: &[u8x16], broadcast: u8x16) -> u64 {
     let mut packed: u64 = 0;
-    for (i, &c) in chunk.iter().enumerate() {
-        let word = c.eq(broadcast).bitset() as u64;
+    for (i, &chunk) in chunks.iter().enumerate() {
+        let word = chunk.eq(broadcast).bitset() as u64;
         packed |= word << (48 - i * 16);
     }
-
     packed
 }
 
-
 #[cfg(test)]
 mod tests {
     use super::*;
 
+    #[test]
+    fn test_init() -> Result<()> {
+        let data = std::fs::read("taxi_zone_lookup.csv")?;
+
+        // todo fix alignment
+        // let csv_reader1 = CsvReader::new(&data[..64]);
+
+        let slice63 = &data[..63];
+
+        let csv_reader1 = CsvReader::new(&slice63);
+
+        assert_eq!(csv_reader1.quotation_bitsets.len(), 1);
+        assert_eq!(csv_reader1.whitespace_bitsets.len(), 1);
+        assert_eq!(csv_reader1.comma_bitsets.len(), 1);
+
+        let mut longer_slice = vec![];
+        longer_slice.extend_from_slice(slice63);
+        longer_slice.push(0x0A);
+        longer_slice.extend_from_slice(&data[63..87]);
+
+        let csv_reader2 = CsvReader::new(&longer_slice);
+        assert_eq!(csv_reader2.quotation_bitsets.len(), 2);
+        assert_eq!(csv_reader2.whitespace_bitsets.len(), 2);
+        assert_eq!(csv_reader2.comma_bitsets.len(), 2);
+
+        assert_eq!(csv_reader1.quotation_bitsets[0], csv_reader2.quotation_bitsets[0]);
+        assert_eq!(csv_reader1.whitespace_bitsets[0], csv_reader2.whitespace_bitsets[0]);
+        assert_eq!(csv_reader1.comma_bitsets[0], csv_reader2.comma_bitsets[0]);
+
+        Ok(())
+    }
     fn check_line(test: &[u8], expected: Vec<Vec<String>>) -> Result<()> {
         let mut reader = CsvReader::new(test);
         let rows = reader
@@ -232,7 +261,7 @@ mod tests {
     #[test]
     fn read_taxi_zone_lookup_header() -> Result<()> {
         let data = b"\"LocationID\",\"Borough\",\"Zone\",\"service_zone\"\n";
-    
+
         check_line(
             data,
             vec![vec![
@@ -242,31 +271,29 @@ mod tests {
                 r#""service_zone""#.to_string(),
             ]],
         )?;
-    
+
         Ok(())
     }
 
     // #[test]
     // fn read_taxi_zone_lookup() -> Result<()> {
     //     let data = std::fs::read("taxi_zone_lookup.csv")?;
-    //     assert_eq!(
-    //         b"\"LocationID\",\"Borough\",\"Zone\",\"service_zone\"\r\n",
-    //         &data[..46]
-    //     );
-    // 
+    //
+    //     // dbg!(String::from_utf8(data[..66].to_vec()).unwrap());
+    //
     //     let rows = CsvReader::new(&data).read()?;
     //     dbg!(&rows);
-    // 
+    //
     //     for row in CsvReader::new(&data).read()? {
     //         let fields = row
     //             .fields()
     //             .into_iter()
     //             .map(|field_range| String::from_utf8(data[field_range.clone()].to_vec()).unwrap())
     //             .collect::<Vec<_>>();
-    // 
+    //
     //         dbg!(fields);
     //     }
-    // 
+    //
     //     Ok(())
     // }