Update

Author: friendlymatthew

Cargo.lock

@@ -1,7 +1,16 @@
 [package]
-name = "bitcsv"
+name = "simdcsv"
+authors = ["Matthew Kim"]
 version = "0.1.0"
 edition = "2024"
+description = "A CSV parser"
+license = "MIT"
 
 [dependencies]
 anyhow = "1.0.98"
+
+
+[profile.release]
+panic = "abort"
+lto = true
+codegen-units = 1

Cargo.toml

@@ -4,6 +4,8 @@ use crate::grammar::Row;
 use crate::u8x16::u8x16;
 use std::ops::Range;
 
+/// [`CsvReader`] holds 3 bits per character in the data set.
+/// To understand csv, you only need to know whether a byte is a quotation, comma, new line delimiter, or something else.
 #[derive(Debug)]
 pub struct CsvReader {
     quotation_bitsets: Vec<u64>,
@@ -13,8 +15,10 @@ pub struct CsvReader {
 
 impl CsvReader {
     pub fn new(data: &[u8]) -> Self {
+        // todo: can you store non-utf8 encoded characters in csv?
+
         let vectors = CsvClassifier::new(data).classify();
-        let capacity = vectors.len() / 4 + 1;
+        let capacity = vectors.len() / 4 + (vectors.len() % 4 != 0) as usize;
 
         let comma_broadcast = u8x16::broadcast(COMMA_CLASS);
         let new_line_broadcast = u8x16::broadcast(NEW_LINE_CLASS);
@@ -251,34 +255,34 @@ mod tests {
         );
     }
 
-    #[test]
-    fn read_taxi_zone_lookup() {
-        let data = r#"
-"LocationID","Borough","Zone","service_zone"
-1,"EWR","Newark Airport","EWR"
-2,"Queens","Jamaica Bay","Boro Zone"
-3,"Bronx","Allerton/Pelham Gardens","Boro Zone""#;
-
-        let mut csv = Vec::new();
-
-        for row in CsvReader::new(data.as_bytes()).read() {
-            let fields = row
-                .fields()
-                .iter()
-                .map(|field_range| {
-                    String::from_utf8(data[field_range.clone()].as_bytes().to_vec()).unwrap()
-                })
-                .collect::<Vec<_>>();
-
-            csv.push(fields);
-        }
-
-        println!("Statistics\ntotal rows: {}", csv.len());
-
-        for (i, row) in csv.iter().enumerate() {
-            println!("row {}\t{}\n", i, row.join("\t"));
-        }
-    }
+    //     #[test]
+    //     fn read_taxi_zone_lookup() {
+    //         let data = r#"
+    // "LocationID","Borough","Zone","service_zone"
+    // 1,"EWR","Newark Airport","EWR"
+    // 2,"Queens","Jamaica Bay","Boro Zone"
+    // 3,"Bronx","Allerton/Pelham Gardens","Boro Zone""#;
+
+    //         let mut csv = Vec::new();
+
+    //         for row in CsvReader::new(data.as_bytes()).read() {
+    //             let fields = row
+    //                 .fields()
+    //                 .iter()
+    //                 .map(|field_range| {
+    //                     String::from_utf8(data[field_range.clone()].as_bytes().to_vec()).unwrap()
+    //                 })
+    //                 .collect::<Vec<_>>();
+
+    //             csv.push(fields);
+    //         }
+
+    //         println!("Statistics\ntotal rows: {}", csv.len());
+
+    //         for (i, row) in csv.iter().enumerate() {
+    //             println!("row {}\t{}\n", i, row.join("\t"));
+    //         }
+    //     }
 
     #[test]
     fn test_mark_inside_quotations() {