Init commit

Author: friendlymatthew

.github/workflows/tests.yml

@@ -0,0 +1,52 @@
+name: CI
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+
+jobs:
+  tests:
+    runs-on: ubuntu-latest-ARM64
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - uses: dtolnay/rust-toolchain@master
+        with:
+          toolchain: "nightly"
+
+      - name: version info
+        run: rustc --version; cargo --version;
+
+      - name: Run all tests
+        run: cargo test
+
+  fmt:
+    runs-on: ubuntu-latest-ARM64
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - uses: dtolnay/rust-toolchain@master
+        with:
+          toolchain: nightly
+
+      - run: |
+          rustup component add rustfmt
+          cargo fmt --all -- --check
+
+  clippy:
+    runs-on: ubuntu-latest-ARM64
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - uses: dtolnay/rust-toolchain@master
+        with:
+          toolchain: "nightly"
+
+      - run: |
+          rustup component add clippy
+          cargo clippy -- -D warnings
+

.gitignore

@@ -0,0 +1 @@
+/target

Cargo.lock

@@ -0,0 +1,16 @@
+[package]
+name = "simdcsv"
+authors = ["Matthew Kim"]
+version = "0.1.0"
+edition = "2024"
+description = "A CSV parser"
+license = "MIT"
+
+[dependencies]
+anyhow = "1.0.98"
+
+
+[profile.release]
+panic = "abort"
+lto = true
+codegen-units = 1

Cargo.toml

@@ -0,0 +1,10 @@
+# csv
+
+There are many kinds of CSV files; this project supports the format described
+in [RFC 4180](https://www.rfc-editor.org/rfc/rfc4180.html).
+
+## Reading
+
+https://www.rfc-editor.org/rfc/rfc4180.html<br>
+https://arxiv.org/pdf/1902.08318<br>
+https://branchfree.org/2019/03/06/code-fragment-finding-quote-pairs-with-carry-less-multiply-pclmulqdq/<br>

README.md

@@ -0,0 +1,132 @@
+pub const COMMA_CLASS: u8 = 1;
+pub const NEW_LINE_CLASS: u8 = 2;
+
+pub const QUOTATION_CLASS: u8 = 3;
+
+pub const LO_LOOKUP: [u8; 16] = [
+    0,
+    0,
+    QUOTATION_CLASS,
+    0,
+    0,
+    0,
+    0,
+    0,
+    0,
+    0,
+    NEW_LINE_CLASS,
+    0,
+    COMMA_CLASS,
+    NEW_LINE_CLASS,
+    0,
+    0,
+];
+pub const HI_LOOKUP: [u8; 16] = [
+    NEW_LINE_CLASS,
+    0,
+    COMMA_CLASS | QUOTATION_CLASS,
+    0,
+    0,
+    0,
+    0,
+    0,
+    0,
+    0,
+    0,
+    0,
+    0,
+    0,
+    0,
+    0,
+];
+
+use crate::u8x16::u8x16;
+
+#[derive(Debug)]
+pub struct CsvClassifier<'a> {
+    cursor: usize,
+    data: &'a [u8],
+}
+
+impl<'a> CsvClassifier<'a> {
+    pub const fn new(data: &'a [u8]) -> Self {
+        Self { cursor: 0, data }
+    }
+
+    pub fn classify(&mut self) -> Vec<u8x16> {
+        let mut bitsets = Vec::new();
+
+        let high_nibble_lookup = u8x16::from_slice_unchecked(&HI_LOOKUP);
+        let low_nibble_lookup = u8x16::from_slice_unchecked(&LO_LOOKUP);
+
+        while self.cursor < self.data.len() {
+            let (lanes, _aligned) = self.load_u8x16();
+
+            let (hi_lanes, lo_lanes) = lanes.nibbles();
+            let res = high_nibble_lookup.classify(hi_lanes) & low_nibble_lookup.classify(lo_lanes);
+
+            bitsets.push(res);
+        }
+
+        bitsets
+    }
+
+    fn load_u8x16(&mut self) -> (u8x16, bool) {
+        if self.cursor + u8x16::LANE_COUNT < self.data.len() {
+            let slice = &self.data[self.cursor..self.cursor + u8x16::LANE_COUNT];
+            self.cursor += u8x16::LANE_COUNT;
+
+            return (u8x16::from_slice_unchecked(slice), true);
+        }
+
+        let slice = &self.data[self.cursor..];
+        self.cursor = self.data.len();
+
+        let mut temp = [0u8; 16];
+        temp[..slice.len()].copy_from_slice(slice);
+
+        let last = self.data[self.data.len() - 1];
+
+        if last != 0x0A && last != 0x0D {
+            temp[slice.len()] = 0x0A;
+        }
+
+        (u8x16::from_slice_unchecked(&temp), false)
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_basic_classify() {
+        let mut classifier = CsvClassifier::new(b"a,b,c\nf,\"g\"");
+        let bitsets = classifier.classify();
+
+        assert_eq!(bitsets.len(), 1);
+        let res: [u8; 16] = bitsets[0].into();
+
+        assert_eq!(
+            res,
+            [
+                0,
+                COMMA_CLASS,
+                0,
+                COMMA_CLASS,
+                0,
+                NEW_LINE_CLASS,
+                0,
+                COMMA_CLASS,
+                QUOTATION_CLASS,
+                0,
+                QUOTATION_CLASS,
+                NEW_LINE_CLASS, // we always add a \n at the end
+                0,
+                0,
+                0,
+                0,
+            ]
+        );
+    }
+}

src/classifier.rs

@@ -0,0 +1,5 @@
+#![warn(clippy::nursery)]
+
+mod classifier;
+pub mod reader;
+mod u8x16;

src/lib.rs

@@ -0,0 +1,15 @@
+use anyhow::anyhow;
+use simdcsv::reader::CsvReader;
+
+fn main() -> anyhow::Result<()> {
+    let mut args = std::env::args().skip(1);
+
+    let data = args.next().ok_or_else(|| anyhow!("No argument passed"))?;
+
+    let mut reader = CsvReader::new(data.as_bytes());
+    let rows = reader.read();
+
+    println!("{:?}", rows);
+
+    Ok(())
+}