python polars 0.14.19

Author: ritchie46

examples/read_csv/benches/streaming.rs

@@ -0,0 +1,26 @@
+use std::fs::File;
+
+use criterion::{criterion_group, criterion_main, Criterion};
+use polars::prelude::*;
+
+fn csv_parsing_benchmark(c: &mut Criterion) {
+    c.bench_function("stream_csv", |b| {
+        b.iter(|| {
+            LazyFrame::scan_parquet(
+                "/home/ritchie46/Downloads/csv-benchmark/yellow_tripdata_2010-01.parquet",
+                Default::default(),
+            )
+            .unwrap()
+            .groupby([col("rate_code")])
+            .agg([
+                col("rate_code").sum(),
+                col("rate_code").first().alias("first"),
+            ])
+            .with_streaming(true)
+            .collect()
+        })
+    });
+}
+
+criterion_group!(benches, csv_parsing_benchmark);
+criterion_main!(benches);

examples/read_csv/src/main.rs

@@ -1,19 +1,18 @@
+use polars::io::mmap::MmapBytesReader;
 use polars::prelude::*;
 
 fn main() -> PolarsResult<()> {
-    let mut df = LazyCsvReader::new("../datasets/foods1.csv")
-        .finish()?
-        .select([
-            // select all columns
-            all(),
-            // and do some aggregations
-            cols(["fats_g", "sugars_g"]).sum().suffix("_summed"),
-        ])
-        .collect()?;
+    let file = std::fs::File::open("/home/ritchie46/Downloads/tpch/tables_scale_100/lineitem.tbl")
+        .unwrap();
+    let file = Box::new(file) as Box<dyn MmapBytesReader>;
+    let df = CsvReader::new(file)
+        .with_delimiter(b'|')
+        .has_header(false)
+        .with_chunk_size(10)
+        .batched(None)
+        .unwrap();
 
-    dbg!(&df);
-
-    write_other_formats(&mut df)?;
+    // write_other_formats(&mut df)?;
     Ok(())
 }
 

py-polars/Cargo.lock

@@ -1,6 +1,6 @@
 [package]
 name = "py-polars"
-version = "0.14.18"
+version = "0.14.19"
 authors = ["ritchie46 <[email protected]>"]
 documentation = "https://pola-rs.github.io/polars/py-polars/html/reference/index.html"
 edition = "2021"