Switch to serde and clean warnings

.cargo/config

@@ -0,0 +1,5 @@
+[build]
+rustflags = ["-C", "link-arg=-undefined", "-C", "link-arg=dynamic_lookup"]
+
+[env]
+CXXFLAGS = "-std=c++11 -Wno-unused-parameter"

Cargo.toml

@@ -16,10 +16,11 @@ crate-type = ["lib"]
 
 
 [dependencies]
-rustc-serialize = "0.3.24"
+serde = { version = "1.0", features = ["derive"] }
+serde_json = "1.0"
 stemmer = "0.3.2"
-unicode-normalization = "0.1.19"
-unicode-segmentation = "1.8.0"
+unicode-normalization = "0.1.25"
+unicode-segmentation = "1.12.0"
 noise_search_deps_rocksdb = "0.1.1"
 varint = "0.9.0"
-uuid = { version = "1.1.2", features = ["v4"] }
+uuid = { version = "1.18.1", features = ["v4"] }

src/bin/debug_bbox.rs

@@ -0,0 +1,74 @@
+use noise_search::index::{Batch, Index, OpenOptions};
+extern crate rocksdb;
+use std::convert::TryInto;
+
+fn main() {
+    // Clean start
+    let dbname = "target/tests/debug_bbox_db";
+    let _ = Index::drop(dbname);
+
+    // Create index
+    let mut index = Index::open(dbname, Some(OpenOptions::Create)).unwrap();
+    let mut batch = Batch::new();
+
+    // Add a point
+    println!("Adding point at [10.9, 48.4]");
+    index.add(r#"{"_id": "point1", "geometry": {"type": "Point", "coordinates": [10.9, 48.4]}}"#, &mut batch).unwrap();
+    index.flush(batch).unwrap();
+
+    println!("\nChecking what's stored in rtree:");
+    let rtree_cf = index.rocks.cf_handle("rtree").unwrap();
+    let iter = index.rocks.iterator_cf(rtree_cf, rocksdb::IteratorMode::Start).unwrap();
+
+    for (key, _value) in iter {
+        println!("Key length: {}", key.len());
+
+        // Parse the key
+        let mut offset = 0;
+
+        // Read keypath length (varint)
+        let keypath_len = key[offset] as usize;
+        offset += 1;
+
+        // Read keypath
+        let keypath = &key[offset..offset + keypath_len];
+        let keypath_str = String::from_utf8_lossy(keypath);
+        println!("Keypath: '{}'", keypath_str);
+        offset += keypath_len;
+
+        // Read IID (u64)
+        let iid_bytes: [u8; 8] = key[offset..offset + 8]
+            .try_into()
+            .expect("iid bytes");
+        let iid = u64::from_le_bytes(iid_bytes);
+        println!("IID: {}", iid);
+        offset += 8;
+
+        // Read bounding box (4 f64 values)
+        if key.len() >= offset + 32 {
+            let mut bbox = [0.0_f64; 4];
+            for (idx, value) in bbox.iter_mut().enumerate() {
+                let start = offset + (idx * 8);
+                let bytes: [u8; 8] = key[start..start + 8]
+                    .try_into()
+                    .expect("bbox bytes");
+                *value = f64::from_le_bytes(bytes);
+            }
+            println!("Bounding box: [{}, {}, {}, {}]", bbox[0], bbox[1], bbox[2], bbox[3]);
+
+            // Check if point [10.9, 48.4] should be inside various test bboxes
+            println!("\nChecking query matches:");
+            let test_bboxes = vec![
+                ([-1000.0, -1000.0, 99.0, 1000.0], "[-1000, -1000, 99, 1000]"),
+                ([0.0, 0.0, 20.0, 50.0], "[0, 0, 20, 50]"),
+                ([10.0, 48.0, 11.0, 49.0], "[10, 48, 11, 49]"),
+            ];
+
+            for (query_bbox, desc) in test_bboxes {
+                let intersects = bbox[0] <= query_bbox[2] && bbox[2] >= query_bbox[0] &&
+                                bbox[1] <= query_bbox[3] && bbox[3] >= query_bbox[1];
+                println!("  Query {}: intersects = {}", desc, intersects);
+            }
+        }
+    }
+}

src/bin/debug_bind_filter.rs

@@ -0,0 +1,64 @@
+use noise_search::index::{Batch, Index, OpenOptions};
+extern crate rocksdb;
+use std::convert::TryInto;
+
+fn main() {
+    let dbname = "target/tests/debug_bind_filter";
+    let _ = Index::drop(dbname);
+
+    let mut index = Index::open(dbname, Some(OpenOptions::Create)).unwrap();
+    let mut batch = Batch::new();
+
+    // Add document with array of points
+    index.add(r#"{"_id": "arraypoint", "area": [{"type": "Point", "coordinates": [10.9, 48.4]}, {"type": "Point", "coordinates": [-5.0, -20.1]}]}"#, &mut batch).unwrap();
+    index.flush(batch).unwrap();
+
+    // Check what's in the rtree
+    println!("=== RTree entries ===");
+    let rtree_cf = index.rocks.cf_handle("rtree").unwrap();
+    let iter = index.rocks.iterator_cf(rtree_cf, rocksdb::IteratorMode::Start).unwrap();
+
+    for (key, value) in iter {
+        // Parse key
+        let mut offset = 0;
+        let keypath_len = key[offset] as usize;
+        offset += 1;
+        let keypath = String::from_utf8_lossy(&key[offset..offset + keypath_len]);
+        offset += keypath_len;
+
+        let iid_bytes: [u8; 8] = key[offset..offset + 8]
+            .try_into()
+            .expect("iid bytes");
+        let iid = u64::from_le_bytes(iid_bytes);
+        offset += 8;
+
+        let mut bbox = [0.0_f64; 4];
+        for (idx, value) in bbox.iter_mut().enumerate() {
+            let start = offset + (idx * 8);
+            let bytes: [u8; 8] = key[start..start + 8]
+                .try_into()
+                .expect("bbox bytes");
+            *value = f64::from_le_bytes(bytes);
+        }
+
+        // Parse value (arraypath)
+        let arraypath = if value.len() >= 8 {
+            let bytes: [u8; 8] = value[..8].try_into().expect("array path bytes");
+            vec![u64::from_le_bytes(bytes)]
+        } else {
+            vec![]
+        };
+
+        println!("Keypath: '{}', IID: {}, BBox: {:?}, ArrayPath: {:?}", keypath, iid, bbox, arraypath);
+    }
+
+    // Check main CF entries for area
+    println!("\n=== Main CF area entries ===");
+    let main_iter = index.rocks.iterator(rocksdb::IteratorMode::Start);
+    for (key, _value) in main_iter {
+        let key_str = String::from_utf8_lossy(&key);
+        if key_str.contains(".area") && key_str.contains("coordinates") {
+            println!("  {}", key_str);
+        }
+    }
+}

src/bin/test_rtree_raw.rs

@@ -0,0 +1,86 @@
+use noise_search::index::{Batch, Index, OpenOptions};
+extern crate rocksdb;
+use rocksdb::IteratorMode;
+use std::convert::TryInto;
+
+fn decode_rtree_key(key: &[u8]) -> (String, u64) {
+    let mut offset = 0;
+
+    // Read keypath length (varint)
+    let keypath_len = key[offset] as usize;
+    offset += 1;
+
+    // Read keypath
+    let keypath = &key[offset..offset + keypath_len];
+    let keypath_str = String::from_utf8_lossy(keypath).to_string();
+    offset += keypath_len;
+
+    // Read IID (u64)
+    let iid_bytes: [u8; 8] = key[offset..offset + 8]
+        .try_into()
+        .expect("iid bytes");
+    let iid = u64::from_le_bytes(iid_bytes);
+
+    (keypath_str, iid)
+}
+
+fn main() {
+    let dbname = "target/tests/test_rtree_raw";
+    let _ = Index::drop(dbname);
+
+    let mut index = Index::open(dbname, Some(OpenOptions::Create)).unwrap();
+    let mut batch = Batch::new();
+
+    // Add test documents
+    println!("Adding documents...");
+    index.add(r#"{"_id": "point1", "geometry": {"type": "Point", "coordinates": [10.9, 48.4]}}"#, &mut batch).unwrap();
+    index.add(r#"{"_id": "point2", "geometry": {"type": "Point", "coordinates": [50.0, 50.0]}}"#, &mut batch).unwrap();
+    index.add(r#"{"_id": "point3", "other": {"type": "Point", "coordinates": [10.0, 10.0]}}"#, &mut batch).unwrap();
+    index.flush(batch).unwrap();
+
+    // Test 1: List all rtree entries
+    println!("\n=== All RTree entries ===");
+    let rtree_cf = index.rocks.cf_handle("rtree").unwrap();
+    let iter = index.rocks.iterator_cf(rtree_cf, IteratorMode::Start).unwrap();
+    for (key, _value) in iter {
+        let (keypath, iid) = decode_rtree_key(&key);
+        println!("  Keypath: '{}', IID: {}", keypath, iid);
+    }
+
+    // Test 2: Test RTree iterator with query
+    println!("\n=== Testing RTree spatial query ===");
+    let snapshot = index.rocks.snapshot();
+
+    // Build a query bbox that should match point1
+    let query_bbox = vec![
+        0.0f64.to_le_bytes().to_vec(),    // west
+        0.0f64.to_le_bytes().to_vec(),    // south
+        20.0f64.to_le_bytes().to_vec(),   // east
+        50.0f64.to_le_bytes().to_vec(),   // north
+    ].concat();
+
+    println!("Query bbox: [0, 0, 20, 50] (should match point1)");
+
+    // Create the full query key
+    let keypath = ".geometry";
+    let mut query = Vec::new();
+    query.push(keypath.len() as u8);
+    query.extend_from_slice(keypath.as_bytes());
+    query.extend_from_slice(&0u64.to_le_bytes());      // min IID
+    query.extend_from_slice(&u64::MAX.to_le_bytes()); // max IID
+    query.extend_from_slice(&query_bbox);
+
+    println!("Query key length: {}", query.len());
+
+    let rtree_iter = snapshot.rtree_iterator(&query);
+    let mut count = 0;
+    for (key, _value) in rtree_iter {
+        count += 1;
+        let (keypath, iid) = decode_rtree_key(&key);
+        println!("  Result {}: Keypath: '{}', IID: {}", count, keypath, iid);
+    }
+
+    if count == 0 {
+        println!("  No results returned by RTree iterator!");
+    }
+}