New remote directory implementation with mmap and userfaultfd (#5)

* Speed up queries for hot caches approx 10x with userfaultfd

* madvise when tantivy slices our mmap

* Super messy but seems to work now without huge pages

* Fix to disregard the result of the uffd copy. More investigation needed.

* Switch to sstable termdicts, hopefully play nice with memory pressure

Cargo.toml

@@ -3,3 +3,7 @@ resolver = "2"
 members = [
   "airmail", "airmail_common", "airmail_index", "airmail_parser", "airmail_service", "turbosm",
 ]
+
+
+[profile.release]
+debug = 1

airmail/Cargo.toml

@@ -8,7 +8,7 @@ edition = "2021"
 [dependencies]
 levenshtein_automata = "0.2.1"
 s2 = "0.0.12"
-tantivy = "0.21.1"
+tantivy = { version = "0.21.1", features = ["quickwit"] }
 tantivy-common = "0.6.0"
 tantivy-fst = "0.4.0"
 tempfile = "3.9.0"
@@ -21,3 +21,6 @@ reqwest = { version = "0.11.24", features = ["blocking"] }
 lru = "0.12.2"
 async-trait = "0.1.77"
 tokio = { version = "1.36.0", features = ["full"] }
+nix = { version = "0.27", features = ["ioctl", "mman"] }
+userfaultfd = { version = "0.8.1", features = ["linux4_14", "linux5_7"] }
+crossbeam = "0.8.4"

airmail/src/directory/TANTIVY_AUTHORS

@@ -0,0 +1,11 @@
+# This is the list of authors of tantivy for copyright purposes.
+Paul Masurel
+Laurentiu Nicola
+Dru Sellers
+Ashley Mannix
+Michael J. Curry
+Jason Wolfe
+# As an employee of Google I am required to add Google LLC
+# in the list of authors, but this project is not affiliated to Google
+# in any other way.
+Google LLC