initial commit for sparse sketch

Author: etwest

include/graph_sketch_driver.h

@@ -156,10 +156,6 @@ class GraphSketchDriver {
         size_t updates = stream->get_update_buffer(update_array, update_array_size);
 
         if (update_array[updates - 1].type == BREAKPOINT) {
-#ifdef VERIFY_SAMPLES_F
-          std::lock_guard<std::mutex> lk(verifier_mtx);
-          verifier->combine(local_verifier);
-#endif
           --updates;
           got_breakpoint = true;
         }
@@ -173,7 +169,13 @@ class GraphSketchDriver {
 #endif
         }
 
-        if (got_breakpoint) return;
+        if (got_breakpoint) {
+#ifdef VERIFY_SAMPLES_F
+          std::lock_guard<std::mutex> lk(verifier_mtx);
+          verifier->combine(local_verifier);
+#endif
+          return;
+        }
       }
     };
 
@@ -204,6 +206,7 @@ class GraphSketchDriver {
   inline void batch_callback(int thr_id, node_id_t src_vertex,
                              const std::vector<node_id_t> &dst_vertices) {
     total_updates += dst_vertices.size();
+    return;
     sketching_alg->apply_update_batch(thr_id, src_vertex, dst_vertices);
   }
 

include/sketch.h

@@ -11,6 +11,19 @@
 #include "util.h"
 #include "bucket.h"
 
+// TODO: Do we want to use row major or column major order?
+// TODO: How do we want to handle raw_bucket_merge() and get_readonly_bucket_ptr()?
+//       These functions are nice for performance because we can skip serialization but aren't
+//       strictly necessary.
+// TODO: It would be nice to preallocate the structure if we know how big its probably going to be.
+//       This would be helpful for delta sketches for example. 
+// TODO: What are we doing with the num_buckets variable? Could be nice to just be the size of 
+//       buckets array. Could also be upperbound on the size.
+
+// A strategy that could work well would be to allocate a chunk of memory some of which is given to
+// the dense region of the sketch and 3 * sizeof(uint64_t) are given to sparse region.
+// 3 -> position, alpha, gamma (could save a little more space by using 16 bits for position)
+
 // enum SerialType {
 //   FULL,
 //   RANGE,
@@ -43,14 +56,34 @@ class Sketch {
   size_t num_samples;      // number of samples we can perform
   size_t cols_per_sample;  // number of columns to use on each sample
   size_t num_columns;      // Total number of columns. (product of above 2)
-  size_t bkt_per_col;      // number of buckets per column
+  size_t bkt_per_col;      // maximum number of buckets per column (max number of rows)
   size_t num_buckets;      // number of total buckets (product of above 2)
 
   size_t sample_idx = 0;   // number of samples performed so far
 
-  // bucket data
+  // bucket data, stored densely
   Bucket* buckets;
 
+#ifndef L0_FULLY_DENSE
+  size_t num_dense_rows = 4;
+
+  // sparse representation of lower levels of Matrix
+  // TODO: Evaluate if this is shit. It probably is
+  std::vector<std::unordered_map<size_t, Bucket>> bucket_buffer;
+  size_t number_of_sparse_buckets = 0;
+  size_t sparse_capacity = 2 * num_columns; // TODO: evaluate implications of this constant
+
+  /**
+   * Reallocates the dense region of the sketch to have a different number of rows
+   * @param new_num_rows  the new number of rows to store densely
+   */
+  void reallocate_dense_region(size_t new_num_rows);
+#endif
+
+  inline Bucket& get_deterministic_bucket() {
+    // TODO: implement this
+  }
+
  public:
   /**
    * The below constructors use vector length as their input. However, in graph sketching our input

src/sketch.cpp

@@ -11,13 +11,20 @@ Sketch::Sketch(vec_t vector_len, uint64_t seed, size_t _samples, size_t _cols) :
   num_columns = num_samples * cols_per_sample;
   bkt_per_col = calc_bkt_per_col(vector_len);
   num_buckets = num_columns * bkt_per_col + 1; // plus 1 for deterministic bucket
+
+#ifndef L0_FULLY_DENSE
+  buckets = new Bucket[num_columns * num_dense_rows];
+  // TODO: AHHHHHHHHH
+#else
   buckets = new Bucket[num_buckets];
 
   // initialize bucket values
   for (size_t i = 0; i < num_buckets; ++i) {
     buckets[i].alpha = 0;
     buckets[i].gamma = 0;
   }
+#endif
+
 }
 
 Sketch::Sketch(vec_t vector_len, uint64_t seed, std::istream &binary_in, size_t _samples,