added compressed index array

sparse_vector/src/main.rs

@@ -10,10 +10,9 @@ use jemalloc_ctl::{stats, epoch};
 #[global_allocator]
 static ALLOC: jemallocator::Jemalloc = jemallocator::Jemalloc;
 
-/// Only stores more efficiently when at least 50% of all elements are zeros
 pub struct SparseVec {
     values: Vec<f64>,
-    indices: Vec<usize>,
+    indices: Vec<u16>,
 }
 
 impl SparseVec {
@@ -22,8 +21,9 @@ impl SparseVec {
         let mut sum = 0.0;
 
         for index in 0..other.indices.len() {
+            let uncompressed_index = decompress_index(index, &self.indices);
             // exponential search for an element in the second vector to have the same index
-            sum += binary_search(self.indices[index], &other.indices, &other.values) * self.values[index];
+            sum += binary_search(uncompressed_index, &other.indices, &other.values) * self.values[index];
         }
 
         sum
@@ -37,11 +37,15 @@ impl SparseVec {
 
         let mut rng = rand::thread_rng();
 
+        let mut last_idx = 0;
+
         for i in 0..non_zero_elements {
             values.push(0.5);
 
-            let idx = i as f32 / non_zero_elements as f32 * (elements as f32 - 4.0) + rng.gen_range(0.0..3.0);
+            let new_idx = i as f32 / non_zero_elements as f32 * (elements as f32 - 4.0) + rng.gen_range(0.0..3.0);
-            indices.push(idx as usize);
+            let compressed_idx = new_idx as usize - last_idx;
+            last_idx = new_idx as usize;
+            indices.push(compressed_idx as u16);
         }
 
         Self {
@@ -52,7 +56,16 @@ impl SparseVec {
 }
 
 #[inline]
-fn binary_search(target: usize, indices: &[usize], values: &[f64]) -> f64 {
+fn decompress_index(index: usize, indices: &[u16]) -> usize {
+    let mut sum = 0;
+    for i in 0..=index {
+        sum += indices[i] as usize;
+    }
+    sum
+}
+
+#[inline]
+fn binary_search(target: usize, indices: &[u16], values: &[f64]) -> f64 {
     let mut range = 0..indices.len();
     loop {
         let mut median = (range.end - range.start) >> 1;
@@ -61,11 +74,13 @@ fn binary_search(target: usize, indices: &[usize], values: &[f64]) -> f64 {
         }
         median += range.start;
 
-        if indices[median] == target {
+        let index = decompress_index(median, indices);
+
+        if index == target {
             return values[median];
         }
 
-        if indices[median] > target {
+        if index > target {
             range.end = median;
         } else {
             range.start = median;
@@ -94,7 +109,7 @@ fn main() {
     const NULL_NON_NULL_RATIO: f64 = 0.02;
 
     let non_zero_elements = (VECTOR_SIZE as f64 * NULL_NON_NULL_RATIO) as usize;
-    let heap_element_size = std::mem::size_of::<f64>() + std::mem::size_of::<usize>();
+    let heap_element_size = std::mem::size_of::<f64>() + std::mem::size_of::<u16>();
 
     println!("Estimated size on heap: {}", ByteSize::b((non_zero_elements * heap_element_size) as u64));
     println!("Size on stack: {} B", std::mem::size_of::<SparseVec>());