Add more sparse tests and framework for enabling sparse math

Makefile

@@ -5,7 +5,7 @@ release:
 	cargo build --release
 
 test:
-	cargo test
+	cargo test --features benchmarking
 
 bench:
 	cargo bench --features benchmarking

src/decoder.rs

@@ -10,7 +10,8 @@ use crate::systematic_constants::extended_source_block_symbols;
 use crate::systematic_constants::num_hdpc_symbols;
 use crate::systematic_constants::num_ldpc_symbols;
 use std::collections::HashSet;
-use crate::matrix::DenseOctetMatrix;
+use crate::matrix::{OctetMatrix, DenseOctetMatrix, SparseOctetMatrix};
+use crate::encoder::SPARSE_MATRIX_THRESHOLD;
 
 pub struct Decoder {
     config: ObjectTransmissionInformation,
@@ -51,6 +52,13 @@ impl Decoder {
         }
     }
 
+    #[cfg(any(test, feature = "benchmarking"))]
+    pub fn set_sparse_threshold(&mut self, value: u32) {
+        for block_decoder in self.block_decoders.iter_mut() {
+            block_decoder.set_sparse_threshold(value);
+        }
+    }
+
     pub fn decode(&mut self, packet: EncodingPacket) -> Option<Vec<u8>> {
         let block_number = packet.payload_id.source_block_number() as usize;
         if self.blocks[block_number].is_none() {
@@ -80,6 +88,7 @@ pub struct SourceBlockDecoder {
     received_source_symbols: u32,
     received_esi: HashSet<u32>,
     decoded: bool,
+    sparse_threshold: u32
 }
 
 impl SourceBlockDecoder {
@@ -98,9 +107,36 @@ impl SourceBlockDecoder {
             received_source_symbols: 0,
             received_esi,
             decoded: false,
+            sparse_threshold: SPARSE_MATRIX_THRESHOLD
         }
     }
 
+    #[cfg(any(test, feature = "benchmarking"))]
+    pub fn set_sparse_threshold(&mut self, value: u32) {
+        self.sparse_threshold = value;
+    }
+
+    fn try_pi_decode(&mut self, constraint_matrix: impl OctetMatrix, symbols: Vec<Symbol>) -> Option<Vec<u8>> {
+        let intermediate_symbols =
+            match fused_inverse_mul_symbols(constraint_matrix, symbols, self.source_block_symbols) {
+                None => return None,
+                Some(s) => s,
+            };
+
+        let mut result = vec![];
+        for i in 0..self.source_block_symbols as usize {
+            if let Some(ref symbol) = self.source_symbols[i] {
+                result.extend(symbol.as_bytes())
+            } else {
+                let rebuilt = self.rebuild_source_symbol(&intermediate_symbols, i as u32);
+                result.extend(rebuilt.as_bytes());
+            }
+        }
+
+        self.decoded = true;
+        return Some(result);
+    }
+
     pub fn decode(&mut self, packet: EncodingPacket) -> Option<Vec<u8>> {
         assert_eq!(
             self.source_block_id,
@@ -163,26 +199,16 @@ impl SourceBlockDecoder {
                 d.push(Symbol::new(repair_packet.data.clone()));
             }
 
-            let constraint_matrix =
-                generate_constraint_matrix::<DenseOctetMatrix>(self.source_block_symbols, &encoded_indices);
-            let intermediate_symbols =
-                match fused_inverse_mul_symbols(constraint_matrix, d, self.source_block_symbols) {
-                    None => return None,
-                    Some(s) => s,
-                };
-
-            let mut result = vec![];
-            for i in 0..self.source_block_symbols as usize {
-                if let Some(ref symbol) = self.source_symbols[i] {
-                    result.extend(symbol.as_bytes())
-                } else {
-                    let rebuilt = self.rebuild_source_symbol(&intermediate_symbols, i as u32);
-                    result.extend(rebuilt.as_bytes());
-                }
+            if extended_source_block_symbols(self.source_block_symbols) >= self.sparse_threshold {
+                let constraint_matrix =
+                    generate_constraint_matrix::<SparseOctetMatrix>(self.source_block_symbols, &encoded_indices);
+                return self.try_pi_decode(constraint_matrix, d);
+            }
+            else {
+                let constraint_matrix =
+                    generate_constraint_matrix::<DenseOctetMatrix>(self.source_block_symbols, &encoded_indices);
+                return self.try_pi_decode(constraint_matrix, d);
             }
-
-            self.decoded = true;
-            return Some(result);
         }
         None
     }

src/encoder.rs

@@ -5,7 +5,7 @@ use crate::base::PayloadId;
 use crate::constraint_matrix::generate_constraint_matrix;
 use crate::pi_solver::fused_inverse_mul_symbols;
 use crate::symbol::Symbol;
-use crate::systematic_constants::calculate_p1;
+use crate::systematic_constants::{calculate_p1, MAX_SOURCE_SYMBOLS_PER_BLOCK};
 use crate::systematic_constants::extended_source_block_symbols;
 use crate::systematic_constants::num_hdpc_symbols;
 use crate::systematic_constants::num_intermediate_symbols;
@@ -13,7 +13,10 @@ use crate::systematic_constants::num_ldpc_symbols;
 use crate::systematic_constants::num_lt_symbols;
 use crate::systematic_constants::num_pi_symbols;
 use crate::ObjectTransmissionInformation;
-use crate::matrix::DenseOctetMatrix;
+use crate::matrix::{DenseOctetMatrix, SparseOctetMatrix};
+
+// Currently disabled, until more testing has been done
+pub const SPARSE_MATRIX_THRESHOLD: u32 = MAX_SOURCE_SYMBOLS_PER_BLOCK + 1;
 
 pub struct Encoder {
     config: ObjectTransmissionInformation,
@@ -106,7 +109,7 @@ impl SourceBlockEncoder {
             .chunks(symbol_size as usize)
             .map(|x| Symbol::new(Vec::from(x)))
             .collect();
-        let intermediate_symbols = gen_intermediate_symbols(&source_symbols, symbol_size as usize);
+        let intermediate_symbols = gen_intermediate_symbols(&source_symbols, symbol_size as usize, SPARSE_MATRIX_THRESHOLD);
         SourceBlockEncoder {
             source_block_id,
             source_symbols,
@@ -154,7 +157,7 @@ impl SourceBlockEncoder {
 
 // See section 5.3.3.4
 #[allow(non_snake_case)]
-fn gen_intermediate_symbols(source_block: &Vec<Symbol>, symbol_size: usize) -> Vec<Symbol> {
+fn gen_intermediate_symbols(source_block: &Vec<Symbol>, symbol_size: usize, sparse_threshold: u32) -> Vec<Symbol> {
     let L = num_intermediate_symbols(source_block.len() as u32);
     let S = num_ldpc_symbols(source_block.len() as u32);
     let H = num_hdpc_symbols(source_block.len() as u32);
@@ -174,8 +177,14 @@ fn gen_intermediate_symbols(source_block: &Vec<Symbol>, symbol_size: usize) -> V
     assert_eq!(D.len(), L as usize);
 
     let indices: Vec<u32> = (0..extended_source_symbols).collect();
-    let A = generate_constraint_matrix::<DenseOctetMatrix>(extended_source_symbols, &indices);
-    fused_inverse_mul_symbols(A, D, extended_source_symbols).unwrap()
+    if extended_source_symbols >= sparse_threshold {
+        let A = generate_constraint_matrix::<SparseOctetMatrix>(extended_source_symbols, &indices);
+        return fused_inverse_mul_symbols(A, D, extended_source_symbols).unwrap()
+    }
+    else {
+        let A = generate_constraint_matrix::<DenseOctetMatrix>(extended_source_symbols, &indices);
+        return fused_inverse_mul_symbols(A, D, extended_source_symbols).unwrap()
+    }
 }
 
 // Enc[] function, as defined in section 5.3.5.3
@@ -226,7 +235,7 @@ mod tests {
     use crate::encoder::enc;
     use crate::encoder::gen_intermediate_symbols;
     use crate::symbol::Symbol;
-    use crate::systematic_constants::num_ldpc_symbols;
+    use crate::systematic_constants::{num_ldpc_symbols, MAX_SOURCE_SYMBOLS_PER_BLOCK};
     use crate::systematic_constants::num_lt_symbols;
     use crate::systematic_constants::num_pi_symbols;
 
@@ -246,9 +255,18 @@ mod tests {
     }
 
     #[test]
-    fn enc_constraint() {
+    fn enc_constraint_dense() {
+        enc_constraint(MAX_SOURCE_SYMBOLS_PER_BLOCK + 1);
+    }
+
+    #[test]
+    fn enc_constraint_sparse() {
+        enc_constraint(0);
+    }
+
+    fn enc_constraint(sparse_threshold: u32) {
         let source_symbols = gen_test_symbols();
-        let intermediate_symbols = gen_intermediate_symbols(&source_symbols, SYMBOL_SIZE);
+        let intermediate_symbols = gen_intermediate_symbols(&source_symbols, SYMBOL_SIZE, sparse_threshold);
 
         // See section 5.3.3.4.1, item 1.
         for i in 0..source_symbols.len() {
@@ -258,10 +276,19 @@ mod tests {
         }
     }
 
-    #[allow(non_snake_case)]
     #[test]
-    fn ldpc_constraint() {
-        let C = gen_intermediate_symbols(&gen_test_symbols(), SYMBOL_SIZE);
+    fn ldpc_constraint_dense() {
+        ldpc_constraint(MAX_SOURCE_SYMBOLS_PER_BLOCK + 1);
+    }
+
+    #[test]
+    fn ldpc_constraint_sparse() {
+        ldpc_constraint(0);
+    }
+
+    #[allow(non_snake_case)]
+    fn ldpc_constraint(sparse_threshold: u32) {
+        let C = gen_intermediate_symbols(&gen_test_symbols(), SYMBOL_SIZE, sparse_threshold);
         let S = num_ldpc_symbols(NUM_SYMBOLS) as usize;
         let P = num_pi_symbols(NUM_SYMBOLS) as usize;
         let W = num_lt_symbols(NUM_SYMBOLS) as usize;

src/systematic_constants.rs

@@ -1,5 +1,5 @@
 // K'_max as defined in section 5.1.2
-const MAX_SOURCE_SYMBOLS_PER_BLOCK: u32 = 56403;
+pub const MAX_SOURCE_SYMBOLS_PER_BLOCK: u32 = 56403;
 
 // Table 2, as defined in section 5.6
 pub const SYSTEMATIC_INDICES_AND_PARAMETERS: [(u32, u32, u32, u32, u32); 477] = [

tests/codec_tests.rs

@@ -8,7 +8,16 @@ mod codec_tests {
     use raptorq::SourceBlockEncoder;
 
     #[test]
-    fn random_erasure() {
+    fn random_erasure_dense() {
+        random_erasure(99_999);
+    }
+
+    #[test]
+    fn random_erasure_sparse() {
+        random_erasure(0);
+    }
+
+    fn random_erasure(sparse_threshold: u32) {
         let elements: usize = rand::thread_rng().gen_range(1, 1_000_000);
         let mut data: Vec<u8> = vec![0; elements];
         for i in 0..elements {
@@ -27,6 +36,7 @@ mod codec_tests {
         packets.truncate(length - 10);
 
         let mut decoder = Decoder::new(encoder.get_config());
+        decoder.set_sparse_threshold(sparse_threshold);
 
         let mut result = None;
         while !packets.is_empty() {
@@ -40,7 +50,16 @@ mod codec_tests {
     }
 
     #[test]
-    fn round_trip() {
+    fn round_trip_dense() {
+        round_trip(99_999);
+    }
+
+    #[test]
+    fn round_trip_sparse() {
+        round_trip(0);
+    }
+
+    fn round_trip(sparse_threshold: u32) {
         let elements = 1024;
         let mut data: Vec<u8> = vec![0; elements];
         for i in 0..elements {
@@ -50,6 +69,7 @@ mod codec_tests {
         let encoder = SourceBlockEncoder::new(1, 8, &data);
 
         let mut decoder = SourceBlockDecoder::new(1, 8, elements as u64);
+        decoder.set_sparse_threshold(sparse_threshold);
 
         let mut result = None;
         for packet in encoder.source_packets() {
@@ -61,7 +81,16 @@ mod codec_tests {
     }
 
     #[test]
-    fn repair() {
+    fn repair_dense() {
+        repair(99_999);
+    }
+
+    #[test]
+    fn repair_sparse() {
+        repair(0);
+    }
+
+    fn repair(sparse_threshold: u32) {
         let elements = 1024;
         let mut data: Vec<u8> = vec![0; elements];
         for i in 0..elements {
@@ -71,6 +100,7 @@ mod codec_tests {
         let encoder = SourceBlockEncoder::new(1, 8, &data);
 
         let mut decoder = SourceBlockDecoder::new(1, 8, elements as u64);
+        decoder.set_sparse_threshold(sparse_threshold);
 
         let mut result = None;
         let mut parsed_packets = 0;