mirror of
https://github.com/cberner/raptorq.git
synced 2024-06-28 09:41:41 +00:00
Add more sparse tests and framework for enabling sparse math
This commit is contained in:
parent
1ff495f445
commit
fca4f14c06
2
Makefile
2
Makefile
@ -5,7 +5,7 @@ release:
|
|||||||
cargo build --release
|
cargo build --release
|
||||||
|
|
||||||
test:
|
test:
|
||||||
cargo test
|
cargo test --features benchmarking
|
||||||
|
|
||||||
bench:
|
bench:
|
||||||
cargo bench --features benchmarking
|
cargo bench --features benchmarking
|
||||||
|
@ -10,7 +10,8 @@ use crate::systematic_constants::extended_source_block_symbols;
|
|||||||
use crate::systematic_constants::num_hdpc_symbols;
|
use crate::systematic_constants::num_hdpc_symbols;
|
||||||
use crate::systematic_constants::num_ldpc_symbols;
|
use crate::systematic_constants::num_ldpc_symbols;
|
||||||
use std::collections::HashSet;
|
use std::collections::HashSet;
|
||||||
use crate::matrix::DenseOctetMatrix;
|
use crate::matrix::{OctetMatrix, DenseOctetMatrix, SparseOctetMatrix};
|
||||||
|
use crate::encoder::SPARSE_MATRIX_THRESHOLD;
|
||||||
|
|
||||||
pub struct Decoder {
|
pub struct Decoder {
|
||||||
config: ObjectTransmissionInformation,
|
config: ObjectTransmissionInformation,
|
||||||
@ -51,6 +52,13 @@ impl Decoder {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(any(test, feature = "benchmarking"))]
|
||||||
|
pub fn set_sparse_threshold(&mut self, value: u32) {
|
||||||
|
for block_decoder in self.block_decoders.iter_mut() {
|
||||||
|
block_decoder.set_sparse_threshold(value);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
pub fn decode(&mut self, packet: EncodingPacket) -> Option<Vec<u8>> {
|
pub fn decode(&mut self, packet: EncodingPacket) -> Option<Vec<u8>> {
|
||||||
let block_number = packet.payload_id.source_block_number() as usize;
|
let block_number = packet.payload_id.source_block_number() as usize;
|
||||||
if self.blocks[block_number].is_none() {
|
if self.blocks[block_number].is_none() {
|
||||||
@ -80,6 +88,7 @@ pub struct SourceBlockDecoder {
|
|||||||
received_source_symbols: u32,
|
received_source_symbols: u32,
|
||||||
received_esi: HashSet<u32>,
|
received_esi: HashSet<u32>,
|
||||||
decoded: bool,
|
decoded: bool,
|
||||||
|
sparse_threshold: u32
|
||||||
}
|
}
|
||||||
|
|
||||||
impl SourceBlockDecoder {
|
impl SourceBlockDecoder {
|
||||||
@ -98,9 +107,36 @@ impl SourceBlockDecoder {
|
|||||||
received_source_symbols: 0,
|
received_source_symbols: 0,
|
||||||
received_esi,
|
received_esi,
|
||||||
decoded: false,
|
decoded: false,
|
||||||
|
sparse_threshold: SPARSE_MATRIX_THRESHOLD
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(any(test, feature = "benchmarking"))]
|
||||||
|
pub fn set_sparse_threshold(&mut self, value: u32) {
|
||||||
|
self.sparse_threshold = value;
|
||||||
|
}
|
||||||
|
|
||||||
|
fn try_pi_decode(&mut self, constraint_matrix: impl OctetMatrix, symbols: Vec<Symbol>) -> Option<Vec<u8>> {
|
||||||
|
let intermediate_symbols =
|
||||||
|
match fused_inverse_mul_symbols(constraint_matrix, symbols, self.source_block_symbols) {
|
||||||
|
None => return None,
|
||||||
|
Some(s) => s,
|
||||||
|
};
|
||||||
|
|
||||||
|
let mut result = vec![];
|
||||||
|
for i in 0..self.source_block_symbols as usize {
|
||||||
|
if let Some(ref symbol) = self.source_symbols[i] {
|
||||||
|
result.extend(symbol.as_bytes())
|
||||||
|
} else {
|
||||||
|
let rebuilt = self.rebuild_source_symbol(&intermediate_symbols, i as u32);
|
||||||
|
result.extend(rebuilt.as_bytes());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
self.decoded = true;
|
||||||
|
return Some(result);
|
||||||
|
}
|
||||||
|
|
||||||
pub fn decode(&mut self, packet: EncodingPacket) -> Option<Vec<u8>> {
|
pub fn decode(&mut self, packet: EncodingPacket) -> Option<Vec<u8>> {
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
self.source_block_id,
|
self.source_block_id,
|
||||||
@ -163,26 +199,16 @@ impl SourceBlockDecoder {
|
|||||||
d.push(Symbol::new(repair_packet.data.clone()));
|
d.push(Symbol::new(repair_packet.data.clone()));
|
||||||
}
|
}
|
||||||
|
|
||||||
let constraint_matrix =
|
if extended_source_block_symbols(self.source_block_symbols) >= self.sparse_threshold {
|
||||||
generate_constraint_matrix::<DenseOctetMatrix>(self.source_block_symbols, &encoded_indices);
|
let constraint_matrix =
|
||||||
let intermediate_symbols =
|
generate_constraint_matrix::<SparseOctetMatrix>(self.source_block_symbols, &encoded_indices);
|
||||||
match fused_inverse_mul_symbols(constraint_matrix, d, self.source_block_symbols) {
|
return self.try_pi_decode(constraint_matrix, d);
|
||||||
None => return None,
|
}
|
||||||
Some(s) => s,
|
else {
|
||||||
};
|
let constraint_matrix =
|
||||||
|
generate_constraint_matrix::<DenseOctetMatrix>(self.source_block_symbols, &encoded_indices);
|
||||||
let mut result = vec![];
|
return self.try_pi_decode(constraint_matrix, d);
|
||||||
for i in 0..self.source_block_symbols as usize {
|
|
||||||
if let Some(ref symbol) = self.source_symbols[i] {
|
|
||||||
result.extend(symbol.as_bytes())
|
|
||||||
} else {
|
|
||||||
let rebuilt = self.rebuild_source_symbol(&intermediate_symbols, i as u32);
|
|
||||||
result.extend(rebuilt.as_bytes());
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
self.decoded = true;
|
|
||||||
return Some(result);
|
|
||||||
}
|
}
|
||||||
None
|
None
|
||||||
}
|
}
|
||||||
|
@ -5,7 +5,7 @@ use crate::base::PayloadId;
|
|||||||
use crate::constraint_matrix::generate_constraint_matrix;
|
use crate::constraint_matrix::generate_constraint_matrix;
|
||||||
use crate::pi_solver::fused_inverse_mul_symbols;
|
use crate::pi_solver::fused_inverse_mul_symbols;
|
||||||
use crate::symbol::Symbol;
|
use crate::symbol::Symbol;
|
||||||
use crate::systematic_constants::calculate_p1;
|
use crate::systematic_constants::{calculate_p1, MAX_SOURCE_SYMBOLS_PER_BLOCK};
|
||||||
use crate::systematic_constants::extended_source_block_symbols;
|
use crate::systematic_constants::extended_source_block_symbols;
|
||||||
use crate::systematic_constants::num_hdpc_symbols;
|
use crate::systematic_constants::num_hdpc_symbols;
|
||||||
use crate::systematic_constants::num_intermediate_symbols;
|
use crate::systematic_constants::num_intermediate_symbols;
|
||||||
@ -13,7 +13,10 @@ use crate::systematic_constants::num_ldpc_symbols;
|
|||||||
use crate::systematic_constants::num_lt_symbols;
|
use crate::systematic_constants::num_lt_symbols;
|
||||||
use crate::systematic_constants::num_pi_symbols;
|
use crate::systematic_constants::num_pi_symbols;
|
||||||
use crate::ObjectTransmissionInformation;
|
use crate::ObjectTransmissionInformation;
|
||||||
use crate::matrix::DenseOctetMatrix;
|
use crate::matrix::{DenseOctetMatrix, SparseOctetMatrix};
|
||||||
|
|
||||||
|
// Currently disabled, until more testing has been done
|
||||||
|
pub const SPARSE_MATRIX_THRESHOLD: u32 = MAX_SOURCE_SYMBOLS_PER_BLOCK + 1;
|
||||||
|
|
||||||
pub struct Encoder {
|
pub struct Encoder {
|
||||||
config: ObjectTransmissionInformation,
|
config: ObjectTransmissionInformation,
|
||||||
@ -106,7 +109,7 @@ impl SourceBlockEncoder {
|
|||||||
.chunks(symbol_size as usize)
|
.chunks(symbol_size as usize)
|
||||||
.map(|x| Symbol::new(Vec::from(x)))
|
.map(|x| Symbol::new(Vec::from(x)))
|
||||||
.collect();
|
.collect();
|
||||||
let intermediate_symbols = gen_intermediate_symbols(&source_symbols, symbol_size as usize);
|
let intermediate_symbols = gen_intermediate_symbols(&source_symbols, symbol_size as usize, SPARSE_MATRIX_THRESHOLD);
|
||||||
SourceBlockEncoder {
|
SourceBlockEncoder {
|
||||||
source_block_id,
|
source_block_id,
|
||||||
source_symbols,
|
source_symbols,
|
||||||
@ -154,7 +157,7 @@ impl SourceBlockEncoder {
|
|||||||
|
|
||||||
// See section 5.3.3.4
|
// See section 5.3.3.4
|
||||||
#[allow(non_snake_case)]
|
#[allow(non_snake_case)]
|
||||||
fn gen_intermediate_symbols(source_block: &Vec<Symbol>, symbol_size: usize) -> Vec<Symbol> {
|
fn gen_intermediate_symbols(source_block: &Vec<Symbol>, symbol_size: usize, sparse_threshold: u32) -> Vec<Symbol> {
|
||||||
let L = num_intermediate_symbols(source_block.len() as u32);
|
let L = num_intermediate_symbols(source_block.len() as u32);
|
||||||
let S = num_ldpc_symbols(source_block.len() as u32);
|
let S = num_ldpc_symbols(source_block.len() as u32);
|
||||||
let H = num_hdpc_symbols(source_block.len() as u32);
|
let H = num_hdpc_symbols(source_block.len() as u32);
|
||||||
@ -174,8 +177,14 @@ fn gen_intermediate_symbols(source_block: &Vec<Symbol>, symbol_size: usize) -> V
|
|||||||
assert_eq!(D.len(), L as usize);
|
assert_eq!(D.len(), L as usize);
|
||||||
|
|
||||||
let indices: Vec<u32> = (0..extended_source_symbols).collect();
|
let indices: Vec<u32> = (0..extended_source_symbols).collect();
|
||||||
let A = generate_constraint_matrix::<DenseOctetMatrix>(extended_source_symbols, &indices);
|
if extended_source_symbols >= sparse_threshold {
|
||||||
fused_inverse_mul_symbols(A, D, extended_source_symbols).unwrap()
|
let A = generate_constraint_matrix::<SparseOctetMatrix>(extended_source_symbols, &indices);
|
||||||
|
return fused_inverse_mul_symbols(A, D, extended_source_symbols).unwrap()
|
||||||
|
}
|
||||||
|
else {
|
||||||
|
let A = generate_constraint_matrix::<DenseOctetMatrix>(extended_source_symbols, &indices);
|
||||||
|
return fused_inverse_mul_symbols(A, D, extended_source_symbols).unwrap()
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Enc[] function, as defined in section 5.3.5.3
|
// Enc[] function, as defined in section 5.3.5.3
|
||||||
@ -226,7 +235,7 @@ mod tests {
|
|||||||
use crate::encoder::enc;
|
use crate::encoder::enc;
|
||||||
use crate::encoder::gen_intermediate_symbols;
|
use crate::encoder::gen_intermediate_symbols;
|
||||||
use crate::symbol::Symbol;
|
use crate::symbol::Symbol;
|
||||||
use crate::systematic_constants::num_ldpc_symbols;
|
use crate::systematic_constants::{num_ldpc_symbols, MAX_SOURCE_SYMBOLS_PER_BLOCK};
|
||||||
use crate::systematic_constants::num_lt_symbols;
|
use crate::systematic_constants::num_lt_symbols;
|
||||||
use crate::systematic_constants::num_pi_symbols;
|
use crate::systematic_constants::num_pi_symbols;
|
||||||
|
|
||||||
@ -246,9 +255,18 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn enc_constraint() {
|
fn enc_constraint_dense() {
|
||||||
|
enc_constraint(MAX_SOURCE_SYMBOLS_PER_BLOCK + 1);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn enc_constraint_sparse() {
|
||||||
|
enc_constraint(0);
|
||||||
|
}
|
||||||
|
|
||||||
|
fn enc_constraint(sparse_threshold: u32) {
|
||||||
let source_symbols = gen_test_symbols();
|
let source_symbols = gen_test_symbols();
|
||||||
let intermediate_symbols = gen_intermediate_symbols(&source_symbols, SYMBOL_SIZE);
|
let intermediate_symbols = gen_intermediate_symbols(&source_symbols, SYMBOL_SIZE, sparse_threshold);
|
||||||
|
|
||||||
// See section 5.3.3.4.1, item 1.
|
// See section 5.3.3.4.1, item 1.
|
||||||
for i in 0..source_symbols.len() {
|
for i in 0..source_symbols.len() {
|
||||||
@ -258,10 +276,19 @@ mod tests {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[allow(non_snake_case)]
|
|
||||||
#[test]
|
#[test]
|
||||||
fn ldpc_constraint() {
|
fn ldpc_constraint_dense() {
|
||||||
let C = gen_intermediate_symbols(&gen_test_symbols(), SYMBOL_SIZE);
|
ldpc_constraint(MAX_SOURCE_SYMBOLS_PER_BLOCK + 1);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn ldpc_constraint_sparse() {
|
||||||
|
ldpc_constraint(0);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[allow(non_snake_case)]
|
||||||
|
fn ldpc_constraint(sparse_threshold: u32) {
|
||||||
|
let C = gen_intermediate_symbols(&gen_test_symbols(), SYMBOL_SIZE, sparse_threshold);
|
||||||
let S = num_ldpc_symbols(NUM_SYMBOLS) as usize;
|
let S = num_ldpc_symbols(NUM_SYMBOLS) as usize;
|
||||||
let P = num_pi_symbols(NUM_SYMBOLS) as usize;
|
let P = num_pi_symbols(NUM_SYMBOLS) as usize;
|
||||||
let W = num_lt_symbols(NUM_SYMBOLS) as usize;
|
let W = num_lt_symbols(NUM_SYMBOLS) as usize;
|
||||||
|
@ -1,5 +1,5 @@
|
|||||||
// K'_max as defined in section 5.1.2
|
// K'_max as defined in section 5.1.2
|
||||||
const MAX_SOURCE_SYMBOLS_PER_BLOCK: u32 = 56403;
|
pub const MAX_SOURCE_SYMBOLS_PER_BLOCK: u32 = 56403;
|
||||||
|
|
||||||
// Table 2, as defined in section 5.6
|
// Table 2, as defined in section 5.6
|
||||||
pub const SYSTEMATIC_INDICES_AND_PARAMETERS: [(u32, u32, u32, u32, u32); 477] = [
|
pub const SYSTEMATIC_INDICES_AND_PARAMETERS: [(u32, u32, u32, u32, u32); 477] = [
|
||||||
|
@ -8,7 +8,16 @@ mod codec_tests {
|
|||||||
use raptorq::SourceBlockEncoder;
|
use raptorq::SourceBlockEncoder;
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn random_erasure() {
|
fn random_erasure_dense() {
|
||||||
|
random_erasure(99_999);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn random_erasure_sparse() {
|
||||||
|
random_erasure(0);
|
||||||
|
}
|
||||||
|
|
||||||
|
fn random_erasure(sparse_threshold: u32) {
|
||||||
let elements: usize = rand::thread_rng().gen_range(1, 1_000_000);
|
let elements: usize = rand::thread_rng().gen_range(1, 1_000_000);
|
||||||
let mut data: Vec<u8> = vec![0; elements];
|
let mut data: Vec<u8> = vec![0; elements];
|
||||||
for i in 0..elements {
|
for i in 0..elements {
|
||||||
@ -27,6 +36,7 @@ mod codec_tests {
|
|||||||
packets.truncate(length - 10);
|
packets.truncate(length - 10);
|
||||||
|
|
||||||
let mut decoder = Decoder::new(encoder.get_config());
|
let mut decoder = Decoder::new(encoder.get_config());
|
||||||
|
decoder.set_sparse_threshold(sparse_threshold);
|
||||||
|
|
||||||
let mut result = None;
|
let mut result = None;
|
||||||
while !packets.is_empty() {
|
while !packets.is_empty() {
|
||||||
@ -40,7 +50,16 @@ mod codec_tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn round_trip() {
|
fn round_trip_dense() {
|
||||||
|
round_trip(99_999);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn round_trip_sparse() {
|
||||||
|
round_trip(0);
|
||||||
|
}
|
||||||
|
|
||||||
|
fn round_trip(sparse_threshold: u32) {
|
||||||
let elements = 1024;
|
let elements = 1024;
|
||||||
let mut data: Vec<u8> = vec![0; elements];
|
let mut data: Vec<u8> = vec![0; elements];
|
||||||
for i in 0..elements {
|
for i in 0..elements {
|
||||||
@ -50,6 +69,7 @@ mod codec_tests {
|
|||||||
let encoder = SourceBlockEncoder::new(1, 8, &data);
|
let encoder = SourceBlockEncoder::new(1, 8, &data);
|
||||||
|
|
||||||
let mut decoder = SourceBlockDecoder::new(1, 8, elements as u64);
|
let mut decoder = SourceBlockDecoder::new(1, 8, elements as u64);
|
||||||
|
decoder.set_sparse_threshold(sparse_threshold);
|
||||||
|
|
||||||
let mut result = None;
|
let mut result = None;
|
||||||
for packet in encoder.source_packets() {
|
for packet in encoder.source_packets() {
|
||||||
@ -61,7 +81,16 @@ mod codec_tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn repair() {
|
fn repair_dense() {
|
||||||
|
repair(99_999);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn repair_sparse() {
|
||||||
|
repair(0);
|
||||||
|
}
|
||||||
|
|
||||||
|
fn repair(sparse_threshold: u32) {
|
||||||
let elements = 1024;
|
let elements = 1024;
|
||||||
let mut data: Vec<u8> = vec![0; elements];
|
let mut data: Vec<u8> = vec![0; elements];
|
||||||
for i in 0..elements {
|
for i in 0..elements {
|
||||||
@ -71,6 +100,7 @@ mod codec_tests {
|
|||||||
let encoder = SourceBlockEncoder::new(1, 8, &data);
|
let encoder = SourceBlockEncoder::new(1, 8, &data);
|
||||||
|
|
||||||
let mut decoder = SourceBlockDecoder::new(1, 8, elements as u64);
|
let mut decoder = SourceBlockDecoder::new(1, 8, elements as u64);
|
||||||
|
decoder.set_sparse_threshold(sparse_threshold);
|
||||||
|
|
||||||
let mut result = None;
|
let mut result = None;
|
||||||
let mut parsed_packets = 0;
|
let mut parsed_packets = 0;
|
||||||
|
Loading…
Reference in New Issue
Block a user