Squashed 'vendor/ruvector/' content from commit b64c2172
git-subtree-dir: vendor/ruvector git-subtree-split: b64c21726f2bb37286d9ee36a7869fef60cc6900
This commit is contained in:
1012
crates/ruvector-temporal-tensor/tests/benchmarks.rs
Normal file
1012
crates/ruvector-temporal-tensor/tests/benchmarks.rs
Normal file
File diff suppressed because it is too large
Load Diff
701
crates/ruvector-temporal-tensor/tests/integration.rs
Normal file
701
crates/ruvector-temporal-tensor/tests/integration.rs
Normal file
@@ -0,0 +1,701 @@
|
||||
//! End-to-end integration tests for the temporal tensor store.
|
||||
//!
|
||||
//! Exercises the full lifecycle: put, get, tier migration, delta compression,
|
||||
//! quantization quality, eviction, checksums, witness logging, and factor
|
||||
//! reconstruction.
|
||||
//!
|
||||
//! Run via: `cargo test -p ruvector-temporal-tensor --test integration`
|
||||
|
||||
use ruvector_temporal_tensor::delta::{
|
||||
compute_delta, decode_delta, encode_delta, DeltaChain, FactorSet,
|
||||
};
|
||||
use ruvector_temporal_tensor::metrics::{TierChangeReason, WitnessEvent, WitnessLog};
|
||||
use ruvector_temporal_tensor::quantizer;
|
||||
use ruvector_temporal_tensor::segment;
|
||||
use ruvector_temporal_tensor::store::{BlockKey, ReconstructPolicy, StoreError, Tier, TieredStore};
|
||||
use ruvector_temporal_tensor::tiering::{self, TierConfig};
|
||||
use ruvector_temporal_tensor::{TemporalTensorCompressor, TierPolicy};
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Deterministic PRNG (LCG) -- no external deps
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Simple linear congruential generator. Constants from Knuth MMIX.
|
||||
struct SimpleRng {
|
||||
state: u64,
|
||||
}
|
||||
|
||||
impl SimpleRng {
|
||||
fn new(seed: u64) -> Self {
|
||||
Self { state: seed }
|
||||
}
|
||||
|
||||
fn next_u64(&mut self) -> u64 {
|
||||
self.state = self
|
||||
.state
|
||||
.wrapping_mul(6_364_136_223_846_793_005)
|
||||
.wrapping_add(1_442_695_040_888_963_407);
|
||||
self.state
|
||||
}
|
||||
|
||||
fn next_f64(&mut self) -> f64 {
|
||||
(self.next_u64() >> 11) as f64 / (1u64 << 53) as f64
|
||||
}
|
||||
|
||||
fn next_f32(&mut self) -> f32 {
|
||||
self.next_f64() as f32
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Helpers
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
fn make_key(tid: u128, idx: u32) -> BlockKey {
|
||||
BlockKey {
|
||||
tensor_id: tid,
|
||||
block_index: idx,
|
||||
}
|
||||
}
|
||||
|
||||
/// Map tiering module Tier to store module Tier.
|
||||
fn tiering_to_store_tier(t: tiering::Tier) -> Tier {
|
||||
match t {
|
||||
tiering::Tier::Tier0 => Tier::Tier0,
|
||||
tiering::Tier::Tier1 => Tier::Tier1,
|
||||
tiering::Tier::Tier2 => Tier::Tier2,
|
||||
tiering::Tier::Tier3 => Tier::Tier3,
|
||||
}
|
||||
}
|
||||
|
||||
// ===========================================================================
|
||||
// 1. Full Lifecycle Test
|
||||
// ===========================================================================
|
||||
|
||||
/// Put 100 blocks as hot, simulate 1000 ticks touching only 10, then verify
|
||||
/// that the 90 untouched blocks migrate to colder tiers.
|
||||
#[test]
|
||||
fn test_full_lifecycle() {
|
||||
let mut store = TieredStore::new(4096);
|
||||
let tier_config = TierConfig::default();
|
||||
let n_elems = 64;
|
||||
|
||||
let mut rng = SimpleRng::new(42);
|
||||
let block_data: Vec<Vec<f32>> = (0..100)
|
||||
.map(|_| (0..n_elems).map(|_| rng.next_f32() * 2.0 - 1.0).collect())
|
||||
.collect();
|
||||
|
||||
// Put 100 blocks as Tier1 (hot).
|
||||
for i in 0..100u32 {
|
||||
store
|
||||
.put(make_key(1, i), &block_data[i as usize], Tier::Tier1, 0)
|
||||
.unwrap();
|
||||
}
|
||||
assert_eq!(store.tier_count(Tier::Tier1), 100);
|
||||
assert_eq!(store.block_count(), 100);
|
||||
|
||||
// Parallel tiering metadata for migration scoring.
|
||||
let mut tiering_metas: Vec<tiering::BlockMeta> =
|
||||
(0..100).map(|_| tiering::BlockMeta::new(0)).collect();
|
||||
|
||||
// Simulate 1000 ticks -- only blocks 0..10 are accessed.
|
||||
for tick in 1..=1000u64 {
|
||||
for i in 0..10 {
|
||||
store.touch(make_key(1, i as u32), tick);
|
||||
tiering::touch(&tier_config, tick, &mut tiering_metas[i]);
|
||||
}
|
||||
for i in 10..100 {
|
||||
tiering::tick_decay(&tier_config, &mut tiering_metas[i]);
|
||||
}
|
||||
}
|
||||
|
||||
// Apply tier migration decisions.
|
||||
let mut migrated = 0u32;
|
||||
for i in 0..100u32 {
|
||||
if let Some(target) = tiering::choose_tier(&tier_config, 1000, &tiering_metas[i as usize]) {
|
||||
let st = tiering_to_store_tier(target);
|
||||
if st != Tier::Tier0 {
|
||||
store
|
||||
.put(make_key(1, i), &block_data[i as usize], st, 1000)
|
||||
.unwrap();
|
||||
migrated += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let tier1 = store.tier_count(Tier::Tier1);
|
||||
let tier2 = store.tier_count(Tier::Tier2);
|
||||
let tier3 = store.tier_count(Tier::Tier3);
|
||||
|
||||
assert!(migrated > 0, "expected migrations, got none");
|
||||
assert!(
|
||||
tier1 < 100,
|
||||
"expected fewer Tier1 blocks after migration, got {}",
|
||||
tier1
|
||||
);
|
||||
assert!(tier1 <= 20, "hot blocks should be ~10, got {}", tier1);
|
||||
assert!(
|
||||
tier2 + tier3 >= 80,
|
||||
"expected >=80 in lower tiers, got {} + {}",
|
||||
tier2,
|
||||
tier3
|
||||
);
|
||||
assert_eq!(store.block_count(), 100);
|
||||
}
|
||||
|
||||
// ===========================================================================
|
||||
// 2. Delta Chain Lifecycle Test
|
||||
// ===========================================================================
|
||||
|
||||
/// Build a delta chain with 5 incremental deltas, reconstruct, compact,
|
||||
/// verify encode/decode roundtrip.
|
||||
#[test]
|
||||
fn test_delta_chain_lifecycle() {
|
||||
let n = 256;
|
||||
let mut rng = SimpleRng::new(99);
|
||||
let base: Vec<f32> = (0..n).map(|_| rng.next_f32() * 2.0 - 1.0).collect();
|
||||
let mut chain = DeltaChain::new(base.clone(), 8);
|
||||
|
||||
// Build 5 incremental deltas (~10% change each).
|
||||
let mut current = base.clone();
|
||||
for epoch in 0..5u64 {
|
||||
let mut next = current.clone();
|
||||
for i in 0..n {
|
||||
if (rng.next_u64() % 10) == 0 {
|
||||
next[i] += (rng.next_f32() - 0.5) * 0.1;
|
||||
}
|
||||
}
|
||||
let delta = compute_delta(¤t, &next, 1, 0, epoch, 0.001, 0.5)
|
||||
.expect("delta should be computable for ~10% change");
|
||||
chain.append(delta).unwrap();
|
||||
current = next;
|
||||
}
|
||||
assert_eq!(chain.chain_len(), 5);
|
||||
|
||||
// Reconstruct and verify accuracy against the final state.
|
||||
let reconstructed = chain.reconstruct();
|
||||
assert_eq!(reconstructed.len(), n);
|
||||
for i in 0..n {
|
||||
let err = (reconstructed[i] - current[i]).abs();
|
||||
assert!(
|
||||
err < 0.01,
|
||||
"recon err at {}: {} vs {} (err={})",
|
||||
i,
|
||||
reconstructed[i],
|
||||
current[i],
|
||||
err
|
||||
);
|
||||
}
|
||||
|
||||
// Encode/decode the last delta and verify roundtrip.
|
||||
let last_delta = compute_delta(&base, ¤t, 1, 0, 99, 0.001, 1.1).unwrap();
|
||||
let encoded = encode_delta(&last_delta);
|
||||
let decoded = decode_delta(&encoded).unwrap();
|
||||
assert_eq!(decoded.header.tensor_id, 1);
|
||||
assert_eq!(decoded.entries.len(), last_delta.entries.len());
|
||||
|
||||
// Compact the chain; delta list drops to 0 but state is preserved.
|
||||
let before_compact = reconstructed.clone();
|
||||
chain.compact();
|
||||
assert_eq!(chain.chain_len(), 0);
|
||||
|
||||
let after_compact = chain.reconstruct();
|
||||
for i in 0..n {
|
||||
let err = (after_compact[i] - before_compact[i]).abs();
|
||||
assert!(
|
||||
err < 1e-6,
|
||||
"compact mismatch at {}: {} vs {}",
|
||||
i,
|
||||
after_compact[i],
|
||||
before_compact[i]
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// ===========================================================================
|
||||
// 3. Quantization Quality Sweep
|
||||
// ===========================================================================
|
||||
|
||||
/// For each bit width (8, 7, 5, 3) verify MSE and max relative error
|
||||
/// stay within ADR-023 bounds.
|
||||
#[test]
|
||||
fn test_quality_sweep_all_tiers() {
|
||||
let n_elems = 256;
|
||||
let mut rng = SimpleRng::new(7777);
|
||||
|
||||
// Sinusoidal + noise with guaranteed minimum magnitude.
|
||||
let data: Vec<f32> = (0..n_elems)
|
||||
.map(|i| {
|
||||
let base = (i as f32 * 0.05).sin();
|
||||
let noise = (rng.next_f32() - 0.5) * 0.1;
|
||||
let val = base + noise;
|
||||
if val.abs() < 0.05 {
|
||||
if val >= 0.0 {
|
||||
0.05 + rng.next_f32() * 0.1
|
||||
} else {
|
||||
-0.05 - rng.next_f32() * 0.1
|
||||
}
|
||||
} else {
|
||||
val
|
||||
}
|
||||
})
|
||||
.collect();
|
||||
|
||||
let max_abs: f32 = data.iter().map(|v| v.abs()).fold(0.0f32, f32::max);
|
||||
|
||||
// Store-backed tiers: (tier, bound_vs_max, label).
|
||||
let store_configs: &[(Tier, f64, &str)] = &[
|
||||
(Tier::Tier1, 0.01, "8-bit/Tier1"),
|
||||
(Tier::Tier2, 0.02, "7-bit/Tier2"),
|
||||
(Tier::Tier3, 0.35, "3-bit/Tier3"),
|
||||
];
|
||||
|
||||
let mut store = TieredStore::new(4096);
|
||||
for &(tier, bound, label) in store_configs {
|
||||
let key = make_key(tier as u128 + 100, 0);
|
||||
store.put(key, &data, tier, 0).unwrap();
|
||||
|
||||
let mut out = vec![0.0f32; n_elems];
|
||||
let n = store.get(key, &mut out, 0).unwrap();
|
||||
assert_eq!(n, n_elems);
|
||||
|
||||
let mut max_rel = 0.0f64;
|
||||
let mut mse = 0.0f64;
|
||||
for i in 0..n_elems {
|
||||
let err = (data[i] - out[i]) as f64;
|
||||
mse += err * err;
|
||||
let rel = err.abs() / max_abs as f64;
|
||||
if rel > max_rel {
|
||||
max_rel = rel;
|
||||
}
|
||||
}
|
||||
mse /= n_elems as f64;
|
||||
|
||||
assert!(
|
||||
max_rel < bound,
|
||||
"{}: max_rel {:.4} >= bound {:.4} (MSE={:.8})",
|
||||
label,
|
||||
max_rel,
|
||||
bound,
|
||||
mse
|
||||
);
|
||||
}
|
||||
|
||||
// 5-bit via groupwise quantizer directly (no store tier for 5-bit).
|
||||
{
|
||||
let scales = quantizer::compute_scales(&data, 64, 5);
|
||||
let mut packed = Vec::new();
|
||||
quantizer::quantize_and_pack(&data, &scales, 64, 5, &mut packed);
|
||||
let mut decoded = Vec::new();
|
||||
quantizer::dequantize(&packed, &scales, 64, 5, n_elems, 1, &mut decoded);
|
||||
|
||||
let mut max_rel = 0.0f64;
|
||||
for i in 0..n_elems {
|
||||
let err = (data[i] - decoded[i]) as f64;
|
||||
let rel = err.abs() / max_abs as f64;
|
||||
if rel > max_rel {
|
||||
max_rel = rel;
|
||||
}
|
||||
}
|
||||
assert!(max_rel < 0.07, "5-bit: max_rel {:.4} >= 0.07", max_rel);
|
||||
}
|
||||
}
|
||||
|
||||
// ===========================================================================
|
||||
// 4. Store Persistence Roundtrip
|
||||
// ===========================================================================
|
||||
|
||||
/// Put 50 blocks with varied data and tiers, get each back and verify data
|
||||
/// and metadata.
|
||||
#[test]
|
||||
fn test_store_put_get_roundtrip() {
|
||||
let mut store = TieredStore::new(4096);
|
||||
let mut rng = SimpleRng::new(1234);
|
||||
let n_elems = 64;
|
||||
let tiers = [Tier::Tier1, Tier::Tier2, Tier::Tier3];
|
||||
|
||||
let mut block_data: Vec<Vec<f32>> = Vec::new();
|
||||
let mut block_tiers: Vec<Tier> = Vec::new();
|
||||
|
||||
for i in 0..50u32 {
|
||||
let d: Vec<f32> = (0..n_elems).map(|_| rng.next_f32() * 2.0 - 1.0).collect();
|
||||
let tier = tiers[(i % 3) as usize];
|
||||
store.put(make_key(42, i), &d, tier, i as u64).unwrap();
|
||||
block_data.push(d);
|
||||
block_tiers.push(tier);
|
||||
}
|
||||
assert_eq!(store.block_count(), 50);
|
||||
|
||||
for i in 0..50u32 {
|
||||
let key = make_key(42, i);
|
||||
let mut out = vec![0.0f32; n_elems];
|
||||
let n = store.get(key, &mut out, i as u64).unwrap();
|
||||
assert_eq!(n, n_elems);
|
||||
|
||||
let meta = store.meta(key).unwrap();
|
||||
assert_eq!(meta.tier, block_tiers[i as usize]);
|
||||
assert_eq!(meta.created_at, i as u64);
|
||||
|
||||
let max_abs: f32 = block_data[i as usize]
|
||||
.iter()
|
||||
.map(|v| v.abs())
|
||||
.fold(0.0f32, f32::max);
|
||||
let tol = match block_tiers[i as usize] {
|
||||
Tier::Tier1 => max_abs * 0.01,
|
||||
Tier::Tier2 => max_abs * 0.02,
|
||||
Tier::Tier3 => max_abs * 0.35,
|
||||
Tier::Tier0 => unreachable!(),
|
||||
}
|
||||
.max(1e-6);
|
||||
|
||||
for j in 0..n_elems {
|
||||
let err = (block_data[i as usize][j] - out[j]).abs();
|
||||
assert!(err < tol, "block {} elem {}: err={} tol={}", i, j, err, tol);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ===========================================================================
|
||||
// 5. Eviction and Tier0
|
||||
// ===========================================================================
|
||||
|
||||
/// Put a block at Tier1, evict it, verify reads fail and metadata reflects
|
||||
/// eviction state.
|
||||
#[test]
|
||||
fn test_eviction_to_tier0() {
|
||||
let mut store = TieredStore::new(4096);
|
||||
let key = make_key(1, 0);
|
||||
let data = vec![1.0f32; 64];
|
||||
|
||||
store.put(key, &data, Tier::Tier1, 0).unwrap();
|
||||
assert_eq!(store.tier_count(Tier::Tier1), 1);
|
||||
assert!(store.total_bytes() > 0);
|
||||
|
||||
store.evict(key, ReconstructPolicy::None).unwrap();
|
||||
|
||||
// Read should fail.
|
||||
let mut out = vec![0.0f32; 64];
|
||||
assert_eq!(store.get(key, &mut out, 1), Err(StoreError::TensorEvicted));
|
||||
|
||||
// Metadata should reflect Tier0.
|
||||
let meta = store.meta(key).unwrap();
|
||||
assert_eq!(meta.tier, Tier::Tier0);
|
||||
assert_eq!(meta.bits, 0);
|
||||
assert_eq!(meta.block_bytes, 0);
|
||||
assert_eq!(meta.reconstruct, ReconstructPolicy::None);
|
||||
|
||||
assert_eq!(store.tier_count(Tier::Tier1), 0);
|
||||
assert_eq!(store.tier_count(Tier::Tier0), 1);
|
||||
assert_eq!(store.block_count(), 1);
|
||||
assert_eq!(store.total_bytes(), 0);
|
||||
}
|
||||
|
||||
// ===========================================================================
|
||||
// 6. Checksum Integrity
|
||||
// ===========================================================================
|
||||
|
||||
/// Verify that checksums are non-zero and deterministic for the same data.
|
||||
#[test]
|
||||
fn test_checksum_integrity() {
|
||||
let mut store = TieredStore::new(4096);
|
||||
let data: Vec<f32> = (0..128).map(|i| (i as f32) * 0.1).collect();
|
||||
|
||||
let key1 = make_key(1, 0);
|
||||
store.put(key1, &data, Tier::Tier1, 0).unwrap();
|
||||
let cksum1 = store.meta(key1).unwrap().checksum;
|
||||
assert_ne!(
|
||||
cksum1, 0,
|
||||
"checksum should be non-zero for non-trivial data"
|
||||
);
|
||||
|
||||
// Same data under a different key produces the same checksum.
|
||||
let key2 = make_key(1, 1);
|
||||
store.put(key2, &data, Tier::Tier1, 0).unwrap();
|
||||
assert_eq!(store.meta(key2).unwrap().checksum, cksum1);
|
||||
|
||||
// Different data produces a different checksum.
|
||||
let other: Vec<f32> = (0..128).map(|i| (i as f32) * 0.2).collect();
|
||||
let key3 = make_key(1, 2);
|
||||
store.put(key3, &other, Tier::Tier1, 0).unwrap();
|
||||
assert_ne!(store.meta(key3).unwrap().checksum, cksum1);
|
||||
}
|
||||
|
||||
// ===========================================================================
|
||||
// 7. Multi-Tensor Store
|
||||
// ===========================================================================
|
||||
|
||||
/// Blocks from 3 different tensor_ids are stored and retrieved independently.
|
||||
#[test]
|
||||
fn test_multiple_tensors() {
|
||||
let mut store = TieredStore::new(4096);
|
||||
let n_elems = 32;
|
||||
let mut rng = SimpleRng::new(555);
|
||||
|
||||
let tensor_ids: [u128; 3] = [100, 200, 300];
|
||||
let mut all_data: Vec<Vec<Vec<f32>>> = Vec::new();
|
||||
|
||||
for &tid in &tensor_ids {
|
||||
let mut tensor_blocks = Vec::new();
|
||||
for blk in 0..5u32 {
|
||||
let d: Vec<f32> = (0..n_elems).map(|_| rng.next_f32() * 2.0 - 1.0).collect();
|
||||
store.put(make_key(tid, blk), &d, Tier::Tier1, 0).unwrap();
|
||||
tensor_blocks.push(d);
|
||||
}
|
||||
all_data.push(tensor_blocks);
|
||||
}
|
||||
assert_eq!(store.block_count(), 15);
|
||||
|
||||
for (t_idx, &tid) in tensor_ids.iter().enumerate() {
|
||||
for blk in 0..5u32 {
|
||||
let key = make_key(tid, blk);
|
||||
let mut out = vec![0.0f32; n_elems];
|
||||
let n = store.get(key, &mut out, 0).unwrap();
|
||||
assert_eq!(n, n_elems);
|
||||
|
||||
let meta = store.meta(key).unwrap();
|
||||
assert_eq!(meta.key.tensor_id, tid);
|
||||
assert_eq!(meta.key.block_index, blk);
|
||||
|
||||
let orig = &all_data[t_idx][blk as usize];
|
||||
let max_abs: f32 = orig.iter().map(|v| v.abs()).fold(0.0f32, f32::max);
|
||||
let tol = (max_abs * 0.01).max(1e-6);
|
||||
for j in 0..n_elems {
|
||||
let err = (orig[j] - out[j]).abs();
|
||||
assert!(err < tol, "tid={} blk={} j={}: err={}", tid, blk, j, err);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ===========================================================================
|
||||
// 8. Stress Test
|
||||
// ===========================================================================
|
||||
|
||||
/// Put 1000 blocks with random tiers, touch random blocks 10000 times,
|
||||
/// verify no panics and all blocks remain readable.
|
||||
#[test]
|
||||
fn test_stress_1000_blocks() {
|
||||
let mut store = TieredStore::new(4096);
|
||||
let mut rng = SimpleRng::new(0xDEADBEEF);
|
||||
let n_elems = 32;
|
||||
let tiers = [Tier::Tier1, Tier::Tier2, Tier::Tier3];
|
||||
|
||||
for i in 0..1000u32 {
|
||||
let d: Vec<f32> = (0..n_elems).map(|_| rng.next_f32() * 2.0 - 1.0).collect();
|
||||
let tier = tiers[(rng.next_u64() % 3) as usize];
|
||||
store.put(make_key(1, i), &d, tier, i as u64).unwrap();
|
||||
}
|
||||
assert_eq!(store.block_count(), 1000);
|
||||
assert!(store.total_bytes() > 0);
|
||||
|
||||
for t in 0..10_000u64 {
|
||||
let idx = (rng.next_u64() % 1000) as u32;
|
||||
store.touch(make_key(1, idx), 1000 + t);
|
||||
}
|
||||
|
||||
for i in 0..1000u32 {
|
||||
let mut out = vec![0.0f32; n_elems];
|
||||
let n = store.get(make_key(1, i), &mut out, 20_000).unwrap();
|
||||
assert_eq!(n, n_elems);
|
||||
for j in 0..n_elems {
|
||||
assert!(out[j].is_finite(), "block {} elem {} not finite", i, j);
|
||||
}
|
||||
}
|
||||
assert!(store.total_bytes() > 0);
|
||||
}
|
||||
|
||||
// ===========================================================================
|
||||
// 9. Compressor + Store Integration
|
||||
// ===========================================================================
|
||||
|
||||
/// Compress frames via TemporalTensorCompressor, decode the segment, store
|
||||
/// each decoded frame as a block, and verify roundtrip.
|
||||
#[test]
|
||||
fn test_compressor_to_store() {
|
||||
let tensor_len = 128u32;
|
||||
let policy = TierPolicy::default();
|
||||
let mut comp = TemporalTensorCompressor::new(policy, tensor_len, 0);
|
||||
comp.set_access(100, 0); // hot -> 8-bit
|
||||
|
||||
let mut rng = SimpleRng::new(0xCAFE);
|
||||
let n_frames = 10usize;
|
||||
|
||||
let frames: Vec<Vec<f32>> = (0..n_frames)
|
||||
.map(|_| {
|
||||
(0..tensor_len as usize)
|
||||
.map(|_| rng.next_f32() * 2.0 - 1.0)
|
||||
.collect()
|
||||
})
|
||||
.collect();
|
||||
|
||||
let mut seg = Vec::new();
|
||||
for (i, frame) in frames.iter().enumerate() {
|
||||
comp.push_frame(frame, (i + 1) as u32, &mut seg);
|
||||
}
|
||||
comp.flush(&mut seg);
|
||||
assert!(!seg.is_empty(), "compressor should produce a segment");
|
||||
|
||||
let mut decoded = Vec::new();
|
||||
segment::decode(&seg, &mut decoded);
|
||||
assert_eq!(decoded.len(), tensor_len as usize * n_frames);
|
||||
|
||||
// Store each decoded frame as a block.
|
||||
let mut store = TieredStore::new(4096);
|
||||
for i in 0..n_frames {
|
||||
let start = i * tensor_len as usize;
|
||||
let end = start + tensor_len as usize;
|
||||
store
|
||||
.put(
|
||||
make_key(50, i as u32),
|
||||
&decoded[start..end],
|
||||
Tier::Tier1,
|
||||
i as u64,
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
assert_eq!(store.block_count(), n_frames);
|
||||
|
||||
// Read back and verify against the decoded data (double quantization).
|
||||
for i in 0..n_frames {
|
||||
let mut out = vec![0.0f32; tensor_len as usize];
|
||||
let n = store
|
||||
.get(make_key(50, i as u32), &mut out, n_frames as u64)
|
||||
.unwrap();
|
||||
assert_eq!(n, tensor_len as usize);
|
||||
|
||||
let start = i * tensor_len as usize;
|
||||
for j in 0..tensor_len as usize {
|
||||
let expected = decoded[start + j];
|
||||
let err = (expected - out[j]).abs();
|
||||
// Double quantization (compressor + store) compounds error.
|
||||
let tol = if expected.abs() > 0.01 {
|
||||
expected.abs() * 0.04
|
||||
} else {
|
||||
0.05
|
||||
};
|
||||
assert!(
|
||||
err < tol,
|
||||
"frame {} elem {}: exp={} got={} err={}",
|
||||
i,
|
||||
j,
|
||||
expected,
|
||||
out[j],
|
||||
err
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ===========================================================================
|
||||
// 10. Factor Reconstruction Quality
|
||||
// ===========================================================================
|
||||
|
||||
/// Create a low-rank matrix, factor it, reconstruct, and verify error is low.
|
||||
#[test]
|
||||
fn test_factor_reconstruction_quality() {
|
||||
let m = 16;
|
||||
let n = 16;
|
||||
|
||||
// Rank-1 matrix: data[i][j] = (i+1)*(j+1) / (m*n).
|
||||
let data: Vec<f32> = (0..m * n)
|
||||
.map(|idx| {
|
||||
let (i, j) = (idx / n, idx % n);
|
||||
(i as f32 + 1.0) * (j as f32 + 1.0) / (m * n) as f32
|
||||
})
|
||||
.collect();
|
||||
|
||||
let factors = FactorSet::from_data(&data, m, n, 1);
|
||||
assert_eq!(factors.m, m);
|
||||
assert_eq!(factors.n, n);
|
||||
assert_eq!(factors.k, 1);
|
||||
|
||||
let reconstructed = factors.reconstruct();
|
||||
assert_eq!(reconstructed.len(), m * n);
|
||||
|
||||
let max_abs: f32 = data.iter().map(|v| v.abs()).fold(0.0f32, f32::max);
|
||||
let mut max_err = 0.0f32;
|
||||
for i in 0..m * n {
|
||||
let err = (data[i] - reconstructed[i]).abs();
|
||||
if err > max_err {
|
||||
max_err = err;
|
||||
}
|
||||
}
|
||||
|
||||
assert!(
|
||||
max_err < max_abs * 0.01,
|
||||
"factor reconstruction error too high: max_err={} (max_abs={})",
|
||||
max_err,
|
||||
max_abs
|
||||
);
|
||||
|
||||
// Factor storage should be smaller than the full matrix.
|
||||
assert!(factors.storage_bytes() > 0);
|
||||
assert!(
|
||||
factors.storage_bytes() < m * n * 4,
|
||||
"factor storage {} should be < original {}",
|
||||
factors.storage_bytes(),
|
||||
m * n * 4
|
||||
);
|
||||
}
|
||||
|
||||
// ===========================================================================
|
||||
// 11. Witness Logging Integration
|
||||
// ===========================================================================
|
||||
|
||||
/// Record access, tier-change, and eviction events; verify counters and
|
||||
/// flip-rate calculation.
|
||||
#[test]
|
||||
fn test_witness_logging() {
|
||||
let mut log = WitnessLog::new(256);
|
||||
let mut store = TieredStore::new(4096);
|
||||
|
||||
let key = make_key(1, 0);
|
||||
store.put(key, &vec![1.0f32; 64], Tier::Tier1, 0).unwrap();
|
||||
|
||||
log.record(
|
||||
0,
|
||||
WitnessEvent::Access {
|
||||
key,
|
||||
score: 0.95,
|
||||
tier: Tier::Tier1,
|
||||
},
|
||||
);
|
||||
log.record(
|
||||
100,
|
||||
WitnessEvent::TierChange {
|
||||
key,
|
||||
from_tier: Tier::Tier1,
|
||||
to_tier: Tier::Tier2,
|
||||
score: 0.45,
|
||||
reason: TierChangeReason::ScoreDowngrade,
|
||||
},
|
||||
);
|
||||
|
||||
store.evict(key, ReconstructPolicy::None).unwrap();
|
||||
log.record(
|
||||
200,
|
||||
WitnessEvent::Eviction {
|
||||
key,
|
||||
score: 0.05,
|
||||
bytes_freed: 64,
|
||||
},
|
||||
);
|
||||
|
||||
assert_eq!(log.len(), 3);
|
||||
assert_eq!(log.count_tier_changes(), 1);
|
||||
assert_eq!(log.count_evictions(), 1);
|
||||
assert_eq!(log.count_checksum_failures(), 0);
|
||||
|
||||
let recent = log.recent(2);
|
||||
assert_eq!(recent.len(), 2);
|
||||
assert_eq!(recent[0].timestamp, 100);
|
||||
assert_eq!(recent[1].timestamp, 200);
|
||||
|
||||
// One tier change across 1 block in the window = flip rate 1.0.
|
||||
let rate = log.tier_flip_rate(300, 1);
|
||||
assert!(
|
||||
(rate - 1.0).abs() < 1e-6,
|
||||
"expected flip rate 1.0, got {}",
|
||||
rate
|
||||
);
|
||||
}
|
||||
225
crates/ruvector-temporal-tensor/tests/persistence_tests.rs
Normal file
225
crates/ruvector-temporal-tensor/tests/persistence_tests.rs
Normal file
@@ -0,0 +1,225 @@
|
||||
#![cfg(feature = "persistence")]
|
||||
|
||||
use ruvector_temporal_tensor::persistence::{FileBlockIO, FileMetaLog};
|
||||
use ruvector_temporal_tensor::store::{
|
||||
BlockIO, BlockKey, BlockMeta, DType, MetaLog, ReconstructPolicy, Tier,
|
||||
};
|
||||
use std::path::PathBuf;
|
||||
|
||||
fn test_dir(name: &str) -> PathBuf {
|
||||
let dir = std::env::temp_dir().join(format!("ruvector_test_{}", name));
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
std::fs::create_dir_all(&dir).unwrap();
|
||||
dir
|
||||
}
|
||||
|
||||
fn cleanup(dir: &PathBuf) {
|
||||
let _ = std::fs::remove_dir_all(dir);
|
||||
}
|
||||
|
||||
fn make_key(id: u128, idx: u32) -> BlockKey {
|
||||
BlockKey {
|
||||
tensor_id: id,
|
||||
block_index: idx,
|
||||
}
|
||||
}
|
||||
|
||||
fn make_meta(key: BlockKey, tier: Tier) -> BlockMeta {
|
||||
BlockMeta {
|
||||
key,
|
||||
dtype: DType::F32,
|
||||
tier,
|
||||
bits: 8,
|
||||
scale: 0.5,
|
||||
zero_point: 0,
|
||||
created_at: 100,
|
||||
last_access_at: 200,
|
||||
access_count: 5,
|
||||
ema_rate: 0.1,
|
||||
window: 0xFF,
|
||||
checksum: 0xDEADBEEF,
|
||||
reconstruct: ReconstructPolicy::None,
|
||||
tier_age: 10,
|
||||
lineage_parent: None,
|
||||
block_bytes: 64,
|
||||
}
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// FileBlockIO tests
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
#[test]
|
||||
fn test_file_block_io_write_read() {
|
||||
let dir = test_dir("block_io_write_read");
|
||||
let mut bio = FileBlockIO::new(&dir).unwrap();
|
||||
|
||||
let key = make_key(1, 0);
|
||||
let data = vec![0xAB, 0xCD, 0xEF, 0x01, 0x23, 0x45, 0x67, 0x89];
|
||||
bio.write_block(Tier::Tier1, key, &data).unwrap();
|
||||
|
||||
let mut dst = vec![0u8; 32];
|
||||
let n = bio.read_block(Tier::Tier1, key, &mut dst).unwrap();
|
||||
assert_eq!(n, data.len());
|
||||
assert_eq!(&dst[..n], &data[..]);
|
||||
|
||||
cleanup(&dir);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_file_block_io_different_tiers() {
|
||||
let dir = test_dir("block_io_tiers");
|
||||
let mut bio = FileBlockIO::new(&dir).unwrap();
|
||||
|
||||
let key = make_key(1, 0);
|
||||
let data1 = vec![1u8; 16];
|
||||
let data2 = vec![2u8; 8];
|
||||
let data3 = vec![3u8; 4];
|
||||
|
||||
bio.write_block(Tier::Tier1, key, &data1).unwrap();
|
||||
bio.write_block(Tier::Tier2, key, &data2).unwrap();
|
||||
bio.write_block(Tier::Tier3, key, &data3).unwrap();
|
||||
|
||||
let mut buf = vec![0u8; 32];
|
||||
|
||||
let n1 = bio.read_block(Tier::Tier1, key, &mut buf).unwrap();
|
||||
assert_eq!(&buf[..n1], &data1[..]);
|
||||
|
||||
let n2 = bio.read_block(Tier::Tier2, key, &mut buf).unwrap();
|
||||
assert_eq!(&buf[..n2], &data2[..]);
|
||||
|
||||
let n3 = bio.read_block(Tier::Tier3, key, &mut buf).unwrap();
|
||||
assert_eq!(&buf[..n3], &data3[..]);
|
||||
|
||||
cleanup(&dir);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_file_block_io_delete() {
|
||||
let dir = test_dir("block_io_delete");
|
||||
let mut bio = FileBlockIO::new(&dir).unwrap();
|
||||
|
||||
let key = make_key(1, 0);
|
||||
bio.write_block(Tier::Tier1, key, &[1, 2, 3]).unwrap();
|
||||
bio.delete_block(Tier::Tier1, key).unwrap();
|
||||
|
||||
let mut buf = vec![0u8; 32];
|
||||
let result = bio.read_block(Tier::Tier1, key, &mut buf);
|
||||
assert!(result.is_err() || result.unwrap() == 0);
|
||||
|
||||
cleanup(&dir);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_file_block_io_overwrite() {
|
||||
let dir = test_dir("block_io_overwrite");
|
||||
let mut bio = FileBlockIO::new(&dir).unwrap();
|
||||
|
||||
let key = make_key(1, 0);
|
||||
bio.write_block(Tier::Tier1, key, &[1, 2, 3]).unwrap();
|
||||
bio.write_block(Tier::Tier1, key, &[4, 5, 6, 7]).unwrap();
|
||||
|
||||
let mut buf = vec![0u8; 32];
|
||||
let n = bio.read_block(Tier::Tier1, key, &mut buf).unwrap();
|
||||
assert_eq!(&buf[..n], &[4, 5, 6, 7]);
|
||||
|
||||
cleanup(&dir);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_file_block_io_missing_key() {
|
||||
let dir = test_dir("block_io_missing");
|
||||
let bio = FileBlockIO::new(&dir).unwrap();
|
||||
|
||||
let mut buf = vec![0u8; 32];
|
||||
let result = bio.read_block(Tier::Tier1, make_key(99, 0), &mut buf);
|
||||
assert!(result.is_err() || result.unwrap() == 0);
|
||||
|
||||
cleanup(&dir);
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// FileMetaLog tests
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
#[test]
|
||||
fn test_file_meta_log_append_get() {
|
||||
let dir = test_dir("meta_log_append");
|
||||
let mut log = FileMetaLog::new(&dir).unwrap();
|
||||
|
||||
let key = make_key(1, 0);
|
||||
let meta = make_meta(key, Tier::Tier1);
|
||||
log.append(&meta).unwrap();
|
||||
|
||||
let retrieved = log.get(key).unwrap();
|
||||
assert_eq!(retrieved.key, key);
|
||||
assert_eq!(retrieved.tier, Tier::Tier1);
|
||||
assert_eq!(retrieved.bits, 8);
|
||||
assert!((retrieved.scale - 0.5).abs() < 1e-6);
|
||||
assert_eq!(retrieved.checksum, 0xDEADBEEF);
|
||||
|
||||
cleanup(&dir);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_file_meta_log_upsert() {
|
||||
let dir = test_dir("meta_log_upsert");
|
||||
let mut log = FileMetaLog::new(&dir).unwrap();
|
||||
|
||||
let key = make_key(1, 0);
|
||||
let meta1 = make_meta(key, Tier::Tier1);
|
||||
log.append(&meta1).unwrap();
|
||||
|
||||
let mut meta2 = make_meta(key, Tier::Tier2);
|
||||
meta2.bits = 7;
|
||||
log.append(&meta2).unwrap();
|
||||
|
||||
let retrieved = log.get(key).unwrap();
|
||||
assert_eq!(retrieved.tier, Tier::Tier2);
|
||||
assert_eq!(retrieved.bits, 7);
|
||||
|
||||
cleanup(&dir);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_file_meta_log_iter() {
|
||||
let dir = test_dir("meta_log_iter");
|
||||
let mut log = FileMetaLog::new(&dir).unwrap();
|
||||
|
||||
for i in 0..5u128 {
|
||||
let key = make_key(i, 0);
|
||||
log.append(&make_meta(key, Tier::Tier1)).unwrap();
|
||||
}
|
||||
|
||||
let count = log.iter().count();
|
||||
assert_eq!(count, 5);
|
||||
|
||||
cleanup(&dir);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_file_meta_log_missing_key() {
|
||||
let dir = test_dir("meta_log_missing");
|
||||
let log = FileMetaLog::new(&dir).unwrap();
|
||||
assert!(log.get(make_key(99, 0)).is_none());
|
||||
|
||||
cleanup(&dir);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_file_meta_log_multiple_blocks_same_tensor() {
|
||||
let dir = test_dir("meta_log_multi_block");
|
||||
let mut log = FileMetaLog::new(&dir).unwrap();
|
||||
|
||||
for idx in 0..3u32 {
|
||||
let key = make_key(1, idx);
|
||||
log.append(&make_meta(key, Tier::Tier1)).unwrap();
|
||||
}
|
||||
|
||||
assert!(log.get(make_key(1, 0)).is_some());
|
||||
assert!(log.get(make_key(1, 1)).is_some());
|
||||
assert!(log.get(make_key(1, 2)).is_some());
|
||||
assert!(log.get(make_key(1, 3)).is_none());
|
||||
|
||||
cleanup(&dir);
|
||||
}
|
||||
821
crates/ruvector-temporal-tensor/tests/property_tests.rs
Normal file
821
crates/ruvector-temporal-tensor/tests/property_tests.rs
Normal file
@@ -0,0 +1,821 @@
|
||||
//! Property-based roundtrip tests for temporal tensor compression.
|
||||
//!
|
||||
//! Verifies quantization roundtrip correctness across many random inputs
|
||||
//! using a deterministic PRNG. No external dependencies.
|
||||
//!
|
||||
//! Run with:
|
||||
//! ```sh
|
||||
//! cargo test --release -p ruvector-temporal-tensor --test property_tests -- --nocapture
|
||||
//! ```
|
||||
|
||||
use ruvector_temporal_tensor::bitpack;
|
||||
use ruvector_temporal_tensor::delta;
|
||||
use ruvector_temporal_tensor::f16;
|
||||
use ruvector_temporal_tensor::quantizer;
|
||||
use ruvector_temporal_tensor::segment;
|
||||
use ruvector_temporal_tensor::tiering::{self, BlockMeta, TierConfig};
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Deterministic PRNG (LCG) -- no external deps
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Simple linear congruential generator. Constants from Knuth MMIX.
|
||||
struct SimpleRng {
|
||||
state: u64,
|
||||
}
|
||||
|
||||
impl SimpleRng {
|
||||
fn new(seed: u64) -> Self {
|
||||
Self { state: seed }
|
||||
}
|
||||
|
||||
fn next_u64(&mut self) -> u64 {
|
||||
self.state = self
|
||||
.state
|
||||
.wrapping_mul(6364136223846793005)
|
||||
.wrapping_add(1442695040888963407);
|
||||
self.state
|
||||
}
|
||||
|
||||
fn next_f32(&mut self) -> f32 {
|
||||
(self.next_u64() >> 40) as f32 / (1u64 << 24) as f32
|
||||
}
|
||||
|
||||
fn next_f32_range(&mut self, lo: f32, hi: f32) -> f32 {
|
||||
lo + self.next_f32() * (hi - lo)
|
||||
}
|
||||
|
||||
fn next_usize_range(&mut self, lo: usize, hi: usize) -> usize {
|
||||
let range = (hi - lo) as u64;
|
||||
if range == 0 {
|
||||
return lo;
|
||||
}
|
||||
lo + (self.next_u64() % range) as usize
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Helpers
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const GROUP_LEN: usize = 64;
|
||||
|
||||
/// Generate a random f32 vector of the given length with values in [lo, hi].
|
||||
fn random_vec(rng: &mut SimpleRng, len: usize, lo: f32, hi: f32) -> Vec<f32> {
|
||||
(0..len).map(|_| rng.next_f32_range(lo, hi)).collect()
|
||||
}
|
||||
|
||||
/// Compute group-level maximum absolute values for error bounding.
|
||||
fn group_max_abs(frame: &[f32], group_len: usize) -> Vec<f32> {
|
||||
frame
|
||||
.chunks(group_len)
|
||||
.map(|chunk| {
|
||||
chunk
|
||||
.iter()
|
||||
.filter(|v| v.is_finite())
|
||||
.map(|v| v.abs())
|
||||
.fold(0.0f32, f32::max)
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 1. Quantize/Dequant Roundtrip Property
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[test]
|
||||
fn prop_roundtrip_error_bounded() {
|
||||
let mut rng = SimpleRng::new(0xDEAD_BEEF_CAFE_BABE);
|
||||
|
||||
// Error bounds as fraction of each group's max absolute value.
|
||||
// The absolute error per element is bounded by:
|
||||
// scale * 1 (one quantization step) + f16 rounding (~0.1% of scale)
|
||||
// where scale = group_max_abs / qmax. So the error fraction of group_max is
|
||||
// approximately 1/qmax + small f16 term.
|
||||
// 8-bit: qmax=127, ~0.8% + margin -> 1%
|
||||
// 7-bit: qmax=63, ~1.6% + margin -> 2%
|
||||
// 5-bit: qmax=15, ~6.7% + margin -> 7%
|
||||
// 3-bit: qmax=3, ~33% + margin -> 35%
|
||||
let bit_configs: &[(u8, f32)] = &[
|
||||
(8, 0.01), // 8-bit: < 1% of group max
|
||||
(7, 0.02), // 7-bit: < 2% of group max
|
||||
(5, 0.07), // 5-bit: < 7% of group max
|
||||
(3, 0.35), // 3-bit: < 35% of group max
|
||||
];
|
||||
|
||||
for trial in 0..1000 {
|
||||
let len = rng.next_usize_range(64, 513); // 64..512 inclusive
|
||||
let frame = random_vec(&mut rng, len, -10.0, 10.0);
|
||||
|
||||
for &(bits, max_err_frac) in bit_configs {
|
||||
let scales = quantizer::compute_scales(&frame, GROUP_LEN, bits);
|
||||
let scales_f32 = quantizer::scales_to_f32(&scales);
|
||||
|
||||
let mut packed = Vec::new();
|
||||
quantizer::quantize_and_pack_f32(&frame, &scales_f32, GROUP_LEN, bits, &mut packed);
|
||||
|
||||
let mut decoded = Vec::new();
|
||||
quantizer::dequantize_f32(
|
||||
&packed,
|
||||
&scales_f32,
|
||||
GROUP_LEN,
|
||||
bits,
|
||||
frame.len(),
|
||||
1,
|
||||
&mut decoded,
|
||||
);
|
||||
|
||||
assert_eq!(
|
||||
decoded.len(),
|
||||
frame.len(),
|
||||
"trial={trial}, bits={bits}: length mismatch"
|
||||
);
|
||||
|
||||
// Compute per-group max absolute value for error bounding.
|
||||
let gmax = group_max_abs(&frame, GROUP_LEN);
|
||||
|
||||
for (i, (&orig, &dec)) in frame.iter().zip(decoded.iter()).enumerate() {
|
||||
let abs_err = (orig - dec).abs();
|
||||
let group_idx = i / GROUP_LEN;
|
||||
let group_m = if group_idx < gmax.len() {
|
||||
gmax[group_idx]
|
||||
} else {
|
||||
1.0
|
||||
};
|
||||
// Bound: max_err_frac * group_max + small absolute floor for near-zero groups.
|
||||
let bound = max_err_frac * group_m + 1e-6;
|
||||
assert!(
|
||||
abs_err <= bound,
|
||||
"trial={trial}, bits={bits}, i={i}: orig={orig}, dec={dec}, \
|
||||
abs_err={abs_err}, bound={bound}, group_max={group_m}"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 2. Bit Packing Roundtrip Property
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[test]
|
||||
fn prop_bitpack_roundtrip() {
|
||||
let mut rng = SimpleRng::new(0x1234_5678_9ABC_DEF0);
|
||||
|
||||
let bit_widths: &[u32] = &[3, 5, 7, 8];
|
||||
|
||||
for _trial in 0..1000 {
|
||||
let count = rng.next_usize_range(1, 513);
|
||||
|
||||
for &bits in bit_widths {
|
||||
let max_val = (1u32 << bits) - 1;
|
||||
let codes: Vec<u32> = (0..count)
|
||||
.map(|_| (rng.next_u64() as u32) % (max_val + 1))
|
||||
.collect();
|
||||
|
||||
let mut packed = Vec::new();
|
||||
bitpack::pack(&codes, bits, &mut packed);
|
||||
|
||||
let mut unpacked = Vec::new();
|
||||
bitpack::unpack(&packed, bits, count, &mut unpacked);
|
||||
|
||||
assert_eq!(
|
||||
codes, unpacked,
|
||||
"bits={bits}, count={count}: pack/unpack mismatch"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 3. Segment Encode/Decode Property
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[test]
|
||||
fn prop_segment_roundtrip() {
|
||||
let mut rng = SimpleRng::new(0xFEED_FACE_DEAD_C0DE);
|
||||
|
||||
let tensor_lens: &[usize] = &[32, 64, 128, 256, 512];
|
||||
let frame_counts: &[usize] = &[1, 2, 5, 10, 20];
|
||||
let bit_widths: &[u8] = &[3, 5, 7, 8];
|
||||
|
||||
for _trial in 0..200 {
|
||||
let tensor_len = tensor_lens[rng.next_usize_range(0, tensor_lens.len())];
|
||||
let frame_count = frame_counts[rng.next_usize_range(0, frame_counts.len())];
|
||||
let bits = bit_widths[rng.next_usize_range(0, bit_widths.len())];
|
||||
|
||||
// Generate the first frame and compute scales from it (shared across frames).
|
||||
let first_frame = random_vec(&mut rng, tensor_len, -5.0, 5.0);
|
||||
let scales = quantizer::compute_scales(&first_frame, GROUP_LEN, bits);
|
||||
let scales_f32 = quantizer::scales_to_f32(&scales);
|
||||
|
||||
// Quantize all frames with the same scales.
|
||||
let mut packed = Vec::new();
|
||||
quantizer::quantize_and_pack_f32(&first_frame, &scales_f32, GROUP_LEN, bits, &mut packed);
|
||||
for _ in 1..frame_count {
|
||||
// Subsequent frames use values within the first frame's range to fit scales.
|
||||
let frame = random_vec(&mut rng, tensor_len, -4.0, 4.0);
|
||||
quantizer::quantize_and_pack_f32(&frame, &scales_f32, GROUP_LEN, bits, &mut packed);
|
||||
}
|
||||
|
||||
// Encode into segment format.
|
||||
let mut seg = Vec::new();
|
||||
segment::encode(
|
||||
bits,
|
||||
GROUP_LEN as u32,
|
||||
tensor_len as u32,
|
||||
frame_count as u32,
|
||||
&scales,
|
||||
&packed,
|
||||
&mut seg,
|
||||
);
|
||||
|
||||
// Decode the segment.
|
||||
let mut decoded = Vec::new();
|
||||
segment::decode(&seg, &mut decoded);
|
||||
|
||||
assert_eq!(
|
||||
decoded.len(),
|
||||
tensor_len * frame_count,
|
||||
"trial={_trial}, bits={bits}, tensor_len={tensor_len}, frames={frame_count}: \
|
||||
decoded length mismatch"
|
||||
);
|
||||
|
||||
// Parse the header and verify metadata.
|
||||
let header = segment::parse_header(&seg).expect("header should parse");
|
||||
assert_eq!(header.bits, bits);
|
||||
assert_eq!(header.tensor_len, tensor_len as u32);
|
||||
assert_eq!(header.frame_count, frame_count as u32);
|
||||
assert_eq!(header.group_len, GROUP_LEN as u32);
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 4. f16 Roundtrip Property
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[test]
|
||||
fn prop_f16_roundtrip() {
|
||||
let mut rng = SimpleRng::new(0xAAAA_BBBB_CCCC_DDDD);
|
||||
|
||||
for _trial in 0..10_000 {
|
||||
// Generate value in scale-relevant range [1e-4, 1e4].
|
||||
let v = rng.next_f32_range(1e-4, 1e4);
|
||||
// Randomly negate half the values.
|
||||
let v = if rng.next_u64() & 1 == 0 { v } else { -v };
|
||||
|
||||
let h = f16::f32_to_f16_bits(v);
|
||||
let back = f16::f16_bits_to_f32(h);
|
||||
|
||||
// f16 has ~0.1% relative error for normal values in this range.
|
||||
let rel_err = ((back - v) / v).abs();
|
||||
assert!(
|
||||
rel_err < 0.002,
|
||||
"trial={_trial}: v={v}, back={back}, rel_err={rel_err}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 5. Delta Compute/Apply Property
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[test]
|
||||
fn prop_delta_apply_recovers_new() {
|
||||
let mut rng = SimpleRng::new(0x0123_4567_89AB_CDEF);
|
||||
|
||||
for trial in 0..500 {
|
||||
let len = rng.next_usize_range(8, 257);
|
||||
let old = random_vec(&mut rng, len, -5.0, 5.0);
|
||||
|
||||
// Create "new" as old with a small number of perturbations.
|
||||
let mut new = old.clone();
|
||||
let num_changes = rng.next_usize_range(1, (len / 4).max(2));
|
||||
for _ in 0..num_changes {
|
||||
let idx = rng.next_usize_range(0, len);
|
||||
new[idx] += rng.next_f32_range(-1.0, 1.0);
|
||||
}
|
||||
|
||||
let threshold = 0.001;
|
||||
let max_change_frac = 0.8;
|
||||
let result =
|
||||
delta::compute_delta(&old, &new, trial as u128, 0, 0, threshold, max_change_frac);
|
||||
|
||||
match result {
|
||||
Some(d) => {
|
||||
// Apply delta to old, verify it approximates new.
|
||||
let mut reconstructed = old.clone();
|
||||
delta::apply_delta(&mut reconstructed, &d);
|
||||
|
||||
for i in 0..len {
|
||||
let err = (reconstructed[i] - new[i]).abs();
|
||||
// Two sources of error:
|
||||
// 1. Entries below threshold are not captured in the delta,
|
||||
// so the reconstruction error for those is up to `threshold`.
|
||||
// 2. Captured entries have i16 quantization error of at most
|
||||
// delta_scale / 2 (half a quantization step).
|
||||
let tolerance = threshold + d.delta_scale * 1.5 + 1e-6;
|
||||
assert!(
|
||||
err <= tolerance,
|
||||
"trial={trial}, i={i}: recon={}, new={}, err={err}, tol={tolerance}",
|
||||
reconstructed[i],
|
||||
new[i]
|
||||
);
|
||||
}
|
||||
}
|
||||
None => {
|
||||
// Delta was too large (>= max_change_fraction).
|
||||
// Verify that indeed many values changed.
|
||||
let changed = old
|
||||
.iter()
|
||||
.zip(new.iter())
|
||||
.filter(|(&o, &n)| (o - n).abs() >= threshold)
|
||||
.count();
|
||||
let fraction = changed as f32 / len as f32;
|
||||
assert!(
|
||||
fraction >= max_change_frac,
|
||||
"trial={trial}: delta was None but change fraction={fraction} < {max_change_frac}"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 6. Compression Ratio Property
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[test]
|
||||
fn prop_compression_ratio_matches_theory() {
|
||||
let mut rng = SimpleRng::new(0xCAFE_D00D_BEEF_FEED);
|
||||
|
||||
let expected: &[(u8, f32)] = &[(8, 3.5), (7, 4.0), (5, 5.5), (3, 8.5)];
|
||||
|
||||
for &(bits, min_ratio) in expected {
|
||||
// Use a 512-element tensor with group_len=64 for consistent measurement.
|
||||
let frame = random_vec(&mut rng, 512, -1.0, 1.0);
|
||||
let scales = quantizer::compute_scales(&frame, GROUP_LEN, bits);
|
||||
let mut packed = Vec::new();
|
||||
quantizer::quantize_and_pack(&frame, &scales, GROUP_LEN, bits, &mut packed);
|
||||
|
||||
let raw_bytes = frame.len() * 4; // f32 = 4 bytes
|
||||
let compressed = packed.len() + scales.len() * 2; // packed data + f16 scales
|
||||
let ratio = raw_bytes as f32 / compressed as f32;
|
||||
|
||||
assert!(
|
||||
ratio >= min_ratio,
|
||||
"bits={bits}: ratio={ratio:.2}x < expected={min_ratio}x \
|
||||
(raw={raw_bytes}, compressed={compressed})"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 7. Score Monotonicity Property
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[test]
|
||||
fn prop_score_monotonic_with_access() {
|
||||
let mut rng = SimpleRng::new(0x7777_8888_9999_AAAA);
|
||||
let config = TierConfig::default();
|
||||
|
||||
for _trial in 0..100 {
|
||||
let start_tick = rng.next_u64() % 1000;
|
||||
let mut meta = BlockMeta::new(start_tick);
|
||||
|
||||
// Score before any touch.
|
||||
let score_before = tiering::compute_score(&config, start_tick, &meta);
|
||||
|
||||
// Touch the block.
|
||||
tiering::touch(&config, start_tick + 1, &mut meta);
|
||||
let score_after_touch = tiering::compute_score(&config, start_tick + 1, &meta);
|
||||
|
||||
// Touching should increase (or at minimum maintain) the score.
|
||||
assert!(
|
||||
score_after_touch >= score_before - 1e-6,
|
||||
"trial={_trial}: score decreased after touch: \
|
||||
before={score_before}, after={score_after_touch}"
|
||||
);
|
||||
|
||||
// Now let time pass without access -- score should decrease.
|
||||
let score_at_touch = tiering::compute_score(&config, start_tick + 1, &meta);
|
||||
let score_later = tiering::compute_score(&config, start_tick + 1000, &meta);
|
||||
|
||||
assert!(
|
||||
score_later <= score_at_touch + 1e-6,
|
||||
"trial={_trial}: score increased without access: \
|
||||
at_touch={score_at_touch}, later={score_later}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 8. Zero Vector Property
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[test]
|
||||
fn prop_zero_vector_roundtrip() {
|
||||
let bit_widths: &[u8] = &[3, 5, 7, 8];
|
||||
|
||||
for &len in &[64, 128, 256, 512] {
|
||||
let frame = vec![0.0f32; len];
|
||||
|
||||
for &bits in bit_widths {
|
||||
let scales = quantizer::compute_scales(&frame, GROUP_LEN, bits);
|
||||
let scales_f32 = quantizer::scales_to_f32(&scales);
|
||||
|
||||
// All scales should be zero for a zero vector.
|
||||
for (i, &s) in scales_f32.iter().enumerate() {
|
||||
assert_eq!(
|
||||
s, 0.0,
|
||||
"len={len}, bits={bits}, group={i}: scale should be 0.0, got {s}"
|
||||
);
|
||||
}
|
||||
|
||||
let mut packed = Vec::new();
|
||||
quantizer::quantize_and_pack_f32(&frame, &scales_f32, GROUP_LEN, bits, &mut packed);
|
||||
|
||||
let mut decoded = Vec::new();
|
||||
quantizer::dequantize_f32(&packed, &scales_f32, GROUP_LEN, bits, len, 1, &mut decoded);
|
||||
|
||||
assert_eq!(decoded.len(), len);
|
||||
for (i, &v) in decoded.iter().enumerate() {
|
||||
assert_eq!(
|
||||
v, 0.0,
|
||||
"len={len}, bits={bits}, i={i}: expected 0.0, got {v}"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 9. Single-Value (Uniform) Vector Property
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[test]
|
||||
fn prop_uniform_vector_roundtrip() {
|
||||
let mut rng = SimpleRng::new(0xBBBB_CCCC_DDDD_EEEE);
|
||||
let bit_widths: &[u8] = &[3, 5, 7, 8];
|
||||
|
||||
for _trial in 0..200 {
|
||||
let len = rng.next_usize_range(64, 513);
|
||||
let value = rng.next_f32_range(-10.0, 10.0);
|
||||
let frame = vec![value; len];
|
||||
|
||||
for &bits in bit_widths {
|
||||
let qmax = bitpack::qmax_from_bits(bits);
|
||||
if qmax == 0 {
|
||||
continue;
|
||||
}
|
||||
|
||||
let scales = quantizer::compute_scales(&frame, GROUP_LEN, bits);
|
||||
let scales_f32 = quantizer::scales_to_f32(&scales);
|
||||
|
||||
let mut packed = Vec::new();
|
||||
quantizer::quantize_and_pack_f32(&frame, &scales_f32, GROUP_LEN, bits, &mut packed);
|
||||
|
||||
let mut decoded = Vec::new();
|
||||
quantizer::dequantize_f32(&packed, &scales_f32, GROUP_LEN, bits, len, 1, &mut decoded);
|
||||
|
||||
assert_eq!(decoded.len(), len);
|
||||
|
||||
// For a uniform vector, the quantization step is value.abs() / qmax.
|
||||
// Max error should be at most half a step (rounding) plus f16 scale error.
|
||||
let step = if value.abs() > 0.0 {
|
||||
value.abs() / qmax as f32
|
||||
} else {
|
||||
0.0
|
||||
};
|
||||
// Allow step/2 plus a small f16 rounding margin.
|
||||
let max_err = step * 0.5 + value.abs() * 0.002 + 1e-6;
|
||||
|
||||
for (i, &dec) in decoded.iter().enumerate() {
|
||||
let err = (dec - value).abs();
|
||||
assert!(
|
||||
err <= max_err,
|
||||
"trial={_trial}, bits={bits}, i={i}: value={value}, dec={dec}, \
|
||||
err={err}, max_err={max_err}, step={step}"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 10. Extreme Value Property
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[test]
|
||||
fn prop_extreme_values_dont_panic() {
|
||||
let bit_widths: &[u8] = &[3, 5, 7, 8];
|
||||
|
||||
// Frames where scales stay within f16 representable range -- decoded values
|
||||
// must be finite.
|
||||
let finite_frames: Vec<Vec<f32>> = vec![
|
||||
// Very small positive values
|
||||
vec![f32::MIN_POSITIVE; 128],
|
||||
// Contains infinities and NaN (quantizer maps non-finite to 0)
|
||||
{
|
||||
let mut v = vec![1.0f32; 128];
|
||||
v[0] = f32::INFINITY;
|
||||
v[1] = f32::NEG_INFINITY;
|
||||
v[2] = f32::NAN;
|
||||
v[3] = -0.0;
|
||||
v
|
||||
},
|
||||
// All subnormal
|
||||
vec![1e-40f32; 128],
|
||||
// Alternating zero and large (within f16 scale range)
|
||||
(0..128)
|
||||
.map(|i| if i % 2 == 0 { 0.0 } else { 1e4 })
|
||||
.collect(),
|
||||
];
|
||||
|
||||
// Frames with magnitudes that overflow f16 scales -- we only assert
|
||||
// no panics and correct output length. The decoded values may be NaN/Inf
|
||||
// because scale overflows to f16 infinity.
|
||||
let overflow_frames: Vec<Vec<f32>> = vec![
|
||||
// All f32::MAX
|
||||
vec![f32::MAX; 128],
|
||||
// All f32::MIN (most negative finite)
|
||||
vec![f32::MIN; 128],
|
||||
// Mixed signs of large magnitude
|
||||
(0..128)
|
||||
.map(|i| if i % 2 == 0 { f32::MAX } else { f32::MIN })
|
||||
.collect(),
|
||||
// Mix of tiny and huge
|
||||
(0..128)
|
||||
.map(|i| {
|
||||
if i % 3 == 0 {
|
||||
f32::MIN_POSITIVE
|
||||
} else if i % 3 == 1 {
|
||||
1e30
|
||||
} else {
|
||||
-1e30
|
||||
}
|
||||
})
|
||||
.collect(),
|
||||
];
|
||||
|
||||
// Test finite-output frames: no panics, correct length, all decoded finite.
|
||||
for (frame_idx, frame) in finite_frames.iter().enumerate() {
|
||||
for &bits in bit_widths {
|
||||
let scales = quantizer::compute_scales(frame, GROUP_LEN, bits);
|
||||
let scales_f32 = quantizer::scales_to_f32(&scales);
|
||||
|
||||
let mut packed = Vec::new();
|
||||
quantizer::quantize_and_pack_f32(frame, &scales_f32, GROUP_LEN, bits, &mut packed);
|
||||
|
||||
let mut decoded = Vec::new();
|
||||
quantizer::dequantize_f32(
|
||||
&packed,
|
||||
&scales_f32,
|
||||
GROUP_LEN,
|
||||
bits,
|
||||
frame.len(),
|
||||
1,
|
||||
&mut decoded,
|
||||
);
|
||||
|
||||
assert_eq!(
|
||||
decoded.len(),
|
||||
frame.len(),
|
||||
"finite frame_idx={frame_idx}, bits={bits}: length mismatch"
|
||||
);
|
||||
|
||||
for (i, &d) in decoded.iter().enumerate() {
|
||||
assert!(
|
||||
d.is_finite(),
|
||||
"finite frame_idx={frame_idx}, bits={bits}, i={i}: \
|
||||
decoded value is not finite: {d}"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Test overflow frames: no panics, correct length (decoded may contain NaN/Inf).
|
||||
for (frame_idx, frame) in overflow_frames.iter().enumerate() {
|
||||
for &bits in bit_widths {
|
||||
let scales = quantizer::compute_scales(frame, GROUP_LEN, bits);
|
||||
let scales_f32 = quantizer::scales_to_f32(&scales);
|
||||
|
||||
let mut packed = Vec::new();
|
||||
quantizer::quantize_and_pack_f32(frame, &scales_f32, GROUP_LEN, bits, &mut packed);
|
||||
|
||||
let mut decoded = Vec::new();
|
||||
quantizer::dequantize_f32(
|
||||
&packed,
|
||||
&scales_f32,
|
||||
GROUP_LEN,
|
||||
bits,
|
||||
frame.len(),
|
||||
1,
|
||||
&mut decoded,
|
||||
);
|
||||
|
||||
assert_eq!(
|
||||
decoded.len(),
|
||||
frame.len(),
|
||||
"overflow frame_idx={frame_idx}, bits={bits}: length mismatch"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// Bitpack roundtrip with boundary codes -- must not panic and must be exact.
|
||||
for &bits in bit_widths {
|
||||
let qmax = bitpack::qmax_from_bits(bits) as u32;
|
||||
if qmax > 0 {
|
||||
let max_code = qmax * 2;
|
||||
let codes: Vec<u32> = (0..128).map(|i| i as u32 % (max_code + 1)).collect();
|
||||
let mut bp = Vec::new();
|
||||
bitpack::pack(&codes, bits as u32, &mut bp);
|
||||
let mut unpacked = Vec::new();
|
||||
bitpack::unpack(&bp, bits as u32, codes.len(), &mut unpacked);
|
||||
assert_eq!(codes, unpacked);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 11. Segment Compression Ratio is Positive
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[test]
|
||||
fn prop_segment_compression_ratio_positive() {
|
||||
let mut rng = SimpleRng::new(0x1111_2222_3333_4444);
|
||||
|
||||
for _trial in 0..100 {
|
||||
let tensor_len = 128;
|
||||
let bits = [3u8, 5, 7, 8][rng.next_usize_range(0, 4)];
|
||||
let frame = random_vec(&mut rng, tensor_len, -1.0, 1.0);
|
||||
|
||||
let scales = quantizer::compute_scales(&frame, GROUP_LEN, bits);
|
||||
let mut packed = Vec::new();
|
||||
quantizer::quantize_and_pack(&frame, &scales, GROUP_LEN, bits, &mut packed);
|
||||
|
||||
let mut seg = Vec::new();
|
||||
segment::encode(
|
||||
bits,
|
||||
GROUP_LEN as u32,
|
||||
tensor_len as u32,
|
||||
1,
|
||||
&scales,
|
||||
&packed,
|
||||
&mut seg,
|
||||
);
|
||||
|
||||
let ratio = segment::compression_ratio(&seg);
|
||||
assert!(
|
||||
ratio > 1.0,
|
||||
"trial={_trial}, bits={bits}: compression ratio {ratio} should be > 1.0"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 12. Single-Frame Decode Matches Full Decode
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[test]
|
||||
fn prop_single_frame_decode_consistency() {
|
||||
let mut rng = SimpleRng::new(0x5555_6666_7777_8888);
|
||||
|
||||
for _trial in 0..100 {
|
||||
let tensor_len = 64;
|
||||
let frame_count = rng.next_usize_range(1, 6);
|
||||
let bits = [3u8, 5, 7, 8][rng.next_usize_range(0, 4)];
|
||||
|
||||
let first_frame = random_vec(&mut rng, tensor_len, -3.0, 3.0);
|
||||
let scales = quantizer::compute_scales(&first_frame, GROUP_LEN, bits);
|
||||
let scales_f32 = quantizer::scales_to_f32(&scales);
|
||||
|
||||
let mut packed = Vec::new();
|
||||
quantizer::quantize_and_pack_f32(&first_frame, &scales_f32, GROUP_LEN, bits, &mut packed);
|
||||
for _ in 1..frame_count {
|
||||
let frame = random_vec(&mut rng, tensor_len, -2.5, 2.5);
|
||||
quantizer::quantize_and_pack_f32(&frame, &scales_f32, GROUP_LEN, bits, &mut packed);
|
||||
}
|
||||
|
||||
let mut seg = Vec::new();
|
||||
segment::encode(
|
||||
bits,
|
||||
GROUP_LEN as u32,
|
||||
tensor_len as u32,
|
||||
frame_count as u32,
|
||||
&scales,
|
||||
&packed,
|
||||
&mut seg,
|
||||
);
|
||||
|
||||
// Full decode.
|
||||
let mut all_decoded = Vec::new();
|
||||
segment::decode(&seg, &mut all_decoded);
|
||||
assert_eq!(all_decoded.len(), tensor_len * frame_count);
|
||||
|
||||
// Single-frame decode should match the corresponding slice.
|
||||
for f in 0..frame_count {
|
||||
let single = segment::decode_single_frame(&seg, f);
|
||||
assert!(
|
||||
single.is_some(),
|
||||
"trial={_trial}, frame={f}: single-frame decode returned None"
|
||||
);
|
||||
let single = single.unwrap();
|
||||
let expected = &all_decoded[f * tensor_len..(f + 1) * tensor_len];
|
||||
assert_eq!(
|
||||
single.len(),
|
||||
expected.len(),
|
||||
"trial={_trial}, frame={f}: length mismatch"
|
||||
);
|
||||
for (i, (&s, &e)) in single.iter().zip(expected.iter()).enumerate() {
|
||||
assert!(
|
||||
(s - e).abs() < 1e-6,
|
||||
"trial={_trial}, frame={f}, i={i}: single={s}, full={e}"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 13. Delta Encode/Decode Binary Roundtrip
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[test]
|
||||
fn prop_delta_encode_decode_binary() {
|
||||
let mut rng = SimpleRng::new(0x9999_0000_1111_2222);
|
||||
|
||||
for trial in 0..500 {
|
||||
let nnz = rng.next_usize_range(0, 100);
|
||||
let entries: Vec<delta::SparseEntry> = (0..nnz)
|
||||
.map(|_| delta::SparseEntry {
|
||||
index: (rng.next_u64() % 65536) as u16,
|
||||
value: (rng.next_u64() % 65536) as i16,
|
||||
})
|
||||
.collect();
|
||||
let scale = rng.next_f32_range(1e-6, 100.0);
|
||||
|
||||
let record = delta::DeltaRecord {
|
||||
header: delta::DeltaHeader {
|
||||
tensor_id: rng.next_u64() as u128 | ((rng.next_u64() as u128) << 64),
|
||||
block_index: rng.next_u64() as u32,
|
||||
base_epoch: rng.next_u64(),
|
||||
nnz: nnz as u16,
|
||||
},
|
||||
delta_scale: scale,
|
||||
entries,
|
||||
};
|
||||
|
||||
let bytes = delta::encode_delta(&record);
|
||||
let decoded = delta::decode_delta(&bytes)
|
||||
.unwrap_or_else(|e| panic!("trial={trial}: decode failed: {e:?}"));
|
||||
|
||||
assert_eq!(decoded.header.tensor_id, record.header.tensor_id);
|
||||
assert_eq!(decoded.header.block_index, record.header.block_index);
|
||||
assert_eq!(decoded.header.base_epoch, record.header.base_epoch);
|
||||
assert_eq!(decoded.header.nnz, record.header.nnz);
|
||||
assert!(
|
||||
(decoded.delta_scale - record.delta_scale).abs() < 1e-10,
|
||||
"trial={trial}: scale mismatch"
|
||||
);
|
||||
assert_eq!(decoded.entries.len(), record.entries.len());
|
||||
for (i, (a, b)) in decoded
|
||||
.entries
|
||||
.iter()
|
||||
.zip(record.entries.iter())
|
||||
.enumerate()
|
||||
{
|
||||
assert_eq!(a.index, b.index, "trial={trial}, entry={i}: index mismatch");
|
||||
assert_eq!(a.value, b.value, "trial={trial}, entry={i}: value mismatch");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 14. Quantization is Deterministic
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[test]
|
||||
fn prop_quantization_deterministic() {
|
||||
let mut rng = SimpleRng::new(0xABCD_EF01_2345_6789);
|
||||
|
||||
for _trial in 0..200 {
|
||||
let len = rng.next_usize_range(64, 257);
|
||||
let frame = random_vec(&mut rng, len, -5.0, 5.0);
|
||||
let bits = [3u8, 5, 7, 8][rng.next_usize_range(0, 4)];
|
||||
|
||||
let scales = quantizer::compute_scales(&frame, GROUP_LEN, bits);
|
||||
let scales_f32 = quantizer::scales_to_f32(&scales);
|
||||
|
||||
let mut packed1 = Vec::new();
|
||||
quantizer::quantize_and_pack_f32(&frame, &scales_f32, GROUP_LEN, bits, &mut packed1);
|
||||
|
||||
let mut packed2 = Vec::new();
|
||||
quantizer::quantize_and_pack_f32(&frame, &scales_f32, GROUP_LEN, bits, &mut packed2);
|
||||
|
||||
assert_eq!(
|
||||
packed1, packed2,
|
||||
"trial={_trial}, bits={bits}: quantization is not deterministic"
|
||||
);
|
||||
}
|
||||
}
|
||||
920
crates/ruvector-temporal-tensor/tests/stress_tests.rs
Normal file
920
crates/ruvector-temporal-tensor/tests/stress_tests.rs
Normal file
@@ -0,0 +1,920 @@
|
||||
//! Stress and fuzz-like tests for temporal tensor compression.
|
||||
//!
|
||||
//! Exercises the storage engine, delta chains, and checksum integrity under
|
||||
//! heavy random workloads using a deterministic PRNG. No external dependencies.
|
||||
//!
|
||||
//! Run with:
|
||||
//! ```sh
|
||||
//! cargo test --release -p ruvector-temporal-tensor --test stress_tests -- --nocapture
|
||||
//! ```
|
||||
|
||||
use ruvector_temporal_tensor::delta::{compute_delta, DeltaChain};
|
||||
use ruvector_temporal_tensor::store::{BlockKey, ReconstructPolicy, StoreError, Tier, TieredStore};
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Deterministic PRNG (LCG) -- same as other test files, no external deps
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Simple linear congruential generator. Constants from Knuth MMIX.
|
||||
struct SimpleRng {
|
||||
state: u64,
|
||||
}
|
||||
|
||||
impl SimpleRng {
|
||||
fn new(seed: u64) -> Self {
|
||||
Self { state: seed }
|
||||
}
|
||||
|
||||
fn next_u64(&mut self) -> u64 {
|
||||
self.state = self
|
||||
.state
|
||||
.wrapping_mul(6_364_136_223_846_793_005)
|
||||
.wrapping_add(1_442_695_040_888_963_407);
|
||||
self.state
|
||||
}
|
||||
|
||||
fn next_f64(&mut self) -> f64 {
|
||||
(self.next_u64() >> 11) as f64 / (1u64 << 53) as f64
|
||||
}
|
||||
|
||||
fn next_f32(&mut self) -> f32 {
|
||||
self.next_f64() as f32
|
||||
}
|
||||
|
||||
fn next_f32_range(&mut self, lo: f32, hi: f32) -> f32 {
|
||||
lo + self.next_f32() * (hi - lo)
|
||||
}
|
||||
|
||||
fn next_usize_range(&mut self, lo: usize, hi: usize) -> usize {
|
||||
let range = (hi - lo) as u64;
|
||||
if range == 0 {
|
||||
return lo;
|
||||
}
|
||||
lo + (self.next_u64() % range) as usize
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Helpers
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
fn make_key(tid: u128, idx: u32) -> BlockKey {
|
||||
BlockKey {
|
||||
tensor_id: tid,
|
||||
block_index: idx,
|
||||
}
|
||||
}
|
||||
|
||||
fn random_tier(rng: &mut SimpleRng) -> Tier {
|
||||
match rng.next_usize_range(0, 3) {
|
||||
0 => Tier::Tier1,
|
||||
1 => Tier::Tier2,
|
||||
_ => Tier::Tier3,
|
||||
}
|
||||
}
|
||||
|
||||
fn random_data(rng: &mut SimpleRng, len: usize) -> Vec<f32> {
|
||||
(0..len).map(|_| rng.next_f32_range(-1.0, 1.0)).collect()
|
||||
}
|
||||
|
||||
// ===========================================================================
|
||||
// 1. Random put/get/evict cycle
|
||||
// ===========================================================================
|
||||
|
||||
/// Exercises the store with 5000 random operations (put 40%, get 30%,
|
||||
/// touch 20%, evict 10%) on a pool of 200 block keys. After all
|
||||
/// iterations the block count must equal `inserted - evicted`.
|
||||
#[test]
|
||||
fn test_random_put_get_evict_cycle() {
|
||||
let mut store = TieredStore::new(4096);
|
||||
let mut rng = SimpleRng::new(0xDEAD_BEEF);
|
||||
|
||||
const NUM_KEYS: usize = 200;
|
||||
const NUM_ITERS: usize = 5_000;
|
||||
const ELEM_COUNT: usize = 64;
|
||||
|
||||
// Track which keys have been inserted and not yet evicted.
|
||||
let mut inserted: std::collections::HashSet<u32> = std::collections::HashSet::new();
|
||||
let mut evicted: std::collections::HashSet<u32> = std::collections::HashSet::new();
|
||||
|
||||
for iter in 0..NUM_ITERS {
|
||||
let roll = rng.next_usize_range(0, 100);
|
||||
let key_idx = rng.next_usize_range(0, NUM_KEYS) as u32;
|
||||
let key = make_key(1, key_idx);
|
||||
let tick = iter as u64;
|
||||
|
||||
if roll < 40 {
|
||||
// PUT (40%)
|
||||
let data = random_data(&mut rng, ELEM_COUNT);
|
||||
let tier = random_tier(&mut rng);
|
||||
store.put(key, &data, tier, tick).unwrap();
|
||||
inserted.insert(key_idx);
|
||||
evicted.remove(&key_idx);
|
||||
} else if roll < 70 {
|
||||
// GET (30%)
|
||||
let mut out = vec![0.0f32; ELEM_COUNT];
|
||||
match store.get(key, &mut out, tick) {
|
||||
Ok(n) => {
|
||||
assert!(n > 0, "get returned 0 elements for an existing block");
|
||||
assert!(n <= ELEM_COUNT);
|
||||
}
|
||||
Err(StoreError::BlockNotFound) => {
|
||||
// Key was never inserted or was evicted -- valid.
|
||||
}
|
||||
Err(StoreError::TensorEvicted) => {
|
||||
// Block was evicted to Tier0 -- valid.
|
||||
assert!(
|
||||
evicted.contains(&key_idx),
|
||||
"TensorEvicted for key not in evicted set"
|
||||
);
|
||||
}
|
||||
Err(e) => {
|
||||
panic!("unexpected error on get at iter {}: {:?}", iter, e);
|
||||
}
|
||||
}
|
||||
} else if roll < 90 {
|
||||
// TOUCH (20%)
|
||||
store.touch(key, tick);
|
||||
} else {
|
||||
// EVICT (10%)
|
||||
match store.evict(key, ReconstructPolicy::None) {
|
||||
Ok(()) => {
|
||||
if inserted.contains(&key_idx) {
|
||||
evicted.insert(key_idx);
|
||||
}
|
||||
}
|
||||
Err(StoreError::BlockNotFound) => {
|
||||
// Key never existed -- valid.
|
||||
}
|
||||
Err(e) => {
|
||||
panic!("unexpected error on evict at iter {}: {:?}", iter, e);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Final invariant: block_count = all unique keys ever put (including evicted ones,
|
||||
// since eviction keeps metadata).
|
||||
let all_known: std::collections::HashSet<u32> = inserted.union(&evicted).copied().collect();
|
||||
assert_eq!(
|
||||
store.block_count(),
|
||||
all_known.len(),
|
||||
"block_count mismatch after random cycle"
|
||||
);
|
||||
|
||||
// Verify: non-evicted blocks are readable.
|
||||
let live_keys: Vec<u32> = inserted.difference(&evicted).copied().collect();
|
||||
for &kid in &live_keys {
|
||||
let mut out = vec![0.0f32; ELEM_COUNT];
|
||||
let key = make_key(1, kid);
|
||||
let result = store.get(key, &mut out, NUM_ITERS as u64);
|
||||
assert!(
|
||||
result.is_ok(),
|
||||
"live block {} should be readable, got {:?}",
|
||||
kid,
|
||||
result
|
||||
);
|
||||
}
|
||||
|
||||
println!(
|
||||
"random_put_get_evict_cycle: {} iters, {} live blocks, {} evicted",
|
||||
NUM_ITERS,
|
||||
live_keys.len(),
|
||||
evicted.len()
|
||||
);
|
||||
}
|
||||
|
||||
// ===========================================================================
|
||||
// 2. Rapid tier oscillation (stress hysteresis)
|
||||
// ===========================================================================
|
||||
|
||||
/// Puts 50 blocks at Tier1, then alternately touches 25 blocks intensively
|
||||
/// (50 touches/tick) and ignores them for 500 ticks. Verifies that all
|
||||
/// blocks remain readable and no panics occur during rapid access-pattern
|
||||
/// changes.
|
||||
#[test]
|
||||
fn test_rapid_tier_oscillation() {
|
||||
let mut store = TieredStore::new(4096);
|
||||
let mut rng = SimpleRng::new(0xCAFE_BABE);
|
||||
|
||||
const NUM_BLOCKS: usize = 50;
|
||||
const ELEM_COUNT: usize = 64;
|
||||
const TOTAL_TICKS: u64 = 500;
|
||||
const HOT_COUNT: usize = 25;
|
||||
const TOUCHES_PER_TICK: usize = 50;
|
||||
|
||||
// Insert all blocks at Tier1.
|
||||
let block_data: Vec<Vec<f32>> = (0..NUM_BLOCKS)
|
||||
.map(|_| random_data(&mut rng, ELEM_COUNT))
|
||||
.collect();
|
||||
|
||||
for i in 0..NUM_BLOCKS {
|
||||
store
|
||||
.put(make_key(2, i as u32), &block_data[i], Tier::Tier1, 0)
|
||||
.unwrap();
|
||||
}
|
||||
assert_eq!(store.block_count(), NUM_BLOCKS);
|
||||
|
||||
// Oscillate: even ticks -> heavy touching of first HOT_COUNT blocks,
|
||||
// odd ticks -> no touching (cold period).
|
||||
for tick in 1..=TOTAL_TICKS {
|
||||
if tick % 2 == 0 {
|
||||
// Hot phase: touch first HOT_COUNT blocks repeatedly.
|
||||
for _ in 0..TOUCHES_PER_TICK {
|
||||
let idx = rng.next_usize_range(0, HOT_COUNT) as u32;
|
||||
store.touch(make_key(2, idx), tick);
|
||||
}
|
||||
}
|
||||
// Odd ticks: silence (no touches).
|
||||
}
|
||||
|
||||
// All blocks must remain readable.
|
||||
for i in 0..NUM_BLOCKS {
|
||||
let key = make_key(2, i as u32);
|
||||
let mut out = vec![0.0f32; ELEM_COUNT];
|
||||
let n = store
|
||||
.get(key, &mut out, TOTAL_TICKS + 1)
|
||||
.unwrap_or_else(|e| panic!("block {} unreadable after oscillation: {:?}", i, e));
|
||||
assert_eq!(n, ELEM_COUNT);
|
||||
// Values must be finite.
|
||||
for (j, &v) in out.iter().enumerate() {
|
||||
assert!(v.is_finite(), "block {} elem {} is non-finite: {}", i, j, v);
|
||||
}
|
||||
}
|
||||
|
||||
// Verify metadata is intact for all blocks.
|
||||
for i in 0..NUM_BLOCKS {
|
||||
let m = store.meta(make_key(2, i as u32)).expect("meta missing");
|
||||
assert!(
|
||||
m.tier == Tier::Tier1 || m.tier == Tier::Tier2 || m.tier == Tier::Tier3,
|
||||
"block {} has unexpected tier {:?}",
|
||||
i,
|
||||
m.tier
|
||||
);
|
||||
}
|
||||
|
||||
println!(
|
||||
"rapid_tier_oscillation: {} ticks, {} blocks, no panics",
|
||||
TOTAL_TICKS, NUM_BLOCKS
|
||||
);
|
||||
}
|
||||
|
||||
// ===========================================================================
|
||||
// 3. Large block stress (memory pressure)
|
||||
// ===========================================================================
|
||||
|
||||
/// Puts 500 blocks of 4096 elements each (total ~8MB at 8-bit), touches
|
||||
/// them randomly, reads them all back verifying finite values, evicts half,
|
||||
/// and verifies the other half is still readable and total_bytes decreased.
|
||||
#[test]
|
||||
fn test_large_block_stress() {
|
||||
let mut store = TieredStore::new(4096);
|
||||
let mut rng = SimpleRng::new(0x1234_5678);
|
||||
|
||||
const NUM_BLOCKS: usize = 500;
|
||||
const ELEM_COUNT: usize = 4096;
|
||||
|
||||
// Insert all blocks at Tier1 (8-bit = 1 byte/elem = 4096 bytes/block).
|
||||
for i in 0..NUM_BLOCKS {
|
||||
let data = random_data(&mut rng, ELEM_COUNT);
|
||||
store
|
||||
.put(make_key(3, i as u32), &data, Tier::Tier1, i as u64)
|
||||
.unwrap();
|
||||
}
|
||||
assert_eq!(store.block_count(), NUM_BLOCKS);
|
||||
|
||||
let bytes_before = store.total_bytes();
|
||||
assert!(
|
||||
bytes_before > 0,
|
||||
"total_bytes should be positive after inserting {} blocks",
|
||||
NUM_BLOCKS
|
||||
);
|
||||
println!(
|
||||
"large_block_stress: {} blocks inserted, total_bytes = {}",
|
||||
NUM_BLOCKS, bytes_before
|
||||
);
|
||||
|
||||
// Touch all blocks randomly.
|
||||
for _ in 0..NUM_BLOCKS {
|
||||
let idx = rng.next_usize_range(0, NUM_BLOCKS) as u32;
|
||||
store.touch(make_key(3, idx), NUM_BLOCKS as u64 + 1);
|
||||
}
|
||||
|
||||
// Read all blocks back and verify finite values.
|
||||
for i in 0..NUM_BLOCKS {
|
||||
let key = make_key(3, i as u32);
|
||||
let mut out = vec![0.0f32; ELEM_COUNT];
|
||||
let n = store
|
||||
.get(key, &mut out, NUM_BLOCKS as u64 + 2)
|
||||
.unwrap_or_else(|e| panic!("block {} unreadable: {:?}", i, e));
|
||||
assert_eq!(n, ELEM_COUNT);
|
||||
for (j, &v) in out.iter().enumerate() {
|
||||
assert!(v.is_finite(), "block {} elem {} is non-finite: {}", i, j, v);
|
||||
}
|
||||
}
|
||||
|
||||
// Evict the first half.
|
||||
for i in 0..(NUM_BLOCKS / 2) {
|
||||
store
|
||||
.evict(make_key(3, i as u32), ReconstructPolicy::None)
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
let bytes_after = store.total_bytes();
|
||||
assert!(
|
||||
bytes_after < bytes_before,
|
||||
"total_bytes should decrease after evicting half: before={}, after={}",
|
||||
bytes_before,
|
||||
bytes_after
|
||||
);
|
||||
|
||||
// Verify the second half is still readable.
|
||||
for i in (NUM_BLOCKS / 2)..NUM_BLOCKS {
|
||||
let key = make_key(3, i as u32);
|
||||
let mut out = vec![0.0f32; ELEM_COUNT];
|
||||
let n = store
|
||||
.get(key, &mut out, NUM_BLOCKS as u64 + 3)
|
||||
.unwrap_or_else(|e| {
|
||||
panic!(
|
||||
"block {} should still be readable after evicting first half: {:?}",
|
||||
i, e
|
||||
)
|
||||
});
|
||||
assert_eq!(n, ELEM_COUNT);
|
||||
}
|
||||
|
||||
// Verify evicted blocks return TensorEvicted.
|
||||
for i in 0..(NUM_BLOCKS / 2) {
|
||||
let key = make_key(3, i as u32);
|
||||
let mut out = vec![0.0f32; ELEM_COUNT];
|
||||
let result = store.get(key, &mut out, NUM_BLOCKS as u64 + 4);
|
||||
assert_eq!(
|
||||
result,
|
||||
Err(StoreError::TensorEvicted),
|
||||
"evicted block {} should return TensorEvicted",
|
||||
i
|
||||
);
|
||||
}
|
||||
|
||||
println!(
|
||||
"large_block_stress: bytes before={}, after={}, reduction={}%",
|
||||
bytes_before,
|
||||
bytes_after,
|
||||
((bytes_before - bytes_after) as f64 / bytes_before as f64 * 100.0) as u32
|
||||
);
|
||||
}
|
||||
|
||||
// ===========================================================================
|
||||
// 4. Delta chain stress
|
||||
// ===========================================================================
|
||||
|
||||
/// Creates a 1024-element base vector, builds a DeltaChain with max_depth=8,
|
||||
/// appends 8 deltas each modifying ~5% of values, reconstructs and verifies
|
||||
/// error < 1%, compacts, rebuilds to max, and checks that an extra append
|
||||
/// yields DeltaChainTooLong.
|
||||
#[test]
|
||||
fn test_delta_chain_stress() {
|
||||
let mut rng = SimpleRng::new(0xABCD_EF01);
|
||||
|
||||
const DIM: usize = 1024;
|
||||
const MAX_DEPTH: u8 = 8;
|
||||
const CHANGE_FRACTION: f32 = 0.05; // ~5% of values per delta
|
||||
|
||||
// Create a base vector with random values in [-1, 1].
|
||||
let base: Vec<f32> = (0..DIM).map(|_| rng.next_f32_range(-1.0, 1.0)).collect();
|
||||
let mut chain = DeltaChain::new(base.clone(), MAX_DEPTH);
|
||||
|
||||
// Build the expected ground-truth by applying modifications cumulatively.
|
||||
let mut truth = base.clone();
|
||||
|
||||
// Append MAX_DEPTH deltas, each modifying ~5% of elements.
|
||||
for epoch in 0..MAX_DEPTH {
|
||||
let mut modified = truth.clone();
|
||||
let num_changes = (DIM as f32 * CHANGE_FRACTION) as usize;
|
||||
for _ in 0..num_changes {
|
||||
let idx = rng.next_usize_range(0, DIM);
|
||||
let perturbation = rng.next_f32_range(-0.1, 0.1);
|
||||
modified[idx] += perturbation;
|
||||
}
|
||||
|
||||
let delta = compute_delta(
|
||||
&truth,
|
||||
&modified,
|
||||
42, // tensor_id
|
||||
0, // block_index
|
||||
epoch as u64, // base_epoch
|
||||
1e-8, // threshold (very small to capture all changes)
|
||||
1.0, // max_change_fraction (allow up to 100%)
|
||||
)
|
||||
.expect("compute_delta should succeed for small changes");
|
||||
|
||||
chain
|
||||
.append(delta)
|
||||
.unwrap_or_else(|e| panic!("append should succeed at depth {}: {:?}", epoch, e));
|
||||
|
||||
truth = modified;
|
||||
}
|
||||
|
||||
assert_eq!(chain.chain_len(), MAX_DEPTH as usize);
|
||||
|
||||
// Reconstruct and verify error < 1%.
|
||||
let reconstructed = chain.reconstruct();
|
||||
assert_eq!(reconstructed.len(), DIM);
|
||||
let mut max_err: f32 = 0.0;
|
||||
for i in 0..DIM {
|
||||
let err = (reconstructed[i] - truth[i]).abs();
|
||||
if err > max_err {
|
||||
max_err = err;
|
||||
}
|
||||
}
|
||||
// The error comes from i16 quantization of deltas; for small perturbations
|
||||
// the relative error should be well under 1% of the value range.
|
||||
let value_range = truth.iter().fold(0.0f32, |acc, &v| acc.max(v.abs()));
|
||||
let relative_max_err = if value_range > 0.0 {
|
||||
max_err / value_range
|
||||
} else {
|
||||
0.0
|
||||
};
|
||||
assert!(
|
||||
relative_max_err < 0.01,
|
||||
"reconstruction error {:.6} ({:.4}%) exceeds 1% of value range {:.4}",
|
||||
max_err,
|
||||
relative_max_err * 100.0,
|
||||
value_range
|
||||
);
|
||||
println!(
|
||||
"delta_chain_stress: max reconstruction error = {:.6} ({:.4}% of range {:.4})",
|
||||
max_err,
|
||||
relative_max_err * 100.0,
|
||||
value_range
|
||||
);
|
||||
|
||||
// Compact: apply all deltas to base, chain_len should become 0.
|
||||
chain.compact();
|
||||
assert_eq!(
|
||||
chain.chain_len(),
|
||||
0,
|
||||
"chain_len should be 0 after compaction"
|
||||
);
|
||||
|
||||
// Verify reconstruction after compaction still yields correct data.
|
||||
let after_compact = chain.reconstruct();
|
||||
for i in 0..DIM {
|
||||
let err = (after_compact[i] - truth[i]).abs();
|
||||
assert!(
|
||||
err < 0.01,
|
||||
"post-compaction error at elem {}: {:.6}",
|
||||
i,
|
||||
err
|
||||
);
|
||||
}
|
||||
|
||||
// Rebuild chain to max depth.
|
||||
let compacted_base = after_compact.clone();
|
||||
let mut chain2 = DeltaChain::new(compacted_base.clone(), MAX_DEPTH);
|
||||
let mut truth2 = compacted_base.clone();
|
||||
for epoch in 0..MAX_DEPTH {
|
||||
let mut modified = truth2.clone();
|
||||
let num_changes = (DIM as f32 * CHANGE_FRACTION) as usize;
|
||||
for _ in 0..num_changes {
|
||||
let idx = rng.next_usize_range(0, DIM);
|
||||
modified[idx] += rng.next_f32_range(-0.05, 0.05);
|
||||
}
|
||||
let delta = compute_delta(&truth2, &modified, 42, 0, epoch as u64, 1e-8, 1.0)
|
||||
.expect("compute_delta should succeed");
|
||||
chain2.append(delta).unwrap();
|
||||
truth2 = modified;
|
||||
}
|
||||
assert_eq!(chain2.chain_len(), MAX_DEPTH as usize);
|
||||
|
||||
// One more append should fail with DeltaChainTooLong.
|
||||
let mut overflow_modified = truth2.clone();
|
||||
overflow_modified[0] += 0.01;
|
||||
let overflow_delta = compute_delta(
|
||||
&truth2,
|
||||
&overflow_modified,
|
||||
42,
|
||||
0,
|
||||
MAX_DEPTH as u64,
|
||||
1e-8,
|
||||
1.0,
|
||||
)
|
||||
.expect("compute_delta for overflow");
|
||||
let result = chain2.append(overflow_delta);
|
||||
assert_eq!(
|
||||
result,
|
||||
Err(StoreError::DeltaChainTooLong),
|
||||
"appending beyond max_depth should return DeltaChainTooLong"
|
||||
);
|
||||
|
||||
// Reconstruct should still work after the failed append.
|
||||
let after_fail = chain2.reconstruct();
|
||||
assert_eq!(after_fail.len(), DIM);
|
||||
for i in 0..DIM {
|
||||
let err = (after_fail[i] - truth2[i]).abs();
|
||||
assert!(
|
||||
err < 0.01,
|
||||
"reconstruction after failed append: elem {} error {:.6}",
|
||||
i,
|
||||
err
|
||||
);
|
||||
}
|
||||
|
||||
println!("delta_chain_stress: all chain operations verified");
|
||||
}
|
||||
|
||||
// ===========================================================================
|
||||
// 5. Checksum sensitivity
|
||||
// ===========================================================================
|
||||
|
||||
/// Verifies that the checksum stored in block metadata is deterministic
|
||||
/// and sensitive to even tiny changes in input data.
|
||||
#[test]
|
||||
fn test_checksum_sensitivity() {
|
||||
let mut store = TieredStore::new(4096);
|
||||
let mut rng = SimpleRng::new(0xFEED_FACE);
|
||||
|
||||
const ELEM_COUNT: usize = 128;
|
||||
let data: Vec<f32> = (0..ELEM_COUNT)
|
||||
.map(|_| rng.next_f32_range(-1.0, 1.0))
|
||||
.collect();
|
||||
|
||||
let key = make_key(5, 0);
|
||||
|
||||
// Put and record the checksum.
|
||||
store.put(key, &data, Tier::Tier1, 0).unwrap();
|
||||
let checksum1 = store.meta(key).unwrap().checksum;
|
||||
|
||||
// Put the same data again with the same key -> same checksum.
|
||||
store.put(key, &data, Tier::Tier1, 1).unwrap();
|
||||
let checksum2 = store.meta(key).unwrap().checksum;
|
||||
assert_eq!(
|
||||
checksum1, checksum2,
|
||||
"identical data should produce identical checksums"
|
||||
);
|
||||
|
||||
// Modify one element by a tiny amount (1e-6), put again.
|
||||
let mut data_tiny = data.clone();
|
||||
data_tiny[ELEM_COUNT / 2] += 1e-6;
|
||||
store.put(key, &data_tiny, Tier::Tier1, 2).unwrap();
|
||||
let checksum3 = store.meta(key).unwrap().checksum;
|
||||
// Note: due to 8-bit quantization, a 1e-6 change on values in [-1,1]
|
||||
// might not change the quantized representation. If it does, checksums
|
||||
// differ; if not, they are the same. We test a larger perturbation below
|
||||
// to guarantee a difference.
|
||||
|
||||
// Modify one element by a larger amount that will definitely change quantized value.
|
||||
let mut data_modified = data.clone();
|
||||
data_modified[ELEM_COUNT / 2] += 0.1;
|
||||
store.put(key, &data_modified, Tier::Tier1, 3).unwrap();
|
||||
let checksum4 = store.meta(key).unwrap().checksum;
|
||||
assert_ne!(
|
||||
checksum1, checksum4,
|
||||
"modifying one element by 0.1 should change the checksum"
|
||||
);
|
||||
|
||||
// Put very different data -> very different checksum.
|
||||
let data_different: Vec<f32> = (0..ELEM_COUNT)
|
||||
.map(|_| rng.next_f32_range(-10.0, 10.0))
|
||||
.collect();
|
||||
store.put(key, &data_different, Tier::Tier1, 4).unwrap();
|
||||
let checksum5 = store.meta(key).unwrap().checksum;
|
||||
assert_ne!(
|
||||
checksum1, checksum5,
|
||||
"very different data should produce a different checksum"
|
||||
);
|
||||
// Also verify it differs from the slightly-modified version.
|
||||
assert_ne!(
|
||||
checksum4, checksum5,
|
||||
"two different datasets should have different checksums"
|
||||
);
|
||||
|
||||
println!(
|
||||
"checksum_sensitivity: c1={:#010X} c2={:#010X} c3={:#010X} c4={:#010X} c5={:#010X}",
|
||||
checksum1, checksum2, checksum3, checksum4, checksum5
|
||||
);
|
||||
}
|
||||
|
||||
// ===========================================================================
|
||||
// 6. Concurrent simulation (simulated multi-reader)
|
||||
// ===========================================================================
|
||||
|
||||
/// Puts 100 blocks, then runs 10 simulated "reader threads" (sequential
|
||||
/// loops) each performing 100 iterations of random touches and reads.
|
||||
/// Verifies all reads succeed and return finite data, and metadata remains
|
||||
/// consistent.
|
||||
#[test]
|
||||
fn test_concurrent_simulation() {
|
||||
let mut store = TieredStore::new(4096);
|
||||
let mut rng = SimpleRng::new(0xC0DE_C0DE);
|
||||
|
||||
const NUM_BLOCKS: usize = 100;
|
||||
const NUM_READERS: usize = 10;
|
||||
const ITERS_PER_READER: usize = 100;
|
||||
const ELEM_COUNT: usize = 64;
|
||||
|
||||
// Insert all blocks.
|
||||
for i in 0..NUM_BLOCKS {
|
||||
let data = random_data(&mut rng, ELEM_COUNT);
|
||||
store
|
||||
.put(make_key(6, i as u32), &data, Tier::Tier1, 0)
|
||||
.unwrap();
|
||||
}
|
||||
assert_eq!(store.block_count(), NUM_BLOCKS);
|
||||
|
||||
let mut total_reads: usize = 0;
|
||||
let mut total_touches: usize = 0;
|
||||
|
||||
// Simulate NUM_READERS concurrent readers.
|
||||
for reader_id in 0..NUM_READERS {
|
||||
let base_tick = (reader_id as u64 + 1) * 1000;
|
||||
for iter in 0..ITERS_PER_READER {
|
||||
let key_idx = rng.next_usize_range(0, NUM_BLOCKS) as u32;
|
||||
let key = make_key(6, key_idx);
|
||||
let tick = base_tick + iter as u64;
|
||||
|
||||
// Touch the block.
|
||||
store.touch(key, tick);
|
||||
total_touches += 1;
|
||||
|
||||
// Read the block.
|
||||
let mut out = vec![0.0f32; ELEM_COUNT];
|
||||
let n = store.get(key, &mut out, tick).unwrap_or_else(|e| {
|
||||
panic!(
|
||||
"reader {} iter {} key {} failed: {:?}",
|
||||
reader_id, iter, key_idx, e
|
||||
)
|
||||
});
|
||||
assert_eq!(n, ELEM_COUNT);
|
||||
total_reads += 1;
|
||||
|
||||
// Verify finite values.
|
||||
for (j, &v) in out.iter().enumerate() {
|
||||
assert!(
|
||||
v.is_finite(),
|
||||
"reader {} iter {} block {} elem {} non-finite: {}",
|
||||
reader_id,
|
||||
iter,
|
||||
key_idx,
|
||||
j,
|
||||
v
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Verify metadata integrity for all blocks.
|
||||
for i in 0..NUM_BLOCKS {
|
||||
let key = make_key(6, i as u32);
|
||||
let m = store.meta(key).expect("meta should exist");
|
||||
assert!(
|
||||
m.tier == Tier::Tier1 || m.tier == Tier::Tier2 || m.tier == Tier::Tier3,
|
||||
"block {} has invalid tier {:?}",
|
||||
i,
|
||||
m.tier
|
||||
);
|
||||
assert!(
|
||||
m.access_count > 0,
|
||||
"block {} should have been accessed at least once",
|
||||
i
|
||||
);
|
||||
}
|
||||
|
||||
println!(
|
||||
"concurrent_simulation: {} readers x {} iters = {} reads, {} touches",
|
||||
NUM_READERS, ITERS_PER_READER, total_reads, total_touches
|
||||
);
|
||||
}
|
||||
|
||||
// ===========================================================================
|
||||
// 7. Extreme tick values
|
||||
// ===========================================================================
|
||||
|
||||
/// Tests behavior at tick value boundaries: 0, u64::MAX-1, and u64::MAX.
|
||||
/// Verifies no overflow or underflow panics in access-pattern tracking.
|
||||
#[test]
|
||||
fn test_extreme_tick_values() {
|
||||
let mut store = TieredStore::new(4096);
|
||||
|
||||
const ELEM_COUNT: usize = 32;
|
||||
let data = vec![0.5f32; ELEM_COUNT];
|
||||
|
||||
// -- Test 1: Put at tick=0, touch at tick=u64::MAX-1 --
|
||||
let key_a = make_key(7, 0);
|
||||
store.put(key_a, &data, Tier::Tier1, 0).unwrap();
|
||||
store.touch(key_a, u64::MAX - 1);
|
||||
|
||||
let meta_a = store.meta(key_a).unwrap();
|
||||
assert_eq!(meta_a.last_access_at, u64::MAX - 1);
|
||||
assert!(
|
||||
meta_a.access_count >= 2,
|
||||
"access_count should reflect put + touch"
|
||||
);
|
||||
|
||||
// Read should still work.
|
||||
let mut out = vec![0.0f32; ELEM_COUNT];
|
||||
let n = store.get(key_a, &mut out, u64::MAX - 1).unwrap();
|
||||
assert_eq!(n, ELEM_COUNT);
|
||||
|
||||
// -- Test 2: Put at tick=u64::MAX --
|
||||
let key_b = make_key(7, 1);
|
||||
store.put(key_b, &data, Tier::Tier1, u64::MAX).unwrap();
|
||||
let meta_b = store.meta(key_b).unwrap();
|
||||
assert_eq!(meta_b.created_at, u64::MAX);
|
||||
assert_eq!(meta_b.last_access_at, u64::MAX);
|
||||
|
||||
// Read at u64::MAX.
|
||||
let mut out2 = vec![0.0f32; ELEM_COUNT];
|
||||
let n2 = store.get(key_b, &mut out2, u64::MAX).unwrap();
|
||||
assert_eq!(n2, ELEM_COUNT);
|
||||
|
||||
// -- Test 3: Touch at tick=0 when last_access=u64::MAX --
|
||||
// This tests that saturating_sub prevents underflow.
|
||||
store.touch(key_b, 0);
|
||||
let meta_b2 = store.meta(key_b).unwrap();
|
||||
// last_access should update to 0 (the tick we passed).
|
||||
// The delta computation uses saturating_sub, so 0 - u64::MAX saturates to 0,
|
||||
// meaning delta=0 and the window/ema are handled without panic.
|
||||
assert_eq!(meta_b2.last_access_at, 0);
|
||||
|
||||
// -- Test 4: Touch at tick=u64::MAX after last_access=0 --
|
||||
store.touch(key_b, u64::MAX);
|
||||
let meta_b3 = store.meta(key_b).unwrap();
|
||||
assert_eq!(meta_b3.last_access_at, u64::MAX);
|
||||
// The delta is u64::MAX, which is >= 64, so window resets to 1.
|
||||
assert_eq!(meta_b3.window, 1);
|
||||
|
||||
// Verify all blocks still readable after extreme tick gymnastics.
|
||||
for i in 0..2u32 {
|
||||
let key = make_key(7, i);
|
||||
let mut out = vec![0.0f32; ELEM_COUNT];
|
||||
let result = store.get(key, &mut out, u64::MAX);
|
||||
assert!(
|
||||
result.is_ok(),
|
||||
"block {} should be readable after extreme ticks: {:?}",
|
||||
i,
|
||||
result
|
||||
);
|
||||
}
|
||||
|
||||
println!("extreme_tick_values: all boundary conditions passed without panic");
|
||||
}
|
||||
|
||||
// ===========================================================================
|
||||
// 8. All tiers coexist
|
||||
// ===========================================================================
|
||||
|
||||
/// Puts 100 blocks in each of Tier1, Tier2, Tier3 (300 total), verifies
|
||||
/// tier counts, reads all blocks verifying accuracy matches tier expectations
|
||||
/// (higher tiers = less quantization error), evicts all Tier3 blocks, and
|
||||
/// verifies Tier1 and Tier2 are still readable.
|
||||
#[test]
|
||||
fn test_all_tiers_coexist() {
|
||||
let mut store = TieredStore::new(4096);
|
||||
let mut rng = SimpleRng::new(0xBAAD_F00D);
|
||||
|
||||
const BLOCKS_PER_TIER: usize = 100;
|
||||
const ELEM_COUNT: usize = 128;
|
||||
|
||||
// Store original data for roundtrip error comparison.
|
||||
let mut originals: Vec<Vec<f32>> = Vec::new();
|
||||
|
||||
// Insert 100 blocks at Tier1 (tensor_id=81).
|
||||
for i in 0..BLOCKS_PER_TIER {
|
||||
let data = random_data(&mut rng, ELEM_COUNT);
|
||||
store
|
||||
.put(make_key(81, i as u32), &data, Tier::Tier1, 0)
|
||||
.unwrap();
|
||||
originals.push(data);
|
||||
}
|
||||
|
||||
// Insert 100 blocks at Tier2 (tensor_id=82).
|
||||
for i in 0..BLOCKS_PER_TIER {
|
||||
let data = random_data(&mut rng, ELEM_COUNT);
|
||||
store
|
||||
.put(make_key(82, i as u32), &data, Tier::Tier2, 0)
|
||||
.unwrap();
|
||||
originals.push(data);
|
||||
}
|
||||
|
||||
// Insert 100 blocks at Tier3 (tensor_id=83).
|
||||
for i in 0..BLOCKS_PER_TIER {
|
||||
let data = random_data(&mut rng, ELEM_COUNT);
|
||||
store
|
||||
.put(make_key(83, i as u32), &data, Tier::Tier3, 0)
|
||||
.unwrap();
|
||||
originals.push(data);
|
||||
}
|
||||
|
||||
// Verify tier counts.
|
||||
assert_eq!(store.tier_count(Tier::Tier1), BLOCKS_PER_TIER);
|
||||
assert_eq!(store.tier_count(Tier::Tier2), BLOCKS_PER_TIER);
|
||||
assert_eq!(store.tier_count(Tier::Tier3), BLOCKS_PER_TIER);
|
||||
assert_eq!(store.block_count(), 3 * BLOCKS_PER_TIER);
|
||||
|
||||
// Read all blocks and compute per-tier max roundtrip error.
|
||||
let mut tier1_max_err: f32 = 0.0;
|
||||
let mut tier2_max_err: f32 = 0.0;
|
||||
let mut tier3_max_err: f32 = 0.0;
|
||||
|
||||
for i in 0..BLOCKS_PER_TIER {
|
||||
// Tier1
|
||||
let key = make_key(81, i as u32);
|
||||
let mut out = vec![0.0f32; ELEM_COUNT];
|
||||
store.get(key, &mut out, 1).unwrap();
|
||||
let orig = &originals[i];
|
||||
for j in 0..ELEM_COUNT {
|
||||
let err = (out[j] - orig[j]).abs();
|
||||
if err > tier1_max_err {
|
||||
tier1_max_err = err;
|
||||
}
|
||||
}
|
||||
|
||||
// Tier2
|
||||
let key = make_key(82, i as u32);
|
||||
store.get(key, &mut out, 1).unwrap();
|
||||
let orig = &originals[BLOCKS_PER_TIER + i];
|
||||
for j in 0..ELEM_COUNT {
|
||||
let err = (out[j] - orig[j]).abs();
|
||||
if err > tier2_max_err {
|
||||
tier2_max_err = err;
|
||||
}
|
||||
}
|
||||
|
||||
// Tier3
|
||||
let key = make_key(83, i as u32);
|
||||
store.get(key, &mut out, 1).unwrap();
|
||||
let orig = &originals[2 * BLOCKS_PER_TIER + i];
|
||||
for j in 0..ELEM_COUNT {
|
||||
let err = (out[j] - orig[j]).abs();
|
||||
if err > tier3_max_err {
|
||||
tier3_max_err = err;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Tier1 (8-bit) should have the lowest error, Tier3 (3-bit) the highest.
|
||||
// Values are in [-1, 1], so 8-bit qmax=127 -> step ~0.0079, 3-bit qmax=3 -> step ~0.33.
|
||||
assert!(
|
||||
tier1_max_err <= tier3_max_err,
|
||||
"Tier1 error ({:.6}) should not exceed Tier3 error ({:.6})",
|
||||
tier1_max_err,
|
||||
tier3_max_err
|
||||
);
|
||||
// Tier3 with 3-bit quantization has significant error for [-1,1] data.
|
||||
assert!(
|
||||
tier3_max_err > 0.0,
|
||||
"Tier3 (3-bit) should have nonzero quantization error"
|
||||
);
|
||||
|
||||
println!(
|
||||
"all_tiers_coexist: tier1_err={:.6}, tier2_err={:.6}, tier3_err={:.6}",
|
||||
tier1_max_err, tier2_max_err, tier3_max_err
|
||||
);
|
||||
|
||||
// Evict all Tier3 blocks.
|
||||
for i in 0..BLOCKS_PER_TIER {
|
||||
store
|
||||
.evict(make_key(83, i as u32), ReconstructPolicy::None)
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
assert_eq!(store.tier_count(Tier::Tier3), 0);
|
||||
assert_eq!(store.tier_count(Tier::Tier0), BLOCKS_PER_TIER);
|
||||
// Total blocks unchanged (eviction preserves metadata).
|
||||
assert_eq!(store.block_count(), 3 * BLOCKS_PER_TIER);
|
||||
|
||||
// Tier1 and Tier2 must still be readable.
|
||||
for i in 0..BLOCKS_PER_TIER {
|
||||
let mut out = vec![0.0f32; ELEM_COUNT];
|
||||
|
||||
let key1 = make_key(81, i as u32);
|
||||
store.get(key1, &mut out, 2).unwrap_or_else(|e| {
|
||||
panic!("Tier1 block {} unreadable after Tier3 eviction: {:?}", i, e)
|
||||
});
|
||||
|
||||
let key2 = make_key(82, i as u32);
|
||||
store.get(key2, &mut out, 2).unwrap_or_else(|e| {
|
||||
panic!("Tier2 block {} unreadable after Tier3 eviction: {:?}", i, e)
|
||||
});
|
||||
}
|
||||
|
||||
// Evicted Tier3 blocks should return TensorEvicted.
|
||||
for i in 0..BLOCKS_PER_TIER {
|
||||
let key = make_key(83, i as u32);
|
||||
let mut out = vec![0.0f32; ELEM_COUNT];
|
||||
let result = store.get(key, &mut out, 2);
|
||||
assert_eq!(
|
||||
result,
|
||||
Err(StoreError::TensorEvicted),
|
||||
"evicted Tier3 block {} should return TensorEvicted",
|
||||
i
|
||||
);
|
||||
}
|
||||
|
||||
println!(
|
||||
"all_tiers_coexist: evicted Tier3, Tier1 ({}) and Tier2 ({}) still intact",
|
||||
store.tier_count(Tier::Tier1),
|
||||
store.tier_count(Tier::Tier2)
|
||||
);
|
||||
}
|
||||
353
crates/ruvector-temporal-tensor/tests/wasm_ffi_test.rs
Normal file
353
crates/ruvector-temporal-tensor/tests/wasm_ffi_test.rs
Normal file
@@ -0,0 +1,353 @@
|
||||
//! FFI interface tests for the temporal tensor store.
|
||||
//!
|
||||
//! These tests exercise the `tts_*` extern "C" functions exposed by
|
||||
//! `store_ffi.rs` through their public API. Because the FFI layer uses
|
||||
//! a single global `STORE_STATE`, tests **must** run sequentially:
|
||||
//!
|
||||
//! ```bash
|
||||
//! cargo test -p ruvector-temporal-tensor --test wasm_ffi_test --features ffi -- --test-threads=1
|
||||
//! ```
|
||||
#![cfg(feature = "ffi")]
|
||||
|
||||
use ruvector_temporal_tensor::store_ffi::{
|
||||
tts_block_count, tts_evict, tts_get, tts_init, tts_put, tts_stats, tts_tier_count, tts_touch,
|
||||
};
|
||||
|
||||
// ── Constants mirrored from store_ffi.rs ────────────────────────────────
|
||||
|
||||
const ERR_BLOCK_NOT_FOUND: i32 = -4;
|
||||
const ERR_BUFFER_TOO_SMALL: i32 = -5;
|
||||
|
||||
/// Binary stats size: 5 * u32 + 2 * u64 = 36 bytes.
|
||||
const STATS_SIZE: usize = 5 * 4 + 2 * 8;
|
||||
|
||||
// ── Helpers ─────────────────────────────────────────────────────────────
|
||||
|
||||
/// Re-initialize the global store with default config before each test.
|
||||
/// This replaces whatever state was left by a previous test.
|
||||
fn reset() {
|
||||
let rc = tts_init(std::ptr::null(), 0);
|
||||
assert_eq!(rc, 0, "tts_init with default config must succeed");
|
||||
}
|
||||
|
||||
/// Read a little-endian u32 from `buf` at the given byte offset.
|
||||
fn read_u32_le(buf: &[u8], off: usize) -> u32 {
|
||||
u32::from_le_bytes([buf[off], buf[off + 1], buf[off + 2], buf[off + 3]])
|
||||
}
|
||||
|
||||
/// Read a little-endian u64 from `buf` at the given byte offset.
|
||||
fn read_u64_le(buf: &[u8], off: usize) -> u64 {
|
||||
let mut arr = [0u8; 8];
|
||||
arr.copy_from_slice(&buf[off..off + 8]);
|
||||
u64::from_le_bytes(arr)
|
||||
}
|
||||
|
||||
// ── Tests ───────────────────────────────────────────────────────────────
|
||||
|
||||
#[test]
|
||||
fn test_ffi_init_and_destroy() {
|
||||
// Calling tts_init with a null pointer and zero length should use
|
||||
// the default TierConfig and return success (0).
|
||||
let rc = tts_init(std::ptr::null(), 0);
|
||||
assert_eq!(rc, 0, "tts_init should return 0 on success");
|
||||
|
||||
// The freshly initialized store must contain zero blocks.
|
||||
assert_eq!(tts_block_count(), 0, "new store should have 0 blocks");
|
||||
|
||||
// Re-initializing must also succeed (replaces old state).
|
||||
let rc2 = tts_init(std::ptr::null(), 0);
|
||||
assert_eq!(rc2, 0, "re-init should succeed");
|
||||
assert_eq!(tts_block_count(), 0, "re-init should reset block count");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_ffi_put_get_roundtrip() {
|
||||
reset();
|
||||
|
||||
// Create 64 f32 values with a clear pattern.
|
||||
let data: Vec<f32> = (0..64).map(|i| (i as f32 - 32.0) * 0.1).collect();
|
||||
|
||||
let rc = tts_put(0, 1, 0, data.as_ptr(), data.len());
|
||||
assert_eq!(rc, 0, "tts_put should return 0 on success");
|
||||
|
||||
let mut out = vec![0.0f32; 64];
|
||||
let n = tts_get(0, 1, 0, out.as_mut_ptr(), out.len());
|
||||
assert_eq!(n, 64, "tts_get should return 64 elements");
|
||||
|
||||
// Verify accuracy. New blocks default to Hot (8-bit quantization)
|
||||
// so the error should be small.
|
||||
let max_abs = data.iter().map(|v| v.abs()).fold(0.0f32, f32::max);
|
||||
for (i, (&orig, &dec)) in data.iter().zip(out.iter()).enumerate() {
|
||||
let err = (orig - dec).abs();
|
||||
assert!(
|
||||
err < max_abs * 0.05,
|
||||
"element {i}: orig={orig}, decoded={dec}, err={err}, tolerance={}",
|
||||
max_abs * 0.05,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_ffi_multi_tensor() {
|
||||
reset();
|
||||
|
||||
let data_a: Vec<f32> = (0..64).map(|i| i as f32 * 0.5).collect();
|
||||
let data_b: Vec<f32> = (0..64).map(|i| -(i as f32) * 0.3).collect();
|
||||
let data_c: Vec<f32> = (0..64).map(|i| (i as f32).sin()).collect();
|
||||
|
||||
// Three different tensor IDs using hi/lo split for u128:
|
||||
// tensor A: hi=0, lo=1 -> tensor_id = 1
|
||||
// tensor B: hi=0, lo=2 -> tensor_id = 2
|
||||
// tensor C: hi=1, lo=0 -> tensor_id = 1 << 64
|
||||
assert_eq!(tts_put(0, 1, 0, data_a.as_ptr(), data_a.len()), 0);
|
||||
assert_eq!(tts_put(0, 2, 0, data_b.as_ptr(), data_b.len()), 0);
|
||||
assert_eq!(tts_put(1, 0, 0, data_c.as_ptr(), data_c.len()), 0);
|
||||
|
||||
assert_eq!(tts_block_count(), 3, "should have 3 blocks total");
|
||||
|
||||
// Read back each tensor independently.
|
||||
let mut out = vec![0.0f32; 64];
|
||||
|
||||
let n_a = tts_get(0, 1, 0, out.as_mut_ptr(), out.len());
|
||||
assert_eq!(n_a, 64);
|
||||
// Spot-check first element of tensor A.
|
||||
assert!(
|
||||
(out[0] - data_a[0]).abs() < 0.5,
|
||||
"tensor A readback mismatch"
|
||||
);
|
||||
|
||||
let n_b = tts_get(0, 2, 0, out.as_mut_ptr(), out.len());
|
||||
assert_eq!(n_b, 64);
|
||||
assert!(
|
||||
(out[0] - data_b[0]).abs() < 0.5,
|
||||
"tensor B readback mismatch"
|
||||
);
|
||||
|
||||
let n_c = tts_get(1, 0, 0, out.as_mut_ptr(), out.len());
|
||||
assert_eq!(n_c, 64);
|
||||
assert!(
|
||||
(out[0] - data_c[0]).abs() < 0.5,
|
||||
"tensor C readback mismatch"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_ffi_eviction() {
|
||||
reset();
|
||||
|
||||
let data = vec![1.0f32; 64];
|
||||
assert_eq!(tts_put(0, 42, 0, data.as_ptr(), data.len()), 0);
|
||||
assert_eq!(tts_block_count(), 1);
|
||||
|
||||
// Evict the block.
|
||||
let rc = tts_evict(0, 42, 0);
|
||||
assert_eq!(rc, 0, "tts_evict should return 0 on success");
|
||||
assert_eq!(tts_block_count(), 0, "evicted block should be gone");
|
||||
|
||||
// A subsequent get should return ERR_BLOCK_NOT_FOUND.
|
||||
let mut out = vec![0.0f32; 64];
|
||||
let rc_get = tts_get(0, 42, 0, out.as_mut_ptr(), out.len());
|
||||
assert_eq!(
|
||||
rc_get, ERR_BLOCK_NOT_FOUND,
|
||||
"get after evict should return block-not-found"
|
||||
);
|
||||
|
||||
// Evicting again should also return block-not-found.
|
||||
let rc2 = tts_evict(0, 42, 0);
|
||||
assert_eq!(rc2, ERR_BLOCK_NOT_FOUND);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_ffi_touch_updates_access() {
|
||||
reset();
|
||||
|
||||
let data = vec![1.0f32; 64];
|
||||
assert_eq!(tts_put(0, 7, 3, data.as_ptr(), data.len()), 0);
|
||||
assert_eq!(tts_block_count(), 1);
|
||||
|
||||
// Touch the block multiple times.
|
||||
for _ in 0..5 {
|
||||
let rc = tts_touch(0, 7, 3);
|
||||
assert_eq!(rc, 0, "tts_touch should return 0 on success");
|
||||
}
|
||||
|
||||
// Block count should remain unchanged (touch does not add/remove blocks).
|
||||
assert_eq!(tts_block_count(), 1, "touch should not change block count");
|
||||
|
||||
// The block should still be readable.
|
||||
let mut out = vec![0.0f32; 64];
|
||||
let n = tts_get(0, 7, 3, out.as_mut_ptr(), out.len());
|
||||
assert_eq!(n, 64, "block should still be readable after touches");
|
||||
|
||||
// Touching a non-existent block should fail.
|
||||
let rc_missing = tts_touch(0, 99, 0);
|
||||
assert_eq!(rc_missing, ERR_BLOCK_NOT_FOUND);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_ffi_tier_counts() {
|
||||
reset();
|
||||
|
||||
// All new blocks are placed in Hot (tier 0) by default.
|
||||
let data = vec![1.0f32; 64];
|
||||
assert_eq!(tts_put(0, 1, 0, data.as_ptr(), data.len()), 0);
|
||||
assert_eq!(tts_put(0, 1, 1, data.as_ptr(), data.len()), 0);
|
||||
assert_eq!(tts_put(0, 2, 0, data.as_ptr(), data.len()), 0);
|
||||
|
||||
assert_eq!(tts_block_count(), 3);
|
||||
assert_eq!(tts_tier_count(0), 3, "all blocks should be Hot");
|
||||
assert_eq!(tts_tier_count(1), 0, "no Warm blocks");
|
||||
assert_eq!(tts_tier_count(2), 0, "no Cool blocks");
|
||||
assert_eq!(tts_tier_count(3), 0, "no Cold blocks");
|
||||
|
||||
// Invalid tier should return an error.
|
||||
assert!(tts_tier_count(99) < 0, "invalid tier should return error");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_ffi_stats_output() {
|
||||
reset();
|
||||
|
||||
let data = vec![1.0f32; 64];
|
||||
assert_eq!(tts_put(0, 1, 0, data.as_ptr(), data.len()), 0);
|
||||
assert_eq!(tts_put(0, 1, 1, data.as_ptr(), data.len()), 0);
|
||||
assert_eq!(tts_put(0, 2, 0, data.as_ptr(), data.len()), 0);
|
||||
|
||||
let mut buf = vec![0u8; STATS_SIZE];
|
||||
let written = tts_stats(buf.as_mut_ptr(), buf.len());
|
||||
assert_eq!(
|
||||
written, STATS_SIZE as i32,
|
||||
"tts_stats should write exactly {STATS_SIZE} bytes"
|
||||
);
|
||||
|
||||
// Parse the binary stats layout:
|
||||
// [block_count:u32][hot:u32][warm:u32][cool:u32][cold:u32]
|
||||
// [total_bytes:u64][tick_count:u64]
|
||||
let block_count = read_u32_le(&buf, 0);
|
||||
let hot = read_u32_le(&buf, 4);
|
||||
let warm = read_u32_le(&buf, 8);
|
||||
let cool = read_u32_le(&buf, 12);
|
||||
let cold = read_u32_le(&buf, 16);
|
||||
let total_bytes = read_u64_le(&buf, 20);
|
||||
let _tick_count = read_u64_le(&buf, 28);
|
||||
|
||||
assert_eq!(block_count, 3, "block_count mismatch");
|
||||
assert_eq!(hot, 3, "hot count mismatch");
|
||||
assert_eq!(warm, 0, "warm count mismatch");
|
||||
assert_eq!(cool, 0, "cool count mismatch");
|
||||
assert_eq!(cold, 0, "cold count mismatch");
|
||||
assert!(total_bytes > 0, "total_bytes should be > 0 after puts");
|
||||
|
||||
// Verify stats rejects a too-small buffer.
|
||||
let mut small_buf = vec![0u8; 4];
|
||||
let rc = tts_stats(small_buf.as_mut_ptr(), small_buf.len());
|
||||
assert_eq!(rc, ERR_BUFFER_TOO_SMALL);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_ffi_put_multiple_blocks_same_tensor() {
|
||||
reset();
|
||||
|
||||
let data = vec![2.5f32; 64];
|
||||
|
||||
// Put 5 blocks for the same tensor (different block indices).
|
||||
for idx in 0..5u32 {
|
||||
let rc = tts_put(0, 10, idx, data.as_ptr(), data.len());
|
||||
assert_eq!(rc, 0, "put block_index={idx} should succeed");
|
||||
}
|
||||
|
||||
assert_eq!(tts_block_count(), 5);
|
||||
|
||||
// Each block should be independently readable.
|
||||
let mut out = vec![0.0f32; 64];
|
||||
for idx in 0..5u32 {
|
||||
let n = tts_get(0, 10, idx, out.as_mut_ptr(), out.len());
|
||||
assert_eq!(n, 64, "block_index={idx} should return 64 elements");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_ffi_overwrite_block() {
|
||||
reset();
|
||||
|
||||
let data1 = vec![1.0f32; 64];
|
||||
assert_eq!(tts_put(0, 5, 0, data1.as_ptr(), data1.len()), 0);
|
||||
|
||||
let data2 = vec![9.0f32; 64];
|
||||
assert_eq!(tts_put(0, 5, 0, data2.as_ptr(), data2.len()), 0);
|
||||
|
||||
// Block count should still be 1 (overwrite, not insert).
|
||||
assert_eq!(tts_block_count(), 1);
|
||||
|
||||
// Should read back the second write.
|
||||
let mut out = vec![0.0f32; 64];
|
||||
let n = tts_get(0, 5, 0, out.as_mut_ptr(), out.len());
|
||||
assert_eq!(n, 64);
|
||||
for &v in &out {
|
||||
assert!(
|
||||
(v - 9.0).abs() < 0.5,
|
||||
"expected ~9.0 after overwrite, got {v}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_ffi_get_buffer_too_small() {
|
||||
reset();
|
||||
|
||||
let data = vec![1.0f32; 64];
|
||||
assert_eq!(tts_put(0, 1, 0, data.as_ptr(), data.len()), 0);
|
||||
|
||||
let mut small_out = vec![0.0f32; 2];
|
||||
let rc = tts_get(0, 1, 0, small_out.as_mut_ptr(), small_out.len());
|
||||
assert_eq!(
|
||||
rc, ERR_BUFFER_TOO_SMALL,
|
||||
"get with undersized buffer should return buffer-too-small"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_ffi_evict_then_reinsert() {
|
||||
reset();
|
||||
|
||||
let data = vec![3.0f32; 64];
|
||||
assert_eq!(tts_put(0, 1, 0, data.as_ptr(), data.len()), 0);
|
||||
assert_eq!(tts_block_count(), 1);
|
||||
|
||||
// Evict.
|
||||
assert_eq!(tts_evict(0, 1, 0), 0);
|
||||
assert_eq!(tts_block_count(), 0);
|
||||
|
||||
// Re-insert at the same key.
|
||||
let data2 = vec![7.0f32; 64];
|
||||
assert_eq!(tts_put(0, 1, 0, data2.as_ptr(), data2.len()), 0);
|
||||
assert_eq!(tts_block_count(), 1);
|
||||
|
||||
// Should read back the new data.
|
||||
let mut out = vec![0.0f32; 64];
|
||||
let n = tts_get(0, 1, 0, out.as_mut_ptr(), out.len());
|
||||
assert_eq!(n, 64);
|
||||
for &v in &out {
|
||||
assert!(
|
||||
(v - 7.0).abs() < 0.5,
|
||||
"expected ~7.0 after re-insert, got {v}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_ffi_large_tensor_id() {
|
||||
reset();
|
||||
|
||||
// Use the full u128 range: hi=u64::MAX, lo=u64::MAX -> tensor_id = u128::MAX.
|
||||
let data = vec![0.5f32; 64];
|
||||
assert_eq!(
|
||||
tts_put(u64::MAX, u64::MAX, 0, data.as_ptr(), data.len()),
|
||||
0,
|
||||
"put with max tensor_id should succeed"
|
||||
);
|
||||
|
||||
let mut out = vec![0.0f32; 64];
|
||||
let n = tts_get(u64::MAX, u64::MAX, 0, out.as_mut_ptr(), out.len());
|
||||
assert_eq!(n, 64, "get with max tensor_id should succeed");
|
||||
}
|
||||
Reference in New Issue
Block a user