Merge commit 'd803bfe2b1fe7f5e219e50ac20d6801a0a58ac75' as 'vendor/ruvector'

vendor/ruvector/crates/ruvector-sparse-inference/tests/unit/predictor_tests.rs

@@ -0,0 +1,159 @@
+//! Unit tests for neuron predictors
+
+use ruvector_sparse_inference::predictor::*;
+
+mod common;
+use common::*;
+
+#[test]
+fn test_lowrank_predictor_creation() {
+    let predictor = LowRankPredictor::new(512, 4096, 128, 0.1);
+    assert_eq!(predictor.input_dim(), 512);
+    assert_eq!(predictor.hidden_dim(), 4096);
+    assert_eq!(predictor.rank(), 128);
+}
+
+#[test]
+fn test_predictor_predicts_active_neurons() {
+    let predictor = create_calibrated_predictor();
+    let input = vec![0.1f32; 512];
+
+    let active = predictor.predict(&input);
+
+    // Should predict some neurons as active
+    assert!(!active.is_empty(), "Predictor should activate some neurons");
+    // Should predict fewer than total neurons (sparsity)
+    assert!(active.len() < 4096, "Predictor should be sparse");
+    // All indices should be valid
+    assert!(active.iter().all(|&i| i < 4096), "All indices should be valid");
+}
+
+#[test]
+fn test_predictor_top_k_mode() {
+    let mut predictor = LowRankPredictor::new(512, 4096, 128, 0.0);
+    predictor.set_top_k(Some(100));
+
+    let input = vec![0.1f32; 512];
+    let active = predictor.predict(&input);
+
+    assert_eq!(active.len(), 100, "Top-K should return exactly K neurons");
+}
+
+#[test]
+fn test_predictor_top_k_larger_than_hidden() {
+    let mut predictor = LowRankPredictor::new(512, 100, 64, 0.0);
+    predictor.set_top_k(Some(200)); // More than hidden_dim
+
+    let input = random_vector(512);
+    let active = predictor.predict(&input);
+
+    // Should return at most hidden_dim neurons
+    assert!(active.len() <= 100);
+}
+
+#[test]
+fn test_predictor_calibration() {
+    let mut predictor = LowRankPredictor::new(512, 4096, 128, 0.5);
+
+    // Initial threshold
+    let initial_threshold = 0.5;
+
+    // Generate calibration data
+    let samples: Vec<_> = (0..100)
+        .map(|_| random_vector(512))
+        .collect();
+
+    let activations: Vec<_> = (0..100)
+        .map(|_| {
+            // Simulate 30% activation rate
+            let num_active = (4096 as f32 * 0.3) as usize;
+            (0..num_active).collect::<Vec<_>>()
+        })
+        .collect();
+
+    predictor.calibrate(&samples, &activations);
+
+    // After calibration, predictor should make better predictions
+    let test_input = random_vector(512);
+    let active = predictor.predict(&test_input);
+    assert!(!active.is_empty(), "Calibrated predictor should activate neurons");
+}
+
+#[test]
+fn test_predictor_different_inputs_different_outputs() {
+    let predictor = LowRankPredictor::new(512, 4096, 128, 0.1);
+
+    let input1 = random_vector(512);
+    let input2 = random_vector(512);
+
+    let active1 = predictor.predict(&input1);
+    let active2 = predictor.predict(&input2);
+
+    // Different inputs should generally produce different activations
+    // (This test might occasionally fail due to randomness, but should pass most of the time)
+    assert_ne!(active1, active2, "Different inputs should produce different activations");
+}
+
+#[test]
+fn test_dense_predictor_activates_all() {
+    let predictor = DensePredictor::new(4096);
+    let input = random_vector(512);
+
+    let active = predictor.predict(&input);
+
+    assert_eq!(active.len(), 4096, "Dense predictor should activate all neurons");
+    assert_eq!(active, (0..4096).collect::<Vec<_>>(), "Should be sequential indices");
+}
+
+#[test]
+fn test_dense_predictor_num_neurons() {
+    let predictor = DensePredictor::new(2048);
+    assert_eq!(predictor.num_neurons(), 2048);
+}
+
+#[test]
+#[should_panic(expected = "Input dimension mismatch")]
+fn test_predictor_wrong_input_dimension() {
+    let predictor = LowRankPredictor::new(512, 4096, 128, 0.1);
+    let wrong_input = vec![0.1f32; 256]; // Wrong dimension
+
+    predictor.predict(&wrong_input);
+}
+
+#[test]
+fn test_predictor_zero_input() {
+    let predictor = LowRankPredictor::new(512, 4096, 128, 0.1);
+    let zero_input = vec![0.0f32; 512];
+
+    let active = predictor.predict(&zero_input);
+
+    // Zero input should still produce some output (might be threshold-dependent)
+    assert!(active.len() <= 4096, "Should not exceed total neurons");
+}
+
+#[test]
+fn test_predictor_extreme_values() {
+    let predictor = LowRankPredictor::new(512, 4096, 128, 0.1);
+
+    // Test with very large values
+    let large_input = vec![1000.0f32; 512];
+    let active_large = predictor.predict(&large_input);
+    assert!(active_large.iter().all(|&i| i < 4096));
+
+    // Test with very small values
+    let small_input = vec![-1000.0f32; 512];
+    let active_small = predictor.predict(&small_input);
+    assert!(active_small.iter().all(|&i| i < 4096));
+}
+
+#[test]
+fn test_predictor_consistent_predictions() {
+    let predictor = LowRankPredictor::new(512, 4096, 128, 0.1);
+    let input = random_vector(512);
+
+    // Same input should produce same output
+    let active1 = predictor.predict(&input);
+    let active2 = predictor.predict(&input);
+
+    assert_eq!(active1, active2, "Same input should produce same output");
+}

vendor/ruvector/crates/ruvector-sparse-inference/tests/unit/quantization_tests.rs

@@ -0,0 +1,193 @@
+//! Unit tests for weight quantization
+
+use ruvector_sparse_inference::memory::quantization::*;
+
+mod common;
+use common::*;
+
+#[test]
+fn test_int8_quantization_roundtrip() {
+    let original = random_vector(1024);
+    let quantized = QuantizedWeights::quantize_int8(&original);
+    let dequantized = quantized.dequantize_row(0);
+
+    // Should be close after dequantization
+    assert_vectors_close(&original, &dequantized, 0.01);
+}
+
+#[test]
+fn test_int8_quantization_dimensions() {
+    let original = random_vector(1024);
+    let quantized = QuantizedWeights::quantize_int8(&original);
+
+    assert_eq!(quantized.nrows(), 1);
+    assert_eq!(quantized.ncols(), 1024);
+}
+
+#[test]
+fn test_int4_quantization_compression() {
+    let original: Vec<f32> = (0..1024).map(|i| (i as f32) * 0.01).collect();
+    let quantized = QuantizedWeights::quantize_int4(&original, 64); // group_size=64
+
+    // Int4 should be significantly smaller than original (4 bytes per f32)
+    let original_size = original.len() * 4;
+    let quantized_size = quantized.size_bytes();
+
+    assert!(quantized_size < original_size / 4,
+        "Int4 quantization should compress data (original: {}, quantized: {})",
+        original_size, quantized_size);
+}
+
+#[test]
+fn test_int4_quantization_roundtrip() {
+    let original: Vec<f32> = (0..256).map(|i| (i as f32) * 0.01).collect();
+    let quantized = QuantizedWeights::quantize_int4(&original, 32);
+    let dequantized = quantized.dequantize_row(0);
+
+    // Int4 has lower precision, so tolerance is higher
+    assert_vectors_close(&original, &dequantized, 0.05);
+}
+
+#[test]
+fn test_int4_different_group_sizes() {
+    let original = random_vector(512);
+
+    for group_size in [16, 32, 64, 128] {
+        let quantized = QuantizedWeights::quantize_int4(&original, group_size);
+        let dequantized = quantized.dequantize_row(0);
+
+        assert_eq!(original.len(), dequantized.len(),
+            "Length mismatch for group_size {}", group_size);
+        assert_vectors_close(&original, &dequantized, 0.1);
+    }
+}
+
+#[test]
+fn test_selective_dequantization() {
+    // Create a larger matrix to test selective dequantization
+    let rows_data: Vec<Vec<f32>> = (0..100)
+        .map(|_| random_vector(512))
+        .collect();
+
+    // For this test, we'll quantize each row separately and store them
+    // (In real implementation, you'd have a multi-row quantization)
+    let quantized = QuantizedWeights::quantize_int8(&rows_data[0]);
+
+    let selected_rows = vec![0];
+    let dequantized = quantized.dequantize_rows(&selected_rows);
+
+    assert_eq!(dequantized.nrows(), selected_rows.len());
+    assert_eq!(dequantized.ncols(), 512);
+}
+
+#[test]
+fn test_quantization_preserves_range() {
+    let original: Vec<f32> = vec![-5.0, -2.5, 0.0, 2.5, 5.0];
+    let quantized = QuantizedWeights::quantize_int8(&original);
+    let dequantized = quantized.dequantize_row(0);
+
+    // Check that min and max are approximately preserved
+    let orig_min = original.iter().cloned().fold(f32::INFINITY, f32::min);
+    let orig_max = original.iter().cloned().fold(f32::NEG_INFINITY, f32::max);
+    let deq_min = dequantized.iter().cloned().fold(f32::INFINITY, f32::min);
+    let deq_max = dequantized.iter().cloned().fold(f32::NEG_INFINITY, f32::max);
+
+    assert!((orig_min - deq_min).abs() < 0.1);
+    assert!((orig_max - deq_max).abs() < 0.1);
+}
+
+#[test]
+fn test_quantization_uniform_values() {
+    let original = vec![3.14f32; 100];
+    let quantized = QuantizedWeights::quantize_int8(&original);
+    let dequantized = quantized.dequantize_row(0);
+
+    // All values should be approximately the same
+    for &val in &dequantized {
+        assert!((val - 3.14).abs() < 0.1);
+    }
+}
+
+#[test]
+fn test_quantization_zero_values() {
+    let original = vec![0.0f32; 100];
+    let quantized = QuantizedWeights::quantize_int8(&original);
+    let dequantized = quantized.dequantize_row(0);
+
+    // All values should be close to zero
+    for &val in &dequantized {
+        assert!(val.abs() < 0.01);
+    }
+}
+
+#[test]
+fn test_int4_odd_length() {
+    // Test with odd number of elements (tests padding)
+    let original = random_vector(513); // Odd number
+    let quantized = QuantizedWeights::quantize_int4(&original, 32);
+    let dequantized = quantized.dequantize_row(0);
+
+    assert_eq!(original.len(), dequantized.len());
+}
+
+#[test]
+fn test_quantization_size_reduction() {
+    let original = random_vector(4096);
+    let original_size = original.len() * std::mem::size_of::<f32>();
+
+    let int8_quantized = QuantizedWeights::quantize_int8(&original);
+    let int8_size = int8_quantized.size_bytes();
+
+    let int4_quantized = QuantizedWeights::quantize_int4(&original, 64);
+    let int4_size = int4_quantized.size_bytes();
+
+    // Verify compression ratios
+    assert!(int8_size < original_size / 2, "Int8 should be ~4x smaller");
+    assert!(int4_size < int8_size, "Int4 should be smaller than Int8");
+}
+
+#[test]
+fn test_multiple_row_dequantization() {
+    let quantized = create_quantized_matrix(100, 512);
+    let rows = vec![10, 50, 99];
+
+    let dequantized = quantized.dequantize_rows(&rows);
+
+    assert_eq!(dequantized.nrows(), rows.len());
+    assert_eq!(dequantized.ncols(), 512);
+
+    // All values should be finite
+    for i in 0..dequantized.nrows() {
+        for j in 0..dequantized.ncols() {
+            assert!(dequantized[[i, j]].is_finite());
+        }
+    }
+}
+
+#[test]
+#[should_panic(expected = "Row index out of bounds")]
+fn test_dequantize_out_of_bounds_row() {
+    let quantized = QuantizedWeights::quantize_int8(&random_vector(512));
+    quantized.dequantize_row(5); // Only 1 row exists
+}
+
+#[test]
+fn test_quantization_large_values() {
+    let original = vec![1000.0, 5000.0, -3000.0, 10000.0];
+    let quantized = QuantizedWeights::quantize_int8(&original);
+    let dequantized = quantized.dequantize_row(0);
+
+    // Should handle large values reasonably
+    assert_vectors_close(&original, &dequantized, 100.0); // Higher tolerance for large values
+}
+
+#[test]
+fn test_int4_group_boundary() {
+    // Test that group boundaries are handled correctly
+    let original = random_vector(128);
+    let quantized = QuantizedWeights::quantize_int4(&original, 32); // 4 groups exactly
+    let dequantized = quantized.dequantize_row(0);
+
+    assert_eq!(original.len(), dequantized.len());
+    assert_vectors_close(&original, &dequantized, 0.1);
+}

vendor/ruvector/crates/ruvector-sparse-inference/tests/unit/sparse_ffn_tests.rs

@@ -0,0 +1,187 @@
+//! Unit tests for sparse feed-forward networks
+
+use ruvector_sparse_inference::sparse::*;
+
+mod common;
+use common::*;
+
+#[test]
+fn test_sparse_ffn_matches_dense() {
+    let ffn = create_test_ffn(512, 2048);
+    let input = random_vector(512);
+    let all_neurons: Vec<usize> = (0..2048).collect();
+
+    let dense_output = ffn.forward_dense(&input);
+    let sparse_output = ffn.forward_sparse(&input, &all_neurons);
+
+    // When all neurons are active, sparse should match dense
+    assert_vectors_close(&dense_output, &sparse_output, 1e-5);
+}
+
+#[test]
+fn test_sparse_ffn_with_subset() {
+    let ffn = create_test_ffn(512, 2048);
+    let input = random_vector(512);
+    let active_neurons: Vec<usize> = (0..1024).collect(); // 50% sparsity
+
+    let output = ffn.forward_sparse(&input, &active_neurons);
+
+    assert_eq!(output.len(), 512, "Output dimension should match input dimension");
+    assert!(output.iter().all(|&x| x.is_finite()), "All outputs should be finite");
+}
+
+#[test]
+fn test_sparse_ffn_empty_activations() {
+    let ffn = create_test_ffn(512, 2048);
+    let input = random_vector(512);
+    let no_neurons: Vec<usize> = vec![];
+
+    let output = ffn.forward_sparse(&input, &no_neurons);
+
+    assert_eq!(output.len(), 512);
+    // With no active neurons, output should be near zero
+    assert!(output.iter().all(|&x| x.abs() < 1e-5), "Output should be near zero with no active neurons");
+}
+
+#[test]
+fn test_different_activations() {
+    for activation in [ActivationType::Relu, ActivationType::Gelu, ActivationType::Silu] {
+        let ffn = SparseFfn::new(512, 2048, activation);
+        let input = random_vector(512);
+        let active: Vec<usize> = (0..500).collect();
+
+        let output = ffn.forward_sparse(&input, &active);
+        assert_eq!(output.len(), 512, "Output dimension should be 512 for {:?}", activation);
+        assert!(output.iter().all(|&x| x.is_finite()), "All outputs should be finite for {:?}", activation);
+    }
+}
+
+#[test]
+fn test_relu_activation_properties() {
+    let ffn = SparseFfn::new(512, 2048, ActivationType::Relu);
+    let input = vec![-1.0f32; 512]; // Negative input
+    let all_neurons: Vec<usize> = (0..2048).collect();
+
+    let output = ffn.forward_dense(&input);
+
+    // ReLU should zero out negative activations
+    // (though final output might still be negative due to w2 projection)
+    assert!(output.iter().all(|&x| x.is_finite()));
+}
+
+#[test]
+fn test_swiglu_paired_neurons() {
+    // SwiGLU uses paired neurons (gate and up projections)
+    let ffn = SwiGLUFfn::new(512, 2048);
+    let input = random_vector(512);
+
+    // Active neurons should be pairs
+    let active_pairs: Vec<usize> = (0..500).map(|i| i * 2).collect();
+    let output = ffn.forward_sparse(&input, &active_pairs);
+
+    assert_eq!(output.len(), 512);
+    assert!(output.iter().all(|&x| x.is_finite()));
+}
+
+#[test]
+fn test_swiglu_matches_dense() {
+    let ffn = SwiGLUFfn::new(512, 2048);
+    let input = random_vector(512);
+    let all_neurons: Vec<usize> = (0..2048).collect();
+
+    let dense_output = ffn.forward_dense(&input);
+    let sparse_output = ffn.forward_sparse(&input, &all_neurons);
+
+    assert_vectors_close(&dense_output, &sparse_output, 1e-5);
+}
+
+#[test]
+fn test_swiglu_empty_activations() {
+    let ffn = SwiGLUFfn::new(512, 2048);
+    let input = random_vector(512);
+    let no_neurons: Vec<usize> = vec![];
+
+    let output = ffn.forward_sparse(&input, &no_neurons);
+
+    assert_eq!(output.len(), 512);
+    assert!(output.iter().all(|&x| x.abs() < 1e-5));
+}
+
+#[test]
+#[should_panic(expected = "Input dimension mismatch")]
+fn test_sparse_ffn_wrong_input_dimension() {
+    let ffn = create_test_ffn(512, 2048);
+    let wrong_input = vec![0.1f32; 256];
+    let active: Vec<usize> = (0..100).collect();
+
+    ffn.forward_sparse(&wrong_input, &active);
+}
+
+#[test]
+fn test_sparse_ffn_out_of_bounds_neurons() {
+    let ffn = create_test_ffn(512, 2048);
+    let input = random_vector(512);
+
+    // Include some out-of-bounds indices
+    let mut active: Vec<usize> = (0..100).collect();
+    active.push(5000); // Out of bounds
+    active.push(10000); // Out of bounds
+
+    let output = ffn.forward_sparse(&input, &active);
+
+    // Should handle gracefully
+    assert_eq!(output.len(), 512);
+    assert!(output.iter().all(|&x| x.is_finite()));
+}
+
+#[test]
+fn test_sparse_ffn_duplicate_neurons() {
+    let ffn = create_test_ffn(512, 2048);
+    let input = random_vector(512);
+
+    // Include duplicate indices
+    let active = vec![10, 20, 10, 30, 20, 10];
+
+    let output = ffn.forward_sparse(&input, &active);
+
+    assert_eq!(output.len(), 512);
+    assert!(output.iter().all(|&x| x.is_finite()));
+}
+
+#[test]
+fn test_sparse_ffn_sparsity_reduces_computation() {
+    let ffn = create_test_ffn(512, 2048);
+    let input = random_vector(512);
+
+    // 10% sparsity
+    let sparse_neurons: Vec<usize> = (0..204).collect();
+
+    let sparse_output = ffn.forward_sparse(&input, &sparse_neurons);
+
+    // Should still produce valid output with much less computation
+    assert_eq!(sparse_output.len(), 512);
+    assert!(sparse_output.iter().all(|&x| x.is_finite()));
+}
+
+#[test]
+fn test_dense_output_deterministic() {
+    let ffn = create_test_ffn(512, 2048);
+    let input = random_vector(512);
+
+    let output1 = ffn.forward_dense(&input);
+    let output2 = ffn.forward_dense(&input);
+
+    assert_vectors_close(&output1, &output2, 1e-10);
+}
+
+#[test]
+fn test_sparse_output_deterministic() {
+    let ffn = create_test_ffn(512, 2048);
+    let input = random_vector(512);
+    let active: Vec<usize> = (0..500).collect();
+
+    let output1 = ffn.forward_sparse(&input, &active);
+    let output2 = ffn.forward_sparse(&input, &active);
+
+    assert_vectors_close(&output1, &output2, 1e-10);
+}