Merge commit 'd803bfe2b1fe7f5e219e50ac20d6801a0a58ac75' as 'vendor/ruvector'

vendor/ruvector/crates/profiling/README.md

@@ -0,0 +1,84 @@
+# Ruvector Performance Profiling
+
+This directory contains profiling scripts, reports, and analysis for Ruvector performance optimization.
+
+## Directory Structure
+
+```
+profiling/
+├── scripts/          # Profiling and benchmarking scripts
+├── reports/          # Generated profiling reports
+├── flamegraphs/      # CPU flamegraphs
+├── memory/           # Memory profiling data
+└── benchmarks/       # Benchmark results
+```
+
+## Profiling Tools
+
+### CPU Profiling
+- **perf**: Linux performance counters
+- **flamegraph**: Visualization of CPU hotspots
+- **cargo-flamegraph**: Integrated Rust profiling
+
+### Memory Profiling
+- **valgrind**: Memory leak detection and profiling
+- **heaptrack**: Heap memory profiling
+- **massif**: Heap profiler
+
+### Cache Profiling
+- **perf-cache**: Cache miss analysis
+- **cachegrind**: Cache simulation
+
+## Quick Start
+
+```bash
+# Install profiling tools
+./scripts/install_tools.sh
+
+# Run CPU profiling
+./scripts/cpu_profile.sh
+
+# Run memory profiling
+./scripts/memory_profile.sh
+
+# Generate flamegraph
+./scripts/generate_flamegraph.sh
+
+# Run comprehensive benchmark suite
+./scripts/benchmark_all.sh
+```
+
+## Performance Targets
+
+- **Throughput**: 50,000+ queries per second (QPS)
+- **Latency**: Sub-millisecond p50 latency (<1ms)
+- **Recall**: 95% recall at high QPS
+- **Memory**: Efficient memory usage with minimal allocations in hot paths
+- **Scalability**: Linear scaling from 1-128 threads
+
+## Optimization Areas
+
+1. **CPU Optimization**
+   - SIMD intrinsics (AVX2/AVX-512)
+   - Target-specific compilation
+   - Hot path optimization
+
+2. **Memory Optimization**
+   - Arena allocation
+   - Object pooling
+   - Zero-copy operations
+
+3. **Cache Optimization**
+   - Structure-of-Arrays layout
+   - Cache-line alignment
+   - Prefetching
+
+4. **Concurrency Optimization**
+   - Lock-free data structures
+   - RwLock optimization
+   - Rayon tuning
+
+5. **Compile-Time Optimization**
+   - Profile-Guided Optimization (PGO)
+   - Link-Time Optimization (LTO)
+   - Target CPU features

vendor/ruvector/crates/profiling/scripts/benchmark_all.sh

@@ -0,0 +1,72 @@
+#!/bin/bash
+# Comprehensive benchmarking script
+
+set -e
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
+BENCHMARK_DIR="$SCRIPT_DIR/../benchmarks"
+
+mkdir -p "$BENCHMARK_DIR"
+
+echo "📊 Running comprehensive benchmark suite..."
+
+cd "$PROJECT_ROOT"
+
+# Get CPU info
+echo "CPU Information:" > "$BENCHMARK_DIR/system_info.txt"
+lscpu >> "$BENCHMARK_DIR/system_info.txt" 2>&1 || true
+echo "" >> "$BENCHMARK_DIR/system_info.txt"
+echo "Memory Information:" >> "$BENCHMARK_DIR/system_info.txt"
+free -h >> "$BENCHMARK_DIR/system_info.txt" 2>&1 || true
+
+# Build with different optimization levels
+echo "Building with optimizations..."
+cargo build --release
+
+# Run criterion benchmarks
+echo "Running criterion benchmarks..."
+cargo bench --bench distance_metrics -- --save-baseline before 2>&1 | tee "$BENCHMARK_DIR/distance_metrics.txt"
+cargo bench --bench hnsw_search -- --save-baseline before 2>&1 | tee "$BENCHMARK_DIR/hnsw_search.txt"
+
+# Test with different thread counts
+echo "Benchmarking with different thread counts..."
+for threads in 1 2 4 8 16 32; do
+    echo "Testing with $threads threads..."
+    RAYON_NUM_THREADS=$threads cargo bench --bench distance_metrics -- --profile-time=5 \
+        2>&1 | tee "$BENCHMARK_DIR/threads_${threads}.txt"
+done
+
+# Generate summary
+echo "Generating benchmark summary..."
+cat > "$BENCHMARK_DIR/summary.txt" << 'EOF'
+# Ruvector Performance Benchmark Summary
+
+## Test Environment
+$(cat system_info.txt)
+
+## Benchmark Results
+
+### Distance Metrics
+$(grep "time:" distance_metrics.txt | head -20)
+
+### HNSW Search
+$(grep "time:" hnsw_search.txt | head -20)
+
+### Thread Scaling
+EOF
+
+for threads in 1 2 4 8 16 32; do
+    echo "#### $threads threads" >> "$BENCHMARK_DIR/summary.txt"
+    grep "time:" "$BENCHMARK_DIR/threads_${threads}.txt" | head -5 >> "$BENCHMARK_DIR/summary.txt" || true
+    echo "" >> "$BENCHMARK_DIR/summary.txt"
+done
+
+echo "✅ Benchmark suite complete!"
+echo "Results saved to: $BENCHMARK_DIR"
+echo ""
+echo "Key files:"
+echo "  - distance_metrics.txt: Distance calculation benchmarks"
+echo "  - hnsw_search.txt: HNSW search benchmarks"
+echo "  - threads_*.txt: Thread scaling tests"
+echo "  - summary.txt: Overall benchmark summary"

vendor/ruvector/crates/profiling/scripts/cpu_profile.sh

@@ -0,0 +1,52 @@
+#!/bin/bash
+# CPU profiling script using perf
+
+set -e
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
+REPORTS_DIR="$SCRIPT_DIR/../reports"
+
+mkdir -p "$REPORTS_DIR"
+
+echo "🔥 Running CPU profiling with perf..."
+echo "Project root: $PROJECT_ROOT"
+
+cd "$PROJECT_ROOT"
+
+# Build in release mode with debug symbols
+echo "Building with debug symbols..."
+CARGO_PROFILE_RELEASE_DEBUG=true cargo build --release
+
+# Run perf record on benchmarks
+echo "Recording performance data..."
+sudo perf record -F 99 -g --call-graph=dwarf \
+    cargo bench --bench distance_metrics -- --profile-time=10 \
+    2>&1 | tee "$REPORTS_DIR/perf_record.log" || true
+
+# Generate perf report
+echo "Generating perf report..."
+sudo perf report --stdio > "$REPORTS_DIR/perf_report.txt" || true
+sudo perf report --stdio --sort=dso,symbol > "$REPORTS_DIR/perf_report_detailed.txt" || true
+
+# Generate annotated source
+echo "Generating annotated source..."
+sudo perf annotate --stdio > "$REPORTS_DIR/perf_annotate.txt" || true
+
+# Analyze cache performance
+echo "Analyzing cache performance..."
+sudo perf stat -e cache-references,cache-misses,L1-dcache-loads,L1-dcache-load-misses \
+    cargo bench --bench distance_metrics -- --profile-time=5 \
+    2>&1 | tee "$REPORTS_DIR/cache_stats.txt" || true
+
+echo "✅ CPU profiling complete!"
+echo "Reports saved to: $REPORTS_DIR"
+echo ""
+echo "Key files:"
+echo "  - perf_report.txt: Overall performance report"
+echo "  - perf_report_detailed.txt: Detailed symbol analysis"
+echo "  - perf_annotate.txt: Annotated source code"
+echo "  - cache_stats.txt: Cache performance statistics"
+
+# Cleanup
+sudo chown -R $USER:$USER perf.data "$REPORTS_DIR" 2>/dev/null || true

vendor/ruvector/crates/profiling/scripts/generate_flamegraph.sh

@@ -0,0 +1,32 @@
+#!/bin/bash
+# Generate flamegraphs for CPU profiling
+
+set -e
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
+FLAMEGRAPH_DIR="$SCRIPT_DIR/../flamegraphs"
+
+mkdir -p "$FLAMEGRAPH_DIR"
+
+echo "🔥 Generating flamegraphs..."
+
+cd "$PROJECT_ROOT"
+
+# Generate flamegraph for distance metrics benchmark
+echo "Flamegraph: distance_metrics..."
+sudo cargo flamegraph --bench distance_metrics --output="$FLAMEGRAPH_DIR/distance_metrics.svg" -- --profile-time=30 || echo "Failed to generate distance_metrics flamegraph"
+
+# Generate flamegraph for HNSW search benchmark
+echo "Flamegraph: hnsw_search..."
+sudo cargo flamegraph --bench hnsw_search --output="$FLAMEGRAPH_DIR/hnsw_search.svg" -- --profile-time=30 || echo "Failed to generate hnsw_search flamegraph"
+
+# Change ownership
+sudo chown -R $USER:$USER "$FLAMEGRAPH_DIR" 2>/dev/null || true
+
+echo "✅ Flamegraph generation complete!"
+echo "Flamegraphs saved to: $FLAMEGRAPH_DIR"
+echo ""
+echo "View flamegraphs:"
+echo "  firefox $FLAMEGRAPH_DIR/distance_metrics.svg"
+echo "  firefox $FLAMEGRAPH_DIR/hnsw_search.svg"

vendor/ruvector/crates/profiling/scripts/install_tools.sh

@@ -0,0 +1,60 @@
+#!/bin/bash
+# Install profiling and benchmarking tools
+
+set -e
+
+echo "Installing Ruvector profiling tools..."
+
+# Install perf (Linux performance tools)
+if ! command -v perf &> /dev/null; then
+    echo "Installing perf..."
+    sudo apt-get update
+    sudo apt-get install -y linux-tools-common linux-tools-generic linux-tools-$(uname -r) || true
+fi
+
+# Install valgrind
+if ! command -v valgrind &> /dev/null; then
+    echo "Installing valgrind..."
+    sudo apt-get install -y valgrind
+fi
+
+# Install heaptrack
+if ! command -v heaptrack &> /dev/null; then
+    echo "Installing heaptrack..."
+    sudo apt-get install -y heaptrack || echo "heaptrack not available, skipping..."
+fi
+
+# Install flamegraph tools
+if ! command -v cargo-flamegraph &> /dev/null; then
+    echo "Installing cargo-flamegraph..."
+    cargo install flamegraph
+fi
+
+# Install cargo benchmarking tools
+if ! command -v cargo-criterion &> /dev/null; then
+    echo "Installing cargo-criterion..."
+    cargo install cargo-criterion || echo "cargo-criterion installation failed, using built-in criterion"
+fi
+
+# Install hyperfine for command-line benchmarking
+if ! command -v hyperfine &> /dev/null; then
+    echo "Installing hyperfine..."
+    cargo install hyperfine
+fi
+
+# Install cargo-bench-cmp for comparing benchmarks
+if ! command -v cargo-bench-cmp &> /dev/null; then
+    echo "Installing cargo-bench-cmp..."
+    cargo install cargo-bench-cmp || echo "cargo-bench-cmp not available, skipping..."
+fi
+
+echo "✅ Profiling tools installation complete!"
+echo ""
+echo "Available tools:"
+echo "  - perf: CPU profiling"
+echo "  - valgrind: Memory profiling"
+echo "  - heaptrack: Heap profiling"
+echo "  - cargo-flamegraph: Flamegraph generation"
+echo "  - hyperfine: Command-line benchmarking"
+echo ""
+echo "Note: Some tools may require sudo privileges to run."

vendor/ruvector/crates/profiling/scripts/memory_profile.sh

@@ -0,0 +1,59 @@
+#!/bin/bash
+# Memory profiling script using valgrind and heaptrack
+
+set -e
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
+MEMORY_DIR="$SCRIPT_DIR/../memory"
+
+mkdir -p "$MEMORY_DIR"
+
+echo "💾 Running memory profiling..."
+
+cd "$PROJECT_ROOT"
+
+# Build in release mode
+echo "Building release binary..."
+cargo build --release --bin ruvector-cli 2>&1 | tee "$MEMORY_DIR/build.log"
+
+# Run valgrind memcheck
+echo "Running valgrind memcheck..."
+valgrind --leak-check=full \
+    --show-leak-kinds=all \
+    --track-origins=yes \
+    --verbose \
+    --log-file="$MEMORY_DIR/valgrind_memcheck.txt" \
+    target/release/ruvector-cli --version || echo "Valgrind memcheck completed with issues"
+
+# Run valgrind massif (heap profiler)
+echo "Running valgrind massif..."
+valgrind --tool=massif \
+    --massif-out-file="$MEMORY_DIR/massif.out" \
+    target/release/ruvector-cli --version || echo "Massif completed"
+
+# Generate massif report
+echo "Generating massif report..."
+ms_print "$MEMORY_DIR/massif.out" > "$MEMORY_DIR/massif_report.txt" || true
+
+# Run heaptrack if available
+if command -v heaptrack &> /dev/null; then
+    echo "Running heaptrack..."
+    heaptrack --output="$MEMORY_DIR/heaptrack.data" \
+        target/release/ruvector-cli --version || echo "Heaptrack completed"
+
+    echo "Analyzing heaptrack data..."
+    heaptrack_print "$MEMORY_DIR/heaptrack.data" > "$MEMORY_DIR/heaptrack_report.txt" || true
+else
+    echo "Heaptrack not available, skipping..."
+fi
+
+echo "✅ Memory profiling complete!"
+echo "Reports saved to: $MEMORY_DIR"
+echo ""
+echo "Key files:"
+echo "  - valgrind_memcheck.txt: Memory leak analysis"
+echo "  - massif_report.txt: Heap usage over time"
+if command -v heaptrack &> /dev/null; then
+    echo "  - heaptrack_report.txt: Detailed heap allocations"
+fi

vendor/ruvector/crates/profiling/scripts/run_all_analysis.sh

@@ -0,0 +1,155 @@
+#!/bin/bash
+# Comprehensive profiling and analysis script
+
+set -e
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
+
+echo "=========================================="
+echo " Ruvector Performance Analysis Suite"
+echo "=========================================="
+echo ""
+echo "Project: $PROJECT_ROOT"
+echo "Date: $(date)"
+echo ""
+
+cd "$PROJECT_ROOT"
+
+# Step 1: Install tools
+echo "Step 1: Checking tools..."
+"$SCRIPT_DIR/install_tools.sh" || echo "Some tools may not be available"
+echo ""
+
+# Step 2: Run benchmarks (baseline)
+echo "Step 2: Running baseline benchmarks..."
+"$SCRIPT_DIR/benchmark_all.sh"
+echo ""
+
+# Step 3: CPU profiling
+echo "Step 3: CPU profiling..."
+"$SCRIPT_DIR/cpu_profile.sh"
+echo ""
+
+# Step 4: Generate flamegraphs
+echo "Step 4: Generating flamegraphs..."
+"$SCRIPT_DIR/generate_flamegraph.sh"
+echo ""
+
+# Step 5: Memory profiling
+echo "Step 5: Memory profiling..."
+"$SCRIPT_DIR/memory_profile.sh"
+echo ""
+
+# Step 6: Generate comprehensive report
+echo "Step 6: Generating comprehensive report..."
+
+REPORT_FILE="$SCRIPT_DIR/../reports/COMPREHENSIVE_REPORT.md"
+
+cat > "$REPORT_FILE" << 'REPORT_HEADER'
+# Ruvector Performance Analysis Report
+
+**Generated**: $(date)
+**System**: $(uname -a)
+**CPU**: $(lscpu | grep "Model name" | sed 's/Model name: *//')
+
+## Executive Summary
+
+This report contains comprehensive performance analysis of Ruvector vector database including:
+
+- CPU profiling and hotspot analysis
+- Memory allocation patterns
+- Cache utilization
+- Thread scaling characteristics
+- SIMD optimization effectiveness
+- Lock-free data structure performance
+
+REPORT_HEADER
+
+echo "" >> "$REPORT_FILE"
+echo "## System Information" >> "$REPORT_FILE"
+echo "" >> "$REPORT_FILE"
+echo '```' >> "$REPORT_FILE"
+cat "$SCRIPT_DIR/../benchmarks/system_info.txt" >> "$REPORT_FILE" 2>/dev/null || echo "System info not available"
+echo '```' >> "$REPORT_FILE"
+echo "" >> "$REPORT_FILE"
+
+echo "## Benchmark Results" >> "$REPORT_FILE"
+echo "" >> "$REPORT_FILE"
+echo "### Distance Metrics Performance" >> "$REPORT_FILE"
+echo '```' >> "$REPORT_FILE"
+grep "time:" "$SCRIPT_DIR/../benchmarks/distance_metrics.txt" | head -20 >> "$REPORT_FILE" 2>/dev/null || echo "Benchmarks not available"
+echo '```' >> "$REPORT_FILE"
+echo "" >> "$REPORT_FILE"
+
+echo "### HNSW Search Performance" >> "$REPORT_FILE"
+echo '```' >> "$REPORT_FILE"
+grep "time:" "$SCRIPT_DIR/../benchmarks/hnsw_search.txt" | head -20 >> "$REPORT_FILE" 2>/dev/null || echo "Benchmarks not available"
+echo '```' >> "$REPORT_FILE"
+echo "" >> "$REPORT_FILE"
+
+echo "### Thread Scaling" >> "$REPORT_FILE"
+echo "" >> "$REPORT_FILE"
+for threads in 1 2 4 8 16 32; do
+    if [ -f "$SCRIPT_DIR/../benchmarks/threads_${threads}.txt" ]; then
+        echo "#### ${threads} threads" >> "$REPORT_FILE"
+        echo '```' >> "$REPORT_FILE"
+        grep "time:" "$SCRIPT_DIR/../benchmarks/threads_${threads}.txt" | head -5 >> "$REPORT_FILE"
+        echo '```' >> "$REPORT_FILE"
+        echo "" >> "$REPORT_FILE"
+    fi
+done
+
+echo "## CPU Profiling Analysis" >> "$REPORT_FILE"
+echo "" >> "$REPORT_FILE"
+echo "### Top Hotspots" >> "$REPORT_FILE"
+echo '```' >> "$REPORT_FILE"
+head -50 "$SCRIPT_DIR/../reports/perf_report.txt" >> "$REPORT_FILE" 2>/dev/null || echo "Perf report not available"
+echo '```' >> "$REPORT_FILE"
+echo "" >> "$REPORT_FILE"
+
+echo "### Cache Performance" >> "$REPORT_FILE"
+echo '```' >> "$REPORT_FILE"
+cat "$SCRIPT_DIR/../reports/cache_stats.txt" >> "$REPORT_FILE" 2>/dev/null || echo "Cache stats not available"
+echo '```' >> "$REPORT_FILE"
+echo "" >> "$REPORT_FILE"
+
+echo "## Memory Analysis" >> "$REPORT_FILE"
+echo "" >> "$REPORT_FILE"
+echo "### Massif Heap Profile" >> "$REPORT_FILE"
+echo '```' >> "$REPORT_FILE"
+head -100 "$SCRIPT_DIR/../memory/massif_report.txt" >> "$REPORT_FILE" 2>/dev/null || echo "Massif report not available"
+echo '```' >> "$REPORT_FILE"
+echo "" >> "$REPORT_FILE"
+
+echo "## Recommendations" >> "$REPORT_FILE"
+echo "" >> "$REPORT_FILE"
+echo "Based on the analysis:" >> "$REPORT_FILE"
+echo "" >> "$REPORT_FILE"
+echo "1. **CPU Optimization**: Review flamegraphs to identify hotspots" >> "$REPORT_FILE"
+echo "2. **Memory Optimization**: Check for allocation patterns in hot paths" >> "$REPORT_FILE"
+echo "3. **Cache Optimization**: Analyze cache miss rates and data structures" >> "$REPORT_FILE"
+echo "4. **Parallelization**: Evaluate thread scaling efficiency" >> "$REPORT_FILE"
+echo "" >> "$REPORT_FILE"
+echo "See detailed optimization guides in /docs/optimization/" >> "$REPORT_FILE"
+echo "" >> "$REPORT_FILE"
+
+echo "✅ Comprehensive analysis complete!"
+echo ""
+echo "=========================================="
+echo " Analysis Results Summary"
+echo "=========================================="
+echo ""
+echo "Reports Directory: $SCRIPT_DIR/../reports"
+echo "Flamegraphs: $SCRIPT_DIR/../flamegraphs"
+echo "Memory Analysis: $SCRIPT_DIR/../memory"
+echo "Benchmarks: $SCRIPT_DIR/../benchmarks"
+echo ""
+echo "📊 Comprehensive Report: $REPORT_FILE"
+echo ""
+echo "Next Steps:"
+echo "1. Review flamegraphs: firefox $SCRIPT_DIR/../flamegraphs/*.svg"
+echo "2. Check benchmark results: cat $SCRIPT_DIR/../benchmarks/summary.txt"
+echo "3. Analyze CPU hotspots: cat $SCRIPT_DIR/../reports/perf_report.txt"
+echo "4. Review memory usage: cat $SCRIPT_DIR/../memory/massif_report.txt"
+echo ""