wifi-densepose/vendor/ruvector/examples/vibecast-7sense/scripts/run_benchmarks.sh

#!/bin/bash
# 7sense Benchmark Runner
#
# This script runs all benchmarks and generates reports.
# Performance targets from ADR-004:
# - HNSW Search: 150x speedup vs brute force, p99 < 50ms
# - Embedding inference: >100 segments/second
# - Batch ingestion: 1M vectors/minute
# - Query latency: <100ms total

set -euo pipefail

# Colors for output
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
NC='\033[0m' # No Color

# Configuration
BASELINE_NAME="${BASELINE_NAME:-main}"
OUTPUT_DIR="${OUTPUT_DIR:-target/criterion}"
REPORT_DIR="${REPORT_DIR:-target/benchmark-reports}"
COMPARE_BASELINE="${COMPARE_BASELINE:-false}"

# Benchmark suites
ALL_BENCHMARKS=(
    "hnsw_benchmark"
    "embedding_benchmark"
    "clustering_benchmark"
    "api_benchmark"
)

print_header() {
    echo ""
    echo "============================================================"
    echo -e "${BLUE}$1${NC}"
    echo "============================================================"
    echo ""
}

print_status() {
    echo -e "${GREEN}[INFO]${NC} $1"
}

print_warning() {
    echo -e "${YELLOW}[WARN]${NC} $1"
}

print_error() {
    echo -e "${RED}[ERROR]${NC} $1"
}

# Create output directories
setup_directories() {
    print_status "Setting up output directories..."
    mkdir -p "$OUTPUT_DIR"
    mkdir -p "$REPORT_DIR"
    mkdir -p "$REPORT_DIR/html"
    mkdir -p "$REPORT_DIR/json"
}

# Run a single benchmark suite
run_benchmark() {
    local bench_name="$1"
    local extra_args="${2:-}"

    print_header "Running $bench_name"

    if [[ "$COMPARE_BASELINE" == "true" ]]; then
        print_status "Comparing against baseline: $BASELINE_NAME"
        cargo bench --bench "$bench_name" -- --baseline "$BASELINE_NAME" $extra_args
    else
        cargo bench --bench "$bench_name" -- $extra_args
    fi

    # Check if benchmark completed successfully
    if [[ $? -eq 0 ]]; then
        print_status "$bench_name completed successfully"
    else
        print_error "$bench_name failed"
        return 1
    fi
}

# Run all benchmarks
run_all_benchmarks() {
    print_header "Running All Benchmarks"

    local failed=()

    for bench in "${ALL_BENCHMARKS[@]}"; do
        if ! run_benchmark "$bench"; then
            failed+=("$bench")
        fi
    done

    if [[ ${#failed[@]} -gt 0 ]]; then
        print_error "Failed benchmarks: ${failed[*]}"
        return 1
    fi

    print_status "All benchmarks completed successfully"
}

# Save baseline for future comparisons
save_baseline() {
    local baseline_name="${1:-$BASELINE_NAME}"

    print_header "Saving Baseline: $baseline_name"

    for bench in "${ALL_BENCHMARKS[@]}"; do
        print_status "Saving baseline for $bench..."
        cargo bench --bench "$bench" -- --save-baseline "$baseline_name"
    done

    print_status "Baseline '$baseline_name' saved"
}

# Run quick benchmarks (reduced sample size)
run_quick() {
    print_header "Running Quick Benchmarks"

    # Set sample size to minimum for quick testing
    export CRITERION_DEBUG=1

    for bench in "${ALL_BENCHMARKS[@]}"; do
        print_status "Quick run: $bench"
        cargo bench --bench "$bench" -- --quick
    done
}

# Run specific benchmark groups
run_hnsw() {
    run_benchmark "hnsw_benchmark" "$@"
}

run_embedding() {
    run_benchmark "embedding_benchmark" "$@"
}

run_clustering() {
    run_benchmark "clustering_benchmark" "$@"
}

run_api() {
    run_benchmark "api_benchmark" "$@"
}

# Generate summary report
generate_summary() {
    print_header "Generating Summary Report"

    local report_file="$REPORT_DIR/summary_$(date +%Y%m%d_%H%M%S).txt"

    {
        echo "7sense Benchmark Summary Report"
        echo "Generated: $(date)"
        echo "============================================================"
        echo ""
        echo "Performance Targets (from ADR-004):"
        echo "  - HNSW Search: 150x speedup vs brute force"
        echo "  - Query Latency p50: < 10ms"
        echo "  - Query Latency p99: < 50ms"
        echo "  - Recall@10: >= 0.95"
        echo "  - Recall@100: >= 0.98"
        echo "  - Embedding inference: >100 segments/second"
        echo "  - Batch ingestion: 1M vectors/minute"
        echo "  - Total query latency: <100ms"
        echo ""
        echo "============================================================"
        echo ""

        # Parse criterion output if available
        if [[ -d "$OUTPUT_DIR" ]]; then
            echo "Benchmark Results:"
            echo ""

            for bench in "${ALL_BENCHMARKS[@]}"; do
                if [[ -d "$OUTPUT_DIR/$bench" ]]; then
                    echo "--- $bench ---"
                    # List all benchmark groups
                    find "$OUTPUT_DIR/$bench" -name "estimates.json" -exec dirname {} \; 2>/dev/null | \
                        while read -r dir; do
                            local name=$(basename "$dir")
                            echo "  $name"
                        done
                    echo ""
                fi
            done
        else
            echo "No benchmark results found. Run benchmarks first."
        fi
    } > "$report_file"

    print_status "Summary saved to: $report_file"
    cat "$report_file"
}

# Run performance analysis tests
run_analysis() {
    print_header "Running Performance Analysis"

    print_status "Running speedup analysis..."
    cargo test --release -- --ignored run_speedup_analysis --nocapture 2>/dev/null || true

    print_status "Running latency analysis..."
    cargo test --release -- --ignored run_latency_analysis --nocapture 2>/dev/null || true

    print_status "Running throughput analysis..."
    cargo test --release -- --ignored run_throughput_analysis --nocapture 2>/dev/null || true

    print_status "Running API latency analysis..."
    cargo test --release -- --ignored run_api_latency_analysis --nocapture 2>/dev/null || true
}

# Clean benchmark artifacts
clean() {
    print_header "Cleaning Benchmark Artifacts"

    rm -rf "$OUTPUT_DIR"
    rm -rf "$REPORT_DIR"

    print_status "Cleaned benchmark artifacts"
}

# Show help
show_help() {
    echo "7sense Benchmark Runner"
    echo ""
    echo "Usage: $0 [command] [options]"
    echo ""
    echo "Commands:"
    echo "  all              Run all benchmarks (default)"
    echo "  quick            Run quick benchmarks with reduced samples"
    echo "  hnsw             Run HNSW benchmarks only"
    echo "  embedding        Run embedding benchmarks only"
    echo "  clustering       Run clustering benchmarks only"
    echo "  api              Run API benchmarks only"
    echo "  save-baseline    Save current results as baseline"
    echo "  compare          Run benchmarks and compare against baseline"
    echo "  analysis         Run detailed performance analysis"
    echo "  summary          Generate summary report"
    echo "  clean            Clean benchmark artifacts"
    echo "  help             Show this help message"
    echo ""
    echo "Environment Variables:"
    echo "  BASELINE_NAME    Name for baseline comparison (default: main)"
    echo "  OUTPUT_DIR       Criterion output directory (default: target/criterion)"
    echo "  REPORT_DIR       Report output directory (default: target/benchmark-reports)"
    echo "  COMPARE_BASELINE Set to 'true' to compare against baseline"
    echo ""
    echo "Examples:"
    echo "  $0                           # Run all benchmarks"
    echo "  $0 hnsw                      # Run HNSW benchmarks only"
    echo "  $0 save-baseline v1.0.0      # Save baseline named 'v1.0.0'"
    echo "  COMPARE_BASELINE=true $0     # Compare against baseline"
    echo ""
    echo "Performance Targets (ADR-004):"
    echo "  - HNSW Search: 150x speedup vs brute force"
    echo "  - Query Latency p99: < 50ms"
    echo "  - Recall@10: >= 0.95"
    echo "  - Embedding: >100 segments/second"
    echo "  - Batch: 1M vectors/minute"
}

# Main entry point
main() {
    local command="${1:-all}"
    shift || true

    setup_directories

    case "$command" in
        all)
            run_all_benchmarks
            generate_summary
            ;;
        quick)
            run_quick
            ;;
        hnsw)
            run_hnsw "$@"
            ;;
        embedding)
            run_embedding "$@"
            ;;
        clustering)
            run_clustering "$@"
            ;;
        api)
            run_api "$@"
            ;;
        save-baseline)
            save_baseline "${1:-$BASELINE_NAME}"
            ;;
        compare)
            COMPARE_BASELINE=true run_all_benchmarks
            ;;
        analysis)
            run_analysis
            ;;
        summary)
            generate_summary
            ;;
        clean)
            clean
            ;;
        help|--help|-h)
            show_help
            ;;
        *)
            print_error "Unknown command: $command"
            show_help
            exit 1
            ;;
    esac
}

main "$@"