Squashed 'vendor/ruvector/' content from commit b64c2172
git-subtree-dir: vendor/ruvector git-subtree-split: b64c21726f2bb37286d9ee36a7869fef60cc6900
This commit is contained in:
ruv
320
examples/vibecast-7sense/scripts/run_benchmarks.sh
Executable file
320
examples/vibecast-7sense/scripts/run_benchmarks.sh
Executable file
@@ -0,0 +1,320 @@
|
||||
#!/bin/bash
|
||||
# 7sense Benchmark Runner
|
||||
#
|
||||
# This script runs all benchmarks and generates reports.
|
||||
# Performance targets from ADR-004:
|
||||
# - HNSW Search: 150x speedup vs brute force, p99 < 50ms
|
||||
# - Embedding inference: >100 segments/second
|
||||
# - Batch ingestion: 1M vectors/minute
|
||||
# - Query latency: <100ms total
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# Colors for output
|
||||
RED='\033[0;31m'
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
BLUE='\033[0;34m'
|
||||
NC='\033[0m' # No Color
|
||||
|
||||
# Configuration
|
||||
BASELINE_NAME="${BASELINE_NAME:-main}"
|
||||
OUTPUT_DIR="${OUTPUT_DIR:-target/criterion}"
|
||||
REPORT_DIR="${REPORT_DIR:-target/benchmark-reports}"
|
||||
COMPARE_BASELINE="${COMPARE_BASELINE:-false}"
|
||||
|
||||
# Benchmark suites
|
||||
ALL_BENCHMARKS=(
|
||||
"hnsw_benchmark"
|
||||
"embedding_benchmark"
|
||||
"clustering_benchmark"
|
||||
"api_benchmark"
|
||||
)
|
||||
|
||||
print_header() {
|
||||
echo ""
|
||||
echo "============================================================"
|
||||
echo -e "${BLUE}$1${NC}"
|
||||
echo "============================================================"
|
||||
echo ""
|
||||
}
|
||||
|
||||
print_status() {
|
||||
echo -e "${GREEN}[INFO]${NC} $1"
|
||||
}
|
||||
|
||||
print_warning() {
|
||||
echo -e "${YELLOW}[WARN]${NC} $1"
|
||||
}
|
||||
|
||||
print_error() {
|
||||
echo -e "${RED}[ERROR]${NC} $1"
|
||||
}
|
||||
|
||||
# Create output directories
|
||||
setup_directories() {
|
||||
print_status "Setting up output directories..."
|
||||
mkdir -p "$OUTPUT_DIR"
|
||||
mkdir -p "$REPORT_DIR"
|
||||
mkdir -p "$REPORT_DIR/html"
|
||||
mkdir -p "$REPORT_DIR/json"
|
||||
}
|
||||
|
||||
# Run a single benchmark suite
|
||||
run_benchmark() {
|
||||
local bench_name="$1"
|
||||
local extra_args="${2:-}"
|
||||
|
||||
print_header "Running $bench_name"
|
||||
|
||||
if [[ "$COMPARE_BASELINE" == "true" ]]; then
|
||||
print_status "Comparing against baseline: $BASELINE_NAME"
|
||||
cargo bench --bench "$bench_name" -- --baseline "$BASELINE_NAME" $extra_args
|
||||
else
|
||||
cargo bench --bench "$bench_name" -- $extra_args
|
||||
fi
|
||||
|
||||
# Check if benchmark completed successfully
|
||||
if [[ $? -eq 0 ]]; then
|
||||
print_status "$bench_name completed successfully"
|
||||
else
|
||||
print_error "$bench_name failed"
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
# Run all benchmarks
|
||||
run_all_benchmarks() {
|
||||
print_header "Running All Benchmarks"
|
||||
|
||||
local failed=()
|
||||
|
||||
for bench in "${ALL_BENCHMARKS[@]}"; do
|
||||
if ! run_benchmark "$bench"; then
|
||||
failed+=("$bench")
|
||||
fi
|
||||
done
|
||||
|
||||
if [[ ${#failed[@]} -gt 0 ]]; then
|
||||
print_error "Failed benchmarks: ${failed[*]}"
|
||||
return 1
|
||||
fi
|
||||
|
||||
print_status "All benchmarks completed successfully"
|
||||
}
|
||||
|
||||
# Save baseline for future comparisons
|
||||
save_baseline() {
|
||||
local baseline_name="${1:-$BASELINE_NAME}"
|
||||
|
||||
print_header "Saving Baseline: $baseline_name"
|
||||
|
||||
for bench in "${ALL_BENCHMARKS[@]}"; do
|
||||
print_status "Saving baseline for $bench..."
|
||||
cargo bench --bench "$bench" -- --save-baseline "$baseline_name"
|
||||
done
|
||||
|
||||
print_status "Baseline '$baseline_name' saved"
|
||||
}
|
||||
|
||||
# Run quick benchmarks (reduced sample size)
|
||||
run_quick() {
|
||||
print_header "Running Quick Benchmarks"
|
||||
|
||||
# Set sample size to minimum for quick testing
|
||||
export CRITERION_DEBUG=1
|
||||
|
||||
for bench in "${ALL_BENCHMARKS[@]}"; do
|
||||
print_status "Quick run: $bench"
|
||||
cargo bench --bench "$bench" -- --quick
|
||||
done
|
||||
}
|
||||
|
||||
# Run specific benchmark groups
|
||||
run_hnsw() {
|
||||
run_benchmark "hnsw_benchmark" "$@"
|
||||
}
|
||||
|
||||
run_embedding() {
|
||||
run_benchmark "embedding_benchmark" "$@"
|
||||
}
|
||||
|
||||
run_clustering() {
|
||||
run_benchmark "clustering_benchmark" "$@"
|
||||
}
|
||||
|
||||
run_api() {
|
||||
run_benchmark "api_benchmark" "$@"
|
||||
}
|
||||
|
||||
# Generate summary report
|
||||
generate_summary() {
|
||||
print_header "Generating Summary Report"
|
||||
|
||||
local report_file="$REPORT_DIR/summary_$(date +%Y%m%d_%H%M%S).txt"
|
||||
|
||||
{
|
||||
echo "7sense Benchmark Summary Report"
|
||||
echo "Generated: $(date)"
|
||||
echo "============================================================"
|
||||
echo ""
|
||||
echo "Performance Targets (from ADR-004):"
|
||||
echo " - HNSW Search: 150x speedup vs brute force"
|
||||
echo " - Query Latency p50: < 10ms"
|
||||
echo " - Query Latency p99: < 50ms"
|
||||
echo " - Recall@10: >= 0.95"
|
||||
echo " - Recall@100: >= 0.98"
|
||||
echo " - Embedding inference: >100 segments/second"
|
||||
echo " - Batch ingestion: 1M vectors/minute"
|
||||
echo " - Total query latency: <100ms"
|
||||
echo ""
|
||||
echo "============================================================"
|
||||
echo ""
|
||||
|
||||
# Parse criterion output if available
|
||||
if [[ -d "$OUTPUT_DIR" ]]; then
|
||||
echo "Benchmark Results:"
|
||||
echo ""
|
||||
|
||||
for bench in "${ALL_BENCHMARKS[@]}"; do
|
||||
if [[ -d "$OUTPUT_DIR/$bench" ]]; then
|
||||
echo "--- $bench ---"
|
||||
# List all benchmark groups
|
||||
find "$OUTPUT_DIR/$bench" -name "estimates.json" -exec dirname {} \; 2>/dev/null | \
|
||||
while read -r dir; do
|
||||
local name=$(basename "$dir")
|
||||
echo " $name"
|
||||
done
|
||||
echo ""
|
||||
fi
|
||||
done
|
||||
else
|
||||
echo "No benchmark results found. Run benchmarks first."
|
||||
fi
|
||||
} > "$report_file"
|
||||
|
||||
print_status "Summary saved to: $report_file"
|
||||
cat "$report_file"
|
||||
}
|
||||
|
||||
# Run performance analysis tests
|
||||
run_analysis() {
|
||||
print_header "Running Performance Analysis"
|
||||
|
||||
print_status "Running speedup analysis..."
|
||||
cargo test --release -- --ignored run_speedup_analysis --nocapture 2>/dev/null || true
|
||||
|
||||
print_status "Running latency analysis..."
|
||||
cargo test --release -- --ignored run_latency_analysis --nocapture 2>/dev/null || true
|
||||
|
||||
print_status "Running throughput analysis..."
|
||||
cargo test --release -- --ignored run_throughput_analysis --nocapture 2>/dev/null || true
|
||||
|
||||
print_status "Running API latency analysis..."
|
||||
cargo test --release -- --ignored run_api_latency_analysis --nocapture 2>/dev/null || true
|
||||
}
|
||||
|
||||
# Clean benchmark artifacts
|
||||
clean() {
|
||||
print_header "Cleaning Benchmark Artifacts"
|
||||
|
||||
rm -rf "$OUTPUT_DIR"
|
||||
rm -rf "$REPORT_DIR"
|
||||
|
||||
print_status "Cleaned benchmark artifacts"
|
||||
}
|
||||
|
||||
# Show help
|
||||
show_help() {
|
||||
echo "7sense Benchmark Runner"
|
||||
echo ""
|
||||
echo "Usage: $0 [command] [options]"
|
||||
echo ""
|
||||
echo "Commands:"
|
||||
echo " all Run all benchmarks (default)"
|
||||
echo " quick Run quick benchmarks with reduced samples"
|
||||
echo " hnsw Run HNSW benchmarks only"
|
||||
echo " embedding Run embedding benchmarks only"
|
||||
echo " clustering Run clustering benchmarks only"
|
||||
echo " api Run API benchmarks only"
|
||||
echo " save-baseline Save current results as baseline"
|
||||
echo " compare Run benchmarks and compare against baseline"
|
||||
echo " analysis Run detailed performance analysis"
|
||||
echo " summary Generate summary report"
|
||||
echo " clean Clean benchmark artifacts"
|
||||
echo " help Show this help message"
|
||||
echo ""
|
||||
echo "Environment Variables:"
|
||||
echo " BASELINE_NAME Name for baseline comparison (default: main)"
|
||||
echo " OUTPUT_DIR Criterion output directory (default: target/criterion)"
|
||||
echo " REPORT_DIR Report output directory (default: target/benchmark-reports)"
|
||||
echo " COMPARE_BASELINE Set to 'true' to compare against baseline"
|
||||
echo ""
|
||||
echo "Examples:"
|
||||
echo " $0 # Run all benchmarks"
|
||||
echo " $0 hnsw # Run HNSW benchmarks only"
|
||||
echo " $0 save-baseline v1.0.0 # Save baseline named 'v1.0.0'"
|
||||
echo " COMPARE_BASELINE=true $0 # Compare against baseline"
|
||||
echo ""
|
||||
echo "Performance Targets (ADR-004):"
|
||||
echo " - HNSW Search: 150x speedup vs brute force"
|
||||
echo " - Query Latency p99: < 50ms"
|
||||
echo " - Recall@10: >= 0.95"
|
||||
echo " - Embedding: >100 segments/second"
|
||||
echo " - Batch: 1M vectors/minute"
|
||||
}
|
||||
|
||||
# Main entry point
|
||||
main() {
|
||||
local command="${1:-all}"
|
||||
shift || true
|
||||
|
||||
setup_directories
|
||||
|
||||
case "$command" in
|
||||
all)
|
||||
run_all_benchmarks
|
||||
generate_summary
|
||||
;;
|
||||
quick)
|
||||
run_quick
|
||||
;;
|
||||
hnsw)
|
||||
run_hnsw "$@"
|
||||
;;
|
||||
embedding)
|
||||
run_embedding "$@"
|
||||
;;
|
||||
clustering)
|
||||
run_clustering "$@"
|
||||
;;
|
||||
api)
|
||||
run_api "$@"
|
||||
;;
|
||||
save-baseline)
|
||||
save_baseline "${1:-$BASELINE_NAME}"
|
||||
;;
|
||||
compare)
|
||||
COMPARE_BASELINE=true run_all_benchmarks
|
||||
;;
|
||||
analysis)
|
||||
run_analysis
|
||||
;;
|
||||
summary)
|
||||
generate_summary
|
||||
;;
|
||||
clean)
|
||||
clean
|
||||
;;
|
||||
help|--help|-h)
|
||||
show_help
|
||||
;;
|
||||
*)
|
||||
print_error "Unknown command: $command"
|
||||
show_help
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
main "$@"
|
||||
Reference in New Issue
Block a user