Merge commit 'd803bfe2b1fe7f5e219e50ac20d6801a0a58ac75' as 'vendor/ruvector'
This commit is contained in:
ruv
327
vendor/ruvector/npm/packages/agentic-synth/benchmark.js
vendored
Executable file
327
vendor/ruvector/npm/packages/agentic-synth/benchmark.js
vendored
Executable file
@@ -0,0 +1,327 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
/**
|
||||
* Comprehensive Benchmark Suite for agentic-synth
|
||||
* Tests: Cache performance, generation speed, memory usage, throughput
|
||||
*/
|
||||
|
||||
import { performance } from 'perf_hooks';
|
||||
import { AgenticSynth } from './dist/index.js';
|
||||
import { CacheManager } from './dist/cache/index.js';
|
||||
|
||||
// Color codes for terminal output
|
||||
const colors = {
|
||||
reset: '\x1b[0m',
|
||||
bright: '\x1b[1m',
|
||||
red: '\x1b[31m',
|
||||
green: '\x1b[32m',
|
||||
yellow: '\x1b[33m',
|
||||
blue: '\x1b[34m',
|
||||
cyan: '\x1b[36m'
|
||||
};
|
||||
|
||||
const c = (color, text) => `${colors[color]}${text}${colors.reset}`;
|
||||
|
||||
console.log(c('cyan', '\n═══════════════════════════════════════════════════'));
|
||||
console.log(c('bright', ' Agentic-Synth Benchmark Suite'));
|
||||
console.log(c('cyan', '═══════════════════════════════════════════════════\n'));
|
||||
|
||||
// Benchmark utilities
|
||||
class BenchmarkRunner {
|
||||
constructor() {
|
||||
this.results = [];
|
||||
}
|
||||
|
||||
async run(name, fn, iterations = 100) {
|
||||
console.log(c('blue', `\n📊 Running: ${name}`));
|
||||
console.log(c('yellow', ` Iterations: ${iterations}`));
|
||||
|
||||
const times = [];
|
||||
const memoryBefore = process.memoryUsage();
|
||||
|
||||
for (let i = 0; i < iterations; i++) {
|
||||
const start = performance.now();
|
||||
await fn();
|
||||
const end = performance.now();
|
||||
times.push(end - start);
|
||||
}
|
||||
|
||||
const memoryAfter = process.memoryUsage();
|
||||
|
||||
const sorted = times.sort((a, b) => a - b);
|
||||
const stats = {
|
||||
name,
|
||||
iterations,
|
||||
min: sorted[0],
|
||||
max: sorted[sorted.length - 1],
|
||||
mean: times.reduce((a, b) => a + b, 0) / times.length,
|
||||
median: sorted[Math.floor(sorted.length / 2)],
|
||||
p95: sorted[Math.floor(sorted.length * 0.95)],
|
||||
p99: sorted[Math.floor(sorted.length * 0.99)],
|
||||
memoryDelta: {
|
||||
heapUsed: (memoryAfter.heapUsed - memoryBefore.heapUsed) / 1024 / 1024,
|
||||
rss: (memoryAfter.rss - memoryBefore.rss) / 1024 / 1024
|
||||
}
|
||||
};
|
||||
|
||||
this.results.push(stats);
|
||||
this.printStats(stats);
|
||||
|
||||
return stats;
|
||||
}
|
||||
|
||||
printStats(stats) {
|
||||
console.log(c('green', ' ✓ Complete'));
|
||||
console.log(` Min: ${c('cyan', stats.min.toFixed(2))}ms`);
|
||||
console.log(` Mean: ${c('cyan', stats.mean.toFixed(2))}ms`);
|
||||
console.log(` Median: ${c('cyan', stats.median.toFixed(2))}ms`);
|
||||
console.log(` P95: ${c('cyan', stats.p95.toFixed(2))}ms`);
|
||||
console.log(` P99: ${c('cyan', stats.p99.toFixed(2))}ms`);
|
||||
console.log(` Max: ${c('cyan', stats.max.toFixed(2))}ms`);
|
||||
console.log(` Memory Δ: ${c('yellow', stats.memoryDelta.heapUsed.toFixed(2))}MB heap`);
|
||||
}
|
||||
|
||||
summary() {
|
||||
console.log(c('cyan', '\n═══════════════════════════════════════════════════'));
|
||||
console.log(c('bright', ' Benchmark Summary'));
|
||||
console.log(c('cyan', '═══════════════════════════════════════════════════\n'));
|
||||
|
||||
console.log(c('bright', 'Performance Results:\n'));
|
||||
|
||||
const table = this.results.map(r => ({
|
||||
'Test': r.name.substring(0, 40),
|
||||
'Mean': `${r.mean.toFixed(2)}ms`,
|
||||
'P95': `${r.p95.toFixed(2)}ms`,
|
||||
'P99': `${r.p99.toFixed(2)}ms`,
|
||||
'Memory': `${r.memoryDelta.heapUsed.toFixed(2)}MB`
|
||||
}));
|
||||
|
||||
console.table(table);
|
||||
|
||||
// Performance ratings
|
||||
console.log(c('bright', '\nPerformance Ratings:\n'));
|
||||
|
||||
this.results.forEach(r => {
|
||||
let rating = '⭐⭐⭐⭐⭐';
|
||||
let status = c('green', 'EXCELLENT');
|
||||
|
||||
if (r.p99 > 1000) {
|
||||
rating = '⭐⭐⭐';
|
||||
status = c('yellow', 'ACCEPTABLE');
|
||||
}
|
||||
if (r.p99 > 2000) {
|
||||
rating = '⭐⭐';
|
||||
status = c('red', 'NEEDS OPTIMIZATION');
|
||||
}
|
||||
|
||||
console.log(` ${rating} ${r.name.substring(0, 35).padEnd(35)} - ${status}`);
|
||||
});
|
||||
|
||||
// Recommendations
|
||||
console.log(c('bright', '\n\nOptimization Recommendations:\n'));
|
||||
|
||||
const slowTests = this.results.filter(r => r.p99 > 100);
|
||||
if (slowTests.length === 0) {
|
||||
console.log(c('green', ' ✓ All benchmarks performing excellently!'));
|
||||
} else {
|
||||
slowTests.forEach(r => {
|
||||
console.log(c('yellow', ` ⚠ ${r.name}:`));
|
||||
if (r.p99 > 1000) {
|
||||
console.log(' - Consider adding caching');
|
||||
console.log(' - Optimize algorithm complexity');
|
||||
}
|
||||
if (r.memoryDelta.heapUsed > 50) {
|
||||
console.log(' - High memory usage detected');
|
||||
console.log(' - Consider memory pooling');
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
console.log(c('cyan', '\n═══════════════════════════════════════════════════\n'));
|
||||
}
|
||||
}
|
||||
|
||||
// Benchmark tests
|
||||
async function runBenchmarks() {
|
||||
const runner = new BenchmarkRunner();
|
||||
|
||||
console.log(c('yellow', 'Preparing benchmark environment...\n'));
|
||||
|
||||
// 1. Cache performance benchmarks
|
||||
console.log(c('bright', '1️⃣ CACHE PERFORMANCE'));
|
||||
|
||||
const cache = new CacheManager({
|
||||
strategy: 'memory',
|
||||
ttl: 3600,
|
||||
maxSize: 1000
|
||||
});
|
||||
|
||||
await runner.run('Cache: Set operation', async () => {
|
||||
await cache.set(`key-${Math.random()}`, { data: 'test-value' });
|
||||
}, 1000);
|
||||
|
||||
// Pre-populate cache
|
||||
for (let i = 0; i < 100; i++) {
|
||||
await cache.set(`test-key-${i}`, { data: `value-${i}` });
|
||||
}
|
||||
|
||||
await runner.run('Cache: Get operation (hit)', async () => {
|
||||
await cache.get(`test-key-${Math.floor(Math.random() * 100)}`);
|
||||
}, 1000);
|
||||
|
||||
await runner.run('Cache: Get operation (miss)', async () => {
|
||||
await cache.get(`missing-key-${Math.random()}`);
|
||||
}, 1000);
|
||||
|
||||
await runner.run('Cache: Has operation', async () => {
|
||||
await cache.has(`test-key-${Math.floor(Math.random() * 100)}`);
|
||||
}, 1000);
|
||||
|
||||
// 2. Configuration benchmarks
|
||||
console.log(c('bright', '\n2️⃣ CONFIGURATION & INITIALIZATION'));
|
||||
|
||||
await runner.run('AgenticSynth: Initialization', async () => {
|
||||
const synth = new AgenticSynth({
|
||||
provider: 'gemini',
|
||||
apiKey: 'test-key',
|
||||
cacheStrategy: 'memory'
|
||||
});
|
||||
}, 100);
|
||||
|
||||
const synth = new AgenticSynth({
|
||||
provider: 'gemini',
|
||||
apiKey: 'test-key',
|
||||
cacheStrategy: 'memory'
|
||||
});
|
||||
|
||||
await runner.run('AgenticSynth: Get config', async () => {
|
||||
synth.getConfig();
|
||||
}, 1000);
|
||||
|
||||
await runner.run('AgenticSynth: Update config', async () => {
|
||||
synth.configure({ cacheTTL: Math.floor(Math.random() * 10000) });
|
||||
}, 100);
|
||||
|
||||
// 3. Type validation benchmarks
|
||||
console.log(c('bright', '\n3️⃣ TYPE VALIDATION'));
|
||||
|
||||
const { SynthConfigSchema } = await import('./dist/index.js');
|
||||
|
||||
await runner.run('Zod: Config validation (valid)', async () => {
|
||||
SynthConfigSchema.parse({
|
||||
provider: 'gemini',
|
||||
apiKey: 'test',
|
||||
cacheStrategy: 'memory'
|
||||
});
|
||||
}, 1000);
|
||||
|
||||
await runner.run('Zod: Config validation (with defaults)', async () => {
|
||||
SynthConfigSchema.parse({
|
||||
provider: 'gemini'
|
||||
});
|
||||
}, 1000);
|
||||
|
||||
// 4. Data structure operations
|
||||
console.log(c('bright', '\n4️⃣ DATA STRUCTURE OPERATIONS'));
|
||||
|
||||
const testData = Array.from({ length: 100 }, (_, i) => ({
|
||||
id: i,
|
||||
name: `user-${i}`,
|
||||
email: `user${i}@example.com`,
|
||||
age: 20 + (i % 50)
|
||||
}));
|
||||
|
||||
await runner.run('JSON: Stringify large object', async () => {
|
||||
JSON.stringify(testData);
|
||||
}, 1000);
|
||||
|
||||
await runner.run('JSON: Parse large object', async () => {
|
||||
JSON.parse(JSON.stringify(testData));
|
||||
}, 1000);
|
||||
|
||||
// 5. Cache key generation
|
||||
console.log(c('bright', '\n5️⃣ CACHE KEY GENERATION'));
|
||||
|
||||
await runner.run('CacheManager: Generate key (simple)', async () => {
|
||||
CacheManager.generateKey('test', { id: 1, type: 'simple' });
|
||||
}, 1000);
|
||||
|
||||
await runner.run('CacheManager: Generate key (complex)', async () => {
|
||||
CacheManager.generateKey('test', {
|
||||
id: 1,
|
||||
type: 'complex',
|
||||
schema: { name: 'string', age: 'number' },
|
||||
options: { count: 10, format: 'json' }
|
||||
});
|
||||
}, 1000);
|
||||
|
||||
// 6. Memory stress test
|
||||
console.log(c('bright', '\n6️⃣ MEMORY STRESS TEST'));
|
||||
|
||||
await runner.run('Memory: Large cache operations', async () => {
|
||||
const tempCache = new CacheManager({
|
||||
strategy: 'memory',
|
||||
ttl: 3600,
|
||||
maxSize: 1000
|
||||
});
|
||||
|
||||
for (let i = 0; i < 100; i++) {
|
||||
await tempCache.set(`key-${i}`, { data: new Array(100).fill(i) });
|
||||
}
|
||||
}, 10);
|
||||
|
||||
// 7. Concurrent operations
|
||||
console.log(c('bright', '\n7️⃣ CONCURRENT OPERATIONS'));
|
||||
|
||||
await runner.run('Concurrency: Parallel cache reads', async () => {
|
||||
await Promise.all(
|
||||
Array.from({ length: 10 }, (_, i) =>
|
||||
cache.get(`test-key-${i}`)
|
||||
)
|
||||
);
|
||||
}, 100);
|
||||
|
||||
await runner.run('Concurrency: Parallel cache writes', async () => {
|
||||
await Promise.all(
|
||||
Array.from({ length: 10 }, (_, i) =>
|
||||
cache.set(`concurrent-${i}`, { value: i })
|
||||
)
|
||||
);
|
||||
}, 100);
|
||||
|
||||
// Print summary
|
||||
runner.summary();
|
||||
|
||||
// Export results
|
||||
const results = {
|
||||
timestamp: new Date().toISOString(),
|
||||
benchmarks: runner.results,
|
||||
environment: {
|
||||
nodeVersion: process.version,
|
||||
platform: process.platform,
|
||||
arch: process.arch,
|
||||
memory: process.memoryUsage()
|
||||
}
|
||||
};
|
||||
|
||||
return results;
|
||||
}
|
||||
|
||||
// Run benchmarks
|
||||
runBenchmarks()
|
||||
.then(results => {
|
||||
// Save results to file
|
||||
import('fs').then(fs => {
|
||||
fs.default.writeFileSync(
|
||||
'benchmark-results.json',
|
||||
JSON.stringify(results, null, 2)
|
||||
);
|
||||
console.log(c('green', '✅ Results saved to benchmark-results.json\n'));
|
||||
});
|
||||
|
||||
process.exit(0);
|
||||
})
|
||||
.catch(error => {
|
||||
console.error(c('red', '\n❌ Benchmark failed:'), error);
|
||||
process.exit(1);
|
||||
});
|
||||
Reference in New Issue
Block a user