Merge commit 'd803bfe2b1fe7f5e219e50ac20d6801a0a58ac75' as 'vendor/ruvector'

vendor/ruvector/npm/packages/agentic-synth/examples/dspy-training-example.ts

@@ -0,0 +1,537 @@
+/**
+ * DSPy Training Session - Usage Example
+ *
+ * Demonstrates how to use the DSPy learning framework for multi-model training
+ * with automatic prompt optimization and benchmarking.
+ *
+ * @example
+ */
+
+import {
+  DSPyTrainingSession,
+  ModelProvider,
+  TrainingPhase,
+  OptimizationEngine,
+  BenchmarkCollector
+} from '../training/dspy-learning-session.js';
+
+/**
+ * Example 1: Basic Training Session
+ */
+async function basicTrainingExample() {
+  console.log('🚀 Starting Basic DSPy Training Session\n');
+
+  // Configure training session with multiple models
+  const session = new DSPyTrainingSession({
+    models: [
+      {
+        provider: ModelProvider.CLAUDE,
+        model: 'claude-sonnet-4',
+        apiKey: process.env.ANTHROPIC_API_KEY || 'sk-ant-test',
+        temperature: 0.7,
+        maxTokens: 1000
+      },
+      {
+        provider: ModelProvider.GPT4,
+        model: 'gpt-4-turbo',
+        apiKey: process.env.OPENAI_API_KEY || 'sk-test',
+        temperature: 0.7,
+        maxTokens: 1000
+      },
+      {
+        provider: ModelProvider.GEMINI,
+        model: 'gemini-2.0-flash-exp',
+        apiKey: process.env.GEMINI_API_KEY || 'test-key',
+        temperature: 0.7,
+        maxTokens: 1000
+      },
+      {
+        provider: ModelProvider.LLAMA,
+        model: 'llama-3.1-70b',
+        apiKey: process.env.TOGETHER_API_KEY || 'test-key',
+        temperature: 0.7,
+        maxTokens: 1000
+      }
+    ],
+    optimizationRounds: 5,
+    convergenceThreshold: 0.95,
+    maxConcurrency: 4,
+    enableCrossLearning: true,
+    enableHooksIntegration: true,
+    costBudget: 10.0, // $10 USD budget
+    timeoutPerIteration: 30000,
+    baselineIterations: 3,
+    benchmarkSamples: 50
+  });
+
+  // Create DSPy signature for the task
+  const optimizer = new OptimizationEngine();
+  const signature = optimizer.createSignature(
+    'product-description',
+    'Generate compelling product descriptions',
+    'high-quality, SEO-optimized product description',
+    {
+      examples: [
+        {
+          input: 'Wireless headphones with noise cancellation',
+          output: 'Premium wireless headphones featuring advanced active noise cancellation technology...'
+        },
+        {
+          input: 'Organic cotton t-shirt',
+          output: 'Sustainably crafted organic cotton t-shirt that combines comfort with environmental responsibility...'
+        }
+      ],
+      constraints: [
+        'min_length:100',
+        'max_length:500',
+        'contains:product benefits',
+        'contains:call to action'
+      ],
+      objectives: [
+        'Maximize engagement',
+        'Optimize for SEO',
+        'Include emotional appeal',
+        'Highlight unique value proposition'
+      ]
+    }
+  );
+
+  // Set up event listeners
+  session.on('start', (data) => {
+    console.log(`📊 Training started - Phase: ${data.phase}`);
+  });
+
+  session.on('phase', (phase) => {
+    console.log(`\n🔄 Phase transition: ${phase}`);
+  });
+
+  session.on('iteration', (result) => {
+    console.log(
+      `  ✓ ${result.modelProvider} - Iteration ${result.iteration}: ` +
+      `Quality: ${result.quality.score.toFixed(3)}, ` +
+      `Latency: ${result.performance.latency.toFixed(0)}ms, ` +
+      `Cost: $${result.performance.cost.toFixed(4)}`
+    );
+  });
+
+  session.on('optimization_round', (round) => {
+    console.log(`\n🔧 Optimization Round ${round}`);
+  });
+
+  session.on('converged', (provider) => {
+    console.log(`  ⭐ ${provider} has converged!`);
+  });
+
+  session.on('benchmark_progress', (data) => {
+    console.log(`  📈 Benchmark progress: ${data.completed}/${data.total}`);
+  });
+
+  session.on('budget_exceeded', (cost) => {
+    console.log(`  ⚠️  Cost budget exceeded: $${cost.toFixed(2)}`);
+  });
+
+  session.on('report', (data) => {
+    console.log('\n📊 Final Report:\n');
+    console.log(data.report);
+    console.log(`\n🏆 Best Model: ${data.bestModel}`);
+    console.log(`💰 Total Cost: $${data.totalCost.toFixed(4)}`);
+    console.log(`⏱️  Duration: ${(data.duration / 1000).toFixed(2)}s`);
+  });
+
+  session.on('complete', (data) => {
+    console.log('\n✅ Training complete!');
+  });
+
+  session.on('error', (error) => {
+    console.error('❌ Error:', error);
+  });
+
+  // Run the training session
+  const basePrompt = `
+Generate a compelling product description that:
+- Highlights key features and benefits
+- Uses persuasive language
+- Includes SEO keywords naturally
+- Has a clear call-to-action
+- Maintains professional tone
+
+Product: {product_name}
+  `.trim();
+
+  await session.run(basePrompt, signature);
+
+  // Get final statistics
+  const stats = session.getStatistics();
+  console.log('\n📊 Final Statistics:', JSON.stringify(stats, null, 2));
+}
+
+/**
+ * Example 2: Advanced Training with Real-time Monitoring
+ */
+async function advancedTrainingExample() {
+  console.log('🚀 Starting Advanced DSPy Training with Real-time Monitoring\n');
+
+  const session = new DSPyTrainingSession({
+    models: [
+      {
+        provider: ModelProvider.CLAUDE,
+        model: 'claude-sonnet-4',
+        apiKey: process.env.ANTHROPIC_API_KEY || 'sk-ant-test'
+      },
+      {
+        provider: ModelProvider.GEMINI,
+        model: 'gemini-2.0-flash-exp',
+        apiKey: process.env.GEMINI_API_KEY || 'test-key'
+      }
+    ],
+    optimizationRounds: 10,
+    enableCrossLearning: true,
+    enableHooksIntegration: true,
+    costBudget: 5.0
+  });
+
+  // Real-time metrics tracking
+  const metricsHistory: any[] = [];
+
+  session.on('metrics', (metrics) => {
+    metricsHistory.push({
+      timestamp: Date.now(),
+      ...metrics
+    });
+
+    // Calculate moving averages
+    if (metricsHistory.length >= 5) {
+      const recent = metricsHistory.slice(-5);
+      const avgQuality = recent.reduce((sum, m) => sum + m.quality.score, 0) / 5;
+      const avgLatency = recent.reduce((sum, m) => sum + m.performance.latency, 0) / 5;
+
+      console.log(
+        `  📊 Moving avg (last 5): Quality: ${avgQuality.toFixed(3)}, ` +
+        `Latency: ${avgLatency.toFixed(0)}ms`
+      );
+    }
+  });
+
+  // Hooks integration monitoring
+  session.on('hooks_integration', (data) => {
+    console.log(`  🔗 Hooks integration: ${data.action} - ${data.key}`);
+  });
+
+  const optimizer = new OptimizationEngine();
+  const signature = optimizer.createSignature(
+    'code-generation',
+    'Generate TypeScript code',
+    'production-ready TypeScript code with types',
+    {
+      constraints: [
+        'contains:type definitions',
+        'contains:error handling',
+        'min_length:50'
+      ],
+      objectives: [
+        'Follow TypeScript best practices',
+        'Include JSDoc comments',
+        'Use modern ES6+ syntax'
+      ]
+    }
+  );
+
+  const basePrompt = `
+Generate production-ready TypeScript code that:
+- Uses strong typing
+- Includes proper error handling
+- Follows best practices
+- Has clear documentation
+
+Task: {task_description}
+  `.trim();
+
+  await session.run(basePrompt, signature);
+}
+
+/**
+ * Example 3: Cost-Optimized Training
+ */
+async function costOptimizedTrainingExample() {
+  console.log('🚀 Starting Cost-Optimized DSPy Training\n');
+
+  // Use only cost-effective models
+  const session = new DSPyTrainingSession({
+    models: [
+      {
+        provider: ModelProvider.GEMINI,
+        model: 'gemini-2.0-flash-exp',
+        apiKey: process.env.GEMINI_API_KEY || 'test-key'
+      },
+      {
+        provider: ModelProvider.LLAMA,
+        model: 'llama-3.1-70b',
+        apiKey: process.env.TOGETHER_API_KEY || 'test-key'
+      }
+    ],
+    optimizationRounds: 3,
+    baselineIterations: 2,
+    benchmarkSamples: 20,
+    costBudget: 1.0, // Strict $1 budget
+    enableCrossLearning: true
+  });
+
+  // Track cost efficiency
+  let totalIterations = 0;
+  let totalQuality = 0;
+
+  session.on('iteration', (result) => {
+    totalIterations++;
+    totalQuality += result.quality.score;
+
+    const avgQuality = totalQuality / totalIterations;
+    const costPerIteration = session.getStatistics().totalCost / totalIterations;
+
+    console.log(
+      `  💰 Iteration ${totalIterations}: ` +
+      `Avg Quality: ${avgQuality.toFixed(3)}, ` +
+      `Cost/iter: $${costPerIteration.toFixed(4)}`
+    );
+  });
+
+  const optimizer = new OptimizationEngine();
+  const signature = optimizer.createSignature(
+    'summary',
+    'Summarize text',
+    'concise summary',
+    {
+      constraints: ['max_length:200'],
+      objectives: ['Maintain key information', 'Use clear language']
+    }
+  );
+
+  const basePrompt = 'Summarize the following text: {text}';
+
+  await session.run(basePrompt, signature);
+
+  const stats = session.getStatistics();
+  console.log(`\n💰 Final cost efficiency: $${stats.totalCost.toFixed(4)} for ${totalIterations} iterations`);
+}
+
+/**
+ * Example 4: Quality-Focused Training
+ */
+async function qualityFocusedTrainingExample() {
+  console.log('🚀 Starting Quality-Focused DSPy Training\n');
+
+  // Use high-quality models with aggressive optimization
+  const session = new DSPyTrainingSession({
+    models: [
+      {
+        provider: ModelProvider.CLAUDE,
+        model: 'claude-sonnet-4',
+        apiKey: process.env.ANTHROPIC_API_KEY || 'sk-ant-test',
+        temperature: 0.3 // Lower temperature for consistency
+      },
+      {
+        provider: ModelProvider.GPT4,
+        model: 'gpt-4-turbo',
+        apiKey: process.env.OPENAI_API_KEY || 'sk-test',
+        temperature: 0.3
+      }
+    ],
+    optimizationRounds: 15, // More rounds for quality
+    convergenceThreshold: 0.98, // Higher threshold
+    baselineIterations: 5,
+    benchmarkSamples: 100,
+    enableCrossLearning: true
+  });
+
+  // Quality monitoring
+  let highQualityCount = 0;
+  let totalCount = 0;
+
+  session.on('iteration', (result) => {
+    totalCount++;
+    if (result.quality.score >= 0.9) {
+      highQualityCount++;
+    }
+
+    const highQualityRate = highQualityCount / totalCount;
+    console.log(
+      `  ⭐ Quality rate: ${(highQualityRate * 100).toFixed(1)}% ` +
+      `(${highQualityCount}/${totalCount} >= 0.9)`
+    );
+  });
+
+  const optimizer = new OptimizationEngine();
+  const signature = optimizer.createSignature(
+    'technical-writing',
+    'Write technical documentation',
+    'clear, accurate technical documentation',
+    {
+      examples: [
+        {
+          input: 'Explain async/await in JavaScript',
+          output: 'Async/await is a modern JavaScript feature that simplifies asynchronous code...'
+        }
+      ],
+      constraints: [
+        'min_length:200',
+        'contains:code examples',
+        'contains:best practices'
+      ],
+      objectives: [
+        'Maximize clarity',
+        'Include practical examples',
+        'Follow technical writing standards',
+        'Ensure accuracy'
+      ]
+    }
+  );
+
+  const basePrompt = `
+Write clear, accurate technical documentation for:
+{topic}
+
+Requirements:
+- Include code examples
+- Explain concepts clearly
+- Follow best practices
+- Provide practical examples
+  `.trim();
+
+  await session.run(basePrompt, signature);
+}
+
+/**
+ * Example 5: Benchmark Comparison
+ */
+async function benchmarkComparisonExample() {
+  console.log('🚀 Starting Benchmark Comparison\n');
+
+  const collector = new BenchmarkCollector();
+
+  // Run training session
+  const session = new DSPyTrainingSession({
+    models: [
+      {
+        provider: ModelProvider.CLAUDE,
+        model: 'claude-sonnet-4',
+        apiKey: process.env.ANTHROPIC_API_KEY || 'sk-ant-test'
+      },
+      {
+        provider: ModelProvider.GPT4,
+        model: 'gpt-4-turbo',
+        apiKey: process.env.OPENAI_API_KEY || 'sk-test'
+      },
+      {
+        provider: ModelProvider.GEMINI,
+        model: 'gemini-2.0-flash-exp',
+        apiKey: process.env.GEMINI_API_KEY || 'test-key'
+      },
+      {
+        provider: ModelProvider.LLAMA,
+        model: 'llama-3.1-70b',
+        apiKey: process.env.TOGETHER_API_KEY || 'test-key'
+      }
+    ],
+    optimizationRounds: 5,
+    benchmarkSamples: 50
+  });
+
+  session.on('iteration', (result) => {
+    collector.addResult(result);
+  });
+
+  session.on('complete', () => {
+    console.log('\n📊 Benchmark Comparison:\n');
+
+    // Get comparison
+    const comparison = collector.getComparison();
+
+    // Display comparison table
+    console.log('Model       | Iterations | Avg Quality | Avg Latency | Total Cost | Improvement');
+    console.log('------------|------------|-------------|-------------|------------|------------');
+
+    for (const [provider, stats] of Object.entries(comparison)) {
+      if (!stats) continue;
+
+      console.log(
+        `${provider.padEnd(11)} | ` +
+        `${String(stats.totalIterations).padEnd(10)} | ` +
+        `${stats.avgQualityScore.toFixed(3).padEnd(11)} | ` +
+        `${stats.avgLatency.toFixed(0).padEnd(11)}ms | ` +
+        `$${stats.totalCost.toFixed(4).padEnd(9)} | ` +
+        `${(stats.improvementRate * 100).toFixed(1)}%`
+      );
+    }
+
+    // Highlight best model
+    const bestModel = collector.getBestModel();
+    console.log(`\n🏆 Winner: ${bestModel}`);
+
+    // Generate full report
+    console.log('\n' + collector.generateReport());
+  });
+
+  const optimizer = new OptimizationEngine();
+  const signature = optimizer.createSignature(
+    'general-task',
+    'Complete task',
+    'high-quality output'
+  );
+
+  const basePrompt = 'Complete the following task: {task}';
+
+  await session.run(basePrompt, signature);
+}
+
+// ============================================================================
+// Main Execution
+// ============================================================================
+
+async function main() {
+  console.log('=' .repeat(80));
+  console.log('DSPy.ts Training Session Examples');
+  console.log('=' .repeat(80));
+  console.log();
+
+  const examples = [
+    { name: 'Basic Training', fn: basicTrainingExample },
+    { name: 'Advanced Monitoring', fn: advancedTrainingExample },
+    { name: 'Cost-Optimized', fn: costOptimizedTrainingExample },
+    { name: 'Quality-Focused', fn: qualityFocusedTrainingExample },
+    { name: 'Benchmark Comparison', fn: benchmarkComparisonExample }
+  ];
+
+  // Run the example specified by command line arg, or default to first
+  const exampleIndex = parseInt(process.argv[2] || '0');
+
+  if (exampleIndex < 0 || exampleIndex >= examples.length) {
+    console.log('Available examples:');
+    examples.forEach((ex, i) => {
+      console.log(`  ${i}: ${ex.name}`);
+    });
+    console.log(`\nUsage: node dspy-training-example.js [0-${examples.length - 1}]`);
+    return;
+  }
+
+  const example = examples[exampleIndex];
+  console.log(`Running: ${example.name}\n`);
+
+  try {
+    await example.fn();
+  } catch (error) {
+    console.error('\n❌ Example failed:', error);
+    process.exit(1);
+  }
+}
+
+// Run if executed directly
+if (import.meta.url === `file://${process.argv[1]}`) {
+  main().catch(console.error);
+}
+
+export {
+  basicTrainingExample,
+  advancedTrainingExample,
+  costOptimizedTrainingExample,
+  qualityFocusedTrainingExample,
+  benchmarkComparisonExample
+};