"use strict"; /** * Capacity Manager - Global Capacity Orchestration * * Handles: * - Cross-region capacity allocation * - Budget-aware scaling decisions * - Priority-based resource allocation * - Graceful degradation strategies * - Traffic shedding when necessary */ Object.defineProperty(exports, "__esModule", { value: true }); exports.CapacityManager = void 0; const child_process_1 = require("child_process"); const util_1 = require("util"); const burst_predictor_1 = require("./burst-predictor"); const reactive_scaler_1 = require("./reactive-scaler"); const execAsync = (0, util_1.promisify)(child_process_1.exec); class CapacityManager { constructor(regions = ['us-central1', 'europe-west1', 'asia-east1'], notifyHook = async (msg) => { await execAsync(`npx claude-flow@alpha hooks notify --message "${msg.replace(/"/g, '\\"')}"`); }) { this.notifyHook = notifyHook; this.regionCapacities = new Map(); this.trafficPriorities = new Map(); this.isPreWarming = false; this.currentDegradationLevel = 'none'; // Initialize region capacities this.initializeRegionCapacities(regions); // Initialize budget config this.budgetConfig = { hourlyBudget: 10000, // $10k/hour dailyBudget: 200000, // $200k/day monthlyBudget: 5000000, // $5M/month currentHourlyCost: 0, currentDailyCost: 0, currentMonthlyCost: 0, warningThreshold: 0.8, // Warn at 80% hardLimit: false // Allow temporary overages }; // Initialize traffic priorities this.trafficPriorities.set('premium', { tier: 'premium', connectionLimit: -1, // Unlimited canShed: false, latencySLA: 30 // 30ms }); this.trafficPriorities.set('standard', { tier: 'standard', connectionLimit: 1000000000, canShed: false, latencySLA: 50 // 50ms }); this.trafficPriorities.set('free', { tier: 'free', connectionLimit: 100000000, canShed: true, latencySLA: 200 // 200ms }); // Initialize predictor and scaler this.predictor = new burst_predictor_1.BurstPredictor(regions, notifyHook); this.scaler = new reactive_scaler_1.ReactiveScaler(regions, notifyHook); } /** * Initialize region capacities with costs */ initializeRegionCapacities(regions) { const costMap = { 'us-central1': 0.50, // $0.50/hour per instance 'us-east1': 0.52, 'us-west1': 0.54, 'europe-west1': 0.55, 'europe-west4': 0.58, 'asia-east1': 0.60, 'asia-southeast1': 0.62, 'south-america-east1': 0.65 }; const priorityMap = { 'us-central1': 10, // Highest priority 'us-east1': 9, 'europe-west1': 9, 'asia-east1': 8, 'us-west1': 7, 'asia-southeast1': 6, 'europe-west4': 6, 'south-america-east1': 5 }; for (const region of regions) { this.regionCapacities.set(region, { region, currentInstances: 10, // Start with min instances maxInstances: 1000, availableInstances: 990, costPerInstance: costMap[region] || 0.50, priority: priorityMap[region] || 5 }); } } /** * Update budget configuration */ updateBudget(config) { this.budgetConfig = { ...this.budgetConfig, ...config }; } /** * Main orchestration loop */ async orchestrate() { // 1. Get predictions const predictions = await this.predictor.predictUpcomingBursts(24); // 2. Check if pre-warming is needed if (predictions.length > 0 && !this.isPreWarming) { await this.handlePreWarming(predictions); } // 3. Process reactive scaling for each region const scalingActions = []; for (const [region, capacity] of this.regionCapacities) { // Get current metrics (in production, fetch from monitoring) const metrics = await this.getCurrentMetrics(region); // Process reactive scaling const action = await this.scaler.processMetrics(metrics); if (action.action !== 'none') { scalingActions.push(action); } } // 4. Apply scaling actions with budget constraints await this.applyScalingActions(scalingActions); // 5. Check budget and apply degradation if needed await this.checkBudgetAndDegrade(); // 6. Generate capacity plan return this.generateCapacityPlan(); } /** * Handle pre-warming for predicted bursts */ async handlePreWarming(predictions) { const now = new Date(); for (const prediction of predictions) { const preWarmTime = new Date(prediction.startTime.getTime() - prediction.preWarmTime * 1000); if (now >= preWarmTime && now < prediction.startTime) { this.isPreWarming = true; await this.notifyHook(`PRE-WARMING: Starting capacity ramp-up for ${prediction.eventName} (${prediction.expectedMultiplier}x load expected)`); // Scale each region to required capacity for (const regionPred of prediction.regions) { const capacity = this.regionCapacities.get(regionPred.region); if (capacity && regionPred.requiredInstances > capacity.currentInstances) { await this.scaleRegion(regionPred.region, regionPred.requiredInstances, 'predictive-prewarm'); } } } } } /** * Apply scaling actions with budget and priority constraints */ async applyScalingActions(actions) { // Sort by urgency and priority const sortedActions = actions.sort((a, b) => { const urgencyScore = { critical: 4, high: 3, normal: 2, low: 1 }; const aScore = urgencyScore[a.urgency]; const bScore = urgencyScore[b.urgency]; if (aScore !== bScore) return bScore - aScore; // Then by region priority const aCapacity = this.regionCapacities.get(a.region); const bCapacity = this.regionCapacities.get(b.region); return bCapacity.priority - aCapacity.priority; }); for (const action of sortedActions) { if (action.action === 'scale-out') { // Check budget before scaling out const canScale = await this.checkBudgetForScaling(action.region, action.toInstances - action.fromInstances); if (canScale) { await this.scaleRegion(action.region, action.toInstances, 'reactive'); } else { await this.notifyHook(`BUDGET LIMIT: Cannot scale ${action.region} - budget exceeded`); // Consider degradation await this.applyDegradation('minor'); } } else if (action.action === 'scale-in') { // Always allow scale-in (saves money) await this.scaleRegion(action.region, action.toInstances, 'reactive'); } } } /** * Scale a specific region */ async scaleRegion(region, targetInstances, reason) { const capacity = this.regionCapacities.get(region); if (!capacity) { throw new Error(`Region ${region} not found`); } const oldInstances = capacity.currentInstances; capacity.currentInstances = Math.min(targetInstances, capacity.maxInstances); capacity.availableInstances = capacity.maxInstances - capacity.currentInstances; // Update budget await this.updateBudgetCosts(); await this.notifyHook(`SCALED: ${region} ${oldInstances} -> ${capacity.currentInstances} instances (${reason})`); // In production, call Terraform or Cloud Run API to actually scale // await this.executeTerraformScale(region, capacity.currentInstances); } /** * Check if budget allows scaling */ async checkBudgetForScaling(region, additionalInstances) { const capacity = this.regionCapacities.get(region); const additionalCost = capacity.costPerInstance * additionalInstances; const newHourlyCost = this.budgetConfig.currentHourlyCost + additionalCost; if (this.budgetConfig.hardLimit) { // Hard limit - don't exceed budget return newHourlyCost <= this.budgetConfig.hourlyBudget; } else { // Soft limit - warn but allow if (newHourlyCost > this.budgetConfig.hourlyBudget * this.budgetConfig.warningThreshold) { await this.notifyHook(`BUDGET WARNING: Approaching hourly budget limit ($${newHourlyCost.toFixed(2)}/$${this.budgetConfig.hourlyBudget})`); } // Allow up to 120% of budget for burst events return newHourlyCost <= this.budgetConfig.hourlyBudget * 1.2; } } /** * Update budget costs based on current capacity */ async updateBudgetCosts() { let totalHourlyCost = 0; for (const capacity of this.regionCapacities.values()) { totalHourlyCost += capacity.currentInstances * capacity.costPerInstance; } this.budgetConfig.currentHourlyCost = totalHourlyCost; this.budgetConfig.currentDailyCost = totalHourlyCost * 24; this.budgetConfig.currentMonthlyCost = totalHourlyCost * 24 * 30; } /** * Check budget and apply degradation if needed */ async checkBudgetAndDegrade() { const hourlyUsage = this.budgetConfig.currentHourlyCost / this.budgetConfig.hourlyBudget; const dailyUsage = this.budgetConfig.currentDailyCost / this.budgetConfig.dailyBudget; if (hourlyUsage > 1.0 || dailyUsage > 1.0) { await this.applyDegradation('major'); } else if (hourlyUsage > 0.9 || dailyUsage > 0.9) { await this.applyDegradation('minor'); } else if (this.currentDegradationLevel !== 'none') { // Recover from degradation await this.applyDegradation('none'); } } /** * Apply degradation strategy */ async applyDegradation(level) { if (level === this.currentDegradationLevel) { return; // Already at this level } const strategy = this.getDegradationStrategy(level); this.currentDegradationLevel = level; await this.notifyHook(`DEGRADATION: ${level.toUpperCase()} - ${strategy.impactDescription}`); // Execute degradation actions for (const action of strategy.actions) { // In production, execute actual degradation (e.g., enable rate limiting, shed traffic) console.log(`Executing: ${action}`); } } /** * Get degradation strategy for a given level */ getDegradationStrategy(level) { const strategies = { none: { level: 'none', actions: ['Restore normal operations'], impactDescription: 'Normal operations - all features available' }, minor: { level: 'minor', actions: [ 'Reduce connection limits for free tier by 20%', 'Increase cache TTL by 2x', 'Defer non-critical background jobs' ], impactDescription: 'Minor impact - free tier users may experience connection limits' }, major: { level: 'major', actions: [ 'Shed 50% of free tier traffic', 'Reduce connection limits for standard tier by 10%', 'Disable non-essential features (recommendations, analytics)', 'Enable aggressive connection pooling' ], impactDescription: 'Major impact - free tier heavily restricted, some features disabled' }, critical: { level: 'critical', actions: [ 'Shed all free tier traffic', 'Reduce standard tier to 50% capacity', 'Premium tier only with reduced features', 'Enable maintenance mode for non-critical services' ], impactDescription: 'Critical - only premium tier with limited functionality' } }; return strategies[level]; } /** * Generate capacity plan */ generateCapacityPlan() { let totalInstances = 0; let totalCost = 0; const regions = []; for (const capacity of this.regionCapacities.values()) { const instances = capacity.currentInstances; const cost = instances * capacity.costPerInstance; const utilization = capacity.currentInstances / capacity.maxInstances; totalInstances += instances; totalCost += cost; regions.push({ region: capacity.region, instances, cost, utilization }); } const budgetRemaining = this.budgetConfig.hourlyBudget - this.budgetConfig.currentHourlyCost; return { timestamp: new Date(), totalInstances, totalCost, regions, budgetRemaining, degradationLevel: this.currentDegradationLevel }; } /** * Get current metrics for a region (mock - would fetch from monitoring in production) */ async getCurrentMetrics(region) { const capacity = this.regionCapacities.get(region); // Mock metrics - in production, fetch from Cloud Monitoring return { region, timestamp: new Date(), cpuUtilization: 0.5 + Math.random() * 0.3, // 50-80% memoryUtilization: 0.4 + Math.random() * 0.3, // 40-70% activeConnections: capacity.currentInstances * 400000 + Math.random() * 100000, requestRate: capacity.currentInstances * 1000, errorRate: 0.001 + Math.random() * 0.004, // 0.1-0.5% p99Latency: 30 + Math.random() * 20, // 30-50ms currentInstances: capacity.currentInstances }; } /** * Get global capacity status */ getGlobalStatus() { let totalInstances = 0; let totalCost = 0; for (const capacity of this.regionCapacities.values()) { totalInstances += capacity.currentInstances; totalCost += capacity.currentInstances * capacity.costPerInstance; } return { totalInstances, totalCost, budgetUsage: totalCost / this.budgetConfig.hourlyBudget, degradationLevel: this.currentDegradationLevel, regions: this.regionCapacities }; } } exports.CapacityManager = CapacityManager; // Example usage if (require.main === module) { const manager = new CapacityManager(); // Run orchestration manager.orchestrate().then(plan => { console.log('\n=== Capacity Plan ==='); console.log(`Timestamp: ${plan.timestamp.toISOString()}`); console.log(`Total Instances: ${plan.totalInstances}`); console.log(`Total Cost: $${plan.totalCost.toFixed(2)}/hour`); console.log(`Budget Remaining: $${plan.budgetRemaining.toFixed(2)}/hour`); console.log(`Degradation Level: ${plan.degradationLevel}`); console.log('\nRegions:'); plan.regions.forEach(r => { console.log(` ${r.region}: ${r.instances} instances ($${r.cost.toFixed(2)}/hr, ${(r.utilization * 100).toFixed(1)}% utilization)`); }); }); } //# sourceMappingURL=capacity-manager.js.map