638 lines
19 KiB
JavaScript
638 lines
19 KiB
JavaScript
/**
|
|
* Real Data Connectors
|
|
*
|
|
* APIs for market data from multiple sources:
|
|
* - Yahoo Finance (free, delayed)
|
|
* - Alpha Vantage (free tier available)
|
|
* - Binance (crypto, real-time)
|
|
* - Polygon.io (stocks, options)
|
|
* - IEX Cloud (stocks)
|
|
*
|
|
* Features:
|
|
* - Rate limiting
|
|
* - Caching
|
|
* - Error handling
|
|
* - Data normalization
|
|
*/
|
|
|
|
// Connector Configuration
|
|
const connectorConfig = {
|
|
// API Keys (set via environment or constructor)
|
|
apiKeys: {
|
|
alphaVantage: process.env.ALPHA_VANTAGE_KEY || '',
|
|
polygon: process.env.POLYGON_KEY || '',
|
|
iex: process.env.IEX_KEY || '',
|
|
binance: process.env.BINANCE_KEY || ''
|
|
},
|
|
|
|
// Rate limits (requests per minute)
|
|
rateLimits: {
|
|
yahoo: 100,
|
|
alphaVantage: 5,
|
|
binance: 1200,
|
|
polygon: 100,
|
|
iex: 100
|
|
},
|
|
|
|
// Cache settings
|
|
cache: {
|
|
enabled: true,
|
|
ttl: 60000, // 1 minute default
|
|
maxSize: 1000
|
|
},
|
|
|
|
// Retry settings
|
|
retry: {
|
|
maxRetries: 3,
|
|
backoffMs: 1000
|
|
}
|
|
};
|
|
|
|
/**
|
|
* Simple LRU Cache
|
|
*/
|
|
class LRUCache {
|
|
constructor(maxSize = 1000, ttl = 60000) {
|
|
this.maxSize = maxSize;
|
|
this.ttl = ttl;
|
|
this.cache = new Map();
|
|
}
|
|
|
|
get(key) {
|
|
const entry = this.cache.get(key);
|
|
if (!entry) return null;
|
|
if (Date.now() - entry.timestamp > this.ttl) {
|
|
this.cache.delete(key);
|
|
return null;
|
|
}
|
|
// Move to end (most recent)
|
|
this.cache.delete(key);
|
|
this.cache.set(key, entry);
|
|
return entry.value;
|
|
}
|
|
|
|
set(key, value) {
|
|
if (this.cache.size >= this.maxSize) {
|
|
// Remove oldest entry
|
|
const firstKey = this.cache.keys().next().value;
|
|
this.cache.delete(firstKey);
|
|
}
|
|
this.cache.set(key, { value, timestamp: Date.now() });
|
|
}
|
|
|
|
clear() {
|
|
this.cache.clear();
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Rate Limiter
|
|
*/
|
|
class RateLimiter {
|
|
constructor(requestsPerMinute) {
|
|
this.requestsPerMinute = requestsPerMinute;
|
|
this.requests = [];
|
|
}
|
|
|
|
async acquire() {
|
|
const now = Date.now();
|
|
// Remove requests older than 1 minute
|
|
this.requests = this.requests.filter(t => now - t < 60000);
|
|
|
|
if (this.requests.length >= this.requestsPerMinute) {
|
|
const waitTime = 60000 - (now - this.requests[0]);
|
|
await new Promise(resolve => setTimeout(resolve, waitTime));
|
|
return this.acquire();
|
|
}
|
|
|
|
this.requests.push(now);
|
|
return true;
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Base Data Connector
|
|
*/
|
|
class BaseConnector {
|
|
constructor(config = {}) {
|
|
this.config = { ...connectorConfig, ...config };
|
|
this.cache = new LRUCache(
|
|
this.config.cache.maxSize,
|
|
this.config.cache.ttl
|
|
);
|
|
this.rateLimiters = {};
|
|
}
|
|
|
|
getRateLimiter(source) {
|
|
if (!this.rateLimiters[source]) {
|
|
this.rateLimiters[source] = new RateLimiter(
|
|
this.config.rateLimits[source] || 100
|
|
);
|
|
}
|
|
return this.rateLimiters[source];
|
|
}
|
|
|
|
async fetchWithRetry(url, options = {}, source = 'default') {
|
|
const cacheKey = `${source}:${url}`;
|
|
|
|
// Check cache
|
|
if (this.config.cache.enabled) {
|
|
const cached = this.cache.get(cacheKey);
|
|
if (cached) return cached;
|
|
}
|
|
|
|
// Rate limit
|
|
await this.getRateLimiter(source).acquire();
|
|
|
|
let lastError;
|
|
for (let i = 0; i < this.config.retry.maxRetries; i++) {
|
|
try {
|
|
const response = await fetch(url, options);
|
|
|
|
if (!response.ok) {
|
|
throw new Error(`HTTP ${response.status}: ${response.statusText}`);
|
|
}
|
|
|
|
const data = await response.json();
|
|
|
|
// Cache result
|
|
if (this.config.cache.enabled) {
|
|
this.cache.set(cacheKey, data);
|
|
}
|
|
|
|
return data;
|
|
} catch (error) {
|
|
lastError = error;
|
|
await new Promise(r => setTimeout(r, this.config.retry.backoffMs * (i + 1)));
|
|
}
|
|
}
|
|
|
|
throw lastError;
|
|
}
|
|
|
|
// Normalize OHLCV data to common format
|
|
normalizeOHLCV(data, source) {
|
|
return data.map(d => ({
|
|
timestamp: new Date(d.timestamp || d.date || d.t).getTime(),
|
|
open: parseFloat(d.open || d.o || d['1. open'] || 0),
|
|
high: parseFloat(d.high || d.h || d['2. high'] || 0),
|
|
low: parseFloat(d.low || d.l || d['3. low'] || 0),
|
|
close: parseFloat(d.close || d.c || d['4. close'] || 0),
|
|
volume: parseFloat(d.volume || d.v || d['5. volume'] || 0),
|
|
source
|
|
}));
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Yahoo Finance Connector (via unofficial API)
|
|
*/
|
|
class YahooFinanceConnector extends BaseConnector {
|
|
constructor(config = {}) {
|
|
super(config);
|
|
this.baseUrl = 'https://query1.finance.yahoo.com/v8/finance';
|
|
}
|
|
|
|
async getQuote(symbol) {
|
|
const url = `${this.baseUrl}/chart/${symbol}?interval=1d&range=1d`;
|
|
const data = await this.fetchWithRetry(url, {}, 'yahoo');
|
|
|
|
if (!data.chart?.result?.[0]) {
|
|
throw new Error(`No data for symbol: ${symbol}`);
|
|
}
|
|
|
|
const result = data.chart.result[0];
|
|
const quote = result.indicators.quote[0];
|
|
const meta = result.meta;
|
|
|
|
return {
|
|
symbol: meta.symbol,
|
|
price: meta.regularMarketPrice,
|
|
previousClose: meta.previousClose,
|
|
change: meta.regularMarketPrice - meta.previousClose,
|
|
changePercent: ((meta.regularMarketPrice - meta.previousClose) / meta.previousClose) * 100,
|
|
volume: quote.volume?.[quote.volume.length - 1] || 0,
|
|
timestamp: Date.now()
|
|
};
|
|
}
|
|
|
|
async getHistorical(symbol, period = '1y', interval = '1d') {
|
|
const url = `${this.baseUrl}/chart/${symbol}?interval=${interval}&range=${period}`;
|
|
const data = await this.fetchWithRetry(url, {}, 'yahoo');
|
|
|
|
if (!data.chart?.result?.[0]) {
|
|
throw new Error(`No data for symbol: ${symbol}`);
|
|
}
|
|
|
|
const result = data.chart.result[0];
|
|
const timestamps = result.timestamp;
|
|
const quote = result.indicators.quote[0];
|
|
|
|
const candles = [];
|
|
for (let i = 0; i < timestamps.length; i++) {
|
|
if (quote.open[i] !== null) {
|
|
candles.push({
|
|
timestamp: timestamps[i] * 1000,
|
|
open: quote.open[i],
|
|
high: quote.high[i],
|
|
low: quote.low[i],
|
|
close: quote.close[i],
|
|
volume: quote.volume[i],
|
|
source: 'yahoo'
|
|
});
|
|
}
|
|
}
|
|
|
|
return candles;
|
|
}
|
|
|
|
async search(query) {
|
|
const url = `https://query2.finance.yahoo.com/v1/finance/search?q=${encodeURIComponent(query)}`;
|
|
const data = await this.fetchWithRetry(url, {}, 'yahoo');
|
|
return data.quotes?.map(q => ({
|
|
symbol: q.symbol,
|
|
name: q.shortname || q.longname,
|
|
type: q.quoteType,
|
|
exchange: q.exchange
|
|
})) || [];
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Alpha Vantage Connector
|
|
*/
|
|
class AlphaVantageConnector extends BaseConnector {
|
|
constructor(config = {}) {
|
|
super(config);
|
|
this.baseUrl = 'https://www.alphavantage.co/query';
|
|
this.apiKey = config.apiKey || this.config.apiKeys.alphaVantage;
|
|
}
|
|
|
|
async getQuote(symbol) {
|
|
if (!this.apiKey) throw new Error('Alpha Vantage API key required');
|
|
|
|
const url = `${this.baseUrl}?function=GLOBAL_QUOTE&symbol=${symbol}&apikey=${this.apiKey}`;
|
|
const data = await this.fetchWithRetry(url, {}, 'alphaVantage');
|
|
|
|
const quote = data['Global Quote'];
|
|
if (!quote) throw new Error(`No data for symbol: ${symbol}`);
|
|
|
|
return {
|
|
symbol: quote['01. symbol'],
|
|
price: parseFloat(quote['05. price']),
|
|
previousClose: parseFloat(quote['08. previous close']),
|
|
change: parseFloat(quote['09. change']),
|
|
changePercent: parseFloat(quote['10. change percent'].replace('%', '')),
|
|
volume: parseInt(quote['06. volume']),
|
|
timestamp: Date.now()
|
|
};
|
|
}
|
|
|
|
async getHistorical(symbol, outputSize = 'compact') {
|
|
if (!this.apiKey) throw new Error('Alpha Vantage API key required');
|
|
|
|
const url = `${this.baseUrl}?function=TIME_SERIES_DAILY&symbol=${symbol}&outputsize=${outputSize}&apikey=${this.apiKey}`;
|
|
const data = await this.fetchWithRetry(url, {}, 'alphaVantage');
|
|
|
|
const timeSeries = data['Time Series (Daily)'];
|
|
if (!timeSeries) throw new Error(`No data for symbol: ${symbol}`);
|
|
|
|
return Object.entries(timeSeries).map(([date, values]) => ({
|
|
timestamp: new Date(date).getTime(),
|
|
open: parseFloat(values['1. open']),
|
|
high: parseFloat(values['2. high']),
|
|
low: parseFloat(values['3. low']),
|
|
close: parseFloat(values['4. close']),
|
|
volume: parseInt(values['5. volume']),
|
|
source: 'alphaVantage'
|
|
})).sort((a, b) => a.timestamp - b.timestamp);
|
|
}
|
|
|
|
async getIntraday(symbol, interval = '5min') {
|
|
if (!this.apiKey) throw new Error('Alpha Vantage API key required');
|
|
|
|
const url = `${this.baseUrl}?function=TIME_SERIES_INTRADAY&symbol=${symbol}&interval=${interval}&apikey=${this.apiKey}`;
|
|
const data = await this.fetchWithRetry(url, {}, 'alphaVantage');
|
|
|
|
const key = `Time Series (${interval})`;
|
|
const timeSeries = data[key];
|
|
if (!timeSeries) throw new Error(`No data for symbol: ${symbol}`);
|
|
|
|
return Object.entries(timeSeries).map(([datetime, values]) => ({
|
|
timestamp: new Date(datetime).getTime(),
|
|
open: parseFloat(values['1. open']),
|
|
high: parseFloat(values['2. high']),
|
|
low: parseFloat(values['3. low']),
|
|
close: parseFloat(values['4. close']),
|
|
volume: parseInt(values['5. volume']),
|
|
source: 'alphaVantage'
|
|
})).sort((a, b) => a.timestamp - b.timestamp);
|
|
}
|
|
|
|
async getSentiment(tickers) {
|
|
if (!this.apiKey) throw new Error('Alpha Vantage API key required');
|
|
|
|
const tickerList = Array.isArray(tickers) ? tickers.join(',') : tickers;
|
|
const url = `${this.baseUrl}?function=NEWS_SENTIMENT&tickers=${tickerList}&apikey=${this.apiKey}`;
|
|
const data = await this.fetchWithRetry(url, {}, 'alphaVantage');
|
|
|
|
return data.feed?.map(item => ({
|
|
title: item.title,
|
|
url: item.url,
|
|
source: item.source,
|
|
summary: item.summary,
|
|
sentiment: item.overall_sentiment_score,
|
|
sentimentLabel: item.overall_sentiment_label,
|
|
tickers: item.ticker_sentiment,
|
|
timestamp: new Date(item.time_published).getTime()
|
|
})) || [];
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Binance Connector (Crypto)
|
|
*/
|
|
class BinanceConnector extends BaseConnector {
|
|
constructor(config = {}) {
|
|
super(config);
|
|
this.baseUrl = 'https://api.binance.com/api/v3';
|
|
this.wsUrl = 'wss://stream.binance.com:9443/ws';
|
|
}
|
|
|
|
async getQuote(symbol) {
|
|
const url = `${this.baseUrl}/ticker/24hr?symbol=${symbol}`;
|
|
const data = await this.fetchWithRetry(url, {}, 'binance');
|
|
|
|
return {
|
|
symbol: data.symbol,
|
|
price: parseFloat(data.lastPrice),
|
|
previousClose: parseFloat(data.prevClosePrice),
|
|
change: parseFloat(data.priceChange),
|
|
changePercent: parseFloat(data.priceChangePercent),
|
|
volume: parseFloat(data.volume),
|
|
quoteVolume: parseFloat(data.quoteVolume),
|
|
high24h: parseFloat(data.highPrice),
|
|
low24h: parseFloat(data.lowPrice),
|
|
timestamp: data.closeTime
|
|
};
|
|
}
|
|
|
|
async getHistorical(symbol, interval = '1d', limit = 500) {
|
|
const url = `${this.baseUrl}/klines?symbol=${symbol}&interval=${interval}&limit=${limit}`;
|
|
const data = await this.fetchWithRetry(url, {}, 'binance');
|
|
|
|
return data.map(candle => ({
|
|
timestamp: candle[0],
|
|
open: parseFloat(candle[1]),
|
|
high: parseFloat(candle[2]),
|
|
low: parseFloat(candle[3]),
|
|
close: parseFloat(candle[4]),
|
|
volume: parseFloat(candle[5]),
|
|
closeTime: candle[6],
|
|
quoteVolume: parseFloat(candle[7]),
|
|
trades: candle[8],
|
|
source: 'binance'
|
|
}));
|
|
}
|
|
|
|
async getOrderBook(symbol, limit = 100) {
|
|
const url = `${this.baseUrl}/depth?symbol=${symbol}&limit=${limit}`;
|
|
const data = await this.fetchWithRetry(url, {}, 'binance');
|
|
|
|
return {
|
|
lastUpdateId: data.lastUpdateId,
|
|
bids: data.bids.map(([price, qty]) => ({
|
|
price: parseFloat(price),
|
|
quantity: parseFloat(qty)
|
|
})),
|
|
asks: data.asks.map(([price, qty]) => ({
|
|
price: parseFloat(price),
|
|
quantity: parseFloat(qty)
|
|
}))
|
|
};
|
|
}
|
|
|
|
async getTrades(symbol, limit = 100) {
|
|
const url = `${this.baseUrl}/trades?symbol=${symbol}&limit=${limit}`;
|
|
const data = await this.fetchWithRetry(url, {}, 'binance');
|
|
|
|
return data.map(trade => ({
|
|
id: trade.id,
|
|
price: parseFloat(trade.price),
|
|
quantity: parseFloat(trade.qty),
|
|
time: trade.time,
|
|
isBuyerMaker: trade.isBuyerMaker
|
|
}));
|
|
}
|
|
|
|
// WebSocket subscription for real-time data
|
|
subscribeToTrades(symbol, callback) {
|
|
const ws = new WebSocket(`${this.wsUrl}/${symbol.toLowerCase()}@trade`);
|
|
|
|
ws.onmessage = (event) => {
|
|
const data = JSON.parse(event.data);
|
|
callback({
|
|
symbol: data.s,
|
|
price: parseFloat(data.p),
|
|
quantity: parseFloat(data.q),
|
|
time: data.T,
|
|
isBuyerMaker: data.m
|
|
});
|
|
};
|
|
|
|
return {
|
|
close: () => ws.close()
|
|
};
|
|
}
|
|
|
|
subscribeToKlines(symbol, interval, callback) {
|
|
const ws = new WebSocket(`${this.wsUrl}/${symbol.toLowerCase()}@kline_${interval}`);
|
|
|
|
ws.onmessage = (event) => {
|
|
const data = JSON.parse(event.data);
|
|
const k = data.k;
|
|
callback({
|
|
symbol: k.s,
|
|
interval: k.i,
|
|
open: parseFloat(k.o),
|
|
high: parseFloat(k.h),
|
|
low: parseFloat(k.l),
|
|
close: parseFloat(k.c),
|
|
volume: parseFloat(k.v),
|
|
isClosed: k.x,
|
|
timestamp: k.t
|
|
});
|
|
};
|
|
|
|
return {
|
|
close: () => ws.close()
|
|
};
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Unified Data Manager
|
|
*/
|
|
class DataManager {
|
|
constructor(config = {}) {
|
|
this.config = { ...connectorConfig, ...config };
|
|
this.connectors = {
|
|
yahoo: new YahooFinanceConnector(config),
|
|
alphaVantage: new AlphaVantageConnector(config),
|
|
binance: new BinanceConnector(config)
|
|
};
|
|
this.preferredSource = config.preferredSource || 'yahoo';
|
|
}
|
|
|
|
// Get connector by name
|
|
getConnector(name) {
|
|
return this.connectors[name];
|
|
}
|
|
|
|
// Smart quote - try preferred source, fallback to others
|
|
async getQuote(symbol, source = null) {
|
|
const sources = source ? [source] : [this.preferredSource, 'yahoo', 'alphaVantage'];
|
|
|
|
for (const src of sources) {
|
|
try {
|
|
const connector = this.connectors[src];
|
|
if (connector) {
|
|
return await connector.getQuote(symbol);
|
|
}
|
|
} catch (error) {
|
|
console.warn(`Quote failed for ${symbol} from ${src}:`, error.message);
|
|
}
|
|
}
|
|
|
|
throw new Error(`Failed to get quote for ${symbol} from all sources`);
|
|
}
|
|
|
|
// Get historical data with source selection
|
|
async getHistorical(symbol, options = {}) {
|
|
const {
|
|
source = this.preferredSource,
|
|
period = '1y',
|
|
interval = '1d'
|
|
} = options;
|
|
|
|
const connector = this.connectors[source];
|
|
if (!connector) throw new Error(`Unknown source: ${source}`);
|
|
|
|
if (source === 'yahoo') {
|
|
return connector.getHistorical(symbol, period, interval);
|
|
} else if (source === 'alphaVantage') {
|
|
return connector.getHistorical(symbol, period === '1y' ? 'full' : 'compact');
|
|
} else if (source === 'binance') {
|
|
return connector.getHistorical(symbol, interval);
|
|
}
|
|
}
|
|
|
|
// Get multiple symbols in parallel
|
|
async getQuotes(symbols) {
|
|
const promises = symbols.map(s => this.getQuote(s).catch(e => ({ symbol: s, error: e.message })));
|
|
return Promise.all(promises);
|
|
}
|
|
|
|
// Get news sentiment
|
|
async getSentiment(symbols, source = 'alphaVantage') {
|
|
const connector = this.connectors[source];
|
|
if (connector?.getSentiment) {
|
|
return connector.getSentiment(symbols);
|
|
}
|
|
return [];
|
|
}
|
|
|
|
// Clear all caches
|
|
clearCache() {
|
|
for (const connector of Object.values(this.connectors)) {
|
|
connector.cache?.clear();
|
|
}
|
|
}
|
|
}
|
|
|
|
// Exports
|
|
export {
|
|
DataManager,
|
|
YahooFinanceConnector,
|
|
AlphaVantageConnector,
|
|
BinanceConnector,
|
|
BaseConnector,
|
|
LRUCache,
|
|
RateLimiter,
|
|
connectorConfig
|
|
};
|
|
|
|
// Demo if run directly
|
|
const isMainModule = import.meta.url === `file://${process.argv[1]}`;
|
|
if (isMainModule) {
|
|
console.log('══════════════════════════════════════════════════════════════════════');
|
|
console.log('DATA CONNECTORS DEMO');
|
|
console.log('══════════════════════════════════════════════════════════════════════\n');
|
|
|
|
console.log('Available Connectors:');
|
|
console.log('──────────────────────────────────────────────────────────────────────');
|
|
console.log(' • Yahoo Finance - Free, delayed quotes, historical data');
|
|
console.log(' • Alpha Vantage - Free tier (5 req/min), sentiment analysis');
|
|
console.log(' • Binance - Real-time crypto, WebSocket support');
|
|
console.log();
|
|
|
|
console.log('Features:');
|
|
console.log('──────────────────────────────────────────────────────────────────────');
|
|
console.log(' • Rate limiting per source');
|
|
console.log(' • LRU caching with TTL');
|
|
console.log(' • Automatic retry with backoff');
|
|
console.log(' • Data normalization to OHLCV format');
|
|
console.log(' • Multi-source fallback');
|
|
console.log();
|
|
|
|
console.log('Example Usage:');
|
|
console.log('──────────────────────────────────────────────────────────────────────');
|
|
console.log(`
|
|
import { DataManager } from './data-connectors.js';
|
|
|
|
const data = new DataManager({
|
|
apiKeys: { alphaVantage: 'YOUR_KEY' }
|
|
});
|
|
|
|
// Get quote
|
|
const quote = await data.getQuote('AAPL');
|
|
|
|
// Get historical data
|
|
const history = await data.getHistorical('AAPL', { period: '1y' });
|
|
|
|
// Get crypto data
|
|
const btc = await data.getQuote('BTCUSDT', 'binance');
|
|
const klines = await data.getHistorical('BTCUSDT', {
|
|
source: 'binance',
|
|
interval: '1h'
|
|
});
|
|
|
|
// Get sentiment
|
|
const sentiment = await data.getSentiment(['AAPL', 'MSFT']);
|
|
`);
|
|
|
|
// Test with mock data (no actual API calls)
|
|
console.log('\nSimulated Output:');
|
|
console.log('──────────────────────────────────────────────────────────────────────');
|
|
|
|
const mockQuote = {
|
|
symbol: 'AAPL',
|
|
price: 178.50,
|
|
previousClose: 177.25,
|
|
change: 1.25,
|
|
changePercent: 0.71,
|
|
volume: 52847300,
|
|
timestamp: Date.now()
|
|
};
|
|
|
|
console.log('Quote (AAPL):');
|
|
console.log(` Price: $${mockQuote.price}`);
|
|
console.log(` Change: $${mockQuote.change} (${mockQuote.changePercent.toFixed(2)}%)`);
|
|
console.log(` Volume: ${mockQuote.volume.toLocaleString()}`);
|
|
|
|
console.log();
|
|
console.log('══════════════════════════════════════════════════════════════════════');
|
|
console.log('Data connectors ready for integration');
|
|
console.log('══════════════════════════════════════════════════════════════════════');
|
|
}
|