Merge commit 'd803bfe2b1fe7f5e219e50ac20d6801a0a58ac75' as 'vendor/ruvector'
This commit is contained in:
ruv
345
vendor/ruvector/npm/packages/ruvector-extensions/src/embeddings.d.ts
vendored
Normal file
345
vendor/ruvector/npm/packages/ruvector-extensions/src/embeddings.d.ts
vendored
Normal file
@@ -0,0 +1,345 @@
|
||||
/**
|
||||
* @fileoverview Comprehensive embeddings integration module for ruvector-extensions
|
||||
* Supports multiple providers: OpenAI, Cohere, Anthropic, and local HuggingFace models
|
||||
*
|
||||
* @module embeddings
|
||||
* @author ruv.io Team <info@ruv.io>
|
||||
* @license MIT
|
||||
*
|
||||
* @example
|
||||
* ```typescript
|
||||
* // OpenAI embeddings
|
||||
* const openai = new OpenAIEmbeddings({ apiKey: 'sk-...' });
|
||||
* const embeddings = await openai.embedTexts(['Hello world', 'Test']);
|
||||
*
|
||||
* // Auto-insert into VectorDB
|
||||
* await embedAndInsert(db, openai, [
|
||||
* { id: '1', text: 'Hello world', metadata: { source: 'test' } }
|
||||
* ]);
|
||||
* ```
|
||||
*/
|
||||
type VectorDB = any;
|
||||
/**
|
||||
* Configuration for retry logic
|
||||
*/
|
||||
export interface RetryConfig {
|
||||
/** Maximum number of retry attempts */
|
||||
maxRetries: number;
|
||||
/** Initial delay in milliseconds before first retry */
|
||||
initialDelay: number;
|
||||
/** Maximum delay in milliseconds between retries */
|
||||
maxDelay: number;
|
||||
/** Multiplier for exponential backoff */
|
||||
backoffMultiplier: number;
|
||||
}
|
||||
/**
|
||||
* Result of an embedding operation
|
||||
*/
|
||||
export interface EmbeddingResult {
|
||||
/** The generated embedding vector */
|
||||
embedding: number[];
|
||||
/** Index of the text in the original batch */
|
||||
index: number;
|
||||
/** Optional token count used */
|
||||
tokens?: number;
|
||||
}
|
||||
/**
|
||||
* Batch result with embeddings and metadata
|
||||
*/
|
||||
export interface BatchEmbeddingResult {
|
||||
/** Array of embedding results */
|
||||
embeddings: EmbeddingResult[];
|
||||
/** Total tokens used (if available) */
|
||||
totalTokens?: number;
|
||||
/** Provider-specific metadata */
|
||||
metadata?: Record<string, unknown>;
|
||||
}
|
||||
/**
|
||||
* Error details for failed embedding operations
|
||||
*/
|
||||
export interface EmbeddingError {
|
||||
/** Error message */
|
||||
message: string;
|
||||
/** Original error object */
|
||||
error: unknown;
|
||||
/** Index of the text that failed (if applicable) */
|
||||
index?: number;
|
||||
/** Whether the error is retryable */
|
||||
retryable: boolean;
|
||||
}
|
||||
/**
|
||||
* Document to embed and insert into VectorDB
|
||||
*/
|
||||
export interface DocumentToEmbed {
|
||||
/** Unique identifier for the document */
|
||||
id: string;
|
||||
/** Text content to embed */
|
||||
text: string;
|
||||
/** Optional metadata to store with the vector */
|
||||
metadata?: Record<string, unknown>;
|
||||
}
|
||||
/**
|
||||
* Abstract base class for embedding providers
|
||||
* All embedding providers must extend this class and implement its methods
|
||||
*/
|
||||
export declare abstract class EmbeddingProvider {
|
||||
protected retryConfig: RetryConfig;
|
||||
/**
|
||||
* Creates a new embedding provider instance
|
||||
* @param retryConfig - Configuration for retry logic
|
||||
*/
|
||||
constructor(retryConfig?: Partial<RetryConfig>);
|
||||
/**
|
||||
* Get the maximum batch size supported by this provider
|
||||
*/
|
||||
abstract getMaxBatchSize(): number;
|
||||
/**
|
||||
* Get the dimension of embeddings produced by this provider
|
||||
*/
|
||||
abstract getDimension(): number;
|
||||
/**
|
||||
* Embed a single text string
|
||||
* @param text - Text to embed
|
||||
* @returns Promise resolving to the embedding vector
|
||||
*/
|
||||
embedText(text: string): Promise<number[]>;
|
||||
/**
|
||||
* Embed multiple texts with automatic batching
|
||||
* @param texts - Array of texts to embed
|
||||
* @returns Promise resolving to batch embedding results
|
||||
*/
|
||||
abstract embedTexts(texts: string[]): Promise<BatchEmbeddingResult>;
|
||||
/**
|
||||
* Execute a function with retry logic
|
||||
* @param fn - Function to execute
|
||||
* @param context - Context description for error messages
|
||||
* @returns Promise resolving to the function result
|
||||
*/
|
||||
protected withRetry<T>(fn: () => Promise<T>, context: string): Promise<T>;
|
||||
/**
|
||||
* Determine if an error is retryable
|
||||
* @param error - Error to check
|
||||
* @returns True if the error should trigger a retry
|
||||
*/
|
||||
protected isRetryableError(error: unknown): boolean;
|
||||
/**
|
||||
* Create a standardized embedding error
|
||||
* @param error - Original error
|
||||
* @param context - Context description
|
||||
* @param retryable - Whether the error is retryable
|
||||
* @returns Formatted error object
|
||||
*/
|
||||
protected createEmbeddingError(error: unknown, context: string, retryable: boolean): EmbeddingError;
|
||||
/**
|
||||
* Sleep for a specified duration
|
||||
* @param ms - Milliseconds to sleep
|
||||
*/
|
||||
protected sleep(ms: number): Promise<void>;
|
||||
/**
|
||||
* Split texts into batches based on max batch size
|
||||
* @param texts - Texts to batch
|
||||
* @returns Array of text batches
|
||||
*/
|
||||
protected createBatches(texts: string[]): string[][];
|
||||
}
|
||||
/**
|
||||
* Configuration for OpenAI embeddings
|
||||
*/
|
||||
export interface OpenAIEmbeddingsConfig {
|
||||
/** OpenAI API key */
|
||||
apiKey: string;
|
||||
/** Model name (default: 'text-embedding-3-small') */
|
||||
model?: string;
|
||||
/** Embedding dimensions (only for text-embedding-3-* models) */
|
||||
dimensions?: number;
|
||||
/** Organization ID (optional) */
|
||||
organization?: string;
|
||||
/** Custom base URL (optional) */
|
||||
baseURL?: string;
|
||||
/** Retry configuration */
|
||||
retryConfig?: Partial<RetryConfig>;
|
||||
}
|
||||
/**
|
||||
* OpenAI embeddings provider
|
||||
* Supports text-embedding-3-small, text-embedding-3-large, and text-embedding-ada-002
|
||||
*/
|
||||
export declare class OpenAIEmbeddings extends EmbeddingProvider {
|
||||
private config;
|
||||
private openai;
|
||||
/**
|
||||
* Creates a new OpenAI embeddings provider
|
||||
* @param config - Configuration options
|
||||
* @throws Error if OpenAI SDK is not installed
|
||||
*/
|
||||
constructor(config: OpenAIEmbeddingsConfig);
|
||||
getMaxBatchSize(): number;
|
||||
getDimension(): number;
|
||||
embedTexts(texts: string[]): Promise<BatchEmbeddingResult>;
|
||||
}
|
||||
/**
|
||||
* Configuration for Cohere embeddings
|
||||
*/
|
||||
export interface CohereEmbeddingsConfig {
|
||||
/** Cohere API key */
|
||||
apiKey: string;
|
||||
/** Model name (default: 'embed-english-v3.0') */
|
||||
model?: string;
|
||||
/** Input type: 'search_document', 'search_query', 'classification', or 'clustering' */
|
||||
inputType?: 'search_document' | 'search_query' | 'classification' | 'clustering';
|
||||
/** Truncate input text if it exceeds model limits */
|
||||
truncate?: 'NONE' | 'START' | 'END';
|
||||
/** Retry configuration */
|
||||
retryConfig?: Partial<RetryConfig>;
|
||||
}
|
||||
/**
|
||||
* Cohere embeddings provider
|
||||
* Supports embed-english-v3.0, embed-multilingual-v3.0, and other Cohere models
|
||||
*/
|
||||
export declare class CohereEmbeddings extends EmbeddingProvider {
|
||||
private config;
|
||||
private cohere;
|
||||
/**
|
||||
* Creates a new Cohere embeddings provider
|
||||
* @param config - Configuration options
|
||||
* @throws Error if Cohere SDK is not installed
|
||||
*/
|
||||
constructor(config: CohereEmbeddingsConfig);
|
||||
getMaxBatchSize(): number;
|
||||
getDimension(): number;
|
||||
embedTexts(texts: string[]): Promise<BatchEmbeddingResult>;
|
||||
}
|
||||
/**
|
||||
* Configuration for Anthropic embeddings via Voyage AI
|
||||
*/
|
||||
export interface AnthropicEmbeddingsConfig {
|
||||
/** Anthropic API key */
|
||||
apiKey: string;
|
||||
/** Model name (default: 'voyage-2') */
|
||||
model?: string;
|
||||
/** Input type for embeddings */
|
||||
inputType?: 'document' | 'query';
|
||||
/** Retry configuration */
|
||||
retryConfig?: Partial<RetryConfig>;
|
||||
}
|
||||
/**
|
||||
* Anthropic embeddings provider using Voyage AI
|
||||
* Anthropic partners with Voyage AI for embeddings
|
||||
*/
|
||||
export declare class AnthropicEmbeddings extends EmbeddingProvider {
|
||||
private config;
|
||||
private anthropic;
|
||||
/**
|
||||
* Creates a new Anthropic embeddings provider
|
||||
* @param config - Configuration options
|
||||
* @throws Error if Anthropic SDK is not installed
|
||||
*/
|
||||
constructor(config: AnthropicEmbeddingsConfig);
|
||||
getMaxBatchSize(): number;
|
||||
getDimension(): number;
|
||||
embedTexts(texts: string[]): Promise<BatchEmbeddingResult>;
|
||||
}
|
||||
/**
|
||||
* Configuration for HuggingFace local embeddings
|
||||
*/
|
||||
export interface HuggingFaceEmbeddingsConfig {
|
||||
/** Model name or path (default: 'sentence-transformers/all-MiniLM-L6-v2') */
|
||||
model?: string;
|
||||
/** Device to run on: 'cpu' or 'cuda' */
|
||||
device?: 'cpu' | 'cuda';
|
||||
/** Normalize embeddings to unit length */
|
||||
normalize?: boolean;
|
||||
/** Batch size for processing */
|
||||
batchSize?: number;
|
||||
/** Retry configuration */
|
||||
retryConfig?: Partial<RetryConfig>;
|
||||
}
|
||||
/**
|
||||
* HuggingFace local embeddings provider
|
||||
* Runs embedding models locally using transformers.js
|
||||
*/
|
||||
export declare class HuggingFaceEmbeddings extends EmbeddingProvider {
|
||||
private config;
|
||||
private pipeline;
|
||||
private initialized;
|
||||
/**
|
||||
* Creates a new HuggingFace local embeddings provider
|
||||
* @param config - Configuration options
|
||||
*/
|
||||
constructor(config?: HuggingFaceEmbeddingsConfig);
|
||||
getMaxBatchSize(): number;
|
||||
getDimension(): number;
|
||||
/**
|
||||
* Initialize the embedding pipeline
|
||||
*/
|
||||
private initialize;
|
||||
embedTexts(texts: string[]): Promise<BatchEmbeddingResult>;
|
||||
}
|
||||
/**
|
||||
* Embed texts and automatically insert them into a VectorDB
|
||||
*
|
||||
* @param db - VectorDB instance to insert into
|
||||
* @param provider - Embedding provider to use
|
||||
* @param documents - Documents to embed and insert
|
||||
* @param options - Additional options
|
||||
* @returns Promise resolving to array of inserted vector IDs
|
||||
*
|
||||
* @example
|
||||
* ```typescript
|
||||
* const openai = new OpenAIEmbeddings({ apiKey: 'sk-...' });
|
||||
* const db = new VectorDB({ dimension: 1536 });
|
||||
*
|
||||
* const ids = await embedAndInsert(db, openai, [
|
||||
* { id: '1', text: 'Hello world', metadata: { source: 'test' } },
|
||||
* { id: '2', text: 'Another document', metadata: { source: 'test' } }
|
||||
* ]);
|
||||
*
|
||||
* console.log('Inserted vector IDs:', ids);
|
||||
* ```
|
||||
*/
|
||||
export declare function embedAndInsert(db: VectorDB, provider: EmbeddingProvider, documents: DocumentToEmbed[], options?: {
|
||||
/** Whether to overwrite existing vectors with same ID */
|
||||
overwrite?: boolean;
|
||||
/** Progress callback */
|
||||
onProgress?: (current: number, total: number) => void;
|
||||
}): Promise<string[]>;
|
||||
/**
|
||||
* Embed a query and search for similar documents in VectorDB
|
||||
*
|
||||
* @param db - VectorDB instance to search
|
||||
* @param provider - Embedding provider to use
|
||||
* @param query - Query text to search for
|
||||
* @param options - Search options
|
||||
* @returns Promise resolving to search results
|
||||
*
|
||||
* @example
|
||||
* ```typescript
|
||||
* const openai = new OpenAIEmbeddings({ apiKey: 'sk-...' });
|
||||
* const db = new VectorDB({ dimension: 1536 });
|
||||
*
|
||||
* const results = await embedAndSearch(db, openai, 'machine learning', {
|
||||
* topK: 5,
|
||||
* threshold: 0.7
|
||||
* });
|
||||
*
|
||||
* console.log('Found documents:', results);
|
||||
* ```
|
||||
*/
|
||||
export declare function embedAndSearch(db: VectorDB, provider: EmbeddingProvider, query: string, options?: {
|
||||
/** Number of results to return */
|
||||
topK?: number;
|
||||
/** Minimum similarity threshold (0-1) */
|
||||
threshold?: number;
|
||||
/** Metadata filter */
|
||||
filter?: Record<string, unknown>;
|
||||
}): Promise<any[]>;
|
||||
declare const _default: {
|
||||
EmbeddingProvider: typeof EmbeddingProvider;
|
||||
OpenAIEmbeddings: typeof OpenAIEmbeddings;
|
||||
CohereEmbeddings: typeof CohereEmbeddings;
|
||||
AnthropicEmbeddings: typeof AnthropicEmbeddings;
|
||||
HuggingFaceEmbeddings: typeof HuggingFaceEmbeddings;
|
||||
embedAndInsert: typeof embedAndInsert;
|
||||
embedAndSearch: typeof embedAndSearch;
|
||||
};
|
||||
export default _default;
|
||||
//# sourceMappingURL=embeddings.d.ts.map
|
||||
1
vendor/ruvector/npm/packages/ruvector-extensions/src/embeddings.d.ts.map
vendored
Normal file
1
vendor/ruvector/npm/packages/ruvector-extensions/src/embeddings.d.ts.map
vendored
Normal file
@@ -0,0 +1 @@
|
||||
{"version":3,"file":"embeddings.d.ts","sourceRoot":"","sources":["embeddings.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;GAmBG;AAGH,KAAK,QAAQ,GAAG,GAAG,CAAC;AAMpB;;GAEG;AACH,MAAM,WAAW,WAAW;IAC1B,uCAAuC;IACvC,UAAU,EAAE,MAAM,CAAC;IACnB,uDAAuD;IACvD,YAAY,EAAE,MAAM,CAAC;IACrB,oDAAoD;IACpD,QAAQ,EAAE,MAAM,CAAC;IACjB,yCAAyC;IACzC,iBAAiB,EAAE,MAAM,CAAC;CAC3B;AAED;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B,qCAAqC;IACrC,SAAS,EAAE,MAAM,EAAE,CAAC;IACpB,8CAA8C;IAC9C,KAAK,EAAE,MAAM,CAAC;IACd,gCAAgC;IAChC,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED;;GAEG;AACH,MAAM,WAAW,oBAAoB;IACnC,iCAAiC;IACjC,UAAU,EAAE,eAAe,EAAE,CAAC;IAC9B,uCAAuC;IACvC,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,iCAAiC;IACjC,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACpC;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,oBAAoB;IACpB,OAAO,EAAE,MAAM,CAAC;IAChB,4BAA4B;IAC5B,KAAK,EAAE,OAAO,CAAC;IACf,oDAAoD;IACpD,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,qCAAqC;IACrC,SAAS,EAAE,OAAO,CAAC;CACpB;AAED;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B,yCAAyC;IACzC,EAAE,EAAE,MAAM,CAAC;IACX,4BAA4B;IAC5B,IAAI,EAAE,MAAM,CAAC;IACb,iDAAiD;IACjD,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACpC;AAMD;;;GAGG;AACH,8BAAsB,iBAAiB;IACrC,SAAS,CAAC,WAAW,EAAE,WAAW,CAAC;IAEnC;;;OAGG;gBACS,WAAW,CAAC,EAAE,OAAO,CAAC,WAAW,CAAC;IAU9C;;OAEG;IACH,QAAQ,CAAC,eAAe,IAAI,MAAM;IAElC;;OAEG;IACH,QAAQ,CAAC,YAAY,IAAI,MAAM;IAE/B;;;;OAIG;IACG,SAAS,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;IAKhD;;;;OAIG;IACH,QAAQ,CAAC,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,oBAAoB,CAAC;IAEnE;;;;;OAKG;cACa,SAAS,CAAC,CAAC,EACzB,EAAE,EAAE,MAAM,OAAO,CAAC,CAAC,CAAC,EACpB,OAAO,EAAE,MAAM,GACd,OAAO,CAAC,CAAC,CAAC;IAgCb;;;;OAIG;IACH,SAAS,CAAC,gBAAgB,CAAC,KAAK,EAAE,OAAO,GAAG,OAAO;IAenD;;;;;;OAMG;IACH,SAAS,CAAC,oBAAoB,CAC5B,KAAK,EAAE,OAAO,EACd,OAAO,EAAE,MAAM,EACf,SAAS,EAAE,OAAO,GACjB,cAAc;IASjB;;;OAGG;IACH,SAAS,CAAC,KAAK,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAI1C;;;;OAIG;IACH,SAAS,CAAC,aAAa,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,MAAM,EAAE,EAAE;CAUrD;AAMD;;GAEG;AACH,MAAM,WAAW,sBAAsB;IACrC,qBAAqB;IACrB,MAAM,EAAE,MAAM,CAAC;IACf,qDAAqD;IACrD,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,gEAAgE;IAChE,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,iCAAiC;IACjC,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,iCAAiC;IACjC,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,0BAA0B;IAC1B,WAAW,CAAC,EAAE,OAAO,CAAC,WAAW,CAAC,CAAC;CACpC;AAED;;;GAGG;AACH,qBAAa,gBAAiB,SAAQ,iBAAiB;IACrD,OAAO,CAAC,MAAM,CAMZ;IACF,OAAO,CAAC,MAAM,CAAM;IAEpB;;;;OAIG;gBACS,MAAM,EAAE,sBAAsB;IA0B1C,eAAe,IAAI,MAAM;IAKzB,YAAY,IAAI,MAAM;IAkBhB,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,oBAAoB,CAAC;CAiDjE;AAMD;;GAEG;AACH,MAAM,WAAW,sBAAsB;IACrC,qBAAqB;IACrB,MAAM,EAAE,MAAM,CAAC;IACf,iDAAiD;IACjD,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,uFAAuF;IACvF,SAAS,CAAC,EAAE,iBAAiB,GAAG,cAAc,GAAG,gBAAgB,GAAG,YAAY,CAAC;IACjF,qDAAqD;IACrD,QAAQ,CAAC,EAAE,MAAM,GAAG,OAAO,GAAG,KAAK,CAAC;IACpC,0BAA0B;IAC1B,WAAW,CAAC,EAAE,OAAO,CAAC,WAAW,CAAC,CAAC;CACpC;AAED;;;GAGG;AACH,qBAAa,gBAAiB,SAAQ,iBAAiB;IACrD,OAAO,CAAC,MAAM,CAKZ;IACF,OAAO,CAAC,MAAM,CAAM;IAEpB;;;;OAIG;gBACS,MAAM,EAAE,sBAAsB;IAuB1C,eAAe,IAAI,MAAM;IAKzB,YAAY,IAAI,MAAM;IAShB,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,oBAAoB,CAAC;CAgDjE;AAMD;;GAEG;AACH,MAAM,WAAW,yBAAyB;IACxC,wBAAwB;IACxB,MAAM,EAAE,MAAM,CAAC;IACf,uCAAuC;IACvC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,gCAAgC;IAChC,SAAS,CAAC,EAAE,UAAU,GAAG,OAAO,CAAC;IACjC,0BAA0B;IAC1B,WAAW,CAAC,EAAE,OAAO,CAAC,WAAW,CAAC,CAAC;CACpC;AAED;;;GAGG;AACH,qBAAa,mBAAoB,SAAQ,iBAAiB;IACxD,OAAO,CAAC,MAAM,CAIZ;IACF,OAAO,CAAC,SAAS,CAAM;IAEvB;;;;OAIG;gBACS,MAAM,EAAE,yBAAyB;IAqB7C,eAAe,IAAI,MAAM;IAKzB,YAAY,IAAI,MAAM;IAKhB,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,oBAAoB,CAAC;CAwDjE;AAMD;;GAEG;AACH,MAAM,WAAW,2BAA2B;IAC1C,6EAA6E;IAC7E,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,wCAAwC;IACxC,MAAM,CAAC,EAAE,KAAK,GAAG,MAAM,CAAC;IACxB,0CAA0C;IAC1C,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,gCAAgC;IAChC,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,0BAA0B;IAC1B,WAAW,CAAC,EAAE,OAAO,CAAC,WAAW,CAAC,CAAC;CACpC;AAED;;;GAGG;AACH,qBAAa,qBAAsB,SAAQ,iBAAiB;IAC1D,OAAO,CAAC,MAAM,CAIZ;IACF,OAAO,CAAC,QAAQ,CAAM;IACtB,OAAO,CAAC,WAAW,CAAkB;IAErC;;;OAGG;gBACS,MAAM,GAAE,2BAAgC;IAUpD,eAAe,IAAI,MAAM;IAIzB,YAAY,IAAI,MAAM;IAMtB;;OAEG;YACW,UAAU;IAoBlB,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,oBAAoB,CAAC;CA2CjE;AAMD;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH,wBAAsB,cAAc,CAClC,EAAE,EAAE,QAAQ,EACZ,QAAQ,EAAE,iBAAiB,EAC3B,SAAS,EAAE,eAAe,EAAE,EAC5B,OAAO,GAAE;IACP,yDAAyD;IACzD,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,wBAAwB;IACxB,UAAU,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,KAAK,IAAI,CAAC;CAClD,GACL,OAAO,CAAC,MAAM,EAAE,CAAC,CAwDnB;AAED;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH,wBAAsB,cAAc,CAClC,EAAE,EAAE,QAAQ,EACZ,QAAQ,EAAE,iBAAiB,EAC3B,KAAK,EAAE,MAAM,EACb,OAAO,GAAE;IACP,kCAAkC;IAClC,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,yCAAyC;IACzC,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,sBAAsB;IACtB,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CAC7B,GACL,OAAO,CAAC,GAAG,EAAE,CAAC,CAahB;;;;;;;;;;AAMD,wBAaE"}
|
||||
621
vendor/ruvector/npm/packages/ruvector-extensions/src/embeddings.js
vendored
Normal file
621
vendor/ruvector/npm/packages/ruvector-extensions/src/embeddings.js
vendored
Normal file
@@ -0,0 +1,621 @@
|
||||
"use strict";
|
||||
/**
|
||||
* @fileoverview Comprehensive embeddings integration module for ruvector-extensions
|
||||
* Supports multiple providers: OpenAI, Cohere, Anthropic, and local HuggingFace models
|
||||
*
|
||||
* @module embeddings
|
||||
* @author ruv.io Team <info@ruv.io>
|
||||
* @license MIT
|
||||
*
|
||||
* @example
|
||||
* ```typescript
|
||||
* // OpenAI embeddings
|
||||
* const openai = new OpenAIEmbeddings({ apiKey: 'sk-...' });
|
||||
* const embeddings = await openai.embedTexts(['Hello world', 'Test']);
|
||||
*
|
||||
* // Auto-insert into VectorDB
|
||||
* await embedAndInsert(db, openai, [
|
||||
* { id: '1', text: 'Hello world', metadata: { source: 'test' } }
|
||||
* ]);
|
||||
* ```
|
||||
*/
|
||||
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
||||
if (k2 === undefined) k2 = k;
|
||||
var desc = Object.getOwnPropertyDescriptor(m, k);
|
||||
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
||||
desc = { enumerable: true, get: function() { return m[k]; } };
|
||||
}
|
||||
Object.defineProperty(o, k2, desc);
|
||||
}) : (function(o, m, k, k2) {
|
||||
if (k2 === undefined) k2 = k;
|
||||
o[k2] = m[k];
|
||||
}));
|
||||
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
||||
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
||||
}) : function(o, v) {
|
||||
o["default"] = v;
|
||||
});
|
||||
var __importStar = (this && this.__importStar) || (function () {
|
||||
var ownKeys = function(o) {
|
||||
ownKeys = Object.getOwnPropertyNames || function (o) {
|
||||
var ar = [];
|
||||
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
|
||||
return ar;
|
||||
};
|
||||
return ownKeys(o);
|
||||
};
|
||||
return function (mod) {
|
||||
if (mod && mod.__esModule) return mod;
|
||||
var result = {};
|
||||
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
|
||||
__setModuleDefault(result, mod);
|
||||
return result;
|
||||
};
|
||||
})();
|
||||
Object.defineProperty(exports, "__esModule", { value: true });
|
||||
exports.HuggingFaceEmbeddings = exports.AnthropicEmbeddings = exports.CohereEmbeddings = exports.OpenAIEmbeddings = exports.EmbeddingProvider = void 0;
|
||||
exports.embedAndInsert = embedAndInsert;
|
||||
exports.embedAndSearch = embedAndSearch;
|
||||
// ============================================================================
|
||||
// Abstract Base Class
|
||||
// ============================================================================
|
||||
/**
|
||||
* Abstract base class for embedding providers
|
||||
* All embedding providers must extend this class and implement its methods
|
||||
*/
|
||||
class EmbeddingProvider {
|
||||
/**
|
||||
* Creates a new embedding provider instance
|
||||
* @param retryConfig - Configuration for retry logic
|
||||
*/
|
||||
constructor(retryConfig) {
|
||||
this.retryConfig = {
|
||||
maxRetries: 3,
|
||||
initialDelay: 1000,
|
||||
maxDelay: 10000,
|
||||
backoffMultiplier: 2,
|
||||
...retryConfig,
|
||||
};
|
||||
}
|
||||
/**
|
||||
* Embed a single text string
|
||||
* @param text - Text to embed
|
||||
* @returns Promise resolving to the embedding vector
|
||||
*/
|
||||
async embedText(text) {
|
||||
const result = await this.embedTexts([text]);
|
||||
return result.embeddings[0].embedding;
|
||||
}
|
||||
/**
|
||||
* Execute a function with retry logic
|
||||
* @param fn - Function to execute
|
||||
* @param context - Context description for error messages
|
||||
* @returns Promise resolving to the function result
|
||||
*/
|
||||
async withRetry(fn, context) {
|
||||
let lastError;
|
||||
let delay = this.retryConfig.initialDelay;
|
||||
for (let attempt = 0; attempt <= this.retryConfig.maxRetries; attempt++) {
|
||||
try {
|
||||
return await fn();
|
||||
}
|
||||
catch (error) {
|
||||
lastError = error;
|
||||
// Check if error is retryable
|
||||
if (!this.isRetryableError(error)) {
|
||||
throw this.createEmbeddingError(error, context, false);
|
||||
}
|
||||
if (attempt < this.retryConfig.maxRetries) {
|
||||
await this.sleep(delay);
|
||||
delay = Math.min(delay * this.retryConfig.backoffMultiplier, this.retryConfig.maxDelay);
|
||||
}
|
||||
}
|
||||
}
|
||||
throw this.createEmbeddingError(lastError, `${context} (after ${this.retryConfig.maxRetries} retries)`, false);
|
||||
}
|
||||
/**
|
||||
* Determine if an error is retryable
|
||||
* @param error - Error to check
|
||||
* @returns True if the error should trigger a retry
|
||||
*/
|
||||
isRetryableError(error) {
|
||||
if (error instanceof Error) {
|
||||
const message = error.message.toLowerCase();
|
||||
// Rate limits, timeouts, and temporary server errors are retryable
|
||||
return (message.includes('rate limit') ||
|
||||
message.includes('timeout') ||
|
||||
message.includes('503') ||
|
||||
message.includes('429') ||
|
||||
message.includes('connection'));
|
||||
}
|
||||
return false;
|
||||
}
|
||||
/**
|
||||
* Create a standardized embedding error
|
||||
* @param error - Original error
|
||||
* @param context - Context description
|
||||
* @param retryable - Whether the error is retryable
|
||||
* @returns Formatted error object
|
||||
*/
|
||||
createEmbeddingError(error, context, retryable) {
|
||||
const message = error instanceof Error ? error.message : String(error);
|
||||
return {
|
||||
message: `${context}: ${message}`,
|
||||
error,
|
||||
retryable,
|
||||
};
|
||||
}
|
||||
/**
|
||||
* Sleep for a specified duration
|
||||
* @param ms - Milliseconds to sleep
|
||||
*/
|
||||
sleep(ms) {
|
||||
return new Promise(resolve => setTimeout(resolve, ms));
|
||||
}
|
||||
/**
|
||||
* Split texts into batches based on max batch size
|
||||
* @param texts - Texts to batch
|
||||
* @returns Array of text batches
|
||||
*/
|
||||
createBatches(texts) {
|
||||
const batches = [];
|
||||
const batchSize = this.getMaxBatchSize();
|
||||
for (let i = 0; i < texts.length; i += batchSize) {
|
||||
batches.push(texts.slice(i, i + batchSize));
|
||||
}
|
||||
return batches;
|
||||
}
|
||||
}
|
||||
exports.EmbeddingProvider = EmbeddingProvider;
|
||||
/**
|
||||
* OpenAI embeddings provider
|
||||
* Supports text-embedding-3-small, text-embedding-3-large, and text-embedding-ada-002
|
||||
*/
|
||||
class OpenAIEmbeddings extends EmbeddingProvider {
|
||||
/**
|
||||
* Creates a new OpenAI embeddings provider
|
||||
* @param config - Configuration options
|
||||
* @throws Error if OpenAI SDK is not installed
|
||||
*/
|
||||
constructor(config) {
|
||||
super(config.retryConfig);
|
||||
this.config = {
|
||||
apiKey: config.apiKey,
|
||||
model: config.model || 'text-embedding-3-small',
|
||||
organization: config.organization,
|
||||
baseURL: config.baseURL,
|
||||
dimensions: config.dimensions,
|
||||
};
|
||||
try {
|
||||
// Dynamic import to support optional peer dependency
|
||||
const OpenAI = require('openai');
|
||||
this.openai = new OpenAI({
|
||||
apiKey: this.config.apiKey,
|
||||
organization: this.config.organization,
|
||||
baseURL: this.config.baseURL,
|
||||
});
|
||||
}
|
||||
catch (error) {
|
||||
throw new Error('OpenAI SDK not found. Install it with: npm install openai');
|
||||
}
|
||||
}
|
||||
getMaxBatchSize() {
|
||||
// OpenAI supports up to 2048 inputs per request
|
||||
return 2048;
|
||||
}
|
||||
getDimension() {
|
||||
// Return configured dimensions or default based on model
|
||||
if (this.config.dimensions) {
|
||||
return this.config.dimensions;
|
||||
}
|
||||
switch (this.config.model) {
|
||||
case 'text-embedding-3-small':
|
||||
return 1536;
|
||||
case 'text-embedding-3-large':
|
||||
return 3072;
|
||||
case 'text-embedding-ada-002':
|
||||
return 1536;
|
||||
default:
|
||||
return 1536;
|
||||
}
|
||||
}
|
||||
async embedTexts(texts) {
|
||||
if (texts.length === 0) {
|
||||
return { embeddings: [] };
|
||||
}
|
||||
const batches = this.createBatches(texts);
|
||||
const allResults = [];
|
||||
let totalTokens = 0;
|
||||
for (let batchIndex = 0; batchIndex < batches.length; batchIndex++) {
|
||||
const batch = batches[batchIndex];
|
||||
const baseIndex = batchIndex * this.getMaxBatchSize();
|
||||
const response = await this.withRetry(async () => {
|
||||
const params = {
|
||||
model: this.config.model,
|
||||
input: batch,
|
||||
};
|
||||
if (this.config.dimensions) {
|
||||
params.dimensions = this.config.dimensions;
|
||||
}
|
||||
return await this.openai.embeddings.create(params);
|
||||
}, `OpenAI embeddings for batch ${batchIndex + 1}/${batches.length}`);
|
||||
totalTokens += response.usage?.total_tokens || 0;
|
||||
for (const item of response.data) {
|
||||
allResults.push({
|
||||
embedding: item.embedding,
|
||||
index: baseIndex + item.index,
|
||||
tokens: response.usage?.total_tokens,
|
||||
});
|
||||
}
|
||||
}
|
||||
return {
|
||||
embeddings: allResults,
|
||||
totalTokens,
|
||||
metadata: {
|
||||
model: this.config.model,
|
||||
provider: 'openai',
|
||||
},
|
||||
};
|
||||
}
|
||||
}
|
||||
exports.OpenAIEmbeddings = OpenAIEmbeddings;
|
||||
/**
|
||||
* Cohere embeddings provider
|
||||
* Supports embed-english-v3.0, embed-multilingual-v3.0, and other Cohere models
|
||||
*/
|
||||
class CohereEmbeddings extends EmbeddingProvider {
|
||||
/**
|
||||
* Creates a new Cohere embeddings provider
|
||||
* @param config - Configuration options
|
||||
* @throws Error if Cohere SDK is not installed
|
||||
*/
|
||||
constructor(config) {
|
||||
super(config.retryConfig);
|
||||
this.config = {
|
||||
apiKey: config.apiKey,
|
||||
model: config.model || 'embed-english-v3.0',
|
||||
inputType: config.inputType,
|
||||
truncate: config.truncate,
|
||||
};
|
||||
try {
|
||||
// Dynamic import to support optional peer dependency
|
||||
const { CohereClient } = require('cohere-ai');
|
||||
this.cohere = new CohereClient({
|
||||
token: this.config.apiKey,
|
||||
});
|
||||
}
|
||||
catch (error) {
|
||||
throw new Error('Cohere SDK not found. Install it with: npm install cohere-ai');
|
||||
}
|
||||
}
|
||||
getMaxBatchSize() {
|
||||
// Cohere supports up to 96 texts per request
|
||||
return 96;
|
||||
}
|
||||
getDimension() {
|
||||
// Cohere v3 models produce 1024-dimensional embeddings
|
||||
if (this.config.model.includes('v3')) {
|
||||
return 1024;
|
||||
}
|
||||
// Earlier models use different dimensions
|
||||
return 4096;
|
||||
}
|
||||
async embedTexts(texts) {
|
||||
if (texts.length === 0) {
|
||||
return { embeddings: [] };
|
||||
}
|
||||
const batches = this.createBatches(texts);
|
||||
const allResults = [];
|
||||
for (let batchIndex = 0; batchIndex < batches.length; batchIndex++) {
|
||||
const batch = batches[batchIndex];
|
||||
const baseIndex = batchIndex * this.getMaxBatchSize();
|
||||
const response = await this.withRetry(async () => {
|
||||
const params = {
|
||||
model: this.config.model,
|
||||
texts: batch,
|
||||
};
|
||||
if (this.config.inputType) {
|
||||
params.inputType = this.config.inputType;
|
||||
}
|
||||
if (this.config.truncate) {
|
||||
params.truncate = this.config.truncate;
|
||||
}
|
||||
return await this.cohere.embed(params);
|
||||
}, `Cohere embeddings for batch ${batchIndex + 1}/${batches.length}`);
|
||||
for (let i = 0; i < response.embeddings.length; i++) {
|
||||
allResults.push({
|
||||
embedding: response.embeddings[i],
|
||||
index: baseIndex + i,
|
||||
});
|
||||
}
|
||||
}
|
||||
return {
|
||||
embeddings: allResults,
|
||||
metadata: {
|
||||
model: this.config.model,
|
||||
provider: 'cohere',
|
||||
},
|
||||
};
|
||||
}
|
||||
}
|
||||
exports.CohereEmbeddings = CohereEmbeddings;
|
||||
/**
|
||||
* Anthropic embeddings provider using Voyage AI
|
||||
* Anthropic partners with Voyage AI for embeddings
|
||||
*/
|
||||
class AnthropicEmbeddings extends EmbeddingProvider {
|
||||
/**
|
||||
* Creates a new Anthropic embeddings provider
|
||||
* @param config - Configuration options
|
||||
* @throws Error if Anthropic SDK is not installed
|
||||
*/
|
||||
constructor(config) {
|
||||
super(config.retryConfig);
|
||||
this.config = {
|
||||
apiKey: config.apiKey,
|
||||
model: config.model || 'voyage-2',
|
||||
inputType: config.inputType,
|
||||
};
|
||||
try {
|
||||
const Anthropic = require('@anthropic-ai/sdk');
|
||||
this.anthropic = new Anthropic({
|
||||
apiKey: this.config.apiKey,
|
||||
});
|
||||
}
|
||||
catch (error) {
|
||||
throw new Error('Anthropic SDK not found. Install it with: npm install @anthropic-ai/sdk');
|
||||
}
|
||||
}
|
||||
getMaxBatchSize() {
|
||||
// Process in smaller batches for Voyage API
|
||||
return 128;
|
||||
}
|
||||
getDimension() {
|
||||
// Voyage-2 produces 1024-dimensional embeddings
|
||||
return 1024;
|
||||
}
|
||||
async embedTexts(texts) {
|
||||
if (texts.length === 0) {
|
||||
return { embeddings: [] };
|
||||
}
|
||||
const batches = this.createBatches(texts);
|
||||
const allResults = [];
|
||||
for (let batchIndex = 0; batchIndex < batches.length; batchIndex++) {
|
||||
const batch = batches[batchIndex];
|
||||
const baseIndex = batchIndex * this.getMaxBatchSize();
|
||||
// Note: As of early 2025, Anthropic uses Voyage AI for embeddings
|
||||
// This is a placeholder for when official API is available
|
||||
const response = await this.withRetry(async () => {
|
||||
// Use Voyage AI API through Anthropic's recommended integration
|
||||
const httpResponse = await fetch('https://api.voyageai.com/v1/embeddings', {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
'Authorization': `Bearer ${this.config.apiKey}`,
|
||||
},
|
||||
body: JSON.stringify({
|
||||
input: batch,
|
||||
model: this.config.model,
|
||||
input_type: this.config.inputType || 'document',
|
||||
}),
|
||||
});
|
||||
if (!httpResponse.ok) {
|
||||
const error = await httpResponse.text();
|
||||
throw new Error(`Voyage API error: ${error}`);
|
||||
}
|
||||
return await httpResponse.json();
|
||||
}, `Anthropic/Voyage embeddings for batch ${batchIndex + 1}/${batches.length}`);
|
||||
for (let i = 0; i < response.data.length; i++) {
|
||||
allResults.push({
|
||||
embedding: response.data[i].embedding,
|
||||
index: baseIndex + i,
|
||||
});
|
||||
}
|
||||
}
|
||||
return {
|
||||
embeddings: allResults,
|
||||
metadata: {
|
||||
model: this.config.model,
|
||||
provider: 'anthropic-voyage',
|
||||
},
|
||||
};
|
||||
}
|
||||
}
|
||||
exports.AnthropicEmbeddings = AnthropicEmbeddings;
|
||||
/**
|
||||
* HuggingFace local embeddings provider
|
||||
* Runs embedding models locally using transformers.js
|
||||
*/
|
||||
class HuggingFaceEmbeddings extends EmbeddingProvider {
|
||||
/**
|
||||
* Creates a new HuggingFace local embeddings provider
|
||||
* @param config - Configuration options
|
||||
*/
|
||||
constructor(config = {}) {
|
||||
super(config.retryConfig);
|
||||
this.initialized = false;
|
||||
this.config = {
|
||||
model: config.model || 'Xenova/all-MiniLM-L6-v2',
|
||||
normalize: config.normalize !== false,
|
||||
batchSize: config.batchSize || 32,
|
||||
};
|
||||
}
|
||||
getMaxBatchSize() {
|
||||
return this.config.batchSize;
|
||||
}
|
||||
getDimension() {
|
||||
// all-MiniLM-L6-v2 produces 384-dimensional embeddings
|
||||
// This should be determined dynamically based on model
|
||||
return 384;
|
||||
}
|
||||
/**
|
||||
* Initialize the embedding pipeline
|
||||
*/
|
||||
async initialize() {
|
||||
if (this.initialized)
|
||||
return;
|
||||
try {
|
||||
// Dynamic import of transformers.js
|
||||
const { pipeline } = await Promise.resolve().then(() => __importStar(require('@xenova/transformers')));
|
||||
this.pipeline = await pipeline('feature-extraction', this.config.model);
|
||||
this.initialized = true;
|
||||
}
|
||||
catch (error) {
|
||||
throw new Error('Transformers.js not found or failed to load. Install it with: npm install @xenova/transformers');
|
||||
}
|
||||
}
|
||||
async embedTexts(texts) {
|
||||
if (texts.length === 0) {
|
||||
return { embeddings: [] };
|
||||
}
|
||||
await this.initialize();
|
||||
const batches = this.createBatches(texts);
|
||||
const allResults = [];
|
||||
for (let batchIndex = 0; batchIndex < batches.length; batchIndex++) {
|
||||
const batch = batches[batchIndex];
|
||||
const baseIndex = batchIndex * this.getMaxBatchSize();
|
||||
const embeddings = await this.withRetry(async () => {
|
||||
const output = await this.pipeline(batch, {
|
||||
pooling: 'mean',
|
||||
normalize: this.config.normalize,
|
||||
});
|
||||
// Convert tensor to array
|
||||
return output.tolist();
|
||||
}, `HuggingFace embeddings for batch ${batchIndex + 1}/${batches.length}`);
|
||||
for (let i = 0; i < embeddings.length; i++) {
|
||||
allResults.push({
|
||||
embedding: embeddings[i],
|
||||
index: baseIndex + i,
|
||||
});
|
||||
}
|
||||
}
|
||||
return {
|
||||
embeddings: allResults,
|
||||
metadata: {
|
||||
model: this.config.model,
|
||||
provider: 'huggingface-local',
|
||||
},
|
||||
};
|
||||
}
|
||||
}
|
||||
exports.HuggingFaceEmbeddings = HuggingFaceEmbeddings;
|
||||
// ============================================================================
|
||||
// Helper Functions
|
||||
// ============================================================================
|
||||
/**
|
||||
* Embed texts and automatically insert them into a VectorDB
|
||||
*
|
||||
* @param db - VectorDB instance to insert into
|
||||
* @param provider - Embedding provider to use
|
||||
* @param documents - Documents to embed and insert
|
||||
* @param options - Additional options
|
||||
* @returns Promise resolving to array of inserted vector IDs
|
||||
*
|
||||
* @example
|
||||
* ```typescript
|
||||
* const openai = new OpenAIEmbeddings({ apiKey: 'sk-...' });
|
||||
* const db = new VectorDB({ dimension: 1536 });
|
||||
*
|
||||
* const ids = await embedAndInsert(db, openai, [
|
||||
* { id: '1', text: 'Hello world', metadata: { source: 'test' } },
|
||||
* { id: '2', text: 'Another document', metadata: { source: 'test' } }
|
||||
* ]);
|
||||
*
|
||||
* console.log('Inserted vector IDs:', ids);
|
||||
* ```
|
||||
*/
|
||||
async function embedAndInsert(db, provider, documents, options = {}) {
|
||||
if (documents.length === 0) {
|
||||
return [];
|
||||
}
|
||||
// Verify dimension compatibility
|
||||
const dbDimension = db.dimension || db.getDimension?.();
|
||||
const providerDimension = provider.getDimension();
|
||||
if (dbDimension && dbDimension !== providerDimension) {
|
||||
throw new Error(`Dimension mismatch: VectorDB expects ${dbDimension} but provider produces ${providerDimension}`);
|
||||
}
|
||||
// Extract texts
|
||||
const texts = documents.map(doc => doc.text);
|
||||
// Generate embeddings
|
||||
const result = await provider.embedTexts(texts);
|
||||
// Insert vectors
|
||||
const insertedIds = [];
|
||||
for (let i = 0; i < documents.length; i++) {
|
||||
const doc = documents[i];
|
||||
const embedding = result.embeddings.find(e => e.index === i);
|
||||
if (!embedding) {
|
||||
throw new Error(`Missing embedding for document at index ${i}`);
|
||||
}
|
||||
// Insert or update vector
|
||||
if (options.overwrite) {
|
||||
await db.upsert({
|
||||
id: doc.id,
|
||||
values: embedding.embedding,
|
||||
metadata: doc.metadata,
|
||||
});
|
||||
}
|
||||
else {
|
||||
await db.insert({
|
||||
id: doc.id,
|
||||
values: embedding.embedding,
|
||||
metadata: doc.metadata,
|
||||
});
|
||||
}
|
||||
insertedIds.push(doc.id);
|
||||
// Call progress callback
|
||||
if (options.onProgress) {
|
||||
options.onProgress(i + 1, documents.length);
|
||||
}
|
||||
}
|
||||
return insertedIds;
|
||||
}
|
||||
/**
|
||||
* Embed a query and search for similar documents in VectorDB
|
||||
*
|
||||
* @param db - VectorDB instance to search
|
||||
* @param provider - Embedding provider to use
|
||||
* @param query - Query text to search for
|
||||
* @param options - Search options
|
||||
* @returns Promise resolving to search results
|
||||
*
|
||||
* @example
|
||||
* ```typescript
|
||||
* const openai = new OpenAIEmbeddings({ apiKey: 'sk-...' });
|
||||
* const db = new VectorDB({ dimension: 1536 });
|
||||
*
|
||||
* const results = await embedAndSearch(db, openai, 'machine learning', {
|
||||
* topK: 5,
|
||||
* threshold: 0.7
|
||||
* });
|
||||
*
|
||||
* console.log('Found documents:', results);
|
||||
* ```
|
||||
*/
|
||||
async function embedAndSearch(db, provider, query, options = {}) {
|
||||
// Generate query embedding
|
||||
const queryEmbedding = await provider.embedText(query);
|
||||
// Search VectorDB
|
||||
const results = await db.search({
|
||||
vector: queryEmbedding,
|
||||
topK: options.topK || 10,
|
||||
threshold: options.threshold,
|
||||
filter: options.filter,
|
||||
});
|
||||
return results;
|
||||
}
|
||||
// ============================================================================
|
||||
// Exports
|
||||
// ============================================================================
|
||||
exports.default = {
|
||||
// Base class
|
||||
EmbeddingProvider,
|
||||
// Providers
|
||||
OpenAIEmbeddings,
|
||||
CohereEmbeddings,
|
||||
AnthropicEmbeddings,
|
||||
HuggingFaceEmbeddings,
|
||||
// Helper functions
|
||||
embedAndInsert,
|
||||
embedAndSearch,
|
||||
};
|
||||
//# sourceMappingURL=embeddings.js.map
|
||||
1
vendor/ruvector/npm/packages/ruvector-extensions/src/embeddings.js.map
vendored
Normal file
1
vendor/ruvector/npm/packages/ruvector-extensions/src/embeddings.js.map
vendored
Normal file
File diff suppressed because one or more lines are too long
926
vendor/ruvector/npm/packages/ruvector-extensions/src/embeddings.ts
vendored
Normal file
926
vendor/ruvector/npm/packages/ruvector-extensions/src/embeddings.ts
vendored
Normal file
@@ -0,0 +1,926 @@
|
||||
/**
|
||||
* @fileoverview Comprehensive embeddings integration module for ruvector-extensions
|
||||
* Supports multiple providers: OpenAI, Cohere, Anthropic, and local HuggingFace models
|
||||
*
|
||||
* @module embeddings
|
||||
* @author ruv.io Team <info@ruv.io>
|
||||
* @license MIT
|
||||
*
|
||||
* @example
|
||||
* ```typescript
|
||||
* // OpenAI embeddings
|
||||
* const openai = new OpenAIEmbeddings({ apiKey: 'sk-...' });
|
||||
* const embeddings = await openai.embedTexts(['Hello world', 'Test']);
|
||||
*
|
||||
* // Auto-insert into VectorDB
|
||||
* await embedAndInsert(db, openai, [
|
||||
* { id: '1', text: 'Hello world', metadata: { source: 'test' } }
|
||||
* ]);
|
||||
* ```
|
||||
*/
|
||||
|
||||
// VectorDB type will be used as any for maximum compatibility
|
||||
type VectorDB = any;
|
||||
|
||||
// ============================================================================
|
||||
// Core Types and Interfaces
|
||||
// ============================================================================
|
||||
|
||||
/**
|
||||
* Configuration for retry logic
|
||||
*/
|
||||
export interface RetryConfig {
|
||||
/** Maximum number of retry attempts */
|
||||
maxRetries: number;
|
||||
/** Initial delay in milliseconds before first retry */
|
||||
initialDelay: number;
|
||||
/** Maximum delay in milliseconds between retries */
|
||||
maxDelay: number;
|
||||
/** Multiplier for exponential backoff */
|
||||
backoffMultiplier: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Result of an embedding operation
|
||||
*/
|
||||
export interface EmbeddingResult {
|
||||
/** The generated embedding vector */
|
||||
embedding: number[];
|
||||
/** Index of the text in the original batch */
|
||||
index: number;
|
||||
/** Optional token count used */
|
||||
tokens?: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Batch result with embeddings and metadata
|
||||
*/
|
||||
export interface BatchEmbeddingResult {
|
||||
/** Array of embedding results */
|
||||
embeddings: EmbeddingResult[];
|
||||
/** Total tokens used (if available) */
|
||||
totalTokens?: number;
|
||||
/** Provider-specific metadata */
|
||||
metadata?: Record<string, unknown>;
|
||||
}
|
||||
|
||||
/**
|
||||
* Error details for failed embedding operations
|
||||
*/
|
||||
export interface EmbeddingError {
|
||||
/** Error message */
|
||||
message: string;
|
||||
/** Original error object */
|
||||
error: unknown;
|
||||
/** Index of the text that failed (if applicable) */
|
||||
index?: number;
|
||||
/** Whether the error is retryable */
|
||||
retryable: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* Document to embed and insert into VectorDB
|
||||
*/
|
||||
export interface DocumentToEmbed {
|
||||
/** Unique identifier for the document */
|
||||
id: string;
|
||||
/** Text content to embed */
|
||||
text: string;
|
||||
/** Optional metadata to store with the vector */
|
||||
metadata?: Record<string, unknown>;
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Abstract Base Class
|
||||
// ============================================================================
|
||||
|
||||
/**
|
||||
* Abstract base class for embedding providers
|
||||
* All embedding providers must extend this class and implement its methods
|
||||
*/
|
||||
export abstract class EmbeddingProvider {
|
||||
protected retryConfig: RetryConfig;
|
||||
|
||||
/**
|
||||
* Creates a new embedding provider instance
|
||||
* @param retryConfig - Configuration for retry logic
|
||||
*/
|
||||
constructor(retryConfig?: Partial<RetryConfig>) {
|
||||
this.retryConfig = {
|
||||
maxRetries: 3,
|
||||
initialDelay: 1000,
|
||||
maxDelay: 10000,
|
||||
backoffMultiplier: 2,
|
||||
...retryConfig,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the maximum batch size supported by this provider
|
||||
*/
|
||||
abstract getMaxBatchSize(): number;
|
||||
|
||||
/**
|
||||
* Get the dimension of embeddings produced by this provider
|
||||
*/
|
||||
abstract getDimension(): number;
|
||||
|
||||
/**
|
||||
* Embed a single text string
|
||||
* @param text - Text to embed
|
||||
* @returns Promise resolving to the embedding vector
|
||||
*/
|
||||
async embedText(text: string): Promise<number[]> {
|
||||
const result = await this.embedTexts([text]);
|
||||
return result.embeddings[0].embedding;
|
||||
}
|
||||
|
||||
/**
|
||||
* Embed multiple texts with automatic batching
|
||||
* @param texts - Array of texts to embed
|
||||
* @returns Promise resolving to batch embedding results
|
||||
*/
|
||||
abstract embedTexts(texts: string[]): Promise<BatchEmbeddingResult>;
|
||||
|
||||
/**
|
||||
* Execute a function with retry logic
|
||||
* @param fn - Function to execute
|
||||
* @param context - Context description for error messages
|
||||
* @returns Promise resolving to the function result
|
||||
*/
|
||||
protected async withRetry<T>(
|
||||
fn: () => Promise<T>,
|
||||
context: string
|
||||
): Promise<T> {
|
||||
let lastError: unknown;
|
||||
let delay = this.retryConfig.initialDelay;
|
||||
|
||||
for (let attempt = 0; attempt <= this.retryConfig.maxRetries; attempt++) {
|
||||
try {
|
||||
return await fn();
|
||||
} catch (error) {
|
||||
lastError = error;
|
||||
|
||||
// Check if error is retryable
|
||||
if (!this.isRetryableError(error)) {
|
||||
throw this.createEmbeddingError(error, context, false);
|
||||
}
|
||||
|
||||
if (attempt < this.retryConfig.maxRetries) {
|
||||
await this.sleep(delay);
|
||||
delay = Math.min(
|
||||
delay * this.retryConfig.backoffMultiplier,
|
||||
this.retryConfig.maxDelay
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
throw this.createEmbeddingError(
|
||||
lastError,
|
||||
`${context} (after ${this.retryConfig.maxRetries} retries)`,
|
||||
false
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Determine if an error is retryable
|
||||
* @param error - Error to check
|
||||
* @returns True if the error should trigger a retry
|
||||
*/
|
||||
protected isRetryableError(error: unknown): boolean {
|
||||
if (error instanceof Error) {
|
||||
const message = error.message.toLowerCase();
|
||||
// Rate limits, timeouts, and temporary server errors are retryable
|
||||
return (
|
||||
message.includes('rate limit') ||
|
||||
message.includes('timeout') ||
|
||||
message.includes('503') ||
|
||||
message.includes('429') ||
|
||||
message.includes('connection')
|
||||
);
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a standardized embedding error
|
||||
* @param error - Original error
|
||||
* @param context - Context description
|
||||
* @param retryable - Whether the error is retryable
|
||||
* @returns Formatted error object
|
||||
*/
|
||||
protected createEmbeddingError(
|
||||
error: unknown,
|
||||
context: string,
|
||||
retryable: boolean
|
||||
): EmbeddingError {
|
||||
const message = error instanceof Error ? error.message : String(error);
|
||||
return {
|
||||
message: `${context}: ${message}`,
|
||||
error,
|
||||
retryable,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Sleep for a specified duration
|
||||
* @param ms - Milliseconds to sleep
|
||||
*/
|
||||
protected sleep(ms: number): Promise<void> {
|
||||
return new Promise(resolve => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
/**
|
||||
* Split texts into batches based on max batch size
|
||||
* @param texts - Texts to batch
|
||||
* @returns Array of text batches
|
||||
*/
|
||||
protected createBatches(texts: string[]): string[][] {
|
||||
const batches: string[][] = [];
|
||||
const batchSize = this.getMaxBatchSize();
|
||||
|
||||
for (let i = 0; i < texts.length; i += batchSize) {
|
||||
batches.push(texts.slice(i, i + batchSize));
|
||||
}
|
||||
|
||||
return batches;
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// OpenAI Embeddings Provider
|
||||
// ============================================================================
|
||||
|
||||
/**
|
||||
* Configuration for OpenAI embeddings
|
||||
*/
|
||||
export interface OpenAIEmbeddingsConfig {
|
||||
/** OpenAI API key */
|
||||
apiKey: string;
|
||||
/** Model name (default: 'text-embedding-3-small') */
|
||||
model?: string;
|
||||
/** Embedding dimensions (only for text-embedding-3-* models) */
|
||||
dimensions?: number;
|
||||
/** Organization ID (optional) */
|
||||
organization?: string;
|
||||
/** Custom base URL (optional) */
|
||||
baseURL?: string;
|
||||
/** Retry configuration */
|
||||
retryConfig?: Partial<RetryConfig>;
|
||||
}
|
||||
|
||||
/**
|
||||
* OpenAI embeddings provider
|
||||
* Supports text-embedding-3-small, text-embedding-3-large, and text-embedding-ada-002
|
||||
*/
|
||||
export class OpenAIEmbeddings extends EmbeddingProvider {
|
||||
private config: {
|
||||
apiKey: string;
|
||||
model: string;
|
||||
organization?: string;
|
||||
baseURL?: string;
|
||||
dimensions?: number;
|
||||
};
|
||||
private openai: any;
|
||||
|
||||
/**
|
||||
* Creates a new OpenAI embeddings provider
|
||||
* @param config - Configuration options
|
||||
* @throws Error if OpenAI SDK is not installed
|
||||
*/
|
||||
constructor(config: OpenAIEmbeddingsConfig) {
|
||||
super(config.retryConfig);
|
||||
|
||||
this.config = {
|
||||
apiKey: config.apiKey,
|
||||
model: config.model || 'text-embedding-3-small',
|
||||
organization: config.organization,
|
||||
baseURL: config.baseURL,
|
||||
dimensions: config.dimensions,
|
||||
};
|
||||
|
||||
try {
|
||||
// Dynamic import to support optional peer dependency
|
||||
const OpenAI = require('openai');
|
||||
this.openai = new OpenAI({
|
||||
apiKey: this.config.apiKey,
|
||||
organization: this.config.organization,
|
||||
baseURL: this.config.baseURL,
|
||||
});
|
||||
} catch (error) {
|
||||
throw new Error(
|
||||
'OpenAI SDK not found. Install it with: npm install openai'
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
getMaxBatchSize(): number {
|
||||
// OpenAI supports up to 2048 inputs per request
|
||||
return 2048;
|
||||
}
|
||||
|
||||
getDimension(): number {
|
||||
// Return configured dimensions or default based on model
|
||||
if (this.config.dimensions) {
|
||||
return this.config.dimensions;
|
||||
}
|
||||
|
||||
switch (this.config.model) {
|
||||
case 'text-embedding-3-small':
|
||||
return 1536;
|
||||
case 'text-embedding-3-large':
|
||||
return 3072;
|
||||
case 'text-embedding-ada-002':
|
||||
return 1536;
|
||||
default:
|
||||
return 1536;
|
||||
}
|
||||
}
|
||||
|
||||
async embedTexts(texts: string[]): Promise<BatchEmbeddingResult> {
|
||||
if (texts.length === 0) {
|
||||
return { embeddings: [] };
|
||||
}
|
||||
|
||||
const batches = this.createBatches(texts);
|
||||
const allResults: EmbeddingResult[] = [];
|
||||
let totalTokens = 0;
|
||||
|
||||
for (let batchIndex = 0; batchIndex < batches.length; batchIndex++) {
|
||||
const batch = batches[batchIndex];
|
||||
const baseIndex = batchIndex * this.getMaxBatchSize();
|
||||
|
||||
const response = await this.withRetry(
|
||||
async () => {
|
||||
const params: any = {
|
||||
model: this.config.model,
|
||||
input: batch,
|
||||
};
|
||||
|
||||
if (this.config.dimensions) {
|
||||
params.dimensions = this.config.dimensions;
|
||||
}
|
||||
|
||||
return await this.openai.embeddings.create(params);
|
||||
},
|
||||
`OpenAI embeddings for batch ${batchIndex + 1}/${batches.length}`
|
||||
);
|
||||
|
||||
totalTokens += response.usage?.total_tokens || 0;
|
||||
|
||||
for (const item of response.data) {
|
||||
allResults.push({
|
||||
embedding: item.embedding,
|
||||
index: baseIndex + item.index,
|
||||
tokens: response.usage?.total_tokens,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
embeddings: allResults,
|
||||
totalTokens,
|
||||
metadata: {
|
||||
model: this.config.model,
|
||||
provider: 'openai',
|
||||
},
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Cohere Embeddings Provider
|
||||
// ============================================================================
|
||||
|
||||
/**
|
||||
* Configuration for Cohere embeddings
|
||||
*/
|
||||
export interface CohereEmbeddingsConfig {
|
||||
/** Cohere API key */
|
||||
apiKey: string;
|
||||
/** Model name (default: 'embed-english-v3.0') */
|
||||
model?: string;
|
||||
/** Input type: 'search_document', 'search_query', 'classification', or 'clustering' */
|
||||
inputType?: 'search_document' | 'search_query' | 'classification' | 'clustering';
|
||||
/** Truncate input text if it exceeds model limits */
|
||||
truncate?: 'NONE' | 'START' | 'END';
|
||||
/** Retry configuration */
|
||||
retryConfig?: Partial<RetryConfig>;
|
||||
}
|
||||
|
||||
/**
|
||||
* Cohere embeddings provider
|
||||
* Supports embed-english-v3.0, embed-multilingual-v3.0, and other Cohere models
|
||||
*/
|
||||
export class CohereEmbeddings extends EmbeddingProvider {
|
||||
private config: {
|
||||
apiKey: string;
|
||||
model: string;
|
||||
inputType?: 'search_document' | 'search_query' | 'classification' | 'clustering';
|
||||
truncate?: 'NONE' | 'START' | 'END';
|
||||
};
|
||||
private cohere: any;
|
||||
|
||||
/**
|
||||
* Creates a new Cohere embeddings provider
|
||||
* @param config - Configuration options
|
||||
* @throws Error if Cohere SDK is not installed
|
||||
*/
|
||||
constructor(config: CohereEmbeddingsConfig) {
|
||||
super(config.retryConfig);
|
||||
|
||||
this.config = {
|
||||
apiKey: config.apiKey,
|
||||
model: config.model || 'embed-english-v3.0',
|
||||
inputType: config.inputType,
|
||||
truncate: config.truncate,
|
||||
};
|
||||
|
||||
try {
|
||||
// Dynamic import to support optional peer dependency
|
||||
const { CohereClient } = require('cohere-ai');
|
||||
this.cohere = new CohereClient({
|
||||
token: this.config.apiKey,
|
||||
});
|
||||
} catch (error) {
|
||||
throw new Error(
|
||||
'Cohere SDK not found. Install it with: npm install cohere-ai'
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
getMaxBatchSize(): number {
|
||||
// Cohere supports up to 96 texts per request
|
||||
return 96;
|
||||
}
|
||||
|
||||
getDimension(): number {
|
||||
// Cohere v3 models produce 1024-dimensional embeddings
|
||||
if (this.config.model.includes('v3')) {
|
||||
return 1024;
|
||||
}
|
||||
// Earlier models use different dimensions
|
||||
return 4096;
|
||||
}
|
||||
|
||||
async embedTexts(texts: string[]): Promise<BatchEmbeddingResult> {
|
||||
if (texts.length === 0) {
|
||||
return { embeddings: [] };
|
||||
}
|
||||
|
||||
const batches = this.createBatches(texts);
|
||||
const allResults: EmbeddingResult[] = [];
|
||||
|
||||
for (let batchIndex = 0; batchIndex < batches.length; batchIndex++) {
|
||||
const batch = batches[batchIndex];
|
||||
const baseIndex = batchIndex * this.getMaxBatchSize();
|
||||
|
||||
const response = await this.withRetry(
|
||||
async () => {
|
||||
const params: any = {
|
||||
model: this.config.model,
|
||||
texts: batch,
|
||||
};
|
||||
|
||||
if (this.config.inputType) {
|
||||
params.inputType = this.config.inputType;
|
||||
}
|
||||
|
||||
if (this.config.truncate) {
|
||||
params.truncate = this.config.truncate;
|
||||
}
|
||||
|
||||
return await this.cohere.embed(params);
|
||||
},
|
||||
`Cohere embeddings for batch ${batchIndex + 1}/${batches.length}`
|
||||
);
|
||||
|
||||
for (let i = 0; i < response.embeddings.length; i++) {
|
||||
allResults.push({
|
||||
embedding: response.embeddings[i],
|
||||
index: baseIndex + i,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
embeddings: allResults,
|
||||
metadata: {
|
||||
model: this.config.model,
|
||||
provider: 'cohere',
|
||||
},
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Anthropic Embeddings Provider
|
||||
// ============================================================================
|
||||
|
||||
/**
|
||||
* Configuration for Anthropic embeddings via Voyage AI
|
||||
*/
|
||||
export interface AnthropicEmbeddingsConfig {
|
||||
/** Anthropic API key */
|
||||
apiKey: string;
|
||||
/** Model name (default: 'voyage-2') */
|
||||
model?: string;
|
||||
/** Input type for embeddings */
|
||||
inputType?: 'document' | 'query';
|
||||
/** Retry configuration */
|
||||
retryConfig?: Partial<RetryConfig>;
|
||||
}
|
||||
|
||||
/**
|
||||
* Anthropic embeddings provider using Voyage AI
|
||||
* Anthropic partners with Voyage AI for embeddings
|
||||
*/
|
||||
export class AnthropicEmbeddings extends EmbeddingProvider {
|
||||
private config: {
|
||||
apiKey: string;
|
||||
model: string;
|
||||
inputType?: 'document' | 'query';
|
||||
};
|
||||
private anthropic: any;
|
||||
|
||||
/**
|
||||
* Creates a new Anthropic embeddings provider
|
||||
* @param config - Configuration options
|
||||
* @throws Error if Anthropic SDK is not installed
|
||||
*/
|
||||
constructor(config: AnthropicEmbeddingsConfig) {
|
||||
super(config.retryConfig);
|
||||
|
||||
this.config = {
|
||||
apiKey: config.apiKey,
|
||||
model: config.model || 'voyage-2',
|
||||
inputType: config.inputType,
|
||||
};
|
||||
|
||||
try {
|
||||
const Anthropic = require('@anthropic-ai/sdk');
|
||||
this.anthropic = new Anthropic({
|
||||
apiKey: this.config.apiKey,
|
||||
});
|
||||
} catch (error) {
|
||||
throw new Error(
|
||||
'Anthropic SDK not found. Install it with: npm install @anthropic-ai/sdk'
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
getMaxBatchSize(): number {
|
||||
// Process in smaller batches for Voyage API
|
||||
return 128;
|
||||
}
|
||||
|
||||
getDimension(): number {
|
||||
// Voyage-2 produces 1024-dimensional embeddings
|
||||
return 1024;
|
||||
}
|
||||
|
||||
async embedTexts(texts: string[]): Promise<BatchEmbeddingResult> {
|
||||
if (texts.length === 0) {
|
||||
return { embeddings: [] };
|
||||
}
|
||||
|
||||
const batches = this.createBatches(texts);
|
||||
const allResults: EmbeddingResult[] = [];
|
||||
|
||||
for (let batchIndex = 0; batchIndex < batches.length; batchIndex++) {
|
||||
const batch = batches[batchIndex];
|
||||
const baseIndex = batchIndex * this.getMaxBatchSize();
|
||||
|
||||
// Note: As of early 2025, Anthropic uses Voyage AI for embeddings
|
||||
// This is a placeholder for when official API is available
|
||||
const response = await this.withRetry(
|
||||
async () => {
|
||||
// Use Voyage AI API through Anthropic's recommended integration
|
||||
const httpResponse = await fetch('https://api.voyageai.com/v1/embeddings', {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
'Authorization': `Bearer ${this.config.apiKey}`,
|
||||
},
|
||||
body: JSON.stringify({
|
||||
input: batch,
|
||||
model: this.config.model,
|
||||
input_type: this.config.inputType || 'document',
|
||||
}),
|
||||
});
|
||||
|
||||
if (!httpResponse.ok) {
|
||||
const error = await httpResponse.text();
|
||||
throw new Error(`Voyage API error: ${error}`);
|
||||
}
|
||||
|
||||
return await httpResponse.json() as { data: Array<{ embedding: number[] }> };
|
||||
},
|
||||
`Anthropic/Voyage embeddings for batch ${batchIndex + 1}/${batches.length}`
|
||||
);
|
||||
|
||||
for (let i = 0; i < response.data.length; i++) {
|
||||
allResults.push({
|
||||
embedding: response.data[i].embedding,
|
||||
index: baseIndex + i,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
embeddings: allResults,
|
||||
metadata: {
|
||||
model: this.config.model,
|
||||
provider: 'anthropic-voyage',
|
||||
},
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// HuggingFace Local Embeddings Provider
|
||||
// ============================================================================
|
||||
|
||||
/**
|
||||
* Configuration for HuggingFace local embeddings
|
||||
*/
|
||||
export interface HuggingFaceEmbeddingsConfig {
|
||||
/** Model name or path (default: 'sentence-transformers/all-MiniLM-L6-v2') */
|
||||
model?: string;
|
||||
/** Device to run on: 'cpu' or 'cuda' */
|
||||
device?: 'cpu' | 'cuda';
|
||||
/** Normalize embeddings to unit length */
|
||||
normalize?: boolean;
|
||||
/** Batch size for processing */
|
||||
batchSize?: number;
|
||||
/** Retry configuration */
|
||||
retryConfig?: Partial<RetryConfig>;
|
||||
}
|
||||
|
||||
/**
|
||||
* HuggingFace local embeddings provider
|
||||
* Runs embedding models locally using transformers.js
|
||||
*/
|
||||
export class HuggingFaceEmbeddings extends EmbeddingProvider {
|
||||
private config: {
|
||||
model: string;
|
||||
normalize: boolean;
|
||||
batchSize: number;
|
||||
};
|
||||
private pipeline: any;
|
||||
private initialized: boolean = false;
|
||||
|
||||
/**
|
||||
* Creates a new HuggingFace local embeddings provider
|
||||
* @param config - Configuration options
|
||||
*/
|
||||
constructor(config: HuggingFaceEmbeddingsConfig = {}) {
|
||||
super(config.retryConfig);
|
||||
|
||||
this.config = {
|
||||
model: config.model || 'Xenova/all-MiniLM-L6-v2',
|
||||
normalize: config.normalize !== false,
|
||||
batchSize: config.batchSize || 32,
|
||||
};
|
||||
}
|
||||
|
||||
getMaxBatchSize(): number {
|
||||
return this.config.batchSize;
|
||||
}
|
||||
|
||||
getDimension(): number {
|
||||
// all-MiniLM-L6-v2 produces 384-dimensional embeddings
|
||||
// This should be determined dynamically based on model
|
||||
return 384;
|
||||
}
|
||||
|
||||
/**
|
||||
* Initialize the embedding pipeline
|
||||
*/
|
||||
private async initialize(): Promise<void> {
|
||||
if (this.initialized) return;
|
||||
|
||||
try {
|
||||
// Dynamic import of transformers.js
|
||||
const { pipeline } = await import('@xenova/transformers');
|
||||
|
||||
this.pipeline = await pipeline(
|
||||
'feature-extraction',
|
||||
this.config.model
|
||||
);
|
||||
|
||||
this.initialized = true;
|
||||
} catch (error) {
|
||||
throw new Error(
|
||||
'Transformers.js not found or failed to load. Install it with: npm install @xenova/transformers'
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
async embedTexts(texts: string[]): Promise<BatchEmbeddingResult> {
|
||||
if (texts.length === 0) {
|
||||
return { embeddings: [] };
|
||||
}
|
||||
|
||||
await this.initialize();
|
||||
|
||||
const batches = this.createBatches(texts);
|
||||
const allResults: EmbeddingResult[] = [];
|
||||
|
||||
for (let batchIndex = 0; batchIndex < batches.length; batchIndex++) {
|
||||
const batch = batches[batchIndex];
|
||||
const baseIndex = batchIndex * this.getMaxBatchSize();
|
||||
|
||||
const embeddings = await this.withRetry(
|
||||
async () => {
|
||||
const output = await this.pipeline(batch, {
|
||||
pooling: 'mean',
|
||||
normalize: this.config.normalize,
|
||||
});
|
||||
|
||||
// Convert tensor to array
|
||||
return output.tolist();
|
||||
},
|
||||
`HuggingFace embeddings for batch ${batchIndex + 1}/${batches.length}`
|
||||
);
|
||||
|
||||
for (let i = 0; i < embeddings.length; i++) {
|
||||
allResults.push({
|
||||
embedding: embeddings[i],
|
||||
index: baseIndex + i,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
embeddings: allResults,
|
||||
metadata: {
|
||||
model: this.config.model,
|
||||
provider: 'huggingface-local',
|
||||
},
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Helper Functions
|
||||
// ============================================================================
|
||||
|
||||
/**
|
||||
* Embed texts and automatically insert them into a VectorDB
|
||||
*
|
||||
* @param db - VectorDB instance to insert into
|
||||
* @param provider - Embedding provider to use
|
||||
* @param documents - Documents to embed and insert
|
||||
* @param options - Additional options
|
||||
* @returns Promise resolving to array of inserted vector IDs
|
||||
*
|
||||
* @example
|
||||
* ```typescript
|
||||
* const openai = new OpenAIEmbeddings({ apiKey: 'sk-...' });
|
||||
* const db = new VectorDB({ dimension: 1536 });
|
||||
*
|
||||
* const ids = await embedAndInsert(db, openai, [
|
||||
* { id: '1', text: 'Hello world', metadata: { source: 'test' } },
|
||||
* { id: '2', text: 'Another document', metadata: { source: 'test' } }
|
||||
* ]);
|
||||
*
|
||||
* console.log('Inserted vector IDs:', ids);
|
||||
* ```
|
||||
*/
|
||||
export async function embedAndInsert(
|
||||
db: VectorDB,
|
||||
provider: EmbeddingProvider,
|
||||
documents: DocumentToEmbed[],
|
||||
options: {
|
||||
/** Whether to overwrite existing vectors with same ID */
|
||||
overwrite?: boolean;
|
||||
/** Progress callback */
|
||||
onProgress?: (current: number, total: number) => void;
|
||||
} = {}
|
||||
): Promise<string[]> {
|
||||
if (documents.length === 0) {
|
||||
return [];
|
||||
}
|
||||
|
||||
// Verify dimension compatibility
|
||||
const dbDimension = (db as any).dimension || db.getDimension?.();
|
||||
const providerDimension = provider.getDimension();
|
||||
|
||||
if (dbDimension && dbDimension !== providerDimension) {
|
||||
throw new Error(
|
||||
`Dimension mismatch: VectorDB expects ${dbDimension} but provider produces ${providerDimension}`
|
||||
);
|
||||
}
|
||||
|
||||
// Extract texts
|
||||
const texts = documents.map(doc => doc.text);
|
||||
|
||||
// Generate embeddings
|
||||
const result = await provider.embedTexts(texts);
|
||||
|
||||
// Insert vectors
|
||||
const insertedIds: string[] = [];
|
||||
|
||||
for (let i = 0; i < documents.length; i++) {
|
||||
const doc = documents[i];
|
||||
const embedding = result.embeddings.find(e => e.index === i);
|
||||
|
||||
if (!embedding) {
|
||||
throw new Error(`Missing embedding for document at index ${i}`);
|
||||
}
|
||||
|
||||
// Insert or update vector
|
||||
if (options.overwrite) {
|
||||
await db.upsert({
|
||||
id: doc.id,
|
||||
values: embedding.embedding,
|
||||
metadata: doc.metadata,
|
||||
});
|
||||
} else {
|
||||
await db.insert({
|
||||
id: doc.id,
|
||||
values: embedding.embedding,
|
||||
metadata: doc.metadata,
|
||||
});
|
||||
}
|
||||
|
||||
insertedIds.push(doc.id);
|
||||
|
||||
// Call progress callback
|
||||
if (options.onProgress) {
|
||||
options.onProgress(i + 1, documents.length);
|
||||
}
|
||||
}
|
||||
|
||||
return insertedIds;
|
||||
}
|
||||
|
||||
/**
|
||||
* Embed a query and search for similar documents in VectorDB
|
||||
*
|
||||
* @param db - VectorDB instance to search
|
||||
* @param provider - Embedding provider to use
|
||||
* @param query - Query text to search for
|
||||
* @param options - Search options
|
||||
* @returns Promise resolving to search results
|
||||
*
|
||||
* @example
|
||||
* ```typescript
|
||||
* const openai = new OpenAIEmbeddings({ apiKey: 'sk-...' });
|
||||
* const db = new VectorDB({ dimension: 1536 });
|
||||
*
|
||||
* const results = await embedAndSearch(db, openai, 'machine learning', {
|
||||
* topK: 5,
|
||||
* threshold: 0.7
|
||||
* });
|
||||
*
|
||||
* console.log('Found documents:', results);
|
||||
* ```
|
||||
*/
|
||||
export async function embedAndSearch(
|
||||
db: VectorDB,
|
||||
provider: EmbeddingProvider,
|
||||
query: string,
|
||||
options: {
|
||||
/** Number of results to return */
|
||||
topK?: number;
|
||||
/** Minimum similarity threshold (0-1) */
|
||||
threshold?: number;
|
||||
/** Metadata filter */
|
||||
filter?: Record<string, unknown>;
|
||||
} = {}
|
||||
): Promise<any[]> {
|
||||
// Generate query embedding
|
||||
const queryEmbedding = await provider.embedText(query);
|
||||
|
||||
// Search VectorDB
|
||||
const results = await db.search({
|
||||
vector: queryEmbedding,
|
||||
topK: options.topK || 10,
|
||||
threshold: options.threshold,
|
||||
filter: options.filter,
|
||||
});
|
||||
|
||||
return results;
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Exports
|
||||
// ============================================================================
|
||||
|
||||
export default {
|
||||
// Base class
|
||||
EmbeddingProvider,
|
||||
|
||||
// Providers
|
||||
OpenAIEmbeddings,
|
||||
CohereEmbeddings,
|
||||
AnthropicEmbeddings,
|
||||
HuggingFaceEmbeddings,
|
||||
|
||||
// Helper functions
|
||||
embedAndInsert,
|
||||
embedAndSearch,
|
||||
};
|
||||
26
vendor/ruvector/npm/packages/ruvector-extensions/src/examples/embeddings-example.d.ts
vendored
Normal file
26
vendor/ruvector/npm/packages/ruvector-extensions/src/examples/embeddings-example.d.ts
vendored
Normal file
@@ -0,0 +1,26 @@
|
||||
/**
|
||||
* @fileoverview Comprehensive examples for the embeddings integration module
|
||||
*
|
||||
* This file demonstrates all features of the ruvector-extensions embeddings module:
|
||||
* - Multiple embedding providers (OpenAI, Cohere, Anthropic, HuggingFace)
|
||||
* - Batch processing
|
||||
* - Error handling and retry logic
|
||||
* - Integration with VectorDB
|
||||
* - Search functionality
|
||||
*
|
||||
* @author ruv.io Team <info@ruv.io>
|
||||
* @license MIT
|
||||
*/
|
||||
declare function example1_OpenAIBasic(): Promise<void>;
|
||||
declare function example2_OpenAICustomDimensions(): Promise<void>;
|
||||
declare function example3_CohereSearchTypes(): Promise<void>;
|
||||
declare function example4_AnthropicVoyage(): Promise<void>;
|
||||
declare function example5_HuggingFaceLocal(): Promise<void>;
|
||||
declare function example6_BatchProcessing(): Promise<void>;
|
||||
declare function example7_ErrorHandling(): Promise<void>;
|
||||
declare function example8_VectorDBInsert(): Promise<void>;
|
||||
declare function example9_VectorDBSearch(): Promise<void>;
|
||||
declare function example10_CompareProviders(): Promise<void>;
|
||||
declare function example11_ProgressiveLoading(): Promise<void>;
|
||||
export { example1_OpenAIBasic, example2_OpenAICustomDimensions, example3_CohereSearchTypes, example4_AnthropicVoyage, example5_HuggingFaceLocal, example6_BatchProcessing, example7_ErrorHandling, example8_VectorDBInsert, example9_VectorDBSearch, example10_CompareProviders, example11_ProgressiveLoading, };
|
||||
//# sourceMappingURL=embeddings-example.d.ts.map
|
||||
1
vendor/ruvector/npm/packages/ruvector-extensions/src/examples/embeddings-example.d.ts.map
vendored
Normal file
1
vendor/ruvector/npm/packages/ruvector-extensions/src/examples/embeddings-example.d.ts.map
vendored
Normal file
@@ -0,0 +1 @@
|
||||
{"version":3,"file":"embeddings-example.d.ts","sourceRoot":"","sources":["embeddings-example.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAgBH,iBAAe,oBAAoB,kBA0BlC;AAMD,iBAAe,+BAA+B,kBAa7C;AAMD,iBAAe,0BAA0B,kBAiCxC;AAMD,iBAAe,wBAAwB,kBAiBtC;AAMD,iBAAe,yBAAyB,kBAoBvC;AAMD,iBAAe,wBAAwB,kBAsBtC;AAMD,iBAAe,sBAAsB,kBAsBpC;AAMD,iBAAe,uBAAuB,kBA8CrC;AAMD,iBAAe,uBAAuB,kBAmCrC;AAMD,iBAAe,0BAA0B,kBA0CxC;AAMD,iBAAe,4BAA4B,kBAgC1C;AAuCD,OAAO,EACL,oBAAoB,EACpB,+BAA+B,EAC/B,0BAA0B,EAC1B,wBAAwB,EACxB,yBAAyB,EACzB,wBAAwB,EACxB,sBAAsB,EACtB,uBAAuB,EACvB,uBAAuB,EACvB,0BAA0B,EAC1B,4BAA4B,GAC7B,CAAC"}
|
||||
364
vendor/ruvector/npm/packages/ruvector-extensions/src/examples/embeddings-example.js
vendored
Normal file
364
vendor/ruvector/npm/packages/ruvector-extensions/src/examples/embeddings-example.js
vendored
Normal file
@@ -0,0 +1,364 @@
|
||||
"use strict";
|
||||
/**
|
||||
* @fileoverview Comprehensive examples for the embeddings integration module
|
||||
*
|
||||
* This file demonstrates all features of the ruvector-extensions embeddings module:
|
||||
* - Multiple embedding providers (OpenAI, Cohere, Anthropic, HuggingFace)
|
||||
* - Batch processing
|
||||
* - Error handling and retry logic
|
||||
* - Integration with VectorDB
|
||||
* - Search functionality
|
||||
*
|
||||
* @author ruv.io Team <info@ruv.io>
|
||||
* @license MIT
|
||||
*/
|
||||
Object.defineProperty(exports, "__esModule", { value: true });
|
||||
exports.example1_OpenAIBasic = example1_OpenAIBasic;
|
||||
exports.example2_OpenAICustomDimensions = example2_OpenAICustomDimensions;
|
||||
exports.example3_CohereSearchTypes = example3_CohereSearchTypes;
|
||||
exports.example4_AnthropicVoyage = example4_AnthropicVoyage;
|
||||
exports.example5_HuggingFaceLocal = example5_HuggingFaceLocal;
|
||||
exports.example6_BatchProcessing = example6_BatchProcessing;
|
||||
exports.example7_ErrorHandling = example7_ErrorHandling;
|
||||
exports.example8_VectorDBInsert = example8_VectorDBInsert;
|
||||
exports.example9_VectorDBSearch = example9_VectorDBSearch;
|
||||
exports.example10_CompareProviders = example10_CompareProviders;
|
||||
exports.example11_ProgressiveLoading = example11_ProgressiveLoading;
|
||||
const embeddings_js_1 = require("../embeddings.js");
|
||||
// ============================================================================
|
||||
// Example 1: OpenAI Embeddings - Basic Usage
|
||||
// ============================================================================
|
||||
async function example1_OpenAIBasic() {
|
||||
console.log('\n=== Example 1: OpenAI Embeddings - Basic Usage ===\n');
|
||||
// Initialize OpenAI embeddings provider
|
||||
const openai = new embeddings_js_1.OpenAIEmbeddings({
|
||||
apiKey: process.env.OPENAI_API_KEY || 'sk-...',
|
||||
model: 'text-embedding-3-small', // 1536 dimensions
|
||||
});
|
||||
// Embed a single text
|
||||
const singleEmbedding = await openai.embedText('Hello, world!');
|
||||
console.log('Single embedding dimension:', singleEmbedding.length);
|
||||
console.log('First 5 values:', singleEmbedding.slice(0, 5));
|
||||
// Embed multiple texts
|
||||
const texts = [
|
||||
'Machine learning is fascinating',
|
||||
'Deep learning uses neural networks',
|
||||
'Natural language processing is important',
|
||||
];
|
||||
const result = await openai.embedTexts(texts);
|
||||
console.log('\nBatch embeddings:');
|
||||
console.log('Total embeddings:', result.embeddings.length);
|
||||
console.log('Total tokens used:', result.totalTokens);
|
||||
console.log('Provider:', result.metadata?.provider);
|
||||
}
|
||||
// ============================================================================
|
||||
// Example 2: OpenAI with Custom Dimensions
|
||||
// ============================================================================
|
||||
async function example2_OpenAICustomDimensions() {
|
||||
console.log('\n=== Example 2: OpenAI with Custom Dimensions ===\n');
|
||||
// Use text-embedding-3-large with custom dimensions
|
||||
const openai = new embeddings_js_1.OpenAIEmbeddings({
|
||||
apiKey: process.env.OPENAI_API_KEY || 'sk-...',
|
||||
model: 'text-embedding-3-large',
|
||||
dimensions: 1024, // Reduce from default 3072 to 1024
|
||||
});
|
||||
const embedding = await openai.embedText('Custom dimension embedding');
|
||||
console.log('Embedding dimension:', embedding.length);
|
||||
console.log('Expected:', openai.getDimension());
|
||||
}
|
||||
// ============================================================================
|
||||
// Example 3: Cohere Embeddings with Search Types
|
||||
// ============================================================================
|
||||
async function example3_CohereSearchTypes() {
|
||||
console.log('\n=== Example 3: Cohere Embeddings with Search Types ===\n');
|
||||
const cohere = new embeddings_js_1.CohereEmbeddings({
|
||||
apiKey: process.env.COHERE_API_KEY || 'your-key',
|
||||
model: 'embed-english-v3.0',
|
||||
});
|
||||
// Embed documents (for storage)
|
||||
const documentEmbedder = new embeddings_js_1.CohereEmbeddings({
|
||||
apiKey: process.env.COHERE_API_KEY || 'your-key',
|
||||
model: 'embed-english-v3.0',
|
||||
inputType: 'search_document',
|
||||
});
|
||||
const documents = [
|
||||
'The Eiffel Tower is in Paris',
|
||||
'The Statue of Liberty is in New York',
|
||||
'The Great Wall is in China',
|
||||
];
|
||||
const docResult = await documentEmbedder.embedTexts(documents);
|
||||
console.log('Document embeddings created:', docResult.embeddings.length);
|
||||
// Embed query (for searching)
|
||||
const queryEmbedder = new embeddings_js_1.CohereEmbeddings({
|
||||
apiKey: process.env.COHERE_API_KEY || 'your-key',
|
||||
model: 'embed-english-v3.0',
|
||||
inputType: 'search_query',
|
||||
});
|
||||
const queryEmbedding = await queryEmbedder.embedText('famous landmarks in France');
|
||||
console.log('Query embedding dimension:', queryEmbedding.length);
|
||||
}
|
||||
// ============================================================================
|
||||
// Example 4: Anthropic/Voyage Embeddings
|
||||
// ============================================================================
|
||||
async function example4_AnthropicVoyage() {
|
||||
console.log('\n=== Example 4: Anthropic/Voyage Embeddings ===\n');
|
||||
const anthropic = new embeddings_js_1.AnthropicEmbeddings({
|
||||
apiKey: process.env.VOYAGE_API_KEY || 'your-voyage-key',
|
||||
model: 'voyage-2',
|
||||
inputType: 'document',
|
||||
});
|
||||
const texts = [
|
||||
'Anthropic develops Claude AI',
|
||||
'Voyage AI provides embedding models',
|
||||
];
|
||||
const result = await anthropic.embedTexts(texts);
|
||||
console.log('Embeddings created:', result.embeddings.length);
|
||||
console.log('Dimension:', anthropic.getDimension());
|
||||
}
|
||||
// ============================================================================
|
||||
// Example 5: HuggingFace Local Embeddings
|
||||
// ============================================================================
|
||||
async function example5_HuggingFaceLocal() {
|
||||
console.log('\n=== Example 5: HuggingFace Local Embeddings ===\n');
|
||||
// Run embeddings locally - no API key needed!
|
||||
const hf = new embeddings_js_1.HuggingFaceEmbeddings({
|
||||
model: 'Xenova/all-MiniLM-L6-v2',
|
||||
normalize: true,
|
||||
batchSize: 32,
|
||||
});
|
||||
const texts = [
|
||||
'Local embeddings are fast',
|
||||
'No API calls required',
|
||||
'Privacy-friendly solution',
|
||||
];
|
||||
console.log('Processing locally...');
|
||||
const result = await hf.embedTexts(texts);
|
||||
console.log('Local embeddings created:', result.embeddings.length);
|
||||
console.log('Dimension:', hf.getDimension());
|
||||
}
|
||||
// ============================================================================
|
||||
// Example 6: Batch Processing Large Datasets
|
||||
// ============================================================================
|
||||
async function example6_BatchProcessing() {
|
||||
console.log('\n=== Example 6: Batch Processing Large Datasets ===\n');
|
||||
const openai = new embeddings_js_1.OpenAIEmbeddings({
|
||||
apiKey: process.env.OPENAI_API_KEY || 'sk-...',
|
||||
});
|
||||
// Generate 1000 sample texts
|
||||
const largeDataset = Array.from({ length: 1000 }, (_, i) => `Document ${i}: Sample text for embedding`);
|
||||
console.log('Processing 1000 texts...');
|
||||
const startTime = Date.now();
|
||||
const result = await openai.embedTexts(largeDataset);
|
||||
const duration = Date.now() - startTime;
|
||||
console.log(`Processed ${result.embeddings.length} texts in ${duration}ms`);
|
||||
console.log(`Average: ${(duration / result.embeddings.length).toFixed(2)}ms per text`);
|
||||
console.log(`Total tokens: ${result.totalTokens}`);
|
||||
}
|
||||
// ============================================================================
|
||||
// Example 7: Error Handling and Retry Logic
|
||||
// ============================================================================
|
||||
async function example7_ErrorHandling() {
|
||||
console.log('\n=== Example 7: Error Handling and Retry Logic ===\n');
|
||||
// Configure custom retry logic
|
||||
const openai = new embeddings_js_1.OpenAIEmbeddings({
|
||||
apiKey: process.env.OPENAI_API_KEY || 'sk-...',
|
||||
retryConfig: {
|
||||
maxRetries: 5,
|
||||
initialDelay: 2000,
|
||||
maxDelay: 30000,
|
||||
backoffMultiplier: 2,
|
||||
},
|
||||
});
|
||||
try {
|
||||
// This will retry on rate limits or temporary errors
|
||||
const result = await openai.embedTexts(['Test text']);
|
||||
console.log('Success! Embeddings created:', result.embeddings.length);
|
||||
}
|
||||
catch (error) {
|
||||
console.error('Failed after retries:', error.message);
|
||||
console.error('Retryable:', error.retryable);
|
||||
}
|
||||
}
|
||||
// ============================================================================
|
||||
// Example 8: Integration with VectorDB - Insert
|
||||
// ============================================================================
|
||||
async function example8_VectorDBInsert() {
|
||||
console.log('\n=== Example 8: Integration with VectorDB - Insert ===\n');
|
||||
// Note: This example assumes VectorDB is available
|
||||
// You'll need to import and initialize VectorDB first
|
||||
const openai = new embeddings_js_1.OpenAIEmbeddings({
|
||||
apiKey: process.env.OPENAI_API_KEY || 'sk-...',
|
||||
});
|
||||
// Sample documents to embed and insert
|
||||
const documents = [
|
||||
{
|
||||
id: 'doc1',
|
||||
text: 'Machine learning enables computers to learn from data',
|
||||
metadata: { category: 'AI', author: 'John Doe' },
|
||||
},
|
||||
{
|
||||
id: 'doc2',
|
||||
text: 'Deep learning uses neural networks with multiple layers',
|
||||
metadata: { category: 'AI', author: 'Jane Smith' },
|
||||
},
|
||||
{
|
||||
id: 'doc3',
|
||||
text: 'Natural language processing helps computers understand text',
|
||||
metadata: { category: 'NLP', author: 'John Doe' },
|
||||
},
|
||||
];
|
||||
// Example usage (uncomment when VectorDB is available):
|
||||
/*
|
||||
const { VectorDB } = await import('ruvector');
|
||||
const db = new VectorDB({ dimension: openai.getDimension() });
|
||||
|
||||
const insertedIds = await embedAndInsert(db, openai, documents, {
|
||||
overwrite: true,
|
||||
onProgress: (current, total) => {
|
||||
console.log(`Progress: ${current}/${total} documents inserted`);
|
||||
},
|
||||
});
|
||||
|
||||
console.log('Inserted document IDs:', insertedIds);
|
||||
*/
|
||||
console.log('Documents prepared:', documents.length);
|
||||
console.log('Ready for insertion when VectorDB is initialized');
|
||||
}
|
||||
// ============================================================================
|
||||
// Example 9: Integration with VectorDB - Search
|
||||
// ============================================================================
|
||||
async function example9_VectorDBSearch() {
|
||||
console.log('\n=== Example 9: Integration with VectorDB - Search ===\n');
|
||||
const openai = new embeddings_js_1.OpenAIEmbeddings({
|
||||
apiKey: process.env.OPENAI_API_KEY || 'sk-...',
|
||||
});
|
||||
// Example usage (uncomment when VectorDB is available):
|
||||
/*
|
||||
const { VectorDB } = await import('ruvector');
|
||||
const db = new VectorDB({ dimension: openai.getDimension() });
|
||||
|
||||
// First, insert some documents (see example 8)
|
||||
// ...
|
||||
|
||||
// Now search for similar documents
|
||||
const results = await embedAndSearch(
|
||||
db,
|
||||
openai,
|
||||
'What is deep learning?',
|
||||
{
|
||||
topK: 5,
|
||||
threshold: 0.7,
|
||||
filter: { category: 'AI' },
|
||||
}
|
||||
);
|
||||
|
||||
console.log('Search results:');
|
||||
results.forEach((result, i) => {
|
||||
console.log(`${i + 1}. ${result.id} (similarity: ${result.score})`);
|
||||
console.log(` Text: ${result.metadata?.text}`);
|
||||
});
|
||||
*/
|
||||
console.log('Search functionality ready when VectorDB is initialized');
|
||||
}
|
||||
// ============================================================================
|
||||
// Example 10: Comparing Multiple Providers
|
||||
// ============================================================================
|
||||
async function example10_CompareProviders() {
|
||||
console.log('\n=== Example 10: Comparing Multiple Providers ===\n');
|
||||
const text = 'Artificial intelligence is transforming the world';
|
||||
// OpenAI
|
||||
const openai = new embeddings_js_1.OpenAIEmbeddings({
|
||||
apiKey: process.env.OPENAI_API_KEY || 'sk-...',
|
||||
});
|
||||
// Cohere
|
||||
const cohere = new embeddings_js_1.CohereEmbeddings({
|
||||
apiKey: process.env.COHERE_API_KEY || 'your-key',
|
||||
});
|
||||
// HuggingFace (local)
|
||||
const hf = new embeddings_js_1.HuggingFaceEmbeddings();
|
||||
// Compare dimensions
|
||||
console.log('Provider dimensions:');
|
||||
console.log('- OpenAI:', openai.getDimension());
|
||||
console.log('- Cohere:', cohere.getDimension());
|
||||
console.log('- HuggingFace:', hf.getDimension());
|
||||
// Compare batch sizes
|
||||
console.log('\nMax batch sizes:');
|
||||
console.log('- OpenAI:', openai.getMaxBatchSize());
|
||||
console.log('- Cohere:', cohere.getMaxBatchSize());
|
||||
console.log('- HuggingFace:', hf.getMaxBatchSize());
|
||||
// Generate embeddings (uncomment to actually run):
|
||||
/*
|
||||
console.log('\nGenerating embeddings...');
|
||||
|
||||
const [openaiResult, cohereResult, hfResult] = await Promise.all([
|
||||
openai.embedText(text),
|
||||
cohere.embedText(text),
|
||||
hf.embedText(text),
|
||||
]);
|
||||
|
||||
console.log('All embeddings generated successfully!');
|
||||
*/
|
||||
}
|
||||
// ============================================================================
|
||||
// Example 11: Progressive Loading with Progress Tracking
|
||||
// ============================================================================
|
||||
async function example11_ProgressiveLoading() {
|
||||
console.log('\n=== Example 11: Progressive Loading with Progress ===\n');
|
||||
const openai = new embeddings_js_1.OpenAIEmbeddings({
|
||||
apiKey: process.env.OPENAI_API_KEY || 'sk-...',
|
||||
});
|
||||
const documents = Array.from({ length: 50 }, (_, i) => ({
|
||||
id: `doc${i}`,
|
||||
text: `Document ${i}: This is sample content for embedding`,
|
||||
metadata: { index: i, batch: Math.floor(i / 10) },
|
||||
}));
|
||||
// Track progress
|
||||
let processed = 0;
|
||||
const progressBar = (current, total) => {
|
||||
const percentage = Math.round((current / total) * 100);
|
||||
const bar = '█'.repeat(percentage / 2) + '░'.repeat(50 - percentage / 2);
|
||||
console.log(`[${bar}] ${percentage}% (${current}/${total})`);
|
||||
};
|
||||
// Example usage (uncomment when VectorDB is available):
|
||||
/*
|
||||
const { VectorDB } = await import('ruvector');
|
||||
const db = new VectorDB({ dimension: openai.getDimension() });
|
||||
|
||||
await embedAndInsert(db, openai, documents, {
|
||||
onProgress: progressBar,
|
||||
});
|
||||
*/
|
||||
console.log('Ready to process', documents.length, 'documents with progress tracking');
|
||||
}
|
||||
// ============================================================================
|
||||
// Main Function - Run All Examples
|
||||
// ============================================================================
|
||||
async function runAllExamples() {
|
||||
console.log('╔════════════════════════════════════════════════════════════╗');
|
||||
console.log('║ RUVector Extensions - Embeddings Integration Examples ║');
|
||||
console.log('╚════════════════════════════════════════════════════════════╝');
|
||||
// Note: Uncomment the examples you want to run
|
||||
// Make sure you have the required API keys set in environment variables
|
||||
try {
|
||||
// await example1_OpenAIBasic();
|
||||
// await example2_OpenAICustomDimensions();
|
||||
// await example3_CohereSearchTypes();
|
||||
// await example4_AnthropicVoyage();
|
||||
// await example5_HuggingFaceLocal();
|
||||
// await example6_BatchProcessing();
|
||||
// await example7_ErrorHandling();
|
||||
// await example8_VectorDBInsert();
|
||||
// await example9_VectorDBSearch();
|
||||
// await example10_CompareProviders();
|
||||
// await example11_ProgressiveLoading();
|
||||
console.log('\n✓ All examples completed successfully!');
|
||||
}
|
||||
catch (error) {
|
||||
console.error('\n✗ Error running examples:', error);
|
||||
}
|
||||
}
|
||||
// Run if executed directly
|
||||
if (import.meta.url === `file://${process.argv[1]}`) {
|
||||
runAllExamples();
|
||||
}
|
||||
//# sourceMappingURL=embeddings-example.js.map
|
||||
1
vendor/ruvector/npm/packages/ruvector-extensions/src/examples/embeddings-example.js.map
vendored
Normal file
1
vendor/ruvector/npm/packages/ruvector-extensions/src/examples/embeddings-example.js.map
vendored
Normal file
File diff suppressed because one or more lines are too long
448
vendor/ruvector/npm/packages/ruvector-extensions/src/examples/embeddings-example.ts
vendored
Normal file
448
vendor/ruvector/npm/packages/ruvector-extensions/src/examples/embeddings-example.ts
vendored
Normal file
@@ -0,0 +1,448 @@
|
||||
/**
|
||||
* @fileoverview Comprehensive examples for the embeddings integration module
|
||||
*
|
||||
* This file demonstrates all features of the ruvector-extensions embeddings module:
|
||||
* - Multiple embedding providers (OpenAI, Cohere, Anthropic, HuggingFace)
|
||||
* - Batch processing
|
||||
* - Error handling and retry logic
|
||||
* - Integration with VectorDB
|
||||
* - Search functionality
|
||||
*
|
||||
* @author ruv.io Team <info@ruv.io>
|
||||
* @license MIT
|
||||
*/
|
||||
|
||||
import {
|
||||
OpenAIEmbeddings,
|
||||
CohereEmbeddings,
|
||||
AnthropicEmbeddings,
|
||||
HuggingFaceEmbeddings,
|
||||
embedAndInsert,
|
||||
embedAndSearch,
|
||||
type DocumentToEmbed,
|
||||
} from '../embeddings.js';
|
||||
|
||||
// ============================================================================
|
||||
// Example 1: OpenAI Embeddings - Basic Usage
|
||||
// ============================================================================
|
||||
|
||||
async function example1_OpenAIBasic() {
|
||||
console.log('\n=== Example 1: OpenAI Embeddings - Basic Usage ===\n');
|
||||
|
||||
// Initialize OpenAI embeddings provider
|
||||
const openai = new OpenAIEmbeddings({
|
||||
apiKey: process.env.OPENAI_API_KEY || 'sk-...',
|
||||
model: 'text-embedding-3-small', // 1536 dimensions
|
||||
});
|
||||
|
||||
// Embed a single text
|
||||
const singleEmbedding = await openai.embedText('Hello, world!');
|
||||
console.log('Single embedding dimension:', singleEmbedding.length);
|
||||
console.log('First 5 values:', singleEmbedding.slice(0, 5));
|
||||
|
||||
// Embed multiple texts
|
||||
const texts = [
|
||||
'Machine learning is fascinating',
|
||||
'Deep learning uses neural networks',
|
||||
'Natural language processing is important',
|
||||
];
|
||||
|
||||
const result = await openai.embedTexts(texts);
|
||||
console.log('\nBatch embeddings:');
|
||||
console.log('Total embeddings:', result.embeddings.length);
|
||||
console.log('Total tokens used:', result.totalTokens);
|
||||
console.log('Provider:', result.metadata?.provider);
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Example 2: OpenAI with Custom Dimensions
|
||||
// ============================================================================
|
||||
|
||||
async function example2_OpenAICustomDimensions() {
|
||||
console.log('\n=== Example 2: OpenAI with Custom Dimensions ===\n');
|
||||
|
||||
// Use text-embedding-3-large with custom dimensions
|
||||
const openai = new OpenAIEmbeddings({
|
||||
apiKey: process.env.OPENAI_API_KEY || 'sk-...',
|
||||
model: 'text-embedding-3-large',
|
||||
dimensions: 1024, // Reduce from default 3072 to 1024
|
||||
});
|
||||
|
||||
const embedding = await openai.embedText('Custom dimension embedding');
|
||||
console.log('Embedding dimension:', embedding.length);
|
||||
console.log('Expected:', openai.getDimension());
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Example 3: Cohere Embeddings with Search Types
|
||||
// ============================================================================
|
||||
|
||||
async function example3_CohereSearchTypes() {
|
||||
console.log('\n=== Example 3: Cohere Embeddings with Search Types ===\n');
|
||||
|
||||
const cohere = new CohereEmbeddings({
|
||||
apiKey: process.env.COHERE_API_KEY || 'your-key',
|
||||
model: 'embed-english-v3.0',
|
||||
});
|
||||
|
||||
// Embed documents (for storage)
|
||||
const documentEmbedder = new CohereEmbeddings({
|
||||
apiKey: process.env.COHERE_API_KEY || 'your-key',
|
||||
model: 'embed-english-v3.0',
|
||||
inputType: 'search_document',
|
||||
});
|
||||
|
||||
const documents = [
|
||||
'The Eiffel Tower is in Paris',
|
||||
'The Statue of Liberty is in New York',
|
||||
'The Great Wall is in China',
|
||||
];
|
||||
|
||||
const docResult = await documentEmbedder.embedTexts(documents);
|
||||
console.log('Document embeddings created:', docResult.embeddings.length);
|
||||
|
||||
// Embed query (for searching)
|
||||
const queryEmbedder = new CohereEmbeddings({
|
||||
apiKey: process.env.COHERE_API_KEY || 'your-key',
|
||||
model: 'embed-english-v3.0',
|
||||
inputType: 'search_query',
|
||||
});
|
||||
|
||||
const queryEmbedding = await queryEmbedder.embedText('famous landmarks in France');
|
||||
console.log('Query embedding dimension:', queryEmbedding.length);
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Example 4: Anthropic/Voyage Embeddings
|
||||
// ============================================================================
|
||||
|
||||
async function example4_AnthropicVoyage() {
|
||||
console.log('\n=== Example 4: Anthropic/Voyage Embeddings ===\n');
|
||||
|
||||
const anthropic = new AnthropicEmbeddings({
|
||||
apiKey: process.env.VOYAGE_API_KEY || 'your-voyage-key',
|
||||
model: 'voyage-2',
|
||||
inputType: 'document',
|
||||
});
|
||||
|
||||
const texts = [
|
||||
'Anthropic develops Claude AI',
|
||||
'Voyage AI provides embedding models',
|
||||
];
|
||||
|
||||
const result = await anthropic.embedTexts(texts);
|
||||
console.log('Embeddings created:', result.embeddings.length);
|
||||
console.log('Dimension:', anthropic.getDimension());
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Example 5: HuggingFace Local Embeddings
|
||||
// ============================================================================
|
||||
|
||||
async function example5_HuggingFaceLocal() {
|
||||
console.log('\n=== Example 5: HuggingFace Local Embeddings ===\n');
|
||||
|
||||
// Run embeddings locally - no API key needed!
|
||||
const hf = new HuggingFaceEmbeddings({
|
||||
model: 'Xenova/all-MiniLM-L6-v2',
|
||||
normalize: true,
|
||||
batchSize: 32,
|
||||
});
|
||||
|
||||
const texts = [
|
||||
'Local embeddings are fast',
|
||||
'No API calls required',
|
||||
'Privacy-friendly solution',
|
||||
];
|
||||
|
||||
console.log('Processing locally...');
|
||||
const result = await hf.embedTexts(texts);
|
||||
console.log('Local embeddings created:', result.embeddings.length);
|
||||
console.log('Dimension:', hf.getDimension());
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Example 6: Batch Processing Large Datasets
|
||||
// ============================================================================
|
||||
|
||||
async function example6_BatchProcessing() {
|
||||
console.log('\n=== Example 6: Batch Processing Large Datasets ===\n');
|
||||
|
||||
const openai = new OpenAIEmbeddings({
|
||||
apiKey: process.env.OPENAI_API_KEY || 'sk-...',
|
||||
});
|
||||
|
||||
// Generate 1000 sample texts
|
||||
const largeDataset = Array.from(
|
||||
{ length: 1000 },
|
||||
(_, i) => `Document ${i}: Sample text for embedding`
|
||||
);
|
||||
|
||||
console.log('Processing 1000 texts...');
|
||||
const startTime = Date.now();
|
||||
|
||||
const result = await openai.embedTexts(largeDataset);
|
||||
|
||||
const duration = Date.now() - startTime;
|
||||
console.log(`Processed ${result.embeddings.length} texts in ${duration}ms`);
|
||||
console.log(`Average: ${(duration / result.embeddings.length).toFixed(2)}ms per text`);
|
||||
console.log(`Total tokens: ${result.totalTokens}`);
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Example 7: Error Handling and Retry Logic
|
||||
// ============================================================================
|
||||
|
||||
async function example7_ErrorHandling() {
|
||||
console.log('\n=== Example 7: Error Handling and Retry Logic ===\n');
|
||||
|
||||
// Configure custom retry logic
|
||||
const openai = new OpenAIEmbeddings({
|
||||
apiKey: process.env.OPENAI_API_KEY || 'sk-...',
|
||||
retryConfig: {
|
||||
maxRetries: 5,
|
||||
initialDelay: 2000,
|
||||
maxDelay: 30000,
|
||||
backoffMultiplier: 2,
|
||||
},
|
||||
});
|
||||
|
||||
try {
|
||||
// This will retry on rate limits or temporary errors
|
||||
const result = await openai.embedTexts(['Test text']);
|
||||
console.log('Success! Embeddings created:', result.embeddings.length);
|
||||
} catch (error: any) {
|
||||
console.error('Failed after retries:', error.message);
|
||||
console.error('Retryable:', error.retryable);
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Example 8: Integration with VectorDB - Insert
|
||||
// ============================================================================
|
||||
|
||||
async function example8_VectorDBInsert() {
|
||||
console.log('\n=== Example 8: Integration with VectorDB - Insert ===\n');
|
||||
|
||||
// Note: This example assumes VectorDB is available
|
||||
// You'll need to import and initialize VectorDB first
|
||||
|
||||
const openai = new OpenAIEmbeddings({
|
||||
apiKey: process.env.OPENAI_API_KEY || 'sk-...',
|
||||
});
|
||||
|
||||
// Sample documents to embed and insert
|
||||
const documents: DocumentToEmbed[] = [
|
||||
{
|
||||
id: 'doc1',
|
||||
text: 'Machine learning enables computers to learn from data',
|
||||
metadata: { category: 'AI', author: 'John Doe' },
|
||||
},
|
||||
{
|
||||
id: 'doc2',
|
||||
text: 'Deep learning uses neural networks with multiple layers',
|
||||
metadata: { category: 'AI', author: 'Jane Smith' },
|
||||
},
|
||||
{
|
||||
id: 'doc3',
|
||||
text: 'Natural language processing helps computers understand text',
|
||||
metadata: { category: 'NLP', author: 'John Doe' },
|
||||
},
|
||||
];
|
||||
|
||||
// Example usage (uncomment when VectorDB is available):
|
||||
/*
|
||||
const { VectorDB } = await import('ruvector');
|
||||
const db = new VectorDB({ dimension: openai.getDimension() });
|
||||
|
||||
const insertedIds = await embedAndInsert(db, openai, documents, {
|
||||
overwrite: true,
|
||||
onProgress: (current, total) => {
|
||||
console.log(`Progress: ${current}/${total} documents inserted`);
|
||||
},
|
||||
});
|
||||
|
||||
console.log('Inserted document IDs:', insertedIds);
|
||||
*/
|
||||
|
||||
console.log('Documents prepared:', documents.length);
|
||||
console.log('Ready for insertion when VectorDB is initialized');
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Example 9: Integration with VectorDB - Search
|
||||
// ============================================================================
|
||||
|
||||
async function example9_VectorDBSearch() {
|
||||
console.log('\n=== Example 9: Integration with VectorDB - Search ===\n');
|
||||
|
||||
const openai = new OpenAIEmbeddings({
|
||||
apiKey: process.env.OPENAI_API_KEY || 'sk-...',
|
||||
});
|
||||
|
||||
// Example usage (uncomment when VectorDB is available):
|
||||
/*
|
||||
const { VectorDB } = await import('ruvector');
|
||||
const db = new VectorDB({ dimension: openai.getDimension() });
|
||||
|
||||
// First, insert some documents (see example 8)
|
||||
// ...
|
||||
|
||||
// Now search for similar documents
|
||||
const results = await embedAndSearch(
|
||||
db,
|
||||
openai,
|
||||
'What is deep learning?',
|
||||
{
|
||||
topK: 5,
|
||||
threshold: 0.7,
|
||||
filter: { category: 'AI' },
|
||||
}
|
||||
);
|
||||
|
||||
console.log('Search results:');
|
||||
results.forEach((result, i) => {
|
||||
console.log(`${i + 1}. ${result.id} (similarity: ${result.score})`);
|
||||
console.log(` Text: ${result.metadata?.text}`);
|
||||
});
|
||||
*/
|
||||
|
||||
console.log('Search functionality ready when VectorDB is initialized');
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Example 10: Comparing Multiple Providers
|
||||
// ============================================================================
|
||||
|
||||
async function example10_CompareProviders() {
|
||||
console.log('\n=== Example 10: Comparing Multiple Providers ===\n');
|
||||
|
||||
const text = 'Artificial intelligence is transforming the world';
|
||||
|
||||
// OpenAI
|
||||
const openai = new OpenAIEmbeddings({
|
||||
apiKey: process.env.OPENAI_API_KEY || 'sk-...',
|
||||
});
|
||||
|
||||
// Cohere
|
||||
const cohere = new CohereEmbeddings({
|
||||
apiKey: process.env.COHERE_API_KEY || 'your-key',
|
||||
});
|
||||
|
||||
// HuggingFace (local)
|
||||
const hf = new HuggingFaceEmbeddings();
|
||||
|
||||
// Compare dimensions
|
||||
console.log('Provider dimensions:');
|
||||
console.log('- OpenAI:', openai.getDimension());
|
||||
console.log('- Cohere:', cohere.getDimension());
|
||||
console.log('- HuggingFace:', hf.getDimension());
|
||||
|
||||
// Compare batch sizes
|
||||
console.log('\nMax batch sizes:');
|
||||
console.log('- OpenAI:', openai.getMaxBatchSize());
|
||||
console.log('- Cohere:', cohere.getMaxBatchSize());
|
||||
console.log('- HuggingFace:', hf.getMaxBatchSize());
|
||||
|
||||
// Generate embeddings (uncomment to actually run):
|
||||
/*
|
||||
console.log('\nGenerating embeddings...');
|
||||
|
||||
const [openaiResult, cohereResult, hfResult] = await Promise.all([
|
||||
openai.embedText(text),
|
||||
cohere.embedText(text),
|
||||
hf.embedText(text),
|
||||
]);
|
||||
|
||||
console.log('All embeddings generated successfully!');
|
||||
*/
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Example 11: Progressive Loading with Progress Tracking
|
||||
// ============================================================================
|
||||
|
||||
async function example11_ProgressiveLoading() {
|
||||
console.log('\n=== Example 11: Progressive Loading with Progress ===\n');
|
||||
|
||||
const openai = new OpenAIEmbeddings({
|
||||
apiKey: process.env.OPENAI_API_KEY || 'sk-...',
|
||||
});
|
||||
|
||||
const documents: DocumentToEmbed[] = Array.from({ length: 50 }, (_, i) => ({
|
||||
id: `doc${i}`,
|
||||
text: `Document ${i}: This is sample content for embedding`,
|
||||
metadata: { index: i, batch: Math.floor(i / 10) },
|
||||
}));
|
||||
|
||||
// Track progress
|
||||
let processed = 0;
|
||||
const progressBar = (current: number, total: number) => {
|
||||
const percentage = Math.round((current / total) * 100);
|
||||
const bar = '█'.repeat(percentage / 2) + '░'.repeat(50 - percentage / 2);
|
||||
console.log(`[${bar}] ${percentage}% (${current}/${total})`);
|
||||
};
|
||||
|
||||
// Example usage (uncomment when VectorDB is available):
|
||||
/*
|
||||
const { VectorDB } = await import('ruvector');
|
||||
const db = new VectorDB({ dimension: openai.getDimension() });
|
||||
|
||||
await embedAndInsert(db, openai, documents, {
|
||||
onProgress: progressBar,
|
||||
});
|
||||
*/
|
||||
|
||||
console.log('Ready to process', documents.length, 'documents with progress tracking');
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Main Function - Run All Examples
|
||||
// ============================================================================
|
||||
|
||||
async function runAllExamples() {
|
||||
console.log('╔════════════════════════════════════════════════════════════╗');
|
||||
console.log('║ RUVector Extensions - Embeddings Integration Examples ║');
|
||||
console.log('╚════════════════════════════════════════════════════════════╝');
|
||||
|
||||
// Note: Uncomment the examples you want to run
|
||||
// Make sure you have the required API keys set in environment variables
|
||||
|
||||
try {
|
||||
// await example1_OpenAIBasic();
|
||||
// await example2_OpenAICustomDimensions();
|
||||
// await example3_CohereSearchTypes();
|
||||
// await example4_AnthropicVoyage();
|
||||
// await example5_HuggingFaceLocal();
|
||||
// await example6_BatchProcessing();
|
||||
// await example7_ErrorHandling();
|
||||
// await example8_VectorDBInsert();
|
||||
// await example9_VectorDBSearch();
|
||||
// await example10_CompareProviders();
|
||||
// await example11_ProgressiveLoading();
|
||||
|
||||
console.log('\n✓ All examples completed successfully!');
|
||||
} catch (error) {
|
||||
console.error('\n✗ Error running examples:', error);
|
||||
}
|
||||
}
|
||||
|
||||
// Run if executed directly
|
||||
if (import.meta.url === `file://${process.argv[1]}`) {
|
||||
runAllExamples();
|
||||
}
|
||||
|
||||
// Export for use in other modules
|
||||
export {
|
||||
example1_OpenAIBasic,
|
||||
example2_OpenAICustomDimensions,
|
||||
example3_CohereSearchTypes,
|
||||
example4_AnthropicVoyage,
|
||||
example5_HuggingFaceLocal,
|
||||
example6_BatchProcessing,
|
||||
example7_ErrorHandling,
|
||||
example8_VectorDBInsert,
|
||||
example9_VectorDBSearch,
|
||||
example10_CompareProviders,
|
||||
example11_ProgressiveLoading,
|
||||
};
|
||||
18
vendor/ruvector/npm/packages/ruvector-extensions/src/examples/persistence-example.d.ts
vendored
Normal file
18
vendor/ruvector/npm/packages/ruvector-extensions/src/examples/persistence-example.d.ts
vendored
Normal file
@@ -0,0 +1,18 @@
|
||||
/**
|
||||
* Example usage of the Database Persistence module
|
||||
*
|
||||
* This example demonstrates all major features:
|
||||
* - Basic save/load operations
|
||||
* - Snapshot management
|
||||
* - Export/import
|
||||
* - Progress callbacks
|
||||
* - Auto-save configuration
|
||||
* - Incremental saves
|
||||
*/
|
||||
declare function example1_BasicSaveLoad(): Promise<void>;
|
||||
declare function example2_SnapshotManagement(): Promise<void>;
|
||||
declare function example3_ExportImport(): Promise<void>;
|
||||
declare function example4_AutoSaveIncremental(): Promise<void>;
|
||||
declare function example5_AdvancedProgress(): Promise<void>;
|
||||
export { example1_BasicSaveLoad, example2_SnapshotManagement, example3_ExportImport, example4_AutoSaveIncremental, example5_AdvancedProgress, };
|
||||
//# sourceMappingURL=persistence-example.d.ts.map
|
||||
1
vendor/ruvector/npm/packages/ruvector-extensions/src/examples/persistence-example.d.ts.map
vendored
Normal file
1
vendor/ruvector/npm/packages/ruvector-extensions/src/examples/persistence-example.d.ts.map
vendored
Normal file
@@ -0,0 +1 @@
|
||||
{"version":3,"file":"persistence-example.d.ts","sourceRoot":"","sources":["persistence-example.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAcH,iBAAe,sBAAsB,kBAiEpC;AAMD,iBAAe,2BAA2B,kBAmEzC;AAMD,iBAAe,qBAAqB,kBAsEnC;AAMD,iBAAe,4BAA4B,kBAwD1C;AAMD,iBAAe,yBAAyB,kBA2EvC;AA0BD,OAAO,EACL,sBAAsB,EACtB,2BAA2B,EAC3B,qBAAqB,EACrB,4BAA4B,EAC5B,yBAAyB,GAC1B,CAAC"}
|
||||
339
vendor/ruvector/npm/packages/ruvector-extensions/src/examples/persistence-example.js
vendored
Normal file
339
vendor/ruvector/npm/packages/ruvector-extensions/src/examples/persistence-example.js
vendored
Normal file
@@ -0,0 +1,339 @@
|
||||
"use strict";
|
||||
/**
|
||||
* Example usage of the Database Persistence module
|
||||
*
|
||||
* This example demonstrates all major features:
|
||||
* - Basic save/load operations
|
||||
* - Snapshot management
|
||||
* - Export/import
|
||||
* - Progress callbacks
|
||||
* - Auto-save configuration
|
||||
* - Incremental saves
|
||||
*/
|
||||
Object.defineProperty(exports, "__esModule", { value: true });
|
||||
exports.example1_BasicSaveLoad = example1_BasicSaveLoad;
|
||||
exports.example2_SnapshotManagement = example2_SnapshotManagement;
|
||||
exports.example3_ExportImport = example3_ExportImport;
|
||||
exports.example4_AutoSaveIncremental = example4_AutoSaveIncremental;
|
||||
exports.example5_AdvancedProgress = example5_AdvancedProgress;
|
||||
const ruvector_1 = require("ruvector");
|
||||
const persistence_js_1 = require("../persistence.js");
|
||||
// ============================================================================
|
||||
// Example 1: Basic Save and Load
|
||||
// ============================================================================
|
||||
async function example1_BasicSaveLoad() {
|
||||
console.log('\n=== Example 1: Basic Save and Load ===\n');
|
||||
// Create a vector database
|
||||
const db = new ruvector_1.VectorDB({
|
||||
dimension: 384,
|
||||
metric: 'cosine',
|
||||
});
|
||||
// Add some sample vectors
|
||||
console.log('Adding sample vectors...');
|
||||
for (let i = 0; i < 1000; i++) {
|
||||
db.insert({
|
||||
id: `doc-${i}`,
|
||||
vector: Array(384).fill(0).map(() => Math.random()),
|
||||
metadata: {
|
||||
category: i % 3 === 0 ? 'A' : i % 3 === 1 ? 'B' : 'C',
|
||||
timestamp: Date.now() - i * 1000,
|
||||
},
|
||||
});
|
||||
}
|
||||
console.log(`Added ${db.stats().count} vectors`);
|
||||
// Create persistence manager
|
||||
const persistence = new persistence_js_1.DatabasePersistence(db, {
|
||||
baseDir: './data/example1',
|
||||
format: 'json',
|
||||
compression: 'gzip',
|
||||
});
|
||||
// Save database with progress tracking
|
||||
console.log('\nSaving database...');
|
||||
const savePath = await persistence.save({
|
||||
onProgress: (progress) => {
|
||||
console.log(` [${progress.percentage}%] ${progress.message}`);
|
||||
},
|
||||
});
|
||||
console.log(`Saved to: ${savePath}`);
|
||||
// Create a new database and load the saved data
|
||||
const db2 = new ruvector_1.VectorDB({ dimension: 384 });
|
||||
const persistence2 = new persistence_js_1.DatabasePersistence(db2, {
|
||||
baseDir: './data/example1',
|
||||
});
|
||||
console.log('\nLoading database...');
|
||||
await persistence2.load({
|
||||
path: savePath,
|
||||
verifyChecksum: true,
|
||||
onProgress: (progress) => {
|
||||
console.log(` [${progress.percentage}%] ${progress.message}`);
|
||||
},
|
||||
});
|
||||
console.log(`Loaded ${db2.stats().count} vectors`);
|
||||
// Verify data integrity
|
||||
const original = db.get('doc-500');
|
||||
const loaded = db2.get('doc-500');
|
||||
console.log('\nData integrity check:');
|
||||
console.log(' Original metadata:', original?.metadata);
|
||||
console.log(' Loaded metadata: ', loaded?.metadata);
|
||||
console.log(' Match:', JSON.stringify(original) === JSON.stringify(loaded) ? '✓' : '✗');
|
||||
}
|
||||
// ============================================================================
|
||||
// Example 2: Snapshot Management
|
||||
// ============================================================================
|
||||
async function example2_SnapshotManagement() {
|
||||
console.log('\n=== Example 2: Snapshot Management ===\n');
|
||||
const db = new ruvector_1.VectorDB({ dimension: 128 });
|
||||
const persistence = new persistence_js_1.DatabasePersistence(db, {
|
||||
baseDir: './data/example2',
|
||||
format: 'binary',
|
||||
compression: 'gzip',
|
||||
maxSnapshots: 5,
|
||||
});
|
||||
// Create initial data
|
||||
console.log('Creating initial dataset...');
|
||||
for (let i = 0; i < 500; i++) {
|
||||
db.insert({
|
||||
id: `v${i}`,
|
||||
vector: Array(128).fill(0).map(() => Math.random()),
|
||||
});
|
||||
}
|
||||
// Create snapshot before major changes
|
||||
console.log('\nCreating snapshot "before-update"...');
|
||||
const snapshot1 = await persistence.createSnapshot('before-update', {
|
||||
description: 'Baseline before adding new vectors',
|
||||
user: 'admin',
|
||||
});
|
||||
console.log(`Snapshot created: ${snapshot1.id}`);
|
||||
console.log(` Name: ${snapshot1.name}`);
|
||||
console.log(` Vectors: ${snapshot1.vectorCount}`);
|
||||
console.log(` Size: ${(0, persistence_js_1.formatFileSize)(snapshot1.fileSize)}`);
|
||||
console.log(` Created: ${(0, persistence_js_1.formatTimestamp)(snapshot1.timestamp)}`);
|
||||
// Make changes
|
||||
console.log('\nAdding more vectors...');
|
||||
for (let i = 500; i < 1000; i++) {
|
||||
db.insert({
|
||||
id: `v${i}`,
|
||||
vector: Array(128).fill(0).map(() => Math.random()),
|
||||
});
|
||||
}
|
||||
// Create another snapshot
|
||||
console.log('\nCreating snapshot "after-update"...');
|
||||
const snapshot2 = await persistence.createSnapshot('after-update');
|
||||
console.log(`Snapshot created: ${snapshot2.id} (${snapshot2.vectorCount} vectors)`);
|
||||
// List all snapshots
|
||||
console.log('\nAll snapshots:');
|
||||
const snapshots = await persistence.listSnapshots();
|
||||
for (const snapshot of snapshots) {
|
||||
console.log(` ${snapshot.name}: ${snapshot.vectorCount} vectors, ${(0, persistence_js_1.formatFileSize)(snapshot.fileSize)}`);
|
||||
}
|
||||
// Restore from first snapshot
|
||||
console.log('\nRestoring from "before-update" snapshot...');
|
||||
await persistence.restoreSnapshot(snapshot1.id, {
|
||||
verifyChecksum: true,
|
||||
onProgress: (p) => console.log(` [${p.percentage}%] ${p.message}`),
|
||||
});
|
||||
console.log(`After restore: ${db.stats().count} vectors`);
|
||||
// Delete a snapshot
|
||||
console.log('\nDeleting snapshot...');
|
||||
await persistence.deleteSnapshot(snapshot2.id);
|
||||
console.log('Snapshot deleted');
|
||||
}
|
||||
// ============================================================================
|
||||
// Example 3: Export and Import
|
||||
// ============================================================================
|
||||
async function example3_ExportImport() {
|
||||
console.log('\n=== Example 3: Export and Import ===\n');
|
||||
// Create source database
|
||||
const sourceDb = new ruvector_1.VectorDB({ dimension: 256 });
|
||||
console.log('Creating source database...');
|
||||
for (let i = 0; i < 2000; i++) {
|
||||
sourceDb.insert({
|
||||
id: `item-${i}`,
|
||||
vector: Array(256).fill(0).map(() => Math.random()),
|
||||
metadata: {
|
||||
type: 'product',
|
||||
price: Math.random() * 100,
|
||||
rating: Math.floor(Math.random() * 5) + 1,
|
||||
},
|
||||
});
|
||||
}
|
||||
const sourcePersistence = new persistence_js_1.DatabasePersistence(sourceDb, {
|
||||
baseDir: './data/example3/source',
|
||||
});
|
||||
// Export to different formats
|
||||
console.log('\nExporting to JSON...');
|
||||
await sourcePersistence.export({
|
||||
path: './data/example3/export/database.json',
|
||||
format: 'json',
|
||||
compress: false,
|
||||
includeIndex: false,
|
||||
onProgress: (p) => console.log(` [${p.percentage}%] ${p.message}`),
|
||||
});
|
||||
console.log('\nExporting to compressed binary...');
|
||||
await sourcePersistence.export({
|
||||
path: './data/example3/export/database.bin.gz',
|
||||
format: 'binary',
|
||||
compress: true,
|
||||
includeIndex: true,
|
||||
});
|
||||
// Import into new database
|
||||
const targetDb = new ruvector_1.VectorDB({ dimension: 256 });
|
||||
const targetPersistence = new persistence_js_1.DatabasePersistence(targetDb, {
|
||||
baseDir: './data/example3/target',
|
||||
});
|
||||
console.log('\nImporting from compressed binary...');
|
||||
await targetPersistence.import({
|
||||
path: './data/example3/export/database.bin.gz',
|
||||
format: 'binary',
|
||||
clear: true,
|
||||
verifyChecksum: true,
|
||||
onProgress: (p) => console.log(` [${p.percentage}%] ${p.message}`),
|
||||
});
|
||||
console.log(`\nImport complete: ${targetDb.stats().count} vectors`);
|
||||
// Test a search to verify data integrity
|
||||
const sampleVector = sourceDb.get('item-100');
|
||||
if (sampleVector) {
|
||||
const results = targetDb.search({
|
||||
vector: sampleVector.vector,
|
||||
k: 1,
|
||||
});
|
||||
console.log('\nData integrity verification:');
|
||||
console.log(' Search for item-100:', results[0]?.id === 'item-100' ? '✓' : '✗');
|
||||
console.log(' Similarity score:', results[0]?.score.toFixed(4));
|
||||
}
|
||||
}
|
||||
// ============================================================================
|
||||
// Example 4: Auto-Save and Incremental Saves
|
||||
// ============================================================================
|
||||
async function example4_AutoSaveIncremental() {
|
||||
console.log('\n=== Example 4: Auto-Save and Incremental Saves ===\n');
|
||||
const db = new ruvector_1.VectorDB({ dimension: 64 });
|
||||
const persistence = new persistence_js_1.DatabasePersistence(db, {
|
||||
baseDir: './data/example4',
|
||||
format: 'json',
|
||||
compression: 'none',
|
||||
incremental: true,
|
||||
autoSaveInterval: 5000, // Auto-save every 5 seconds
|
||||
maxSnapshots: 3,
|
||||
});
|
||||
console.log('Auto-save enabled (every 5 seconds)');
|
||||
console.log('Incremental saves enabled');
|
||||
// Add initial batch
|
||||
console.log('\nAdding initial batch (500 vectors)...');
|
||||
for (let i = 0; i < 500; i++) {
|
||||
db.insert({
|
||||
id: `vec-${i}`,
|
||||
vector: Array(64).fill(0).map(() => Math.random()),
|
||||
});
|
||||
}
|
||||
// Manual incremental save
|
||||
console.log('\nPerforming initial save...');
|
||||
await persistence.save();
|
||||
// Simulate ongoing operations
|
||||
console.log('\nAdding more vectors...');
|
||||
for (let i = 500; i < 600; i++) {
|
||||
db.insert({
|
||||
id: `vec-${i}`,
|
||||
vector: Array(64).fill(0).map(() => Math.random()),
|
||||
});
|
||||
}
|
||||
// Incremental save (only saves changes)
|
||||
console.log('\nPerforming incremental save...');
|
||||
const incrementalPath = await persistence.saveIncremental();
|
||||
if (incrementalPath) {
|
||||
console.log(`Incremental save completed: ${incrementalPath}`);
|
||||
}
|
||||
else {
|
||||
console.log('No changes detected (skip)');
|
||||
}
|
||||
// Wait for auto-save to trigger
|
||||
console.log('\nWaiting for auto-save (5 seconds)...');
|
||||
await new Promise(resolve => setTimeout(resolve, 6000));
|
||||
// Cleanup
|
||||
console.log('\nShutting down (final save)...');
|
||||
await persistence.shutdown();
|
||||
console.log('Shutdown complete');
|
||||
}
|
||||
// ============================================================================
|
||||
// Example 5: Advanced Progress Tracking
|
||||
// ============================================================================
|
||||
async function example5_AdvancedProgress() {
|
||||
console.log('\n=== Example 5: Advanced Progress Tracking ===\n');
|
||||
const db = new ruvector_1.VectorDB({ dimension: 512 });
|
||||
// Create large dataset
|
||||
console.log('Creating large dataset (5000 vectors)...');
|
||||
const startTime = Date.now();
|
||||
for (let i = 0; i < 5000; i++) {
|
||||
db.insert({
|
||||
id: `large-${i}`,
|
||||
vector: Array(512).fill(0).map(() => Math.random()),
|
||||
metadata: {
|
||||
batch: Math.floor(i / 100),
|
||||
index: i,
|
||||
},
|
||||
});
|
||||
}
|
||||
console.log(`Dataset created in ${Date.now() - startTime}ms`);
|
||||
const persistence = new persistence_js_1.DatabasePersistence(db, {
|
||||
baseDir: './data/example5',
|
||||
format: 'binary',
|
||||
compression: 'gzip',
|
||||
batchSize: 500, // Process in batches of 500
|
||||
});
|
||||
// Custom progress handler with detailed stats
|
||||
let lastUpdate = Date.now();
|
||||
const progressHandler = (progress) => {
|
||||
const now = Date.now();
|
||||
const elapsed = now - lastUpdate;
|
||||
if (elapsed > 100) { // Update max every 100ms
|
||||
const bar = '█'.repeat(Math.floor(progress.percentage / 2)) +
|
||||
'░'.repeat(50 - Math.floor(progress.percentage / 2));
|
||||
process.stdout.write(`\r [${bar}] ${progress.percentage}% - ${progress.message}`.padEnd(100));
|
||||
lastUpdate = now;
|
||||
}
|
||||
};
|
||||
// Save with detailed progress
|
||||
console.log('\nSaving with progress tracking:');
|
||||
const saveStart = Date.now();
|
||||
await persistence.save({
|
||||
compress: true,
|
||||
onProgress: progressHandler,
|
||||
});
|
||||
console.log(`\n\nSave completed in ${Date.now() - saveStart}ms`);
|
||||
// Load with progress
|
||||
const db2 = new ruvector_1.VectorDB({ dimension: 512 });
|
||||
const persistence2 = new persistence_js_1.DatabasePersistence(db2, {
|
||||
baseDir: './data/example5',
|
||||
});
|
||||
console.log('\nLoading with progress tracking:');
|
||||
const loadStart = Date.now();
|
||||
await persistence2.load({
|
||||
path: './data/example5/database.bin.gz',
|
||||
verifyChecksum: true,
|
||||
onProgress: progressHandler,
|
||||
});
|
||||
console.log(`\n\nLoad completed in ${Date.now() - loadStart}ms`);
|
||||
console.log(`Loaded ${db2.stats().count} vectors`);
|
||||
}
|
||||
// ============================================================================
|
||||
// Run All Examples
|
||||
// ============================================================================
|
||||
async function runAllExamples() {
|
||||
try {
|
||||
await example1_BasicSaveLoad();
|
||||
await example2_SnapshotManagement();
|
||||
await example3_ExportImport();
|
||||
await example4_AutoSaveIncremental();
|
||||
await example5_AdvancedProgress();
|
||||
console.log('\n\n✓ All examples completed successfully!\n');
|
||||
}
|
||||
catch (error) {
|
||||
console.error('\n✗ Error running examples:', error);
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
// Run examples if executed directly
|
||||
if (import.meta.url === `file://${process.argv[1]}`) {
|
||||
runAllExamples();
|
||||
}
|
||||
//# sourceMappingURL=persistence-example.js.map
|
||||
1
vendor/ruvector/npm/packages/ruvector-extensions/src/examples/persistence-example.js.map
vendored
Normal file
1
vendor/ruvector/npm/packages/ruvector-extensions/src/examples/persistence-example.js.map
vendored
Normal file
File diff suppressed because one or more lines are too long
414
vendor/ruvector/npm/packages/ruvector-extensions/src/examples/persistence-example.ts
vendored
Normal file
414
vendor/ruvector/npm/packages/ruvector-extensions/src/examples/persistence-example.ts
vendored
Normal file
@@ -0,0 +1,414 @@
|
||||
/**
|
||||
* Example usage of the Database Persistence module
|
||||
*
|
||||
* This example demonstrates all major features:
|
||||
* - Basic save/load operations
|
||||
* - Snapshot management
|
||||
* - Export/import
|
||||
* - Progress callbacks
|
||||
* - Auto-save configuration
|
||||
* - Incremental saves
|
||||
*/
|
||||
|
||||
import { VectorDB } from 'ruvector';
|
||||
import {
|
||||
DatabasePersistence,
|
||||
formatFileSize,
|
||||
formatTimestamp,
|
||||
estimateMemoryUsage,
|
||||
} from '../persistence.js';
|
||||
|
||||
// ============================================================================
|
||||
// Example 1: Basic Save and Load
|
||||
// ============================================================================
|
||||
|
||||
async function example1_BasicSaveLoad() {
|
||||
console.log('\n=== Example 1: Basic Save and Load ===\n');
|
||||
|
||||
// Create a vector database
|
||||
const db = new VectorDB({
|
||||
dimension: 384,
|
||||
metric: 'cosine',
|
||||
});
|
||||
|
||||
// Add some sample vectors
|
||||
console.log('Adding sample vectors...');
|
||||
for (let i = 0; i < 1000; i++) {
|
||||
db.insert({
|
||||
id: `doc-${i}`,
|
||||
vector: Array(384).fill(0).map(() => Math.random()),
|
||||
metadata: {
|
||||
category: i % 3 === 0 ? 'A' : i % 3 === 1 ? 'B' : 'C',
|
||||
timestamp: Date.now() - i * 1000,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
console.log(`Added ${db.stats().count} vectors`);
|
||||
|
||||
// Create persistence manager
|
||||
const persistence = new DatabasePersistence(db, {
|
||||
baseDir: './data/example1',
|
||||
format: 'json',
|
||||
compression: 'gzip',
|
||||
});
|
||||
|
||||
// Save database with progress tracking
|
||||
console.log('\nSaving database...');
|
||||
const savePath = await persistence.save({
|
||||
onProgress: (progress) => {
|
||||
console.log(` [${progress.percentage}%] ${progress.message}`);
|
||||
},
|
||||
});
|
||||
|
||||
console.log(`Saved to: ${savePath}`);
|
||||
|
||||
// Create a new database and load the saved data
|
||||
const db2 = new VectorDB({ dimension: 384 });
|
||||
const persistence2 = new DatabasePersistence(db2, {
|
||||
baseDir: './data/example1',
|
||||
});
|
||||
|
||||
console.log('\nLoading database...');
|
||||
await persistence2.load({
|
||||
path: savePath,
|
||||
verifyChecksum: true,
|
||||
onProgress: (progress) => {
|
||||
console.log(` [${progress.percentage}%] ${progress.message}`);
|
||||
},
|
||||
});
|
||||
|
||||
console.log(`Loaded ${db2.stats().count} vectors`);
|
||||
|
||||
// Verify data integrity
|
||||
const original = db.get('doc-500');
|
||||
const loaded = db2.get('doc-500');
|
||||
console.log('\nData integrity check:');
|
||||
console.log(' Original metadata:', original?.metadata);
|
||||
console.log(' Loaded metadata: ', loaded?.metadata);
|
||||
console.log(' Match:', JSON.stringify(original) === JSON.stringify(loaded) ? '✓' : '✗');
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Example 2: Snapshot Management
|
||||
// ============================================================================
|
||||
|
||||
async function example2_SnapshotManagement() {
|
||||
console.log('\n=== Example 2: Snapshot Management ===\n');
|
||||
|
||||
const db = new VectorDB({ dimension: 128 });
|
||||
const persistence = new DatabasePersistence(db, {
|
||||
baseDir: './data/example2',
|
||||
format: 'binary',
|
||||
compression: 'gzip',
|
||||
maxSnapshots: 5,
|
||||
});
|
||||
|
||||
// Create initial data
|
||||
console.log('Creating initial dataset...');
|
||||
for (let i = 0; i < 500; i++) {
|
||||
db.insert({
|
||||
id: `v${i}`,
|
||||
vector: Array(128).fill(0).map(() => Math.random()),
|
||||
});
|
||||
}
|
||||
|
||||
// Create snapshot before major changes
|
||||
console.log('\nCreating snapshot "before-update"...');
|
||||
const snapshot1 = await persistence.createSnapshot('before-update', {
|
||||
description: 'Baseline before adding new vectors',
|
||||
user: 'admin',
|
||||
});
|
||||
|
||||
console.log(`Snapshot created: ${snapshot1.id}`);
|
||||
console.log(` Name: ${snapshot1.name}`);
|
||||
console.log(` Vectors: ${snapshot1.vectorCount}`);
|
||||
console.log(` Size: ${formatFileSize(snapshot1.fileSize)}`);
|
||||
console.log(` Created: ${formatTimestamp(snapshot1.timestamp)}`);
|
||||
|
||||
// Make changes
|
||||
console.log('\nAdding more vectors...');
|
||||
for (let i = 500; i < 1000; i++) {
|
||||
db.insert({
|
||||
id: `v${i}`,
|
||||
vector: Array(128).fill(0).map(() => Math.random()),
|
||||
});
|
||||
}
|
||||
|
||||
// Create another snapshot
|
||||
console.log('\nCreating snapshot "after-update"...');
|
||||
const snapshot2 = await persistence.createSnapshot('after-update');
|
||||
console.log(`Snapshot created: ${snapshot2.id} (${snapshot2.vectorCount} vectors)`);
|
||||
|
||||
// List all snapshots
|
||||
console.log('\nAll snapshots:');
|
||||
const snapshots = await persistence.listSnapshots();
|
||||
for (const snapshot of snapshots) {
|
||||
console.log(` ${snapshot.name}: ${snapshot.vectorCount} vectors, ${formatFileSize(snapshot.fileSize)}`);
|
||||
}
|
||||
|
||||
// Restore from first snapshot
|
||||
console.log('\nRestoring from "before-update" snapshot...');
|
||||
await persistence.restoreSnapshot(snapshot1.id, {
|
||||
verifyChecksum: true,
|
||||
onProgress: (p) => console.log(` [${p.percentage}%] ${p.message}`),
|
||||
});
|
||||
|
||||
console.log(`After restore: ${db.stats().count} vectors`);
|
||||
|
||||
// Delete a snapshot
|
||||
console.log('\nDeleting snapshot...');
|
||||
await persistence.deleteSnapshot(snapshot2.id);
|
||||
console.log('Snapshot deleted');
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Example 3: Export and Import
|
||||
// ============================================================================
|
||||
|
||||
async function example3_ExportImport() {
|
||||
console.log('\n=== Example 3: Export and Import ===\n');
|
||||
|
||||
// Create source database
|
||||
const sourceDb = new VectorDB({ dimension: 256 });
|
||||
console.log('Creating source database...');
|
||||
|
||||
for (let i = 0; i < 2000; i++) {
|
||||
sourceDb.insert({
|
||||
id: `item-${i}`,
|
||||
vector: Array(256).fill(0).map(() => Math.random()),
|
||||
metadata: {
|
||||
type: 'product',
|
||||
price: Math.random() * 100,
|
||||
rating: Math.floor(Math.random() * 5) + 1,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
const sourcePersistence = new DatabasePersistence(sourceDb, {
|
||||
baseDir: './data/example3/source',
|
||||
});
|
||||
|
||||
// Export to different formats
|
||||
console.log('\nExporting to JSON...');
|
||||
await sourcePersistence.export({
|
||||
path: './data/example3/export/database.json',
|
||||
format: 'json',
|
||||
compress: false,
|
||||
includeIndex: false,
|
||||
onProgress: (p) => console.log(` [${p.percentage}%] ${p.message}`),
|
||||
});
|
||||
|
||||
console.log('\nExporting to compressed binary...');
|
||||
await sourcePersistence.export({
|
||||
path: './data/example3/export/database.bin.gz',
|
||||
format: 'binary',
|
||||
compress: true,
|
||||
includeIndex: true,
|
||||
});
|
||||
|
||||
// Import into new database
|
||||
const targetDb = new VectorDB({ dimension: 256 });
|
||||
const targetPersistence = new DatabasePersistence(targetDb, {
|
||||
baseDir: './data/example3/target',
|
||||
});
|
||||
|
||||
console.log('\nImporting from compressed binary...');
|
||||
await targetPersistence.import({
|
||||
path: './data/example3/export/database.bin.gz',
|
||||
format: 'binary',
|
||||
clear: true,
|
||||
verifyChecksum: true,
|
||||
onProgress: (p) => console.log(` [${p.percentage}%] ${p.message}`),
|
||||
});
|
||||
|
||||
console.log(`\nImport complete: ${targetDb.stats().count} vectors`);
|
||||
|
||||
// Test a search to verify data integrity
|
||||
const sampleVector = sourceDb.get('item-100');
|
||||
if (sampleVector) {
|
||||
const results = targetDb.search({
|
||||
vector: sampleVector.vector,
|
||||
k: 1,
|
||||
});
|
||||
|
||||
console.log('\nData integrity verification:');
|
||||
console.log(' Search for item-100:', results[0]?.id === 'item-100' ? '✓' : '✗');
|
||||
console.log(' Similarity score:', results[0]?.score.toFixed(4));
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Example 4: Auto-Save and Incremental Saves
|
||||
// ============================================================================
|
||||
|
||||
async function example4_AutoSaveIncremental() {
|
||||
console.log('\n=== Example 4: Auto-Save and Incremental Saves ===\n');
|
||||
|
||||
const db = new VectorDB({ dimension: 64 });
|
||||
const persistence = new DatabasePersistence(db, {
|
||||
baseDir: './data/example4',
|
||||
format: 'json',
|
||||
compression: 'none',
|
||||
incremental: true,
|
||||
autoSaveInterval: 5000, // Auto-save every 5 seconds
|
||||
maxSnapshots: 3,
|
||||
});
|
||||
|
||||
console.log('Auto-save enabled (every 5 seconds)');
|
||||
console.log('Incremental saves enabled');
|
||||
|
||||
// Add initial batch
|
||||
console.log('\nAdding initial batch (500 vectors)...');
|
||||
for (let i = 0; i < 500; i++) {
|
||||
db.insert({
|
||||
id: `vec-${i}`,
|
||||
vector: Array(64).fill(0).map(() => Math.random()),
|
||||
});
|
||||
}
|
||||
|
||||
// Manual incremental save
|
||||
console.log('\nPerforming initial save...');
|
||||
await persistence.save();
|
||||
|
||||
// Simulate ongoing operations
|
||||
console.log('\nAdding more vectors...');
|
||||
for (let i = 500; i < 600; i++) {
|
||||
db.insert({
|
||||
id: `vec-${i}`,
|
||||
vector: Array(64).fill(0).map(() => Math.random()),
|
||||
});
|
||||
}
|
||||
|
||||
// Incremental save (only saves changes)
|
||||
console.log('\nPerforming incremental save...');
|
||||
const incrementalPath = await persistence.saveIncremental();
|
||||
|
||||
if (incrementalPath) {
|
||||
console.log(`Incremental save completed: ${incrementalPath}`);
|
||||
} else {
|
||||
console.log('No changes detected (skip)');
|
||||
}
|
||||
|
||||
// Wait for auto-save to trigger
|
||||
console.log('\nWaiting for auto-save (5 seconds)...');
|
||||
await new Promise(resolve => setTimeout(resolve, 6000));
|
||||
|
||||
// Cleanup
|
||||
console.log('\nShutting down (final save)...');
|
||||
await persistence.shutdown();
|
||||
console.log('Shutdown complete');
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Example 5: Advanced Progress Tracking
|
||||
// ============================================================================
|
||||
|
||||
async function example5_AdvancedProgress() {
|
||||
console.log('\n=== Example 5: Advanced Progress Tracking ===\n');
|
||||
|
||||
const db = new VectorDB({ dimension: 512 });
|
||||
|
||||
// Create large dataset
|
||||
console.log('Creating large dataset (5000 vectors)...');
|
||||
const startTime = Date.now();
|
||||
|
||||
for (let i = 0; i < 5000; i++) {
|
||||
db.insert({
|
||||
id: `large-${i}`,
|
||||
vector: Array(512).fill(0).map(() => Math.random()),
|
||||
metadata: {
|
||||
batch: Math.floor(i / 100),
|
||||
index: i,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
console.log(`Dataset created in ${Date.now() - startTime}ms`);
|
||||
|
||||
const persistence = new DatabasePersistence(db, {
|
||||
baseDir: './data/example5',
|
||||
format: 'binary',
|
||||
compression: 'gzip',
|
||||
batchSize: 500, // Process in batches of 500
|
||||
});
|
||||
|
||||
// Custom progress handler with detailed stats
|
||||
let lastUpdate = Date.now();
|
||||
const progressHandler = (progress: any) => {
|
||||
const now = Date.now();
|
||||
const elapsed = now - lastUpdate;
|
||||
|
||||
if (elapsed > 100) { // Update max every 100ms
|
||||
const bar = '█'.repeat(Math.floor(progress.percentage / 2)) +
|
||||
'░'.repeat(50 - Math.floor(progress.percentage / 2));
|
||||
|
||||
process.stdout.write(
|
||||
`\r [${bar}] ${progress.percentage}% - ${progress.message}`.padEnd(100)
|
||||
);
|
||||
|
||||
lastUpdate = now;
|
||||
}
|
||||
};
|
||||
|
||||
// Save with detailed progress
|
||||
console.log('\nSaving with progress tracking:');
|
||||
const saveStart = Date.now();
|
||||
|
||||
await persistence.save({
|
||||
compress: true,
|
||||
onProgress: progressHandler,
|
||||
});
|
||||
|
||||
console.log(`\n\nSave completed in ${Date.now() - saveStart}ms`);
|
||||
|
||||
// Load with progress
|
||||
const db2 = new VectorDB({ dimension: 512 });
|
||||
const persistence2 = new DatabasePersistence(db2, {
|
||||
baseDir: './data/example5',
|
||||
});
|
||||
|
||||
console.log('\nLoading with progress tracking:');
|
||||
const loadStart = Date.now();
|
||||
|
||||
await persistence2.load({
|
||||
path: './data/example5/database.bin.gz',
|
||||
verifyChecksum: true,
|
||||
onProgress: progressHandler,
|
||||
});
|
||||
|
||||
console.log(`\n\nLoad completed in ${Date.now() - loadStart}ms`);
|
||||
console.log(`Loaded ${db2.stats().count} vectors`);
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Run All Examples
|
||||
// ============================================================================
|
||||
|
||||
async function runAllExamples() {
|
||||
try {
|
||||
await example1_BasicSaveLoad();
|
||||
await example2_SnapshotManagement();
|
||||
await example3_ExportImport();
|
||||
await example4_AutoSaveIncremental();
|
||||
await example5_AdvancedProgress();
|
||||
|
||||
console.log('\n\n✓ All examples completed successfully!\n');
|
||||
} catch (error) {
|
||||
console.error('\n✗ Error running examples:', error);
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
// Run examples if executed directly
|
||||
if (import.meta.url === `file://${process.argv[1]}`) {
|
||||
runAllExamples();
|
||||
}
|
||||
|
||||
export {
|
||||
example1_BasicSaveLoad,
|
||||
example2_SnapshotManagement,
|
||||
example3_ExportImport,
|
||||
example4_AutoSaveIncremental,
|
||||
example5_AdvancedProgress,
|
||||
};
|
||||
49
vendor/ruvector/npm/packages/ruvector-extensions/src/examples/temporal-example.d.ts
vendored
Normal file
49
vendor/ruvector/npm/packages/ruvector-extensions/src/examples/temporal-example.d.ts
vendored
Normal file
@@ -0,0 +1,49 @@
|
||||
/**
|
||||
* Temporal Tracking Module - Usage Examples
|
||||
*
|
||||
* Demonstrates various features of the temporal tracking system
|
||||
* including version management, change tracking, time-travel queries,
|
||||
* and visualization data generation.
|
||||
*/
|
||||
/**
|
||||
* Example 1: Basic Version Management
|
||||
*/
|
||||
declare function basicVersionManagement(): Promise<void>;
|
||||
/**
|
||||
* Example 2: Time-Travel Queries
|
||||
*/
|
||||
declare function timeTravelQueries(): Promise<void>;
|
||||
/**
|
||||
* Example 3: Version Comparison and Diffing
|
||||
*/
|
||||
declare function versionComparison(): Promise<void>;
|
||||
/**
|
||||
* Example 4: Version Reverting
|
||||
*/
|
||||
declare function versionReverting(): Promise<void>;
|
||||
/**
|
||||
* Example 5: Visualization Data
|
||||
*/
|
||||
declare function visualizationData(): Promise<void>;
|
||||
/**
|
||||
* Example 6: Audit Logging
|
||||
*/
|
||||
declare function auditLogging(): Promise<void>;
|
||||
/**
|
||||
* Example 7: Storage Management
|
||||
*/
|
||||
declare function storageManagement(): Promise<void>;
|
||||
/**
|
||||
* Example 8: Backup and Restore
|
||||
*/
|
||||
declare function backupAndRestore(): Promise<void>;
|
||||
/**
|
||||
* Example 9: Event-Driven Architecture
|
||||
*/
|
||||
declare function eventDrivenArchitecture(): Promise<void>;
|
||||
/**
|
||||
* Run all examples
|
||||
*/
|
||||
declare function runAllExamples(): Promise<void>;
|
||||
export { basicVersionManagement, timeTravelQueries, versionComparison, versionReverting, visualizationData, auditLogging, storageManagement, backupAndRestore, eventDrivenArchitecture, runAllExamples };
|
||||
//# sourceMappingURL=temporal-example.d.ts.map
|
||||
1
vendor/ruvector/npm/packages/ruvector-extensions/src/examples/temporal-example.d.ts.map
vendored
Normal file
1
vendor/ruvector/npm/packages/ruvector-extensions/src/examples/temporal-example.d.ts.map
vendored
Normal file
@@ -0,0 +1 @@
|
||||
{"version":3,"file":"temporal-example.d.ts","sourceRoot":"","sources":["temporal-example.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAWH;;GAEG;AACH,iBAAe,sBAAsB,kBAiFpC;AAED;;GAEG;AACH,iBAAe,iBAAiB,kBAqD/B;AAED;;GAEG;AACH,iBAAe,iBAAiB,kBAqE/B;AAED;;GAEG;AACH,iBAAe,gBAAgB,kBA0D9B;AAED;;GAEG;AACH,iBAAe,iBAAiB,kBA8C/B;AAED;;GAEG;AACH,iBAAe,YAAY,kBAgC1B;AAED;;GAEG;AACH,iBAAe,iBAAiB,kBAyC/B;AAED;;GAEG;AACH,iBAAe,gBAAgB,kBA4C9B;AAED;;GAEG;AACH,iBAAe,uBAAuB,kBAoCrC;AAED;;GAEG;AACH,iBAAe,cAAc,kBAiB5B;AAOD,OAAO,EACL,sBAAsB,EACtB,iBAAiB,EACjB,iBAAiB,EACjB,gBAAgB,EAChB,iBAAiB,EACjB,YAAY,EACZ,iBAAiB,EACjB,gBAAgB,EAChB,uBAAuB,EACvB,cAAc,EACf,CAAC"}
|
||||
466
vendor/ruvector/npm/packages/ruvector-extensions/src/examples/temporal-example.js
vendored
Normal file
466
vendor/ruvector/npm/packages/ruvector-extensions/src/examples/temporal-example.js
vendored
Normal file
@@ -0,0 +1,466 @@
|
||||
"use strict";
|
||||
/**
|
||||
* Temporal Tracking Module - Usage Examples
|
||||
*
|
||||
* Demonstrates various features of the temporal tracking system
|
||||
* including version management, change tracking, time-travel queries,
|
||||
* and visualization data generation.
|
||||
*/
|
||||
Object.defineProperty(exports, "__esModule", { value: true });
|
||||
exports.basicVersionManagement = basicVersionManagement;
|
||||
exports.timeTravelQueries = timeTravelQueries;
|
||||
exports.versionComparison = versionComparison;
|
||||
exports.versionReverting = versionReverting;
|
||||
exports.visualizationData = visualizationData;
|
||||
exports.auditLogging = auditLogging;
|
||||
exports.storageManagement = storageManagement;
|
||||
exports.backupAndRestore = backupAndRestore;
|
||||
exports.eventDrivenArchitecture = eventDrivenArchitecture;
|
||||
exports.runAllExamples = runAllExamples;
|
||||
const temporal_js_1 = require("../temporal.js");
|
||||
/**
|
||||
* Example 1: Basic Version Management
|
||||
*/
|
||||
async function basicVersionManagement() {
|
||||
console.log('=== Example 1: Basic Version Management ===\n');
|
||||
const tracker = new temporal_js_1.TemporalTracker();
|
||||
// Create initial schema version
|
||||
tracker.trackChange({
|
||||
type: temporal_js_1.ChangeType.ADDITION,
|
||||
path: 'nodes.User',
|
||||
before: null,
|
||||
after: {
|
||||
name: 'User',
|
||||
properties: ['id', 'name', 'email']
|
||||
},
|
||||
timestamp: Date.now()
|
||||
});
|
||||
tracker.trackChange({
|
||||
type: temporal_js_1.ChangeType.ADDITION,
|
||||
path: 'edges.FOLLOWS',
|
||||
before: null,
|
||||
after: {
|
||||
name: 'FOLLOWS',
|
||||
from: 'User',
|
||||
to: 'User'
|
||||
},
|
||||
timestamp: Date.now()
|
||||
});
|
||||
const v1 = await tracker.createVersion({
|
||||
description: 'Initial schema with User nodes and FOLLOWS edges',
|
||||
tags: ['v1.0', 'production'],
|
||||
author: 'system'
|
||||
});
|
||||
console.log('Created version:', v1.id);
|
||||
console.log('Changes:', v1.changes.length);
|
||||
console.log('Tags:', v1.tags);
|
||||
console.log();
|
||||
// Add more entities
|
||||
tracker.trackChange({
|
||||
type: temporal_js_1.ChangeType.ADDITION,
|
||||
path: 'nodes.Post',
|
||||
before: null,
|
||||
after: {
|
||||
name: 'Post',
|
||||
properties: ['id', 'title', 'content', 'authorId']
|
||||
},
|
||||
timestamp: Date.now()
|
||||
});
|
||||
tracker.trackChange({
|
||||
type: temporal_js_1.ChangeType.ADDITION,
|
||||
path: 'edges.POSTED',
|
||||
before: null,
|
||||
after: {
|
||||
name: 'POSTED',
|
||||
from: 'User',
|
||||
to: 'Post'
|
||||
},
|
||||
timestamp: Date.now()
|
||||
});
|
||||
const v2 = await tracker.createVersion({
|
||||
description: 'Added Post nodes and POSTED edges',
|
||||
tags: ['v1.1'],
|
||||
author: 'developer'
|
||||
});
|
||||
console.log('Created version:', v2.id);
|
||||
console.log('Changes:', v2.changes.length);
|
||||
console.log();
|
||||
// List all versions
|
||||
const allVersions = tracker.listVersions();
|
||||
console.log('Total versions:', allVersions.length);
|
||||
allVersions.forEach(v => {
|
||||
console.log(`- ${v.description} (${v.tags.join(', ')})`);
|
||||
});
|
||||
console.log();
|
||||
}
|
||||
/**
|
||||
* Example 2: Time-Travel Queries
|
||||
*/
|
||||
async function timeTravelQueries() {
|
||||
console.log('=== Example 2: Time-Travel Queries ===\n');
|
||||
const tracker = new temporal_js_1.TemporalTracker();
|
||||
// Create multiple versions over time
|
||||
tracker.trackChange({
|
||||
type: temporal_js_1.ChangeType.ADDITION,
|
||||
path: 'config.maxUsers',
|
||||
before: null,
|
||||
after: 100,
|
||||
timestamp: Date.now()
|
||||
});
|
||||
const v1 = await tracker.createVersion({
|
||||
description: 'Set max users to 100',
|
||||
tags: ['config-v1']
|
||||
});
|
||||
console.log(`Version 1 created at ${new Date(v1.timestamp).toISOString()}`);
|
||||
// Wait a bit and make changes
|
||||
await new Promise(resolve => setTimeout(resolve, 100));
|
||||
tracker.trackChange({
|
||||
type: temporal_js_1.ChangeType.MODIFICATION,
|
||||
path: 'config.maxUsers',
|
||||
before: 100,
|
||||
after: 500,
|
||||
timestamp: Date.now()
|
||||
});
|
||||
const v2 = await tracker.createVersion({
|
||||
description: 'Increased max users to 500',
|
||||
tags: ['config-v2']
|
||||
});
|
||||
console.log(`Version 2 created at ${new Date(v2.timestamp).toISOString()}`);
|
||||
// Query at different timestamps
|
||||
const stateAtV1 = await tracker.queryAtTimestamp(v1.timestamp);
|
||||
console.log('\nState at version 1:', JSON.stringify(stateAtV1, null, 2));
|
||||
const stateAtV2 = await tracker.queryAtTimestamp(v2.timestamp);
|
||||
console.log('\nState at version 2:', JSON.stringify(stateAtV2, null, 2));
|
||||
// Query with path filter
|
||||
const configOnly = await tracker.queryAtTimestamp({
|
||||
timestamp: v2.timestamp,
|
||||
pathPattern: /^config\./
|
||||
});
|
||||
console.log('\nFiltered state (config only):', JSON.stringify(configOnly, null, 2));
|
||||
console.log();
|
||||
}
|
||||
/**
|
||||
* Example 3: Version Comparison and Diffing
|
||||
*/
|
||||
async function versionComparison() {
|
||||
console.log('=== Example 3: Version Comparison and Diffing ===\n');
|
||||
const tracker = new temporal_js_1.TemporalTracker();
|
||||
// Create initial state
|
||||
tracker.trackChange({
|
||||
type: temporal_js_1.ChangeType.ADDITION,
|
||||
path: 'schema.version',
|
||||
before: null,
|
||||
after: '1.0.0',
|
||||
timestamp: Date.now()
|
||||
});
|
||||
tracker.trackChange({
|
||||
type: temporal_js_1.ChangeType.ADDITION,
|
||||
path: 'schema.entities.User',
|
||||
before: null,
|
||||
after: { fields: ['id', 'name'] },
|
||||
timestamp: Date.now()
|
||||
});
|
||||
const v1 = await tracker.createVersion({
|
||||
description: 'Initial schema',
|
||||
tags: ['schema-v1']
|
||||
});
|
||||
// Make multiple changes
|
||||
tracker.trackChange({
|
||||
type: temporal_js_1.ChangeType.MODIFICATION,
|
||||
path: 'schema.version',
|
||||
before: '1.0.0',
|
||||
after: '2.0.0',
|
||||
timestamp: Date.now()
|
||||
});
|
||||
tracker.trackChange({
|
||||
type: temporal_js_1.ChangeType.MODIFICATION,
|
||||
path: 'schema.entities.User',
|
||||
before: { fields: ['id', 'name'] },
|
||||
after: { fields: ['id', 'name', 'email', 'createdAt'] },
|
||||
timestamp: Date.now()
|
||||
});
|
||||
tracker.trackChange({
|
||||
type: temporal_js_1.ChangeType.ADDITION,
|
||||
path: 'schema.entities.Post',
|
||||
before: null,
|
||||
after: { fields: ['id', 'title', 'content'] },
|
||||
timestamp: Date.now()
|
||||
});
|
||||
const v2 = await tracker.createVersion({
|
||||
description: 'Schema v2 with enhanced User and new Post',
|
||||
tags: ['schema-v2']
|
||||
});
|
||||
// Compare versions
|
||||
const diff = await tracker.compareVersions(v1.id, v2.id);
|
||||
console.log('Diff from v1 to v2:');
|
||||
console.log('Summary:', JSON.stringify(diff.summary, null, 2));
|
||||
console.log('\nChanges:');
|
||||
diff.changes.forEach(change => {
|
||||
console.log(`- ${change.type}: ${change.path}`);
|
||||
if (change.before !== null)
|
||||
console.log(` Before: ${JSON.stringify(change.before)}`);
|
||||
if (change.after !== null)
|
||||
console.log(` After: ${JSON.stringify(change.after)}`);
|
||||
});
|
||||
console.log();
|
||||
}
|
||||
/**
|
||||
* Example 4: Version Reverting
|
||||
*/
|
||||
async function versionReverting() {
|
||||
console.log('=== Example 4: Version Reverting ===\n');
|
||||
const tracker = new temporal_js_1.TemporalTracker();
|
||||
// Create progression of versions
|
||||
tracker.trackChange({
|
||||
type: temporal_js_1.ChangeType.ADDITION,
|
||||
path: 'feature.experimentalMode',
|
||||
before: null,
|
||||
after: false,
|
||||
timestamp: Date.now()
|
||||
});
|
||||
const v1 = await tracker.createVersion({
|
||||
description: 'Initial stable version',
|
||||
tags: ['stable', 'v1.0']
|
||||
});
|
||||
console.log('v1 created:', v1.description);
|
||||
// Enable experimental feature
|
||||
tracker.trackChange({
|
||||
type: temporal_js_1.ChangeType.MODIFICATION,
|
||||
path: 'feature.experimentalMode',
|
||||
before: false,
|
||||
after: true,
|
||||
timestamp: Date.now()
|
||||
});
|
||||
tracker.trackChange({
|
||||
type: temporal_js_1.ChangeType.ADDITION,
|
||||
path: 'feature.betaFeatures',
|
||||
before: null,
|
||||
after: ['feature1', 'feature2'],
|
||||
timestamp: Date.now()
|
||||
});
|
||||
const v2 = await tracker.createVersion({
|
||||
description: 'Experimental features enabled',
|
||||
tags: ['experimental', 'v2.0']
|
||||
});
|
||||
console.log('v2 created:', v2.description);
|
||||
// Current state
|
||||
const currentState = await tracker.queryAtTimestamp(Date.now());
|
||||
console.log('\nCurrent state:', JSON.stringify(currentState, null, 2));
|
||||
// Revert to stable version
|
||||
const revertVersion = await tracker.revertToVersion(v1.id);
|
||||
console.log('\nReverted to v1, created new version:', revertVersion.id);
|
||||
console.log('Revert description:', revertVersion.description);
|
||||
// Check state after revert
|
||||
const revertedState = await tracker.queryAtTimestamp(Date.now());
|
||||
console.log('\nState after revert:', JSON.stringify(revertedState, null, 2));
|
||||
console.log();
|
||||
}
|
||||
/**
|
||||
* Example 5: Visualization Data
|
||||
*/
|
||||
async function visualizationData() {
|
||||
console.log('=== Example 5: Visualization Data ===\n');
|
||||
const tracker = new temporal_js_1.TemporalTracker();
|
||||
// Create several versions with various changes
|
||||
for (let i = 0; i < 5; i++) {
|
||||
const changeCount = Math.floor(Math.random() * 5) + 1;
|
||||
for (let j = 0; j < changeCount; j++) {
|
||||
tracker.trackChange({
|
||||
type: [temporal_js_1.ChangeType.ADDITION, temporal_js_1.ChangeType.MODIFICATION, temporal_js_1.ChangeType.DELETION][j % 3],
|
||||
path: `data.entity${i}.field${j}`,
|
||||
before: j > 0 ? `value${j - 1}` : null,
|
||||
after: j < changeCount - 1 ? `value${j}` : null,
|
||||
timestamp: Date.now()
|
||||
});
|
||||
}
|
||||
await tracker.createVersion({
|
||||
description: `Version ${i + 1} with ${changeCount} changes`,
|
||||
tags: [`v${i + 1}`],
|
||||
author: `developer${(i % 3) + 1}`
|
||||
});
|
||||
await new Promise(resolve => setTimeout(resolve, 50));
|
||||
}
|
||||
// Get visualization data
|
||||
const vizData = tracker.getVisualizationData();
|
||||
console.log('Timeline:');
|
||||
vizData.timeline.forEach(item => {
|
||||
console.log(`- ${new Date(item.timestamp).toISOString()}: ${item.description}`);
|
||||
console.log(` Changes: ${item.changeCount}, Tags: ${item.tags.join(', ')}`);
|
||||
});
|
||||
console.log('\nTop Hotspots:');
|
||||
vizData.hotspots.slice(0, 5).forEach(hotspot => {
|
||||
console.log(`- ${hotspot.path}: ${hotspot.changeCount} changes`);
|
||||
});
|
||||
console.log('\nVersion Graph:');
|
||||
console.log('Nodes:', vizData.versionGraph.nodes.length);
|
||||
console.log('Edges:', vizData.versionGraph.edges.length);
|
||||
console.log();
|
||||
}
|
||||
/**
|
||||
* Example 6: Audit Logging
|
||||
*/
|
||||
async function auditLogging() {
|
||||
console.log('=== Example 6: Audit Logging ===\n');
|
||||
const tracker = new temporal_js_1.TemporalTracker();
|
||||
// Listen to audit events
|
||||
tracker.on('auditLogged', (entry) => {
|
||||
console.log(`[AUDIT] ${entry.operation} - ${entry.status} at ${new Date(entry.timestamp).toISOString()}`);
|
||||
});
|
||||
// Perform various operations
|
||||
tracker.trackChange({
|
||||
type: temporal_js_1.ChangeType.ADDITION,
|
||||
path: 'test.data',
|
||||
before: null,
|
||||
after: 'value',
|
||||
timestamp: Date.now()
|
||||
});
|
||||
await tracker.createVersion({
|
||||
description: 'Test version',
|
||||
tags: ['test']
|
||||
});
|
||||
// Get audit log
|
||||
const auditLog = tracker.getAuditLog(10);
|
||||
console.log('\nRecent Audit Entries:');
|
||||
auditLog.forEach(entry => {
|
||||
console.log(`- ${entry.operation}: ${entry.status}`);
|
||||
console.log(` Details:`, JSON.stringify(entry.details, null, 2));
|
||||
});
|
||||
console.log();
|
||||
}
|
||||
/**
|
||||
* Example 7: Storage Management
|
||||
*/
|
||||
async function storageManagement() {
|
||||
console.log('=== Example 7: Storage Management ===\n');
|
||||
const tracker = new temporal_js_1.TemporalTracker();
|
||||
// Create multiple versions
|
||||
for (let i = 0; i < 10; i++) {
|
||||
tracker.trackChange({
|
||||
type: temporal_js_1.ChangeType.ADDITION,
|
||||
path: `data.item${i}`,
|
||||
before: null,
|
||||
after: `value${i}`,
|
||||
timestamp: Date.now()
|
||||
});
|
||||
await tracker.createVersion({
|
||||
description: `Version ${i + 1}`,
|
||||
tags: i < 3 ? ['important'] : []
|
||||
});
|
||||
await new Promise(resolve => setTimeout(resolve, 10));
|
||||
}
|
||||
// Get storage stats before pruning
|
||||
const statsBefore = tracker.getStorageStats();
|
||||
console.log('Storage stats before pruning:');
|
||||
console.log(`- Versions: ${statsBefore.versionCount}`);
|
||||
console.log(`- Total changes: ${statsBefore.totalChanges}`);
|
||||
console.log(`- Estimated size: ${(statsBefore.estimatedSizeBytes / 1024).toFixed(2)} KB`);
|
||||
// Prune old versions, keeping last 5 and preserving tagged ones
|
||||
tracker.pruneVersions(5, ['baseline', 'important']);
|
||||
// Get storage stats after pruning
|
||||
const statsAfter = tracker.getStorageStats();
|
||||
console.log('\nStorage stats after pruning:');
|
||||
console.log(`- Versions: ${statsAfter.versionCount}`);
|
||||
console.log(`- Total changes: ${statsAfter.totalChanges}`);
|
||||
console.log(`- Estimated size: ${(statsAfter.estimatedSizeBytes / 1024).toFixed(2)} KB`);
|
||||
console.log(`- Space saved: ${((statsBefore.estimatedSizeBytes - statsAfter.estimatedSizeBytes) / 1024).toFixed(2)} KB`);
|
||||
console.log();
|
||||
}
|
||||
/**
|
||||
* Example 8: Backup and Restore
|
||||
*/
|
||||
async function backupAndRestore() {
|
||||
console.log('=== Example 8: Backup and Restore ===\n');
|
||||
const tracker1 = new temporal_js_1.TemporalTracker();
|
||||
// Create some versions
|
||||
tracker1.trackChange({
|
||||
type: temporal_js_1.ChangeType.ADDITION,
|
||||
path: 'important.data',
|
||||
before: null,
|
||||
after: { critical: true, value: 42 },
|
||||
timestamp: Date.now()
|
||||
});
|
||||
await tracker1.createVersion({
|
||||
description: 'Important data version',
|
||||
tags: ['production', 'critical']
|
||||
});
|
||||
// Export backup
|
||||
const backup = tracker1.exportBackup();
|
||||
console.log('Backup created:');
|
||||
console.log(`- Versions: ${backup.versions.length}`);
|
||||
console.log(`- Audit entries: ${backup.auditLog.length}`);
|
||||
console.log(`- Exported at: ${new Date(backup.exportedAt).toISOString()}`);
|
||||
// Create new tracker and import
|
||||
const tracker2 = new temporal_js_1.TemporalTracker();
|
||||
tracker2.importBackup(backup);
|
||||
console.log('\nBackup restored to new tracker:');
|
||||
const restoredVersions = tracker2.listVersions();
|
||||
console.log(`- Restored versions: ${restoredVersions.length}`);
|
||||
restoredVersions.forEach(v => {
|
||||
console.log(` - ${v.description} (${v.tags.join(', ')})`);
|
||||
});
|
||||
// Verify data integrity
|
||||
const originalState = await tracker1.queryAtTimestamp(Date.now());
|
||||
const restoredState = await tracker2.queryAtTimestamp(Date.now());
|
||||
console.log('\nData integrity check:');
|
||||
console.log(`- States match: ${JSON.stringify(originalState) === JSON.stringify(restoredState)}`);
|
||||
console.log();
|
||||
}
|
||||
/**
|
||||
* Example 9: Event-Driven Architecture
|
||||
*/
|
||||
async function eventDrivenArchitecture() {
|
||||
console.log('=== Example 9: Event-Driven Architecture ===\n');
|
||||
const tracker = new temporal_js_1.TemporalTracker();
|
||||
// Set up event listeners
|
||||
tracker.on('versionCreated', (version) => {
|
||||
console.log(`✓ Version created: ${version.description}`);
|
||||
console.log(` ID: ${version.id}, Changes: ${version.changes.length}`);
|
||||
});
|
||||
tracker.on('changeTracked', (change) => {
|
||||
console.log(`→ Change tracked: ${change.type} at ${change.path}`);
|
||||
});
|
||||
tracker.on('versionReverted', (fromVersion, toVersion) => {
|
||||
console.log(`⟲ Reverted from ${fromVersion} to ${toVersion}`);
|
||||
});
|
||||
// Perform operations that trigger events
|
||||
console.log('Tracking changes...');
|
||||
tracker.trackChange({
|
||||
type: temporal_js_1.ChangeType.ADDITION,
|
||||
path: 'events.example',
|
||||
before: null,
|
||||
after: 'test',
|
||||
timestamp: Date.now()
|
||||
});
|
||||
console.log('\nCreating version...');
|
||||
await tracker.createVersion({
|
||||
description: 'Event demo version',
|
||||
tags: ['demo']
|
||||
});
|
||||
console.log();
|
||||
}
|
||||
/**
|
||||
* Run all examples
|
||||
*/
|
||||
async function runAllExamples() {
|
||||
try {
|
||||
await basicVersionManagement();
|
||||
await timeTravelQueries();
|
||||
await versionComparison();
|
||||
await versionReverting();
|
||||
await visualizationData();
|
||||
await auditLogging();
|
||||
await storageManagement();
|
||||
await backupAndRestore();
|
||||
await eventDrivenArchitecture();
|
||||
console.log('✓ All examples completed successfully!');
|
||||
}
|
||||
catch (error) {
|
||||
console.error('Error running examples:', error);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
// Run if executed directly
|
||||
if (import.meta.url === `file://${process.argv[1]}`) {
|
||||
runAllExamples().catch(console.error);
|
||||
}
|
||||
//# sourceMappingURL=temporal-example.js.map
|
||||
1
vendor/ruvector/npm/packages/ruvector-extensions/src/examples/temporal-example.js.map
vendored
Normal file
1
vendor/ruvector/npm/packages/ruvector-extensions/src/examples/temporal-example.js.map
vendored
Normal file
File diff suppressed because one or more lines are too long
561
vendor/ruvector/npm/packages/ruvector-extensions/src/examples/temporal-example.ts
vendored
Normal file
561
vendor/ruvector/npm/packages/ruvector-extensions/src/examples/temporal-example.ts
vendored
Normal file
@@ -0,0 +1,561 @@
|
||||
/**
|
||||
* Temporal Tracking Module - Usage Examples
|
||||
*
|
||||
* Demonstrates various features of the temporal tracking system
|
||||
* including version management, change tracking, time-travel queries,
|
||||
* and visualization data generation.
|
||||
*/
|
||||
|
||||
import {
|
||||
TemporalTracker,
|
||||
ChangeType,
|
||||
type Change,
|
||||
type Version,
|
||||
type QueryOptions,
|
||||
type VisualizationData
|
||||
} from '../temporal.js';
|
||||
|
||||
/**
|
||||
* Example 1: Basic Version Management
|
||||
*/
|
||||
async function basicVersionManagement() {
|
||||
console.log('=== Example 1: Basic Version Management ===\n');
|
||||
|
||||
const tracker = new TemporalTracker();
|
||||
|
||||
// Create initial schema version
|
||||
tracker.trackChange({
|
||||
type: ChangeType.ADDITION,
|
||||
path: 'nodes.User',
|
||||
before: null,
|
||||
after: {
|
||||
name: 'User',
|
||||
properties: ['id', 'name', 'email']
|
||||
},
|
||||
timestamp: Date.now()
|
||||
});
|
||||
|
||||
tracker.trackChange({
|
||||
type: ChangeType.ADDITION,
|
||||
path: 'edges.FOLLOWS',
|
||||
before: null,
|
||||
after: {
|
||||
name: 'FOLLOWS',
|
||||
from: 'User',
|
||||
to: 'User'
|
||||
},
|
||||
timestamp: Date.now()
|
||||
});
|
||||
|
||||
const v1 = await tracker.createVersion({
|
||||
description: 'Initial schema with User nodes and FOLLOWS edges',
|
||||
tags: ['v1.0', 'production'],
|
||||
author: 'system'
|
||||
});
|
||||
|
||||
console.log('Created version:', v1.id);
|
||||
console.log('Changes:', v1.changes.length);
|
||||
console.log('Tags:', v1.tags);
|
||||
console.log();
|
||||
|
||||
// Add more entities
|
||||
tracker.trackChange({
|
||||
type: ChangeType.ADDITION,
|
||||
path: 'nodes.Post',
|
||||
before: null,
|
||||
after: {
|
||||
name: 'Post',
|
||||
properties: ['id', 'title', 'content', 'authorId']
|
||||
},
|
||||
timestamp: Date.now()
|
||||
});
|
||||
|
||||
tracker.trackChange({
|
||||
type: ChangeType.ADDITION,
|
||||
path: 'edges.POSTED',
|
||||
before: null,
|
||||
after: {
|
||||
name: 'POSTED',
|
||||
from: 'User',
|
||||
to: 'Post'
|
||||
},
|
||||
timestamp: Date.now()
|
||||
});
|
||||
|
||||
const v2 = await tracker.createVersion({
|
||||
description: 'Added Post nodes and POSTED edges',
|
||||
tags: ['v1.1'],
|
||||
author: 'developer'
|
||||
});
|
||||
|
||||
console.log('Created version:', v2.id);
|
||||
console.log('Changes:', v2.changes.length);
|
||||
console.log();
|
||||
|
||||
// List all versions
|
||||
const allVersions = tracker.listVersions();
|
||||
console.log('Total versions:', allVersions.length);
|
||||
allVersions.forEach(v => {
|
||||
console.log(`- ${v.description} (${v.tags.join(', ')})`);
|
||||
});
|
||||
console.log();
|
||||
}
|
||||
|
||||
/**
|
||||
* Example 2: Time-Travel Queries
|
||||
*/
|
||||
async function timeTravelQueries() {
|
||||
console.log('=== Example 2: Time-Travel Queries ===\n');
|
||||
|
||||
const tracker = new TemporalTracker();
|
||||
|
||||
// Create multiple versions over time
|
||||
tracker.trackChange({
|
||||
type: ChangeType.ADDITION,
|
||||
path: 'config.maxUsers',
|
||||
before: null,
|
||||
after: 100,
|
||||
timestamp: Date.now()
|
||||
});
|
||||
|
||||
const v1 = await tracker.createVersion({
|
||||
description: 'Set max users to 100',
|
||||
tags: ['config-v1']
|
||||
});
|
||||
|
||||
console.log(`Version 1 created at ${new Date(v1.timestamp).toISOString()}`);
|
||||
|
||||
// Wait a bit and make changes
|
||||
await new Promise(resolve => setTimeout(resolve, 100));
|
||||
|
||||
tracker.trackChange({
|
||||
type: ChangeType.MODIFICATION,
|
||||
path: 'config.maxUsers',
|
||||
before: 100,
|
||||
after: 500,
|
||||
timestamp: Date.now()
|
||||
});
|
||||
|
||||
const v2 = await tracker.createVersion({
|
||||
description: 'Increased max users to 500',
|
||||
tags: ['config-v2']
|
||||
});
|
||||
|
||||
console.log(`Version 2 created at ${new Date(v2.timestamp).toISOString()}`);
|
||||
|
||||
// Query at different timestamps
|
||||
const stateAtV1 = await tracker.queryAtTimestamp(v1.timestamp);
|
||||
console.log('\nState at version 1:', JSON.stringify(stateAtV1, null, 2));
|
||||
|
||||
const stateAtV2 = await tracker.queryAtTimestamp(v2.timestamp);
|
||||
console.log('\nState at version 2:', JSON.stringify(stateAtV2, null, 2));
|
||||
|
||||
// Query with path filter
|
||||
const configOnly = await tracker.queryAtTimestamp({
|
||||
timestamp: v2.timestamp,
|
||||
pathPattern: /^config\./
|
||||
});
|
||||
console.log('\nFiltered state (config only):', JSON.stringify(configOnly, null, 2));
|
||||
console.log();
|
||||
}
|
||||
|
||||
/**
|
||||
* Example 3: Version Comparison and Diffing
|
||||
*/
|
||||
async function versionComparison() {
|
||||
console.log('=== Example 3: Version Comparison and Diffing ===\n');
|
||||
|
||||
const tracker = new TemporalTracker();
|
||||
|
||||
// Create initial state
|
||||
tracker.trackChange({
|
||||
type: ChangeType.ADDITION,
|
||||
path: 'schema.version',
|
||||
before: null,
|
||||
after: '1.0.0',
|
||||
timestamp: Date.now()
|
||||
});
|
||||
|
||||
tracker.trackChange({
|
||||
type: ChangeType.ADDITION,
|
||||
path: 'schema.entities.User',
|
||||
before: null,
|
||||
after: { fields: ['id', 'name'] },
|
||||
timestamp: Date.now()
|
||||
});
|
||||
|
||||
const v1 = await tracker.createVersion({
|
||||
description: 'Initial schema',
|
||||
tags: ['schema-v1']
|
||||
});
|
||||
|
||||
// Make multiple changes
|
||||
tracker.trackChange({
|
||||
type: ChangeType.MODIFICATION,
|
||||
path: 'schema.version',
|
||||
before: '1.0.0',
|
||||
after: '2.0.0',
|
||||
timestamp: Date.now()
|
||||
});
|
||||
|
||||
tracker.trackChange({
|
||||
type: ChangeType.MODIFICATION,
|
||||
path: 'schema.entities.User',
|
||||
before: { fields: ['id', 'name'] },
|
||||
after: { fields: ['id', 'name', 'email', 'createdAt'] },
|
||||
timestamp: Date.now()
|
||||
});
|
||||
|
||||
tracker.trackChange({
|
||||
type: ChangeType.ADDITION,
|
||||
path: 'schema.entities.Post',
|
||||
before: null,
|
||||
after: { fields: ['id', 'title', 'content'] },
|
||||
timestamp: Date.now()
|
||||
});
|
||||
|
||||
const v2 = await tracker.createVersion({
|
||||
description: 'Schema v2 with enhanced User and new Post',
|
||||
tags: ['schema-v2']
|
||||
});
|
||||
|
||||
// Compare versions
|
||||
const diff = await tracker.compareVersions(v1.id, v2.id);
|
||||
|
||||
console.log('Diff from v1 to v2:');
|
||||
console.log('Summary:', JSON.stringify(diff.summary, null, 2));
|
||||
console.log('\nChanges:');
|
||||
diff.changes.forEach(change => {
|
||||
console.log(`- ${change.type}: ${change.path}`);
|
||||
if (change.before !== null) console.log(` Before: ${JSON.stringify(change.before)}`);
|
||||
if (change.after !== null) console.log(` After: ${JSON.stringify(change.after)}`);
|
||||
});
|
||||
console.log();
|
||||
}
|
||||
|
||||
/**
|
||||
* Example 4: Version Reverting
|
||||
*/
|
||||
async function versionReverting() {
|
||||
console.log('=== Example 4: Version Reverting ===\n');
|
||||
|
||||
const tracker = new TemporalTracker();
|
||||
|
||||
// Create progression of versions
|
||||
tracker.trackChange({
|
||||
type: ChangeType.ADDITION,
|
||||
path: 'feature.experimentalMode',
|
||||
before: null,
|
||||
after: false,
|
||||
timestamp: Date.now()
|
||||
});
|
||||
|
||||
const v1 = await tracker.createVersion({
|
||||
description: 'Initial stable version',
|
||||
tags: ['stable', 'v1.0']
|
||||
});
|
||||
|
||||
console.log('v1 created:', v1.description);
|
||||
|
||||
// Enable experimental feature
|
||||
tracker.trackChange({
|
||||
type: ChangeType.MODIFICATION,
|
||||
path: 'feature.experimentalMode',
|
||||
before: false,
|
||||
after: true,
|
||||
timestamp: Date.now()
|
||||
});
|
||||
|
||||
tracker.trackChange({
|
||||
type: ChangeType.ADDITION,
|
||||
path: 'feature.betaFeatures',
|
||||
before: null,
|
||||
after: ['feature1', 'feature2'],
|
||||
timestamp: Date.now()
|
||||
});
|
||||
|
||||
const v2 = await tracker.createVersion({
|
||||
description: 'Experimental features enabled',
|
||||
tags: ['experimental', 'v2.0']
|
||||
});
|
||||
|
||||
console.log('v2 created:', v2.description);
|
||||
|
||||
// Current state
|
||||
const currentState = await tracker.queryAtTimestamp(Date.now());
|
||||
console.log('\nCurrent state:', JSON.stringify(currentState, null, 2));
|
||||
|
||||
// Revert to stable version
|
||||
const revertVersion = await tracker.revertToVersion(v1.id);
|
||||
console.log('\nReverted to v1, created new version:', revertVersion.id);
|
||||
console.log('Revert description:', revertVersion.description);
|
||||
|
||||
// Check state after revert
|
||||
const revertedState = await tracker.queryAtTimestamp(Date.now());
|
||||
console.log('\nState after revert:', JSON.stringify(revertedState, null, 2));
|
||||
console.log();
|
||||
}
|
||||
|
||||
/**
|
||||
* Example 5: Visualization Data
|
||||
*/
|
||||
async function visualizationData() {
|
||||
console.log('=== Example 5: Visualization Data ===\n');
|
||||
|
||||
const tracker = new TemporalTracker();
|
||||
|
||||
// Create several versions with various changes
|
||||
for (let i = 0; i < 5; i++) {
|
||||
const changeCount = Math.floor(Math.random() * 5) + 1;
|
||||
|
||||
for (let j = 0; j < changeCount; j++) {
|
||||
tracker.trackChange({
|
||||
type: [ChangeType.ADDITION, ChangeType.MODIFICATION, ChangeType.DELETION][j % 3],
|
||||
path: `data.entity${i}.field${j}`,
|
||||
before: j > 0 ? `value${j - 1}` : null,
|
||||
after: j < changeCount - 1 ? `value${j}` : null,
|
||||
timestamp: Date.now()
|
||||
});
|
||||
}
|
||||
|
||||
await tracker.createVersion({
|
||||
description: `Version ${i + 1} with ${changeCount} changes`,
|
||||
tags: [`v${i + 1}`],
|
||||
author: `developer${(i % 3) + 1}`
|
||||
});
|
||||
|
||||
await new Promise(resolve => setTimeout(resolve, 50));
|
||||
}
|
||||
|
||||
// Get visualization data
|
||||
const vizData = tracker.getVisualizationData();
|
||||
|
||||
console.log('Timeline:');
|
||||
vizData.timeline.forEach(item => {
|
||||
console.log(`- ${new Date(item.timestamp).toISOString()}: ${item.description}`);
|
||||
console.log(` Changes: ${item.changeCount}, Tags: ${item.tags.join(', ')}`);
|
||||
});
|
||||
|
||||
console.log('\nTop Hotspots:');
|
||||
vizData.hotspots.slice(0, 5).forEach(hotspot => {
|
||||
console.log(`- ${hotspot.path}: ${hotspot.changeCount} changes`);
|
||||
});
|
||||
|
||||
console.log('\nVersion Graph:');
|
||||
console.log('Nodes:', vizData.versionGraph.nodes.length);
|
||||
console.log('Edges:', vizData.versionGraph.edges.length);
|
||||
console.log();
|
||||
}
|
||||
|
||||
/**
|
||||
* Example 6: Audit Logging
|
||||
*/
|
||||
async function auditLogging() {
|
||||
console.log('=== Example 6: Audit Logging ===\n');
|
||||
|
||||
const tracker = new TemporalTracker();
|
||||
|
||||
// Listen to audit events
|
||||
tracker.on('auditLogged', (entry) => {
|
||||
console.log(`[AUDIT] ${entry.operation} - ${entry.status} at ${new Date(entry.timestamp).toISOString()}`);
|
||||
});
|
||||
|
||||
// Perform various operations
|
||||
tracker.trackChange({
|
||||
type: ChangeType.ADDITION,
|
||||
path: 'test.data',
|
||||
before: null,
|
||||
after: 'value',
|
||||
timestamp: Date.now()
|
||||
});
|
||||
|
||||
await tracker.createVersion({
|
||||
description: 'Test version',
|
||||
tags: ['test']
|
||||
});
|
||||
|
||||
// Get audit log
|
||||
const auditLog = tracker.getAuditLog(10);
|
||||
console.log('\nRecent Audit Entries:');
|
||||
auditLog.forEach(entry => {
|
||||
console.log(`- ${entry.operation}: ${entry.status}`);
|
||||
console.log(` Details:`, JSON.stringify(entry.details, null, 2));
|
||||
});
|
||||
console.log();
|
||||
}
|
||||
|
||||
/**
|
||||
* Example 7: Storage Management
|
||||
*/
|
||||
async function storageManagement() {
|
||||
console.log('=== Example 7: Storage Management ===\n');
|
||||
|
||||
const tracker = new TemporalTracker();
|
||||
|
||||
// Create multiple versions
|
||||
for (let i = 0; i < 10; i++) {
|
||||
tracker.trackChange({
|
||||
type: ChangeType.ADDITION,
|
||||
path: `data.item${i}`,
|
||||
before: null,
|
||||
after: `value${i}`,
|
||||
timestamp: Date.now()
|
||||
});
|
||||
|
||||
await tracker.createVersion({
|
||||
description: `Version ${i + 1}`,
|
||||
tags: i < 3 ? ['important'] : []
|
||||
});
|
||||
|
||||
await new Promise(resolve => setTimeout(resolve, 10));
|
||||
}
|
||||
|
||||
// Get storage stats before pruning
|
||||
const statsBefore = tracker.getStorageStats();
|
||||
console.log('Storage stats before pruning:');
|
||||
console.log(`- Versions: ${statsBefore.versionCount}`);
|
||||
console.log(`- Total changes: ${statsBefore.totalChanges}`);
|
||||
console.log(`- Estimated size: ${(statsBefore.estimatedSizeBytes / 1024).toFixed(2)} KB`);
|
||||
|
||||
// Prune old versions, keeping last 5 and preserving tagged ones
|
||||
tracker.pruneVersions(5, ['baseline', 'important']);
|
||||
|
||||
// Get storage stats after pruning
|
||||
const statsAfter = tracker.getStorageStats();
|
||||
console.log('\nStorage stats after pruning:');
|
||||
console.log(`- Versions: ${statsAfter.versionCount}`);
|
||||
console.log(`- Total changes: ${statsAfter.totalChanges}`);
|
||||
console.log(`- Estimated size: ${(statsAfter.estimatedSizeBytes / 1024).toFixed(2)} KB`);
|
||||
console.log(`- Space saved: ${((statsBefore.estimatedSizeBytes - statsAfter.estimatedSizeBytes) / 1024).toFixed(2)} KB`);
|
||||
console.log();
|
||||
}
|
||||
|
||||
/**
|
||||
* Example 8: Backup and Restore
|
||||
*/
|
||||
async function backupAndRestore() {
|
||||
console.log('=== Example 8: Backup and Restore ===\n');
|
||||
|
||||
const tracker1 = new TemporalTracker();
|
||||
|
||||
// Create some versions
|
||||
tracker1.trackChange({
|
||||
type: ChangeType.ADDITION,
|
||||
path: 'important.data',
|
||||
before: null,
|
||||
after: { critical: true, value: 42 },
|
||||
timestamp: Date.now()
|
||||
});
|
||||
|
||||
await tracker1.createVersion({
|
||||
description: 'Important data version',
|
||||
tags: ['production', 'critical']
|
||||
});
|
||||
|
||||
// Export backup
|
||||
const backup = tracker1.exportBackup();
|
||||
console.log('Backup created:');
|
||||
console.log(`- Versions: ${backup.versions.length}`);
|
||||
console.log(`- Audit entries: ${backup.auditLog.length}`);
|
||||
console.log(`- Exported at: ${new Date(backup.exportedAt).toISOString()}`);
|
||||
|
||||
// Create new tracker and import
|
||||
const tracker2 = new TemporalTracker();
|
||||
tracker2.importBackup(backup);
|
||||
|
||||
console.log('\nBackup restored to new tracker:');
|
||||
const restoredVersions = tracker2.listVersions();
|
||||
console.log(`- Restored versions: ${restoredVersions.length}`);
|
||||
restoredVersions.forEach(v => {
|
||||
console.log(` - ${v.description} (${v.tags.join(', ')})`);
|
||||
});
|
||||
|
||||
// Verify data integrity
|
||||
const originalState = await tracker1.queryAtTimestamp(Date.now());
|
||||
const restoredState = await tracker2.queryAtTimestamp(Date.now());
|
||||
|
||||
console.log('\nData integrity check:');
|
||||
console.log(`- States match: ${JSON.stringify(originalState) === JSON.stringify(restoredState)}`);
|
||||
console.log();
|
||||
}
|
||||
|
||||
/**
|
||||
* Example 9: Event-Driven Architecture
|
||||
*/
|
||||
async function eventDrivenArchitecture() {
|
||||
console.log('=== Example 9: Event-Driven Architecture ===\n');
|
||||
|
||||
const tracker = new TemporalTracker();
|
||||
|
||||
// Set up event listeners
|
||||
tracker.on('versionCreated', (version) => {
|
||||
console.log(`✓ Version created: ${version.description}`);
|
||||
console.log(` ID: ${version.id}, Changes: ${version.changes.length}`);
|
||||
});
|
||||
|
||||
tracker.on('changeTracked', (change) => {
|
||||
console.log(`→ Change tracked: ${change.type} at ${change.path}`);
|
||||
});
|
||||
|
||||
tracker.on('versionReverted', (fromVersion, toVersion) => {
|
||||
console.log(`⟲ Reverted from ${fromVersion} to ${toVersion}`);
|
||||
});
|
||||
|
||||
// Perform operations that trigger events
|
||||
console.log('Tracking changes...');
|
||||
tracker.trackChange({
|
||||
type: ChangeType.ADDITION,
|
||||
path: 'events.example',
|
||||
before: null,
|
||||
after: 'test',
|
||||
timestamp: Date.now()
|
||||
});
|
||||
|
||||
console.log('\nCreating version...');
|
||||
await tracker.createVersion({
|
||||
description: 'Event demo version',
|
||||
tags: ['demo']
|
||||
});
|
||||
|
||||
console.log();
|
||||
}
|
||||
|
||||
/**
|
||||
* Run all examples
|
||||
*/
|
||||
async function runAllExamples() {
|
||||
try {
|
||||
await basicVersionManagement();
|
||||
await timeTravelQueries();
|
||||
await versionComparison();
|
||||
await versionReverting();
|
||||
await visualizationData();
|
||||
await auditLogging();
|
||||
await storageManagement();
|
||||
await backupAndRestore();
|
||||
await eventDrivenArchitecture();
|
||||
|
||||
console.log('✓ All examples completed successfully!');
|
||||
} catch (error) {
|
||||
console.error('Error running examples:', error);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
// Run if executed directly
|
||||
if (import.meta.url === `file://${process.argv[1]}`) {
|
||||
runAllExamples().catch(console.error);
|
||||
}
|
||||
|
||||
export {
|
||||
basicVersionManagement,
|
||||
timeTravelQueries,
|
||||
versionComparison,
|
||||
versionReverting,
|
||||
visualizationData,
|
||||
auditLogging,
|
||||
storageManagement,
|
||||
backupAndRestore,
|
||||
eventDrivenArchitecture,
|
||||
runAllExamples
|
||||
};
|
||||
2
vendor/ruvector/npm/packages/ruvector-extensions/src/examples/ui-example.d.ts
vendored
Normal file
2
vendor/ruvector/npm/packages/ruvector-extensions/src/examples/ui-example.d.ts
vendored
Normal file
@@ -0,0 +1,2 @@
|
||||
export {};
|
||||
//# sourceMappingURL=ui-example.d.ts.map
|
||||
1
vendor/ruvector/npm/packages/ruvector-extensions/src/examples/ui-example.d.ts.map
vendored
Normal file
1
vendor/ruvector/npm/packages/ruvector-extensions/src/examples/ui-example.d.ts.map
vendored
Normal file
@@ -0,0 +1 @@
|
||||
{"version":3,"file":"ui-example.d.ts","sourceRoot":"","sources":["ui-example.ts"],"names":[],"mappings":""}
|
||||
121
vendor/ruvector/npm/packages/ruvector-extensions/src/examples/ui-example.js
vendored
Normal file
121
vendor/ruvector/npm/packages/ruvector-extensions/src/examples/ui-example.js
vendored
Normal file
@@ -0,0 +1,121 @@
|
||||
"use strict";
|
||||
Object.defineProperty(exports, "__esModule", { value: true });
|
||||
const ruvector_1 = require("ruvector");
|
||||
const ui_server_js_1 = require("../ui-server.js");
|
||||
/**
|
||||
* Example: Interactive Graph Explorer UI
|
||||
*
|
||||
* This example demonstrates how to launch the interactive web UI
|
||||
* for exploring vector embeddings as a force-directed graph.
|
||||
*/
|
||||
async function main() {
|
||||
console.log('🚀 Starting RuVector Graph Explorer Example\n');
|
||||
// Initialize database
|
||||
const db = new ruvector_1.VectorDB({
|
||||
dimension: 384,
|
||||
distanceMetric: 'cosine'
|
||||
});
|
||||
console.log('📊 Populating database with sample data...\n');
|
||||
// Create sample embeddings with different categories
|
||||
const categories = ['research', 'code', 'documentation', 'test'];
|
||||
const sampleData = [];
|
||||
for (let i = 0; i < 50; i++) {
|
||||
const category = categories[i % categories.length];
|
||||
// Generate random embedding with some structure
|
||||
const baseVector = Array.from({ length: 384 }, () => Math.random() - 0.5);
|
||||
// Add category-specific bias to make similar items cluster
|
||||
const categoryBias = i % categories.length;
|
||||
for (let j = 0; j < 96; j++) {
|
||||
baseVector[j + categoryBias * 96] += 0.5;
|
||||
}
|
||||
// Normalize vector
|
||||
const magnitude = Math.sqrt(baseVector.reduce((sum, val) => sum + val * val, 0));
|
||||
const embedding = baseVector.map(val => val / magnitude);
|
||||
const id = `node-${i.toString().padStart(3, '0')}`;
|
||||
const metadata = {
|
||||
label: `${category} ${i}`,
|
||||
category,
|
||||
timestamp: Date.now() - Math.random() * 86400000 * 30,
|
||||
importance: Math.random(),
|
||||
tags: [category, `tag-${Math.floor(Math.random() * 5)}`]
|
||||
};
|
||||
sampleData.push({ id, embedding, metadata });
|
||||
}
|
||||
// Add all vectors to database
|
||||
for (const { id, embedding, metadata } of sampleData) {
|
||||
await db.add(id, embedding, metadata);
|
||||
}
|
||||
console.log(`✅ Added ${sampleData.length} sample nodes\n`);
|
||||
// Get database statistics
|
||||
const stats = await db.getStats();
|
||||
console.log('📈 Database Statistics:');
|
||||
console.log(` Total vectors: ${stats.totalVectors}`);
|
||||
console.log(` Dimension: ${stats.dimension}`);
|
||||
console.log(` Distance metric: ${stats.distanceMetric}\n`);
|
||||
// Start UI server
|
||||
console.log('🌐 Starting UI server...\n');
|
||||
const port = parseInt(process.env.PORT || '3000');
|
||||
const server = await (0, ui_server_js_1.startUIServer)(db, port);
|
||||
console.log('✨ UI Features:');
|
||||
console.log(' • Interactive force-directed graph visualization');
|
||||
console.log(' • Drag nodes to reposition');
|
||||
console.log(' • Zoom and pan with mouse/touch');
|
||||
console.log(' • Search nodes by ID or metadata');
|
||||
console.log(' • Click nodes to view metadata');
|
||||
console.log(' • Double-click or use "Find Similar" to highlight similar nodes');
|
||||
console.log(' • Export graph as PNG or SVG');
|
||||
console.log(' • Real-time updates via WebSocket');
|
||||
console.log(' • Responsive design for mobile devices\n');
|
||||
console.log('💡 Try these actions:');
|
||||
console.log(' 1. Search for "research" to filter nodes');
|
||||
console.log(' 2. Click any node to see its metadata');
|
||||
console.log(' 3. Click "Find Similar Nodes" to discover connections');
|
||||
console.log(' 4. Adjust the similarity threshold slider');
|
||||
console.log(' 5. Export the visualization as PNG or SVG\n');
|
||||
// Demonstrate adding nodes in real-time
|
||||
console.log('🔄 Adding nodes in real-time (every 10 seconds)...\n');
|
||||
let counter = 50;
|
||||
const interval = setInterval(async () => {
|
||||
const category = categories[counter % categories.length];
|
||||
const baseVector = Array.from({ length: 384 }, () => Math.random() - 0.5);
|
||||
const categoryBias = counter % categories.length;
|
||||
for (let j = 0; j < 96; j++) {
|
||||
baseVector[j + categoryBias * 96] += 0.5;
|
||||
}
|
||||
const magnitude = Math.sqrt(baseVector.reduce((sum, val) => sum + val * val, 0));
|
||||
const embedding = baseVector.map(val => val / magnitude);
|
||||
const id = `node-${counter.toString().padStart(3, '0')}`;
|
||||
const metadata = {
|
||||
label: `${category} ${counter}`,
|
||||
category,
|
||||
timestamp: Date.now(),
|
||||
importance: Math.random(),
|
||||
tags: [category, `tag-${Math.floor(Math.random() * 5)}`]
|
||||
};
|
||||
await db.add(id, embedding, metadata);
|
||||
// Notify UI of update
|
||||
server.notifyGraphUpdate();
|
||||
console.log(`✅ Added new node: ${id} (${category})`);
|
||||
counter++;
|
||||
// Stop after adding 10 more nodes
|
||||
if (counter >= 60) {
|
||||
clearInterval(interval);
|
||||
console.log('\n✨ Real-time updates complete!\n');
|
||||
}
|
||||
}, 10000);
|
||||
// Handle graceful shutdown
|
||||
process.on('SIGINT', async () => {
|
||||
console.log('\n\n🛑 Shutting down gracefully...');
|
||||
clearInterval(interval);
|
||||
await server.stop();
|
||||
await db.close();
|
||||
console.log('👋 Goodbye!\n');
|
||||
process.exit(0);
|
||||
});
|
||||
}
|
||||
// Run example
|
||||
main().catch(error => {
|
||||
console.error('❌ Error:', error);
|
||||
process.exit(1);
|
||||
});
|
||||
//# sourceMappingURL=ui-example.js.map
|
||||
1
vendor/ruvector/npm/packages/ruvector-extensions/src/examples/ui-example.js.map
vendored
Normal file
1
vendor/ruvector/npm/packages/ruvector-extensions/src/examples/ui-example.js.map
vendored
Normal file
File diff suppressed because one or more lines are too long
146
vendor/ruvector/npm/packages/ruvector-extensions/src/examples/ui-example.ts
vendored
Normal file
146
vendor/ruvector/npm/packages/ruvector-extensions/src/examples/ui-example.ts
vendored
Normal file
@@ -0,0 +1,146 @@
|
||||
import { VectorDB } from 'ruvector';
|
||||
import { startUIServer } from '../ui-server.js';
|
||||
|
||||
/**
|
||||
* Example: Interactive Graph Explorer UI
|
||||
*
|
||||
* This example demonstrates how to launch the interactive web UI
|
||||
* for exploring vector embeddings as a force-directed graph.
|
||||
*/
|
||||
async function main() {
|
||||
console.log('🚀 Starting RuVector Graph Explorer Example\n');
|
||||
|
||||
// Initialize database
|
||||
const db = new VectorDB({
|
||||
dimension: 384,
|
||||
distanceMetric: 'cosine'
|
||||
});
|
||||
|
||||
console.log('📊 Populating database with sample data...\n');
|
||||
|
||||
// Create sample embeddings with different categories
|
||||
const categories = ['research', 'code', 'documentation', 'test'];
|
||||
const sampleData = [];
|
||||
|
||||
for (let i = 0; i < 50; i++) {
|
||||
const category = categories[i % categories.length];
|
||||
|
||||
// Generate random embedding with some structure
|
||||
const baseVector = Array.from({ length: 384 }, () => Math.random() - 0.5);
|
||||
|
||||
// Add category-specific bias to make similar items cluster
|
||||
const categoryBias = i % categories.length;
|
||||
for (let j = 0; j < 96; j++) {
|
||||
baseVector[j + categoryBias * 96] += 0.5;
|
||||
}
|
||||
|
||||
// Normalize vector
|
||||
const magnitude = Math.sqrt(baseVector.reduce((sum, val) => sum + val * val, 0));
|
||||
const embedding = baseVector.map(val => val / magnitude);
|
||||
|
||||
const id = `node-${i.toString().padStart(3, '0')}`;
|
||||
const metadata = {
|
||||
label: `${category} ${i}`,
|
||||
category,
|
||||
timestamp: Date.now() - Math.random() * 86400000 * 30,
|
||||
importance: Math.random(),
|
||||
tags: [category, `tag-${Math.floor(Math.random() * 5)}`]
|
||||
};
|
||||
|
||||
sampleData.push({ id, embedding, metadata });
|
||||
}
|
||||
|
||||
// Add all vectors to database
|
||||
for (const { id, embedding, metadata } of sampleData) {
|
||||
await db.add(id, embedding, metadata);
|
||||
}
|
||||
|
||||
console.log(`✅ Added ${sampleData.length} sample nodes\n`);
|
||||
|
||||
// Get database statistics
|
||||
const stats = await db.getStats();
|
||||
console.log('📈 Database Statistics:');
|
||||
console.log(` Total vectors: ${stats.totalVectors}`);
|
||||
console.log(` Dimension: ${stats.dimension}`);
|
||||
console.log(` Distance metric: ${stats.distanceMetric}\n`);
|
||||
|
||||
// Start UI server
|
||||
console.log('🌐 Starting UI server...\n');
|
||||
|
||||
const port = parseInt(process.env.PORT || '3000');
|
||||
const server = await startUIServer(db, port);
|
||||
|
||||
console.log('✨ UI Features:');
|
||||
console.log(' • Interactive force-directed graph visualization');
|
||||
console.log(' • Drag nodes to reposition');
|
||||
console.log(' • Zoom and pan with mouse/touch');
|
||||
console.log(' • Search nodes by ID or metadata');
|
||||
console.log(' • Click nodes to view metadata');
|
||||
console.log(' • Double-click or use "Find Similar" to highlight similar nodes');
|
||||
console.log(' • Export graph as PNG or SVG');
|
||||
console.log(' • Real-time updates via WebSocket');
|
||||
console.log(' • Responsive design for mobile devices\n');
|
||||
|
||||
console.log('💡 Try these actions:');
|
||||
console.log(' 1. Search for "research" to filter nodes');
|
||||
console.log(' 2. Click any node to see its metadata');
|
||||
console.log(' 3. Click "Find Similar Nodes" to discover connections');
|
||||
console.log(' 4. Adjust the similarity threshold slider');
|
||||
console.log(' 5. Export the visualization as PNG or SVG\n');
|
||||
|
||||
// Demonstrate adding nodes in real-time
|
||||
console.log('🔄 Adding nodes in real-time (every 10 seconds)...\n');
|
||||
|
||||
let counter = 50;
|
||||
const interval = setInterval(async () => {
|
||||
const category = categories[counter % categories.length];
|
||||
const baseVector = Array.from({ length: 384 }, () => Math.random() - 0.5);
|
||||
|
||||
const categoryBias = counter % categories.length;
|
||||
for (let j = 0; j < 96; j++) {
|
||||
baseVector[j + categoryBias * 96] += 0.5;
|
||||
}
|
||||
|
||||
const magnitude = Math.sqrt(baseVector.reduce((sum, val) => sum + val * val, 0));
|
||||
const embedding = baseVector.map(val => val / magnitude);
|
||||
|
||||
const id = `node-${counter.toString().padStart(3, '0')}`;
|
||||
const metadata = {
|
||||
label: `${category} ${counter}`,
|
||||
category,
|
||||
timestamp: Date.now(),
|
||||
importance: Math.random(),
|
||||
tags: [category, `tag-${Math.floor(Math.random() * 5)}`]
|
||||
};
|
||||
|
||||
await db.add(id, embedding, metadata);
|
||||
|
||||
// Notify UI of update
|
||||
server.notifyGraphUpdate();
|
||||
|
||||
console.log(`✅ Added new node: ${id} (${category})`);
|
||||
counter++;
|
||||
|
||||
// Stop after adding 10 more nodes
|
||||
if (counter >= 60) {
|
||||
clearInterval(interval);
|
||||
console.log('\n✨ Real-time updates complete!\n');
|
||||
}
|
||||
}, 10000);
|
||||
|
||||
// Handle graceful shutdown
|
||||
process.on('SIGINT', async () => {
|
||||
console.log('\n\n🛑 Shutting down gracefully...');
|
||||
clearInterval(interval);
|
||||
await server.stop();
|
||||
await db.close();
|
||||
console.log('👋 Goodbye!\n');
|
||||
process.exit(0);
|
||||
});
|
||||
}
|
||||
|
||||
// Run example
|
||||
main().catch(error => {
|
||||
console.error('❌ Error:', error);
|
||||
process.exit(1);
|
||||
});
|
||||
399
vendor/ruvector/npm/packages/ruvector-extensions/src/exporters.d.ts
vendored
Normal file
399
vendor/ruvector/npm/packages/ruvector-extensions/src/exporters.d.ts
vendored
Normal file
@@ -0,0 +1,399 @@
|
||||
/**
|
||||
* Graph Export Module for ruvector-extensions
|
||||
*
|
||||
* Provides export functionality to multiple graph formats:
|
||||
* - GraphML (XML-based graph format)
|
||||
* - GEXF (Graph Exchange XML Format for Gephi)
|
||||
* - Neo4j (Cypher queries)
|
||||
* - D3.js JSON (web visualization)
|
||||
* - NetworkX (Python graph library)
|
||||
*
|
||||
* Features:
|
||||
* - Full TypeScript types and interfaces
|
||||
* - Streaming exports for large graphs
|
||||
* - Configurable export options
|
||||
* - Support for node attributes and edge weights
|
||||
* - Error handling and validation
|
||||
*
|
||||
* @module exporters
|
||||
*/
|
||||
import { Writable } from 'stream';
|
||||
import type { VectorEntry } from 'ruvector';
|
||||
type VectorDBInstance = any;
|
||||
/**
|
||||
* Graph node representing a vector entry
|
||||
*/
|
||||
export interface GraphNode {
|
||||
/** Unique node identifier */
|
||||
id: string;
|
||||
/** Node label/name */
|
||||
label?: string;
|
||||
/** Vector embedding */
|
||||
vector?: number[];
|
||||
/** Node attributes/metadata */
|
||||
attributes?: Record<string, any>;
|
||||
}
|
||||
/**
|
||||
* Graph edge representing similarity between nodes
|
||||
*/
|
||||
export interface GraphEdge {
|
||||
/** Source node ID */
|
||||
source: string;
|
||||
/** Target node ID */
|
||||
target: string;
|
||||
/** Edge weight (similarity score) */
|
||||
weight: number;
|
||||
/** Edge type/label */
|
||||
type?: string;
|
||||
/** Edge attributes */
|
||||
attributes?: Record<string, any>;
|
||||
}
|
||||
/**
|
||||
* Complete graph structure
|
||||
*/
|
||||
export interface Graph {
|
||||
/** Graph nodes */
|
||||
nodes: GraphNode[];
|
||||
/** Graph edges */
|
||||
edges: GraphEdge[];
|
||||
/** Graph-level metadata */
|
||||
metadata?: Record<string, any>;
|
||||
}
|
||||
/**
|
||||
* Export configuration options
|
||||
*/
|
||||
export interface ExportOptions {
|
||||
/** Include vector embeddings in export */
|
||||
includeVectors?: boolean;
|
||||
/** Include metadata/attributes */
|
||||
includeMetadata?: boolean;
|
||||
/** Maximum number of neighbors per node */
|
||||
maxNeighbors?: number;
|
||||
/** Minimum similarity threshold for edges */
|
||||
threshold?: number;
|
||||
/** Graph title/name */
|
||||
graphName?: string;
|
||||
/** Graph description */
|
||||
graphDescription?: string;
|
||||
/** Enable streaming mode for large graphs */
|
||||
streaming?: boolean;
|
||||
/** Custom attribute mappings */
|
||||
attributeMapping?: Record<string, string>;
|
||||
}
|
||||
/**
|
||||
* Export format types
|
||||
*/
|
||||
export type ExportFormat = 'graphml' | 'gexf' | 'neo4j' | 'd3' | 'networkx';
|
||||
/**
|
||||
* Export result containing output and metadata
|
||||
*/
|
||||
export interface ExportResult {
|
||||
/** Export format used */
|
||||
format: ExportFormat;
|
||||
/** Exported data (string or object depending on format) */
|
||||
data: string | object;
|
||||
/** Number of nodes exported */
|
||||
nodeCount: number;
|
||||
/** Number of edges exported */
|
||||
edgeCount: number;
|
||||
/** Export metadata */
|
||||
metadata?: Record<string, any>;
|
||||
}
|
||||
/**
|
||||
* Build a graph from VectorDB by computing similarity between vectors
|
||||
*
|
||||
* @param db - VectorDB instance
|
||||
* @param options - Export options
|
||||
* @returns Graph structure
|
||||
*
|
||||
* @example
|
||||
* ```typescript
|
||||
* const graph = buildGraphFromVectorDB(db, {
|
||||
* maxNeighbors: 5,
|
||||
* threshold: 0.7,
|
||||
* includeVectors: false
|
||||
* });
|
||||
* ```
|
||||
*/
|
||||
export declare function buildGraphFromVectorDB(db: VectorDBInstance, options?: ExportOptions): Graph;
|
||||
/**
|
||||
* Build a graph from a list of vector entries
|
||||
*
|
||||
* @param entries - Array of vector entries
|
||||
* @param options - Export options
|
||||
* @returns Graph structure
|
||||
*
|
||||
* @example
|
||||
* ```typescript
|
||||
* const entries = [...]; // Your vector entries
|
||||
* const graph = buildGraphFromEntries(entries, {
|
||||
* maxNeighbors: 5,
|
||||
* threshold: 0.7
|
||||
* });
|
||||
* ```
|
||||
*/
|
||||
export declare function buildGraphFromEntries(entries: VectorEntry[], options?: ExportOptions): Graph;
|
||||
/**
|
||||
* Compute cosine similarity between two vectors
|
||||
*/
|
||||
declare function cosineSimilarity(a: number[], b: number[]): number;
|
||||
/**
|
||||
* Export graph to GraphML format (XML-based)
|
||||
*
|
||||
* GraphML is a comprehensive and easy-to-use file format for graphs.
|
||||
* It's supported by many graph analysis tools including Gephi, NetworkX, and igraph.
|
||||
*
|
||||
* @param graph - Graph to export
|
||||
* @param options - Export options
|
||||
* @returns GraphML XML string
|
||||
*
|
||||
* @example
|
||||
* ```typescript
|
||||
* const graphml = exportToGraphML(graph, {
|
||||
* graphName: 'Vector Similarity Graph',
|
||||
* includeVectors: false
|
||||
* });
|
||||
* console.log(graphml);
|
||||
* ```
|
||||
*/
|
||||
export declare function exportToGraphML(graph: Graph, options?: ExportOptions): string;
|
||||
/**
|
||||
* Stream graph to GraphML format
|
||||
*
|
||||
* @param graph - Graph to export
|
||||
* @param stream - Writable stream
|
||||
* @param options - Export options
|
||||
*
|
||||
* @example
|
||||
* ```typescript
|
||||
* import { createWriteStream } from 'fs';
|
||||
* const stream = createWriteStream('graph.graphml');
|
||||
* await streamToGraphML(graph, stream);
|
||||
* ```
|
||||
*/
|
||||
export declare function streamToGraphML(graph: Graph, stream: Writable, options?: ExportOptions): Promise<void>;
|
||||
/**
|
||||
* Export graph to GEXF format (Gephi)
|
||||
*
|
||||
* GEXF (Graph Exchange XML Format) is designed for Gephi, a popular
|
||||
* graph visualization tool. It supports rich graph attributes and dynamics.
|
||||
*
|
||||
* @param graph - Graph to export
|
||||
* @param options - Export options
|
||||
* @returns GEXF XML string
|
||||
*
|
||||
* @example
|
||||
* ```typescript
|
||||
* const gexf = exportToGEXF(graph, {
|
||||
* graphName: 'Vector Network',
|
||||
* graphDescription: 'Similarity network of embeddings'
|
||||
* });
|
||||
* ```
|
||||
*/
|
||||
export declare function exportToGEXF(graph: Graph, options?: ExportOptions): string;
|
||||
/**
|
||||
* Export graph to Neo4j Cypher queries
|
||||
*
|
||||
* Generates Cypher CREATE statements that can be executed in Neo4j
|
||||
* to import the graph structure.
|
||||
*
|
||||
* @param graph - Graph to export
|
||||
* @param options - Export options
|
||||
* @returns Cypher query string
|
||||
*
|
||||
* @example
|
||||
* ```typescript
|
||||
* const cypher = exportToNeo4j(graph, {
|
||||
* includeVectors: true,
|
||||
* includeMetadata: true
|
||||
* });
|
||||
* // Execute in Neo4j shell or driver
|
||||
* ```
|
||||
*/
|
||||
export declare function exportToNeo4j(graph: Graph, options?: ExportOptions): string;
|
||||
/**
|
||||
* Export graph to Neo4j JSON format (for neo4j-admin import)
|
||||
*
|
||||
* @param graph - Graph to export
|
||||
* @param options - Export options
|
||||
* @returns Neo4j JSON import format
|
||||
*/
|
||||
export declare function exportToNeo4jJSON(graph: Graph, options?: ExportOptions): {
|
||||
nodes: any[];
|
||||
relationships: any[];
|
||||
};
|
||||
/**
|
||||
* Export graph to D3.js JSON format
|
||||
*
|
||||
* Creates a JSON structure suitable for D3.js force-directed graphs
|
||||
* and other D3 visualizations.
|
||||
*
|
||||
* @param graph - Graph to export
|
||||
* @param options - Export options
|
||||
* @returns D3.js compatible JSON object
|
||||
*
|
||||
* @example
|
||||
* ```typescript
|
||||
* const d3Graph = exportToD3(graph);
|
||||
* // Use in D3.js force simulation
|
||||
* const simulation = d3.forceSimulation(d3Graph.nodes)
|
||||
* .force("link", d3.forceLink(d3Graph.links));
|
||||
* ```
|
||||
*/
|
||||
export declare function exportToD3(graph: Graph, options?: ExportOptions): {
|
||||
nodes: any[];
|
||||
links: any[];
|
||||
};
|
||||
/**
|
||||
* Export graph to D3.js hierarchy format
|
||||
*
|
||||
* Creates a hierarchical JSON structure for D3.js tree layouts.
|
||||
* Requires a root node to be specified.
|
||||
*
|
||||
* @param graph - Graph to export
|
||||
* @param rootId - ID of the root node
|
||||
* @param options - Export options
|
||||
* @returns D3.js hierarchy object
|
||||
*/
|
||||
export declare function exportToD3Hierarchy(graph: Graph, rootId: string, options?: ExportOptions): any;
|
||||
/**
|
||||
* Export graph to NetworkX JSON format
|
||||
*
|
||||
* Creates node-link JSON format compatible with NetworkX's
|
||||
* node_link_graph() function.
|
||||
*
|
||||
* @param graph - Graph to export
|
||||
* @param options - Export options
|
||||
* @returns NetworkX JSON object
|
||||
*
|
||||
* @example
|
||||
* ```typescript
|
||||
* const nxGraph = exportToNetworkX(graph);
|
||||
* // In Python:
|
||||
* // import json
|
||||
* // import networkx as nx
|
||||
* // with open('graph.json') as f:
|
||||
* // G = nx.node_link_graph(json.load(f))
|
||||
* ```
|
||||
*/
|
||||
export declare function exportToNetworkX(graph: Graph, options?: ExportOptions): any;
|
||||
/**
|
||||
* Export graph to NetworkX edge list format
|
||||
*
|
||||
* Creates a simple text format with one edge per line.
|
||||
* Format: source target weight
|
||||
*
|
||||
* @param graph - Graph to export
|
||||
* @returns Edge list string
|
||||
*/
|
||||
export declare function exportToNetworkXEdgeList(graph: Graph): string;
|
||||
/**
|
||||
* Export graph to NetworkX adjacency list format
|
||||
*
|
||||
* @param graph - Graph to export
|
||||
* @returns Adjacency list string
|
||||
*/
|
||||
export declare function exportToNetworkXAdjacencyList(graph: Graph): string;
|
||||
/**
|
||||
* Export graph to specified format
|
||||
*
|
||||
* Universal export function that routes to the appropriate format exporter.
|
||||
*
|
||||
* @param graph - Graph to export
|
||||
* @param format - Target export format
|
||||
* @param options - Export options
|
||||
* @returns Export result with data and metadata
|
||||
*
|
||||
* @example
|
||||
* ```typescript
|
||||
* // Export to GraphML
|
||||
* const result = exportGraph(graph, 'graphml', {
|
||||
* graphName: 'My Graph',
|
||||
* includeVectors: false
|
||||
* });
|
||||
* console.log(result.data);
|
||||
*
|
||||
* // Export to D3.js
|
||||
* const d3Result = exportGraph(graph, 'd3');
|
||||
* // d3Result.data is a JSON object
|
||||
* ```
|
||||
*/
|
||||
export declare function exportGraph(graph: Graph, format: ExportFormat, options?: ExportOptions): ExportResult;
|
||||
/**
|
||||
* Base class for streaming graph exporters
|
||||
*/
|
||||
export declare abstract class StreamingExporter {
|
||||
protected stream: Writable;
|
||||
protected options: ExportOptions;
|
||||
constructor(stream: Writable, options?: ExportOptions);
|
||||
protected write(data: string): Promise<void>;
|
||||
abstract start(): Promise<void>;
|
||||
abstract addNode(node: GraphNode): Promise<void>;
|
||||
abstract addEdge(edge: GraphEdge): Promise<void>;
|
||||
abstract end(): Promise<void>;
|
||||
}
|
||||
/**
|
||||
* Streaming GraphML exporter
|
||||
*
|
||||
* @example
|
||||
* ```typescript
|
||||
* const stream = createWriteStream('graph.graphml');
|
||||
* const exporter = new GraphMLStreamExporter(stream);
|
||||
*
|
||||
* await exporter.start();
|
||||
* for (const node of nodes) {
|
||||
* await exporter.addNode(node);
|
||||
* }
|
||||
* for (const edge of edges) {
|
||||
* await exporter.addEdge(edge);
|
||||
* }
|
||||
* await exporter.end();
|
||||
* ```
|
||||
*/
|
||||
export declare class GraphMLStreamExporter extends StreamingExporter {
|
||||
private nodeAttributesDefined;
|
||||
start(): Promise<void>;
|
||||
addNode(node: GraphNode): Promise<void>;
|
||||
addEdge(edge: GraphEdge): Promise<void>;
|
||||
end(): Promise<void>;
|
||||
}
|
||||
/**
|
||||
* Streaming D3.js JSON exporter
|
||||
*/
|
||||
export declare class D3StreamExporter extends StreamingExporter {
|
||||
private firstNode;
|
||||
private firstEdge;
|
||||
private nodePhase;
|
||||
start(): Promise<void>;
|
||||
addNode(node: GraphNode): Promise<void>;
|
||||
addEdge(edge: GraphEdge): Promise<void>;
|
||||
end(): Promise<void>;
|
||||
}
|
||||
/**
|
||||
* Validate graph structure
|
||||
*
|
||||
* @param graph - Graph to validate
|
||||
* @throws Error if graph is invalid
|
||||
*/
|
||||
export declare function validateGraph(graph: Graph): void;
|
||||
declare const _default: {
|
||||
buildGraphFromEntries: typeof buildGraphFromEntries;
|
||||
buildGraphFromVectorDB: typeof buildGraphFromVectorDB;
|
||||
exportToGraphML: typeof exportToGraphML;
|
||||
exportToGEXF: typeof exportToGEXF;
|
||||
exportToNeo4j: typeof exportToNeo4j;
|
||||
exportToNeo4jJSON: typeof exportToNeo4jJSON;
|
||||
exportToD3: typeof exportToD3;
|
||||
exportToD3Hierarchy: typeof exportToD3Hierarchy;
|
||||
exportToNetworkX: typeof exportToNetworkX;
|
||||
exportToNetworkXEdgeList: typeof exportToNetworkXEdgeList;
|
||||
exportToNetworkXAdjacencyList: typeof exportToNetworkXAdjacencyList;
|
||||
exportGraph: typeof exportGraph;
|
||||
GraphMLStreamExporter: typeof GraphMLStreamExporter;
|
||||
D3StreamExporter: typeof D3StreamExporter;
|
||||
streamToGraphML: typeof streamToGraphML;
|
||||
validateGraph: typeof validateGraph;
|
||||
cosineSimilarity: typeof cosineSimilarity;
|
||||
};
|
||||
export default _default;
|
||||
//# sourceMappingURL=exporters.d.ts.map
|
||||
1
vendor/ruvector/npm/packages/ruvector-extensions/src/exporters.d.ts.map
vendored
Normal file
1
vendor/ruvector/npm/packages/ruvector-extensions/src/exporters.d.ts.map
vendored
Normal file
@@ -0,0 +1 @@
|
||||
{"version":3,"file":"exporters.d.ts","sourceRoot":"","sources":["exporters.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;GAkBG;AAEH,OAAO,EAAE,QAAQ,EAAE,MAAM,QAAQ,CAAC;AAClC,OAAO,KAAK,EAAE,WAAW,EAAgB,MAAM,UAAU,CAAC;AAG1D,KAAK,gBAAgB,GAAG,GAAG,CAAC;AAM5B;;GAEG;AACH,MAAM,WAAW,SAAS;IACxB,6BAA6B;IAC7B,EAAE,EAAE,MAAM,CAAC;IACX,sBAAsB;IACtB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,uBAAuB;IACvB,MAAM,CAAC,EAAE,MAAM,EAAE,CAAC;IAClB,+BAA+B;IAC/B,UAAU,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;CAClC;AAED;;GAEG;AACH,MAAM,WAAW,SAAS;IACxB,qBAAqB;IACrB,MAAM,EAAE,MAAM,CAAC;IACf,qBAAqB;IACrB,MAAM,EAAE,MAAM,CAAC;IACf,qCAAqC;IACrC,MAAM,EAAE,MAAM,CAAC;IACf,sBAAsB;IACtB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,sBAAsB;IACtB,UAAU,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;CAClC;AAED;;GAEG;AACH,MAAM,WAAW,KAAK;IACpB,kBAAkB;IAClB,KAAK,EAAE,SAAS,EAAE,CAAC;IACnB,kBAAkB;IAClB,KAAK,EAAE,SAAS,EAAE,CAAC;IACnB,2BAA2B;IAC3B,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;CAChC;AAED;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,0CAA0C;IAC1C,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB,kCAAkC;IAClC,eAAe,CAAC,EAAE,OAAO,CAAC;IAC1B,2CAA2C;IAC3C,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,6CAA6C;IAC7C,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,uBAAuB;IACvB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,wBAAwB;IACxB,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,6CAA6C;IAC7C,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,gCAAgC;IAChC,gBAAgB,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CAC3C;AAED;;GAEG;AACH,MAAM,MAAM,YAAY,GAAG,SAAS,GAAG,MAAM,GAAG,OAAO,GAAG,IAAI,GAAG,UAAU,CAAC;AAE5E;;GAEG;AACH,MAAM,WAAW,YAAY;IAC3B,yBAAyB;IACzB,MAAM,EAAE,YAAY,CAAC;IACrB,2DAA2D;IAC3D,IAAI,EAAE,MAAM,GAAG,MAAM,CAAC;IACtB,+BAA+B;IAC/B,SAAS,EAAE,MAAM,CAAC;IAClB,+BAA+B;IAC/B,SAAS,EAAE,MAAM,CAAC;IAClB,sBAAsB;IACtB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;CAChC;AAMD;;;;;;;;;;;;;;;GAeG;AACH,wBAAgB,sBAAsB,CACpC,EAAE,EAAE,gBAAgB,EACpB,OAAO,GAAE,aAAkB,GAC1B,KAAK,CAsBP;AAED;;;;;;;;;;;;;;;GAeG;AACH,wBAAgB,qBAAqB,CACnC,OAAO,EAAE,WAAW,EAAE,EACtB,OAAO,GAAE,aAAkB,GAC1B,KAAK,CAoEP;AAED;;GAEG;AACH,iBAAS,gBAAgB,CAAC,CAAC,EAAE,MAAM,EAAE,EAAE,CAAC,EAAE,MAAM,EAAE,GAAG,MAAM,CAuB1D;AAMD;;;;;;;;;;;;;;;;;;GAkBG;AACH,wBAAgB,eAAe,CAC7B,KAAK,EAAE,KAAK,EACZ,OAAO,GAAE,aAAkB,GAC1B,MAAM,CA6ER;AAED;;;;;;;;;;;;;GAaG;AACH,wBAAsB,eAAe,CACnC,KAAK,EAAE,KAAK,EACZ,MAAM,EAAE,QAAQ,EAChB,OAAO,GAAE,aAAkB,GAC1B,OAAO,CAAC,IAAI,CAAC,CASf;AAMD;;;;;;;;;;;;;;;;;GAiBG;AACH,wBAAgB,YAAY,CAC1B,KAAK,EAAE,KAAK,EACZ,OAAO,GAAE,aAAkB,GAC1B,MAAM,CAsGR;AAMD;;;;;;;;;;;;;;;;;;GAkBG;AACH,wBAAgB,aAAa,CAC3B,KAAK,EAAE,KAAK,EACZ,OAAO,GAAE,aAAkB,GAC1B,MAAM,CAuDR;AAED;;;;;;GAMG;AACH,wBAAgB,iBAAiB,CAC/B,KAAK,EAAE,KAAK,EACZ,OAAO,GAAE,aAAkB,GAC1B;IAAE,KAAK,EAAE,GAAG,EAAE,CAAC;IAAC,aAAa,EAAE,GAAG,EAAE,CAAA;CAAE,CAqCxC;AAMD;;;;;;;;;;;;;;;;;GAiBG;AACH,wBAAgB,UAAU,CACxB,KAAK,EAAE,KAAK,EACZ,OAAO,GAAE,aAAkB,GAC1B;IAAE,KAAK,EAAE,GAAG,EAAE,CAAC;IAAC,KAAK,EAAE,GAAG,EAAE,CAAA;CAAE,CA6BhC;AAED;;;;;;;;;;GAUG;AACH,wBAAgB,mBAAmB,CACjC,KAAK,EAAE,KAAK,EACZ,MAAM,EAAE,MAAM,EACd,OAAO,GAAE,aAAkB,GAC1B,GAAG,CA2CL;AAMD;;;;;;;;;;;;;;;;;;;GAmBG;AACH,wBAAgB,gBAAgB,CAC9B,KAAK,EAAE,KAAK,EACZ,OAAO,GAAE,aAAkB,GAC1B,GAAG,CA4BL;AAED;;;;;;;;GAQG;AACH,wBAAgB,wBAAwB,CAAC,KAAK,EAAE,KAAK,GAAG,MAAM,CAQ7D;AAED;;;;;GAKG;AACH,wBAAgB,6BAA6B,CAAC,KAAK,EAAE,KAAK,GAAG,MAAM,CAuBlE;AAMD;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACH,wBAAgB,WAAW,CACzB,KAAK,EAAE,KAAK,EACZ,MAAM,EAAE,YAAY,EACpB,OAAO,GAAE,aAAkB,GAC1B,YAAY,CAuCd;AAMD;;GAEG;AACH,8BAAsB,iBAAiB;IACrC,SAAS,CAAC,MAAM,EAAE,QAAQ,CAAC;IAC3B,SAAS,CAAC,OAAO,EAAE,aAAa,CAAC;gBAErB,MAAM,EAAE,QAAQ,EAAE,OAAO,GAAE,aAAkB;IAKzD,SAAS,CAAC,KAAK,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAS5C,QAAQ,CAAC,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAC/B,QAAQ,CAAC,OAAO,CAAC,IAAI,EAAE,SAAS,GAAG,OAAO,CAAC,IAAI,CAAC;IAChD,QAAQ,CAAC,OAAO,CAAC,IAAI,EAAE,SAAS,GAAG,OAAO,CAAC,IAAI,CAAC;IAChD,QAAQ,CAAC,GAAG,IAAI,OAAO,CAAC,IAAI,CAAC;CAC9B;AAED;;;;;;;;;;;;;;;;;GAiBG;AACH,qBAAa,qBAAsB,SAAQ,iBAAiB;IAC1D,OAAO,CAAC,qBAAqB,CAAS;IAEhC,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAatB,OAAO,CAAC,IAAI,EAAE,SAAS,GAAG,OAAO,CAAC,IAAI,CAAC;IAWvC,OAAO,CAAC,IAAI,EAAE,SAAS,GAAG,OAAO,CAAC,IAAI,CAAC;IAQvC,GAAG,IAAI,OAAO,CAAC,IAAI,CAAC;CAI3B;AAED;;GAEG;AACH,qBAAa,gBAAiB,SAAQ,iBAAiB;IACrD,OAAO,CAAC,SAAS,CAAQ;IACzB,OAAO,CAAC,SAAS,CAAQ;IACzB,OAAO,CAAC,SAAS,CAAQ;IAEnB,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAItB,OAAO,CAAC,IAAI,EAAE,SAAS,GAAG,OAAO,CAAC,IAAI,CAAC;IAiBvC,OAAO,CAAC,IAAI,EAAE,SAAS,GAAG,OAAO,CAAC,IAAI,CAAC;IAmBvC,GAAG,IAAI,OAAO,CAAC,IAAI,CAAC;CAM3B;AAyBD;;;;;GAKG;AACH,wBAAgB,aAAa,CAAC,KAAK,EAAE,KAAK,GAAG,IAAI,CAkChD;;;;;;;;;;;;;;;;;;;;AAMD,wBA2BE"}
|
||||
931
vendor/ruvector/npm/packages/ruvector-extensions/src/exporters.js
vendored
Normal file
931
vendor/ruvector/npm/packages/ruvector-extensions/src/exporters.js
vendored
Normal file
@@ -0,0 +1,931 @@
|
||||
"use strict";
|
||||
/**
|
||||
* Graph Export Module for ruvector-extensions
|
||||
*
|
||||
* Provides export functionality to multiple graph formats:
|
||||
* - GraphML (XML-based graph format)
|
||||
* - GEXF (Graph Exchange XML Format for Gephi)
|
||||
* - Neo4j (Cypher queries)
|
||||
* - D3.js JSON (web visualization)
|
||||
* - NetworkX (Python graph library)
|
||||
*
|
||||
* Features:
|
||||
* - Full TypeScript types and interfaces
|
||||
* - Streaming exports for large graphs
|
||||
* - Configurable export options
|
||||
* - Support for node attributes and edge weights
|
||||
* - Error handling and validation
|
||||
*
|
||||
* @module exporters
|
||||
*/
|
||||
Object.defineProperty(exports, "__esModule", { value: true });
|
||||
exports.D3StreamExporter = exports.GraphMLStreamExporter = exports.StreamingExporter = void 0;
|
||||
exports.buildGraphFromVectorDB = buildGraphFromVectorDB;
|
||||
exports.buildGraphFromEntries = buildGraphFromEntries;
|
||||
exports.exportToGraphML = exportToGraphML;
|
||||
exports.streamToGraphML = streamToGraphML;
|
||||
exports.exportToGEXF = exportToGEXF;
|
||||
exports.exportToNeo4j = exportToNeo4j;
|
||||
exports.exportToNeo4jJSON = exportToNeo4jJSON;
|
||||
exports.exportToD3 = exportToD3;
|
||||
exports.exportToD3Hierarchy = exportToD3Hierarchy;
|
||||
exports.exportToNetworkX = exportToNetworkX;
|
||||
exports.exportToNetworkXEdgeList = exportToNetworkXEdgeList;
|
||||
exports.exportToNetworkXAdjacencyList = exportToNetworkXAdjacencyList;
|
||||
exports.exportGraph = exportGraph;
|
||||
exports.validateGraph = validateGraph;
|
||||
// ============================================================================
|
||||
// Graph Builder
|
||||
// ============================================================================
|
||||
/**
|
||||
* Build a graph from VectorDB by computing similarity between vectors
|
||||
*
|
||||
* @param db - VectorDB instance
|
||||
* @param options - Export options
|
||||
* @returns Graph structure
|
||||
*
|
||||
* @example
|
||||
* ```typescript
|
||||
* const graph = buildGraphFromVectorDB(db, {
|
||||
* maxNeighbors: 5,
|
||||
* threshold: 0.7,
|
||||
* includeVectors: false
|
||||
* });
|
||||
* ```
|
||||
*/
|
||||
function buildGraphFromVectorDB(db, options = {}) {
|
||||
const { maxNeighbors = 10, threshold = 0.0, includeVectors = false, includeMetadata = true } = options;
|
||||
const stats = db.stats();
|
||||
const nodes = [];
|
||||
const edges = [];
|
||||
const processedIds = new Set();
|
||||
// Get all vectors by searching with a dummy query
|
||||
// Since we don't have a list() method, we'll need to build the graph incrementally
|
||||
// This is a limitation - in practice, you'd want to add a list() or getAllIds() method to VectorDB
|
||||
// For now, we'll create a helper function that needs to be called with pre-fetched entries
|
||||
throw new Error('buildGraphFromVectorDB requires VectorDB to have a list() or getAllIds() method. ' +
|
||||
'Please use buildGraphFromEntries() instead with pre-fetched vector entries.');
|
||||
}
|
||||
/**
|
||||
* Build a graph from a list of vector entries
|
||||
*
|
||||
* @param entries - Array of vector entries
|
||||
* @param options - Export options
|
||||
* @returns Graph structure
|
||||
*
|
||||
* @example
|
||||
* ```typescript
|
||||
* const entries = [...]; // Your vector entries
|
||||
* const graph = buildGraphFromEntries(entries, {
|
||||
* maxNeighbors: 5,
|
||||
* threshold: 0.7
|
||||
* });
|
||||
* ```
|
||||
*/
|
||||
function buildGraphFromEntries(entries, options = {}) {
|
||||
const { maxNeighbors = 10, threshold = 0.0, includeVectors = false, includeMetadata = true } = options;
|
||||
const nodes = [];
|
||||
const edges = [];
|
||||
// Create nodes
|
||||
for (const entry of entries) {
|
||||
const node = {
|
||||
id: entry.id,
|
||||
label: entry.metadata?.name || entry.metadata?.label || entry.id
|
||||
};
|
||||
if (includeVectors) {
|
||||
node.vector = entry.vector;
|
||||
}
|
||||
if (includeMetadata && entry.metadata) {
|
||||
node.attributes = { ...entry.metadata };
|
||||
}
|
||||
nodes.push(node);
|
||||
}
|
||||
// Create edges by computing similarity
|
||||
for (let i = 0; i < entries.length; i++) {
|
||||
const neighbors = [];
|
||||
for (let j = 0; j < entries.length; j++) {
|
||||
if (i === j)
|
||||
continue;
|
||||
const similarity = cosineSimilarity(entries[i].vector, entries[j].vector);
|
||||
if (similarity >= threshold) {
|
||||
neighbors.push({ index: j, similarity });
|
||||
}
|
||||
}
|
||||
// Sort by similarity and take top k
|
||||
neighbors.sort((a, b) => b.similarity - a.similarity);
|
||||
const topNeighbors = neighbors.slice(0, maxNeighbors);
|
||||
// Create edges
|
||||
for (const neighbor of topNeighbors) {
|
||||
edges.push({
|
||||
source: entries[i].id,
|
||||
target: entries[neighbor.index].id,
|
||||
weight: neighbor.similarity,
|
||||
type: 'similarity'
|
||||
});
|
||||
}
|
||||
}
|
||||
return {
|
||||
nodes,
|
||||
edges,
|
||||
metadata: {
|
||||
nodeCount: nodes.length,
|
||||
edgeCount: edges.length,
|
||||
threshold,
|
||||
maxNeighbors
|
||||
}
|
||||
};
|
||||
}
|
||||
/**
|
||||
* Compute cosine similarity between two vectors
|
||||
*/
|
||||
function cosineSimilarity(a, b) {
|
||||
if (a.length !== b.length) {
|
||||
throw new Error('Vectors must have the same dimension');
|
||||
}
|
||||
let dotProduct = 0;
|
||||
let normA = 0;
|
||||
let normB = 0;
|
||||
for (let i = 0; i < a.length; i++) {
|
||||
dotProduct += a[i] * b[i];
|
||||
normA += a[i] * a[i];
|
||||
normB += b[i] * b[i];
|
||||
}
|
||||
normA = Math.sqrt(normA);
|
||||
normB = Math.sqrt(normB);
|
||||
if (normA === 0 || normB === 0) {
|
||||
return 0;
|
||||
}
|
||||
return dotProduct / (normA * normB);
|
||||
}
|
||||
// ============================================================================
|
||||
// GraphML Exporter
|
||||
// ============================================================================
|
||||
/**
|
||||
* Export graph to GraphML format (XML-based)
|
||||
*
|
||||
* GraphML is a comprehensive and easy-to-use file format for graphs.
|
||||
* It's supported by many graph analysis tools including Gephi, NetworkX, and igraph.
|
||||
*
|
||||
* @param graph - Graph to export
|
||||
* @param options - Export options
|
||||
* @returns GraphML XML string
|
||||
*
|
||||
* @example
|
||||
* ```typescript
|
||||
* const graphml = exportToGraphML(graph, {
|
||||
* graphName: 'Vector Similarity Graph',
|
||||
* includeVectors: false
|
||||
* });
|
||||
* console.log(graphml);
|
||||
* ```
|
||||
*/
|
||||
function exportToGraphML(graph, options = {}) {
|
||||
const { graphName = 'VectorGraph', includeVectors = false } = options;
|
||||
let xml = '<?xml version="1.0" encoding="UTF-8"?>\n';
|
||||
xml += '<graphml xmlns="http://graphml.graphdrawing.org/xmlns"\n';
|
||||
xml += ' xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"\n';
|
||||
xml += ' xsi:schemaLocation="http://graphml.graphdrawing.org/xmlns\n';
|
||||
xml += ' http://graphml.graphdrawing.org/xmlns/1.0/graphml.xsd">\n\n';
|
||||
// Define node attributes
|
||||
xml += ' <!-- Node attributes -->\n';
|
||||
xml += ' <key id="label" for="node" attr.name="label" attr.type="string"/>\n';
|
||||
if (includeVectors) {
|
||||
xml += ' <key id="vector" for="node" attr.name="vector" attr.type="string"/>\n';
|
||||
}
|
||||
// Collect all unique node attributes
|
||||
const nodeAttrs = new Set();
|
||||
for (const node of graph.nodes) {
|
||||
if (node.attributes) {
|
||||
Object.keys(node.attributes).forEach(key => nodeAttrs.add(key));
|
||||
}
|
||||
}
|
||||
Array.from(nodeAttrs).forEach(attr => {
|
||||
xml += ` <key id="node_${escapeXML(attr)}" for="node" attr.name="${escapeXML(attr)}" attr.type="string"/>\n`;
|
||||
});
|
||||
// Define edge attributes
|
||||
xml += '\n <!-- Edge attributes -->\n';
|
||||
xml += ' <key id="weight" for="edge" attr.name="weight" attr.type="double"/>\n';
|
||||
xml += ' <key id="type" for="edge" attr.name="type" attr.type="string"/>\n';
|
||||
// Start graph
|
||||
xml += `\n <graph id="${escapeXML(graphName)}" edgedefault="directed">\n\n`;
|
||||
// Add nodes
|
||||
xml += ' <!-- Nodes -->\n';
|
||||
for (const node of graph.nodes) {
|
||||
xml += ` <node id="${escapeXML(node.id)}">\n`;
|
||||
if (node.label) {
|
||||
xml += ` <data key="label">${escapeXML(node.label)}</data>\n`;
|
||||
}
|
||||
if (includeVectors && node.vector) {
|
||||
xml += ` <data key="vector">${escapeXML(JSON.stringify(node.vector))}</data>\n`;
|
||||
}
|
||||
if (node.attributes) {
|
||||
for (const [key, value] of Object.entries(node.attributes)) {
|
||||
xml += ` <data key="node_${escapeXML(key)}">${escapeXML(String(value))}</data>\n`;
|
||||
}
|
||||
}
|
||||
xml += ' </node>\n';
|
||||
}
|
||||
// Add edges
|
||||
xml += '\n <!-- Edges -->\n';
|
||||
for (let i = 0; i < graph.edges.length; i++) {
|
||||
const edge = graph.edges[i];
|
||||
xml += ` <edge id="e${i}" source="${escapeXML(edge.source)}" target="${escapeXML(edge.target)}">\n`;
|
||||
xml += ` <data key="weight">${edge.weight}</data>\n`;
|
||||
if (edge.type) {
|
||||
xml += ` <data key="type">${escapeXML(edge.type)}</data>\n`;
|
||||
}
|
||||
xml += ' </edge>\n';
|
||||
}
|
||||
xml += ' </graph>\n';
|
||||
xml += '</graphml>\n';
|
||||
return xml;
|
||||
}
|
||||
/**
|
||||
* Stream graph to GraphML format
|
||||
*
|
||||
* @param graph - Graph to export
|
||||
* @param stream - Writable stream
|
||||
* @param options - Export options
|
||||
*
|
||||
* @example
|
||||
* ```typescript
|
||||
* import { createWriteStream } from 'fs';
|
||||
* const stream = createWriteStream('graph.graphml');
|
||||
* await streamToGraphML(graph, stream);
|
||||
* ```
|
||||
*/
|
||||
async function streamToGraphML(graph, stream, options = {}) {
|
||||
const graphml = exportToGraphML(graph, options);
|
||||
return new Promise((resolve, reject) => {
|
||||
stream.write(graphml, (err) => {
|
||||
if (err)
|
||||
reject(err);
|
||||
else
|
||||
resolve();
|
||||
});
|
||||
});
|
||||
}
|
||||
// ============================================================================
|
||||
// GEXF Exporter
|
||||
// ============================================================================
|
||||
/**
|
||||
* Export graph to GEXF format (Gephi)
|
||||
*
|
||||
* GEXF (Graph Exchange XML Format) is designed for Gephi, a popular
|
||||
* graph visualization tool. It supports rich graph attributes and dynamics.
|
||||
*
|
||||
* @param graph - Graph to export
|
||||
* @param options - Export options
|
||||
* @returns GEXF XML string
|
||||
*
|
||||
* @example
|
||||
* ```typescript
|
||||
* const gexf = exportToGEXF(graph, {
|
||||
* graphName: 'Vector Network',
|
||||
* graphDescription: 'Similarity network of embeddings'
|
||||
* });
|
||||
* ```
|
||||
*/
|
||||
function exportToGEXF(graph, options = {}) {
|
||||
const { graphName = 'VectorGraph', graphDescription = 'Vector similarity graph', includeVectors = false } = options;
|
||||
const timestamp = new Date().toISOString();
|
||||
let xml = '<?xml version="1.0" encoding="UTF-8"?>\n';
|
||||
xml += '<gexf xmlns="http://www.gexf.net/1.2draft"\n';
|
||||
xml += ' xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"\n';
|
||||
xml += ' xsi:schemaLocation="http://www.gexf.net/1.2draft\n';
|
||||
xml += ' http://www.gexf.net/1.2draft/gexf.xsd"\n';
|
||||
xml += ' version="1.2">\n\n';
|
||||
xml += ` <meta lastmodifieddate="${timestamp}">\n`;
|
||||
xml += ` <creator>ruvector-extensions</creator>\n`;
|
||||
xml += ` <description>${escapeXML(graphDescription)}</description>\n`;
|
||||
xml += ' </meta>\n\n';
|
||||
xml += ' <graph mode="static" defaultedgetype="directed">\n\n';
|
||||
// Node attributes
|
||||
xml += ' <attributes class="node">\n';
|
||||
xml += ' <attribute id="0" title="label" type="string"/>\n';
|
||||
let attrId = 1;
|
||||
const nodeAttrMap = new Map();
|
||||
if (includeVectors) {
|
||||
xml += ` <attribute id="${attrId}" title="vector" type="string"/>\n`;
|
||||
nodeAttrMap.set('vector', attrId++);
|
||||
}
|
||||
// Collect unique node attributes
|
||||
const nodeAttrs = new Set();
|
||||
for (const node of graph.nodes) {
|
||||
if (node.attributes) {
|
||||
Object.keys(node.attributes).forEach(key => nodeAttrs.add(key));
|
||||
}
|
||||
}
|
||||
Array.from(nodeAttrs).forEach(attr => {
|
||||
xml += ` <attribute id="${attrId}" title="${escapeXML(attr)}" type="string"/>\n`;
|
||||
nodeAttrMap.set(attr, attrId++);
|
||||
});
|
||||
xml += ' </attributes>\n\n';
|
||||
// Edge attributes
|
||||
xml += ' <attributes class="edge">\n';
|
||||
xml += ' <attribute id="0" title="weight" type="double"/>\n';
|
||||
xml += ' <attribute id="1" title="type" type="string"/>\n';
|
||||
xml += ' </attributes>\n\n';
|
||||
// Nodes
|
||||
xml += ' <nodes>\n';
|
||||
for (const node of graph.nodes) {
|
||||
xml += ` <node id="${escapeXML(node.id)}" label="${escapeXML(node.label || node.id)}">\n`;
|
||||
xml += ' <attvalues>\n';
|
||||
if (includeVectors && node.vector) {
|
||||
const vectorId = nodeAttrMap.get('vector');
|
||||
xml += ` <attvalue for="${vectorId}" value="${escapeXML(JSON.stringify(node.vector))}"/>\n`;
|
||||
}
|
||||
if (node.attributes) {
|
||||
for (const [key, value] of Object.entries(node.attributes)) {
|
||||
const attrIdForKey = nodeAttrMap.get(key);
|
||||
if (attrIdForKey !== undefined) {
|
||||
xml += ` <attvalue for="${attrIdForKey}" value="${escapeXML(String(value))}"/>\n`;
|
||||
}
|
||||
}
|
||||
}
|
||||
xml += ' </attvalues>\n';
|
||||
xml += ' </node>\n';
|
||||
}
|
||||
xml += ' </nodes>\n\n';
|
||||
// Edges
|
||||
xml += ' <edges>\n';
|
||||
for (let i = 0; i < graph.edges.length; i++) {
|
||||
const edge = graph.edges[i];
|
||||
xml += ` <edge id="${i}" source="${escapeXML(edge.source)}" target="${escapeXML(edge.target)}" weight="${edge.weight}">\n`;
|
||||
xml += ' <attvalues>\n';
|
||||
xml += ` <attvalue for="0" value="${edge.weight}"/>\n`;
|
||||
if (edge.type) {
|
||||
xml += ` <attvalue for="1" value="${escapeXML(edge.type)}"/>\n`;
|
||||
}
|
||||
xml += ' </attvalues>\n';
|
||||
xml += ' </edge>\n';
|
||||
}
|
||||
xml += ' </edges>\n\n';
|
||||
xml += ' </graph>\n';
|
||||
xml += '</gexf>\n';
|
||||
return xml;
|
||||
}
|
||||
// ============================================================================
|
||||
// Neo4j Exporter
|
||||
// ============================================================================
|
||||
/**
|
||||
* Export graph to Neo4j Cypher queries
|
||||
*
|
||||
* Generates Cypher CREATE statements that can be executed in Neo4j
|
||||
* to import the graph structure.
|
||||
*
|
||||
* @param graph - Graph to export
|
||||
* @param options - Export options
|
||||
* @returns Cypher query string
|
||||
*
|
||||
* @example
|
||||
* ```typescript
|
||||
* const cypher = exportToNeo4j(graph, {
|
||||
* includeVectors: true,
|
||||
* includeMetadata: true
|
||||
* });
|
||||
* // Execute in Neo4j shell or driver
|
||||
* ```
|
||||
*/
|
||||
function exportToNeo4j(graph, options = {}) {
|
||||
const { includeVectors = false, includeMetadata = true } = options;
|
||||
let cypher = '// Neo4j Cypher Import Script\n';
|
||||
cypher += '// Generated by ruvector-extensions\n\n';
|
||||
cypher += '// Clear existing data (optional - uncomment if needed)\n';
|
||||
cypher += '// MATCH (n) DETACH DELETE n;\n\n';
|
||||
// Create constraint for unique IDs
|
||||
cypher += '// Create constraint for unique node IDs\n';
|
||||
cypher += 'CREATE CONSTRAINT vector_id IF NOT EXISTS FOR (v:Vector) REQUIRE v.id IS UNIQUE;\n\n';
|
||||
// Create nodes
|
||||
cypher += '// Create nodes\n';
|
||||
for (const node of graph.nodes) {
|
||||
const props = [`id: "${escapeCypher(node.id)}"`];
|
||||
if (node.label) {
|
||||
props.push(`label: "${escapeCypher(node.label)}"`);
|
||||
}
|
||||
if (includeVectors && node.vector) {
|
||||
props.push(`vector: [${node.vector.join(', ')}]`);
|
||||
}
|
||||
if (includeMetadata && node.attributes) {
|
||||
for (const [key, value] of Object.entries(node.attributes)) {
|
||||
const cypherValue = typeof value === 'string'
|
||||
? `"${escapeCypher(value)}"`
|
||||
: JSON.stringify(value);
|
||||
props.push(`${escapeCypher(key)}: ${cypherValue}`);
|
||||
}
|
||||
}
|
||||
cypher += `CREATE (:Vector {${props.join(', ')}});\n`;
|
||||
}
|
||||
cypher += '\n// Create relationships\n';
|
||||
// Create edges
|
||||
for (const edge of graph.edges) {
|
||||
const relType = edge.type ? escapeCypher(edge.type.toUpperCase()) : 'SIMILAR_TO';
|
||||
cypher += `MATCH (a:Vector {id: "${escapeCypher(edge.source)}"}), `;
|
||||
cypher += `(b:Vector {id: "${escapeCypher(edge.target)}"})\n`;
|
||||
cypher += `CREATE (a)-[:${relType} {weight: ${edge.weight}}]->(b);\n`;
|
||||
}
|
||||
cypher += '\n// Create indexes for performance\n';
|
||||
cypher += 'CREATE INDEX vector_label IF NOT EXISTS FOR (v:Vector) ON (v.label);\n\n';
|
||||
cypher += '// Verify import\n';
|
||||
cypher += 'MATCH (n:Vector) RETURN count(n) as nodeCount;\n';
|
||||
cypher += 'MATCH ()-[r]->() RETURN count(r) as edgeCount;\n';
|
||||
return cypher;
|
||||
}
|
||||
/**
|
||||
* Export graph to Neo4j JSON format (for neo4j-admin import)
|
||||
*
|
||||
* @param graph - Graph to export
|
||||
* @param options - Export options
|
||||
* @returns Neo4j JSON import format
|
||||
*/
|
||||
function exportToNeo4jJSON(graph, options = {}) {
|
||||
const { includeVectors = false, includeMetadata = true } = options;
|
||||
const nodes = graph.nodes.map(node => {
|
||||
const props = { id: node.id };
|
||||
if (node.label)
|
||||
props.label = node.label;
|
||||
if (includeVectors && node.vector)
|
||||
props.vector = node.vector;
|
||||
if (includeMetadata && node.attributes)
|
||||
Object.assign(props, node.attributes);
|
||||
return {
|
||||
type: 'node',
|
||||
id: node.id,
|
||||
labels: ['Vector'],
|
||||
properties: props
|
||||
};
|
||||
});
|
||||
const relationships = graph.edges.map((edge, i) => ({
|
||||
type: 'relationship',
|
||||
id: `e${i}`,
|
||||
label: edge.type || 'SIMILAR_TO',
|
||||
start: {
|
||||
id: edge.source,
|
||||
labels: ['Vector']
|
||||
},
|
||||
end: {
|
||||
id: edge.target,
|
||||
labels: ['Vector']
|
||||
},
|
||||
properties: {
|
||||
weight: edge.weight,
|
||||
...(edge.attributes || {})
|
||||
}
|
||||
}));
|
||||
return { nodes, relationships };
|
||||
}
|
||||
// ============================================================================
|
||||
// D3.js Exporter
|
||||
// ============================================================================
|
||||
/**
|
||||
* Export graph to D3.js JSON format
|
||||
*
|
||||
* Creates a JSON structure suitable for D3.js force-directed graphs
|
||||
* and other D3 visualizations.
|
||||
*
|
||||
* @param graph - Graph to export
|
||||
* @param options - Export options
|
||||
* @returns D3.js compatible JSON object
|
||||
*
|
||||
* @example
|
||||
* ```typescript
|
||||
* const d3Graph = exportToD3(graph);
|
||||
* // Use in D3.js force simulation
|
||||
* const simulation = d3.forceSimulation(d3Graph.nodes)
|
||||
* .force("link", d3.forceLink(d3Graph.links));
|
||||
* ```
|
||||
*/
|
||||
function exportToD3(graph, options = {}) {
|
||||
const { includeVectors = false, includeMetadata = true } = options;
|
||||
const nodes = graph.nodes.map(node => {
|
||||
const d3Node = {
|
||||
id: node.id,
|
||||
name: node.label || node.id
|
||||
};
|
||||
if (includeVectors && node.vector) {
|
||||
d3Node.vector = node.vector;
|
||||
}
|
||||
if (includeMetadata && node.attributes) {
|
||||
Object.assign(d3Node, node.attributes);
|
||||
}
|
||||
return d3Node;
|
||||
});
|
||||
const links = graph.edges.map(edge => ({
|
||||
source: edge.source,
|
||||
target: edge.target,
|
||||
value: edge.weight,
|
||||
type: edge.type || 'similarity',
|
||||
...(edge.attributes || {})
|
||||
}));
|
||||
return { nodes, links };
|
||||
}
|
||||
/**
|
||||
* Export graph to D3.js hierarchy format
|
||||
*
|
||||
* Creates a hierarchical JSON structure for D3.js tree layouts.
|
||||
* Requires a root node to be specified.
|
||||
*
|
||||
* @param graph - Graph to export
|
||||
* @param rootId - ID of the root node
|
||||
* @param options - Export options
|
||||
* @returns D3.js hierarchy object
|
||||
*/
|
||||
function exportToD3Hierarchy(graph, rootId, options = {}) {
|
||||
const { includeMetadata = true } = options;
|
||||
// Build adjacency map
|
||||
const adjacency = new Map();
|
||||
for (const edge of graph.edges) {
|
||||
if (!adjacency.has(edge.source)) {
|
||||
adjacency.set(edge.source, new Set());
|
||||
}
|
||||
adjacency.get(edge.source).add(edge.target);
|
||||
}
|
||||
// Find node by ID
|
||||
const nodeMap = new Map(graph.nodes.map(n => [n.id, n]));
|
||||
const visited = new Set();
|
||||
function buildHierarchy(nodeId) {
|
||||
if (visited.has(nodeId))
|
||||
return null;
|
||||
visited.add(nodeId);
|
||||
const node = nodeMap.get(nodeId);
|
||||
if (!node)
|
||||
return null;
|
||||
const hierarchyNode = {
|
||||
name: node.label || node.id,
|
||||
id: node.id
|
||||
};
|
||||
if (includeMetadata && node.attributes) {
|
||||
Object.assign(hierarchyNode, node.attributes);
|
||||
}
|
||||
const children = adjacency.get(nodeId);
|
||||
if (children && children.size > 0) {
|
||||
hierarchyNode.children = Array.from(children)
|
||||
.map(childId => buildHierarchy(childId))
|
||||
.filter(child => child !== null);
|
||||
}
|
||||
return hierarchyNode;
|
||||
}
|
||||
return buildHierarchy(rootId);
|
||||
}
|
||||
// ============================================================================
|
||||
// NetworkX Exporter
|
||||
// ============================================================================
|
||||
/**
|
||||
* Export graph to NetworkX JSON format
|
||||
*
|
||||
* Creates node-link JSON format compatible with NetworkX's
|
||||
* node_link_graph() function.
|
||||
*
|
||||
* @param graph - Graph to export
|
||||
* @param options - Export options
|
||||
* @returns NetworkX JSON object
|
||||
*
|
||||
* @example
|
||||
* ```typescript
|
||||
* const nxGraph = exportToNetworkX(graph);
|
||||
* // In Python:
|
||||
* // import json
|
||||
* // import networkx as nx
|
||||
* // with open('graph.json') as f:
|
||||
* // G = nx.node_link_graph(json.load(f))
|
||||
* ```
|
||||
*/
|
||||
function exportToNetworkX(graph, options = {}) {
|
||||
const { includeVectors = false, includeMetadata = true } = options;
|
||||
const nodes = graph.nodes.map(node => {
|
||||
const nxNode = { id: node.id };
|
||||
if (node.label)
|
||||
nxNode.label = node.label;
|
||||
if (includeVectors && node.vector)
|
||||
nxNode.vector = node.vector;
|
||||
if (includeMetadata && node.attributes)
|
||||
Object.assign(nxNode, node.attributes);
|
||||
return nxNode;
|
||||
});
|
||||
const links = graph.edges.map(edge => ({
|
||||
source: edge.source,
|
||||
target: edge.target,
|
||||
weight: edge.weight,
|
||||
type: edge.type || 'similarity',
|
||||
...(edge.attributes || {})
|
||||
}));
|
||||
return {
|
||||
directed: true,
|
||||
multigraph: false,
|
||||
graph: graph.metadata || {},
|
||||
nodes,
|
||||
links
|
||||
};
|
||||
}
|
||||
/**
|
||||
* Export graph to NetworkX edge list format
|
||||
*
|
||||
* Creates a simple text format with one edge per line.
|
||||
* Format: source target weight
|
||||
*
|
||||
* @param graph - Graph to export
|
||||
* @returns Edge list string
|
||||
*/
|
||||
function exportToNetworkXEdgeList(graph) {
|
||||
let edgeList = '# Source Target Weight\n';
|
||||
for (const edge of graph.edges) {
|
||||
edgeList += `${edge.source} ${edge.target} ${edge.weight}\n`;
|
||||
}
|
||||
return edgeList;
|
||||
}
|
||||
/**
|
||||
* Export graph to NetworkX adjacency list format
|
||||
*
|
||||
* @param graph - Graph to export
|
||||
* @returns Adjacency list string
|
||||
*/
|
||||
function exportToNetworkXAdjacencyList(graph) {
|
||||
const adjacency = new Map();
|
||||
// Build adjacency structure
|
||||
for (const edge of graph.edges) {
|
||||
if (!adjacency.has(edge.source)) {
|
||||
adjacency.set(edge.source, []);
|
||||
}
|
||||
adjacency.get(edge.source).push({
|
||||
target: edge.target,
|
||||
weight: edge.weight
|
||||
});
|
||||
}
|
||||
let adjList = '# Adjacency List\n';
|
||||
Array.from(adjacency.entries()).forEach(([source, neighbors]) => {
|
||||
const neighborStr = neighbors
|
||||
.map(n => `${n.target}:${n.weight}`)
|
||||
.join(' ');
|
||||
adjList += `${source} ${neighborStr}\n`;
|
||||
});
|
||||
return adjList;
|
||||
}
|
||||
// ============================================================================
|
||||
// Unified Export Function
|
||||
// ============================================================================
|
||||
/**
|
||||
* Export graph to specified format
|
||||
*
|
||||
* Universal export function that routes to the appropriate format exporter.
|
||||
*
|
||||
* @param graph - Graph to export
|
||||
* @param format - Target export format
|
||||
* @param options - Export options
|
||||
* @returns Export result with data and metadata
|
||||
*
|
||||
* @example
|
||||
* ```typescript
|
||||
* // Export to GraphML
|
||||
* const result = exportGraph(graph, 'graphml', {
|
||||
* graphName: 'My Graph',
|
||||
* includeVectors: false
|
||||
* });
|
||||
* console.log(result.data);
|
||||
*
|
||||
* // Export to D3.js
|
||||
* const d3Result = exportGraph(graph, 'd3');
|
||||
* // d3Result.data is a JSON object
|
||||
* ```
|
||||
*/
|
||||
function exportGraph(graph, format, options = {}) {
|
||||
let data;
|
||||
switch (format) {
|
||||
case 'graphml':
|
||||
data = exportToGraphML(graph, options);
|
||||
break;
|
||||
case 'gexf':
|
||||
data = exportToGEXF(graph, options);
|
||||
break;
|
||||
case 'neo4j':
|
||||
data = exportToNeo4j(graph, options);
|
||||
break;
|
||||
case 'd3':
|
||||
data = exportToD3(graph, options);
|
||||
break;
|
||||
case 'networkx':
|
||||
data = exportToNetworkX(graph, options);
|
||||
break;
|
||||
default:
|
||||
throw new Error(`Unsupported export format: ${format}`);
|
||||
}
|
||||
return {
|
||||
format,
|
||||
data,
|
||||
nodeCount: graph.nodes.length,
|
||||
edgeCount: graph.edges.length,
|
||||
metadata: {
|
||||
timestamp: new Date().toISOString(),
|
||||
options,
|
||||
...graph.metadata
|
||||
}
|
||||
};
|
||||
}
|
||||
// ============================================================================
|
||||
// Streaming Exporters
|
||||
// ============================================================================
|
||||
/**
|
||||
* Base class for streaming graph exporters
|
||||
*/
|
||||
class StreamingExporter {
|
||||
constructor(stream, options = {}) {
|
||||
this.stream = stream;
|
||||
this.options = options;
|
||||
}
|
||||
write(data) {
|
||||
return new Promise((resolve, reject) => {
|
||||
this.stream.write(data, (err) => {
|
||||
if (err)
|
||||
reject(err);
|
||||
else
|
||||
resolve();
|
||||
});
|
||||
});
|
||||
}
|
||||
}
|
||||
exports.StreamingExporter = StreamingExporter;
|
||||
/**
|
||||
* Streaming GraphML exporter
|
||||
*
|
||||
* @example
|
||||
* ```typescript
|
||||
* const stream = createWriteStream('graph.graphml');
|
||||
* const exporter = new GraphMLStreamExporter(stream);
|
||||
*
|
||||
* await exporter.start();
|
||||
* for (const node of nodes) {
|
||||
* await exporter.addNode(node);
|
||||
* }
|
||||
* for (const edge of edges) {
|
||||
* await exporter.addEdge(edge);
|
||||
* }
|
||||
* await exporter.end();
|
||||
* ```
|
||||
*/
|
||||
class GraphMLStreamExporter extends StreamingExporter {
|
||||
constructor() {
|
||||
super(...arguments);
|
||||
this.nodeAttributesDefined = false;
|
||||
}
|
||||
async start() {
|
||||
let xml = '<?xml version="1.0" encoding="UTF-8"?>\n';
|
||||
xml += '<graphml xmlns="http://graphml.graphdrawing.org/xmlns"\n';
|
||||
xml += ' xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"\n';
|
||||
xml += ' xsi:schemaLocation="http://graphml.graphdrawing.org/xmlns\n';
|
||||
xml += ' http://graphml.graphdrawing.org/xmlns/1.0/graphml.xsd">\n\n';
|
||||
xml += ' <key id="label" for="node" attr.name="label" attr.type="string"/>\n';
|
||||
xml += ' <key id="weight" for="edge" attr.name="weight" attr.type="double"/>\n\n';
|
||||
xml += ` <graph id="${escapeXML(this.options.graphName || 'VectorGraph')}" edgedefault="directed">\n\n`;
|
||||
await this.write(xml);
|
||||
}
|
||||
async addNode(node) {
|
||||
let xml = ` <node id="${escapeXML(node.id)}">\n`;
|
||||
if (node.label) {
|
||||
xml += ` <data key="label">${escapeXML(node.label)}</data>\n`;
|
||||
}
|
||||
xml += ' </node>\n';
|
||||
await this.write(xml);
|
||||
}
|
||||
async addEdge(edge) {
|
||||
const edgeId = `e_${edge.source}_${edge.target}`;
|
||||
let xml = ` <edge id="${escapeXML(edgeId)}" source="${escapeXML(edge.source)}" target="${escapeXML(edge.target)}">\n`;
|
||||
xml += ` <data key="weight">${edge.weight}</data>\n`;
|
||||
xml += ' </edge>\n';
|
||||
await this.write(xml);
|
||||
}
|
||||
async end() {
|
||||
const xml = ' </graph>\n</graphml>\n';
|
||||
await this.write(xml);
|
||||
}
|
||||
}
|
||||
exports.GraphMLStreamExporter = GraphMLStreamExporter;
|
||||
/**
|
||||
* Streaming D3.js JSON exporter
|
||||
*/
|
||||
class D3StreamExporter extends StreamingExporter {
|
||||
constructor() {
|
||||
super(...arguments);
|
||||
this.firstNode = true;
|
||||
this.firstEdge = true;
|
||||
this.nodePhase = true;
|
||||
}
|
||||
async start() {
|
||||
await this.write('{"nodes":[');
|
||||
}
|
||||
async addNode(node) {
|
||||
if (!this.nodePhase) {
|
||||
throw new Error('Cannot add nodes after edges have been added');
|
||||
}
|
||||
const d3Node = {
|
||||
id: node.id,
|
||||
name: node.label || node.id,
|
||||
...node.attributes
|
||||
};
|
||||
const prefix = this.firstNode ? '' : ',';
|
||||
this.firstNode = false;
|
||||
await this.write(prefix + JSON.stringify(d3Node));
|
||||
}
|
||||
async addEdge(edge) {
|
||||
if (this.nodePhase) {
|
||||
this.nodePhase = false;
|
||||
await this.write('],"links":[');
|
||||
}
|
||||
const d3Link = {
|
||||
source: edge.source,
|
||||
target: edge.target,
|
||||
value: edge.weight,
|
||||
type: edge.type || 'similarity'
|
||||
};
|
||||
const prefix = this.firstEdge ? '' : ',';
|
||||
this.firstEdge = false;
|
||||
await this.write(prefix + JSON.stringify(d3Link));
|
||||
}
|
||||
async end() {
|
||||
if (this.nodePhase) {
|
||||
await this.write('],"links":[');
|
||||
}
|
||||
await this.write(']}');
|
||||
}
|
||||
}
|
||||
exports.D3StreamExporter = D3StreamExporter;
|
||||
// ============================================================================
|
||||
// Utility Functions
|
||||
// ============================================================================
|
||||
/**
|
||||
* Escape XML special characters
|
||||
*/
|
||||
function escapeXML(str) {
|
||||
return str
|
||||
.replace(/&/g, '&')
|
||||
.replace(/</g, '<')
|
||||
.replace(/>/g, '>')
|
||||
.replace(/"/g, '"')
|
||||
.replace(/'/g, ''');
|
||||
}
|
||||
/**
|
||||
* Escape Cypher special characters
|
||||
*/
|
||||
function escapeCypher(str) {
|
||||
return str.replace(/"/g, '\\"').replace(/\\/g, '\\\\');
|
||||
}
|
||||
/**
|
||||
* Validate graph structure
|
||||
*
|
||||
* @param graph - Graph to validate
|
||||
* @throws Error if graph is invalid
|
||||
*/
|
||||
function validateGraph(graph) {
|
||||
if (!graph.nodes || !Array.isArray(graph.nodes)) {
|
||||
throw new Error('Graph must have a nodes array');
|
||||
}
|
||||
if (!graph.edges || !Array.isArray(graph.edges)) {
|
||||
throw new Error('Graph must have an edges array');
|
||||
}
|
||||
const nodeIds = new Set(graph.nodes.map(n => n.id));
|
||||
for (const node of graph.nodes) {
|
||||
if (!node.id) {
|
||||
throw new Error('All nodes must have an id');
|
||||
}
|
||||
}
|
||||
for (const edge of graph.edges) {
|
||||
if (!edge.source || !edge.target) {
|
||||
throw new Error('All edges must have source and target');
|
||||
}
|
||||
if (!nodeIds.has(edge.source)) {
|
||||
throw new Error(`Edge references non-existent source node: ${edge.source}`);
|
||||
}
|
||||
if (!nodeIds.has(edge.target)) {
|
||||
throw new Error(`Edge references non-existent target node: ${edge.target}`);
|
||||
}
|
||||
if (typeof edge.weight !== 'number') {
|
||||
throw new Error('All edges must have a numeric weight');
|
||||
}
|
||||
}
|
||||
}
|
||||
// ============================================================================
|
||||
// Exports
|
||||
// ============================================================================
|
||||
exports.default = {
|
||||
// Graph builders
|
||||
buildGraphFromEntries,
|
||||
buildGraphFromVectorDB,
|
||||
// Format exporters
|
||||
exportToGraphML,
|
||||
exportToGEXF,
|
||||
exportToNeo4j,
|
||||
exportToNeo4jJSON,
|
||||
exportToD3,
|
||||
exportToD3Hierarchy,
|
||||
exportToNetworkX,
|
||||
exportToNetworkXEdgeList,
|
||||
exportToNetworkXAdjacencyList,
|
||||
// Unified export
|
||||
exportGraph,
|
||||
// Streaming exporters
|
||||
GraphMLStreamExporter,
|
||||
D3StreamExporter,
|
||||
streamToGraphML,
|
||||
// Utilities
|
||||
validateGraph,
|
||||
cosineSimilarity
|
||||
};
|
||||
//# sourceMappingURL=exporters.js.map
|
||||
1
vendor/ruvector/npm/packages/ruvector-extensions/src/exporters.js.map
vendored
Normal file
1
vendor/ruvector/npm/packages/ruvector-extensions/src/exporters.js.map
vendored
Normal file
File diff suppressed because one or more lines are too long
1213
vendor/ruvector/npm/packages/ruvector-extensions/src/exporters.ts
vendored
Normal file
1213
vendor/ruvector/npm/packages/ruvector-extensions/src/exporters.ts
vendored
Normal file
File diff suppressed because it is too large
Load Diff
1
vendor/ruvector/npm/packages/ruvector-extensions/src/index.d.ts.map
vendored
Normal file
1
vendor/ruvector/npm/packages/ruvector-extensions/src/index.d.ts.map
vendored
Normal file
@@ -0,0 +1 @@
|
||||
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAGH,OAAO,EAEL,iBAAiB,EAGjB,gBAAgB,EAChB,gBAAgB,EAChB,mBAAmB,EACnB,qBAAqB,EAGrB,cAAc,EACd,cAAc,EAGd,KAAK,WAAW,EAChB,KAAK,eAAe,EACpB,KAAK,oBAAoB,EACzB,KAAK,cAAc,EACnB,KAAK,eAAe,EACpB,KAAK,sBAAsB,EAC3B,KAAK,sBAAsB,EAC3B,KAAK,yBAAyB,EAC9B,KAAK,2BAA2B,GACjC,MAAM,iBAAiB,CAAC;AAGzB,OAAO,EAAE,OAAO,IAAI,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAGxD,OAAO,EAEL,qBAAqB,EACrB,sBAAsB,EAGtB,eAAe,EACf,YAAY,EACZ,aAAa,EACb,iBAAiB,EACjB,UAAU,EACV,mBAAmB,EACnB,gBAAgB,EAChB,wBAAwB,EACxB,6BAA6B,EAG7B,WAAW,EAGX,qBAAqB,EACrB,gBAAgB,EAChB,eAAe,EAGf,aAAa,EAGb,KAAK,KAAK,EACV,KAAK,SAAS,EACd,KAAK,SAAS,EACd,KAAK,aAAa,EAClB,KAAK,YAAY,EACjB,KAAK,YAAY,EAClB,MAAM,gBAAgB,CAAC;AAGxB,OAAO,EAEL,eAAe,EAGf,eAAe,EAGf,UAAU,EAGV,QAAQ,EACR,SAAS,EAGT,KAAK,MAAM,EACX,KAAK,OAAO,EACZ,KAAK,WAAW,EAChB,KAAK,aAAa,EAClB,KAAK,oBAAoB,EACzB,KAAK,YAAY,EACjB,KAAK,iBAAiB,EACtB,KAAK,qBAAqB,GAC3B,MAAM,eAAe,CAAC;AAGvB,OAAO,EAEL,QAAQ,EAGR,aAAa,EAGb,KAAK,SAAS,IAAI,WAAW,EAC7B,KAAK,SAAS,EACd,KAAK,SAAS,GACf,MAAM,gBAAgB,CAAC"}
|
||||
1
vendor/ruvector/npm/packages/ruvector-extensions/src/index.js.map
vendored
Normal file
1
vendor/ruvector/npm/packages/ruvector-extensions/src/index.js.map
vendored
Normal file
@@ -0,0 +1 @@
|
||||
{"version":3,"file":"index.js","sourceRoot":"","sources":["index.ts"],"names":[],"mappings":";AAAA;;;;;;;;;GASG;;;;;;AAEH,2BAA2B;AAC3B,iDAwByB;AAvBvB,aAAa;AACb,kHAAA,iBAAiB,OAAA;AAEjB,2BAA2B;AAC3B,iHAAA,gBAAgB,OAAA;AAChB,iHAAA,gBAAgB,OAAA;AAChB,oHAAA,mBAAmB,OAAA;AACnB,sHAAA,qBAAqB,OAAA;AAErB,mBAAmB;AACnB,+GAAA,cAAc,OAAA;AACd,+GAAA,cAAc,OAAA;AAchB,oCAAoC;AACpC,iDAAwD;AAA/C,4HAAA,OAAO,OAAc;AAE9B,gCAAgC;AAChC,+CAkCwB;AAjCtB,iBAAiB;AACjB,qHAAA,qBAAqB,OAAA;AACrB,sHAAA,sBAAsB,OAAA;AAEtB,mBAAmB;AACnB,+GAAA,eAAe,OAAA;AACf,4GAAA,YAAY,OAAA;AACZ,6GAAA,aAAa,OAAA;AACb,iHAAA,iBAAiB,OAAA;AACjB,0GAAA,UAAU,OAAA;AACV,mHAAA,mBAAmB,OAAA;AACnB,gHAAA,gBAAgB,OAAA;AAChB,wHAAA,wBAAwB,OAAA;AACxB,6HAAA,6BAA6B,OAAA;AAE7B,iBAAiB;AACjB,2GAAA,WAAW,OAAA;AAEX,sBAAsB;AACtB,qHAAA,qBAAqB,OAAA;AACrB,gHAAA,gBAAgB,OAAA;AAChB,+GAAA,eAAe,OAAA;AAEf,YAAY;AACZ,6GAAA,aAAa,OAAA;AAWf,kCAAkC;AAClC,6CAuBuB;AAtBrB,aAAa;AACb,8GAAA,eAAe,OAAA;AAEf,qBAAqB;AACrB,8GAAA,eAAe,OAAA;AAEf,QAAQ;AACR,yGAAA,UAAU,OAAA;AAEV,cAAc;AACd,uGAAA,QAAQ,OAAA;AACR,wGAAA,SAAS,OAAA;AAaX,0BAA0B;AAC1B,+CAWwB;AAVtB,aAAa;AACb,wGAAA,QAAQ,OAAA;AAER,kBAAkB;AAClB,6GAAA,aAAa,OAAA"}
|
||||
118
vendor/ruvector/npm/packages/ruvector-extensions/src/index.ts
vendored
Normal file
118
vendor/ruvector/npm/packages/ruvector-extensions/src/index.ts
vendored
Normal file
@@ -0,0 +1,118 @@
|
||||
/**
|
||||
* @fileoverview ruvector-extensions - Advanced features for ruvector
|
||||
*
|
||||
* Provides embeddings integration, UI components, export utilities,
|
||||
* temporal tracking, and persistence layers for ruvector vector database.
|
||||
*
|
||||
* @module ruvector-extensions
|
||||
* @author ruv.io Team <info@ruv.io>
|
||||
* @license MIT
|
||||
*/
|
||||
|
||||
// Export embeddings module
|
||||
export {
|
||||
// Base class
|
||||
EmbeddingProvider,
|
||||
|
||||
// Provider implementations
|
||||
OpenAIEmbeddings,
|
||||
CohereEmbeddings,
|
||||
AnthropicEmbeddings,
|
||||
HuggingFaceEmbeddings,
|
||||
|
||||
// Helper functions
|
||||
embedAndInsert,
|
||||
embedAndSearch,
|
||||
|
||||
// Types and interfaces
|
||||
type RetryConfig,
|
||||
type EmbeddingResult,
|
||||
type BatchEmbeddingResult,
|
||||
type EmbeddingError,
|
||||
type DocumentToEmbed,
|
||||
type OpenAIEmbeddingsConfig,
|
||||
type CohereEmbeddingsConfig,
|
||||
type AnthropicEmbeddingsConfig,
|
||||
type HuggingFaceEmbeddingsConfig,
|
||||
} from './embeddings.js';
|
||||
|
||||
// Re-export default for convenience
|
||||
export { default as embeddings } from './embeddings.js';
|
||||
|
||||
// Export graph exporters module
|
||||
export {
|
||||
// Graph builders
|
||||
buildGraphFromEntries,
|
||||
buildGraphFromVectorDB,
|
||||
|
||||
// Format exporters
|
||||
exportToGraphML,
|
||||
exportToGEXF,
|
||||
exportToNeo4j,
|
||||
exportToNeo4jJSON,
|
||||
exportToD3,
|
||||
exportToD3Hierarchy,
|
||||
exportToNetworkX,
|
||||
exportToNetworkXEdgeList,
|
||||
exportToNetworkXAdjacencyList,
|
||||
|
||||
// Unified export
|
||||
exportGraph,
|
||||
|
||||
// Streaming exporters
|
||||
GraphMLStreamExporter,
|
||||
D3StreamExporter,
|
||||
streamToGraphML,
|
||||
|
||||
// Utilities
|
||||
validateGraph,
|
||||
|
||||
// Types
|
||||
type Graph,
|
||||
type GraphNode,
|
||||
type GraphEdge,
|
||||
type ExportOptions,
|
||||
type ExportFormat,
|
||||
type ExportResult
|
||||
} from './exporters.js';
|
||||
|
||||
// Export temporal tracking module
|
||||
export {
|
||||
// Main class
|
||||
TemporalTracker,
|
||||
|
||||
// Singleton instance
|
||||
temporalTracker,
|
||||
|
||||
// Enums
|
||||
ChangeType,
|
||||
|
||||
// Type guards
|
||||
isChange,
|
||||
isVersion,
|
||||
|
||||
// Types
|
||||
type Change,
|
||||
type Version,
|
||||
type VersionDiff,
|
||||
type AuditLogEntry,
|
||||
type CreateVersionOptions,
|
||||
type QueryOptions,
|
||||
type VisualizationData,
|
||||
type TemporalTrackerEvents,
|
||||
} from './temporal.js';
|
||||
|
||||
// Export UI server module
|
||||
export {
|
||||
// Main class
|
||||
UIServer,
|
||||
|
||||
// Helper function
|
||||
startUIServer,
|
||||
|
||||
// Types
|
||||
type GraphNode as UIGraphNode,
|
||||
type GraphLink,
|
||||
type GraphData,
|
||||
} from "./ui-server.js";
|
||||
|
||||
336
vendor/ruvector/npm/packages/ruvector-extensions/src/persistence.d.ts
vendored
Normal file
336
vendor/ruvector/npm/packages/ruvector-extensions/src/persistence.d.ts
vendored
Normal file
@@ -0,0 +1,336 @@
|
||||
/**
|
||||
* Database Persistence Module for ruvector-extensions
|
||||
*
|
||||
* Provides comprehensive database persistence capabilities including:
|
||||
* - Multiple save formats (JSON, Binary/MessagePack, SQLite)
|
||||
* - Incremental saves (only changed data)
|
||||
* - Snapshot management (create, list, restore, delete)
|
||||
* - Export/import functionality
|
||||
* - Compression support
|
||||
* - Progress callbacks for large operations
|
||||
*
|
||||
* @module persistence
|
||||
*/
|
||||
import type { VectorEntry, DbOptions, DbStats } from 'ruvector';
|
||||
type VectorDBInstance = any;
|
||||
/**
|
||||
* Supported persistence formats
|
||||
*/
|
||||
export type PersistenceFormat = 'json' | 'binary' | 'sqlite';
|
||||
/**
|
||||
* Compression algorithms
|
||||
*/
|
||||
export type CompressionType = 'none' | 'gzip' | 'brotli';
|
||||
/**
|
||||
* Progress callback for long-running operations
|
||||
*/
|
||||
export type ProgressCallback = (progress: {
|
||||
/** Operation being performed */
|
||||
operation: string;
|
||||
/** Current progress (0-100) */
|
||||
percentage: number;
|
||||
/** Number of items processed */
|
||||
current: number;
|
||||
/** Total items to process */
|
||||
total: number;
|
||||
/** Human-readable message */
|
||||
message: string;
|
||||
}) => void;
|
||||
/**
|
||||
* Persistence configuration options
|
||||
*/
|
||||
export interface PersistenceOptions {
|
||||
/** Base directory for persistence files */
|
||||
baseDir: string;
|
||||
/** Default format for saves */
|
||||
format?: PersistenceFormat;
|
||||
/** Enable compression */
|
||||
compression?: CompressionType;
|
||||
/** Enable incremental saves */
|
||||
incremental?: boolean;
|
||||
/** Auto-save interval in milliseconds (0 = disabled) */
|
||||
autoSaveInterval?: number;
|
||||
/** Maximum number of snapshots to keep */
|
||||
maxSnapshots?: number;
|
||||
/** Batch size for large operations */
|
||||
batchSize?: number;
|
||||
}
|
||||
/**
|
||||
* Database snapshot metadata
|
||||
*/
|
||||
export interface SnapshotMetadata {
|
||||
/** Snapshot identifier */
|
||||
id: string;
|
||||
/** Human-readable name */
|
||||
name: string;
|
||||
/** Creation timestamp */
|
||||
timestamp: number;
|
||||
/** Vector count at snapshot time */
|
||||
vectorCount: number;
|
||||
/** Database dimension */
|
||||
dimension: number;
|
||||
/** Format used */
|
||||
format: PersistenceFormat;
|
||||
/** Whether compressed */
|
||||
compressed: boolean;
|
||||
/** File size in bytes */
|
||||
fileSize: number;
|
||||
/** Checksum for integrity */
|
||||
checksum: string;
|
||||
/** Additional metadata */
|
||||
metadata?: Record<string, any>;
|
||||
}
|
||||
/**
|
||||
* Serialized database state
|
||||
*/
|
||||
export interface DatabaseState {
|
||||
/** Format version for compatibility */
|
||||
version: string;
|
||||
/** Database configuration */
|
||||
options: DbOptions;
|
||||
/** Database statistics */
|
||||
stats: DbStats;
|
||||
/** Vector entries */
|
||||
vectors: VectorEntry[];
|
||||
/** Index state (opaque) */
|
||||
indexState?: any;
|
||||
/** Additional metadata */
|
||||
metadata?: Record<string, any>;
|
||||
/** Timestamp of save */
|
||||
timestamp: number;
|
||||
/** Checksum for integrity */
|
||||
checksum?: string;
|
||||
}
|
||||
/**
|
||||
* Export options
|
||||
*/
|
||||
export interface ExportOptions {
|
||||
/** Output file path */
|
||||
path: string;
|
||||
/** Export format */
|
||||
format?: PersistenceFormat;
|
||||
/** Enable compression */
|
||||
compress?: boolean;
|
||||
/** Include index state */
|
||||
includeIndex?: boolean;
|
||||
/** Progress callback */
|
||||
onProgress?: ProgressCallback;
|
||||
}
|
||||
/**
|
||||
* Import options
|
||||
*/
|
||||
export interface ImportOptions {
|
||||
/** Input file path */
|
||||
path: string;
|
||||
/** Expected format (auto-detect if not specified) */
|
||||
format?: PersistenceFormat;
|
||||
/** Whether to clear database before import */
|
||||
clear?: boolean;
|
||||
/** Verify checksum */
|
||||
verifyChecksum?: boolean;
|
||||
/** Progress callback */
|
||||
onProgress?: ProgressCallback;
|
||||
}
|
||||
/**
|
||||
* Main persistence manager for VectorDB instances
|
||||
*
|
||||
* @example
|
||||
* ```typescript
|
||||
* const db = new VectorDB({ dimension: 384 });
|
||||
* const persistence = new DatabasePersistence(db, {
|
||||
* baseDir: './data',
|
||||
* format: 'binary',
|
||||
* compression: 'gzip',
|
||||
* incremental: true
|
||||
* });
|
||||
*
|
||||
* // Save database
|
||||
* await persistence.save({ onProgress: (p) => console.log(p.message) });
|
||||
*
|
||||
* // Create snapshot
|
||||
* const snapshot = await persistence.createSnapshot('before-update');
|
||||
*
|
||||
* // Restore from snapshot
|
||||
* await persistence.restoreSnapshot(snapshot.id);
|
||||
* ```
|
||||
*/
|
||||
export declare class DatabasePersistence {
|
||||
private db;
|
||||
private options;
|
||||
private incrementalState;
|
||||
private autoSaveTimer;
|
||||
/**
|
||||
* Create a new database persistence manager
|
||||
*
|
||||
* @param db - VectorDB instance to manage
|
||||
* @param options - Persistence configuration
|
||||
*/
|
||||
constructor(db: VectorDBInstance, options: PersistenceOptions);
|
||||
/**
|
||||
* Initialize persistence system
|
||||
*/
|
||||
private initialize;
|
||||
/**
|
||||
* Save database to disk
|
||||
*
|
||||
* @param options - Save options
|
||||
* @returns Path to saved file
|
||||
*/
|
||||
save(options?: {
|
||||
path?: string;
|
||||
format?: PersistenceFormat;
|
||||
compress?: boolean;
|
||||
onProgress?: ProgressCallback;
|
||||
}): Promise<string>;
|
||||
/**
|
||||
* Save only changed data (incremental save)
|
||||
*
|
||||
* @param options - Save options
|
||||
* @returns Path to saved file or null if no changes
|
||||
*/
|
||||
saveIncremental(options?: {
|
||||
path?: string;
|
||||
format?: PersistenceFormat;
|
||||
onProgress?: ProgressCallback;
|
||||
}): Promise<string | null>;
|
||||
/**
|
||||
* Load database from disk
|
||||
*
|
||||
* @param options - Load options
|
||||
*/
|
||||
load(options: {
|
||||
path: string;
|
||||
format?: PersistenceFormat;
|
||||
verifyChecksum?: boolean;
|
||||
onProgress?: ProgressCallback;
|
||||
}): Promise<void>;
|
||||
/**
|
||||
* Create a snapshot of the current database state
|
||||
*
|
||||
* @param name - Human-readable snapshot name
|
||||
* @param metadata - Additional metadata to store
|
||||
* @returns Snapshot metadata
|
||||
*/
|
||||
createSnapshot(name: string, metadata?: Record<string, any>): Promise<SnapshotMetadata>;
|
||||
/**
|
||||
* List all available snapshots
|
||||
*
|
||||
* @returns Array of snapshot metadata, sorted by timestamp (newest first)
|
||||
*/
|
||||
listSnapshots(): Promise<SnapshotMetadata[]>;
|
||||
/**
|
||||
* Restore database from a snapshot
|
||||
*
|
||||
* @param snapshotId - Snapshot ID to restore
|
||||
* @param options - Restore options
|
||||
*/
|
||||
restoreSnapshot(snapshotId: string, options?: {
|
||||
verifyChecksum?: boolean;
|
||||
onProgress?: ProgressCallback;
|
||||
}): Promise<void>;
|
||||
/**
|
||||
* Delete a snapshot
|
||||
*
|
||||
* @param snapshotId - Snapshot ID to delete
|
||||
*/
|
||||
deleteSnapshot(snapshotId: string): Promise<void>;
|
||||
/**
|
||||
* Export database to a file
|
||||
*
|
||||
* @param options - Export options
|
||||
*/
|
||||
export(options: ExportOptions): Promise<void>;
|
||||
/**
|
||||
* Import database from a file
|
||||
*
|
||||
* @param options - Import options
|
||||
*/
|
||||
import(options: ImportOptions): Promise<void>;
|
||||
/**
|
||||
* Start automatic saves at configured interval
|
||||
*/
|
||||
startAutoSave(): void;
|
||||
/**
|
||||
* Stop automatic saves
|
||||
*/
|
||||
stopAutoSave(): void;
|
||||
/**
|
||||
* Cleanup and shutdown
|
||||
*/
|
||||
shutdown(): Promise<void>;
|
||||
/**
|
||||
* Serialize database to state object
|
||||
*/
|
||||
private serializeDatabase;
|
||||
/**
|
||||
* Deserialize state object into database
|
||||
*/
|
||||
private deserializeDatabase;
|
||||
/**
|
||||
* Write state to file in specified format
|
||||
*/
|
||||
private writeStateToFile;
|
||||
/**
|
||||
* Read state from file in specified format
|
||||
*/
|
||||
private readStateFromFile;
|
||||
/**
|
||||
* Get all vector IDs from database
|
||||
*/
|
||||
private getAllVectorIds;
|
||||
/**
|
||||
* Compute checksum of state object
|
||||
*/
|
||||
private computeChecksum;
|
||||
/**
|
||||
* Compute checksum of file
|
||||
*/
|
||||
private computeFileChecksum;
|
||||
/**
|
||||
* Detect file format from extension
|
||||
*/
|
||||
private detectFormat;
|
||||
/**
|
||||
* Check if data is compressed
|
||||
*/
|
||||
private isCompressed;
|
||||
/**
|
||||
* Get default save path
|
||||
*/
|
||||
private getDefaultSavePath;
|
||||
/**
|
||||
* Load incremental state
|
||||
*/
|
||||
private loadIncrementalState;
|
||||
/**
|
||||
* Update incremental state after save
|
||||
*/
|
||||
private updateIncrementalState;
|
||||
/**
|
||||
* Clean up old snapshots beyond max limit
|
||||
*/
|
||||
private cleanupOldSnapshots;
|
||||
}
|
||||
/**
|
||||
* Format file size in human-readable format
|
||||
*
|
||||
* @param bytes - File size in bytes
|
||||
* @returns Formatted string (e.g., "1.5 MB")
|
||||
*/
|
||||
export declare function formatFileSize(bytes: number): string;
|
||||
/**
|
||||
* Format timestamp as ISO string
|
||||
*
|
||||
* @param timestamp - Unix timestamp in milliseconds
|
||||
* @returns ISO formatted date string
|
||||
*/
|
||||
export declare function formatTimestamp(timestamp: number): string;
|
||||
/**
|
||||
* Estimate memory usage of database state
|
||||
*
|
||||
* @param state - Database state
|
||||
* @returns Estimated memory usage in bytes
|
||||
*/
|
||||
export declare function estimateMemoryUsage(state: DatabaseState): number;
|
||||
export {};
|
||||
//# sourceMappingURL=persistence.d.ts.map
|
||||
1
vendor/ruvector/npm/packages/ruvector-extensions/src/persistence.d.ts.map
vendored
Normal file
1
vendor/ruvector/npm/packages/ruvector-extensions/src/persistence.d.ts.map
vendored
Normal file
@@ -0,0 +1 @@
|
||||
{"version":3,"file":"persistence.d.ts","sourceRoot":"","sources":["persistence.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAOH,OAAO,KAAK,EAAE,WAAW,EAAE,SAAS,EAAE,OAAO,EAAE,MAAM,UAAU,CAAC;AAGhE,KAAK,gBAAgB,GAAG,GAAG,CAAC;AAM5B;;GAEG;AACH,MAAM,MAAM,iBAAiB,GAAG,MAAM,GAAG,QAAQ,GAAG,QAAQ,CAAC;AAE7D;;GAEG;AACH,MAAM,MAAM,eAAe,GAAG,MAAM,GAAG,MAAM,GAAG,QAAQ,CAAC;AAEzD;;GAEG;AACH,MAAM,MAAM,gBAAgB,GAAG,CAAC,QAAQ,EAAE;IACxC,gCAAgC;IAChC,SAAS,EAAE,MAAM,CAAC;IAClB,+BAA+B;IAC/B,UAAU,EAAE,MAAM,CAAC;IACnB,gCAAgC;IAChC,OAAO,EAAE,MAAM,CAAC;IAChB,6BAA6B;IAC7B,KAAK,EAAE,MAAM,CAAC;IACd,6BAA6B;IAC7B,OAAO,EAAE,MAAM,CAAC;CACjB,KAAK,IAAI,CAAC;AAEX;;GAEG;AACH,MAAM,WAAW,kBAAkB;IACjC,2CAA2C;IAC3C,OAAO,EAAE,MAAM,CAAC;IAChB,+BAA+B;IAC/B,MAAM,CAAC,EAAE,iBAAiB,CAAC;IAC3B,yBAAyB;IACzB,WAAW,CAAC,EAAE,eAAe,CAAC;IAC9B,+BAA+B;IAC/B,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB,wDAAwD;IACxD,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,0CAA0C;IAC1C,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,sCAAsC;IACtC,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,0BAA0B;IAC1B,EAAE,EAAE,MAAM,CAAC;IACX,0BAA0B;IAC1B,IAAI,EAAE,MAAM,CAAC;IACb,yBAAyB;IACzB,SAAS,EAAE,MAAM,CAAC;IAClB,oCAAoC;IACpC,WAAW,EAAE,MAAM,CAAC;IACpB,yBAAyB;IACzB,SAAS,EAAE,MAAM,CAAC;IAClB,kBAAkB;IAClB,MAAM,EAAE,iBAAiB,CAAC;IAC1B,yBAAyB;IACzB,UAAU,EAAE,OAAO,CAAC;IACpB,yBAAyB;IACzB,QAAQ,EAAE,MAAM,CAAC;IACjB,6BAA6B;IAC7B,QAAQ,EAAE,MAAM,CAAC;IACjB,0BAA0B;IAC1B,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;CAChC;AAED;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,uCAAuC;IACvC,OAAO,EAAE,MAAM,CAAC;IAChB,6BAA6B;IAC7B,OAAO,EAAE,SAAS,CAAC;IACnB,0BAA0B;IAC1B,KAAK,EAAE,OAAO,CAAC;IACf,qBAAqB;IACrB,OAAO,EAAE,WAAW,EAAE,CAAC;IACvB,2BAA2B;IAC3B,UAAU,CAAC,EAAE,GAAG,CAAC;IACjB,0BAA0B;IAC1B,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;IAC/B,wBAAwB;IACxB,SAAS,EAAE,MAAM,CAAC;IAClB,6BAA6B;IAC7B,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAcD;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,uBAAuB;IACvB,IAAI,EAAE,MAAM,CAAC;IACb,oBAAoB;IACpB,MAAM,CAAC,EAAE,iBAAiB,CAAC;IAC3B,yBAAyB;IACzB,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,0BAA0B;IAC1B,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB,wBAAwB;IACxB,UAAU,CAAC,EAAE,gBAAgB,CAAC;CAC/B;AAED;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,sBAAsB;IACtB,IAAI,EAAE,MAAM,CAAC;IACb,qDAAqD;IACrD,MAAM,CAAC,EAAE,iBAAiB,CAAC;IAC3B,8CAA8C;IAC9C,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,sBAAsB;IACtB,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB,wBAAwB;IACxB,UAAU,CAAC,EAAE,gBAAgB,CAAC;CAC/B;AAMD;;;;;;;;;;;;;;;;;;;;;;GAsBG;AACH,qBAAa,mBAAmB;IAC9B,OAAO,CAAC,EAAE,CAAmB;IAC7B,OAAO,CAAC,OAAO,CAA+B;IAC9C,OAAO,CAAC,gBAAgB,CAAiC;IACzD,OAAO,CAAC,aAAa,CAA+B;IAEpD;;;;;OAKG;gBACS,EAAE,EAAE,gBAAgB,EAAE,OAAO,EAAE,kBAAkB;IAe7D;;OAEG;YACW,UAAU;IAoBxB;;;;;OAKG;IACG,IAAI,CAAC,OAAO,GAAE;QAClB,IAAI,CAAC,EAAE,MAAM,CAAC;QACd,MAAM,CAAC,EAAE,iBAAiB,CAAC;QAC3B,QAAQ,CAAC,EAAE,OAAO,CAAC;QACnB,UAAU,CAAC,EAAE,gBAAgB,CAAC;KAC1B,GAAG,OAAO,CAAC,MAAM,CAAC;IAoCxB;;;;;OAKG;IACG,eAAe,CAAC,OAAO,GAAE;QAC7B,IAAI,CAAC,EAAE,MAAM,CAAC;QACd,MAAM,CAAC,EAAE,iBAAiB,CAAC;QAC3B,UAAU,CAAC,EAAE,gBAAgB,CAAC;KAC1B,GAAG,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC;IAmC/B;;;;OAIG;IACG,IAAI,CAAC,OAAO,EAAE;QAClB,IAAI,EAAE,MAAM,CAAC;QACb,MAAM,CAAC,EAAE,iBAAiB,CAAC;QAC3B,cAAc,CAAC,EAAE,OAAO,CAAC;QACzB,UAAU,CAAC,EAAE,gBAAgB,CAAC;KAC/B,GAAG,OAAO,CAAC,IAAI,CAAC;IAiDjB;;;;;;OAMG;IACG,cAAc,CAClB,IAAI,EAAE,MAAM,EACZ,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,GAC7B,OAAO,CAAC,gBAAgB,CAAC;IA+C5B;;;;OAIG;IACG,aAAa,IAAI,OAAO,CAAC,gBAAgB,EAAE,CAAC;IAelD;;;;;OAKG;IACG,eAAe,CACnB,UAAU,EAAE,MAAM,EAClB,OAAO,GAAE;QACP,cAAc,CAAC,EAAE,OAAO,CAAC;QACzB,UAAU,CAAC,EAAE,gBAAgB,CAAC;KAC1B,GACL,OAAO,CAAC,IAAI,CAAC;IAuChB;;;;OAIG;IACG,cAAc,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAwBvD;;;;OAIG;IACG,MAAM,CAAC,OAAO,EAAE,aAAa,GAAG,OAAO,CAAC,IAAI,CAAC;IAanD;;;;OAIG;IACG,MAAM,CAAC,OAAO,EAAE,aAAa,GAAG,OAAO,CAAC,IAAI,CAAC;IAiBnD;;OAEG;IACH,aAAa,IAAI,IAAI;IAkBrB;;OAEG;IACH,YAAY,IAAI,IAAI;IAOpB;;OAEG;IACG,QAAQ,IAAI,OAAO,CAAC,IAAI,CAAC;IAa/B;;OAEG;YACW,iBAAiB;IAwE/B;;OAEG;YACW,mBAAmB;IAyDjC;;OAEG;YACW,gBAAgB;IA4C9B;;OAEG;YACW,iBAAiB;IAmC/B;;OAEG;YACW,eAAe;IAoB7B;;OAEG;IACH,OAAO,CAAC,eAAe;IAMvB;;OAEG;YACW,mBAAmB;IAWjC;;OAEG;IACH,OAAO,CAAC,YAAY;IAQpB;;OAEG;IACH,OAAO,CAAC,YAAY;IAOpB;;OAEG;IACH,OAAO,CAAC,kBAAkB;IAM1B;;OAEG;YACW,oBAAoB;IAelC;;OAEG;YACW,sBAAsB;IAmBpC;;OAEG;YACW,mBAAmB;CAalC;AAMD;;;;;GAKG;AACH,wBAAgB,cAAc,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAWpD;AAED;;;;;GAKG;AACH,wBAAgB,eAAe,CAAC,SAAS,EAAE,MAAM,GAAG,MAAM,CAEzD;AAED;;;;;GAKG;AACH,wBAAgB,mBAAmB,CAAC,KAAK,EAAE,aAAa,GAAG,MAAM,CAQhE"}
|
||||
774
vendor/ruvector/npm/packages/ruvector-extensions/src/persistence.js
vendored
Normal file
774
vendor/ruvector/npm/packages/ruvector-extensions/src/persistence.js
vendored
Normal file
@@ -0,0 +1,774 @@
|
||||
"use strict";
|
||||
/**
|
||||
* Database Persistence Module for ruvector-extensions
|
||||
*
|
||||
* Provides comprehensive database persistence capabilities including:
|
||||
* - Multiple save formats (JSON, Binary/MessagePack, SQLite)
|
||||
* - Incremental saves (only changed data)
|
||||
* - Snapshot management (create, list, restore, delete)
|
||||
* - Export/import functionality
|
||||
* - Compression support
|
||||
* - Progress callbacks for large operations
|
||||
*
|
||||
* @module persistence
|
||||
*/
|
||||
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
||||
if (k2 === undefined) k2 = k;
|
||||
var desc = Object.getOwnPropertyDescriptor(m, k);
|
||||
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
||||
desc = { enumerable: true, get: function() { return m[k]; } };
|
||||
}
|
||||
Object.defineProperty(o, k2, desc);
|
||||
}) : (function(o, m, k, k2) {
|
||||
if (k2 === undefined) k2 = k;
|
||||
o[k2] = m[k];
|
||||
}));
|
||||
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
||||
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
||||
}) : function(o, v) {
|
||||
o["default"] = v;
|
||||
});
|
||||
var __importStar = (this && this.__importStar) || (function () {
|
||||
var ownKeys = function(o) {
|
||||
ownKeys = Object.getOwnPropertyNames || function (o) {
|
||||
var ar = [];
|
||||
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
|
||||
return ar;
|
||||
};
|
||||
return ownKeys(o);
|
||||
};
|
||||
return function (mod) {
|
||||
if (mod && mod.__esModule) return mod;
|
||||
var result = {};
|
||||
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
|
||||
__setModuleDefault(result, mod);
|
||||
return result;
|
||||
};
|
||||
})();
|
||||
Object.defineProperty(exports, "__esModule", { value: true });
|
||||
exports.DatabasePersistence = void 0;
|
||||
exports.formatFileSize = formatFileSize;
|
||||
exports.formatTimestamp = formatTimestamp;
|
||||
exports.estimateMemoryUsage = estimateMemoryUsage;
|
||||
const fs_1 = require("fs");
|
||||
const fs_2 = require("fs");
|
||||
const path = __importStar(require("path"));
|
||||
const crypto = __importStar(require("crypto"));
|
||||
// ============================================================================
|
||||
// Database Persistence Manager
|
||||
// ============================================================================
|
||||
/**
|
||||
* Main persistence manager for VectorDB instances
|
||||
*
|
||||
* @example
|
||||
* ```typescript
|
||||
* const db = new VectorDB({ dimension: 384 });
|
||||
* const persistence = new DatabasePersistence(db, {
|
||||
* baseDir: './data',
|
||||
* format: 'binary',
|
||||
* compression: 'gzip',
|
||||
* incremental: true
|
||||
* });
|
||||
*
|
||||
* // Save database
|
||||
* await persistence.save({ onProgress: (p) => console.log(p.message) });
|
||||
*
|
||||
* // Create snapshot
|
||||
* const snapshot = await persistence.createSnapshot('before-update');
|
||||
*
|
||||
* // Restore from snapshot
|
||||
* await persistence.restoreSnapshot(snapshot.id);
|
||||
* ```
|
||||
*/
|
||||
class DatabasePersistence {
|
||||
/**
|
||||
* Create a new database persistence manager
|
||||
*
|
||||
* @param db - VectorDB instance to manage
|
||||
* @param options - Persistence configuration
|
||||
*/
|
||||
constructor(db, options) {
|
||||
this.incrementalState = null;
|
||||
this.autoSaveTimer = null;
|
||||
this.db = db;
|
||||
this.options = {
|
||||
baseDir: options.baseDir,
|
||||
format: options.format || 'json',
|
||||
compression: options.compression || 'none',
|
||||
incremental: options.incremental ?? false,
|
||||
autoSaveInterval: options.autoSaveInterval ?? 0,
|
||||
maxSnapshots: options.maxSnapshots ?? 10,
|
||||
batchSize: options.batchSize ?? 1000,
|
||||
};
|
||||
this.initialize();
|
||||
}
|
||||
/**
|
||||
* Initialize persistence system
|
||||
*/
|
||||
async initialize() {
|
||||
// Create base directory if it doesn't exist
|
||||
await fs_1.promises.mkdir(this.options.baseDir, { recursive: true });
|
||||
await fs_1.promises.mkdir(path.join(this.options.baseDir, 'snapshots'), { recursive: true });
|
||||
// Start auto-save if configured
|
||||
if (this.options.autoSaveInterval > 0) {
|
||||
this.startAutoSave();
|
||||
}
|
||||
// Load incremental state if exists
|
||||
if (this.options.incremental) {
|
||||
await this.loadIncrementalState();
|
||||
}
|
||||
}
|
||||
// ==========================================================================
|
||||
// Save Operations
|
||||
// ==========================================================================
|
||||
/**
|
||||
* Save database to disk
|
||||
*
|
||||
* @param options - Save options
|
||||
* @returns Path to saved file
|
||||
*/
|
||||
async save(options = {}) {
|
||||
const format = options.format || this.options.format;
|
||||
const compress = options.compress ?? (this.options.compression !== 'none');
|
||||
const savePath = options.path || this.getDefaultSavePath(format, compress);
|
||||
const state = await this.serializeDatabase(options.onProgress);
|
||||
if (options.onProgress) {
|
||||
options.onProgress({
|
||||
operation: 'save',
|
||||
percentage: 80,
|
||||
current: 4,
|
||||
total: 5,
|
||||
message: 'Writing to disk...',
|
||||
});
|
||||
}
|
||||
await this.writeStateToFile(state, savePath, format, compress);
|
||||
if (this.options.incremental) {
|
||||
await this.updateIncrementalState(state);
|
||||
}
|
||||
if (options.onProgress) {
|
||||
options.onProgress({
|
||||
operation: 'save',
|
||||
percentage: 100,
|
||||
current: 5,
|
||||
total: 5,
|
||||
message: 'Save completed',
|
||||
});
|
||||
}
|
||||
return savePath;
|
||||
}
|
||||
/**
|
||||
* Save only changed data (incremental save)
|
||||
*
|
||||
* @param options - Save options
|
||||
* @returns Path to saved file or null if no changes
|
||||
*/
|
||||
async saveIncremental(options = {}) {
|
||||
if (!this.incrementalState) {
|
||||
// First save, do full save
|
||||
return this.save(options);
|
||||
}
|
||||
const stats = this.db.stats();
|
||||
const currentVectors = await this.getAllVectorIds();
|
||||
// Detect changes
|
||||
const added = currentVectors.filter(id => !this.incrementalState.vectorIds.has(id));
|
||||
const removed = Array.from(this.incrementalState.vectorIds).filter(id => !currentVectors.includes(id));
|
||||
if (added.length === 0 && removed.length === 0) {
|
||||
// No changes
|
||||
return null;
|
||||
}
|
||||
if (options.onProgress) {
|
||||
options.onProgress({
|
||||
operation: 'incremental-save',
|
||||
percentage: 20,
|
||||
current: 1,
|
||||
total: 5,
|
||||
message: `Found ${added.length} new and ${removed.length} removed vectors`,
|
||||
});
|
||||
}
|
||||
// For now, do a full save with changes
|
||||
// In a production system, you'd implement delta encoding
|
||||
return this.save(options);
|
||||
}
|
||||
/**
|
||||
* Load database from disk
|
||||
*
|
||||
* @param options - Load options
|
||||
*/
|
||||
async load(options) {
|
||||
const format = options.format || this.detectFormat(options.path);
|
||||
if (options.onProgress) {
|
||||
options.onProgress({
|
||||
operation: 'load',
|
||||
percentage: 10,
|
||||
current: 1,
|
||||
total: 5,
|
||||
message: 'Reading from disk...',
|
||||
});
|
||||
}
|
||||
const state = await this.readStateFromFile(options.path, format);
|
||||
if (options.verifyChecksum && state.checksum) {
|
||||
if (options.onProgress) {
|
||||
options.onProgress({
|
||||
operation: 'load',
|
||||
percentage: 30,
|
||||
current: 2,
|
||||
total: 5,
|
||||
message: 'Verifying checksum...',
|
||||
});
|
||||
}
|
||||
const computed = this.computeChecksum(state);
|
||||
if (computed !== state.checksum) {
|
||||
throw new Error('Checksum verification failed - file may be corrupted');
|
||||
}
|
||||
}
|
||||
await this.deserializeDatabase(state, options.onProgress);
|
||||
if (options.onProgress) {
|
||||
options.onProgress({
|
||||
operation: 'load',
|
||||
percentage: 100,
|
||||
current: 5,
|
||||
total: 5,
|
||||
message: 'Load completed',
|
||||
});
|
||||
}
|
||||
}
|
||||
// ==========================================================================
|
||||
// Snapshot Management
|
||||
// ==========================================================================
|
||||
/**
|
||||
* Create a snapshot of the current database state
|
||||
*
|
||||
* @param name - Human-readable snapshot name
|
||||
* @param metadata - Additional metadata to store
|
||||
* @returns Snapshot metadata
|
||||
*/
|
||||
async createSnapshot(name, metadata) {
|
||||
const id = crypto.randomUUID();
|
||||
const timestamp = Date.now();
|
||||
const stats = this.db.stats();
|
||||
const snapshotPath = path.join(this.options.baseDir, 'snapshots', `${id}.${this.options.format}`);
|
||||
await this.save({
|
||||
path: snapshotPath,
|
||||
format: this.options.format,
|
||||
compress: this.options.compression !== 'none',
|
||||
});
|
||||
const fileStats = await fs_1.promises.stat(snapshotPath);
|
||||
const checksum = await this.computeFileChecksum(snapshotPath);
|
||||
const snapshotMetadata = {
|
||||
id,
|
||||
name,
|
||||
timestamp,
|
||||
vectorCount: stats.count,
|
||||
dimension: stats.dimension,
|
||||
format: this.options.format,
|
||||
compressed: this.options.compression !== 'none',
|
||||
fileSize: fileStats.size,
|
||||
checksum,
|
||||
metadata,
|
||||
};
|
||||
// Save metadata
|
||||
const metadataPath = path.join(this.options.baseDir, 'snapshots', `${id}.meta.json`);
|
||||
await fs_1.promises.writeFile(metadataPath, JSON.stringify(snapshotMetadata, null, 2));
|
||||
// Clean up old snapshots
|
||||
await this.cleanupOldSnapshots();
|
||||
return snapshotMetadata;
|
||||
}
|
||||
/**
|
||||
* List all available snapshots
|
||||
*
|
||||
* @returns Array of snapshot metadata, sorted by timestamp (newest first)
|
||||
*/
|
||||
async listSnapshots() {
|
||||
const snapshotsDir = path.join(this.options.baseDir, 'snapshots');
|
||||
const files = await fs_1.promises.readdir(snapshotsDir);
|
||||
const metadataFiles = files.filter(f => f.endsWith('.meta.json'));
|
||||
const snapshots = [];
|
||||
for (const file of metadataFiles) {
|
||||
const content = await fs_1.promises.readFile(path.join(snapshotsDir, file), 'utf-8');
|
||||
snapshots.push(JSON.parse(content));
|
||||
}
|
||||
return snapshots.sort((a, b) => b.timestamp - a.timestamp);
|
||||
}
|
||||
/**
|
||||
* Restore database from a snapshot
|
||||
*
|
||||
* @param snapshotId - Snapshot ID to restore
|
||||
* @param options - Restore options
|
||||
*/
|
||||
async restoreSnapshot(snapshotId, options = {}) {
|
||||
const snapshotsDir = path.join(this.options.baseDir, 'snapshots');
|
||||
const metadataPath = path.join(snapshotsDir, `${snapshotId}.meta.json`);
|
||||
let metadata;
|
||||
try {
|
||||
const content = await fs_1.promises.readFile(metadataPath, 'utf-8');
|
||||
metadata = JSON.parse(content);
|
||||
}
|
||||
catch (error) {
|
||||
throw new Error(`Snapshot ${snapshotId} not found`);
|
||||
}
|
||||
const snapshotPath = path.join(snapshotsDir, `${snapshotId}.${metadata.format}`);
|
||||
if (options.verifyChecksum) {
|
||||
if (options.onProgress) {
|
||||
options.onProgress({
|
||||
operation: 'restore',
|
||||
percentage: 10,
|
||||
current: 1,
|
||||
total: 5,
|
||||
message: 'Verifying snapshot integrity...',
|
||||
});
|
||||
}
|
||||
const checksum = await this.computeFileChecksum(snapshotPath);
|
||||
if (checksum !== metadata.checksum) {
|
||||
throw new Error('Snapshot checksum verification failed - file may be corrupted');
|
||||
}
|
||||
}
|
||||
await this.load({
|
||||
path: snapshotPath,
|
||||
format: metadata.format,
|
||||
verifyChecksum: false, // Already verified above if needed
|
||||
onProgress: options.onProgress,
|
||||
});
|
||||
}
|
||||
/**
|
||||
* Delete a snapshot
|
||||
*
|
||||
* @param snapshotId - Snapshot ID to delete
|
||||
*/
|
||||
async deleteSnapshot(snapshotId) {
|
||||
const snapshotsDir = path.join(this.options.baseDir, 'snapshots');
|
||||
const metadataPath = path.join(snapshotsDir, `${snapshotId}.meta.json`);
|
||||
let metadata;
|
||||
try {
|
||||
const content = await fs_1.promises.readFile(metadataPath, 'utf-8');
|
||||
metadata = JSON.parse(content);
|
||||
}
|
||||
catch (error) {
|
||||
throw new Error(`Snapshot ${snapshotId} not found`);
|
||||
}
|
||||
const snapshotPath = path.join(snapshotsDir, `${snapshotId}.${metadata.format}`);
|
||||
await Promise.all([
|
||||
fs_1.promises.unlink(snapshotPath).catch(() => { }),
|
||||
fs_1.promises.unlink(metadataPath).catch(() => { }),
|
||||
]);
|
||||
}
|
||||
// ==========================================================================
|
||||
// Export/Import
|
||||
// ==========================================================================
|
||||
/**
|
||||
* Export database to a file
|
||||
*
|
||||
* @param options - Export options
|
||||
*/
|
||||
async export(options) {
|
||||
const format = options.format || 'json';
|
||||
const compress = options.compress ?? false;
|
||||
const state = await this.serializeDatabase(options.onProgress);
|
||||
if (!options.includeIndex) {
|
||||
delete state.indexState;
|
||||
}
|
||||
await this.writeStateToFile(state, options.path, format, compress);
|
||||
}
|
||||
/**
|
||||
* Import database from a file
|
||||
*
|
||||
* @param options - Import options
|
||||
*/
|
||||
async import(options) {
|
||||
if (options.clear) {
|
||||
this.db.clear();
|
||||
}
|
||||
await this.load({
|
||||
path: options.path,
|
||||
format: options.format,
|
||||
verifyChecksum: options.verifyChecksum,
|
||||
onProgress: options.onProgress,
|
||||
});
|
||||
}
|
||||
// ==========================================================================
|
||||
// Auto-Save
|
||||
// ==========================================================================
|
||||
/**
|
||||
* Start automatic saves at configured interval
|
||||
*/
|
||||
startAutoSave() {
|
||||
if (this.autoSaveTimer) {
|
||||
return; // Already running
|
||||
}
|
||||
this.autoSaveTimer = setInterval(async () => {
|
||||
try {
|
||||
if (this.options.incremental) {
|
||||
await this.saveIncremental();
|
||||
}
|
||||
else {
|
||||
await this.save();
|
||||
}
|
||||
}
|
||||
catch (error) {
|
||||
console.error('Auto-save failed:', error);
|
||||
}
|
||||
}, this.options.autoSaveInterval);
|
||||
}
|
||||
/**
|
||||
* Stop automatic saves
|
||||
*/
|
||||
stopAutoSave() {
|
||||
if (this.autoSaveTimer) {
|
||||
clearInterval(this.autoSaveTimer);
|
||||
this.autoSaveTimer = null;
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Cleanup and shutdown
|
||||
*/
|
||||
async shutdown() {
|
||||
this.stopAutoSave();
|
||||
// Do final save if auto-save was enabled
|
||||
if (this.options.autoSaveInterval > 0) {
|
||||
await this.save();
|
||||
}
|
||||
}
|
||||
// ==========================================================================
|
||||
// Private Helper Methods
|
||||
// ==========================================================================
|
||||
/**
|
||||
* Serialize database to state object
|
||||
*/
|
||||
async serializeDatabase(onProgress) {
|
||||
if (onProgress) {
|
||||
onProgress({
|
||||
operation: 'serialize',
|
||||
percentage: 10,
|
||||
current: 1,
|
||||
total: 5,
|
||||
message: 'Collecting database statistics...',
|
||||
});
|
||||
}
|
||||
const stats = this.db.stats();
|
||||
const vectors = [];
|
||||
if (onProgress) {
|
||||
onProgress({
|
||||
operation: 'serialize',
|
||||
percentage: 30,
|
||||
current: 2,
|
||||
total: 5,
|
||||
message: 'Extracting vectors...',
|
||||
});
|
||||
}
|
||||
// Extract all vectors
|
||||
const vectorIds = await this.getAllVectorIds();
|
||||
for (let i = 0; i < vectorIds.length; i++) {
|
||||
const vector = this.db.get(vectorIds[i]);
|
||||
if (vector) {
|
||||
vectors.push(vector);
|
||||
}
|
||||
if (onProgress && i % this.options.batchSize === 0) {
|
||||
const percentage = 30 + Math.floor((i / vectorIds.length) * 40);
|
||||
onProgress({
|
||||
operation: 'serialize',
|
||||
percentage,
|
||||
current: i,
|
||||
total: vectorIds.length,
|
||||
message: `Extracted ${i}/${vectorIds.length} vectors...`,
|
||||
});
|
||||
}
|
||||
}
|
||||
const state = {
|
||||
version: '1.0.0',
|
||||
options: {
|
||||
dimension: stats.dimension,
|
||||
metric: stats.metric,
|
||||
},
|
||||
stats,
|
||||
vectors,
|
||||
timestamp: Date.now(),
|
||||
};
|
||||
if (onProgress) {
|
||||
onProgress({
|
||||
operation: 'serialize',
|
||||
percentage: 90,
|
||||
current: 4,
|
||||
total: 5,
|
||||
message: 'Computing checksum...',
|
||||
});
|
||||
}
|
||||
state.checksum = this.computeChecksum(state);
|
||||
return state;
|
||||
}
|
||||
/**
|
||||
* Deserialize state object into database
|
||||
*/
|
||||
async deserializeDatabase(state, onProgress) {
|
||||
if (onProgress) {
|
||||
onProgress({
|
||||
operation: 'deserialize',
|
||||
percentage: 40,
|
||||
current: 2,
|
||||
total: 5,
|
||||
message: 'Clearing existing data...',
|
||||
});
|
||||
}
|
||||
this.db.clear();
|
||||
if (onProgress) {
|
||||
onProgress({
|
||||
operation: 'deserialize',
|
||||
percentage: 50,
|
||||
current: 3,
|
||||
total: 5,
|
||||
message: 'Inserting vectors...',
|
||||
});
|
||||
}
|
||||
// Insert vectors in batches
|
||||
for (let i = 0; i < state.vectors.length; i += this.options.batchSize) {
|
||||
const batch = state.vectors.slice(i, i + this.options.batchSize);
|
||||
this.db.insertBatch(batch);
|
||||
if (onProgress) {
|
||||
const percentage = 50 + Math.floor((i / state.vectors.length) * 40);
|
||||
onProgress({
|
||||
operation: 'deserialize',
|
||||
percentage,
|
||||
current: i,
|
||||
total: state.vectors.length,
|
||||
message: `Inserted ${i}/${state.vectors.length} vectors...`,
|
||||
});
|
||||
}
|
||||
}
|
||||
if (onProgress) {
|
||||
onProgress({
|
||||
operation: 'deserialize',
|
||||
percentage: 95,
|
||||
current: 4,
|
||||
total: 5,
|
||||
message: 'Rebuilding index...',
|
||||
});
|
||||
}
|
||||
// Rebuild index
|
||||
this.db.buildIndex();
|
||||
}
|
||||
/**
|
||||
* Write state to file in specified format
|
||||
*/
|
||||
async writeStateToFile(state, filePath, format, compress) {
|
||||
await fs_1.promises.mkdir(path.dirname(filePath), { recursive: true });
|
||||
let data;
|
||||
switch (format) {
|
||||
case 'json':
|
||||
data = Buffer.from(JSON.stringify(state, null, compress ? 0 : 2));
|
||||
break;
|
||||
case 'binary':
|
||||
// Use simple JSON for now - in production, use MessagePack
|
||||
data = Buffer.from(JSON.stringify(state));
|
||||
break;
|
||||
case 'sqlite':
|
||||
// SQLite implementation would go here
|
||||
throw new Error('SQLite format not yet implemented');
|
||||
default:
|
||||
throw new Error(`Unsupported format: ${format}`);
|
||||
}
|
||||
if (compress) {
|
||||
const { gzip, brotliCompress } = await Promise.resolve().then(() => __importStar(require('zlib')));
|
||||
const { promisify } = await Promise.resolve().then(() => __importStar(require('util')));
|
||||
if (this.options.compression === 'gzip') {
|
||||
const gzipAsync = promisify(gzip);
|
||||
data = await gzipAsync(data);
|
||||
}
|
||||
else if (this.options.compression === 'brotli') {
|
||||
const brotliAsync = promisify(brotliCompress);
|
||||
data = await brotliAsync(data);
|
||||
}
|
||||
}
|
||||
await fs_1.promises.writeFile(filePath, data);
|
||||
}
|
||||
/**
|
||||
* Read state from file in specified format
|
||||
*/
|
||||
async readStateFromFile(filePath, format) {
|
||||
let data = await fs_1.promises.readFile(filePath);
|
||||
// Detect and decompress if needed
|
||||
if (this.isCompressed(data)) {
|
||||
const { gunzip, brotliDecompress } = await Promise.resolve().then(() => __importStar(require('zlib')));
|
||||
const { promisify } = await Promise.resolve().then(() => __importStar(require('util')));
|
||||
// Try gzip first
|
||||
try {
|
||||
const gunzipAsync = promisify(gunzip);
|
||||
data = await gunzipAsync(data);
|
||||
}
|
||||
catch {
|
||||
// Try brotli
|
||||
const brotliAsync = promisify(brotliDecompress);
|
||||
data = await brotliAsync(data);
|
||||
}
|
||||
}
|
||||
switch (format) {
|
||||
case 'json':
|
||||
case 'binary':
|
||||
return JSON.parse(data.toString());
|
||||
case 'sqlite':
|
||||
throw new Error('SQLite format not yet implemented');
|
||||
default:
|
||||
throw new Error(`Unsupported format: ${format}`);
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Get all vector IDs from database
|
||||
*/
|
||||
async getAllVectorIds() {
|
||||
// This is a workaround - in production, VectorDB should provide an iterator
|
||||
const stats = this.db.stats();
|
||||
const ids = [];
|
||||
// Try to get vectors by attempting sequential IDs
|
||||
// This is inefficient and should be replaced with a proper API
|
||||
for (let i = 0; i < stats.count * 2; i++) {
|
||||
const vector = this.db.get(String(i));
|
||||
if (vector) {
|
||||
ids.push(vector.id);
|
||||
}
|
||||
if (ids.length >= stats.count) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
return ids;
|
||||
}
|
||||
/**
|
||||
* Compute checksum of state object
|
||||
*/
|
||||
computeChecksum(state) {
|
||||
const { checksum, ...stateWithoutChecksum } = state;
|
||||
const data = JSON.stringify(stateWithoutChecksum);
|
||||
return crypto.createHash('sha256').update(data).digest('hex');
|
||||
}
|
||||
/**
|
||||
* Compute checksum of file
|
||||
*/
|
||||
async computeFileChecksum(filePath) {
|
||||
return new Promise((resolve, reject) => {
|
||||
const hash = crypto.createHash('sha256');
|
||||
const stream = (0, fs_2.createReadStream)(filePath);
|
||||
stream.on('data', data => hash.update(data));
|
||||
stream.on('end', () => resolve(hash.digest('hex')));
|
||||
stream.on('error', reject);
|
||||
});
|
||||
}
|
||||
/**
|
||||
* Detect file format from extension
|
||||
*/
|
||||
detectFormat(filePath) {
|
||||
const ext = path.extname(filePath).toLowerCase();
|
||||
if (ext === '.json')
|
||||
return 'json';
|
||||
if (ext === '.bin' || ext === '.msgpack')
|
||||
return 'binary';
|
||||
if (ext === '.db' || ext === '.sqlite')
|
||||
return 'sqlite';
|
||||
return this.options.format;
|
||||
}
|
||||
/**
|
||||
* Check if data is compressed
|
||||
*/
|
||||
isCompressed(data) {
|
||||
// Gzip magic number: 1f 8b
|
||||
if (data[0] === 0x1f && data[1] === 0x8b)
|
||||
return true;
|
||||
// Brotli doesn't have a magic number, but we can try to decompress
|
||||
return false;
|
||||
}
|
||||
/**
|
||||
* Get default save path
|
||||
*/
|
||||
getDefaultSavePath(format, compress) {
|
||||
const ext = format === 'json' ? 'json' : format === 'binary' ? 'bin' : 'db';
|
||||
const compressExt = compress ? `.${this.options.compression}` : '';
|
||||
return path.join(this.options.baseDir, `database.${ext}${compressExt}`);
|
||||
}
|
||||
/**
|
||||
* Load incremental state
|
||||
*/
|
||||
async loadIncrementalState() {
|
||||
const statePath = path.join(this.options.baseDir, '.incremental.json');
|
||||
try {
|
||||
const content = await fs_1.promises.readFile(statePath, 'utf-8');
|
||||
const data = JSON.parse(content);
|
||||
this.incrementalState = {
|
||||
lastSave: data.lastSave,
|
||||
vectorIds: new Set(data.vectorIds),
|
||||
checksum: data.checksum,
|
||||
};
|
||||
}
|
||||
catch {
|
||||
// No incremental state yet
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Update incremental state after save
|
||||
*/
|
||||
async updateIncrementalState(state) {
|
||||
const vectorIds = state.vectors.map(v => v.id);
|
||||
this.incrementalState = {
|
||||
lastSave: Date.now(),
|
||||
vectorIds: new Set(vectorIds),
|
||||
checksum: state.checksum || '',
|
||||
};
|
||||
const statePath = path.join(this.options.baseDir, '.incremental.json');
|
||||
await fs_1.promises.writeFile(statePath, JSON.stringify({
|
||||
lastSave: this.incrementalState.lastSave,
|
||||
vectorIds: Array.from(this.incrementalState.vectorIds),
|
||||
checksum: this.incrementalState.checksum,
|
||||
}));
|
||||
}
|
||||
/**
|
||||
* Clean up old snapshots beyond max limit
|
||||
*/
|
||||
async cleanupOldSnapshots() {
|
||||
const snapshots = await this.listSnapshots();
|
||||
if (snapshots.length <= this.options.maxSnapshots) {
|
||||
return;
|
||||
}
|
||||
const toDelete = snapshots.slice(this.options.maxSnapshots);
|
||||
for (const snapshot of toDelete) {
|
||||
await this.deleteSnapshot(snapshot.id);
|
||||
}
|
||||
}
|
||||
}
|
||||
exports.DatabasePersistence = DatabasePersistence;
|
||||
// ============================================================================
|
||||
// Utility Functions
|
||||
// ============================================================================
|
||||
/**
|
||||
* Format file size in human-readable format
|
||||
*
|
||||
* @param bytes - File size in bytes
|
||||
* @returns Formatted string (e.g., "1.5 MB")
|
||||
*/
|
||||
function formatFileSize(bytes) {
|
||||
const units = ['B', 'KB', 'MB', 'GB', 'TB'];
|
||||
let size = bytes;
|
||||
let unitIndex = 0;
|
||||
while (size >= 1024 && unitIndex < units.length - 1) {
|
||||
size /= 1024;
|
||||
unitIndex++;
|
||||
}
|
||||
return `${size.toFixed(2)} ${units[unitIndex]}`;
|
||||
}
|
||||
/**
|
||||
* Format timestamp as ISO string
|
||||
*
|
||||
* @param timestamp - Unix timestamp in milliseconds
|
||||
* @returns ISO formatted date string
|
||||
*/
|
||||
function formatTimestamp(timestamp) {
|
||||
return new Date(timestamp).toISOString();
|
||||
}
|
||||
/**
|
||||
* Estimate memory usage of database state
|
||||
*
|
||||
* @param state - Database state
|
||||
* @returns Estimated memory usage in bytes
|
||||
*/
|
||||
function estimateMemoryUsage(state) {
|
||||
// Rough estimation
|
||||
const vectorSize = state.stats.dimension * 4; // 4 bytes per float
|
||||
const metadataSize = 100; // Average metadata size
|
||||
const totalVectorSize = state.vectors.length * (vectorSize + metadataSize);
|
||||
const overheadSize = JSON.stringify(state).length;
|
||||
return totalVectorSize + overheadSize;
|
||||
}
|
||||
//# sourceMappingURL=persistence.js.map
|
||||
1
vendor/ruvector/npm/packages/ruvector-extensions/src/persistence.js.map
vendored
Normal file
1
vendor/ruvector/npm/packages/ruvector-extensions/src/persistence.js.map
vendored
Normal file
File diff suppressed because one or more lines are too long
1040
vendor/ruvector/npm/packages/ruvector-extensions/src/persistence.ts
vendored
Normal file
1040
vendor/ruvector/npm/packages/ruvector-extensions/src/persistence.ts
vendored
Normal file
File diff suppressed because it is too large
Load Diff
390
vendor/ruvector/npm/packages/ruvector-extensions/src/temporal.d.ts
vendored
Normal file
390
vendor/ruvector/npm/packages/ruvector-extensions/src/temporal.d.ts
vendored
Normal file
@@ -0,0 +1,390 @@
|
||||
/**
|
||||
* Temporal Tracking Module for RUVector
|
||||
*
|
||||
* Provides comprehensive version control, change tracking, and time-travel capabilities
|
||||
* for ontology and database evolution over time.
|
||||
*
|
||||
* @module temporal
|
||||
* @author ruv.io Team
|
||||
* @license MIT
|
||||
*/
|
||||
import { EventEmitter } from 'events';
|
||||
/**
|
||||
* Represents the type of change in a version
|
||||
*/
|
||||
export declare enum ChangeType {
|
||||
ADDITION = "addition",
|
||||
DELETION = "deletion",
|
||||
MODIFICATION = "modification",
|
||||
METADATA = "metadata"
|
||||
}
|
||||
/**
|
||||
* Represents a single change in the database
|
||||
*/
|
||||
export interface Change {
|
||||
/** Type of change */
|
||||
type: ChangeType;
|
||||
/** Path to the changed entity (e.g., "nodes.User", "edges.FOLLOWS") */
|
||||
path: string;
|
||||
/** Previous value (null for additions) */
|
||||
before: any;
|
||||
/** New value (null for deletions) */
|
||||
after: any;
|
||||
/** Timestamp of the change */
|
||||
timestamp: number;
|
||||
/** Optional metadata about the change */
|
||||
metadata?: Record<string, any>;
|
||||
}
|
||||
/**
|
||||
* Represents a version snapshot with delta encoding
|
||||
*/
|
||||
export interface Version {
|
||||
/** Unique version identifier */
|
||||
id: string;
|
||||
/** Parent version ID (null for initial version) */
|
||||
parentId: string | null;
|
||||
/** Version creation timestamp */
|
||||
timestamp: number;
|
||||
/** Human-readable version description */
|
||||
description: string;
|
||||
/** List of changes from parent version (delta encoding) */
|
||||
changes: Change[];
|
||||
/** Version tags for easy reference */
|
||||
tags: string[];
|
||||
/** User or system that created the version */
|
||||
author?: string;
|
||||
/** Checksum for integrity verification */
|
||||
checksum: string;
|
||||
/** Additional metadata */
|
||||
metadata: Record<string, any>;
|
||||
}
|
||||
/**
|
||||
* Represents a diff between two versions
|
||||
*/
|
||||
export interface VersionDiff {
|
||||
/** Source version ID */
|
||||
fromVersion: string;
|
||||
/** Target version ID */
|
||||
toVersion: string;
|
||||
/** List of changes between versions */
|
||||
changes: Change[];
|
||||
/** Summary statistics */
|
||||
summary: {
|
||||
additions: number;
|
||||
deletions: number;
|
||||
modifications: number;
|
||||
};
|
||||
/** Timestamp of diff generation */
|
||||
generatedAt: number;
|
||||
}
|
||||
/**
|
||||
* Audit log entry for tracking all operations
|
||||
*/
|
||||
export interface AuditLogEntry {
|
||||
/** Unique log entry ID */
|
||||
id: string;
|
||||
/** Operation type */
|
||||
operation: 'create' | 'revert' | 'query' | 'compare' | 'tag' | 'prune';
|
||||
/** Target version ID */
|
||||
versionId?: string;
|
||||
/** Timestamp of the operation */
|
||||
timestamp: number;
|
||||
/** User or system that performed the operation */
|
||||
actor?: string;
|
||||
/** Operation result status */
|
||||
status: 'success' | 'failure' | 'partial';
|
||||
/** Error message if operation failed */
|
||||
error?: string;
|
||||
/** Additional operation details */
|
||||
details: Record<string, any>;
|
||||
}
|
||||
/**
|
||||
* Options for creating a new version
|
||||
*/
|
||||
export interface CreateVersionOptions {
|
||||
/** Version description */
|
||||
description: string;
|
||||
/** Optional tags for the version */
|
||||
tags?: string[];
|
||||
/** Author of the version */
|
||||
author?: string;
|
||||
/** Additional metadata */
|
||||
metadata?: Record<string, any>;
|
||||
}
|
||||
/**
|
||||
* Options for querying historical data
|
||||
*/
|
||||
export interface QueryOptions {
|
||||
/** Target timestamp for time-travel query */
|
||||
timestamp?: number;
|
||||
/** Target version ID */
|
||||
versionId?: string;
|
||||
/** Filter by path pattern */
|
||||
pathPattern?: RegExp;
|
||||
/** Include metadata in results */
|
||||
includeMetadata?: boolean;
|
||||
}
|
||||
/**
|
||||
* Visualization data for change history
|
||||
*/
|
||||
export interface VisualizationData {
|
||||
/** Version timeline */
|
||||
timeline: Array<{
|
||||
versionId: string;
|
||||
timestamp: number;
|
||||
description: string;
|
||||
changeCount: number;
|
||||
tags: string[];
|
||||
}>;
|
||||
/** Change frequency over time */
|
||||
changeFrequency: Array<{
|
||||
timestamp: number;
|
||||
count: number;
|
||||
type: ChangeType;
|
||||
}>;
|
||||
/** Most frequently changed paths */
|
||||
hotspots: Array<{
|
||||
path: string;
|
||||
changeCount: number;
|
||||
lastChanged: number;
|
||||
}>;
|
||||
/** Version graph (parent-child relationships) */
|
||||
versionGraph: {
|
||||
nodes: Array<{
|
||||
id: string;
|
||||
label: string;
|
||||
timestamp: number;
|
||||
}>;
|
||||
edges: Array<{
|
||||
from: string;
|
||||
to: string;
|
||||
}>;
|
||||
};
|
||||
}
|
||||
/**
|
||||
* Temporal Tracker Events
|
||||
*/
|
||||
export interface TemporalTrackerEvents {
|
||||
versionCreated: [version: Version];
|
||||
versionReverted: [fromVersion: string, toVersion: string];
|
||||
changeTracked: [change: Change];
|
||||
auditLogged: [entry: AuditLogEntry];
|
||||
error: [error: Error];
|
||||
}
|
||||
/**
|
||||
* TemporalTracker - Main class for temporal tracking functionality
|
||||
*
|
||||
* Provides version management, change tracking, time-travel queries,
|
||||
* and audit logging for database evolution over time.
|
||||
*
|
||||
* @example
|
||||
* ```typescript
|
||||
* const tracker = new TemporalTracker();
|
||||
*
|
||||
* // Create initial version
|
||||
* const v1 = await tracker.createVersion({
|
||||
* description: 'Initial schema',
|
||||
* tags: ['v1.0']
|
||||
* });
|
||||
*
|
||||
* // Track changes
|
||||
* tracker.trackChange({
|
||||
* type: ChangeType.ADDITION,
|
||||
* path: 'nodes.User',
|
||||
* before: null,
|
||||
* after: { name: 'User', properties: ['id', 'name'] },
|
||||
* timestamp: Date.now()
|
||||
* });
|
||||
*
|
||||
* // Create new version with tracked changes
|
||||
* const v2 = await tracker.createVersion({
|
||||
* description: 'Added User node',
|
||||
* tags: ['v1.1']
|
||||
* });
|
||||
*
|
||||
* // Time-travel query
|
||||
* const snapshot = await tracker.queryAtTimestamp(v1.timestamp);
|
||||
*
|
||||
* // Compare versions
|
||||
* const diff = await tracker.compareVersions(v1.id, v2.id);
|
||||
* ```
|
||||
*/
|
||||
export declare class TemporalTracker extends EventEmitter {
|
||||
private versions;
|
||||
private currentState;
|
||||
private pendingChanges;
|
||||
private auditLog;
|
||||
private tagIndex;
|
||||
private pathIndex;
|
||||
constructor();
|
||||
/**
|
||||
* Initialize with a baseline empty version
|
||||
*/
|
||||
private initializeBaseline;
|
||||
/**
|
||||
* Generate a unique ID
|
||||
*/
|
||||
private generateId;
|
||||
/**
|
||||
* Calculate checksum for data integrity
|
||||
*/
|
||||
private calculateChecksum;
|
||||
/**
|
||||
* Index a version for fast lookups
|
||||
*/
|
||||
private indexVersion;
|
||||
/**
|
||||
* Track a change to be included in the next version
|
||||
*
|
||||
* @param change - The change to track
|
||||
* @emits changeTracked
|
||||
*/
|
||||
trackChange(change: Change): void;
|
||||
/**
|
||||
* Create a new version with all pending changes
|
||||
*
|
||||
* @param options - Version creation options
|
||||
* @returns The created version
|
||||
* @emits versionCreated
|
||||
*/
|
||||
createVersion(options: CreateVersionOptions): Promise<Version>;
|
||||
/**
|
||||
* Apply a change to the state object
|
||||
*/
|
||||
private applyChange;
|
||||
/**
|
||||
* Get the current (latest) version
|
||||
*/
|
||||
private getCurrentVersion;
|
||||
/**
|
||||
* List all versions, optionally filtered by tags
|
||||
*
|
||||
* @param tags - Optional tags to filter by
|
||||
* @returns Array of versions
|
||||
*/
|
||||
listVersions(tags?: string[]): Version[];
|
||||
/**
|
||||
* Get a specific version by ID
|
||||
*
|
||||
* @param versionId - Version ID
|
||||
* @returns The version or null if not found
|
||||
*/
|
||||
getVersion(versionId: string): Version | null;
|
||||
/**
|
||||
* Compare two versions and generate a diff
|
||||
*
|
||||
* @param fromVersionId - Source version ID
|
||||
* @param toVersionId - Target version ID
|
||||
* @returns Version diff
|
||||
*/
|
||||
compareVersions(fromVersionId: string, toVersionId: string): Promise<VersionDiff>;
|
||||
/**
|
||||
* Generate diff between two states
|
||||
*/
|
||||
private generateDiff;
|
||||
/**
|
||||
* Revert to a specific version
|
||||
*
|
||||
* @param versionId - Target version ID
|
||||
* @returns The new current version (revert creates a new version)
|
||||
* @emits versionReverted
|
||||
*/
|
||||
revertToVersion(versionId: string): Promise<Version>;
|
||||
/**
|
||||
* Reconstruct the database state at a specific version
|
||||
*
|
||||
* @param versionId - Target version ID
|
||||
* @returns Reconstructed state
|
||||
*/
|
||||
private reconstructStateAt;
|
||||
/**
|
||||
* Query the database state at a specific timestamp or version
|
||||
*
|
||||
* @param options - Query options
|
||||
* @returns Reconstructed state at the specified time/version
|
||||
*/
|
||||
queryAtTimestamp(timestamp: number): Promise<any>;
|
||||
queryAtTimestamp(options: QueryOptions): Promise<any>;
|
||||
/**
|
||||
* Filter state by path pattern
|
||||
*/
|
||||
private filterByPath;
|
||||
/**
|
||||
* Strip metadata from state
|
||||
*/
|
||||
private stripMetadata;
|
||||
/**
|
||||
* Add tags to a version
|
||||
*
|
||||
* @param versionId - Version ID
|
||||
* @param tags - Tags to add
|
||||
*/
|
||||
addTags(versionId: string, tags: string[]): void;
|
||||
/**
|
||||
* Get visualization data for change history
|
||||
*
|
||||
* @returns Visualization data
|
||||
*/
|
||||
getVisualizationData(): VisualizationData;
|
||||
/**
|
||||
* Get audit log entries
|
||||
*
|
||||
* @param limit - Maximum number of entries to return
|
||||
* @returns Audit log entries
|
||||
*/
|
||||
getAuditLog(limit?: number): AuditLogEntry[];
|
||||
/**
|
||||
* Log an audit entry
|
||||
*/
|
||||
private logAudit;
|
||||
/**
|
||||
* Prune old versions to save space
|
||||
*
|
||||
* @param keepCount - Number of recent versions to keep
|
||||
* @param preserveTags - Tags to preserve regardless of age
|
||||
*/
|
||||
pruneVersions(keepCount: number, preserveTags?: string[]): void;
|
||||
/**
|
||||
* Export all versions and audit log for backup
|
||||
*
|
||||
* @returns Serializable backup data
|
||||
*/
|
||||
exportBackup(): {
|
||||
versions: Version[];
|
||||
auditLog: AuditLogEntry[];
|
||||
currentState: any;
|
||||
exportedAt: number;
|
||||
};
|
||||
/**
|
||||
* Import versions and state from backup
|
||||
*
|
||||
* @param backup - Backup data to import
|
||||
*/
|
||||
importBackup(backup: ReturnType<typeof this.exportBackup>): void;
|
||||
/**
|
||||
* Get storage statistics
|
||||
*
|
||||
* @returns Storage statistics
|
||||
*/
|
||||
getStorageStats(): {
|
||||
versionCount: number;
|
||||
totalChanges: number;
|
||||
auditLogSize: number;
|
||||
estimatedSizeBytes: number;
|
||||
oldestVersion: number;
|
||||
newestVersion: number;
|
||||
};
|
||||
}
|
||||
/**
|
||||
* Export singleton instance for convenience
|
||||
*/
|
||||
export declare const temporalTracker: TemporalTracker;
|
||||
/**
|
||||
* Type guard for Change
|
||||
*/
|
||||
export declare function isChange(obj: any): obj is Change;
|
||||
/**
|
||||
* Type guard for Version
|
||||
*/
|
||||
export declare function isVersion(obj: any): obj is Version;
|
||||
//# sourceMappingURL=temporal.d.ts.map
|
||||
1
vendor/ruvector/npm/packages/ruvector-extensions/src/temporal.d.ts.map
vendored
Normal file
1
vendor/ruvector/npm/packages/ruvector-extensions/src/temporal.d.ts.map
vendored
Normal file
File diff suppressed because one or more lines are too long
797
vendor/ruvector/npm/packages/ruvector-extensions/src/temporal.js
vendored
Normal file
797
vendor/ruvector/npm/packages/ruvector-extensions/src/temporal.js
vendored
Normal file
@@ -0,0 +1,797 @@
|
||||
"use strict";
|
||||
/**
|
||||
* Temporal Tracking Module for RUVector
|
||||
*
|
||||
* Provides comprehensive version control, change tracking, and time-travel capabilities
|
||||
* for ontology and database evolution over time.
|
||||
*
|
||||
* @module temporal
|
||||
* @author ruv.io Team
|
||||
* @license MIT
|
||||
*/
|
||||
Object.defineProperty(exports, "__esModule", { value: true });
|
||||
exports.temporalTracker = exports.TemporalTracker = exports.ChangeType = void 0;
|
||||
exports.isChange = isChange;
|
||||
exports.isVersion = isVersion;
|
||||
const events_1 = require("events");
|
||||
const crypto_1 = require("crypto");
|
||||
/**
|
||||
* Represents the type of change in a version
|
||||
*/
|
||||
var ChangeType;
|
||||
(function (ChangeType) {
|
||||
ChangeType["ADDITION"] = "addition";
|
||||
ChangeType["DELETION"] = "deletion";
|
||||
ChangeType["MODIFICATION"] = "modification";
|
||||
ChangeType["METADATA"] = "metadata";
|
||||
})(ChangeType || (exports.ChangeType = ChangeType = {}));
|
||||
/**
|
||||
* TemporalTracker - Main class for temporal tracking functionality
|
||||
*
|
||||
* Provides version management, change tracking, time-travel queries,
|
||||
* and audit logging for database evolution over time.
|
||||
*
|
||||
* @example
|
||||
* ```typescript
|
||||
* const tracker = new TemporalTracker();
|
||||
*
|
||||
* // Create initial version
|
||||
* const v1 = await tracker.createVersion({
|
||||
* description: 'Initial schema',
|
||||
* tags: ['v1.0']
|
||||
* });
|
||||
*
|
||||
* // Track changes
|
||||
* tracker.trackChange({
|
||||
* type: ChangeType.ADDITION,
|
||||
* path: 'nodes.User',
|
||||
* before: null,
|
||||
* after: { name: 'User', properties: ['id', 'name'] },
|
||||
* timestamp: Date.now()
|
||||
* });
|
||||
*
|
||||
* // Create new version with tracked changes
|
||||
* const v2 = await tracker.createVersion({
|
||||
* description: 'Added User node',
|
||||
* tags: ['v1.1']
|
||||
* });
|
||||
*
|
||||
* // Time-travel query
|
||||
* const snapshot = await tracker.queryAtTimestamp(v1.timestamp);
|
||||
*
|
||||
* // Compare versions
|
||||
* const diff = await tracker.compareVersions(v1.id, v2.id);
|
||||
* ```
|
||||
*/
|
||||
class TemporalTracker extends events_1.EventEmitter {
|
||||
constructor() {
|
||||
super();
|
||||
this.versions = new Map();
|
||||
this.currentState = {};
|
||||
this.pendingChanges = [];
|
||||
this.auditLog = [];
|
||||
this.tagIndex = new Map(); // tag -> versionIds
|
||||
this.pathIndex = new Map(); // path -> changes
|
||||
this.initializeBaseline();
|
||||
}
|
||||
/**
|
||||
* Initialize with a baseline empty version
|
||||
*/
|
||||
initializeBaseline() {
|
||||
const baseline = {
|
||||
id: this.generateId(),
|
||||
parentId: null,
|
||||
timestamp: 0, // Baseline is always at timestamp 0
|
||||
description: 'Baseline version',
|
||||
changes: [],
|
||||
tags: ['baseline'],
|
||||
checksum: this.calculateChecksum({}),
|
||||
metadata: {}
|
||||
};
|
||||
this.versions.set(baseline.id, baseline);
|
||||
this.indexVersion(baseline);
|
||||
}
|
||||
/**
|
||||
* Generate a unique ID
|
||||
*/
|
||||
generateId() {
|
||||
return `${Date.now()}-${Math.random().toString(36).substr(2, 9)}`;
|
||||
}
|
||||
/**
|
||||
* Calculate checksum for data integrity
|
||||
*/
|
||||
calculateChecksum(data) {
|
||||
const hash = (0, crypto_1.createHash)('sha256');
|
||||
hash.update(JSON.stringify(data));
|
||||
return hash.digest('hex');
|
||||
}
|
||||
/**
|
||||
* Index a version for fast lookups
|
||||
*/
|
||||
indexVersion(version) {
|
||||
// Index tags
|
||||
version.tags.forEach(tag => {
|
||||
if (!this.tagIndex.has(tag)) {
|
||||
this.tagIndex.set(tag, new Set());
|
||||
}
|
||||
this.tagIndex.get(tag).add(version.id);
|
||||
});
|
||||
// Index changes by path
|
||||
version.changes.forEach(change => {
|
||||
if (!this.pathIndex.has(change.path)) {
|
||||
this.pathIndex.set(change.path, []);
|
||||
}
|
||||
this.pathIndex.get(change.path).push(change);
|
||||
});
|
||||
}
|
||||
/**
|
||||
* Track a change to be included in the next version
|
||||
*
|
||||
* @param change - The change to track
|
||||
* @emits changeTracked
|
||||
*/
|
||||
trackChange(change) {
|
||||
this.pendingChanges.push(change);
|
||||
this.emit('changeTracked', change);
|
||||
}
|
||||
/**
|
||||
* Create a new version with all pending changes
|
||||
*
|
||||
* @param options - Version creation options
|
||||
* @returns The created version
|
||||
* @emits versionCreated
|
||||
*/
|
||||
async createVersion(options) {
|
||||
const startTime = Date.now();
|
||||
try {
|
||||
// Get current version (latest)
|
||||
const currentVersion = this.getCurrentVersion();
|
||||
// Reconstruct current state from all versions
|
||||
if (currentVersion) {
|
||||
this.currentState = await this.reconstructStateAt(currentVersion.id);
|
||||
}
|
||||
// Apply pending changes to current state
|
||||
this.pendingChanges.forEach(change => {
|
||||
this.applyChange(this.currentState, change);
|
||||
});
|
||||
// Create new version
|
||||
const version = {
|
||||
id: this.generateId(),
|
||||
parentId: currentVersion?.id || null,
|
||||
timestamp: Date.now(),
|
||||
description: options.description,
|
||||
changes: [...this.pendingChanges],
|
||||
tags: options.tags || [],
|
||||
author: options.author,
|
||||
checksum: this.calculateChecksum(this.currentState),
|
||||
metadata: options.metadata || {}
|
||||
};
|
||||
// Store version
|
||||
this.versions.set(version.id, version);
|
||||
this.indexVersion(version);
|
||||
// Clear pending changes
|
||||
this.pendingChanges = [];
|
||||
// Log audit entry
|
||||
this.logAudit({
|
||||
operation: 'create',
|
||||
versionId: version.id,
|
||||
status: 'success',
|
||||
details: {
|
||||
description: options.description,
|
||||
changeCount: version.changes.length,
|
||||
duration: Date.now() - startTime
|
||||
}
|
||||
});
|
||||
this.emit('versionCreated', version);
|
||||
return version;
|
||||
}
|
||||
catch (error) {
|
||||
this.logAudit({
|
||||
operation: 'create',
|
||||
status: 'failure',
|
||||
error: error instanceof Error ? error.message : String(error),
|
||||
details: { options }
|
||||
});
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Apply a change to the state object
|
||||
*/
|
||||
applyChange(state, change) {
|
||||
const pathParts = change.path.split('.');
|
||||
let current = state;
|
||||
// Navigate to parent
|
||||
for (let i = 0; i < pathParts.length - 1; i++) {
|
||||
if (!(pathParts[i] in current)) {
|
||||
current[pathParts[i]] = {};
|
||||
}
|
||||
current = current[pathParts[i]];
|
||||
}
|
||||
const key = pathParts[pathParts.length - 1];
|
||||
// Apply change
|
||||
switch (change.type) {
|
||||
case ChangeType.ADDITION:
|
||||
case ChangeType.MODIFICATION:
|
||||
// Deep clone to avoid reference issues
|
||||
current[key] = JSON.parse(JSON.stringify(change.after));
|
||||
break;
|
||||
case ChangeType.DELETION:
|
||||
delete current[key];
|
||||
break;
|
||||
case ChangeType.METADATA:
|
||||
if (!current[key])
|
||||
current[key] = {};
|
||||
Object.assign(current[key], JSON.parse(JSON.stringify(change.after)));
|
||||
break;
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Get the current (latest) version
|
||||
*/
|
||||
getCurrentVersion() {
|
||||
if (this.versions.size === 0)
|
||||
return null;
|
||||
const versions = Array.from(this.versions.values());
|
||||
return versions.reduce((latest, current) => current.timestamp > latest.timestamp ? current : latest);
|
||||
}
|
||||
/**
|
||||
* List all versions, optionally filtered by tags
|
||||
*
|
||||
* @param tags - Optional tags to filter by
|
||||
* @returns Array of versions
|
||||
*/
|
||||
listVersions(tags) {
|
||||
let versionIds = null;
|
||||
// Filter by tags if provided
|
||||
if (tags && tags.length > 0) {
|
||||
versionIds = new Set();
|
||||
tags.forEach(tag => {
|
||||
const taggedVersions = this.tagIndex.get(tag);
|
||||
if (taggedVersions) {
|
||||
taggedVersions.forEach(id => versionIds.add(id));
|
||||
}
|
||||
});
|
||||
}
|
||||
const versions = Array.from(this.versions.values());
|
||||
const filtered = versionIds
|
||||
? versions.filter(v => versionIds.has(v.id))
|
||||
: versions;
|
||||
return filtered.sort((a, b) => b.timestamp - a.timestamp);
|
||||
}
|
||||
/**
|
||||
* Get a specific version by ID
|
||||
*
|
||||
* @param versionId - Version ID
|
||||
* @returns The version or null if not found
|
||||
*/
|
||||
getVersion(versionId) {
|
||||
return this.versions.get(versionId) || null;
|
||||
}
|
||||
/**
|
||||
* Compare two versions and generate a diff
|
||||
*
|
||||
* @param fromVersionId - Source version ID
|
||||
* @param toVersionId - Target version ID
|
||||
* @returns Version diff
|
||||
*/
|
||||
async compareVersions(fromVersionId, toVersionId) {
|
||||
const startTime = Date.now();
|
||||
try {
|
||||
const fromVersion = this.versions.get(fromVersionId);
|
||||
const toVersion = this.versions.get(toVersionId);
|
||||
if (!fromVersion || !toVersion) {
|
||||
throw new Error('Version not found');
|
||||
}
|
||||
// Reconstruct state at both versions
|
||||
const fromState = await this.reconstructStateAt(fromVersionId);
|
||||
const toState = await this.reconstructStateAt(toVersionId);
|
||||
// Generate diff
|
||||
const changes = this.generateDiff(fromState, toState, '');
|
||||
// Calculate summary
|
||||
const summary = {
|
||||
additions: changes.filter(c => c.type === ChangeType.ADDITION).length,
|
||||
deletions: changes.filter(c => c.type === ChangeType.DELETION).length,
|
||||
modifications: changes.filter(c => c.type === ChangeType.MODIFICATION).length
|
||||
};
|
||||
const diff = {
|
||||
fromVersion: fromVersionId,
|
||||
toVersion: toVersionId,
|
||||
changes,
|
||||
summary,
|
||||
generatedAt: Date.now()
|
||||
};
|
||||
this.logAudit({
|
||||
operation: 'compare',
|
||||
status: 'success',
|
||||
details: {
|
||||
fromVersion: fromVersionId,
|
||||
toVersion: toVersionId,
|
||||
changeCount: changes.length,
|
||||
duration: Date.now() - startTime
|
||||
}
|
||||
});
|
||||
return diff;
|
||||
}
|
||||
catch (error) {
|
||||
this.logAudit({
|
||||
operation: 'compare',
|
||||
status: 'failure',
|
||||
error: error instanceof Error ? error.message : String(error),
|
||||
details: { fromVersionId, toVersionId }
|
||||
});
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Generate diff between two states
|
||||
*/
|
||||
generateDiff(from, to, path) {
|
||||
const changes = [];
|
||||
const timestamp = Date.now();
|
||||
// Check all keys in 'to' state
|
||||
for (const key in to) {
|
||||
const currentPath = path ? `${path}.${key}` : key;
|
||||
const fromValue = from?.[key];
|
||||
const toValue = to[key];
|
||||
if (!(key in (from || {}))) {
|
||||
// Addition
|
||||
changes.push({
|
||||
type: ChangeType.ADDITION,
|
||||
path: currentPath,
|
||||
before: null,
|
||||
after: toValue,
|
||||
timestamp
|
||||
});
|
||||
}
|
||||
else if (typeof toValue === 'object' && toValue !== null && !Array.isArray(toValue)) {
|
||||
// Recurse into object
|
||||
changes.push(...this.generateDiff(fromValue, toValue, currentPath));
|
||||
}
|
||||
else if (JSON.stringify(fromValue) !== JSON.stringify(toValue)) {
|
||||
// Modification
|
||||
changes.push({
|
||||
type: ChangeType.MODIFICATION,
|
||||
path: currentPath,
|
||||
before: fromValue,
|
||||
after: toValue,
|
||||
timestamp
|
||||
});
|
||||
}
|
||||
}
|
||||
// Check for deletions
|
||||
for (const key in from) {
|
||||
if (!(key in to)) {
|
||||
const currentPath = path ? `${path}.${key}` : key;
|
||||
changes.push({
|
||||
type: ChangeType.DELETION,
|
||||
path: currentPath,
|
||||
before: from[key],
|
||||
after: null,
|
||||
timestamp
|
||||
});
|
||||
}
|
||||
}
|
||||
return changes;
|
||||
}
|
||||
/**
|
||||
* Revert to a specific version
|
||||
*
|
||||
* @param versionId - Target version ID
|
||||
* @returns The new current version (revert creates a new version)
|
||||
* @emits versionReverted
|
||||
*/
|
||||
async revertToVersion(versionId) {
|
||||
const startTime = Date.now();
|
||||
const currentVersion = this.getCurrentVersion();
|
||||
try {
|
||||
const targetVersion = this.versions.get(versionId);
|
||||
if (!targetVersion) {
|
||||
throw new Error('Target version not found');
|
||||
}
|
||||
// Reconstruct state at target version
|
||||
const targetState = await this.reconstructStateAt(versionId);
|
||||
// Generate changes from current to target
|
||||
const revertChanges = this.generateDiff(this.currentState, targetState, '');
|
||||
// Create new version with revert changes
|
||||
this.pendingChanges = revertChanges;
|
||||
const revertVersion = await this.createVersion({
|
||||
description: `Revert to version: ${targetVersion.description}`,
|
||||
tags: ['revert'],
|
||||
metadata: {
|
||||
revertedFrom: currentVersion?.id,
|
||||
revertedTo: versionId
|
||||
}
|
||||
});
|
||||
this.logAudit({
|
||||
operation: 'revert',
|
||||
versionId: revertVersion.id,
|
||||
status: 'success',
|
||||
details: {
|
||||
targetVersion: versionId,
|
||||
changeCount: revertChanges.length,
|
||||
duration: Date.now() - startTime
|
||||
}
|
||||
});
|
||||
this.emit('versionReverted', currentVersion?.id || '', versionId);
|
||||
return revertVersion;
|
||||
}
|
||||
catch (error) {
|
||||
this.logAudit({
|
||||
operation: 'revert',
|
||||
status: 'failure',
|
||||
error: error instanceof Error ? error.message : String(error),
|
||||
details: { versionId }
|
||||
});
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Reconstruct the database state at a specific version
|
||||
*
|
||||
* @param versionId - Target version ID
|
||||
* @returns Reconstructed state
|
||||
*/
|
||||
async reconstructStateAt(versionId) {
|
||||
const version = this.versions.get(versionId);
|
||||
if (!version) {
|
||||
throw new Error('Version not found');
|
||||
}
|
||||
// Build version chain from baseline to target
|
||||
const chain = [];
|
||||
let current = version;
|
||||
while (current) {
|
||||
chain.unshift(current);
|
||||
current = current.parentId ? this.versions.get(current.parentId) || null : null;
|
||||
}
|
||||
// Apply changes in sequence to a fresh state
|
||||
const state = {};
|
||||
for (const v of chain) {
|
||||
v.changes.forEach(change => {
|
||||
this.applyChange(state, change);
|
||||
});
|
||||
}
|
||||
// Deep clone to avoid reference issues
|
||||
return JSON.parse(JSON.stringify(state));
|
||||
}
|
||||
async queryAtTimestamp(timestampOrOptions) {
|
||||
const startTime = Date.now();
|
||||
try {
|
||||
const options = typeof timestampOrOptions === 'number'
|
||||
? { timestamp: timestampOrOptions }
|
||||
: timestampOrOptions;
|
||||
let targetVersion = null;
|
||||
if (options.versionId) {
|
||||
targetVersion = this.versions.get(options.versionId) || null;
|
||||
}
|
||||
else if (options.timestamp) {
|
||||
// Find version closest to timestamp
|
||||
const versions = Array.from(this.versions.values())
|
||||
.filter(v => v.timestamp <= options.timestamp)
|
||||
.sort((a, b) => b.timestamp - a.timestamp);
|
||||
targetVersion = versions[0] || null;
|
||||
}
|
||||
if (!targetVersion) {
|
||||
throw new Error('No version found matching criteria');
|
||||
}
|
||||
let state = await this.reconstructStateAt(targetVersion.id);
|
||||
// Apply path filter if provided
|
||||
if (options.pathPattern) {
|
||||
state = this.filterByPath(state, options.pathPattern, '');
|
||||
}
|
||||
// Strip metadata if not requested
|
||||
if (!options.includeMetadata) {
|
||||
state = this.stripMetadata(state);
|
||||
}
|
||||
this.logAudit({
|
||||
operation: 'query',
|
||||
versionId: targetVersion.id,
|
||||
status: 'success',
|
||||
details: {
|
||||
options,
|
||||
duration: Date.now() - startTime
|
||||
}
|
||||
});
|
||||
return state;
|
||||
}
|
||||
catch (error) {
|
||||
this.logAudit({
|
||||
operation: 'query',
|
||||
status: 'failure',
|
||||
error: error instanceof Error ? error.message : String(error),
|
||||
details: { options: timestampOrOptions }
|
||||
});
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Filter state by path pattern
|
||||
*/
|
||||
filterByPath(state, pattern, currentPath) {
|
||||
const filtered = {};
|
||||
for (const key in state) {
|
||||
const path = currentPath ? `${currentPath}.${key}` : key;
|
||||
if (pattern.test(path)) {
|
||||
filtered[key] = state[key];
|
||||
}
|
||||
else if (typeof state[key] === 'object' && state[key] !== null) {
|
||||
const nested = this.filterByPath(state[key], pattern, path);
|
||||
if (Object.keys(nested).length > 0) {
|
||||
filtered[key] = nested;
|
||||
}
|
||||
}
|
||||
}
|
||||
return filtered;
|
||||
}
|
||||
/**
|
||||
* Strip metadata from state
|
||||
*/
|
||||
stripMetadata(state) {
|
||||
const cleaned = Array.isArray(state) ? [] : {};
|
||||
for (const key in state) {
|
||||
if (key === 'metadata')
|
||||
continue;
|
||||
if (typeof state[key] === 'object' && state[key] !== null) {
|
||||
cleaned[key] = this.stripMetadata(state[key]);
|
||||
}
|
||||
else {
|
||||
cleaned[key] = state[key];
|
||||
}
|
||||
}
|
||||
return cleaned;
|
||||
}
|
||||
/**
|
||||
* Add tags to a version
|
||||
*
|
||||
* @param versionId - Version ID
|
||||
* @param tags - Tags to add
|
||||
*/
|
||||
addTags(versionId, tags) {
|
||||
const version = this.versions.get(versionId);
|
||||
if (!version) {
|
||||
throw new Error('Version not found');
|
||||
}
|
||||
tags.forEach(tag => {
|
||||
if (!version.tags.includes(tag)) {
|
||||
version.tags.push(tag);
|
||||
if (!this.tagIndex.has(tag)) {
|
||||
this.tagIndex.set(tag, new Set());
|
||||
}
|
||||
this.tagIndex.get(tag).add(versionId);
|
||||
}
|
||||
});
|
||||
this.logAudit({
|
||||
operation: 'tag',
|
||||
versionId,
|
||||
status: 'success',
|
||||
details: { tags }
|
||||
});
|
||||
}
|
||||
/**
|
||||
* Get visualization data for change history
|
||||
*
|
||||
* @returns Visualization data
|
||||
*/
|
||||
getVisualizationData() {
|
||||
const versions = Array.from(this.versions.values());
|
||||
// Timeline
|
||||
const timeline = versions
|
||||
.sort((a, b) => a.timestamp - b.timestamp)
|
||||
.map(v => ({
|
||||
versionId: v.id,
|
||||
timestamp: v.timestamp,
|
||||
description: v.description,
|
||||
changeCount: v.changes.length,
|
||||
tags: v.tags
|
||||
}));
|
||||
// Change frequency
|
||||
const frequencyMap = new Map();
|
||||
versions.forEach(v => {
|
||||
const hourBucket = Math.floor(v.timestamp / (1000 * 60 * 60)) * (1000 * 60 * 60);
|
||||
if (!frequencyMap.has(hourBucket)) {
|
||||
frequencyMap.set(hourBucket, new Map());
|
||||
}
|
||||
const bucket = frequencyMap.get(hourBucket);
|
||||
v.changes.forEach(change => {
|
||||
bucket.set(change.type, (bucket.get(change.type) || 0) + 1);
|
||||
});
|
||||
});
|
||||
const changeFrequency = [];
|
||||
frequencyMap.forEach((typeCounts, timestamp) => {
|
||||
typeCounts.forEach((count, type) => {
|
||||
changeFrequency.push({ timestamp, count, type });
|
||||
});
|
||||
});
|
||||
// Hotspots
|
||||
const pathStats = new Map();
|
||||
this.pathIndex.forEach((changes, path) => {
|
||||
const lastChange = changes[changes.length - 1];
|
||||
pathStats.set(path, {
|
||||
count: changes.length,
|
||||
lastChanged: lastChange.timestamp
|
||||
});
|
||||
});
|
||||
const hotspots = Array.from(pathStats.entries())
|
||||
.map(([path, stats]) => ({
|
||||
path,
|
||||
changeCount: stats.count,
|
||||
lastChanged: stats.lastChanged
|
||||
}))
|
||||
.sort((a, b) => b.changeCount - a.changeCount)
|
||||
.slice(0, 20);
|
||||
// Version graph
|
||||
const versionGraph = {
|
||||
nodes: versions.map(v => ({
|
||||
id: v.id,
|
||||
label: v.description,
|
||||
timestamp: v.timestamp
|
||||
})),
|
||||
edges: versions
|
||||
.filter(v => v.parentId)
|
||||
.map(v => ({
|
||||
from: v.parentId,
|
||||
to: v.id
|
||||
}))
|
||||
};
|
||||
return {
|
||||
timeline,
|
||||
changeFrequency,
|
||||
hotspots,
|
||||
versionGraph
|
||||
};
|
||||
}
|
||||
/**
|
||||
* Get audit log entries
|
||||
*
|
||||
* @param limit - Maximum number of entries to return
|
||||
* @returns Audit log entries
|
||||
*/
|
||||
getAuditLog(limit) {
|
||||
const sorted = [...this.auditLog].sort((a, b) => b.timestamp - a.timestamp);
|
||||
return limit ? sorted.slice(0, limit) : sorted;
|
||||
}
|
||||
/**
|
||||
* Log an audit entry
|
||||
*/
|
||||
logAudit(entry) {
|
||||
const auditEntry = {
|
||||
id: this.generateId(),
|
||||
timestamp: Date.now(),
|
||||
...entry
|
||||
};
|
||||
this.auditLog.push(auditEntry);
|
||||
this.emit('auditLogged', auditEntry);
|
||||
}
|
||||
/**
|
||||
* Prune old versions to save space
|
||||
*
|
||||
* @param keepCount - Number of recent versions to keep
|
||||
* @param preserveTags - Tags to preserve regardless of age
|
||||
*/
|
||||
pruneVersions(keepCount, preserveTags = ['baseline']) {
|
||||
const versions = Array.from(this.versions.values())
|
||||
.sort((a, b) => b.timestamp - a.timestamp);
|
||||
const toDelete = [];
|
||||
versions.forEach((version, index) => {
|
||||
// Keep recent versions
|
||||
if (index < keepCount)
|
||||
return;
|
||||
// Keep tagged versions
|
||||
if (version.tags.some(tag => preserveTags.includes(tag)))
|
||||
return;
|
||||
// Keep if any child version exists
|
||||
const hasChildren = versions.some(v => v.parentId === version.id);
|
||||
if (hasChildren)
|
||||
return;
|
||||
toDelete.push(version.id);
|
||||
});
|
||||
// Delete versions
|
||||
toDelete.forEach(id => {
|
||||
const version = this.versions.get(id);
|
||||
if (version) {
|
||||
// Remove from indices
|
||||
version.tags.forEach(tag => {
|
||||
this.tagIndex.get(tag)?.delete(id);
|
||||
});
|
||||
this.versions.delete(id);
|
||||
}
|
||||
});
|
||||
this.logAudit({
|
||||
operation: 'prune',
|
||||
status: 'success',
|
||||
details: {
|
||||
deletedCount: toDelete.length,
|
||||
keepCount,
|
||||
preserveTags
|
||||
}
|
||||
});
|
||||
}
|
||||
/**
|
||||
* Export all versions and audit log for backup
|
||||
*
|
||||
* @returns Serializable backup data
|
||||
*/
|
||||
exportBackup() {
|
||||
return {
|
||||
versions: Array.from(this.versions.values()),
|
||||
auditLog: this.auditLog,
|
||||
currentState: this.currentState,
|
||||
exportedAt: Date.now()
|
||||
};
|
||||
}
|
||||
/**
|
||||
* Import versions and state from backup
|
||||
*
|
||||
* @param backup - Backup data to import
|
||||
*/
|
||||
importBackup(backup) {
|
||||
// Clear existing data
|
||||
this.versions.clear();
|
||||
this.tagIndex.clear();
|
||||
this.pathIndex.clear();
|
||||
this.auditLog = [];
|
||||
this.pendingChanges = [];
|
||||
// Import versions
|
||||
backup.versions.forEach(version => {
|
||||
this.versions.set(version.id, version);
|
||||
this.indexVersion(version);
|
||||
});
|
||||
// Import audit log
|
||||
this.auditLog = [...backup.auditLog];
|
||||
// Import current state
|
||||
this.currentState = backup.currentState;
|
||||
this.logAudit({
|
||||
operation: 'create',
|
||||
status: 'success',
|
||||
details: {
|
||||
importedVersions: backup.versions.length,
|
||||
importedAuditEntries: backup.auditLog.length,
|
||||
importedFrom: backup.exportedAt
|
||||
}
|
||||
});
|
||||
}
|
||||
/**
|
||||
* Get storage statistics
|
||||
*
|
||||
* @returns Storage statistics
|
||||
*/
|
||||
getStorageStats() {
|
||||
const versions = Array.from(this.versions.values());
|
||||
const totalChanges = versions.reduce((sum, v) => sum + v.changes.length, 0);
|
||||
const backup = this.exportBackup();
|
||||
const estimatedSizeBytes = JSON.stringify(backup).length;
|
||||
return {
|
||||
versionCount: versions.length,
|
||||
totalChanges,
|
||||
auditLogSize: this.auditLog.length,
|
||||
estimatedSizeBytes,
|
||||
oldestVersion: Math.min(...versions.map(v => v.timestamp)),
|
||||
newestVersion: Math.max(...versions.map(v => v.timestamp))
|
||||
};
|
||||
}
|
||||
}
|
||||
exports.TemporalTracker = TemporalTracker;
|
||||
/**
|
||||
* Export singleton instance for convenience
|
||||
*/
|
||||
exports.temporalTracker = new TemporalTracker();
|
||||
/**
|
||||
* Type guard for Change
|
||||
*/
|
||||
function isChange(obj) {
|
||||
return obj &&
|
||||
typeof obj.type === 'string' &&
|
||||
typeof obj.path === 'string' &&
|
||||
typeof obj.timestamp === 'number';
|
||||
}
|
||||
/**
|
||||
* Type guard for Version
|
||||
*/
|
||||
function isVersion(obj) {
|
||||
return obj &&
|
||||
typeof obj.id === 'string' &&
|
||||
typeof obj.timestamp === 'number' &&
|
||||
Array.isArray(obj.changes) &&
|
||||
Array.isArray(obj.tags);
|
||||
}
|
||||
//# sourceMappingURL=temporal.js.map
|
||||
1
vendor/ruvector/npm/packages/ruvector-extensions/src/temporal.js.map
vendored
Normal file
1
vendor/ruvector/npm/packages/ruvector-extensions/src/temporal.js.map
vendored
Normal file
File diff suppressed because one or more lines are too long
1059
vendor/ruvector/npm/packages/ruvector-extensions/src/temporal.ts
vendored
Normal file
1059
vendor/ruvector/npm/packages/ruvector-extensions/src/temporal.ts
vendored
Normal file
File diff suppressed because it is too large
Load Diff
39
vendor/ruvector/npm/packages/ruvector-extensions/src/ui-server.d.ts
vendored
Normal file
39
vendor/ruvector/npm/packages/ruvector-extensions/src/ui-server.d.ts
vendored
Normal file
@@ -0,0 +1,39 @@
|
||||
export interface GraphNode {
|
||||
id: string;
|
||||
label?: string;
|
||||
metadata?: Record<string, any>;
|
||||
x?: number;
|
||||
y?: number;
|
||||
}
|
||||
export interface GraphLink {
|
||||
source: string;
|
||||
target: string;
|
||||
similarity: number;
|
||||
}
|
||||
export interface GraphData {
|
||||
nodes: GraphNode[];
|
||||
links: GraphLink[];
|
||||
}
|
||||
export declare class UIServer {
|
||||
private app;
|
||||
private server;
|
||||
private wss;
|
||||
private db;
|
||||
private clients;
|
||||
private port;
|
||||
constructor(db: any, port?: number);
|
||||
private setupMiddleware;
|
||||
private setupRoutes;
|
||||
private setupWebSocket;
|
||||
private handleWebSocketMessage;
|
||||
private broadcast;
|
||||
private getGraphData;
|
||||
private searchNodes;
|
||||
private findSimilarNodes;
|
||||
private getNodeDetails;
|
||||
start(): Promise<void>;
|
||||
stop(): Promise<void>;
|
||||
notifyGraphUpdate(): void;
|
||||
}
|
||||
export declare function startUIServer(db: any, port?: number): Promise<UIServer>;
|
||||
//# sourceMappingURL=ui-server.d.ts.map
|
||||
1
vendor/ruvector/npm/packages/ruvector-extensions/src/ui-server.d.ts.map
vendored
Normal file
1
vendor/ruvector/npm/packages/ruvector-extensions/src/ui-server.d.ts.map
vendored
Normal file
@@ -0,0 +1 @@
|
||||
{"version":3,"file":"ui-server.d.ts","sourceRoot":"","sources":["ui-server.ts"],"names":[],"mappings":"AAMA,MAAM,WAAW,SAAS;IACtB,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;IAC/B,CAAC,CAAC,EAAE,MAAM,CAAC;IACX,CAAC,CAAC,EAAE,MAAM,CAAC;CACd;AAED,MAAM,WAAW,SAAS;IACtB,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,EAAE,MAAM,CAAC;CACtB;AAED,MAAM,WAAW,SAAS;IACtB,KAAK,EAAE,SAAS,EAAE,CAAC;IACnB,KAAK,EAAE,SAAS,EAAE,CAAC;CACtB;AAED,qBAAa,QAAQ;IACjB,OAAO,CAAC,GAAG,CAAsB;IACjC,OAAO,CAAC,MAAM,CAAM;IACpB,OAAO,CAAC,GAAG,CAAkB;IAC7B,OAAO,CAAC,EAAE,CAAM;IAChB,OAAO,CAAC,OAAO,CAAiB;IAChC,OAAO,CAAC,IAAI,CAAS;gBAET,EAAE,EAAE,GAAG,EAAE,IAAI,GAAE,MAAa;IAcxC,OAAO,CAAC,eAAe;IAuBvB,OAAO,CAAC,WAAW;IAsInB,OAAO,CAAC,cAAc;YAoCR,sBAAsB;IAsCpC,OAAO,CAAC,SAAS;YASH,YAAY;YA+CZ,WAAW;YA+BX,gBAAgB;YA2BhB,cAAc;IAWrB,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAmBtB,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAgBrB,iBAAiB,IAAI,IAAI;CAOnC;AAGD,wBAAsB,aAAa,CAAC,EAAE,EAAE,GAAG,EAAE,IAAI,GAAE,MAAa,GAAG,OAAO,CAAC,QAAQ,CAAC,CAInF"}
|
||||
382
vendor/ruvector/npm/packages/ruvector-extensions/src/ui-server.js
vendored
Normal file
382
vendor/ruvector/npm/packages/ruvector-extensions/src/ui-server.js
vendored
Normal file
@@ -0,0 +1,382 @@
|
||||
"use strict";
|
||||
var __importDefault = (this && this.__importDefault) || function (mod) {
|
||||
return (mod && mod.__esModule) ? mod : { "default": mod };
|
||||
};
|
||||
Object.defineProperty(exports, "__esModule", { value: true });
|
||||
exports.UIServer = void 0;
|
||||
exports.startUIServer = startUIServer;
|
||||
const express_1 = __importDefault(require("express"));
|
||||
const http_1 = require("http");
|
||||
const ws_1 = require("ws");
|
||||
const path_1 = __importDefault(require("path"));
|
||||
class UIServer {
|
||||
constructor(db, port = 3000) {
|
||||
this.db = db;
|
||||
this.port = port;
|
||||
this.clients = new Set();
|
||||
this.app = (0, express_1.default)();
|
||||
this.server = (0, http_1.createServer)(this.app);
|
||||
this.wss = new ws_1.WebSocketServer({ server: this.server });
|
||||
this.setupMiddleware();
|
||||
this.setupRoutes();
|
||||
this.setupWebSocket();
|
||||
}
|
||||
setupMiddleware() {
|
||||
// JSON parsing
|
||||
this.app.use(express_1.default.json());
|
||||
// CORS
|
||||
this.app.use((req, res, next) => {
|
||||
res.header('Access-Control-Allow-Origin', '*');
|
||||
res.header('Access-Control-Allow-Methods', 'GET, POST, PUT, DELETE, OPTIONS');
|
||||
res.header('Access-Control-Allow-Headers', 'Content-Type, Authorization');
|
||||
next();
|
||||
});
|
||||
// Static files
|
||||
const uiPath = path_1.default.join(__dirname, 'ui');
|
||||
this.app.use(express_1.default.static(uiPath));
|
||||
// Logging
|
||||
this.app.use((req, res, next) => {
|
||||
console.log(`${new Date().toISOString()} ${req.method} ${req.path}`);
|
||||
next();
|
||||
});
|
||||
}
|
||||
setupRoutes() {
|
||||
// Health check
|
||||
this.app.get('/health', (req, res) => {
|
||||
res.json({
|
||||
status: 'ok',
|
||||
timestamp: Date.now(),
|
||||
version: '1.0.0'
|
||||
});
|
||||
});
|
||||
// Get full graph data
|
||||
this.app.get('/api/graph', async (req, res) => {
|
||||
try {
|
||||
const maxNodes = parseInt(req.query.max) || 100;
|
||||
const graphData = await this.getGraphData(maxNodes);
|
||||
res.json(graphData);
|
||||
}
|
||||
catch (error) {
|
||||
console.error('Error fetching graph:', error);
|
||||
res.status(500).json({
|
||||
error: 'Failed to fetch graph data',
|
||||
message: error instanceof Error ? error.message : 'Unknown error'
|
||||
});
|
||||
}
|
||||
});
|
||||
// Search nodes
|
||||
this.app.get('/api/search', async (req, res) => {
|
||||
try {
|
||||
const query = req.query.q;
|
||||
if (!query) {
|
||||
return res.status(400).json({ error: 'Query parameter required' });
|
||||
}
|
||||
const results = await this.searchNodes(query);
|
||||
res.json({ results, count: results.length });
|
||||
}
|
||||
catch (error) {
|
||||
console.error('Search error:', error);
|
||||
res.status(500).json({
|
||||
error: 'Search failed',
|
||||
message: error instanceof Error ? error.message : 'Unknown error'
|
||||
});
|
||||
}
|
||||
});
|
||||
// Find similar nodes
|
||||
this.app.get('/api/similarity/:nodeId', async (req, res) => {
|
||||
try {
|
||||
const { nodeId } = req.params;
|
||||
const threshold = parseFloat(req.query.threshold) || 0.5;
|
||||
const limit = parseInt(req.query.limit) || 10;
|
||||
const similar = await this.findSimilarNodes(nodeId, threshold, limit);
|
||||
res.json({
|
||||
nodeId,
|
||||
similar,
|
||||
count: similar.length,
|
||||
threshold
|
||||
});
|
||||
}
|
||||
catch (error) {
|
||||
console.error('Similarity search error:', error);
|
||||
res.status(500).json({
|
||||
error: 'Similarity search failed',
|
||||
message: error instanceof Error ? error.message : 'Unknown error'
|
||||
});
|
||||
}
|
||||
});
|
||||
// Get node details
|
||||
this.app.get('/api/nodes/:nodeId', async (req, res) => {
|
||||
try {
|
||||
const { nodeId } = req.params;
|
||||
const node = await this.getNodeDetails(nodeId);
|
||||
if (!node) {
|
||||
return res.status(404).json({ error: 'Node not found' });
|
||||
}
|
||||
res.json(node);
|
||||
}
|
||||
catch (error) {
|
||||
console.error('Error fetching node:', error);
|
||||
res.status(500).json({
|
||||
error: 'Failed to fetch node',
|
||||
message: error instanceof Error ? error.message : 'Unknown error'
|
||||
});
|
||||
}
|
||||
});
|
||||
// Add new node (for testing)
|
||||
this.app.post('/api/nodes', async (req, res) => {
|
||||
try {
|
||||
const { id, embedding, metadata } = req.body;
|
||||
if (!id || !embedding) {
|
||||
return res.status(400).json({ error: 'ID and embedding required' });
|
||||
}
|
||||
await this.db.add(id, embedding, metadata);
|
||||
// Notify all clients
|
||||
this.broadcast({
|
||||
type: 'node_added',
|
||||
payload: { id, metadata }
|
||||
});
|
||||
res.status(201).json({ success: true, id });
|
||||
}
|
||||
catch (error) {
|
||||
console.error('Error adding node:', error);
|
||||
res.status(500).json({
|
||||
error: 'Failed to add node',
|
||||
message: error instanceof Error ? error.message : 'Unknown error'
|
||||
});
|
||||
}
|
||||
});
|
||||
// Database statistics
|
||||
this.app.get('/api/stats', async (req, res) => {
|
||||
try {
|
||||
const stats = await this.db.getStats();
|
||||
res.json(stats);
|
||||
}
|
||||
catch (error) {
|
||||
console.error('Error fetching stats:', error);
|
||||
res.status(500).json({
|
||||
error: 'Failed to fetch statistics',
|
||||
message: error instanceof Error ? error.message : 'Unknown error'
|
||||
});
|
||||
}
|
||||
});
|
||||
// Serve UI
|
||||
this.app.get('*', (req, res) => {
|
||||
res.sendFile(path_1.default.join(__dirname, 'ui', 'index.html'));
|
||||
});
|
||||
}
|
||||
setupWebSocket() {
|
||||
this.wss.on('connection', (ws) => {
|
||||
console.log('New WebSocket client connected');
|
||||
this.clients.add(ws);
|
||||
ws.on('message', async (message) => {
|
||||
try {
|
||||
const data = JSON.parse(message.toString());
|
||||
await this.handleWebSocketMessage(ws, data);
|
||||
}
|
||||
catch (error) {
|
||||
console.error('WebSocket message error:', error);
|
||||
ws.send(JSON.stringify({
|
||||
type: 'error',
|
||||
message: 'Invalid message format'
|
||||
}));
|
||||
}
|
||||
});
|
||||
ws.on('close', () => {
|
||||
console.log('WebSocket client disconnected');
|
||||
this.clients.delete(ws);
|
||||
});
|
||||
ws.on('error', (error) => {
|
||||
console.error('WebSocket error:', error);
|
||||
this.clients.delete(ws);
|
||||
});
|
||||
// Send initial connection message
|
||||
ws.send(JSON.stringify({
|
||||
type: 'connected',
|
||||
message: 'Connected to RuVector UI Server'
|
||||
}));
|
||||
});
|
||||
}
|
||||
async handleWebSocketMessage(ws, data) {
|
||||
switch (data.type) {
|
||||
case 'subscribe':
|
||||
// Handle subscription to updates
|
||||
ws.send(JSON.stringify({
|
||||
type: 'subscribed',
|
||||
message: 'Subscribed to graph updates'
|
||||
}));
|
||||
break;
|
||||
case 'request_graph':
|
||||
const graphData = await this.getGraphData(data.maxNodes || 100);
|
||||
ws.send(JSON.stringify({
|
||||
type: 'graph_data',
|
||||
payload: graphData
|
||||
}));
|
||||
break;
|
||||
case 'similarity_query':
|
||||
const similar = await this.findSimilarNodes(data.nodeId, data.threshold || 0.5, data.limit || 10);
|
||||
ws.send(JSON.stringify({
|
||||
type: 'similarity_result',
|
||||
payload: { nodeId: data.nodeId, similar }
|
||||
}));
|
||||
break;
|
||||
default:
|
||||
ws.send(JSON.stringify({
|
||||
type: 'error',
|
||||
message: 'Unknown message type'
|
||||
}));
|
||||
}
|
||||
}
|
||||
broadcast(message) {
|
||||
const messageStr = JSON.stringify(message);
|
||||
this.clients.forEach(client => {
|
||||
if (client.readyState === ws_1.WebSocket.OPEN) {
|
||||
client.send(messageStr);
|
||||
}
|
||||
});
|
||||
}
|
||||
async getGraphData(maxNodes) {
|
||||
// Get all vectors from database
|
||||
const vectors = await this.db.list();
|
||||
const nodes = [];
|
||||
const links = [];
|
||||
const nodeMap = new Map();
|
||||
// Limit nodes
|
||||
const limitedVectors = vectors.slice(0, maxNodes);
|
||||
// Create nodes
|
||||
for (const vector of limitedVectors) {
|
||||
const node = {
|
||||
id: vector.id,
|
||||
label: vector.metadata?.label || vector.id.substring(0, 8),
|
||||
metadata: vector.metadata
|
||||
};
|
||||
nodes.push(node);
|
||||
nodeMap.set(vector.id, node);
|
||||
}
|
||||
// Create links based on similarity
|
||||
for (let i = 0; i < limitedVectors.length; i++) {
|
||||
const sourceVector = limitedVectors[i];
|
||||
// Find top 5 similar nodes
|
||||
const similar = await this.db.query(sourceVector.embedding, { topK: 6 });
|
||||
for (const result of similar) {
|
||||
// Skip self-links and already processed pairs
|
||||
if (result.id === sourceVector.id)
|
||||
continue;
|
||||
if (!nodeMap.has(result.id))
|
||||
continue;
|
||||
// Only add links above threshold
|
||||
if (result.similarity > 0.3) {
|
||||
links.push({
|
||||
source: sourceVector.id,
|
||||
target: result.id,
|
||||
similarity: result.similarity
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
return { nodes, links };
|
||||
}
|
||||
async searchNodes(query) {
|
||||
const vectors = await this.db.list();
|
||||
const results = [];
|
||||
for (const vector of vectors) {
|
||||
// Search in ID
|
||||
if (vector.id.toLowerCase().includes(query.toLowerCase())) {
|
||||
results.push({
|
||||
id: vector.id,
|
||||
label: vector.metadata?.label,
|
||||
metadata: vector.metadata
|
||||
});
|
||||
continue;
|
||||
}
|
||||
// Search in metadata
|
||||
if (vector.metadata) {
|
||||
const metadataStr = JSON.stringify(vector.metadata).toLowerCase();
|
||||
if (metadataStr.includes(query.toLowerCase())) {
|
||||
results.push({
|
||||
id: vector.id,
|
||||
label: vector.metadata.label,
|
||||
metadata: vector.metadata
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
return results;
|
||||
}
|
||||
async findSimilarNodes(nodeId, threshold, limit) {
|
||||
// Get the source node
|
||||
const sourceVector = await this.db.get(nodeId);
|
||||
if (!sourceVector) {
|
||||
throw new Error('Node not found');
|
||||
}
|
||||
// Query similar nodes
|
||||
const results = await this.db.query(sourceVector.embedding, {
|
||||
topK: limit + 1
|
||||
});
|
||||
// Filter and format results
|
||||
return results
|
||||
.filter((r) => r.id !== nodeId && r.similarity >= threshold)
|
||||
.slice(0, limit)
|
||||
.map((r) => ({
|
||||
id: r.id,
|
||||
similarity: r.similarity,
|
||||
metadata: r.metadata
|
||||
}));
|
||||
}
|
||||
async getNodeDetails(nodeId) {
|
||||
const vector = await this.db.get(nodeId);
|
||||
if (!vector)
|
||||
return null;
|
||||
return {
|
||||
id: vector.id,
|
||||
label: vector.metadata?.label,
|
||||
metadata: vector.metadata
|
||||
};
|
||||
}
|
||||
start() {
|
||||
return new Promise((resolve) => {
|
||||
this.server.listen(this.port, () => {
|
||||
console.log(`
|
||||
╔════════════════════════════════════════════════════════════╗
|
||||
║ RuVector Graph Explorer UI Server ║
|
||||
╚════════════════════════════════════════════════════════════╝
|
||||
|
||||
🌐 Server running at: http://localhost:${this.port}
|
||||
📊 WebSocket: ws://localhost:${this.port}
|
||||
🗄️ Database: Connected
|
||||
|
||||
Open your browser and navigate to http://localhost:${this.port}
|
||||
`);
|
||||
resolve();
|
||||
});
|
||||
});
|
||||
}
|
||||
stop() {
|
||||
return new Promise((resolve) => {
|
||||
// Close WebSocket connections
|
||||
this.clients.forEach(client => client.close());
|
||||
// Close WebSocket server
|
||||
this.wss.close(() => {
|
||||
// Close HTTP server
|
||||
this.server.close(() => {
|
||||
console.log('UI Server stopped');
|
||||
resolve();
|
||||
});
|
||||
});
|
||||
});
|
||||
}
|
||||
notifyGraphUpdate() {
|
||||
// Broadcast update to all clients
|
||||
this.broadcast({
|
||||
type: 'update',
|
||||
message: 'Graph data updated'
|
||||
});
|
||||
}
|
||||
}
|
||||
exports.UIServer = UIServer;
|
||||
// Example usage
|
||||
async function startUIServer(db, port = 3000) {
|
||||
const server = new UIServer(db, port);
|
||||
await server.start();
|
||||
return server;
|
||||
}
|
||||
//# sourceMappingURL=ui-server.js.map
|
||||
1
vendor/ruvector/npm/packages/ruvector-extensions/src/ui-server.js.map
vendored
Normal file
1
vendor/ruvector/npm/packages/ruvector-extensions/src/ui-server.js.map
vendored
Normal file
File diff suppressed because one or more lines are too long
453
vendor/ruvector/npm/packages/ruvector-extensions/src/ui-server.ts
vendored
Normal file
453
vendor/ruvector/npm/packages/ruvector-extensions/src/ui-server.ts
vendored
Normal file
@@ -0,0 +1,453 @@
|
||||
import express, { Request, Response } from 'express';
|
||||
import { createServer } from 'http';
|
||||
import { WebSocketServer, WebSocket } from 'ws';
|
||||
import path from 'path';
|
||||
import type { VectorDB } from 'ruvector';
|
||||
|
||||
export interface GraphNode {
|
||||
id: string;
|
||||
label?: string;
|
||||
metadata?: Record<string, any>;
|
||||
x?: number;
|
||||
y?: number;
|
||||
}
|
||||
|
||||
export interface GraphLink {
|
||||
source: string;
|
||||
target: string;
|
||||
similarity: number;
|
||||
}
|
||||
|
||||
export interface GraphData {
|
||||
nodes: GraphNode[];
|
||||
links: GraphLink[];
|
||||
}
|
||||
|
||||
export class UIServer {
|
||||
private app: express.Application;
|
||||
private server: any;
|
||||
private wss: WebSocketServer;
|
||||
private db: any;
|
||||
private clients: Set<WebSocket>;
|
||||
private port: number;
|
||||
|
||||
constructor(db: any, port: number = 3000) {
|
||||
this.db = db;
|
||||
this.port = port;
|
||||
this.clients = new Set();
|
||||
|
||||
this.app = express();
|
||||
this.server = createServer(this.app);
|
||||
this.wss = new WebSocketServer({ server: this.server });
|
||||
|
||||
this.setupMiddleware();
|
||||
this.setupRoutes();
|
||||
this.setupWebSocket();
|
||||
}
|
||||
|
||||
private setupMiddleware(): void {
|
||||
// JSON parsing
|
||||
this.app.use(express.json());
|
||||
|
||||
// CORS
|
||||
this.app.use((req, res, next) => {
|
||||
res.header('Access-Control-Allow-Origin', '*');
|
||||
res.header('Access-Control-Allow-Methods', 'GET, POST, PUT, DELETE, OPTIONS');
|
||||
res.header('Access-Control-Allow-Headers', 'Content-Type, Authorization');
|
||||
next();
|
||||
});
|
||||
|
||||
// Static files
|
||||
const uiPath = path.join(__dirname, 'ui');
|
||||
this.app.use(express.static(uiPath));
|
||||
|
||||
// Logging
|
||||
this.app.use((req, res, next) => {
|
||||
console.log(`${new Date().toISOString()} ${req.method} ${req.path}`);
|
||||
next();
|
||||
});
|
||||
}
|
||||
|
||||
private setupRoutes(): void {
|
||||
// Health check
|
||||
this.app.get('/health', (req: Request, res: Response) => {
|
||||
res.json({
|
||||
status: 'ok',
|
||||
timestamp: Date.now(),
|
||||
version: '1.0.0'
|
||||
});
|
||||
});
|
||||
|
||||
// Get full graph data
|
||||
this.app.get('/api/graph', async (req: Request, res: Response) => {
|
||||
try {
|
||||
const maxNodes = parseInt(req.query.max as string) || 100;
|
||||
const graphData = await this.getGraphData(maxNodes);
|
||||
res.json(graphData);
|
||||
} catch (error) {
|
||||
console.error('Error fetching graph:', error);
|
||||
res.status(500).json({
|
||||
error: 'Failed to fetch graph data',
|
||||
message: error instanceof Error ? error.message : 'Unknown error'
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
// Search nodes
|
||||
this.app.get('/api/search', async (req: Request, res: Response) => {
|
||||
try {
|
||||
const query = req.query.q as string;
|
||||
if (!query) {
|
||||
return res.status(400).json({ error: 'Query parameter required' });
|
||||
}
|
||||
|
||||
const results = await this.searchNodes(query);
|
||||
res.json({ results, count: results.length });
|
||||
} catch (error) {
|
||||
console.error('Search error:', error);
|
||||
res.status(500).json({
|
||||
error: 'Search failed',
|
||||
message: error instanceof Error ? error.message : 'Unknown error'
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
// Find similar nodes
|
||||
this.app.get('/api/similarity/:nodeId', async (req: Request, res: Response) => {
|
||||
try {
|
||||
const { nodeId } = req.params;
|
||||
const threshold = parseFloat(req.query.threshold as string) || 0.5;
|
||||
const limit = parseInt(req.query.limit as string) || 10;
|
||||
|
||||
const similar = await this.findSimilarNodes(nodeId, threshold, limit);
|
||||
res.json({
|
||||
nodeId,
|
||||
similar,
|
||||
count: similar.length,
|
||||
threshold
|
||||
});
|
||||
} catch (error) {
|
||||
console.error('Similarity search error:', error);
|
||||
res.status(500).json({
|
||||
error: 'Similarity search failed',
|
||||
message: error instanceof Error ? error.message : 'Unknown error'
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
// Get node details
|
||||
this.app.get('/api/nodes/:nodeId', async (req: Request, res: Response) => {
|
||||
try {
|
||||
const { nodeId } = req.params;
|
||||
const node = await this.getNodeDetails(nodeId);
|
||||
|
||||
if (!node) {
|
||||
return res.status(404).json({ error: 'Node not found' });
|
||||
}
|
||||
|
||||
res.json(node);
|
||||
} catch (error) {
|
||||
console.error('Error fetching node:', error);
|
||||
res.status(500).json({
|
||||
error: 'Failed to fetch node',
|
||||
message: error instanceof Error ? error.message : 'Unknown error'
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
// Add new node (for testing)
|
||||
this.app.post('/api/nodes', async (req: Request, res: Response) => {
|
||||
try {
|
||||
const { id, embedding, metadata } = req.body;
|
||||
|
||||
if (!id || !embedding) {
|
||||
return res.status(400).json({ error: 'ID and embedding required' });
|
||||
}
|
||||
|
||||
await this.db.add(id, embedding, metadata);
|
||||
|
||||
// Notify all clients
|
||||
this.broadcast({
|
||||
type: 'node_added',
|
||||
payload: { id, metadata }
|
||||
});
|
||||
|
||||
res.status(201).json({ success: true, id });
|
||||
} catch (error) {
|
||||
console.error('Error adding node:', error);
|
||||
res.status(500).json({
|
||||
error: 'Failed to add node',
|
||||
message: error instanceof Error ? error.message : 'Unknown error'
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
// Database statistics
|
||||
this.app.get('/api/stats', async (req: Request, res: Response) => {
|
||||
try {
|
||||
const stats = await this.db.getStats();
|
||||
res.json(stats);
|
||||
} catch (error) {
|
||||
console.error('Error fetching stats:', error);
|
||||
res.status(500).json({
|
||||
error: 'Failed to fetch statistics',
|
||||
message: error instanceof Error ? error.message : 'Unknown error'
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
// Serve UI
|
||||
this.app.get('*', (req: Request, res: Response) => {
|
||||
res.sendFile(path.join(__dirname, 'ui', 'index.html'));
|
||||
});
|
||||
}
|
||||
|
||||
private setupWebSocket(): void {
|
||||
this.wss.on('connection', (ws: WebSocket) => {
|
||||
console.log('New WebSocket client connected');
|
||||
this.clients.add(ws);
|
||||
|
||||
ws.on('message', async (message: string) => {
|
||||
try {
|
||||
const data = JSON.parse(message.toString());
|
||||
await this.handleWebSocketMessage(ws, data);
|
||||
} catch (error) {
|
||||
console.error('WebSocket message error:', error);
|
||||
ws.send(JSON.stringify({
|
||||
type: 'error',
|
||||
message: 'Invalid message format'
|
||||
}));
|
||||
}
|
||||
});
|
||||
|
||||
ws.on('close', () => {
|
||||
console.log('WebSocket client disconnected');
|
||||
this.clients.delete(ws);
|
||||
});
|
||||
|
||||
ws.on('error', (error) => {
|
||||
console.error('WebSocket error:', error);
|
||||
this.clients.delete(ws);
|
||||
});
|
||||
|
||||
// Send initial connection message
|
||||
ws.send(JSON.stringify({
|
||||
type: 'connected',
|
||||
message: 'Connected to RuVector UI Server'
|
||||
}));
|
||||
});
|
||||
}
|
||||
|
||||
private async handleWebSocketMessage(ws: WebSocket, data: any): Promise<void> {
|
||||
switch (data.type) {
|
||||
case 'subscribe':
|
||||
// Handle subscription to updates
|
||||
ws.send(JSON.stringify({
|
||||
type: 'subscribed',
|
||||
message: 'Subscribed to graph updates'
|
||||
}));
|
||||
break;
|
||||
|
||||
case 'request_graph':
|
||||
const graphData = await this.getGraphData(data.maxNodes || 100);
|
||||
ws.send(JSON.stringify({
|
||||
type: 'graph_data',
|
||||
payload: graphData
|
||||
}));
|
||||
break;
|
||||
|
||||
case 'similarity_query':
|
||||
const similar = await this.findSimilarNodes(
|
||||
data.nodeId,
|
||||
data.threshold || 0.5,
|
||||
data.limit || 10
|
||||
);
|
||||
ws.send(JSON.stringify({
|
||||
type: 'similarity_result',
|
||||
payload: { nodeId: data.nodeId, similar }
|
||||
}));
|
||||
break;
|
||||
|
||||
default:
|
||||
ws.send(JSON.stringify({
|
||||
type: 'error',
|
||||
message: 'Unknown message type'
|
||||
}));
|
||||
}
|
||||
}
|
||||
|
||||
private broadcast(message: any): void {
|
||||
const messageStr = JSON.stringify(message);
|
||||
this.clients.forEach(client => {
|
||||
if (client.readyState === WebSocket.OPEN) {
|
||||
client.send(messageStr);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
private async getGraphData(maxNodes: number): Promise<GraphData> {
|
||||
// Get all vectors from database
|
||||
const vectors = await this.db.list();
|
||||
const nodes: GraphNode[] = [];
|
||||
const links: GraphLink[] = [];
|
||||
const nodeMap = new Map<string, GraphNode>();
|
||||
|
||||
// Limit nodes
|
||||
const limitedVectors = vectors.slice(0, maxNodes);
|
||||
|
||||
// Create nodes
|
||||
for (const vector of limitedVectors) {
|
||||
const node: GraphNode = {
|
||||
id: vector.id,
|
||||
label: vector.metadata?.label || vector.id.substring(0, 8),
|
||||
metadata: vector.metadata
|
||||
};
|
||||
nodes.push(node);
|
||||
nodeMap.set(vector.id, node);
|
||||
}
|
||||
|
||||
// Create links based on similarity
|
||||
for (let i = 0; i < limitedVectors.length; i++) {
|
||||
const sourceVector = limitedVectors[i];
|
||||
|
||||
// Find top 5 similar nodes
|
||||
const similar = await this.db.query(sourceVector.embedding, { topK: 6 });
|
||||
|
||||
for (const result of similar) {
|
||||
// Skip self-links and already processed pairs
|
||||
if (result.id === sourceVector.id) continue;
|
||||
if (!nodeMap.has(result.id)) continue;
|
||||
|
||||
// Only add links above threshold
|
||||
if (result.similarity > 0.3) {
|
||||
links.push({
|
||||
source: sourceVector.id,
|
||||
target: result.id,
|
||||
similarity: result.similarity
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return { nodes, links };
|
||||
}
|
||||
|
||||
private async searchNodes(query: string): Promise<GraphNode[]> {
|
||||
const vectors = await this.db.list();
|
||||
const results: GraphNode[] = [];
|
||||
|
||||
for (const vector of vectors) {
|
||||
// Search in ID
|
||||
if (vector.id.toLowerCase().includes(query.toLowerCase())) {
|
||||
results.push({
|
||||
id: vector.id,
|
||||
label: vector.metadata?.label,
|
||||
metadata: vector.metadata
|
||||
});
|
||||
continue;
|
||||
}
|
||||
|
||||
// Search in metadata
|
||||
if (vector.metadata) {
|
||||
const metadataStr = JSON.stringify(vector.metadata).toLowerCase();
|
||||
if (metadataStr.includes(query.toLowerCase())) {
|
||||
results.push({
|
||||
id: vector.id,
|
||||
label: vector.metadata.label,
|
||||
metadata: vector.metadata
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return results;
|
||||
}
|
||||
|
||||
private async findSimilarNodes(
|
||||
nodeId: string,
|
||||
threshold: number,
|
||||
limit: number
|
||||
): Promise<Array<GraphNode & { similarity: number }>> {
|
||||
// Get the source node
|
||||
const sourceVector = await this.db.get(nodeId);
|
||||
if (!sourceVector) {
|
||||
throw new Error('Node not found');
|
||||
}
|
||||
|
||||
// Query similar nodes
|
||||
const results = await this.db.query(sourceVector.embedding, {
|
||||
topK: limit + 1
|
||||
});
|
||||
|
||||
// Filter and format results
|
||||
return results
|
||||
.filter((r: any) => r.id !== nodeId && r.similarity >= threshold)
|
||||
.slice(0, limit)
|
||||
.map((r: any) => ({
|
||||
id: r.id,
|
||||
similarity: r.similarity,
|
||||
metadata: r.metadata
|
||||
}));
|
||||
}
|
||||
|
||||
private async getNodeDetails(nodeId: string): Promise<GraphNode | null> {
|
||||
const vector = await this.db.get(nodeId);
|
||||
if (!vector) return null;
|
||||
|
||||
return {
|
||||
id: vector.id,
|
||||
label: vector.metadata?.label,
|
||||
metadata: vector.metadata
|
||||
};
|
||||
}
|
||||
|
||||
public start(): Promise<void> {
|
||||
return new Promise((resolve) => {
|
||||
this.server.listen(this.port, () => {
|
||||
console.log(`
|
||||
╔════════════════════════════════════════════════════════════╗
|
||||
║ RuVector Graph Explorer UI Server ║
|
||||
╚════════════════════════════════════════════════════════════╝
|
||||
|
||||
🌐 Server running at: http://localhost:${this.port}
|
||||
📊 WebSocket: ws://localhost:${this.port}
|
||||
🗄️ Database: Connected
|
||||
|
||||
Open your browser and navigate to http://localhost:${this.port}
|
||||
`);
|
||||
resolve();
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
public stop(): Promise<void> {
|
||||
return new Promise((resolve) => {
|
||||
// Close WebSocket connections
|
||||
this.clients.forEach(client => client.close());
|
||||
|
||||
// Close WebSocket server
|
||||
this.wss.close(() => {
|
||||
// Close HTTP server
|
||||
this.server.close(() => {
|
||||
console.log('UI Server stopped');
|
||||
resolve();
|
||||
});
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
public notifyGraphUpdate(): void {
|
||||
// Broadcast update to all clients
|
||||
this.broadcast({
|
||||
type: 'update',
|
||||
message: 'Graph data updated'
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// Example usage
|
||||
export async function startUIServer(db: any, port: number = 3000): Promise<UIServer> {
|
||||
const server = new UIServer(db, port);
|
||||
await server.start();
|
||||
return server;
|
||||
}
|
||||
582
vendor/ruvector/npm/packages/ruvector-extensions/src/ui/app.js
vendored
Normal file
582
vendor/ruvector/npm/packages/ruvector-extensions/src/ui/app.js
vendored
Normal file
@@ -0,0 +1,582 @@
|
||||
// RuVector Graph Explorer - Client-side Application
|
||||
class GraphExplorer {
|
||||
constructor() {
|
||||
this.nodes = [];
|
||||
this.links = [];
|
||||
this.simulation = null;
|
||||
this.svg = null;
|
||||
this.g = null;
|
||||
this.zoom = null;
|
||||
this.selectedNode = null;
|
||||
this.ws = null;
|
||||
this.apiUrl = window.location.origin;
|
||||
|
||||
this.init();
|
||||
}
|
||||
|
||||
async init() {
|
||||
this.setupUI();
|
||||
this.setupD3();
|
||||
this.setupWebSocket();
|
||||
this.setupEventListeners();
|
||||
await this.loadInitialData();
|
||||
}
|
||||
|
||||
setupUI() {
|
||||
// Show loading overlay
|
||||
this.showLoading(true);
|
||||
|
||||
// Update connection status
|
||||
this.updateConnectionStatus('connecting');
|
||||
}
|
||||
|
||||
setupD3() {
|
||||
const container = d3.select('#graph-canvas');
|
||||
const width = container.node().getBoundingClientRect().width;
|
||||
const height = container.node().getBoundingClientRect().height;
|
||||
|
||||
// Create SVG
|
||||
this.svg = container.append('svg')
|
||||
.attr('width', width)
|
||||
.attr('height', height)
|
||||
.style('background', 'transparent');
|
||||
|
||||
// Create zoom behavior
|
||||
this.zoom = d3.zoom()
|
||||
.scaleExtent([0.1, 10])
|
||||
.on('zoom', (event) => {
|
||||
this.g.attr('transform', event.transform);
|
||||
});
|
||||
|
||||
this.svg.call(this.zoom);
|
||||
|
||||
// Create main group
|
||||
this.g = this.svg.append('g');
|
||||
|
||||
// Create force simulation
|
||||
this.simulation = d3.forceSimulation()
|
||||
.force('link', d3.forceLink().id(d => d.id).distance(100))
|
||||
.force('charge', d3.forceManyBody().strength(-300))
|
||||
.force('center', d3.forceCenter(width / 2, height / 2))
|
||||
.force('collision', d3.forceCollide().radius(30));
|
||||
}
|
||||
|
||||
setupWebSocket() {
|
||||
const protocol = window.location.protocol === 'https:' ? 'wss:' : 'ws:';
|
||||
const wsUrl = `${protocol}//${window.location.host}`;
|
||||
|
||||
this.ws = new WebSocket(wsUrl);
|
||||
|
||||
this.ws.onopen = () => {
|
||||
console.log('WebSocket connected');
|
||||
this.updateConnectionStatus('connected');
|
||||
this.showToast('Connected to server', 'success');
|
||||
};
|
||||
|
||||
this.ws.onmessage = (event) => {
|
||||
const data = JSON.parse(event.data);
|
||||
this.handleWebSocketMessage(data);
|
||||
};
|
||||
|
||||
this.ws.onerror = (error) => {
|
||||
console.error('WebSocket error:', error);
|
||||
this.updateConnectionStatus('error');
|
||||
this.showToast('Connection error', 'error');
|
||||
};
|
||||
|
||||
this.ws.onclose = () => {
|
||||
console.log('WebSocket disconnected');
|
||||
this.updateConnectionStatus('disconnected');
|
||||
this.showToast('Disconnected from server', 'warning');
|
||||
|
||||
// Attempt reconnection after 3 seconds
|
||||
setTimeout(() => this.setupWebSocket(), 3000);
|
||||
};
|
||||
}
|
||||
|
||||
handleWebSocketMessage(data) {
|
||||
switch (data.type) {
|
||||
case 'update':
|
||||
this.handleGraphUpdate(data.payload);
|
||||
break;
|
||||
case 'node_added':
|
||||
this.handleNodeAdded(data.payload);
|
||||
break;
|
||||
case 'node_updated':
|
||||
this.handleNodeUpdated(data.payload);
|
||||
break;
|
||||
case 'similarity_result':
|
||||
this.handleSimilarityResult(data.payload);
|
||||
break;
|
||||
default:
|
||||
console.log('Unknown message type:', data.type);
|
||||
}
|
||||
}
|
||||
|
||||
async loadInitialData() {
|
||||
try {
|
||||
const response = await fetch(`${this.apiUrl}/api/graph`);
|
||||
if (!response.ok) throw new Error('Failed to load graph data');
|
||||
|
||||
const data = await response.json();
|
||||
this.updateGraph(data.nodes, data.links);
|
||||
this.showLoading(false);
|
||||
this.showToast('Graph loaded successfully', 'success');
|
||||
} catch (error) {
|
||||
console.error('Error loading data:', error);
|
||||
this.showLoading(false);
|
||||
this.showToast('Failed to load graph data', 'error');
|
||||
}
|
||||
}
|
||||
|
||||
updateGraph(nodes, links) {
|
||||
this.nodes = nodes;
|
||||
this.links = links;
|
||||
|
||||
this.updateStatistics();
|
||||
this.renderGraph();
|
||||
}
|
||||
|
||||
renderGraph() {
|
||||
// Remove existing elements
|
||||
this.g.selectAll('.link').remove();
|
||||
this.g.selectAll('.node').remove();
|
||||
this.g.selectAll('.node-label').remove();
|
||||
|
||||
// Create links
|
||||
const link = this.g.selectAll('.link')
|
||||
.data(this.links)
|
||||
.enter().append('line')
|
||||
.attr('class', 'link')
|
||||
.attr('stroke-width', d => Math.sqrt(d.similarity * 5) || 1);
|
||||
|
||||
// Create nodes
|
||||
const node = this.g.selectAll('.node')
|
||||
.data(this.nodes)
|
||||
.enter().append('circle')
|
||||
.attr('class', 'node')
|
||||
.attr('r', 15)
|
||||
.attr('fill', d => this.getNodeColor(d))
|
||||
.call(this.drag(this.simulation))
|
||||
.on('click', (event, d) => this.handleNodeClick(event, d))
|
||||
.on('dblclick', (event, d) => this.handleNodeDoubleClick(event, d));
|
||||
|
||||
// Create labels
|
||||
const label = this.g.selectAll('.node-label')
|
||||
.data(this.nodes)
|
||||
.enter().append('text')
|
||||
.attr('class', 'node-label')
|
||||
.attr('dy', -20)
|
||||
.text(d => d.label || d.id.substring(0, 8));
|
||||
|
||||
// Update simulation
|
||||
this.simulation.nodes(this.nodes);
|
||||
this.simulation.force('link').links(this.links);
|
||||
|
||||
this.simulation.on('tick', () => {
|
||||
link
|
||||
.attr('x1', d => d.source.x)
|
||||
.attr('y1', d => d.source.y)
|
||||
.attr('x2', d => d.target.x)
|
||||
.attr('y2', d => d.target.y);
|
||||
|
||||
node
|
||||
.attr('cx', d => d.x)
|
||||
.attr('cy', d => d.y);
|
||||
|
||||
label
|
||||
.attr('x', d => d.x)
|
||||
.attr('y', d => d.y);
|
||||
});
|
||||
|
||||
this.simulation.alpha(1).restart();
|
||||
}
|
||||
|
||||
getNodeColor(node) {
|
||||
// Color based on metadata or cluster
|
||||
if (node.metadata && node.metadata.category) {
|
||||
const categories = ['research', 'code', 'documentation', 'test'];
|
||||
const index = categories.indexOf(node.metadata.category);
|
||||
const colors = ['#667eea', '#f093fb', '#4caf50', '#ff9800'];
|
||||
return colors[index] || '#667eea';
|
||||
}
|
||||
return '#667eea';
|
||||
}
|
||||
|
||||
drag(simulation) {
|
||||
function dragstarted(event) {
|
||||
if (!event.active) simulation.alphaTarget(0.3).restart();
|
||||
event.subject.fx = event.subject.x;
|
||||
event.subject.fy = event.subject.y;
|
||||
}
|
||||
|
||||
function dragged(event) {
|
||||
event.subject.fx = event.x;
|
||||
event.subject.fy = event.y;
|
||||
}
|
||||
|
||||
function dragended(event) {
|
||||
if (!event.active) simulation.alphaTarget(0);
|
||||
event.subject.fx = null;
|
||||
event.subject.fy = null;
|
||||
}
|
||||
|
||||
return d3.drag()
|
||||
.on('start', dragstarted)
|
||||
.on('drag', dragged)
|
||||
.on('end', dragended);
|
||||
}
|
||||
|
||||
handleNodeClick(event, node) {
|
||||
event.stopPropagation();
|
||||
|
||||
// Deselect previous node
|
||||
this.g.selectAll('.node').classed('selected', false);
|
||||
|
||||
// Select new node
|
||||
this.selectedNode = node;
|
||||
d3.select(event.currentTarget).classed('selected', true);
|
||||
|
||||
// Show metadata panel
|
||||
this.showMetadata(node);
|
||||
this.updateStatistics();
|
||||
}
|
||||
|
||||
handleNodeDoubleClick(event, node) {
|
||||
event.stopPropagation();
|
||||
this.findSimilarNodes(node.id);
|
||||
}
|
||||
|
||||
showMetadata(node) {
|
||||
const panel = document.getElementById('metadata-panel');
|
||||
const content = document.getElementById('metadata-content');
|
||||
|
||||
let html = `
|
||||
<div class="metadata-item">
|
||||
<strong>ID:</strong>
|
||||
<div>${node.id}</div>
|
||||
</div>
|
||||
`;
|
||||
|
||||
if (node.metadata) {
|
||||
for (const [key, value] of Object.entries(node.metadata)) {
|
||||
html += `
|
||||
<div class="metadata-item">
|
||||
<strong>${key}:</strong>
|
||||
<div>${JSON.stringify(value, null, 2)}</div>
|
||||
</div>
|
||||
`;
|
||||
}
|
||||
}
|
||||
|
||||
content.innerHTML = html;
|
||||
panel.style.display = 'block';
|
||||
}
|
||||
|
||||
async findSimilarNodes(nodeId) {
|
||||
if (!nodeId && this.selectedNode) {
|
||||
nodeId = this.selectedNode.id;
|
||||
}
|
||||
|
||||
if (!nodeId) {
|
||||
this.showToast('Please select a node first', 'warning');
|
||||
return;
|
||||
}
|
||||
|
||||
this.showLoading(true);
|
||||
|
||||
try {
|
||||
const minSimilarity = parseFloat(document.getElementById('min-similarity').value);
|
||||
const response = await fetch(
|
||||
`${this.apiUrl}/api/similarity/${nodeId}?threshold=${minSimilarity}`
|
||||
);
|
||||
|
||||
if (!response.ok) throw new Error('Failed to find similar nodes');
|
||||
|
||||
const data = await response.json();
|
||||
this.highlightSimilarNodes(data.similar);
|
||||
this.showToast(`Found ${data.similar.length} similar nodes`, 'success');
|
||||
} catch (error) {
|
||||
console.error('Error finding similar nodes:', error);
|
||||
this.showToast('Failed to find similar nodes', 'error');
|
||||
} finally {
|
||||
this.showLoading(false);
|
||||
}
|
||||
}
|
||||
|
||||
highlightSimilarNodes(similarNodes) {
|
||||
// Reset highlights
|
||||
this.g.selectAll('.node').classed('highlighted', false);
|
||||
this.g.selectAll('.link').classed('highlighted', false);
|
||||
|
||||
const similarIds = new Set(similarNodes.map(n => n.id));
|
||||
|
||||
// Highlight nodes
|
||||
this.g.selectAll('.node')
|
||||
.classed('highlighted', d => similarIds.has(d.id));
|
||||
|
||||
// Highlight links
|
||||
this.g.selectAll('.link')
|
||||
.classed('highlighted', d =>
|
||||
similarIds.has(d.source.id) && similarIds.has(d.target.id)
|
||||
);
|
||||
}
|
||||
|
||||
async searchNodes(query) {
|
||||
if (!query.trim()) {
|
||||
this.renderGraph();
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
const response = await fetch(
|
||||
`${this.apiUrl}/api/search?q=${encodeURIComponent(query)}`
|
||||
);
|
||||
|
||||
if (!response.ok) throw new Error('Search failed');
|
||||
|
||||
const data = await response.json();
|
||||
this.highlightSearchResults(data.results);
|
||||
this.showToast(`Found ${data.results.length} matches`, 'success');
|
||||
} catch (error) {
|
||||
console.error('Search error:', error);
|
||||
this.showToast('Search failed', 'error');
|
||||
}
|
||||
}
|
||||
|
||||
highlightSearchResults(results) {
|
||||
const resultIds = new Set(results.map(r => r.id));
|
||||
|
||||
this.g.selectAll('.node')
|
||||
.style('opacity', d => resultIds.has(d.id) ? 1 : 0.2);
|
||||
|
||||
this.g.selectAll('.link')
|
||||
.style('opacity', d =>
|
||||
resultIds.has(d.source.id) || resultIds.has(d.target.id) ? 0.6 : 0.1
|
||||
);
|
||||
}
|
||||
|
||||
updateStatistics() {
|
||||
document.getElementById('stat-nodes').textContent = this.nodes.length;
|
||||
document.getElementById('stat-edges').textContent = this.links.length;
|
||||
document.getElementById('stat-selected').textContent =
|
||||
this.selectedNode ? this.selectedNode.id.substring(0, 8) : 'None';
|
||||
}
|
||||
|
||||
updateConnectionStatus(status) {
|
||||
const statusEl = document.getElementById('connection-status');
|
||||
const dot = statusEl.querySelector('.status-dot');
|
||||
const text = statusEl.querySelector('.status-text');
|
||||
|
||||
const statusMap = {
|
||||
connecting: { text: 'Connecting...', class: '' },
|
||||
connected: { text: 'Connected', class: 'connected' },
|
||||
disconnected: { text: 'Disconnected', class: '' },
|
||||
error: { text: 'Error', class: '' }
|
||||
};
|
||||
|
||||
const config = statusMap[status] || statusMap.disconnected;
|
||||
text.textContent = config.text;
|
||||
dot.className = `status-dot ${config.class}`;
|
||||
}
|
||||
|
||||
showLoading(show) {
|
||||
const overlay = document.getElementById('loading-overlay');
|
||||
overlay.classList.toggle('hidden', !show);
|
||||
}
|
||||
|
||||
showToast(message, type = 'info') {
|
||||
const container = document.getElementById('toast-container');
|
||||
const toast = document.createElement('div');
|
||||
toast.className = `toast ${type}`;
|
||||
toast.textContent = message;
|
||||
|
||||
container.appendChild(toast);
|
||||
|
||||
setTimeout(() => {
|
||||
toast.style.animation = 'slideIn 0.3s ease-out reverse';
|
||||
setTimeout(() => toast.remove(), 300);
|
||||
}, 3000);
|
||||
}
|
||||
|
||||
async exportPNG() {
|
||||
try {
|
||||
const svgElement = this.svg.node();
|
||||
const canvas = document.createElement('canvas');
|
||||
const ctx = canvas.getContext('2d');
|
||||
|
||||
const bbox = svgElement.getBBox();
|
||||
canvas.width = bbox.width + 40;
|
||||
canvas.height = bbox.height + 40;
|
||||
|
||||
// Fill background
|
||||
ctx.fillStyle = '#1a1a2e';
|
||||
ctx.fillRect(0, 0, canvas.width, canvas.height);
|
||||
|
||||
const svgString = new XMLSerializer().serializeToString(svgElement);
|
||||
const img = new Image();
|
||||
const blob = new Blob([svgString], { type: 'image/svg+xml' });
|
||||
const url = URL.createObjectURL(blob);
|
||||
|
||||
img.onload = () => {
|
||||
ctx.drawImage(img, 20, 20);
|
||||
canvas.toBlob((blob) => {
|
||||
const link = document.createElement('a');
|
||||
link.download = `graph-${Date.now()}.png`;
|
||||
link.href = URL.createObjectURL(blob);
|
||||
link.click();
|
||||
URL.revokeObjectURL(url);
|
||||
this.showToast('Graph exported as PNG', 'success');
|
||||
});
|
||||
};
|
||||
|
||||
img.src = url;
|
||||
} catch (error) {
|
||||
console.error('Export error:', error);
|
||||
this.showToast('Failed to export PNG', 'error');
|
||||
}
|
||||
}
|
||||
|
||||
exportSVG() {
|
||||
try {
|
||||
const svgElement = this.svg.node();
|
||||
const svgString = new XMLSerializer().serializeToString(svgElement);
|
||||
const blob = new Blob([svgString], { type: 'image/svg+xml' });
|
||||
|
||||
const link = document.createElement('a');
|
||||
link.download = `graph-${Date.now()}.svg`;
|
||||
link.href = URL.createObjectURL(blob);
|
||||
link.click();
|
||||
|
||||
this.showToast('Graph exported as SVG', 'success');
|
||||
} catch (error) {
|
||||
console.error('Export error:', error);
|
||||
this.showToast('Failed to export SVG', 'error');
|
||||
}
|
||||
}
|
||||
|
||||
resetView() {
|
||||
this.svg.transition()
|
||||
.duration(750)
|
||||
.call(this.zoom.transform, d3.zoomIdentity);
|
||||
}
|
||||
|
||||
fitView() {
|
||||
const bounds = this.g.node().getBBox();
|
||||
const parent = this.svg.node().getBoundingClientRect();
|
||||
const fullWidth = parent.width;
|
||||
const fullHeight = parent.height;
|
||||
const width = bounds.width;
|
||||
const height = bounds.height;
|
||||
|
||||
const midX = bounds.x + width / 2;
|
||||
const midY = bounds.y + height / 2;
|
||||
|
||||
const scale = 0.85 / Math.max(width / fullWidth, height / fullHeight);
|
||||
const translate = [fullWidth / 2 - scale * midX, fullHeight / 2 - scale * midY];
|
||||
|
||||
this.svg.transition()
|
||||
.duration(750)
|
||||
.call(this.zoom.transform, d3.zoomIdentity.translate(translate[0], translate[1]).scale(scale));
|
||||
}
|
||||
|
||||
zoomIn() {
|
||||
this.svg.transition().call(this.zoom.scaleBy, 1.3);
|
||||
}
|
||||
|
||||
zoomOut() {
|
||||
this.svg.transition().call(this.zoom.scaleBy, 0.7);
|
||||
}
|
||||
|
||||
setupEventListeners() {
|
||||
// Search
|
||||
const searchInput = document.getElementById('node-search');
|
||||
let searchTimeout;
|
||||
searchInput.addEventListener('input', (e) => {
|
||||
clearTimeout(searchTimeout);
|
||||
searchTimeout = setTimeout(() => this.searchNodes(e.target.value), 300);
|
||||
});
|
||||
|
||||
document.getElementById('clear-search').addEventListener('click', () => {
|
||||
searchInput.value = '';
|
||||
this.renderGraph();
|
||||
});
|
||||
|
||||
// Filters
|
||||
const similaritySlider = document.getElementById('min-similarity');
|
||||
similaritySlider.addEventListener('input', (e) => {
|
||||
document.getElementById('similarity-value').textContent =
|
||||
parseFloat(e.target.value).toFixed(2);
|
||||
});
|
||||
|
||||
document.getElementById('apply-filters').addEventListener('click', () => {
|
||||
this.loadInitialData();
|
||||
});
|
||||
|
||||
// Metadata panel
|
||||
document.getElementById('find-similar').addEventListener('click', () => {
|
||||
this.findSimilarNodes();
|
||||
});
|
||||
|
||||
document.getElementById('close-metadata').addEventListener('click', () => {
|
||||
document.getElementById('metadata-panel').style.display = 'none';
|
||||
this.selectedNode = null;
|
||||
this.g.selectAll('.node').classed('selected', false);
|
||||
this.updateStatistics();
|
||||
});
|
||||
|
||||
// Export
|
||||
document.getElementById('export-png').addEventListener('click', () => this.exportPNG());
|
||||
document.getElementById('export-svg').addEventListener('click', () => this.exportSVG());
|
||||
|
||||
// View controls
|
||||
document.getElementById('reset-view').addEventListener('click', () => this.resetView());
|
||||
document.getElementById('zoom-in').addEventListener('click', () => this.zoomIn());
|
||||
document.getElementById('zoom-out').addEventListener('click', () => this.zoomOut());
|
||||
document.getElementById('fit-view').addEventListener('click', () => this.fitView());
|
||||
|
||||
// Window resize
|
||||
window.addEventListener('resize', () => {
|
||||
const container = d3.select('#graph-canvas');
|
||||
const width = container.node().getBoundingClientRect().width;
|
||||
const height = container.node().getBoundingClientRect().height;
|
||||
|
||||
this.svg
|
||||
.attr('width', width)
|
||||
.attr('height', height);
|
||||
|
||||
this.simulation
|
||||
.force('center', d3.forceCenter(width / 2, height / 2))
|
||||
.alpha(0.3)
|
||||
.restart();
|
||||
});
|
||||
}
|
||||
|
||||
handleGraphUpdate(data) {
|
||||
this.updateGraph(data.nodes, data.links);
|
||||
}
|
||||
|
||||
handleNodeAdded(node) {
|
||||
this.nodes.push(node);
|
||||
this.renderGraph();
|
||||
this.showToast('New node added', 'info');
|
||||
}
|
||||
|
||||
handleNodeUpdated(node) {
|
||||
const index = this.nodes.findIndex(n => n.id === node.id);
|
||||
if (index !== -1) {
|
||||
this.nodes[index] = { ...this.nodes[index], ...node };
|
||||
this.renderGraph();
|
||||
this.showToast('Node updated', 'info');
|
||||
}
|
||||
}
|
||||
|
||||
handleSimilarityResult(data) {
|
||||
this.highlightSimilarNodes(data.similar);
|
||||
}
|
||||
}
|
||||
|
||||
// Initialize application when DOM is ready
|
||||
document.addEventListener('DOMContentLoaded', () => {
|
||||
window.graphExplorer = new GraphExplorer();
|
||||
});
|
||||
127
vendor/ruvector/npm/packages/ruvector-extensions/src/ui/index.html
vendored
Normal file
127
vendor/ruvector/npm/packages/ruvector-extensions/src/ui/index.html
vendored
Normal file
@@ -0,0 +1,127 @@
|
||||
<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||
<title>RuVector - Graph Explorer</title>
|
||||
<link rel="stylesheet" href="styles.css">
|
||||
<script src="https://d3js.org/d3.v7.min.js"></script>
|
||||
</head>
|
||||
<body>
|
||||
<div class="app-container">
|
||||
<!-- Header -->
|
||||
<header class="app-header">
|
||||
<div class="header-content">
|
||||
<h1>🔍 RuVector Graph Explorer</h1>
|
||||
<div class="header-controls">
|
||||
<button id="export-png" class="btn btn-secondary" title="Export as PNG">
|
||||
📷 PNG
|
||||
</button>
|
||||
<button id="export-svg" class="btn btn-secondary" title="Export as SVG">
|
||||
📊 SVG
|
||||
</button>
|
||||
<button id="reset-view" class="btn btn-secondary" title="Reset View">
|
||||
🔄 Reset
|
||||
</button>
|
||||
<div class="connection-status" id="connection-status">
|
||||
<span class="status-dot"></span>
|
||||
<span class="status-text">Connecting...</span>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</header>
|
||||
|
||||
<!-- Main Content -->
|
||||
<div class="main-content">
|
||||
<!-- Sidebar -->
|
||||
<aside class="sidebar">
|
||||
<div class="sidebar-section">
|
||||
<h2>Search & Filter</h2>
|
||||
<div class="search-box">
|
||||
<input
|
||||
type="text"
|
||||
id="node-search"
|
||||
placeholder="Search nodes by ID or metadata..."
|
||||
class="search-input"
|
||||
>
|
||||
<button id="clear-search" class="btn-icon" title="Clear search">✕</button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="sidebar-section">
|
||||
<h2>Filters</h2>
|
||||
<div class="filter-group">
|
||||
<label for="min-similarity">Min Similarity:</label>
|
||||
<input
|
||||
type="range"
|
||||
id="min-similarity"
|
||||
min="0"
|
||||
max="1"
|
||||
step="0.01"
|
||||
value="0.5"
|
||||
>
|
||||
<span id="similarity-value">0.50</span>
|
||||
</div>
|
||||
<div class="filter-group">
|
||||
<label for="max-nodes">Max Nodes:</label>
|
||||
<input
|
||||
type="number"
|
||||
id="max-nodes"
|
||||
min="10"
|
||||
max="1000"
|
||||
step="10"
|
||||
value="100"
|
||||
>
|
||||
</div>
|
||||
<button id="apply-filters" class="btn btn-primary">Apply Filters</button>
|
||||
</div>
|
||||
|
||||
<div class="sidebar-section">
|
||||
<h2>Statistics</h2>
|
||||
<div class="stats">
|
||||
<div class="stat-item">
|
||||
<span class="stat-label">Nodes:</span>
|
||||
<span class="stat-value" id="stat-nodes">0</span>
|
||||
</div>
|
||||
<div class="stat-item">
|
||||
<span class="stat-label">Edges:</span>
|
||||
<span class="stat-value" id="stat-edges">0</span>
|
||||
</div>
|
||||
<div class="stat-item">
|
||||
<span class="stat-label">Selected:</span>
|
||||
<span class="stat-value" id="stat-selected">None</span>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="sidebar-section" id="metadata-panel" style="display: none;">
|
||||
<h2>Node Details</h2>
|
||||
<div id="metadata-content"></div>
|
||||
<button id="find-similar" class="btn btn-primary">Find Similar Nodes</button>
|
||||
<button id="close-metadata" class="btn btn-secondary">Close</button>
|
||||
</div>
|
||||
</aside>
|
||||
|
||||
<!-- Graph Canvas -->
|
||||
<main class="graph-container">
|
||||
<div id="graph-canvas"></div>
|
||||
<div class="graph-controls">
|
||||
<button id="zoom-in" class="btn-icon" title="Zoom In">+</button>
|
||||
<button id="zoom-out" class="btn-icon" title="Zoom Out">−</button>
|
||||
<button id="fit-view" class="btn-icon" title="Fit to View">⊡</button>
|
||||
</div>
|
||||
<div class="loading-overlay" id="loading-overlay">
|
||||
<div class="spinner"></div>
|
||||
<p>Loading graph data...</p>
|
||||
</div>
|
||||
</main>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Toast Notifications -->
|
||||
<div id="toast-container"></div>
|
||||
|
||||
<!-- Scripts -->
|
||||
<script src="app.js"></script>
|
||||
</body>
|
||||
</html>
|
||||
512
vendor/ruvector/npm/packages/ruvector-extensions/src/ui/styles.css
vendored
Normal file
512
vendor/ruvector/npm/packages/ruvector-extensions/src/ui/styles.css
vendored
Normal file
@@ -0,0 +1,512 @@
|
||||
/* Reset & Base Styles */
|
||||
* {
|
||||
margin: 0;
|
||||
padding: 0;
|
||||
box-sizing: border-box;
|
||||
}
|
||||
|
||||
:root {
|
||||
--primary-color: #667eea;
|
||||
--secondary-color: #764ba2;
|
||||
--accent-color: #f093fb;
|
||||
--bg-dark: #1a1a2e;
|
||||
--bg-medium: #16213e;
|
||||
--bg-light: #0f3460;
|
||||
--text-primary: #eee;
|
||||
--text-secondary: #aaa;
|
||||
--border-color: #333;
|
||||
--success-color: #4caf50;
|
||||
--warning-color: #ff9800;
|
||||
--error-color: #f44336;
|
||||
--shadow: 0 4px 6px rgba(0, 0, 0, 0.3);
|
||||
}
|
||||
|
||||
body {
|
||||
font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, Oxygen, Ubuntu, Cantarell, sans-serif;
|
||||
background: linear-gradient(135deg, var(--bg-dark) 0%, var(--bg-medium) 100%);
|
||||
color: var(--text-primary);
|
||||
overflow: hidden;
|
||||
}
|
||||
|
||||
/* App Layout */
|
||||
.app-container {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
height: 100vh;
|
||||
}
|
||||
|
||||
/* Header */
|
||||
.app-header {
|
||||
background: var(--bg-medium);
|
||||
border-bottom: 2px solid var(--border-color);
|
||||
box-shadow: var(--shadow);
|
||||
z-index: 100;
|
||||
}
|
||||
|
||||
.header-content {
|
||||
display: flex;
|
||||
justify-content: space-between;
|
||||
align-items: center;
|
||||
padding: 1rem 2rem;
|
||||
max-width: 100%;
|
||||
}
|
||||
|
||||
.app-header h1 {
|
||||
font-size: 1.5rem;
|
||||
background: linear-gradient(135deg, var(--primary-color), var(--accent-color));
|
||||
-webkit-background-clip: text;
|
||||
-webkit-text-fill-color: transparent;
|
||||
background-clip: text;
|
||||
}
|
||||
|
||||
.header-controls {
|
||||
display: flex;
|
||||
gap: 0.5rem;
|
||||
align-items: center;
|
||||
}
|
||||
|
||||
/* Connection Status */
|
||||
.connection-status {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 0.5rem;
|
||||
padding: 0.5rem 1rem;
|
||||
background: var(--bg-light);
|
||||
border-radius: 20px;
|
||||
font-size: 0.875rem;
|
||||
}
|
||||
|
||||
.status-dot {
|
||||
width: 8px;
|
||||
height: 8px;
|
||||
border-radius: 50%;
|
||||
background: var(--error-color);
|
||||
animation: pulse 2s infinite;
|
||||
}
|
||||
|
||||
.status-dot.connected {
|
||||
background: var(--success-color);
|
||||
animation: none;
|
||||
}
|
||||
|
||||
@keyframes pulse {
|
||||
0%, 100% { opacity: 1; }
|
||||
50% { opacity: 0.5; }
|
||||
}
|
||||
|
||||
/* Main Content */
|
||||
.main-content {
|
||||
display: flex;
|
||||
flex: 1;
|
||||
overflow: hidden;
|
||||
}
|
||||
|
||||
/* Sidebar */
|
||||
.sidebar {
|
||||
width: 320px;
|
||||
background: var(--bg-medium);
|
||||
border-right: 2px solid var(--border-color);
|
||||
overflow-y: auto;
|
||||
padding: 1.5rem;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 1.5rem;
|
||||
}
|
||||
|
||||
.sidebar-section {
|
||||
background: var(--bg-light);
|
||||
padding: 1.5rem;
|
||||
border-radius: 12px;
|
||||
box-shadow: var(--shadow);
|
||||
}
|
||||
|
||||
.sidebar-section h2 {
|
||||
font-size: 1.1rem;
|
||||
margin-bottom: 1rem;
|
||||
color: var(--primary-color);
|
||||
}
|
||||
|
||||
/* Search Box */
|
||||
.search-box {
|
||||
display: flex;
|
||||
gap: 0.5rem;
|
||||
}
|
||||
|
||||
.search-input {
|
||||
flex: 1;
|
||||
padding: 0.75rem;
|
||||
background: var(--bg-dark);
|
||||
border: 2px solid var(--border-color);
|
||||
border-radius: 8px;
|
||||
color: var(--text-primary);
|
||||
font-size: 0.9rem;
|
||||
transition: border-color 0.3s;
|
||||
}
|
||||
|
||||
.search-input:focus {
|
||||
outline: none;
|
||||
border-color: var(--primary-color);
|
||||
}
|
||||
|
||||
/* Filters */
|
||||
.filter-group {
|
||||
margin-bottom: 1rem;
|
||||
}
|
||||
|
||||
.filter-group label {
|
||||
display: block;
|
||||
margin-bottom: 0.5rem;
|
||||
font-size: 0.9rem;
|
||||
color: var(--text-secondary);
|
||||
}
|
||||
|
||||
.filter-group input[type="range"] {
|
||||
width: 100%;
|
||||
margin-right: 0.5rem;
|
||||
}
|
||||
|
||||
.filter-group input[type="number"] {
|
||||
width: 100%;
|
||||
padding: 0.5rem;
|
||||
background: var(--bg-dark);
|
||||
border: 2px solid var(--border-color);
|
||||
border-radius: 8px;
|
||||
color: var(--text-primary);
|
||||
}
|
||||
|
||||
#similarity-value {
|
||||
font-weight: bold;
|
||||
color: var(--accent-color);
|
||||
}
|
||||
|
||||
/* Statistics */
|
||||
.stats {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 0.75rem;
|
||||
}
|
||||
|
||||
.stat-item {
|
||||
display: flex;
|
||||
justify-content: space-between;
|
||||
padding: 0.75rem;
|
||||
background: var(--bg-dark);
|
||||
border-radius: 8px;
|
||||
}
|
||||
|
||||
.stat-label {
|
||||
color: var(--text-secondary);
|
||||
}
|
||||
|
||||
.stat-value {
|
||||
font-weight: bold;
|
||||
color: var(--accent-color);
|
||||
}
|
||||
|
||||
/* Metadata Panel */
|
||||
#metadata-content {
|
||||
margin-bottom: 1rem;
|
||||
max-height: 300px;
|
||||
overflow-y: auto;
|
||||
}
|
||||
|
||||
.metadata-item {
|
||||
padding: 0.75rem;
|
||||
background: var(--bg-dark);
|
||||
border-radius: 8px;
|
||||
margin-bottom: 0.5rem;
|
||||
}
|
||||
|
||||
.metadata-item strong {
|
||||
color: var(--primary-color);
|
||||
display: block;
|
||||
margin-bottom: 0.25rem;
|
||||
}
|
||||
|
||||
/* Buttons */
|
||||
.btn {
|
||||
padding: 0.75rem 1.5rem;
|
||||
border: none;
|
||||
border-radius: 8px;
|
||||
font-size: 0.9rem;
|
||||
font-weight: 600;
|
||||
cursor: pointer;
|
||||
transition: all 0.3s;
|
||||
text-transform: uppercase;
|
||||
letter-spacing: 0.5px;
|
||||
}
|
||||
|
||||
.btn-primary {
|
||||
background: linear-gradient(135deg, var(--primary-color), var(--secondary-color));
|
||||
color: white;
|
||||
width: 100%;
|
||||
}
|
||||
|
||||
.btn-primary:hover {
|
||||
transform: translateY(-2px);
|
||||
box-shadow: 0 6px 12px rgba(102, 126, 234, 0.4);
|
||||
}
|
||||
|
||||
.btn-secondary {
|
||||
background: var(--bg-light);
|
||||
color: var(--text-primary);
|
||||
border: 2px solid var(--border-color);
|
||||
}
|
||||
|
||||
.btn-secondary:hover {
|
||||
border-color: var(--primary-color);
|
||||
background: var(--bg-medium);
|
||||
}
|
||||
|
||||
.btn-icon {
|
||||
width: 40px;
|
||||
height: 40px;
|
||||
border: none;
|
||||
border-radius: 50%;
|
||||
background: var(--bg-light);
|
||||
color: var(--text-primary);
|
||||
font-size: 1.2rem;
|
||||
cursor: pointer;
|
||||
transition: all 0.3s;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
}
|
||||
|
||||
.btn-icon:hover {
|
||||
background: var(--primary-color);
|
||||
transform: scale(1.1);
|
||||
}
|
||||
|
||||
/* Graph Container */
|
||||
.graph-container {
|
||||
flex: 1;
|
||||
position: relative;
|
||||
overflow: hidden;
|
||||
}
|
||||
|
||||
#graph-canvas {
|
||||
width: 100%;
|
||||
height: 100%;
|
||||
}
|
||||
|
||||
.graph-controls {
|
||||
position: absolute;
|
||||
bottom: 2rem;
|
||||
right: 2rem;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 0.5rem;
|
||||
}
|
||||
|
||||
/* Loading Overlay */
|
||||
.loading-overlay {
|
||||
position: absolute;
|
||||
top: 0;
|
||||
left: 0;
|
||||
right: 0;
|
||||
bottom: 0;
|
||||
background: rgba(26, 26, 46, 0.9);
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
z-index: 1000;
|
||||
}
|
||||
|
||||
.loading-overlay.hidden {
|
||||
display: none;
|
||||
}
|
||||
|
||||
.spinner {
|
||||
width: 60px;
|
||||
height: 60px;
|
||||
border: 4px solid var(--border-color);
|
||||
border-top: 4px solid var(--primary-color);
|
||||
border-radius: 50%;
|
||||
animation: spin 1s linear infinite;
|
||||
}
|
||||
|
||||
@keyframes spin {
|
||||
0% { transform: rotate(0deg); }
|
||||
100% { transform: rotate(360deg); }
|
||||
}
|
||||
|
||||
/* Toast Notifications */
|
||||
#toast-container {
|
||||
position: fixed;
|
||||
top: 5rem;
|
||||
right: 2rem;
|
||||
z-index: 2000;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 0.5rem;
|
||||
}
|
||||
|
||||
.toast {
|
||||
padding: 1rem 1.5rem;
|
||||
background: var(--bg-medium);
|
||||
border-left: 4px solid var(--primary-color);
|
||||
border-radius: 8px;
|
||||
box-shadow: var(--shadow);
|
||||
animation: slideIn 0.3s ease-out;
|
||||
min-width: 250px;
|
||||
}
|
||||
|
||||
.toast.success {
|
||||
border-left-color: var(--success-color);
|
||||
}
|
||||
|
||||
.toast.error {
|
||||
border-left-color: var(--error-color);
|
||||
}
|
||||
|
||||
.toast.warning {
|
||||
border-left-color: var(--warning-color);
|
||||
}
|
||||
|
||||
@keyframes slideIn {
|
||||
from {
|
||||
transform: translateX(400px);
|
||||
opacity: 0;
|
||||
}
|
||||
to {
|
||||
transform: translateX(0);
|
||||
opacity: 1;
|
||||
}
|
||||
}
|
||||
|
||||
/* Graph Styles */
|
||||
.node {
|
||||
cursor: pointer;
|
||||
stroke: var(--bg-dark);
|
||||
stroke-width: 2px;
|
||||
transition: all 0.3s;
|
||||
}
|
||||
|
||||
.node:hover {
|
||||
stroke: var(--accent-color);
|
||||
stroke-width: 3px;
|
||||
}
|
||||
|
||||
.node.selected {
|
||||
stroke: var(--primary-color);
|
||||
stroke-width: 4px;
|
||||
}
|
||||
|
||||
.node.highlighted {
|
||||
stroke: var(--success-color);
|
||||
stroke-width: 3px;
|
||||
}
|
||||
|
||||
.link {
|
||||
stroke: var(--border-color);
|
||||
stroke-opacity: 0.6;
|
||||
stroke-width: 1.5px;
|
||||
}
|
||||
|
||||
.link.highlighted {
|
||||
stroke: var(--primary-color);
|
||||
stroke-opacity: 1;
|
||||
stroke-width: 2.5px;
|
||||
}
|
||||
|
||||
.node-label {
|
||||
font-size: 11px;
|
||||
fill: var(--text-primary);
|
||||
text-anchor: middle;
|
||||
pointer-events: none;
|
||||
user-select: none;
|
||||
}
|
||||
|
||||
/* Responsive Design */
|
||||
@media (max-width: 1024px) {
|
||||
.sidebar {
|
||||
width: 280px;
|
||||
}
|
||||
|
||||
.header-content {
|
||||
padding: 1rem;
|
||||
}
|
||||
|
||||
.app-header h1 {
|
||||
font-size: 1.2rem;
|
||||
}
|
||||
}
|
||||
|
||||
@media (max-width: 768px) {
|
||||
.main-content {
|
||||
flex-direction: column;
|
||||
}
|
||||
|
||||
.sidebar {
|
||||
width: 100%;
|
||||
max-height: 40vh;
|
||||
border-right: none;
|
||||
border-bottom: 2px solid var(--border-color);
|
||||
}
|
||||
|
||||
.header-content {
|
||||
flex-direction: column;
|
||||
gap: 1rem;
|
||||
}
|
||||
|
||||
.header-controls {
|
||||
width: 100%;
|
||||
justify-content: space-between;
|
||||
}
|
||||
|
||||
.graph-controls {
|
||||
bottom: 1rem;
|
||||
right: 1rem;
|
||||
}
|
||||
|
||||
#toast-container {
|
||||
right: 1rem;
|
||||
left: 1rem;
|
||||
}
|
||||
|
||||
.btn {
|
||||
padding: 0.6rem 1rem;
|
||||
font-size: 0.8rem;
|
||||
}
|
||||
}
|
||||
|
||||
@media (max-width: 480px) {
|
||||
.sidebar {
|
||||
padding: 1rem;
|
||||
}
|
||||
|
||||
.sidebar-section {
|
||||
padding: 1rem;
|
||||
}
|
||||
|
||||
.app-header h1 {
|
||||
font-size: 1rem;
|
||||
}
|
||||
|
||||
.btn-icon {
|
||||
width: 35px;
|
||||
height: 35px;
|
||||
font-size: 1rem;
|
||||
}
|
||||
}
|
||||
|
||||
/* Scrollbar Styling */
|
||||
::-webkit-scrollbar {
|
||||
width: 8px;
|
||||
height: 8px;
|
||||
}
|
||||
|
||||
::-webkit-scrollbar-track {
|
||||
background: var(--bg-dark);
|
||||
}
|
||||
|
||||
::-webkit-scrollbar-thumb {
|
||||
background: var(--border-color);
|
||||
border-radius: 4px;
|
||||
}
|
||||
|
||||
::-webkit-scrollbar-thumb:hover {
|
||||
background: var(--primary-color);
|
||||
}
|
||||
Reference in New Issue
Block a user