Merge commit 'd803bfe2b1fe7f5e219e50ac20d6801a0a58ac75' as 'vendor/ruvector'
This commit is contained in:
ruv
696
vendor/ruvector/examples/edge-net/pkg/models/model-registry.js
vendored
Normal file
696
vendor/ruvector/examples/edge-net/pkg/models/model-registry.js
vendored
Normal file
@@ -0,0 +1,696 @@
|
||||
/**
|
||||
* @ruvector/edge-net Model Registry
|
||||
*
|
||||
* Manages model metadata, versions, dependencies, and discovery
|
||||
* for the distributed model distribution infrastructure.
|
||||
*
|
||||
* @module @ruvector/edge-net/models/model-registry
|
||||
*/
|
||||
|
||||
import { EventEmitter } from 'events';
|
||||
import { createHash, randomBytes } from 'crypto';
|
||||
import { promises as fs } from 'fs';
|
||||
import path from 'path';
|
||||
|
||||
// ============================================
|
||||
// SEMVER UTILITIES
|
||||
// ============================================
|
||||
|
||||
/**
|
||||
* Parse a semver version string
|
||||
* @param {string} version - Version string (e.g., "1.2.3", "1.0.0-beta.1")
|
||||
* @returns {object} Parsed version object
|
||||
*/
|
||||
export function parseSemver(version) {
|
||||
const match = String(version).match(
|
||||
/^(\d+)\.(\d+)\.(\d+)(?:-([a-zA-Z0-9.-]+))?(?:\+([a-zA-Z0-9.-]+))?$/
|
||||
);
|
||||
|
||||
if (!match) {
|
||||
throw new Error(`Invalid semver: ${version}`);
|
||||
}
|
||||
|
||||
return {
|
||||
major: parseInt(match[1], 10),
|
||||
minor: parseInt(match[2], 10),
|
||||
patch: parseInt(match[3], 10),
|
||||
prerelease: match[4] || null,
|
||||
build: match[5] || null,
|
||||
raw: version,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Compare two semver versions
|
||||
* @param {string} a - First version
|
||||
* @param {string} b - Second version
|
||||
* @returns {number} -1 if a < b, 0 if equal, 1 if a > b
|
||||
*/
|
||||
export function compareSemver(a, b) {
|
||||
const va = parseSemver(a);
|
||||
const vb = parseSemver(b);
|
||||
|
||||
if (va.major !== vb.major) return va.major - vb.major;
|
||||
if (va.minor !== vb.minor) return va.minor - vb.minor;
|
||||
if (va.patch !== vb.patch) return va.patch - vb.patch;
|
||||
|
||||
// Prerelease versions have lower precedence
|
||||
if (va.prerelease && !vb.prerelease) return -1;
|
||||
if (!va.prerelease && vb.prerelease) return 1;
|
||||
if (va.prerelease && vb.prerelease) {
|
||||
return va.prerelease.localeCompare(vb.prerelease);
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if version satisfies a version range
|
||||
* Supports: "1.0.0", "^1.0.0", "~1.0.0", ">=1.0.0", "1.x", "*"
|
||||
* @param {string} version - Version to check
|
||||
* @param {string} range - Version range
|
||||
* @returns {boolean}
|
||||
*/
|
||||
export function satisfiesSemver(version, range) {
|
||||
const v = parseSemver(version);
|
||||
|
||||
// Exact match
|
||||
if (range === version) return true;
|
||||
|
||||
// Wildcard
|
||||
if (range === '*' || range === 'latest') return true;
|
||||
|
||||
// X-range: 1.x, 1.2.x
|
||||
const xMatch = range.match(/^(\d+)(?:\.(\d+))?\.x$/);
|
||||
if (xMatch) {
|
||||
const major = parseInt(xMatch[1], 10);
|
||||
const minor = xMatch[2] ? parseInt(xMatch[2], 10) : null;
|
||||
if (v.major !== major) return false;
|
||||
if (minor !== null && v.minor !== minor) return false;
|
||||
return true;
|
||||
}
|
||||
|
||||
// Caret range: ^1.0.0 (compatible with)
|
||||
if (range.startsWith('^')) {
|
||||
const r = parseSemver(range.slice(1));
|
||||
if (v.major !== r.major) return false;
|
||||
if (v.major === 0) {
|
||||
if (v.minor !== r.minor) return false;
|
||||
return v.patch >= r.patch;
|
||||
}
|
||||
return compareSemver(version, range.slice(1)) >= 0;
|
||||
}
|
||||
|
||||
// Tilde range: ~1.0.0 (approximately equivalent)
|
||||
if (range.startsWith('~')) {
|
||||
const r = parseSemver(range.slice(1));
|
||||
if (v.major !== r.major) return false;
|
||||
if (v.minor !== r.minor) return false;
|
||||
return v.patch >= r.patch;
|
||||
}
|
||||
|
||||
// Comparison ranges: >=1.0.0, >1.0.0, <=1.0.0, <1.0.0
|
||||
if (range.startsWith('>=')) {
|
||||
return compareSemver(version, range.slice(2)) >= 0;
|
||||
}
|
||||
if (range.startsWith('>')) {
|
||||
return compareSemver(version, range.slice(1)) > 0;
|
||||
}
|
||||
if (range.startsWith('<=')) {
|
||||
return compareSemver(version, range.slice(2)) <= 0;
|
||||
}
|
||||
if (range.startsWith('<')) {
|
||||
return compareSemver(version, range.slice(1)) < 0;
|
||||
}
|
||||
|
||||
// Fallback to exact match
|
||||
return compareSemver(version, range) === 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the latest version from a list
|
||||
* @param {string[]} versions - List of version strings
|
||||
* @returns {string} Latest version
|
||||
*/
|
||||
export function getLatestVersion(versions) {
|
||||
if (!versions || versions.length === 0) return null;
|
||||
return versions.sort((a, b) => compareSemver(b, a))[0];
|
||||
}
|
||||
|
||||
// ============================================
|
||||
// MODEL METADATA
|
||||
// ============================================
|
||||
|
||||
/**
|
||||
* Model metadata structure
|
||||
* @typedef {object} ModelMetadata
|
||||
* @property {string} name - Model identifier (e.g., "phi-1.5-int4")
|
||||
* @property {string} version - Semantic version
|
||||
* @property {number} size - Model size in bytes
|
||||
* @property {string} hash - SHA256 hash for integrity
|
||||
* @property {string} format - Model format (onnx, safetensors, gguf)
|
||||
* @property {string[]} capabilities - Model capabilities
|
||||
* @property {object} sources - Download sources (gcs, ipfs, cdn)
|
||||
* @property {object} dependencies - Base models and adapters
|
||||
* @property {object} quantization - Quantization details
|
||||
* @property {object} metadata - Additional metadata
|
||||
*/
|
||||
|
||||
/**
|
||||
* Create a model metadata object
|
||||
* @param {object} options - Model options
|
||||
* @returns {ModelMetadata}
|
||||
*/
|
||||
export function createModelMetadata(options) {
|
||||
const {
|
||||
name,
|
||||
version = '1.0.0',
|
||||
size = 0,
|
||||
hash = '',
|
||||
format = 'onnx',
|
||||
capabilities = [],
|
||||
sources = {},
|
||||
dependencies = {},
|
||||
quantization = null,
|
||||
metadata = {},
|
||||
} = options;
|
||||
|
||||
if (!name) {
|
||||
throw new Error('Model name is required');
|
||||
}
|
||||
|
||||
// Validate version
|
||||
parseSemver(version);
|
||||
|
||||
return {
|
||||
name,
|
||||
version,
|
||||
size,
|
||||
hash,
|
||||
format,
|
||||
capabilities: Array.isArray(capabilities) ? capabilities : [capabilities],
|
||||
sources: {
|
||||
gcs: sources.gcs || null,
|
||||
ipfs: sources.ipfs || null,
|
||||
cdn: sources.cdn || null,
|
||||
...sources,
|
||||
},
|
||||
dependencies: {
|
||||
base: dependencies.base || null,
|
||||
adapters: dependencies.adapters || [],
|
||||
...dependencies,
|
||||
},
|
||||
quantization: quantization ? {
|
||||
type: quantization.type || 'int4',
|
||||
bits: quantization.bits || 4,
|
||||
blockSize: quantization.blockSize || 32,
|
||||
symmetric: quantization.symmetric ?? true,
|
||||
} : null,
|
||||
metadata: {
|
||||
createdAt: metadata.createdAt || new Date().toISOString(),
|
||||
updatedAt: metadata.updatedAt || new Date().toISOString(),
|
||||
author: metadata.author || 'RuVector',
|
||||
license: metadata.license || 'Apache-2.0',
|
||||
description: metadata.description || '',
|
||||
tags: metadata.tags || [],
|
||||
...metadata,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
// ============================================
|
||||
// MODEL REGISTRY
|
||||
// ============================================
|
||||
|
||||
/**
|
||||
* ModelRegistry - Manages model metadata, versions, and dependencies
|
||||
*/
|
||||
export class ModelRegistry extends EventEmitter {
|
||||
/**
|
||||
* Create a new ModelRegistry
|
||||
* @param {object} options - Registry options
|
||||
*/
|
||||
constructor(options = {}) {
|
||||
super();
|
||||
|
||||
this.id = `registry-${randomBytes(6).toString('hex')}`;
|
||||
this.registryPath = options.registryPath || null;
|
||||
|
||||
// Model storage: { modelName: { version: ModelMetadata } }
|
||||
this.models = new Map();
|
||||
|
||||
// Dependency graph
|
||||
this.dependencies = new Map();
|
||||
|
||||
// Search index
|
||||
this.searchIndex = {
|
||||
byCapability: new Map(),
|
||||
byFormat: new Map(),
|
||||
byTag: new Map(),
|
||||
};
|
||||
|
||||
// Stats
|
||||
this.stats = {
|
||||
totalModels: 0,
|
||||
totalVersions: 0,
|
||||
totalSize: 0,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Register a new model or version
|
||||
* @param {object} modelData - Model metadata
|
||||
* @returns {ModelMetadata}
|
||||
*/
|
||||
register(modelData) {
|
||||
const metadata = createModelMetadata(modelData);
|
||||
const { name, version } = metadata;
|
||||
|
||||
// Get or create model entry
|
||||
if (!this.models.has(name)) {
|
||||
this.models.set(name, new Map());
|
||||
this.stats.totalModels++;
|
||||
}
|
||||
|
||||
const versions = this.models.get(name);
|
||||
|
||||
// Check if version exists
|
||||
if (versions.has(version)) {
|
||||
this.emit('warning', {
|
||||
type: 'version_exists',
|
||||
model: name,
|
||||
version,
|
||||
});
|
||||
}
|
||||
|
||||
// Store metadata
|
||||
versions.set(version, metadata);
|
||||
this.stats.totalVersions++;
|
||||
this.stats.totalSize += metadata.size;
|
||||
|
||||
// Update search index
|
||||
this._indexModel(metadata);
|
||||
|
||||
// Update dependency graph
|
||||
this._updateDependencies(metadata);
|
||||
|
||||
this.emit('registered', { name, version, metadata });
|
||||
|
||||
return metadata;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get model metadata
|
||||
* @param {string} name - Model name
|
||||
* @param {string} version - Version (default: latest)
|
||||
* @returns {ModelMetadata|null}
|
||||
*/
|
||||
get(name, version = 'latest') {
|
||||
const versions = this.models.get(name);
|
||||
if (!versions) return null;
|
||||
|
||||
if (version === 'latest') {
|
||||
const latest = getLatestVersion([...versions.keys()]);
|
||||
return latest ? versions.get(latest) : null;
|
||||
}
|
||||
|
||||
// Check for exact match first
|
||||
if (versions.has(version)) {
|
||||
return versions.get(version);
|
||||
}
|
||||
|
||||
// Try to find matching version in range
|
||||
for (const [v, metadata] of versions) {
|
||||
if (satisfiesSemver(v, version)) {
|
||||
return metadata;
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* List all versions of a model
|
||||
* @param {string} name - Model name
|
||||
* @returns {string[]}
|
||||
*/
|
||||
listVersions(name) {
|
||||
const versions = this.models.get(name);
|
||||
if (!versions) return [];
|
||||
|
||||
return [...versions.keys()].sort((a, b) => compareSemver(b, a));
|
||||
}
|
||||
|
||||
/**
|
||||
* List all registered models
|
||||
* @returns {string[]}
|
||||
*/
|
||||
listModels() {
|
||||
return [...this.models.keys()];
|
||||
}
|
||||
|
||||
/**
|
||||
* Search for models
|
||||
* @param {object} criteria - Search criteria
|
||||
* @returns {ModelMetadata[]}
|
||||
*/
|
||||
search(criteria = {}) {
|
||||
const {
|
||||
name = null,
|
||||
capability = null,
|
||||
format = null,
|
||||
tag = null,
|
||||
minVersion = null,
|
||||
maxVersion = null,
|
||||
maxSize = null,
|
||||
query = null,
|
||||
} = criteria;
|
||||
|
||||
let results = [];
|
||||
|
||||
// Start with all models or filtered by name
|
||||
if (name) {
|
||||
const versions = this.models.get(name);
|
||||
if (versions) {
|
||||
results = [...versions.values()];
|
||||
}
|
||||
} else {
|
||||
// Collect all model versions
|
||||
for (const versions of this.models.values()) {
|
||||
results.push(...versions.values());
|
||||
}
|
||||
}
|
||||
|
||||
// Filter by capability
|
||||
if (capability) {
|
||||
results = results.filter(m =>
|
||||
m.capabilities.includes(capability)
|
||||
);
|
||||
}
|
||||
|
||||
// Filter by format
|
||||
if (format) {
|
||||
results = results.filter(m => m.format === format);
|
||||
}
|
||||
|
||||
// Filter by tag
|
||||
if (tag) {
|
||||
results = results.filter(m =>
|
||||
m.metadata.tags && m.metadata.tags.includes(tag)
|
||||
);
|
||||
}
|
||||
|
||||
// Filter by version range
|
||||
if (minVersion) {
|
||||
results = results.filter(m =>
|
||||
compareSemver(m.version, minVersion) >= 0
|
||||
);
|
||||
}
|
||||
|
||||
if (maxVersion) {
|
||||
results = results.filter(m =>
|
||||
compareSemver(m.version, maxVersion) <= 0
|
||||
);
|
||||
}
|
||||
|
||||
// Filter by size
|
||||
if (maxSize) {
|
||||
results = results.filter(m => m.size <= maxSize);
|
||||
}
|
||||
|
||||
// Text search
|
||||
if (query) {
|
||||
const q = query.toLowerCase();
|
||||
results = results.filter(m =>
|
||||
m.name.toLowerCase().includes(q) ||
|
||||
m.metadata.description?.toLowerCase().includes(q) ||
|
||||
m.metadata.tags?.some(t => t.toLowerCase().includes(q))
|
||||
);
|
||||
}
|
||||
|
||||
return results;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get models by capability
|
||||
* @param {string} capability - Capability to search for
|
||||
* @returns {ModelMetadata[]}
|
||||
*/
|
||||
getByCapability(capability) {
|
||||
const models = this.searchIndex.byCapability.get(capability);
|
||||
if (!models) return [];
|
||||
|
||||
return models.map(key => {
|
||||
const [name, version] = key.split('@');
|
||||
return this.get(name, version);
|
||||
}).filter(Boolean);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all dependencies for a model
|
||||
* @param {string} name - Model name
|
||||
* @param {string} version - Version
|
||||
* @param {boolean} recursive - Include transitive dependencies
|
||||
* @returns {ModelMetadata[]}
|
||||
*/
|
||||
getDependencies(name, version = 'latest', recursive = true) {
|
||||
const model = this.get(name, version);
|
||||
if (!model) return [];
|
||||
|
||||
const deps = [];
|
||||
const visited = new Set();
|
||||
const queue = [model];
|
||||
|
||||
while (queue.length > 0) {
|
||||
const current = queue.shift();
|
||||
const key = `${current.name}@${current.version}`;
|
||||
|
||||
if (visited.has(key)) continue;
|
||||
visited.add(key);
|
||||
|
||||
// Add base model
|
||||
if (current.dependencies.base) {
|
||||
const [baseName, baseVersion] = current.dependencies.base.split('@');
|
||||
const baseDep = this.get(baseName, baseVersion || 'latest');
|
||||
if (baseDep) {
|
||||
deps.push(baseDep);
|
||||
if (recursive) queue.push(baseDep);
|
||||
}
|
||||
}
|
||||
|
||||
// Add adapters
|
||||
if (current.dependencies.adapters) {
|
||||
for (const adapter of current.dependencies.adapters) {
|
||||
const [adapterName, adapterVersion] = adapter.split('@');
|
||||
const adapterDep = this.get(adapterName, adapterVersion || 'latest');
|
||||
if (adapterDep) {
|
||||
deps.push(adapterDep);
|
||||
if (recursive) queue.push(adapterDep);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return deps;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get dependents (models that depend on this one)
|
||||
* @param {string} name - Model name
|
||||
* @param {string} version - Version
|
||||
* @returns {ModelMetadata[]}
|
||||
*/
|
||||
getDependents(name, version = 'latest') {
|
||||
const key = version === 'latest'
|
||||
? name
|
||||
: `${name}@${version}`;
|
||||
|
||||
const dependents = [];
|
||||
|
||||
for (const [depKey, dependencies] of this.dependencies) {
|
||||
if (dependencies.includes(key) || dependencies.includes(name)) {
|
||||
const [modelName, modelVersion] = depKey.split('@');
|
||||
const model = this.get(modelName, modelVersion);
|
||||
if (model) dependents.push(model);
|
||||
}
|
||||
}
|
||||
|
||||
return dependents;
|
||||
}
|
||||
|
||||
/**
|
||||
* Compute hash for a model file
|
||||
* @param {Buffer|Uint8Array} data - Model data
|
||||
* @returns {string} SHA256 hash
|
||||
*/
|
||||
static computeHash(data) {
|
||||
return `sha256:${createHash('sha256').update(data).digest('hex')}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Verify model integrity
|
||||
* @param {string} name - Model name
|
||||
* @param {string} version - Version
|
||||
* @param {Buffer|Uint8Array} data - Model data
|
||||
* @returns {boolean}
|
||||
*/
|
||||
verify(name, version, data) {
|
||||
const model = this.get(name, version);
|
||||
if (!model) return false;
|
||||
|
||||
const computedHash = ModelRegistry.computeHash(data);
|
||||
return model.hash === computedHash;
|
||||
}
|
||||
|
||||
/**
|
||||
* Update search index for a model
|
||||
* @private
|
||||
*/
|
||||
_indexModel(metadata) {
|
||||
const key = `${metadata.name}@${metadata.version}`;
|
||||
|
||||
// Index by capability
|
||||
for (const cap of metadata.capabilities) {
|
||||
if (!this.searchIndex.byCapability.has(cap)) {
|
||||
this.searchIndex.byCapability.set(cap, []);
|
||||
}
|
||||
this.searchIndex.byCapability.get(cap).push(key);
|
||||
}
|
||||
|
||||
// Index by format
|
||||
if (!this.searchIndex.byFormat.has(metadata.format)) {
|
||||
this.searchIndex.byFormat.set(metadata.format, []);
|
||||
}
|
||||
this.searchIndex.byFormat.get(metadata.format).push(key);
|
||||
|
||||
// Index by tags
|
||||
if (metadata.metadata.tags) {
|
||||
for (const tag of metadata.metadata.tags) {
|
||||
if (!this.searchIndex.byTag.has(tag)) {
|
||||
this.searchIndex.byTag.set(tag, []);
|
||||
}
|
||||
this.searchIndex.byTag.get(tag).push(key);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Update dependency graph
|
||||
* @private
|
||||
*/
|
||||
_updateDependencies(metadata) {
|
||||
const key = `${metadata.name}@${metadata.version}`;
|
||||
const deps = [];
|
||||
|
||||
if (metadata.dependencies.base) {
|
||||
deps.push(metadata.dependencies.base);
|
||||
}
|
||||
|
||||
if (metadata.dependencies.adapters) {
|
||||
deps.push(...metadata.dependencies.adapters);
|
||||
}
|
||||
|
||||
if (deps.length > 0) {
|
||||
this.dependencies.set(key, deps);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Export registry to JSON
|
||||
* @returns {object}
|
||||
*/
|
||||
export() {
|
||||
const models = {};
|
||||
|
||||
for (const [name, versions] of this.models) {
|
||||
models[name] = {};
|
||||
for (const [version, metadata] of versions) {
|
||||
models[name][version] = metadata;
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
version: '1.0.0',
|
||||
generatedAt: new Date().toISOString(),
|
||||
stats: this.stats,
|
||||
models,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Import registry from JSON
|
||||
* @param {object} data - Registry data
|
||||
*/
|
||||
import(data) {
|
||||
if (!data.models) return;
|
||||
|
||||
for (const [name, versions] of Object.entries(data.models)) {
|
||||
for (const [version, metadata] of Object.entries(versions)) {
|
||||
this.register({
|
||||
...metadata,
|
||||
name,
|
||||
version,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Save registry to file
|
||||
* @param {string} filePath - File path
|
||||
*/
|
||||
async save(filePath = null) {
|
||||
const targetPath = filePath || this.registryPath;
|
||||
if (!targetPath) {
|
||||
throw new Error('No registry path specified');
|
||||
}
|
||||
|
||||
const data = JSON.stringify(this.export(), null, 2);
|
||||
await fs.writeFile(targetPath, data, 'utf-8');
|
||||
|
||||
this.emit('saved', { path: targetPath });
|
||||
}
|
||||
|
||||
/**
|
||||
* Load registry from file
|
||||
* @param {string} filePath - File path
|
||||
*/
|
||||
async load(filePath = null) {
|
||||
const targetPath = filePath || this.registryPath;
|
||||
if (!targetPath) {
|
||||
throw new Error('No registry path specified');
|
||||
}
|
||||
|
||||
try {
|
||||
const data = await fs.readFile(targetPath, 'utf-8');
|
||||
this.import(JSON.parse(data));
|
||||
this.emit('loaded', { path: targetPath });
|
||||
} catch (error) {
|
||||
if (error.code === 'ENOENT') {
|
||||
this.emit('warning', { message: 'Registry file not found, starting fresh' });
|
||||
} else {
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get registry statistics
|
||||
* @returns {object}
|
||||
*/
|
||||
getStats() {
|
||||
return {
|
||||
...this.stats,
|
||||
capabilities: this.searchIndex.byCapability.size,
|
||||
formats: this.searchIndex.byFormat.size,
|
||||
tags: this.searchIndex.byTag.size,
|
||||
dependencyEdges: this.dependencies.size,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================
|
||||
// DEFAULT EXPORT
|
||||
// ============================================
|
||||
|
||||
export default ModelRegistry;
|
||||
Reference in New Issue
Block a user