Merge commit 'd803bfe2b1fe7f5e219e50ac20d6801a0a58ac75' as 'vendor/ruvector'
This commit is contained in:
1008
vendor/ruvector/examples/edge-net/pkg/models/adapter-hub.js
vendored
Normal file
1008
vendor/ruvector/examples/edge-net/pkg/models/adapter-hub.js
vendored
Normal file
File diff suppressed because it is too large
Load Diff
792
vendor/ruvector/examples/edge-net/pkg/models/adapter-security.js
vendored
Normal file
792
vendor/ruvector/examples/edge-net/pkg/models/adapter-security.js
vendored
Normal file
@@ -0,0 +1,792 @@
|
||||
/**
|
||||
* @ruvector/edge-net Adapter Security
|
||||
*
|
||||
* Security for MicroLoRA adapters:
|
||||
* - Quarantine before activation
|
||||
* - Local evaluation gating
|
||||
* - Base model matching
|
||||
* - Signature verification
|
||||
* - Merge lineage tracking
|
||||
*
|
||||
* Invariant: Adapters never applied without full verification.
|
||||
*
|
||||
* @module @ruvector/edge-net/models/adapter-security
|
||||
*/
|
||||
|
||||
import { createHash } from 'crypto';
|
||||
import { canonicalize, hashCanonical, TrustRoot, ManifestVerifier } from './integrity.js';
|
||||
|
||||
// ============================================================================
|
||||
// ADAPTER VERIFICATION
|
||||
// ============================================================================
|
||||
|
||||
/**
|
||||
* Adapter verification rules
|
||||
*/
|
||||
export const ADAPTER_REQUIREMENTS = Object.freeze({
|
||||
// Base model must match exactly
|
||||
requireExactBaseMatch: true,
|
||||
|
||||
// Checksum must match manifest
|
||||
requireChecksumMatch: true,
|
||||
|
||||
// Signature must be verified
|
||||
requireSignature: true,
|
||||
|
||||
// Must pass local evaluation OR have trusted quality proof
|
||||
requireQualityGate: true,
|
||||
|
||||
// Minimum evaluation score to pass gate (0-1)
|
||||
minEvaluationScore: 0.7,
|
||||
|
||||
// Maximum adapter size relative to base model
|
||||
maxAdapterSizeRatio: 0.1, // 10% of base model
|
||||
|
||||
// Trusted quality proof publishers
|
||||
trustedQualityProvers: ['ruvector-eval-2024', 'community-eval-2024'],
|
||||
});
|
||||
|
||||
/**
|
||||
* Adapter manifest structure
|
||||
*/
|
||||
export function createAdapterManifest(adapter) {
|
||||
return {
|
||||
schemaVersion: '2.0.0',
|
||||
adapter: {
|
||||
id: adapter.id,
|
||||
name: adapter.name,
|
||||
version: adapter.version,
|
||||
baseModelId: adapter.baseModelId,
|
||||
baseModelVersion: adapter.baseModelVersion,
|
||||
rank: adapter.rank,
|
||||
alpha: adapter.alpha,
|
||||
targetModules: adapter.targetModules || ['q_proj', 'v_proj'],
|
||||
},
|
||||
artifacts: [{
|
||||
path: adapter.path,
|
||||
size: adapter.size,
|
||||
sha256: adapter.sha256,
|
||||
format: 'safetensors',
|
||||
}],
|
||||
quality: {
|
||||
evaluationScore: adapter.evaluationScore,
|
||||
evaluationDataset: adapter.evaluationDataset,
|
||||
evaluationProof: adapter.evaluationProof,
|
||||
domain: adapter.domain,
|
||||
capabilities: adapter.capabilities,
|
||||
},
|
||||
lineage: adapter.lineage || null,
|
||||
provenance: {
|
||||
creator: adapter.creator,
|
||||
createdAt: adapter.createdAt || new Date().toISOString(),
|
||||
trainedOn: adapter.trainedOn,
|
||||
trainingConfig: adapter.trainingConfig,
|
||||
},
|
||||
integrity: {
|
||||
manifestHash: null, // Computed
|
||||
signatures: [],
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// QUARANTINE SYSTEM
|
||||
// ============================================================================
|
||||
|
||||
/**
|
||||
* Quarantine states for adapters
|
||||
*/
|
||||
export const QuarantineState = Object.freeze({
|
||||
PENDING: 'pending',
|
||||
EVALUATING: 'evaluating',
|
||||
PASSED: 'passed',
|
||||
FAILED: 'failed',
|
||||
TRUSTED: 'trusted', // Has trusted quality proof
|
||||
});
|
||||
|
||||
/**
|
||||
* Quarantine manager for adapter verification
|
||||
*/
|
||||
export class AdapterQuarantine {
|
||||
constructor(options = {}) {
|
||||
this.trustRoot = options.trustRoot || new TrustRoot();
|
||||
this.requirements = { ...ADAPTER_REQUIREMENTS, ...options.requirements };
|
||||
|
||||
// Quarantined adapters awaiting evaluation
|
||||
this.quarantine = new Map();
|
||||
|
||||
// Approved adapters
|
||||
this.approved = new Map();
|
||||
|
||||
// Failed adapters (blocked)
|
||||
this.blocked = new Map();
|
||||
|
||||
// Evaluation test sets by domain
|
||||
this.testSets = new Map();
|
||||
}
|
||||
|
||||
/**
|
||||
* Register a test set for a domain
|
||||
*/
|
||||
registerTestSet(domain, testCases) {
|
||||
this.testSets.set(domain, testCases);
|
||||
}
|
||||
|
||||
/**
|
||||
* Quarantine an adapter for evaluation
|
||||
*/
|
||||
async quarantineAdapter(manifest, adapterData) {
|
||||
const adapterId = manifest.adapter.id;
|
||||
|
||||
// 1. Verify checksum
|
||||
const actualHash = createHash('sha256')
|
||||
.update(Buffer.from(adapterData))
|
||||
.digest('hex');
|
||||
|
||||
if (actualHash !== manifest.artifacts[0].sha256) {
|
||||
const failure = {
|
||||
adapterId,
|
||||
reason: 'checksum_mismatch',
|
||||
expected: manifest.artifacts[0].sha256,
|
||||
actual: actualHash,
|
||||
timestamp: Date.now(),
|
||||
};
|
||||
this.blocked.set(adapterId, failure);
|
||||
return { state: QuarantineState.FAILED, failure };
|
||||
}
|
||||
|
||||
// 2. Verify signature if required
|
||||
if (this.requirements.requireSignature) {
|
||||
const sigResult = this._verifySignatures(manifest);
|
||||
if (!sigResult.valid) {
|
||||
const failure = {
|
||||
adapterId,
|
||||
reason: 'invalid_signature',
|
||||
details: sigResult.errors,
|
||||
timestamp: Date.now(),
|
||||
};
|
||||
this.blocked.set(adapterId, failure);
|
||||
return { state: QuarantineState.FAILED, failure };
|
||||
}
|
||||
}
|
||||
|
||||
// 3. Check for trusted quality proof
|
||||
if (manifest.quality?.evaluationProof) {
|
||||
const proofValid = await this._verifyQualityProof(manifest);
|
||||
if (proofValid) {
|
||||
this.approved.set(adapterId, {
|
||||
manifest,
|
||||
state: QuarantineState.TRUSTED,
|
||||
approvedAt: Date.now(),
|
||||
});
|
||||
return { state: QuarantineState.TRUSTED };
|
||||
}
|
||||
}
|
||||
|
||||
// 4. Add to quarantine for local evaluation
|
||||
this.quarantine.set(adapterId, {
|
||||
manifest,
|
||||
adapterData,
|
||||
state: QuarantineState.PENDING,
|
||||
quarantinedAt: Date.now(),
|
||||
});
|
||||
|
||||
return { state: QuarantineState.PENDING };
|
||||
}
|
||||
|
||||
/**
|
||||
* Evaluate a quarantined adapter locally
|
||||
*/
|
||||
async evaluateAdapter(adapterId, inferenceSession) {
|
||||
const quarantined = this.quarantine.get(adapterId);
|
||||
if (!quarantined) {
|
||||
throw new Error(`Adapter ${adapterId} not in quarantine`);
|
||||
}
|
||||
|
||||
quarantined.state = QuarantineState.EVALUATING;
|
||||
|
||||
const manifest = quarantined.manifest;
|
||||
const domain = manifest.quality?.domain || 'general';
|
||||
|
||||
// Get test set for domain
|
||||
const testSet = this.testSets.get(domain) || this._getDefaultTestSet();
|
||||
|
||||
if (testSet.length === 0) {
|
||||
throw new Error(`No test set available for domain: ${domain}`);
|
||||
}
|
||||
|
||||
// Run evaluation
|
||||
const results = await this._runEvaluation(
|
||||
quarantined.adapterData,
|
||||
testSet,
|
||||
inferenceSession,
|
||||
manifest.adapter.baseModelId
|
||||
);
|
||||
|
||||
// Check if passed
|
||||
const passed = results.score >= this.requirements.minEvaluationScore;
|
||||
|
||||
if (passed) {
|
||||
this.quarantine.delete(adapterId);
|
||||
this.approved.set(adapterId, {
|
||||
manifest,
|
||||
state: QuarantineState.PASSED,
|
||||
evaluationResults: results,
|
||||
approvedAt: Date.now(),
|
||||
});
|
||||
return { state: QuarantineState.PASSED, results };
|
||||
} else {
|
||||
this.quarantine.delete(adapterId);
|
||||
this.blocked.set(adapterId, {
|
||||
adapterId,
|
||||
reason: 'evaluation_failed',
|
||||
score: results.score,
|
||||
required: this.requirements.minEvaluationScore,
|
||||
timestamp: Date.now(),
|
||||
});
|
||||
return { state: QuarantineState.FAILED, results };
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if an adapter can be used
|
||||
*/
|
||||
canUseAdapter(adapterId, baseModelId) {
|
||||
const approved = this.approved.get(adapterId);
|
||||
if (!approved) {
|
||||
return { allowed: false, reason: 'not_approved' };
|
||||
}
|
||||
|
||||
// Verify base model match
|
||||
if (this.requirements.requireExactBaseMatch) {
|
||||
const expectedBase = approved.manifest.adapter.baseModelId;
|
||||
if (expectedBase !== baseModelId) {
|
||||
return {
|
||||
allowed: false,
|
||||
reason: 'base_model_mismatch',
|
||||
expected: expectedBase,
|
||||
actual: baseModelId,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
return { allowed: true, state: approved.state };
|
||||
}
|
||||
|
||||
/**
|
||||
* Get approved adapter data
|
||||
*/
|
||||
getApprovedAdapter(adapterId) {
|
||||
return this.approved.get(adapterId) || null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Verify signatures on adapter manifest
|
||||
*/
|
||||
_verifySignatures(manifest) {
|
||||
if (!manifest.integrity?.signatures?.length) {
|
||||
return { valid: false, errors: ['No signatures present'] };
|
||||
}
|
||||
|
||||
return this.trustRoot.verifySignatureThreshold(
|
||||
manifest.integrity.signatures,
|
||||
1 // At least one valid signature for adapters
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Verify a trusted quality proof
|
||||
*/
|
||||
async _verifyQualityProof(manifest) {
|
||||
const proof = manifest.quality.evaluationProof;
|
||||
if (!proof) return false;
|
||||
|
||||
// Check if prover is trusted
|
||||
if (!this.requirements.trustedQualityProvers.includes(proof.proverId)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
// Verify proof signature
|
||||
const proofPayload = {
|
||||
adapterId: manifest.adapter.id,
|
||||
evaluationScore: manifest.quality.evaluationScore,
|
||||
evaluationDataset: manifest.quality.evaluationDataset,
|
||||
timestamp: proof.timestamp,
|
||||
};
|
||||
|
||||
// In production, verify actual signature here
|
||||
return proof.signature && proof.proverId;
|
||||
}
|
||||
|
||||
/**
|
||||
* Run local evaluation on adapter
|
||||
*/
|
||||
async _runEvaluation(adapterData, testSet, inferenceSession, baseModelId) {
|
||||
const results = {
|
||||
total: testSet.length,
|
||||
passed: 0,
|
||||
failed: 0,
|
||||
errors: 0,
|
||||
details: [],
|
||||
};
|
||||
|
||||
for (const testCase of testSet) {
|
||||
try {
|
||||
// Apply adapter temporarily
|
||||
await inferenceSession.loadAdapter(adapterData, { temporary: true });
|
||||
|
||||
// Run inference
|
||||
const output = await inferenceSession.generate(testCase.input, {
|
||||
maxTokens: testCase.maxTokens || 64,
|
||||
});
|
||||
|
||||
// Check against expected
|
||||
const passed = this._checkOutput(output, testCase.expected, testCase.criteria);
|
||||
|
||||
results.details.push({
|
||||
input: testCase.input.slice(0, 50),
|
||||
passed,
|
||||
});
|
||||
|
||||
if (passed) {
|
||||
results.passed++;
|
||||
} else {
|
||||
results.failed++;
|
||||
}
|
||||
|
||||
// Unload temporary adapter
|
||||
await inferenceSession.unloadAdapter();
|
||||
} catch (error) {
|
||||
results.errors++;
|
||||
results.details.push({
|
||||
input: testCase.input.slice(0, 50),
|
||||
error: error.message,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
results.score = results.passed / results.total;
|
||||
return results;
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if output matches expected criteria
|
||||
*/
|
||||
_checkOutput(output, expected, criteria = 'contains') {
|
||||
const outputLower = output.toLowerCase();
|
||||
const expectedLower = expected.toLowerCase();
|
||||
|
||||
switch (criteria) {
|
||||
case 'exact':
|
||||
return output.trim() === expected.trim();
|
||||
case 'contains':
|
||||
return outputLower.includes(expectedLower);
|
||||
case 'startsWith':
|
||||
return outputLower.startsWith(expectedLower);
|
||||
case 'regex':
|
||||
return new RegExp(expected).test(output);
|
||||
default:
|
||||
return outputLower.includes(expectedLower);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get default test set for unknown domains
|
||||
*/
|
||||
_getDefaultTestSet() {
|
||||
return [
|
||||
{
|
||||
input: 'Hello, how are you?',
|
||||
expected: 'hello',
|
||||
criteria: 'contains',
|
||||
},
|
||||
{
|
||||
input: 'What is 2 + 2?',
|
||||
expected: '4',
|
||||
criteria: 'contains',
|
||||
},
|
||||
{
|
||||
input: 'Translate to French: hello',
|
||||
expected: 'bonjour',
|
||||
criteria: 'contains',
|
||||
},
|
||||
];
|
||||
}
|
||||
|
||||
/**
|
||||
* Export quarantine state
|
||||
*/
|
||||
export() {
|
||||
return {
|
||||
quarantine: Array.from(this.quarantine.entries()),
|
||||
approved: Array.from(this.approved.entries()),
|
||||
blocked: Array.from(this.blocked.entries()),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Import quarantine state
|
||||
*/
|
||||
import(data) {
|
||||
if (data.quarantine) {
|
||||
this.quarantine = new Map(data.quarantine);
|
||||
}
|
||||
if (data.approved) {
|
||||
this.approved = new Map(data.approved);
|
||||
}
|
||||
if (data.blocked) {
|
||||
this.blocked = new Map(data.blocked);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// MERGE LINEAGE TRACKING
|
||||
// ============================================================================
|
||||
|
||||
/**
|
||||
* Lineage entry for merged adapters
|
||||
*/
|
||||
export function createMergeLineage(options) {
|
||||
return {
|
||||
parentAdapterIds: options.parentIds,
|
||||
mergeMethod: options.method, // 'ties', 'dare', 'task_arithmetic', 'linear'
|
||||
mergeParameters: options.parameters, // Method-specific params
|
||||
mergeSeed: options.seed || Math.floor(Math.random() * 2 ** 32),
|
||||
evaluationMetrics: options.metrics || {},
|
||||
mergerIdentity: options.mergerId,
|
||||
mergeTimestamp: new Date().toISOString(),
|
||||
signature: null, // To be filled after signing
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Lineage tracker for adapter merges
|
||||
*/
|
||||
export class AdapterLineage {
|
||||
constructor(options = {}) {
|
||||
this.trustRoot = options.trustRoot || new TrustRoot();
|
||||
|
||||
// DAG of adapter lineage
|
||||
this.lineageGraph = new Map();
|
||||
|
||||
// Root adapters (no parents)
|
||||
this.roots = new Set();
|
||||
}
|
||||
|
||||
/**
|
||||
* Register a new adapter in lineage
|
||||
*/
|
||||
registerAdapter(adapterId, manifest) {
|
||||
const lineage = manifest.lineage;
|
||||
|
||||
const node = {
|
||||
adapterId,
|
||||
version: manifest.adapter.version,
|
||||
baseModelId: manifest.adapter.baseModelId,
|
||||
parents: lineage?.parentAdapterIds || [],
|
||||
children: [],
|
||||
lineage,
|
||||
registeredAt: Date.now(),
|
||||
};
|
||||
|
||||
this.lineageGraph.set(adapterId, node);
|
||||
|
||||
// Update parent-child relationships
|
||||
if (node.parents.length === 0) {
|
||||
this.roots.add(adapterId);
|
||||
} else {
|
||||
for (const parentId of node.parents) {
|
||||
const parent = this.lineageGraph.get(parentId);
|
||||
if (parent) {
|
||||
parent.children.push(adapterId);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return node;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get full ancestry path for an adapter
|
||||
*/
|
||||
getAncestry(adapterId) {
|
||||
const ancestry = [];
|
||||
const visited = new Set();
|
||||
const queue = [adapterId];
|
||||
|
||||
while (queue.length > 0) {
|
||||
const current = queue.shift();
|
||||
if (visited.has(current)) continue;
|
||||
visited.add(current);
|
||||
|
||||
const node = this.lineageGraph.get(current);
|
||||
if (node) {
|
||||
ancestry.push({
|
||||
adapterId: current,
|
||||
version: node.version,
|
||||
baseModelId: node.baseModelId,
|
||||
mergeMethod: node.lineage?.mergeMethod,
|
||||
});
|
||||
|
||||
for (const parentId of node.parents) {
|
||||
queue.push(parentId);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return ancestry;
|
||||
}
|
||||
|
||||
/**
|
||||
* Verify lineage integrity
|
||||
*/
|
||||
verifyLineage(adapterId) {
|
||||
const node = this.lineageGraph.get(adapterId);
|
||||
if (!node) {
|
||||
return { valid: false, error: 'Adapter not found' };
|
||||
}
|
||||
|
||||
const errors = [];
|
||||
|
||||
// Check all parents exist
|
||||
for (const parentId of node.parents) {
|
||||
if (!this.lineageGraph.has(parentId)) {
|
||||
errors.push(`Missing parent: ${parentId}`);
|
||||
}
|
||||
}
|
||||
|
||||
// Verify lineage signature if present
|
||||
if (node.lineage?.signature) {
|
||||
// In production, verify actual signature
|
||||
const sigValid = true; // Placeholder
|
||||
if (!sigValid) {
|
||||
errors.push('Invalid lineage signature');
|
||||
}
|
||||
}
|
||||
|
||||
// Check for circular references
|
||||
const hasCircle = this._detectCircle(adapterId, new Set());
|
||||
if (hasCircle) {
|
||||
errors.push('Circular lineage detected');
|
||||
}
|
||||
|
||||
return {
|
||||
valid: errors.length === 0,
|
||||
errors,
|
||||
ancestry: this.getAncestry(adapterId),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Detect circular references in lineage
|
||||
*/
|
||||
_detectCircle(adapterId, visited) {
|
||||
if (visited.has(adapterId)) return true;
|
||||
visited.add(adapterId);
|
||||
|
||||
const node = this.lineageGraph.get(adapterId);
|
||||
if (!node) return false;
|
||||
|
||||
for (const parentId of node.parents) {
|
||||
if (this._detectCircle(parentId, new Set(visited))) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get descendants of an adapter
|
||||
*/
|
||||
getDescendants(adapterId) {
|
||||
const descendants = [];
|
||||
const queue = [adapterId];
|
||||
const visited = new Set();
|
||||
|
||||
while (queue.length > 0) {
|
||||
const current = queue.shift();
|
||||
if (visited.has(current)) continue;
|
||||
visited.add(current);
|
||||
|
||||
const node = this.lineageGraph.get(current);
|
||||
if (node) {
|
||||
for (const childId of node.children) {
|
||||
descendants.push(childId);
|
||||
queue.push(childId);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return descendants;
|
||||
}
|
||||
|
||||
/**
|
||||
* Compute reproducibility hash for a merge
|
||||
*/
|
||||
computeReproducibilityHash(lineage) {
|
||||
const payload = {
|
||||
parents: lineage.parentAdapterIds.sort(),
|
||||
method: lineage.mergeMethod,
|
||||
parameters: lineage.mergeParameters,
|
||||
seed: lineage.mergeSeed,
|
||||
};
|
||||
return hashCanonical(payload);
|
||||
}
|
||||
|
||||
/**
|
||||
* Export lineage graph
|
||||
*/
|
||||
export() {
|
||||
return {
|
||||
nodes: Array.from(this.lineageGraph.entries()),
|
||||
roots: Array.from(this.roots),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Import lineage graph
|
||||
*/
|
||||
import(data) {
|
||||
if (data.nodes) {
|
||||
this.lineageGraph = new Map(data.nodes);
|
||||
}
|
||||
if (data.roots) {
|
||||
this.roots = new Set(data.roots);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// ADAPTER POOL WITH SECURITY
|
||||
// ============================================================================
|
||||
|
||||
/**
|
||||
* Secure adapter pool with quarantine integration
|
||||
*/
|
||||
export class SecureAdapterPool {
|
||||
constructor(options = {}) {
|
||||
this.maxSlots = options.maxSlots || 16;
|
||||
this.quarantine = new AdapterQuarantine(options);
|
||||
this.lineage = new AdapterLineage(options);
|
||||
|
||||
// Active adapters (LRU)
|
||||
this.activeAdapters = new Map();
|
||||
this.accessOrder = [];
|
||||
}
|
||||
|
||||
/**
|
||||
* Add adapter with full security checks
|
||||
*/
|
||||
async addAdapter(manifest, adapterData, inferenceSession = null) {
|
||||
const adapterId = manifest.adapter.id;
|
||||
|
||||
// 1. Quarantine and verify
|
||||
const quarantineResult = await this.quarantine.quarantineAdapter(manifest, adapterData);
|
||||
|
||||
if (quarantineResult.state === QuarantineState.FAILED) {
|
||||
throw new Error(`Adapter blocked: ${quarantineResult.failure.reason}`);
|
||||
}
|
||||
|
||||
// 2. If not trusted, run local evaluation
|
||||
if (quarantineResult.state === QuarantineState.PENDING) {
|
||||
if (!inferenceSession) {
|
||||
throw new Error('Inference session required for local evaluation');
|
||||
}
|
||||
|
||||
const evalResult = await this.quarantine.evaluateAdapter(adapterId, inferenceSession);
|
||||
if (evalResult.state === QuarantineState.FAILED) {
|
||||
throw new Error(`Adapter failed evaluation: score ${evalResult.results.score}`);
|
||||
}
|
||||
}
|
||||
|
||||
// 3. Register in lineage
|
||||
this.lineage.registerAdapter(adapterId, manifest);
|
||||
|
||||
// 4. Add to active pool
|
||||
await this._addToPool(adapterId, adapterData, manifest);
|
||||
|
||||
return { adapterId, state: 'active' };
|
||||
}
|
||||
|
||||
/**
|
||||
* Get an adapter if allowed
|
||||
*/
|
||||
getAdapter(adapterId, baseModelId) {
|
||||
// Check if can use
|
||||
const check = this.quarantine.canUseAdapter(adapterId, baseModelId);
|
||||
if (!check.allowed) {
|
||||
return { allowed: false, reason: check.reason };
|
||||
}
|
||||
|
||||
// Get from pool
|
||||
const adapter = this.activeAdapters.get(adapterId);
|
||||
if (!adapter) {
|
||||
return { allowed: false, reason: 'not_in_pool' };
|
||||
}
|
||||
|
||||
// Update access order
|
||||
this._updateAccessOrder(adapterId);
|
||||
|
||||
return { allowed: true, adapter };
|
||||
}
|
||||
|
||||
/**
|
||||
* Add to pool with LRU eviction
|
||||
*/
|
||||
async _addToPool(adapterId, adapterData, manifest) {
|
||||
// Evict if at capacity
|
||||
while (this.activeAdapters.size >= this.maxSlots) {
|
||||
const evictId = this.accessOrder.shift();
|
||||
this.activeAdapters.delete(evictId);
|
||||
}
|
||||
|
||||
this.activeAdapters.set(adapterId, {
|
||||
data: adapterData,
|
||||
manifest,
|
||||
loadedAt: Date.now(),
|
||||
});
|
||||
|
||||
this._updateAccessOrder(adapterId);
|
||||
}
|
||||
|
||||
/**
|
||||
* Update LRU access order
|
||||
*/
|
||||
_updateAccessOrder(adapterId) {
|
||||
const index = this.accessOrder.indexOf(adapterId);
|
||||
if (index > -1) {
|
||||
this.accessOrder.splice(index, 1);
|
||||
}
|
||||
this.accessOrder.push(adapterId);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get pool statistics
|
||||
*/
|
||||
getStats() {
|
||||
return {
|
||||
activeCount: this.activeAdapters.size,
|
||||
maxSlots: this.maxSlots,
|
||||
quarantinedCount: this.quarantine.quarantine.size,
|
||||
approvedCount: this.quarantine.approved.size,
|
||||
blockedCount: this.quarantine.blocked.size,
|
||||
lineageNodes: this.lineage.lineageGraph.size,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// EXPORTS
|
||||
// ============================================================================
|
||||
|
||||
export default {
|
||||
ADAPTER_REQUIREMENTS,
|
||||
QuarantineState,
|
||||
createAdapterManifest,
|
||||
AdapterQuarantine,
|
||||
createMergeLineage,
|
||||
AdapterLineage,
|
||||
SecureAdapterPool,
|
||||
};
|
||||
688
vendor/ruvector/examples/edge-net/pkg/models/benchmark.js
vendored
Normal file
688
vendor/ruvector/examples/edge-net/pkg/models/benchmark.js
vendored
Normal file
@@ -0,0 +1,688 @@
|
||||
/**
|
||||
* @ruvector/edge-net Benchmark Utilities
|
||||
*
|
||||
* Comprehensive benchmarking for model optimization
|
||||
*
|
||||
* @module @ruvector/edge-net/models/benchmark
|
||||
*/
|
||||
|
||||
import { EventEmitter } from 'events';
|
||||
import { ModelOptimizer, TARGET_MODELS, QUANTIZATION_CONFIGS } from './model-optimizer.js';
|
||||
|
||||
// ============================================
|
||||
// BENCHMARK CONFIGURATION
|
||||
// ============================================
|
||||
|
||||
/**
|
||||
* Benchmark profiles for different scenarios
|
||||
*/
|
||||
export const BENCHMARK_PROFILES = {
|
||||
'quick': {
|
||||
iterations: 50,
|
||||
warmupIterations: 5,
|
||||
inputSizes: [[1, 128]],
|
||||
quantMethods: ['int8'],
|
||||
},
|
||||
'standard': {
|
||||
iterations: 100,
|
||||
warmupIterations: 10,
|
||||
inputSizes: [[1, 128], [1, 512], [4, 256]],
|
||||
quantMethods: ['int8', 'int4', 'fp16'],
|
||||
},
|
||||
'comprehensive': {
|
||||
iterations: 500,
|
||||
warmupIterations: 50,
|
||||
inputSizes: [[1, 64], [1, 128], [1, 256], [1, 512], [1, 1024], [4, 256], [8, 128]],
|
||||
quantMethods: ['int8', 'int4', 'fp16', 'int8-fp16-mixed'],
|
||||
},
|
||||
'edge-device': {
|
||||
iterations: 100,
|
||||
warmupIterations: 10,
|
||||
inputSizes: [[1, 128], [1, 256]],
|
||||
quantMethods: ['int4'],
|
||||
memoryLimit: 512, // MB
|
||||
},
|
||||
'accuracy-focus': {
|
||||
iterations: 200,
|
||||
warmupIterations: 20,
|
||||
inputSizes: [[1, 512]],
|
||||
quantMethods: ['fp16', 'int8'],
|
||||
measureAccuracy: true,
|
||||
},
|
||||
};
|
||||
|
||||
// ============================================
|
||||
// ACCURACY MEASUREMENT
|
||||
// ============================================
|
||||
|
||||
/**
|
||||
* Accuracy metrics for quantized models
|
||||
*/
|
||||
export class AccuracyMeter {
|
||||
constructor() {
|
||||
this.predictions = [];
|
||||
this.groundTruth = [];
|
||||
this.originalOutputs = [];
|
||||
this.quantizedOutputs = [];
|
||||
}
|
||||
|
||||
/**
|
||||
* Add prediction pair for accuracy measurement
|
||||
*/
|
||||
addPrediction(original, quantized, groundTruth = null) {
|
||||
this.originalOutputs.push(original);
|
||||
this.quantizedOutputs.push(quantized);
|
||||
if (groundTruth !== null) {
|
||||
this.groundTruth.push(groundTruth);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Compute Mean Squared Error
|
||||
*/
|
||||
computeMSE() {
|
||||
if (this.originalOutputs.length === 0) return 0;
|
||||
|
||||
let totalMSE = 0;
|
||||
let count = 0;
|
||||
|
||||
for (let i = 0; i < this.originalOutputs.length; i++) {
|
||||
const orig = this.originalOutputs[i];
|
||||
const quant = this.quantizedOutputs[i];
|
||||
|
||||
let mse = 0;
|
||||
const len = Math.min(orig.length, quant.length);
|
||||
for (let j = 0; j < len; j++) {
|
||||
const diff = orig[j] - quant[j];
|
||||
mse += diff * diff;
|
||||
}
|
||||
totalMSE += mse / len;
|
||||
count++;
|
||||
}
|
||||
|
||||
return totalMSE / count;
|
||||
}
|
||||
|
||||
/**
|
||||
* Compute cosine similarity between original and quantized
|
||||
*/
|
||||
computeCosineSimilarity() {
|
||||
if (this.originalOutputs.length === 0) return 1.0;
|
||||
|
||||
let totalSim = 0;
|
||||
|
||||
for (let i = 0; i < this.originalOutputs.length; i++) {
|
||||
const orig = this.originalOutputs[i];
|
||||
const quant = this.quantizedOutputs[i];
|
||||
|
||||
let dot = 0, normA = 0, normB = 0;
|
||||
const len = Math.min(orig.length, quant.length);
|
||||
|
||||
for (let j = 0; j < len; j++) {
|
||||
dot += orig[j] * quant[j];
|
||||
normA += orig[j] * orig[j];
|
||||
normB += quant[j] * quant[j];
|
||||
}
|
||||
|
||||
totalSim += dot / (Math.sqrt(normA) * Math.sqrt(normB) + 1e-8);
|
||||
}
|
||||
|
||||
return totalSim / this.originalOutputs.length;
|
||||
}
|
||||
|
||||
/**
|
||||
* Compute max absolute error
|
||||
*/
|
||||
computeMaxError() {
|
||||
let maxError = 0;
|
||||
|
||||
for (let i = 0; i < this.originalOutputs.length; i++) {
|
||||
const orig = this.originalOutputs[i];
|
||||
const quant = this.quantizedOutputs[i];
|
||||
const len = Math.min(orig.length, quant.length);
|
||||
|
||||
for (let j = 0; j < len; j++) {
|
||||
maxError = Math.max(maxError, Math.abs(orig[j] - quant[j]));
|
||||
}
|
||||
}
|
||||
|
||||
return maxError;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get comprehensive accuracy metrics
|
||||
*/
|
||||
getMetrics() {
|
||||
const mse = this.computeMSE();
|
||||
|
||||
return {
|
||||
mse,
|
||||
rmse: Math.sqrt(mse),
|
||||
cosineSimilarity: this.computeCosineSimilarity(),
|
||||
maxError: this.computeMaxError(),
|
||||
samples: this.originalOutputs.length,
|
||||
accuracyRetained: this.computeCosineSimilarity() * 100,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Reset meter
|
||||
*/
|
||||
reset() {
|
||||
this.predictions = [];
|
||||
this.groundTruth = [];
|
||||
this.originalOutputs = [];
|
||||
this.quantizedOutputs = [];
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================
|
||||
// LATENCY PROFILER
|
||||
// ============================================
|
||||
|
||||
/**
|
||||
* Detailed latency profiling
|
||||
*/
|
||||
export class LatencyProfiler {
|
||||
constructor() {
|
||||
this.measurements = new Map();
|
||||
}
|
||||
|
||||
/**
|
||||
* Start timing a section
|
||||
*/
|
||||
start(label) {
|
||||
if (!this.measurements.has(label)) {
|
||||
this.measurements.set(label, {
|
||||
samples: [],
|
||||
running: null,
|
||||
});
|
||||
}
|
||||
this.measurements.get(label).running = performance.now();
|
||||
}
|
||||
|
||||
/**
|
||||
* End timing a section
|
||||
*/
|
||||
end(label) {
|
||||
const entry = this.measurements.get(label);
|
||||
if (entry && entry.running !== null) {
|
||||
const duration = performance.now() - entry.running;
|
||||
entry.samples.push(duration);
|
||||
entry.running = null;
|
||||
return duration;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get statistics for a label
|
||||
*/
|
||||
getStats(label) {
|
||||
const entry = this.measurements.get(label);
|
||||
if (!entry || entry.samples.length === 0) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const samples = [...entry.samples].sort((a, b) => a - b);
|
||||
const sum = samples.reduce((a, b) => a + b, 0);
|
||||
|
||||
return {
|
||||
label,
|
||||
count: samples.length,
|
||||
mean: sum / samples.length,
|
||||
median: samples[Math.floor(samples.length / 2)],
|
||||
min: samples[0],
|
||||
max: samples[samples.length - 1],
|
||||
p95: samples[Math.floor(samples.length * 0.95)],
|
||||
p99: samples[Math.floor(samples.length * 0.99)],
|
||||
std: Math.sqrt(samples.reduce((acc, v) => acc + Math.pow(v - sum / samples.length, 2), 0) / samples.length),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all statistics
|
||||
*/
|
||||
getAllStats() {
|
||||
const stats = {};
|
||||
for (const label of this.measurements.keys()) {
|
||||
stats[label] = this.getStats(label);
|
||||
}
|
||||
return stats;
|
||||
}
|
||||
|
||||
/**
|
||||
* Reset profiler
|
||||
*/
|
||||
reset() {
|
||||
this.measurements.clear();
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================
|
||||
// MEMORY PROFILER
|
||||
// ============================================
|
||||
|
||||
/**
|
||||
* Memory usage profiler
|
||||
*/
|
||||
export class MemoryProfiler {
|
||||
constructor() {
|
||||
this.snapshots = [];
|
||||
this.peakMemory = 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* Take memory snapshot
|
||||
*/
|
||||
snapshot(label = 'snapshot') {
|
||||
const memUsage = this.getMemoryUsage();
|
||||
const snapshot = {
|
||||
label,
|
||||
timestamp: Date.now(),
|
||||
...memUsage,
|
||||
};
|
||||
|
||||
this.snapshots.push(snapshot);
|
||||
this.peakMemory = Math.max(this.peakMemory, memUsage.heapUsed);
|
||||
|
||||
return snapshot;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get current memory usage
|
||||
*/
|
||||
getMemoryUsage() {
|
||||
if (typeof process !== 'undefined' && process.memoryUsage) {
|
||||
const usage = process.memoryUsage();
|
||||
return {
|
||||
heapUsed: usage.heapUsed / (1024 * 1024),
|
||||
heapTotal: usage.heapTotal / (1024 * 1024),
|
||||
external: usage.external / (1024 * 1024),
|
||||
rss: usage.rss / (1024 * 1024),
|
||||
};
|
||||
}
|
||||
|
||||
// Browser fallback
|
||||
if (typeof performance !== 'undefined' && performance.memory) {
|
||||
return {
|
||||
heapUsed: performance.memory.usedJSHeapSize / (1024 * 1024),
|
||||
heapTotal: performance.memory.totalJSHeapSize / (1024 * 1024),
|
||||
external: 0,
|
||||
rss: 0,
|
||||
};
|
||||
}
|
||||
|
||||
return { heapUsed: 0, heapTotal: 0, external: 0, rss: 0 };
|
||||
}
|
||||
|
||||
/**
|
||||
* Get memory delta between two snapshots
|
||||
*/
|
||||
getDelta(startLabel, endLabel) {
|
||||
const start = this.snapshots.find(s => s.label === startLabel);
|
||||
const end = this.snapshots.find(s => s.label === endLabel);
|
||||
|
||||
if (!start || !end) return null;
|
||||
|
||||
return {
|
||||
heapDelta: end.heapUsed - start.heapUsed,
|
||||
timeDelta: end.timestamp - start.timestamp,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Get profiler summary
|
||||
*/
|
||||
getSummary() {
|
||||
return {
|
||||
snapshots: this.snapshots.length,
|
||||
peakMemoryMB: this.peakMemory,
|
||||
currentMemoryMB: this.getMemoryUsage().heapUsed,
|
||||
history: this.snapshots,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Reset profiler
|
||||
*/
|
||||
reset() {
|
||||
this.snapshots = [];
|
||||
this.peakMemory = 0;
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================
|
||||
// COMPREHENSIVE BENCHMARK RUNNER
|
||||
// ============================================
|
||||
|
||||
/**
|
||||
* ComprehensiveBenchmark - Full benchmark suite for model optimization
|
||||
*/
|
||||
export class ComprehensiveBenchmark extends EventEmitter {
|
||||
constructor(options = {}) {
|
||||
super();
|
||||
this.optimizer = options.optimizer || new ModelOptimizer();
|
||||
this.latencyProfiler = new LatencyProfiler();
|
||||
this.memoryProfiler = new MemoryProfiler();
|
||||
this.accuracyMeter = new AccuracyMeter();
|
||||
this.results = [];
|
||||
}
|
||||
|
||||
/**
|
||||
* Run benchmark suite on a model
|
||||
*/
|
||||
async runSuite(model, profile = 'standard') {
|
||||
const profileConfig = BENCHMARK_PROFILES[profile] || BENCHMARK_PROFILES.standard;
|
||||
const modelConfig = TARGET_MODELS[model];
|
||||
|
||||
if (!modelConfig) {
|
||||
throw new Error(`Unknown model: ${model}`);
|
||||
}
|
||||
|
||||
this.emit('suite:start', { model, profile });
|
||||
|
||||
const suiteResults = {
|
||||
model,
|
||||
profile,
|
||||
modelConfig,
|
||||
timestamp: new Date().toISOString(),
|
||||
benchmarks: [],
|
||||
};
|
||||
|
||||
// Memory baseline
|
||||
this.memoryProfiler.snapshot('baseline');
|
||||
|
||||
// Benchmark each quantization method
|
||||
for (const method of profileConfig.quantMethods) {
|
||||
const methodResult = await this.benchmarkQuantization(
|
||||
model,
|
||||
method,
|
||||
profileConfig
|
||||
);
|
||||
suiteResults.benchmarks.push(methodResult);
|
||||
}
|
||||
|
||||
// Memory after benchmarks
|
||||
this.memoryProfiler.snapshot('after-benchmarks');
|
||||
|
||||
// Add memory profile
|
||||
suiteResults.memory = this.memoryProfiler.getSummary();
|
||||
|
||||
// Add summary
|
||||
suiteResults.summary = this.generateSummary(suiteResults);
|
||||
|
||||
this.results.push(suiteResults);
|
||||
this.emit('suite:complete', suiteResults);
|
||||
|
||||
return suiteResults;
|
||||
}
|
||||
|
||||
/**
|
||||
* Benchmark a specific quantization method
|
||||
*/
|
||||
async benchmarkQuantization(model, method, config) {
|
||||
this.emit('benchmark:start', { model, method });
|
||||
|
||||
const quantConfig = QUANTIZATION_CONFIGS[method];
|
||||
const modelConfig = TARGET_MODELS[model];
|
||||
|
||||
// Quantize model
|
||||
this.latencyProfiler.start('quantization');
|
||||
const quantResult = await this.optimizer.quantize(model, method);
|
||||
this.latencyProfiler.end('quantization');
|
||||
|
||||
// Simulate inference benchmarks for each input size
|
||||
const inferenceBenchmarks = [];
|
||||
|
||||
for (const inputSize of config.inputSizes) {
|
||||
const batchSize = inputSize[0];
|
||||
const seqLen = inputSize[1];
|
||||
|
||||
this.latencyProfiler.start(`inference-${batchSize}x${seqLen}`);
|
||||
|
||||
// Warmup
|
||||
for (let i = 0; i < config.warmupIterations; i++) {
|
||||
await this.simulateInference(modelConfig, batchSize, seqLen, method);
|
||||
}
|
||||
|
||||
// Measure
|
||||
const times = [];
|
||||
for (let i = 0; i < config.iterations; i++) {
|
||||
const start = performance.now();
|
||||
await this.simulateInference(modelConfig, batchSize, seqLen, method);
|
||||
times.push(performance.now() - start);
|
||||
}
|
||||
|
||||
this.latencyProfiler.end(`inference-${batchSize}x${seqLen}`);
|
||||
|
||||
times.sort((a, b) => a - b);
|
||||
|
||||
inferenceBenchmarks.push({
|
||||
inputSize: `${batchSize}x${seqLen}`,
|
||||
iterations: config.iterations,
|
||||
meanMs: times.reduce((a, b) => a + b) / times.length,
|
||||
medianMs: times[Math.floor(times.length / 2)],
|
||||
p95Ms: times[Math.floor(times.length * 0.95)],
|
||||
minMs: times[0],
|
||||
maxMs: times[times.length - 1],
|
||||
tokensPerSecond: (seqLen * batchSize * 1000) / (times.reduce((a, b) => a + b) / times.length),
|
||||
});
|
||||
}
|
||||
|
||||
// Measure accuracy if requested
|
||||
let accuracyMetrics = null;
|
||||
if (config.measureAccuracy) {
|
||||
// Generate test outputs
|
||||
for (let i = 0; i < 100; i++) {
|
||||
const original = new Float32Array(modelConfig.hiddenSize).map(() => Math.random());
|
||||
const quantized = this.simulateQuantizedOutput(original, method);
|
||||
this.accuracyMeter.addPrediction(Array.from(original), Array.from(quantized));
|
||||
}
|
||||
accuracyMetrics = this.accuracyMeter.getMetrics();
|
||||
this.accuracyMeter.reset();
|
||||
}
|
||||
|
||||
const result = {
|
||||
method,
|
||||
quantization: quantResult,
|
||||
inference: inferenceBenchmarks,
|
||||
accuracy: accuracyMetrics,
|
||||
latencyProfile: this.latencyProfiler.getAllStats(),
|
||||
compression: {
|
||||
original: modelConfig.originalSize,
|
||||
quantized: modelConfig.originalSize / quantConfig.compression,
|
||||
ratio: quantConfig.compression,
|
||||
},
|
||||
recommendation: this.getRecommendation(model, method, inferenceBenchmarks),
|
||||
};
|
||||
|
||||
this.emit('benchmark:complete', result);
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Simulate model inference
|
||||
*/
|
||||
async simulateInference(config, batchSize, seqLen, method) {
|
||||
// Base latency depends on model size and batch
|
||||
const quantConfig = QUANTIZATION_CONFIGS[method];
|
||||
const baseLatency = (config.originalSize / 100) * (batchSize * seqLen / 512);
|
||||
const speedup = quantConfig?.speedup || 1;
|
||||
|
||||
const latency = baseLatency / speedup;
|
||||
await new Promise(resolve => setTimeout(resolve, latency));
|
||||
|
||||
return new Float32Array(config.hiddenSize).map(() => Math.random());
|
||||
}
|
||||
|
||||
/**
|
||||
* Simulate quantized output with added noise
|
||||
*/
|
||||
simulateQuantizedOutput(original, method) {
|
||||
const quantConfig = QUANTIZATION_CONFIGS[method];
|
||||
const noise = quantConfig?.accuracyLoss || 0.01;
|
||||
|
||||
return new Float32Array(original.length).map((_, i) => {
|
||||
return original[i] + (Math.random() - 0.5) * 2 * noise;
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate recommendation based on benchmark results
|
||||
*/
|
||||
getRecommendation(model, method, inferenceBenchmarks) {
|
||||
const modelConfig = TARGET_MODELS[model];
|
||||
const quantConfig = QUANTIZATION_CONFIGS[method];
|
||||
|
||||
const avgLatency = inferenceBenchmarks.reduce((a, b) => a + b.meanMs, 0) / inferenceBenchmarks.length;
|
||||
const targetMet = (modelConfig.originalSize / quantConfig.compression) <= modelConfig.targetSize;
|
||||
|
||||
let score = 0;
|
||||
let reasons = [];
|
||||
|
||||
// Size target met
|
||||
if (targetMet) {
|
||||
score += 30;
|
||||
reasons.push('Meets size target');
|
||||
}
|
||||
|
||||
// Good latency
|
||||
if (avgLatency < 10) {
|
||||
score += 30;
|
||||
reasons.push('Excellent latency (<10ms)');
|
||||
} else if (avgLatency < 50) {
|
||||
score += 20;
|
||||
reasons.push('Good latency (<50ms)');
|
||||
}
|
||||
|
||||
// Low accuracy loss
|
||||
if (quantConfig.accuracyLoss < 0.02) {
|
||||
score += 25;
|
||||
reasons.push('Minimal accuracy loss (<2%)');
|
||||
} else if (quantConfig.accuracyLoss < 0.05) {
|
||||
score += 15;
|
||||
reasons.push('Acceptable accuracy loss (<5%)');
|
||||
}
|
||||
|
||||
// Compression ratio
|
||||
if (quantConfig.compression >= 4) {
|
||||
score += 15;
|
||||
reasons.push('High compression (4x+)');
|
||||
}
|
||||
|
||||
return {
|
||||
score,
|
||||
rating: score >= 80 ? 'Excellent' : score >= 60 ? 'Good' : score >= 40 ? 'Acceptable' : 'Poor',
|
||||
reasons,
|
||||
recommended: score >= 60,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate suite summary
|
||||
*/
|
||||
generateSummary(suiteResults) {
|
||||
const benchmarks = suiteResults.benchmarks;
|
||||
|
||||
// Find best method
|
||||
let bestMethod = null;
|
||||
let bestScore = 0;
|
||||
|
||||
for (const b of benchmarks) {
|
||||
if (b.recommendation.score > bestScore) {
|
||||
bestScore = b.recommendation.score;
|
||||
bestMethod = b.method;
|
||||
}
|
||||
}
|
||||
|
||||
// Calculate averages
|
||||
const avgLatency = benchmarks.reduce((sum, b) => {
|
||||
return sum + b.inference.reduce((s, i) => s + i.meanMs, 0) / b.inference.length;
|
||||
}, 0) / benchmarks.length;
|
||||
|
||||
return {
|
||||
modelKey: suiteResults.model,
|
||||
modelType: suiteResults.modelConfig.type,
|
||||
originalSizeMB: suiteResults.modelConfig.originalSize,
|
||||
targetSizeMB: suiteResults.modelConfig.targetSize,
|
||||
bestMethod,
|
||||
bestScore,
|
||||
avgLatencyMs: avgLatency,
|
||||
methodsEvaluated: benchmarks.length,
|
||||
recommendation: bestMethod ? `Use ${bestMethod} quantization for optimal edge deployment` : 'No suitable method found',
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Run benchmarks on all target models
|
||||
*/
|
||||
async runAllModels(profile = 'standard') {
|
||||
const allResults = [];
|
||||
|
||||
for (const modelKey of Object.keys(TARGET_MODELS)) {
|
||||
try {
|
||||
const result = await this.runSuite(modelKey, profile);
|
||||
allResults.push(result);
|
||||
} catch (error) {
|
||||
allResults.push({
|
||||
model: modelKey,
|
||||
error: error.message,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
timestamp: new Date().toISOString(),
|
||||
profile,
|
||||
results: allResults,
|
||||
summary: this.generateOverallSummary(allResults),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate overall summary for all models
|
||||
*/
|
||||
generateOverallSummary(allResults) {
|
||||
const successful = allResults.filter(r => !r.error);
|
||||
|
||||
return {
|
||||
totalModels: allResults.length,
|
||||
successfulBenchmarks: successful.length,
|
||||
failedBenchmarks: allResults.length - successful.length,
|
||||
recommendations: successful.map(r => ({
|
||||
model: r.model,
|
||||
bestMethod: r.summary?.bestMethod,
|
||||
score: r.summary?.bestScore,
|
||||
})),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Export results to JSON
|
||||
*/
|
||||
exportResults() {
|
||||
return {
|
||||
exported: new Date().toISOString(),
|
||||
results: this.results,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Reset benchmark state
|
||||
*/
|
||||
reset() {
|
||||
this.latencyProfiler.reset();
|
||||
this.memoryProfiler.reset();
|
||||
this.accuracyMeter.reset();
|
||||
this.results = [];
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================
|
||||
// EXPORTS
|
||||
// ============================================
|
||||
|
||||
// BENCHMARK_PROFILES already exported at declaration (line 19)
|
||||
export default ComprehensiveBenchmark;
|
||||
791
vendor/ruvector/examples/edge-net/pkg/models/distribution.js
vendored
Normal file
791
vendor/ruvector/examples/edge-net/pkg/models/distribution.js
vendored
Normal file
@@ -0,0 +1,791 @@
|
||||
/**
|
||||
* @ruvector/edge-net Distribution Manager
|
||||
*
|
||||
* Handles model distribution across multiple sources:
|
||||
* - Google Cloud Storage (GCS)
|
||||
* - IPFS (via web3.storage or nft.storage)
|
||||
* - CDN with fallback support
|
||||
*
|
||||
* Features:
|
||||
* - Integrity verification (SHA256)
|
||||
* - Progress tracking for large files
|
||||
* - Automatic source failover
|
||||
*
|
||||
* @module @ruvector/edge-net/models/distribution
|
||||
*/
|
||||
|
||||
import { EventEmitter } from 'events';
|
||||
import { createHash, randomBytes } from 'crypto';
|
||||
import { promises as fs } from 'fs';
|
||||
import path from 'path';
|
||||
import https from 'https';
|
||||
import http from 'http';
|
||||
import { URL } from 'url';
|
||||
|
||||
// ============================================
|
||||
// CONSTANTS
|
||||
// ============================================
|
||||
|
||||
const DEFAULT_GCS_BUCKET = 'ruvector-models';
|
||||
const DEFAULT_CDN_BASE = 'https://models.ruvector.dev';
|
||||
const DEFAULT_IPFS_GATEWAY = 'https://w3s.link/ipfs';
|
||||
|
||||
const CHUNK_SIZE = 1024 * 1024; // 1MB chunks for streaming
|
||||
const MAX_RETRIES = 3;
|
||||
const RETRY_DELAY_MS = 1000;
|
||||
|
||||
// ============================================
|
||||
// SOURCE TYPES
|
||||
// ============================================
|
||||
|
||||
/**
|
||||
* Source priority order (lower = higher priority)
|
||||
*/
|
||||
export const SOURCE_PRIORITY = {
|
||||
cdn: 1,
|
||||
gcs: 2,
|
||||
ipfs: 3,
|
||||
fallback: 99,
|
||||
};
|
||||
|
||||
/**
|
||||
* Source URL patterns
|
||||
*/
|
||||
export const SOURCE_PATTERNS = {
|
||||
gcs: /^gs:\/\/([^/]+)\/(.+)$/,
|
||||
ipfs: /^ipfs:\/\/(.+)$/,
|
||||
http: /^https?:\/\/.+$/,
|
||||
};
|
||||
|
||||
// ============================================
|
||||
// PROGRESS TRACKER
|
||||
// ============================================
|
||||
|
||||
/**
|
||||
* Progress tracker for file transfers
|
||||
*/
|
||||
export class ProgressTracker extends EventEmitter {
|
||||
constructor(totalBytes = 0) {
|
||||
super();
|
||||
this.totalBytes = totalBytes;
|
||||
this.bytesTransferred = 0;
|
||||
this.startTime = Date.now();
|
||||
this.lastUpdateTime = Date.now();
|
||||
this.lastBytesTransferred = 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* Update progress
|
||||
* @param {number} bytes - Bytes transferred in this chunk
|
||||
*/
|
||||
update(bytes) {
|
||||
this.bytesTransferred += bytes;
|
||||
const now = Date.now();
|
||||
|
||||
// Calculate speed (bytes per second)
|
||||
const timeDelta = (now - this.lastUpdateTime) / 1000;
|
||||
const bytesDelta = this.bytesTransferred - this.lastBytesTransferred;
|
||||
const speed = timeDelta > 0 ? bytesDelta / timeDelta : 0;
|
||||
|
||||
// Calculate ETA
|
||||
const remaining = this.totalBytes - this.bytesTransferred;
|
||||
const eta = speed > 0 ? remaining / speed : 0;
|
||||
|
||||
const progress = {
|
||||
bytesTransferred: this.bytesTransferred,
|
||||
totalBytes: this.totalBytes,
|
||||
percent: this.totalBytes > 0
|
||||
? Math.round((this.bytesTransferred / this.totalBytes) * 100)
|
||||
: 0,
|
||||
speed: Math.round(speed),
|
||||
speedMBps: Math.round(speed / (1024 * 1024) * 100) / 100,
|
||||
eta: Math.round(eta),
|
||||
elapsed: Math.round((now - this.startTime) / 1000),
|
||||
};
|
||||
|
||||
this.lastUpdateTime = now;
|
||||
this.lastBytesTransferred = this.bytesTransferred;
|
||||
|
||||
this.emit('progress', progress);
|
||||
|
||||
if (this.bytesTransferred >= this.totalBytes) {
|
||||
this.emit('complete', progress);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Mark as complete
|
||||
*/
|
||||
complete() {
|
||||
this.bytesTransferred = this.totalBytes;
|
||||
const elapsed = (Date.now() - this.startTime) / 1000;
|
||||
|
||||
this.emit('complete', {
|
||||
bytesTransferred: this.bytesTransferred,
|
||||
totalBytes: this.totalBytes,
|
||||
percent: 100,
|
||||
elapsed: Math.round(elapsed),
|
||||
averageSpeed: Math.round(this.totalBytes / elapsed),
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Mark as failed
|
||||
* @param {Error} error - Failure error
|
||||
*/
|
||||
fail(error) {
|
||||
this.emit('error', {
|
||||
error,
|
||||
bytesTransferred: this.bytesTransferred,
|
||||
totalBytes: this.totalBytes,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================
|
||||
// DISTRIBUTION MANAGER
|
||||
// ============================================
|
||||
|
||||
/**
|
||||
* DistributionManager - Manages model uploads and downloads
|
||||
*/
|
||||
export class DistributionManager extends EventEmitter {
|
||||
/**
|
||||
* Create a new DistributionManager
|
||||
* @param {object} options - Configuration options
|
||||
*/
|
||||
constructor(options = {}) {
|
||||
super();
|
||||
|
||||
this.id = `dist-${randomBytes(6).toString('hex')}`;
|
||||
|
||||
// GCS configuration
|
||||
this.gcsConfig = {
|
||||
bucket: options.gcsBucket || DEFAULT_GCS_BUCKET,
|
||||
projectId: options.gcsProjectId || process.env.GCS_PROJECT_ID,
|
||||
keyFilePath: options.gcsKeyFile || process.env.GOOGLE_APPLICATION_CREDENTIALS,
|
||||
};
|
||||
|
||||
// IPFS configuration
|
||||
this.ipfsConfig = {
|
||||
gateway: options.ipfsGateway || DEFAULT_IPFS_GATEWAY,
|
||||
web3StorageToken: options.web3StorageToken || process.env.WEB3_STORAGE_TOKEN,
|
||||
nftStorageToken: options.nftStorageToken || process.env.NFT_STORAGE_TOKEN,
|
||||
};
|
||||
|
||||
// CDN configuration
|
||||
this.cdnConfig = {
|
||||
baseUrl: options.cdnBaseUrl || DEFAULT_CDN_BASE,
|
||||
fallbackUrls: options.cdnFallbacks || [],
|
||||
};
|
||||
|
||||
// Download cache (in-flight downloads)
|
||||
this.activeDownloads = new Map();
|
||||
|
||||
// Stats
|
||||
this.stats = {
|
||||
uploads: 0,
|
||||
downloads: 0,
|
||||
bytesUploaded: 0,
|
||||
bytesDownloaded: 0,
|
||||
failures: 0,
|
||||
};
|
||||
}
|
||||
|
||||
// ============================================
|
||||
// URL GENERATION
|
||||
// ============================================
|
||||
|
||||
/**
|
||||
* Generate CDN URL for a model
|
||||
* @param {string} modelName - Model name
|
||||
* @param {string} version - Model version
|
||||
* @param {string} filename - Filename
|
||||
* @returns {string}
|
||||
*/
|
||||
getCdnUrl(modelName, version, filename = null) {
|
||||
const file = filename || `${modelName}.onnx`;
|
||||
return `${this.cdnConfig.baseUrl}/${modelName}/${version}/${file}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate GCS URL for a model
|
||||
* @param {string} modelName - Model name
|
||||
* @param {string} version - Model version
|
||||
* @param {string} filename - Filename
|
||||
* @returns {string}
|
||||
*/
|
||||
getGcsUrl(modelName, version, filename = null) {
|
||||
const file = filename || `${modelName}.onnx`;
|
||||
return `gs://${this.gcsConfig.bucket}/${modelName}/${version}/${file}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate IPFS URL from CID
|
||||
* @param {string} cid - IPFS Content ID
|
||||
* @returns {string}
|
||||
*/
|
||||
getIpfsUrl(cid) {
|
||||
return `ipfs://${cid}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate HTTP gateway URL for IPFS
|
||||
* @param {string} cid - IPFS Content ID
|
||||
* @returns {string}
|
||||
*/
|
||||
getIpfsGatewayUrl(cid) {
|
||||
// Handle both ipfs:// URLs and raw CIDs
|
||||
const cleanCid = cid.replace(/^ipfs:\/\//, '');
|
||||
return `${this.ipfsConfig.gateway}/${cleanCid}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate all source URLs for a model
|
||||
* @param {object} sources - Source configuration from metadata
|
||||
* @param {string} modelName - Model name
|
||||
* @param {string} version - Version
|
||||
* @returns {object[]} Sorted list of sources with URLs
|
||||
*/
|
||||
generateSourceUrls(sources, modelName, version) {
|
||||
const urls = [];
|
||||
|
||||
// CDN (highest priority)
|
||||
if (sources.cdn) {
|
||||
urls.push({
|
||||
type: 'cdn',
|
||||
url: sources.cdn,
|
||||
priority: SOURCE_PRIORITY.cdn,
|
||||
});
|
||||
} else {
|
||||
// Auto-generate CDN URL
|
||||
urls.push({
|
||||
type: 'cdn',
|
||||
url: this.getCdnUrl(modelName, version),
|
||||
priority: SOURCE_PRIORITY.cdn,
|
||||
});
|
||||
}
|
||||
|
||||
// GCS
|
||||
if (sources.gcs) {
|
||||
const gcsMatch = sources.gcs.match(SOURCE_PATTERNS.gcs);
|
||||
if (gcsMatch) {
|
||||
// Convert gs:// to HTTPS URL
|
||||
const [, bucket, path] = gcsMatch;
|
||||
urls.push({
|
||||
type: 'gcs',
|
||||
url: `https://storage.googleapis.com/${bucket}/${path}`,
|
||||
originalUrl: sources.gcs,
|
||||
priority: SOURCE_PRIORITY.gcs,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// IPFS
|
||||
if (sources.ipfs) {
|
||||
urls.push({
|
||||
type: 'ipfs',
|
||||
url: this.getIpfsGatewayUrl(sources.ipfs),
|
||||
originalUrl: sources.ipfs,
|
||||
priority: SOURCE_PRIORITY.ipfs,
|
||||
});
|
||||
}
|
||||
|
||||
// Fallback URLs
|
||||
for (const fallback of this.cdnConfig.fallbackUrls) {
|
||||
urls.push({
|
||||
type: 'fallback',
|
||||
url: `${fallback}/${modelName}/${version}/${modelName}.onnx`,
|
||||
priority: SOURCE_PRIORITY.fallback,
|
||||
});
|
||||
}
|
||||
|
||||
// Sort by priority
|
||||
return urls.sort((a, b) => a.priority - b.priority);
|
||||
}
|
||||
|
||||
// ============================================
|
||||
// DOWNLOAD
|
||||
// ============================================
|
||||
|
||||
/**
|
||||
* Download a model from the best available source
|
||||
* @param {object} metadata - Model metadata
|
||||
* @param {object} options - Download options
|
||||
* @returns {Promise<Buffer>}
|
||||
*/
|
||||
async download(metadata, options = {}) {
|
||||
const { name, version, sources, size, hash } = metadata;
|
||||
const key = `${name}@${version}`;
|
||||
|
||||
// Check for in-flight download
|
||||
if (this.activeDownloads.has(key)) {
|
||||
return this.activeDownloads.get(key);
|
||||
}
|
||||
|
||||
const downloadPromise = this._executeDownload(metadata, options);
|
||||
this.activeDownloads.set(key, downloadPromise);
|
||||
|
||||
try {
|
||||
const result = await downloadPromise;
|
||||
return result;
|
||||
} finally {
|
||||
this.activeDownloads.delete(key);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Execute the download with fallback
|
||||
* @private
|
||||
*/
|
||||
async _executeDownload(metadata, options = {}) {
|
||||
const { name, version, sources, size, hash } = metadata;
|
||||
const sourceUrls = this.generateSourceUrls(sources, name, version);
|
||||
|
||||
const progress = new ProgressTracker(size);
|
||||
|
||||
if (options.onProgress) {
|
||||
progress.on('progress', options.onProgress);
|
||||
}
|
||||
|
||||
let lastError = null;
|
||||
|
||||
for (const source of sourceUrls) {
|
||||
try {
|
||||
this.emit('download_attempt', { source, model: name, version });
|
||||
|
||||
const data = await this._downloadFromUrl(source.url, {
|
||||
...options,
|
||||
progress,
|
||||
expectedSize: size,
|
||||
});
|
||||
|
||||
// Verify integrity
|
||||
if (hash) {
|
||||
const computedHash = `sha256:${createHash('sha256').update(data).digest('hex')}`;
|
||||
if (computedHash !== hash) {
|
||||
throw new Error(`Hash mismatch: expected ${hash}, got ${computedHash}`);
|
||||
}
|
||||
}
|
||||
|
||||
this.stats.downloads++;
|
||||
this.stats.bytesDownloaded += data.length;
|
||||
|
||||
progress.complete();
|
||||
|
||||
this.emit('download_complete', {
|
||||
source,
|
||||
model: name,
|
||||
version,
|
||||
size: data.length,
|
||||
});
|
||||
|
||||
return data;
|
||||
|
||||
} catch (error) {
|
||||
lastError = error;
|
||||
this.emit('download_failed', {
|
||||
source,
|
||||
model: name,
|
||||
version,
|
||||
error: error.message,
|
||||
});
|
||||
|
||||
// Continue to next source
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
this.stats.failures++;
|
||||
progress.fail(lastError);
|
||||
|
||||
throw new Error(`Failed to download ${name}@${version} from all sources: ${lastError?.message}`);
|
||||
}
|
||||
|
||||
/**
|
||||
* Download from a URL with streaming and progress
|
||||
* @private
|
||||
*/
|
||||
_downloadFromUrl(url, options = {}) {
|
||||
return new Promise((resolve, reject) => {
|
||||
const { progress, expectedSize, timeout = 60000 } = options;
|
||||
const parsedUrl = new URL(url);
|
||||
const protocol = parsedUrl.protocol === 'https:' ? https : http;
|
||||
|
||||
const chunks = [];
|
||||
let bytesReceived = 0;
|
||||
|
||||
const request = protocol.get(url, {
|
||||
timeout,
|
||||
headers: {
|
||||
'User-Agent': 'RuVector-EdgeNet/1.0',
|
||||
'Accept': 'application/octet-stream',
|
||||
},
|
||||
}, (response) => {
|
||||
// Handle redirects
|
||||
if (response.statusCode >= 300 && response.statusCode < 400 && response.headers.location) {
|
||||
this._downloadFromUrl(response.headers.location, options)
|
||||
.then(resolve)
|
||||
.catch(reject);
|
||||
return;
|
||||
}
|
||||
|
||||
if (response.statusCode !== 200) {
|
||||
reject(new Error(`HTTP ${response.statusCode}: ${response.statusMessage}`));
|
||||
return;
|
||||
}
|
||||
|
||||
const contentLength = parseInt(response.headers['content-length'] || expectedSize || 0, 10);
|
||||
if (progress && contentLength) {
|
||||
progress.totalBytes = contentLength;
|
||||
}
|
||||
|
||||
response.on('data', (chunk) => {
|
||||
chunks.push(chunk);
|
||||
bytesReceived += chunk.length;
|
||||
|
||||
if (progress) {
|
||||
progress.update(chunk.length);
|
||||
}
|
||||
});
|
||||
|
||||
response.on('end', () => {
|
||||
const data = Buffer.concat(chunks);
|
||||
resolve(data);
|
||||
});
|
||||
|
||||
response.on('error', reject);
|
||||
});
|
||||
|
||||
request.on('error', reject);
|
||||
request.on('timeout', () => {
|
||||
request.destroy();
|
||||
reject(new Error('Request timeout'));
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Download to a file with streaming
|
||||
* @param {object} metadata - Model metadata
|
||||
* @param {string} destPath - Destination file path
|
||||
* @param {object} options - Download options
|
||||
*/
|
||||
async downloadToFile(metadata, destPath, options = {}) {
|
||||
const data = await this.download(metadata, options);
|
||||
|
||||
// Ensure directory exists
|
||||
const dir = path.dirname(destPath);
|
||||
await fs.mkdir(dir, { recursive: true });
|
||||
|
||||
await fs.writeFile(destPath, data);
|
||||
|
||||
return {
|
||||
path: destPath,
|
||||
size: data.length,
|
||||
};
|
||||
}
|
||||
|
||||
// ============================================
|
||||
// UPLOAD
|
||||
// ============================================
|
||||
|
||||
/**
|
||||
* Upload a model to Google Cloud Storage
|
||||
* @param {Buffer} data - Model data
|
||||
* @param {string} modelName - Model name
|
||||
* @param {string} version - Version
|
||||
* @param {object} options - Upload options
|
||||
* @returns {Promise<string>} GCS URL
|
||||
*/
|
||||
async uploadToGcs(data, modelName, version, options = {}) {
|
||||
const { filename = `${modelName}.onnx` } = options;
|
||||
const gcsPath = `${modelName}/${version}/${filename}`;
|
||||
|
||||
// Check for @google-cloud/storage
|
||||
let storage;
|
||||
try {
|
||||
const { Storage } = await import('@google-cloud/storage');
|
||||
storage = new Storage({
|
||||
projectId: this.gcsConfig.projectId,
|
||||
keyFilename: this.gcsConfig.keyFilePath,
|
||||
});
|
||||
} catch (error) {
|
||||
throw new Error('GCS upload requires @google-cloud/storage package');
|
||||
}
|
||||
|
||||
const bucket = storage.bucket(this.gcsConfig.bucket);
|
||||
const file = bucket.file(gcsPath);
|
||||
|
||||
const progress = new ProgressTracker(data.length);
|
||||
|
||||
if (options.onProgress) {
|
||||
progress.on('progress', options.onProgress);
|
||||
}
|
||||
|
||||
await new Promise((resolve, reject) => {
|
||||
const stream = file.createWriteStream({
|
||||
metadata: {
|
||||
contentType: 'application/octet-stream',
|
||||
metadata: {
|
||||
modelName,
|
||||
version,
|
||||
hash: `sha256:${createHash('sha256').update(data).digest('hex')}`,
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
stream.on('error', reject);
|
||||
stream.on('finish', resolve);
|
||||
|
||||
// Write in chunks for progress tracking
|
||||
let offset = 0;
|
||||
const writeChunk = () => {
|
||||
while (offset < data.length) {
|
||||
const end = Math.min(offset + CHUNK_SIZE, data.length);
|
||||
const chunk = data.slice(offset, end);
|
||||
|
||||
if (!stream.write(chunk)) {
|
||||
offset = end;
|
||||
stream.once('drain', writeChunk);
|
||||
return;
|
||||
}
|
||||
|
||||
progress.update(chunk.length);
|
||||
offset = end;
|
||||
}
|
||||
stream.end();
|
||||
};
|
||||
|
||||
writeChunk();
|
||||
});
|
||||
|
||||
progress.complete();
|
||||
this.stats.uploads++;
|
||||
this.stats.bytesUploaded += data.length;
|
||||
|
||||
const gcsUrl = this.getGcsUrl(modelName, version, filename);
|
||||
|
||||
this.emit('upload_complete', {
|
||||
type: 'gcs',
|
||||
url: gcsUrl,
|
||||
model: modelName,
|
||||
version,
|
||||
size: data.length,
|
||||
});
|
||||
|
||||
return gcsUrl;
|
||||
}
|
||||
|
||||
/**
|
||||
* Upload a model to IPFS via web3.storage
|
||||
* @param {Buffer} data - Model data
|
||||
* @param {string} modelName - Model name
|
||||
* @param {string} version - Version
|
||||
* @param {object} options - Upload options
|
||||
* @returns {Promise<string>} IPFS CID
|
||||
*/
|
||||
async uploadToIpfs(data, modelName, version, options = {}) {
|
||||
const { filename = `${modelName}.onnx`, provider = 'web3storage' } = options;
|
||||
|
||||
let cid;
|
||||
|
||||
if (provider === 'web3storage' && this.ipfsConfig.web3StorageToken) {
|
||||
cid = await this._uploadToWeb3Storage(data, filename);
|
||||
} else if (provider === 'nftstorage' && this.ipfsConfig.nftStorageToken) {
|
||||
cid = await this._uploadToNftStorage(data, filename);
|
||||
} else {
|
||||
throw new Error('No IPFS provider configured. Set WEB3_STORAGE_TOKEN or NFT_STORAGE_TOKEN');
|
||||
}
|
||||
|
||||
this.stats.uploads++;
|
||||
this.stats.bytesUploaded += data.length;
|
||||
|
||||
const ipfsUrl = this.getIpfsUrl(cid);
|
||||
|
||||
this.emit('upload_complete', {
|
||||
type: 'ipfs',
|
||||
url: ipfsUrl,
|
||||
cid,
|
||||
model: modelName,
|
||||
version,
|
||||
size: data.length,
|
||||
});
|
||||
|
||||
return ipfsUrl;
|
||||
}
|
||||
|
||||
/**
|
||||
* Upload to web3.storage
|
||||
* @private
|
||||
*/
|
||||
async _uploadToWeb3Storage(data, filename) {
|
||||
// web3.storage API upload
|
||||
const response = await this._httpRequest('https://api.web3.storage/upload', {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Authorization': `Bearer ${this.ipfsConfig.web3StorageToken}`,
|
||||
'X-Name': filename,
|
||||
},
|
||||
body: data,
|
||||
});
|
||||
|
||||
if (!response.cid) {
|
||||
throw new Error('web3.storage upload failed: no CID returned');
|
||||
}
|
||||
|
||||
return response.cid;
|
||||
}
|
||||
|
||||
/**
|
||||
* Upload to nft.storage
|
||||
* @private
|
||||
*/
|
||||
async _uploadToNftStorage(data, filename) {
|
||||
// nft.storage API upload
|
||||
const response = await this._httpRequest('https://api.nft.storage/upload', {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Authorization': `Bearer ${this.ipfsConfig.nftStorageToken}`,
|
||||
},
|
||||
body: data,
|
||||
});
|
||||
|
||||
if (!response.value?.cid) {
|
||||
throw new Error('nft.storage upload failed: no CID returned');
|
||||
}
|
||||
|
||||
return response.value.cid;
|
||||
}
|
||||
|
||||
/**
|
||||
* Make an HTTP request
|
||||
* @private
|
||||
*/
|
||||
_httpRequest(url, options = {}) {
|
||||
return new Promise((resolve, reject) => {
|
||||
const parsedUrl = new URL(url);
|
||||
const protocol = parsedUrl.protocol === 'https:' ? https : http;
|
||||
|
||||
const requestOptions = {
|
||||
method: options.method || 'GET',
|
||||
headers: options.headers || {},
|
||||
hostname: parsedUrl.hostname,
|
||||
path: parsedUrl.pathname + parsedUrl.search,
|
||||
port: parsedUrl.port,
|
||||
};
|
||||
|
||||
const request = protocol.request(requestOptions, (response) => {
|
||||
const chunks = [];
|
||||
|
||||
response.on('data', chunk => chunks.push(chunk));
|
||||
response.on('end', () => {
|
||||
const body = Buffer.concat(chunks).toString('utf-8');
|
||||
|
||||
if (response.statusCode >= 400) {
|
||||
reject(new Error(`HTTP ${response.statusCode}: ${body}`));
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
resolve(JSON.parse(body));
|
||||
} catch {
|
||||
resolve(body);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
request.on('error', reject);
|
||||
|
||||
if (options.body) {
|
||||
request.write(options.body);
|
||||
}
|
||||
|
||||
request.end();
|
||||
});
|
||||
}
|
||||
|
||||
// ============================================
|
||||
// INTEGRITY VERIFICATION
|
||||
// ============================================
|
||||
|
||||
/**
|
||||
* Compute SHA256 hash of data
|
||||
* @param {Buffer} data - Data to hash
|
||||
* @returns {string} Hash string with sha256: prefix
|
||||
*/
|
||||
computeHash(data) {
|
||||
return `sha256:${createHash('sha256').update(data).digest('hex')}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Verify data integrity against expected hash
|
||||
* @param {Buffer} data - Data to verify
|
||||
* @param {string} expectedHash - Expected hash
|
||||
* @returns {boolean}
|
||||
*/
|
||||
verifyIntegrity(data, expectedHash) {
|
||||
const computed = this.computeHash(data);
|
||||
return computed === expectedHash;
|
||||
}
|
||||
|
||||
/**
|
||||
* Verify a downloaded model
|
||||
* @param {Buffer} data - Model data
|
||||
* @param {object} metadata - Model metadata
|
||||
* @returns {object} Verification result
|
||||
*/
|
||||
verifyModel(data, metadata) {
|
||||
const result = {
|
||||
valid: true,
|
||||
checks: [],
|
||||
};
|
||||
|
||||
// Size check
|
||||
if (metadata.size) {
|
||||
const sizeMatch = data.length === metadata.size;
|
||||
result.checks.push({
|
||||
type: 'size',
|
||||
expected: metadata.size,
|
||||
actual: data.length,
|
||||
passed: sizeMatch,
|
||||
});
|
||||
if (!sizeMatch) result.valid = false;
|
||||
}
|
||||
|
||||
// Hash check
|
||||
if (metadata.hash) {
|
||||
const hashMatch = this.verifyIntegrity(data, metadata.hash);
|
||||
result.checks.push({
|
||||
type: 'hash',
|
||||
expected: metadata.hash,
|
||||
actual: this.computeHash(data),
|
||||
passed: hashMatch,
|
||||
});
|
||||
if (!hashMatch) result.valid = false;
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
// ============================================
|
||||
// STATS AND INFO
|
||||
// ============================================
|
||||
|
||||
/**
|
||||
* Get distribution manager stats
|
||||
* @returns {object}
|
||||
*/
|
||||
getStats() {
|
||||
return {
|
||||
...this.stats,
|
||||
activeDownloads: this.activeDownloads.size,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================
|
||||
// DEFAULT EXPORT
|
||||
// ============================================
|
||||
|
||||
export default DistributionManager;
|
||||
753
vendor/ruvector/examples/edge-net/pkg/models/integrity.js
vendored
Normal file
753
vendor/ruvector/examples/edge-net/pkg/models/integrity.js
vendored
Normal file
@@ -0,0 +1,753 @@
|
||||
/**
|
||||
* @ruvector/edge-net Model Integrity System
|
||||
*
|
||||
* Content-addressed integrity with:
|
||||
* - Canonical JSON signing
|
||||
* - Threshold signatures with trust roots
|
||||
* - Merkle chunk verification for streaming
|
||||
* - Transparency log integration
|
||||
*
|
||||
* Design principle: Manifest is truth, everything else is replaceable.
|
||||
*
|
||||
* @module @ruvector/edge-net/models/integrity
|
||||
*/
|
||||
|
||||
import { createHash } from 'crypto';
|
||||
|
||||
// ============================================================================
|
||||
// CANONICAL JSON
|
||||
// ============================================================================
|
||||
|
||||
/**
|
||||
* Canonical JSON encoding for deterministic signing.
|
||||
* - Keys sorted lexicographically
|
||||
* - No whitespace
|
||||
* - Unicode escaped consistently
|
||||
* - Numbers without trailing zeros
|
||||
*/
|
||||
export function canonicalize(obj) {
|
||||
if (obj === null || obj === undefined) {
|
||||
return 'null';
|
||||
}
|
||||
|
||||
if (typeof obj === 'boolean') {
|
||||
return obj ? 'true' : 'false';
|
||||
}
|
||||
|
||||
if (typeof obj === 'number') {
|
||||
if (!Number.isFinite(obj)) {
|
||||
throw new Error('Cannot canonicalize Infinity or NaN');
|
||||
}
|
||||
// Use JSON for consistent number formatting
|
||||
return JSON.stringify(obj);
|
||||
}
|
||||
|
||||
if (typeof obj === 'string') {
|
||||
// Escape unicode consistently
|
||||
return JSON.stringify(obj).replace(/[\u007f-\uffff]/g, (c) => {
|
||||
return '\\u' + ('0000' + c.charCodeAt(0).toString(16)).slice(-4);
|
||||
});
|
||||
}
|
||||
|
||||
if (Array.isArray(obj)) {
|
||||
const elements = obj.map(canonicalize);
|
||||
return '[' + elements.join(',') + ']';
|
||||
}
|
||||
|
||||
if (typeof obj === 'object') {
|
||||
const keys = Object.keys(obj).sort();
|
||||
const pairs = keys
|
||||
.filter(k => obj[k] !== undefined)
|
||||
.map(k => canonicalize(k) + ':' + canonicalize(obj[k]));
|
||||
return '{' + pairs.join(',') + '}';
|
||||
}
|
||||
|
||||
throw new Error(`Cannot canonicalize type: ${typeof obj}`);
|
||||
}
|
||||
|
||||
/**
|
||||
* Hash canonical JSON bytes
|
||||
*/
|
||||
export function hashCanonical(obj, algorithm = 'sha256') {
|
||||
const canonical = canonicalize(obj);
|
||||
const hash = createHash(algorithm);
|
||||
hash.update(canonical, 'utf8');
|
||||
return hash.digest('hex');
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// TRUST ROOT
|
||||
// ============================================================================
|
||||
|
||||
/**
|
||||
* Built-in root keys shipped with SDK.
|
||||
* These are the only keys trusted by default.
|
||||
*/
|
||||
export const BUILTIN_ROOT_KEYS = Object.freeze({
|
||||
'ruvector-root-2024': {
|
||||
keyId: 'ruvector-root-2024',
|
||||
algorithm: 'ed25519',
|
||||
publicKey: 'MCowBQYDK2VwAyEAaGVsbG8td29ybGQta2V5LXBsYWNlaG9sZGVy', // Placeholder
|
||||
validFrom: '2024-01-01T00:00:00Z',
|
||||
validUntil: '2030-01-01T00:00:00Z',
|
||||
capabilities: ['sign-manifest', 'sign-adapter', 'delegate'],
|
||||
},
|
||||
'ruvector-models-2024': {
|
||||
keyId: 'ruvector-models-2024',
|
||||
algorithm: 'ed25519',
|
||||
publicKey: 'MCowBQYDK2VwAyEAbW9kZWxzLWtleS1wbGFjZWhvbGRlcg==', // Placeholder
|
||||
validFrom: '2024-01-01T00:00:00Z',
|
||||
validUntil: '2026-01-01T00:00:00Z',
|
||||
capabilities: ['sign-manifest'],
|
||||
delegatedBy: 'ruvector-root-2024',
|
||||
},
|
||||
});
|
||||
|
||||
/**
|
||||
* Trust root configuration
|
||||
*/
|
||||
export class TrustRoot {
|
||||
constructor(options = {}) {
|
||||
// Start with built-in keys
|
||||
this.trustedKeys = new Map();
|
||||
for (const [id, key] of Object.entries(BUILTIN_ROOT_KEYS)) {
|
||||
this.trustedKeys.set(id, key);
|
||||
}
|
||||
|
||||
// Add enterprise keys if configured
|
||||
if (options.enterpriseKeys) {
|
||||
for (const key of options.enterpriseKeys) {
|
||||
this.addEnterpriseKey(key);
|
||||
}
|
||||
}
|
||||
|
||||
// Revocation list
|
||||
this.revokedKeys = new Set(options.revokedKeys || []);
|
||||
|
||||
// Minimum signatures required for official releases
|
||||
this.minimumSignaturesRequired = options.minimumSignaturesRequired || 1;
|
||||
|
||||
// Threshold for high-security operations (e.g., new root key)
|
||||
this.thresholdSignaturesRequired = options.thresholdSignaturesRequired || 2;
|
||||
}
|
||||
|
||||
/**
|
||||
* Add an enterprise root key (for private deployments)
|
||||
*/
|
||||
addEnterpriseKey(key) {
|
||||
if (!key.keyId || !key.publicKey) {
|
||||
throw new Error('Enterprise key must have keyId and publicKey');
|
||||
}
|
||||
|
||||
// Verify delegation chain if not self-signed
|
||||
if (key.delegatedBy && key.delegationSignature) {
|
||||
const delegator = this.trustedKeys.get(key.delegatedBy);
|
||||
if (!delegator) {
|
||||
throw new Error(`Unknown delegator: ${key.delegatedBy}`);
|
||||
}
|
||||
if (!delegator.capabilities.includes('delegate')) {
|
||||
throw new Error(`Key ${key.delegatedBy} cannot delegate`);
|
||||
}
|
||||
// In production, verify delegationSignature here
|
||||
}
|
||||
|
||||
this.trustedKeys.set(key.keyId, {
|
||||
...key,
|
||||
isEnterprise: true,
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Revoke a key
|
||||
*/
|
||||
revokeKey(keyId, reason) {
|
||||
this.revokedKeys.add(keyId);
|
||||
console.warn(`[TrustRoot] Key revoked: ${keyId} - ${reason}`);
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if a key is trusted for a capability
|
||||
*/
|
||||
isKeyTrusted(keyId, capability = 'sign-manifest') {
|
||||
if (this.revokedKeys.has(keyId)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
const key = this.trustedKeys.get(keyId);
|
||||
if (!key) {
|
||||
return false;
|
||||
}
|
||||
|
||||
// Check validity period
|
||||
const now = new Date();
|
||||
if (key.validFrom && new Date(key.validFrom) > now) {
|
||||
return false;
|
||||
}
|
||||
if (key.validUntil && new Date(key.validUntil) < now) {
|
||||
return false;
|
||||
}
|
||||
|
||||
// Check capability
|
||||
if (!key.capabilities.includes(capability)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get public key for verification
|
||||
*/
|
||||
getPublicKey(keyId) {
|
||||
const key = this.trustedKeys.get(keyId);
|
||||
if (!key || this.revokedKeys.has(keyId)) {
|
||||
return null;
|
||||
}
|
||||
return key.publicKey;
|
||||
}
|
||||
|
||||
/**
|
||||
* Verify signature set meets threshold
|
||||
*/
|
||||
verifySignatureThreshold(signatures, requiredCount = null) {
|
||||
const required = requiredCount || this.minimumSignaturesRequired;
|
||||
let validCount = 0;
|
||||
const validSigners = [];
|
||||
|
||||
for (const sig of signatures) {
|
||||
if (this.isKeyTrusted(sig.keyId, 'sign-manifest')) {
|
||||
// In production, verify actual signature here
|
||||
validCount++;
|
||||
validSigners.push(sig.keyId);
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
valid: validCount >= required,
|
||||
validCount,
|
||||
required,
|
||||
validSigners,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Export current trust configuration
|
||||
*/
|
||||
export() {
|
||||
return {
|
||||
trustedKeys: Object.fromEntries(this.trustedKeys),
|
||||
revokedKeys: Array.from(this.revokedKeys),
|
||||
minimumSignaturesRequired: this.minimumSignaturesRequired,
|
||||
thresholdSignaturesRequired: this.thresholdSignaturesRequired,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// MERKLE CHUNK VERIFICATION
|
||||
// ============================================================================
|
||||
|
||||
/**
|
||||
* Compute Merkle tree from chunk hashes
|
||||
*/
|
||||
export function computeMerkleRoot(chunkHashes) {
|
||||
if (chunkHashes.length === 0) {
|
||||
return hashCanonical({ empty: true });
|
||||
}
|
||||
|
||||
if (chunkHashes.length === 1) {
|
||||
return chunkHashes[0];
|
||||
}
|
||||
|
||||
// Build tree bottom-up
|
||||
let level = [...chunkHashes];
|
||||
|
||||
while (level.length > 1) {
|
||||
const nextLevel = [];
|
||||
for (let i = 0; i < level.length; i += 2) {
|
||||
const left = level[i];
|
||||
const right = level[i + 1] || left; // Duplicate last if odd
|
||||
const combined = createHash('sha256')
|
||||
.update(left, 'hex')
|
||||
.update(right, 'hex')
|
||||
.digest('hex');
|
||||
nextLevel.push(combined);
|
||||
}
|
||||
level = nextLevel;
|
||||
}
|
||||
|
||||
return level[0];
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate Merkle proof for a chunk
|
||||
*/
|
||||
export function generateMerkleProof(chunkHashes, chunkIndex) {
|
||||
const proof = [];
|
||||
let level = [...chunkHashes];
|
||||
let index = chunkIndex;
|
||||
|
||||
while (level.length > 1) {
|
||||
const isRight = index % 2 === 1;
|
||||
const siblingIndex = isRight ? index - 1 : index + 1;
|
||||
|
||||
if (siblingIndex < level.length) {
|
||||
proof.push({
|
||||
hash: level[siblingIndex],
|
||||
position: isRight ? 'left' : 'right',
|
||||
});
|
||||
} else {
|
||||
// Odd number, sibling is self
|
||||
proof.push({
|
||||
hash: level[index],
|
||||
position: 'right',
|
||||
});
|
||||
}
|
||||
|
||||
// Move up
|
||||
const nextLevel = [];
|
||||
for (let i = 0; i < level.length; i += 2) {
|
||||
const left = level[i];
|
||||
const right = level[i + 1] || left;
|
||||
nextLevel.push(
|
||||
createHash('sha256')
|
||||
.update(left, 'hex')
|
||||
.update(right, 'hex')
|
||||
.digest('hex')
|
||||
);
|
||||
}
|
||||
level = nextLevel;
|
||||
index = Math.floor(index / 2);
|
||||
}
|
||||
|
||||
return proof;
|
||||
}
|
||||
|
||||
/**
|
||||
* Verify a chunk against Merkle root
|
||||
*/
|
||||
export function verifyMerkleProof(chunkHash, chunkIndex, proof, merkleRoot) {
|
||||
let computed = chunkHash;
|
||||
|
||||
for (const step of proof) {
|
||||
const left = step.position === 'left' ? step.hash : computed;
|
||||
const right = step.position === 'right' ? step.hash : computed;
|
||||
computed = createHash('sha256')
|
||||
.update(left, 'hex')
|
||||
.update(right, 'hex')
|
||||
.digest('hex');
|
||||
}
|
||||
|
||||
return computed === merkleRoot;
|
||||
}
|
||||
|
||||
/**
|
||||
* Chunk a buffer and compute hashes
|
||||
*/
|
||||
export function chunkAndHash(buffer, chunkSize = 256 * 1024) {
|
||||
const chunks = [];
|
||||
const hashes = [];
|
||||
|
||||
for (let offset = 0; offset < buffer.length; offset += chunkSize) {
|
||||
const chunk = buffer.slice(offset, offset + chunkSize);
|
||||
chunks.push(chunk);
|
||||
hashes.push(
|
||||
createHash('sha256').update(chunk).digest('hex')
|
||||
);
|
||||
}
|
||||
|
||||
return {
|
||||
chunks,
|
||||
chunkHashes: hashes,
|
||||
chunkSize,
|
||||
chunkCount: chunks.length,
|
||||
totalSize: buffer.length,
|
||||
merkleRoot: computeMerkleRoot(hashes),
|
||||
};
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// MANIFEST INTEGRITY
|
||||
// ============================================================================
|
||||
|
||||
/**
|
||||
* Integrity block for manifests
|
||||
*/
|
||||
export function createIntegrityBlock(manifest, chunkInfo) {
|
||||
// Create the signed payload (everything except signatures)
|
||||
const signedPayload = {
|
||||
model: manifest.model,
|
||||
version: manifest.version,
|
||||
artifacts: manifest.artifacts,
|
||||
provenance: manifest.provenance,
|
||||
capabilities: manifest.capabilities,
|
||||
timestamp: new Date().toISOString(),
|
||||
};
|
||||
|
||||
const signedPayloadHash = hashCanonical(signedPayload);
|
||||
|
||||
return {
|
||||
manifestHash: hashCanonical(manifest),
|
||||
signedPayloadHash,
|
||||
merkleRoot: chunkInfo.merkleRoot,
|
||||
chunking: {
|
||||
chunkSize: chunkInfo.chunkSize,
|
||||
chunkCount: chunkInfo.chunkCount,
|
||||
chunkHashes: chunkInfo.chunkHashes,
|
||||
},
|
||||
signatures: [], // To be filled by signing process
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Provenance block for manifests
|
||||
*/
|
||||
export function createProvenanceBlock(options = {}) {
|
||||
return {
|
||||
builtBy: {
|
||||
tool: options.tool || '@ruvector/model-optimizer',
|
||||
version: options.toolVersion || '1.0.0',
|
||||
commit: options.commit || 'unknown',
|
||||
},
|
||||
optimizationRecipeHash: options.recipeHash || null,
|
||||
calibrationDatasetHash: options.calibrationHash || null,
|
||||
parentLineage: options.parentLineage || null,
|
||||
buildTimestamp: new Date().toISOString(),
|
||||
environment: {
|
||||
platform: process.platform,
|
||||
arch: process.arch,
|
||||
nodeVersion: process.version,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Full manifest with integrity
|
||||
*/
|
||||
export function createSecureManifest(model, artifacts, options = {}) {
|
||||
const manifest = {
|
||||
schemaVersion: '2.0.0',
|
||||
model: {
|
||||
id: model.id,
|
||||
name: model.name,
|
||||
version: model.version,
|
||||
type: model.type, // 'embedding' | 'generation'
|
||||
tier: model.tier, // 'micro' | 'small' | 'large'
|
||||
capabilities: model.capabilities || [],
|
||||
memoryRequirement: model.memoryRequirement,
|
||||
},
|
||||
artifacts: artifacts.map(a => ({
|
||||
path: a.path,
|
||||
size: a.size,
|
||||
sha256: a.sha256,
|
||||
format: a.format,
|
||||
quantization: a.quantization,
|
||||
})),
|
||||
distribution: {
|
||||
gcs: options.gcsUrl,
|
||||
ipfs: options.ipfsCid,
|
||||
fallbackUrls: options.fallbackUrls || [],
|
||||
},
|
||||
provenance: createProvenanceBlock(options.provenance || {}),
|
||||
capabilities: model.capabilities || [],
|
||||
};
|
||||
|
||||
// Add integrity block if chunk info provided
|
||||
if (options.chunkInfo) {
|
||||
manifest.integrity = createIntegrityBlock(manifest, options.chunkInfo);
|
||||
}
|
||||
|
||||
// Add trust metadata
|
||||
manifest.trust = {
|
||||
trustedKeySetId: options.trustedKeySetId || 'ruvector-default-2024',
|
||||
minimumSignaturesRequired: options.minimumSignaturesRequired || 1,
|
||||
};
|
||||
|
||||
return manifest;
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// MANIFEST VERIFICATION
|
||||
// ============================================================================
|
||||
|
||||
/**
|
||||
* Verify a manifest's integrity
|
||||
*/
|
||||
export class ManifestVerifier {
|
||||
constructor(trustRoot = null) {
|
||||
this.trustRoot = trustRoot || new TrustRoot();
|
||||
}
|
||||
|
||||
/**
|
||||
* Full verification of a manifest
|
||||
*/
|
||||
verify(manifest) {
|
||||
const errors = [];
|
||||
const warnings = [];
|
||||
|
||||
// 1. Schema version check
|
||||
if (!manifest.schemaVersion || manifest.schemaVersion < '2.0.0') {
|
||||
warnings.push('Manifest uses old schema version');
|
||||
}
|
||||
|
||||
// 2. Verify integrity block
|
||||
if (manifest.integrity) {
|
||||
// Check manifest hash
|
||||
const computed = hashCanonical(manifest);
|
||||
// Note: manifestHash is computed before adding integrity, so we skip this
|
||||
|
||||
// Check signed payload hash
|
||||
const signedPayload = {
|
||||
model: manifest.model,
|
||||
version: manifest.version,
|
||||
artifacts: manifest.artifacts,
|
||||
provenance: manifest.provenance,
|
||||
capabilities: manifest.capabilities,
|
||||
timestamp: manifest.integrity.timestamp,
|
||||
};
|
||||
const computedPayloadHash = hashCanonical(signedPayload);
|
||||
|
||||
// 3. Verify signatures meet threshold
|
||||
if (manifest.integrity.signatures?.length > 0) {
|
||||
const sigResult = this.trustRoot.verifySignatureThreshold(
|
||||
manifest.integrity.signatures,
|
||||
manifest.trust?.minimumSignaturesRequired
|
||||
);
|
||||
|
||||
if (!sigResult.valid) {
|
||||
errors.push(`Insufficient valid signatures: ${sigResult.validCount}/${sigResult.required}`);
|
||||
}
|
||||
} else {
|
||||
warnings.push('No signatures present');
|
||||
}
|
||||
|
||||
// 4. Verify Merkle root matches chunk hashes
|
||||
if (manifest.integrity.chunking) {
|
||||
const computedRoot = computeMerkleRoot(manifest.integrity.chunking.chunkHashes);
|
||||
if (computedRoot !== manifest.integrity.merkleRoot) {
|
||||
errors.push('Merkle root mismatch');
|
||||
}
|
||||
}
|
||||
} else {
|
||||
warnings.push('No integrity block present');
|
||||
}
|
||||
|
||||
// 5. Check provenance
|
||||
if (!manifest.provenance) {
|
||||
warnings.push('No provenance information');
|
||||
}
|
||||
|
||||
// 6. Check required fields
|
||||
if (!manifest.model?.id) errors.push('Missing model.id');
|
||||
if (!manifest.model?.version) errors.push('Missing model.version');
|
||||
if (!manifest.artifacts?.length) errors.push('No artifacts defined');
|
||||
|
||||
return {
|
||||
valid: errors.length === 0,
|
||||
errors,
|
||||
warnings,
|
||||
trust: manifest.trust,
|
||||
provenance: manifest.provenance,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Verify a single chunk during streaming download
|
||||
*/
|
||||
verifyChunk(chunkData, chunkIndex, manifest) {
|
||||
if (!manifest.integrity?.chunking) {
|
||||
return { valid: false, error: 'No chunking info in manifest' };
|
||||
}
|
||||
|
||||
const expectedHash = manifest.integrity.chunking.chunkHashes[chunkIndex];
|
||||
if (!expectedHash) {
|
||||
return { valid: false, error: `No hash for chunk ${chunkIndex}` };
|
||||
}
|
||||
|
||||
const actualHash = createHash('sha256').update(chunkData).digest('hex');
|
||||
|
||||
if (actualHash !== expectedHash) {
|
||||
return {
|
||||
valid: false,
|
||||
error: `Chunk ${chunkIndex} hash mismatch`,
|
||||
expected: expectedHash,
|
||||
actual: actualHash,
|
||||
};
|
||||
}
|
||||
|
||||
return { valid: true, chunkIndex, hash: actualHash };
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// TRANSPARENCY LOG
|
||||
// ============================================================================
|
||||
|
||||
/**
|
||||
* Entry in the transparency log
|
||||
*/
|
||||
export function createLogEntry(manifest, publisherKeyId) {
|
||||
return {
|
||||
manifestHash: hashCanonical(manifest),
|
||||
modelId: manifest.model.id,
|
||||
version: manifest.model.version,
|
||||
publisherKeyId,
|
||||
timestamp: new Date().toISOString(),
|
||||
signedPayloadHash: manifest.integrity?.signedPayloadHash,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Simple append-only transparency log
|
||||
* In production, this would be backed by a Merkle tree or blockchain
|
||||
*/
|
||||
export class TransparencyLog {
|
||||
constructor(options = {}) {
|
||||
this.entries = [];
|
||||
this.indexByModel = new Map();
|
||||
this.indexByHash = new Map();
|
||||
this.logRoot = null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Append an entry to the log
|
||||
*/
|
||||
append(entry) {
|
||||
const index = this.entries.length;
|
||||
|
||||
// Compute log entry hash including previous
|
||||
const logEntryHash = hashCanonical({
|
||||
...entry,
|
||||
index,
|
||||
previousHash: this.logRoot,
|
||||
});
|
||||
|
||||
const fullEntry = {
|
||||
...entry,
|
||||
index,
|
||||
previousHash: this.logRoot,
|
||||
logEntryHash,
|
||||
};
|
||||
|
||||
this.entries.push(fullEntry);
|
||||
this.logRoot = logEntryHash;
|
||||
|
||||
// Update indexes
|
||||
if (!this.indexByModel.has(entry.modelId)) {
|
||||
this.indexByModel.set(entry.modelId, []);
|
||||
}
|
||||
this.indexByModel.get(entry.modelId).push(index);
|
||||
this.indexByHash.set(entry.manifestHash, index);
|
||||
|
||||
return fullEntry;
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate inclusion proof
|
||||
*/
|
||||
getInclusionProof(manifestHash) {
|
||||
const index = this.indexByHash.get(manifestHash);
|
||||
if (index === undefined) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const entry = this.entries[index];
|
||||
const proof = [];
|
||||
|
||||
// Simple chain proof (in production, use Merkle tree)
|
||||
for (let i = index; i < this.entries.length; i++) {
|
||||
proof.push({
|
||||
index: i,
|
||||
logEntryHash: this.entries[i].logEntryHash,
|
||||
});
|
||||
}
|
||||
|
||||
return {
|
||||
entry,
|
||||
proof,
|
||||
currentRoot: this.logRoot,
|
||||
logLength: this.entries.length,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Verify inclusion proof
|
||||
*/
|
||||
verifyInclusionProof(proof) {
|
||||
if (!proof || !proof.entry || !proof.proof.length) {
|
||||
return false;
|
||||
}
|
||||
|
||||
// Verify chain
|
||||
let expectedHash = proof.entry.logEntryHash;
|
||||
for (let i = 1; i < proof.proof.length; i++) {
|
||||
const entry = proof.proof[i];
|
||||
// Verify chain continuity
|
||||
if (i < proof.proof.length - 1) {
|
||||
// Each entry should reference the previous
|
||||
}
|
||||
}
|
||||
|
||||
return proof.proof[proof.proof.length - 1].logEntryHash === proof.currentRoot;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get history for a model
|
||||
*/
|
||||
getModelHistory(modelId) {
|
||||
const indices = this.indexByModel.get(modelId) || [];
|
||||
return indices.map(i => this.entries[i]);
|
||||
}
|
||||
|
||||
/**
|
||||
* Export log for persistence
|
||||
*/
|
||||
export() {
|
||||
return {
|
||||
entries: this.entries,
|
||||
logRoot: this.logRoot,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Import log
|
||||
*/
|
||||
import(data) {
|
||||
this.entries = data.entries || [];
|
||||
this.logRoot = data.logRoot;
|
||||
|
||||
// Rebuild indexes
|
||||
this.indexByModel.clear();
|
||||
this.indexByHash.clear();
|
||||
|
||||
for (let i = 0; i < this.entries.length; i++) {
|
||||
const entry = this.entries[i];
|
||||
if (!this.indexByModel.has(entry.modelId)) {
|
||||
this.indexByModel.set(entry.modelId, []);
|
||||
}
|
||||
this.indexByModel.get(entry.modelId).push(i);
|
||||
this.indexByHash.set(entry.manifestHash, i);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// EXPORTS
|
||||
// ============================================================================
|
||||
|
||||
export default {
|
||||
canonicalize,
|
||||
hashCanonical,
|
||||
TrustRoot,
|
||||
BUILTIN_ROOT_KEYS,
|
||||
computeMerkleRoot,
|
||||
generateMerkleProof,
|
||||
verifyMerkleProof,
|
||||
chunkAndHash,
|
||||
createIntegrityBlock,
|
||||
createProvenanceBlock,
|
||||
createSecureManifest,
|
||||
ManifestVerifier,
|
||||
createLogEntry,
|
||||
TransparencyLog,
|
||||
};
|
||||
725
vendor/ruvector/examples/edge-net/pkg/models/loader.js
vendored
Normal file
725
vendor/ruvector/examples/edge-net/pkg/models/loader.js
vendored
Normal file
@@ -0,0 +1,725 @@
|
||||
/**
|
||||
* @ruvector/edge-net Model Loader
|
||||
*
|
||||
* Tiered model loading with:
|
||||
* - Memory-aware model selection
|
||||
* - Streaming chunk verification
|
||||
* - Multi-source fallback (GCS → IPFS → P2P)
|
||||
* - IndexedDB caching
|
||||
*
|
||||
* Design: Registry returns manifest only, client derives URLs from manifest.
|
||||
*
|
||||
* @module @ruvector/edge-net/models/loader
|
||||
*/
|
||||
|
||||
import { createHash } from 'crypto';
|
||||
import { ManifestVerifier, verifyMerkleProof, computeMerkleRoot } from './integrity.js';
|
||||
|
||||
// ============================================================================
|
||||
// MODEL TIERS
|
||||
// ============================================================================
|
||||
|
||||
/**
|
||||
* Model tier definitions with memory requirements
|
||||
*/
|
||||
export const MODEL_TIERS = Object.freeze({
|
||||
micro: {
|
||||
name: 'micro',
|
||||
maxSize: 100 * 1024 * 1024, // 100MB
|
||||
minMemory: 256 * 1024 * 1024, // 256MB available
|
||||
description: 'Embeddings and small tasks',
|
||||
priority: 1,
|
||||
},
|
||||
small: {
|
||||
name: 'small',
|
||||
maxSize: 500 * 1024 * 1024, // 500MB
|
||||
minMemory: 1024 * 1024 * 1024, // 1GB available
|
||||
description: 'Balanced capability',
|
||||
priority: 2,
|
||||
},
|
||||
large: {
|
||||
name: 'large',
|
||||
maxSize: 1500 * 1024 * 1024, // 1.5GB
|
||||
minMemory: 4096 * 1024 * 1024, // 4GB available
|
||||
description: 'Full capability',
|
||||
priority: 3,
|
||||
},
|
||||
});
|
||||
|
||||
/**
|
||||
* Capability priorities for model selection
|
||||
*/
|
||||
export const CAPABILITY_PRIORITIES = Object.freeze({
|
||||
embed: 1, // Always prioritize embeddings
|
||||
retrieve: 2, // Then retrieval
|
||||
generate: 3, // Generation only when needed
|
||||
code: 4, // Specialized capabilities
|
||||
multilingual: 5,
|
||||
});
|
||||
|
||||
// ============================================================================
|
||||
// MEMORY DETECTION
|
||||
// ============================================================================
|
||||
|
||||
/**
|
||||
* Detect available memory for model loading
|
||||
*/
|
||||
export function detectAvailableMemory() {
|
||||
// Browser environment
|
||||
if (typeof navigator !== 'undefined' && navigator.deviceMemory) {
|
||||
return navigator.deviceMemory * 1024 * 1024 * 1024;
|
||||
}
|
||||
|
||||
// Node.js environment
|
||||
if (typeof process !== 'undefined' && process.memoryUsage) {
|
||||
const usage = process.memoryUsage();
|
||||
// Estimate available as total minus current usage
|
||||
const total = require('os').totalmem?.() || 4 * 1024 * 1024 * 1024;
|
||||
return Math.max(0, total - usage.heapUsed);
|
||||
}
|
||||
|
||||
// Default to 2GB as conservative estimate
|
||||
return 2 * 1024 * 1024 * 1024;
|
||||
}
|
||||
|
||||
/**
|
||||
* Select appropriate tier based on device capabilities
|
||||
*/
|
||||
export function selectTier(requiredCapabilities = ['embed'], preferredTier = null) {
|
||||
const available = detectAvailableMemory();
|
||||
|
||||
// Find highest tier that fits in memory
|
||||
const viableTiers = Object.values(MODEL_TIERS)
|
||||
.filter(tier => tier.minMemory <= available)
|
||||
.sort((a, b) => b.priority - a.priority);
|
||||
|
||||
if (viableTiers.length === 0) {
|
||||
console.warn('[ModelLoader] Insufficient memory for any tier, using micro');
|
||||
return MODEL_TIERS.micro;
|
||||
}
|
||||
|
||||
// Respect preferred tier if viable
|
||||
if (preferredTier && viableTiers.find(t => t.name === preferredTier)) {
|
||||
return MODEL_TIERS[preferredTier];
|
||||
}
|
||||
|
||||
// Otherwise use highest viable
|
||||
return viableTiers[0];
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// CACHE MANAGER
|
||||
// ============================================================================
|
||||
|
||||
/**
|
||||
* IndexedDB-based cache for models and chunks
|
||||
*/
|
||||
export class ModelCache {
|
||||
constructor(options = {}) {
|
||||
this.dbName = options.dbName || 'ruvector-models';
|
||||
this.version = options.version || 1;
|
||||
this.db = null;
|
||||
this.maxCacheSize = options.maxCacheSize || 2 * 1024 * 1024 * 1024; // 2GB
|
||||
}
|
||||
|
||||
async open() {
|
||||
if (this.db) return this.db;
|
||||
|
||||
return new Promise((resolve, reject) => {
|
||||
const request = indexedDB.open(this.dbName, this.version);
|
||||
|
||||
request.onerror = () => reject(request.error);
|
||||
|
||||
request.onsuccess = () => {
|
||||
this.db = request.result;
|
||||
resolve(this.db);
|
||||
};
|
||||
|
||||
request.onupgradeneeded = (event) => {
|
||||
const db = event.target.result;
|
||||
|
||||
// Store for complete models
|
||||
if (!db.objectStoreNames.contains('models')) {
|
||||
const store = db.createObjectStore('models', { keyPath: 'id' });
|
||||
store.createIndex('hash', 'hash', { unique: true });
|
||||
store.createIndex('lastAccess', 'lastAccess');
|
||||
}
|
||||
|
||||
// Store for individual chunks (for streaming)
|
||||
if (!db.objectStoreNames.contains('chunks')) {
|
||||
const store = db.createObjectStore('chunks', { keyPath: 'id' });
|
||||
store.createIndex('modelId', 'modelId');
|
||||
}
|
||||
|
||||
// Store for manifests
|
||||
if (!db.objectStoreNames.contains('manifests')) {
|
||||
db.createObjectStore('manifests', { keyPath: 'modelId' });
|
||||
}
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
async get(modelId) {
|
||||
await this.open();
|
||||
|
||||
return new Promise((resolve, reject) => {
|
||||
const tx = this.db.transaction('models', 'readonly');
|
||||
const store = tx.objectStore('models');
|
||||
const request = store.get(modelId);
|
||||
|
||||
request.onsuccess = () => {
|
||||
const result = request.result;
|
||||
if (result) {
|
||||
// Update last access
|
||||
this.updateLastAccess(modelId);
|
||||
}
|
||||
resolve(result);
|
||||
};
|
||||
request.onerror = () => reject(request.error);
|
||||
});
|
||||
}
|
||||
|
||||
async put(modelId, data, manifest) {
|
||||
await this.open();
|
||||
await this.ensureSpace(data.byteLength || data.length);
|
||||
|
||||
return new Promise((resolve, reject) => {
|
||||
const tx = this.db.transaction(['models', 'manifests'], 'readwrite');
|
||||
|
||||
const modelStore = tx.objectStore('models');
|
||||
modelStore.put({
|
||||
id: modelId,
|
||||
data,
|
||||
hash: manifest.integrity?.merkleRoot || 'unknown',
|
||||
size: data.byteLength || data.length,
|
||||
lastAccess: Date.now(),
|
||||
cachedAt: Date.now(),
|
||||
});
|
||||
|
||||
const manifestStore = tx.objectStore('manifests');
|
||||
manifestStore.put({
|
||||
modelId,
|
||||
manifest,
|
||||
cachedAt: Date.now(),
|
||||
});
|
||||
|
||||
tx.oncomplete = () => resolve();
|
||||
tx.onerror = () => reject(tx.error);
|
||||
});
|
||||
}
|
||||
|
||||
async getChunk(modelId, chunkIndex) {
|
||||
await this.open();
|
||||
|
||||
return new Promise((resolve, reject) => {
|
||||
const tx = this.db.transaction('chunks', 'readonly');
|
||||
const store = tx.objectStore('chunks');
|
||||
const request = store.get(`${modelId}:${chunkIndex}`);
|
||||
|
||||
request.onsuccess = () => resolve(request.result?.data);
|
||||
request.onerror = () => reject(request.error);
|
||||
});
|
||||
}
|
||||
|
||||
async putChunk(modelId, chunkIndex, data, hash) {
|
||||
await this.open();
|
||||
|
||||
return new Promise((resolve, reject) => {
|
||||
const tx = this.db.transaction('chunks', 'readwrite');
|
||||
const store = tx.objectStore('chunks');
|
||||
store.put({
|
||||
id: `${modelId}:${chunkIndex}`,
|
||||
modelId,
|
||||
chunkIndex,
|
||||
data,
|
||||
hash,
|
||||
cachedAt: Date.now(),
|
||||
});
|
||||
|
||||
tx.oncomplete = () => resolve();
|
||||
tx.onerror = () => reject(tx.error);
|
||||
});
|
||||
}
|
||||
|
||||
async updateLastAccess(modelId) {
|
||||
await this.open();
|
||||
|
||||
return new Promise((resolve) => {
|
||||
const tx = this.db.transaction('models', 'readwrite');
|
||||
const store = tx.objectStore('models');
|
||||
const request = store.get(modelId);
|
||||
|
||||
request.onsuccess = () => {
|
||||
if (request.result) {
|
||||
request.result.lastAccess = Date.now();
|
||||
store.put(request.result);
|
||||
}
|
||||
resolve();
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
async ensureSpace(needed) {
|
||||
await this.open();
|
||||
|
||||
// Get current usage
|
||||
const estimate = await navigator.storage?.estimate?.();
|
||||
const used = estimate?.usage || 0;
|
||||
|
||||
if (used + needed > this.maxCacheSize) {
|
||||
await this.evictLRU(needed);
|
||||
}
|
||||
}
|
||||
|
||||
async evictLRU(needed) {
|
||||
await this.open();
|
||||
|
||||
return new Promise((resolve, reject) => {
|
||||
const tx = this.db.transaction('models', 'readwrite');
|
||||
const store = tx.objectStore('models');
|
||||
const index = store.index('lastAccess');
|
||||
const request = index.openCursor();
|
||||
|
||||
let freed = 0;
|
||||
|
||||
request.onsuccess = (event) => {
|
||||
const cursor = event.target.result;
|
||||
if (cursor && freed < needed) {
|
||||
freed += cursor.value.size || 0;
|
||||
cursor.delete();
|
||||
cursor.continue();
|
||||
} else {
|
||||
resolve(freed);
|
||||
}
|
||||
};
|
||||
request.onerror = () => reject(request.error);
|
||||
});
|
||||
}
|
||||
|
||||
async getCacheStats() {
|
||||
await this.open();
|
||||
|
||||
return new Promise((resolve, reject) => {
|
||||
const tx = this.db.transaction('models', 'readonly');
|
||||
const store = tx.objectStore('models');
|
||||
const request = store.getAll();
|
||||
|
||||
request.onsuccess = () => {
|
||||
const models = request.result;
|
||||
const totalSize = models.reduce((sum, m) => sum + (m.size || 0), 0);
|
||||
resolve({
|
||||
modelCount: models.length,
|
||||
totalSize,
|
||||
models: models.map(m => ({
|
||||
id: m.id,
|
||||
size: m.size,
|
||||
lastAccess: m.lastAccess,
|
||||
})),
|
||||
});
|
||||
};
|
||||
request.onerror = () => reject(request.error);
|
||||
});
|
||||
}
|
||||
|
||||
async clear() {
|
||||
await this.open();
|
||||
|
||||
return new Promise((resolve, reject) => {
|
||||
const tx = this.db.transaction(['models', 'chunks', 'manifests'], 'readwrite');
|
||||
tx.objectStore('models').clear();
|
||||
tx.objectStore('chunks').clear();
|
||||
tx.objectStore('manifests').clear();
|
||||
tx.oncomplete = () => resolve();
|
||||
tx.onerror = () => reject(tx.error);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// MODEL LOADER
|
||||
// ============================================================================
|
||||
|
||||
/**
|
||||
* Model loader with tiered selection and chunk verification
|
||||
*/
|
||||
export class ModelLoader {
|
||||
constructor(options = {}) {
|
||||
this.cache = new ModelCache(options.cache);
|
||||
this.verifier = new ManifestVerifier(options.trustRoot);
|
||||
this.registryUrl = options.registryUrl || 'https://models.ruvector.dev';
|
||||
|
||||
// Loading state
|
||||
this.loadingModels = new Map();
|
||||
this.loadedModels = new Map();
|
||||
|
||||
// Callbacks
|
||||
this.onProgress = options.onProgress || (() => {});
|
||||
this.onError = options.onError || console.error;
|
||||
|
||||
// Source preference order
|
||||
this.sourceOrder = options.sourceOrder || ['cache', 'gcs', 'ipfs', 'p2p'];
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch manifest from registry (registry only returns manifest, not URLs)
|
||||
*/
|
||||
async fetchManifest(modelId) {
|
||||
// Check cache first
|
||||
const cached = await this.cache.get(modelId);
|
||||
if (cached?.manifest) {
|
||||
return cached.manifest;
|
||||
}
|
||||
|
||||
// Fetch from registry
|
||||
const response = await fetch(`${this.registryUrl}/v2/manifests/${modelId}`);
|
||||
if (!response.ok) {
|
||||
throw new Error(`Failed to fetch manifest: ${response.status}`);
|
||||
}
|
||||
|
||||
const manifest = await response.json();
|
||||
|
||||
// Verify manifest
|
||||
const verification = this.verifier.verify(manifest);
|
||||
if (!verification.valid) {
|
||||
throw new Error(`Invalid manifest: ${verification.errors.join(', ')}`);
|
||||
}
|
||||
|
||||
if (verification.warnings.length > 0) {
|
||||
console.warn('[ModelLoader] Manifest warnings:', verification.warnings);
|
||||
}
|
||||
|
||||
return manifest;
|
||||
}
|
||||
|
||||
/**
|
||||
* Select best model for required capabilities
|
||||
*/
|
||||
async selectModel(requiredCapabilities, options = {}) {
|
||||
const tier = selectTier(requiredCapabilities, options.preferredTier);
|
||||
|
||||
// Fetch model catalog for this tier
|
||||
const response = await fetch(`${this.registryUrl}/v2/catalog?tier=${tier.name}`);
|
||||
if (!response.ok) {
|
||||
throw new Error(`Failed to fetch catalog: ${response.status}`);
|
||||
}
|
||||
|
||||
const catalog = await response.json();
|
||||
|
||||
// Filter by capabilities
|
||||
const candidates = catalog.models.filter(m => {
|
||||
const hasCapabilities = requiredCapabilities.every(cap =>
|
||||
m.capabilities?.includes(cap)
|
||||
);
|
||||
const fitsMemory = m.memoryRequirement <= detectAvailableMemory();
|
||||
return hasCapabilities && fitsMemory;
|
||||
});
|
||||
|
||||
if (candidates.length === 0) {
|
||||
throw new Error(`No model found for capabilities: ${requiredCapabilities.join(', ')}`);
|
||||
}
|
||||
|
||||
// Sort by capability priority (prefer embeddings over generation)
|
||||
candidates.sort((a, b) => {
|
||||
const aPriority = Math.min(...a.capabilities.map(c => CAPABILITY_PRIORITIES[c] || 10));
|
||||
const bPriority = Math.min(...b.capabilities.map(c => CAPABILITY_PRIORITIES[c] || 10));
|
||||
return aPriority - bPriority;
|
||||
});
|
||||
|
||||
return candidates[0];
|
||||
}
|
||||
|
||||
/**
|
||||
* Load a model with chunk verification
|
||||
*/
|
||||
async load(modelId, options = {}) {
|
||||
// Return if already loaded
|
||||
if (this.loadedModels.has(modelId)) {
|
||||
return this.loadedModels.get(modelId);
|
||||
}
|
||||
|
||||
// Return existing promise if loading
|
||||
if (this.loadingModels.has(modelId)) {
|
||||
return this.loadingModels.get(modelId);
|
||||
}
|
||||
|
||||
const loadPromise = this._loadInternal(modelId, options);
|
||||
this.loadingModels.set(modelId, loadPromise);
|
||||
|
||||
try {
|
||||
const result = await loadPromise;
|
||||
this.loadedModels.set(modelId, result);
|
||||
return result;
|
||||
} finally {
|
||||
this.loadingModels.delete(modelId);
|
||||
}
|
||||
}
|
||||
|
||||
async _loadInternal(modelId, options) {
|
||||
// 1. Get manifest
|
||||
const manifest = await this.fetchManifest(modelId);
|
||||
|
||||
// 2. Memory check
|
||||
const available = detectAvailableMemory();
|
||||
if (manifest.model.memoryRequirement > available) {
|
||||
throw new Error(
|
||||
`Insufficient memory: need ${manifest.model.memoryRequirement}, have ${available}`
|
||||
);
|
||||
}
|
||||
|
||||
// 3. Check cache
|
||||
const cached = await this.cache.get(modelId);
|
||||
if (cached?.data) {
|
||||
// Verify cached data against manifest
|
||||
if (cached.hash === manifest.integrity?.merkleRoot) {
|
||||
this.onProgress({ modelId, status: 'cached', progress: 1 });
|
||||
return { manifest, data: cached.data, source: 'cache' };
|
||||
}
|
||||
// Cache invalid, continue to download
|
||||
}
|
||||
|
||||
// 4. Download with chunk verification
|
||||
const artifact = manifest.artifacts[0]; // Primary artifact
|
||||
const data = await this._downloadWithVerification(modelId, manifest, artifact, options);
|
||||
|
||||
// 5. Cache the result
|
||||
await this.cache.put(modelId, data, manifest);
|
||||
|
||||
return { manifest, data, source: options.source || 'remote' };
|
||||
}
|
||||
|
||||
/**
|
||||
* Download with streaming chunk verification
|
||||
*/
|
||||
async _downloadWithVerification(modelId, manifest, artifact, options) {
|
||||
const sources = this._getSourceUrls(manifest, artifact);
|
||||
|
||||
for (const source of sources) {
|
||||
try {
|
||||
const data = await this._downloadFromSource(
|
||||
modelId,
|
||||
source,
|
||||
manifest,
|
||||
artifact
|
||||
);
|
||||
return data;
|
||||
} catch (error) {
|
||||
console.warn(`[ModelLoader] Source failed: ${source.type}`, error.message);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
throw new Error('All download sources failed');
|
||||
}
|
||||
|
||||
/**
|
||||
* Get ordered source URLs from manifest
|
||||
*/
|
||||
_getSourceUrls(manifest, artifact) {
|
||||
const sources = [];
|
||||
const dist = manifest.distribution || {};
|
||||
|
||||
for (const sourceType of this.sourceOrder) {
|
||||
if (sourceType === 'gcs' && dist.gcs) {
|
||||
sources.push({ type: 'gcs', url: dist.gcs });
|
||||
}
|
||||
if (sourceType === 'ipfs' && dist.ipfs) {
|
||||
sources.push({
|
||||
type: 'ipfs',
|
||||
url: `https://ipfs.io/ipfs/${dist.ipfs}`,
|
||||
cid: dist.ipfs,
|
||||
});
|
||||
}
|
||||
if (sourceType === 'p2p') {
|
||||
// P2P would be handled separately
|
||||
sources.push({ type: 'p2p', url: null });
|
||||
}
|
||||
}
|
||||
|
||||
// Add fallbacks
|
||||
if (dist.fallbackUrls) {
|
||||
for (const url of dist.fallbackUrls) {
|
||||
sources.push({ type: 'fallback', url });
|
||||
}
|
||||
}
|
||||
|
||||
return sources;
|
||||
}
|
||||
|
||||
/**
|
||||
* Download from a specific source with chunk verification
|
||||
*/
|
||||
async _downloadFromSource(modelId, source, manifest, artifact) {
|
||||
if (source.type === 'p2p') {
|
||||
return this._downloadFromP2P(modelId, manifest, artifact);
|
||||
}
|
||||
|
||||
const response = await fetch(source.url);
|
||||
if (!response.ok) {
|
||||
throw new Error(`HTTP ${response.status}`);
|
||||
}
|
||||
|
||||
const contentLength = parseInt(response.headers.get('content-length') || '0');
|
||||
const chunking = manifest.integrity?.chunking;
|
||||
|
||||
if (chunking && response.body) {
|
||||
// Streaming download with chunk verification
|
||||
return this._streamWithVerification(
|
||||
modelId,
|
||||
response.body,
|
||||
manifest,
|
||||
contentLength
|
||||
);
|
||||
} else {
|
||||
// Simple download
|
||||
const buffer = await response.arrayBuffer();
|
||||
|
||||
// Verify full file hash
|
||||
if (artifact.sha256) {
|
||||
const hash = createHash('sha256')
|
||||
.update(Buffer.from(buffer))
|
||||
.digest('hex');
|
||||
if (hash !== artifact.sha256) {
|
||||
throw new Error('File hash mismatch');
|
||||
}
|
||||
}
|
||||
|
||||
return buffer;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Stream download with chunk-by-chunk verification
|
||||
*/
|
||||
async _streamWithVerification(modelId, body, manifest, totalSize) {
|
||||
const chunking = manifest.integrity.chunking;
|
||||
const chunkSize = chunking.chunkSize;
|
||||
const expectedChunks = chunking.chunkCount;
|
||||
|
||||
const reader = body.getReader();
|
||||
const chunks = [];
|
||||
let buffer = new Uint8Array(0);
|
||||
let chunkIndex = 0;
|
||||
let bytesReceived = 0;
|
||||
|
||||
while (true) {
|
||||
const { done, value } = await reader.read();
|
||||
|
||||
if (done) break;
|
||||
|
||||
// Append to buffer
|
||||
const newBuffer = new Uint8Array(buffer.length + value.length);
|
||||
newBuffer.set(buffer);
|
||||
newBuffer.set(value, buffer.length);
|
||||
buffer = newBuffer;
|
||||
bytesReceived += value.length;
|
||||
|
||||
// Process complete chunks
|
||||
while (buffer.length >= chunkSize || (bytesReceived === totalSize && buffer.length > 0)) {
|
||||
const isLastChunk = bytesReceived === totalSize && buffer.length <= chunkSize;
|
||||
const thisChunkSize = isLastChunk ? buffer.length : chunkSize;
|
||||
const chunkData = buffer.slice(0, thisChunkSize);
|
||||
buffer = buffer.slice(thisChunkSize);
|
||||
|
||||
// Verify chunk
|
||||
const verification = this.verifier.verifyChunk(chunkData, chunkIndex, manifest);
|
||||
if (!verification.valid) {
|
||||
throw new Error(`Chunk verification failed: ${verification.error}`);
|
||||
}
|
||||
|
||||
chunks.push(chunkData);
|
||||
chunkIndex++;
|
||||
|
||||
// Cache chunk for resume capability
|
||||
await this.cache.putChunk(
|
||||
modelId,
|
||||
chunkIndex - 1,
|
||||
chunkData,
|
||||
verification.hash
|
||||
);
|
||||
|
||||
// Progress callback
|
||||
this.onProgress({
|
||||
modelId,
|
||||
status: 'downloading',
|
||||
progress: bytesReceived / totalSize,
|
||||
chunksVerified: chunkIndex,
|
||||
totalChunks: expectedChunks,
|
||||
});
|
||||
|
||||
if (isLastChunk) break;
|
||||
}
|
||||
}
|
||||
|
||||
// Verify Merkle root
|
||||
const chunkHashes = chunking.chunkHashes;
|
||||
const computedRoot = computeMerkleRoot(chunkHashes);
|
||||
if (computedRoot !== manifest.integrity.merkleRoot) {
|
||||
throw new Error('Merkle root verification failed');
|
||||
}
|
||||
|
||||
// Combine chunks
|
||||
const totalLength = chunks.reduce((sum, c) => sum + c.length, 0);
|
||||
const result = new Uint8Array(totalLength);
|
||||
let offset = 0;
|
||||
for (const chunk of chunks) {
|
||||
result.set(chunk, offset);
|
||||
offset += chunk.length;
|
||||
}
|
||||
|
||||
this.onProgress({
|
||||
modelId,
|
||||
status: 'complete',
|
||||
progress: 1,
|
||||
verified: true,
|
||||
});
|
||||
|
||||
return result.buffer;
|
||||
}
|
||||
|
||||
/**
|
||||
* Download from P2P network (placeholder)
|
||||
*/
|
||||
async _downloadFromP2P(modelId, manifest, artifact) {
|
||||
// Would integrate with WebRTC P2P network
|
||||
throw new Error('P2P download not implemented');
|
||||
}
|
||||
|
||||
/**
|
||||
* Preload a model in the background
|
||||
*/
|
||||
async preload(modelId) {
|
||||
try {
|
||||
await this.load(modelId);
|
||||
} catch (error) {
|
||||
console.warn(`[ModelLoader] Preload failed for ${modelId}:`, error.message);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Unload a model from memory
|
||||
*/
|
||||
unload(modelId) {
|
||||
this.loadedModels.delete(modelId);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get cache statistics
|
||||
*/
|
||||
async getCacheStats() {
|
||||
return this.cache.getCacheStats();
|
||||
}
|
||||
|
||||
/**
|
||||
* Clear all cached models
|
||||
*/
|
||||
async clearCache() {
|
||||
await this.cache.clear();
|
||||
this.loadedModels.clear();
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// EXPORTS
|
||||
// ============================================================================
|
||||
|
||||
export default ModelLoader;
|
||||
1298
vendor/ruvector/examples/edge-net/pkg/models/microlora.js
vendored
Normal file
1298
vendor/ruvector/examples/edge-net/pkg/models/microlora.js
vendored
Normal file
File diff suppressed because it is too large
Load Diff
922
vendor/ruvector/examples/edge-net/pkg/models/model-loader.js
vendored
Normal file
922
vendor/ruvector/examples/edge-net/pkg/models/model-loader.js
vendored
Normal file
@@ -0,0 +1,922 @@
|
||||
/**
|
||||
* @ruvector/edge-net Model Loader
|
||||
*
|
||||
* Smart model loading with:
|
||||
* - IndexedDB caching
|
||||
* - Automatic source selection (CDN -> GCS -> IPFS -> fallback)
|
||||
* - Streaming download with progress
|
||||
* - Model validation before use
|
||||
* - Lazy loading support
|
||||
*
|
||||
* @module @ruvector/edge-net/models/model-loader
|
||||
*/
|
||||
|
||||
import { EventEmitter } from 'events';
|
||||
import { createHash, randomBytes } from 'crypto';
|
||||
import { promises as fs } from 'fs';
|
||||
import path from 'path';
|
||||
|
||||
import { ModelRegistry } from './model-registry.js';
|
||||
import { DistributionManager, ProgressTracker } from './distribution.js';
|
||||
|
||||
// ============================================
|
||||
// CONSTANTS
|
||||
// ============================================
|
||||
|
||||
const DEFAULT_CACHE_DIR = process.env.HOME
|
||||
? `${process.env.HOME}/.ruvector/models/cache`
|
||||
: '/tmp/.ruvector/models/cache';
|
||||
|
||||
const CACHE_VERSION = 1;
|
||||
const MAX_CACHE_SIZE_BYTES = 10 * 1024 * 1024 * 1024; // 10GB default
|
||||
const CACHE_CLEANUP_THRESHOLD = 0.9; // Cleanup when 90% full
|
||||
|
||||
// ============================================
|
||||
// CACHE STORAGE INTERFACE
|
||||
// ============================================
|
||||
|
||||
/**
|
||||
* Cache storage interface for different backends
|
||||
*/
|
||||
class CacheStorage {
|
||||
async get(key) { throw new Error('Not implemented'); }
|
||||
async set(key, value, metadata) { throw new Error('Not implemented'); }
|
||||
async delete(key) { throw new Error('Not implemented'); }
|
||||
async has(key) { throw new Error('Not implemented'); }
|
||||
async list() { throw new Error('Not implemented'); }
|
||||
async getMetadata(key) { throw new Error('Not implemented'); }
|
||||
async clear() { throw new Error('Not implemented'); }
|
||||
async getSize() { throw new Error('Not implemented'); }
|
||||
}
|
||||
|
||||
// ============================================
|
||||
// FILE SYSTEM CACHE
|
||||
// ============================================
|
||||
|
||||
/**
|
||||
* File system-based cache storage for Node.js
|
||||
*/
|
||||
class FileSystemCache extends CacheStorage {
|
||||
constructor(cacheDir) {
|
||||
super();
|
||||
this.cacheDir = cacheDir;
|
||||
this.metadataDir = path.join(cacheDir, '.metadata');
|
||||
this.initialized = false;
|
||||
}
|
||||
|
||||
async init() {
|
||||
if (this.initialized) return;
|
||||
await fs.mkdir(this.cacheDir, { recursive: true });
|
||||
await fs.mkdir(this.metadataDir, { recursive: true });
|
||||
this.initialized = true;
|
||||
}
|
||||
|
||||
_getFilePath(key) {
|
||||
// Sanitize key for filesystem
|
||||
const safeKey = key.replace(/[^a-zA-Z0-9._-]/g, '_');
|
||||
return path.join(this.cacheDir, safeKey);
|
||||
}
|
||||
|
||||
_getMetadataPath(key) {
|
||||
const safeKey = key.replace(/[^a-zA-Z0-9._-]/g, '_');
|
||||
return path.join(this.metadataDir, `${safeKey}.json`);
|
||||
}
|
||||
|
||||
async get(key) {
|
||||
await this.init();
|
||||
const filePath = this._getFilePath(key);
|
||||
|
||||
try {
|
||||
const data = await fs.readFile(filePath);
|
||||
|
||||
// Update access time in metadata
|
||||
await this._updateAccessTime(key);
|
||||
|
||||
return data;
|
||||
} catch (error) {
|
||||
if (error.code === 'ENOENT') return null;
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
async set(key, value, metadata = {}) {
|
||||
await this.init();
|
||||
const filePath = this._getFilePath(key);
|
||||
const metadataPath = this._getMetadataPath(key);
|
||||
|
||||
// Write data
|
||||
await fs.writeFile(filePath, value);
|
||||
|
||||
// Write metadata
|
||||
const fullMetadata = {
|
||||
key,
|
||||
size: value.length,
|
||||
hash: `sha256:${createHash('sha256').update(value).digest('hex')}`,
|
||||
createdAt: new Date().toISOString(),
|
||||
accessedAt: new Date().toISOString(),
|
||||
accessCount: 1,
|
||||
cacheVersion: CACHE_VERSION,
|
||||
...metadata,
|
||||
};
|
||||
|
||||
await fs.writeFile(metadataPath, JSON.stringify(fullMetadata, null, 2));
|
||||
|
||||
return fullMetadata;
|
||||
}
|
||||
|
||||
async delete(key) {
|
||||
await this.init();
|
||||
const filePath = this._getFilePath(key);
|
||||
const metadataPath = this._getMetadataPath(key);
|
||||
|
||||
try {
|
||||
await fs.unlink(filePath);
|
||||
await fs.unlink(metadataPath).catch(() => {});
|
||||
return true;
|
||||
} catch (error) {
|
||||
if (error.code === 'ENOENT') return false;
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
async has(key) {
|
||||
await this.init();
|
||||
const filePath = this._getFilePath(key);
|
||||
|
||||
try {
|
||||
await fs.access(filePath);
|
||||
return true;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
async list() {
|
||||
await this.init();
|
||||
|
||||
try {
|
||||
const files = await fs.readdir(this.cacheDir);
|
||||
return files.filter(f => !f.startsWith('.'));
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
async getMetadata(key) {
|
||||
await this.init();
|
||||
const metadataPath = this._getMetadataPath(key);
|
||||
|
||||
try {
|
||||
const data = await fs.readFile(metadataPath, 'utf-8');
|
||||
return JSON.parse(data);
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
async _updateAccessTime(key) {
|
||||
const metadataPath = this._getMetadataPath(key);
|
||||
|
||||
try {
|
||||
const data = await fs.readFile(metadataPath, 'utf-8');
|
||||
const metadata = JSON.parse(data);
|
||||
|
||||
metadata.accessedAt = new Date().toISOString();
|
||||
metadata.accessCount = (metadata.accessCount || 0) + 1;
|
||||
|
||||
await fs.writeFile(metadataPath, JSON.stringify(metadata, null, 2));
|
||||
} catch {
|
||||
// Ignore metadata update errors
|
||||
}
|
||||
}
|
||||
|
||||
async clear() {
|
||||
await this.init();
|
||||
const files = await this.list();
|
||||
|
||||
for (const file of files) {
|
||||
await this.delete(file);
|
||||
}
|
||||
}
|
||||
|
||||
async getSize() {
|
||||
await this.init();
|
||||
const files = await this.list();
|
||||
let totalSize = 0;
|
||||
|
||||
for (const file of files) {
|
||||
const filePath = this._getFilePath(file);
|
||||
try {
|
||||
const stats = await fs.stat(filePath);
|
||||
totalSize += stats.size;
|
||||
} catch {
|
||||
// Ignore missing files
|
||||
}
|
||||
}
|
||||
|
||||
return totalSize;
|
||||
}
|
||||
|
||||
async getEntriesWithMetadata() {
|
||||
await this.init();
|
||||
const files = await this.list();
|
||||
const entries = [];
|
||||
|
||||
for (const file of files) {
|
||||
const metadata = await this.getMetadata(file);
|
||||
if (metadata) {
|
||||
entries.push(metadata);
|
||||
}
|
||||
}
|
||||
|
||||
return entries;
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================
|
||||
// INDEXEDDB CACHE (BROWSER)
|
||||
// ============================================
|
||||
|
||||
/**
|
||||
* IndexedDB-based cache storage for browsers
|
||||
*/
|
||||
class IndexedDBCache extends CacheStorage {
|
||||
constructor(dbName = 'ruvector-models') {
|
||||
super();
|
||||
this.dbName = dbName;
|
||||
this.storeName = 'models';
|
||||
this.metadataStoreName = 'metadata';
|
||||
this.db = null;
|
||||
}
|
||||
|
||||
async init() {
|
||||
if (this.db) return;
|
||||
|
||||
if (typeof indexedDB === 'undefined') {
|
||||
throw new Error('IndexedDB not available');
|
||||
}
|
||||
|
||||
return new Promise((resolve, reject) => {
|
||||
const request = indexedDB.open(this.dbName, CACHE_VERSION);
|
||||
|
||||
request.onerror = () => reject(request.error);
|
||||
|
||||
request.onsuccess = () => {
|
||||
this.db = request.result;
|
||||
resolve();
|
||||
};
|
||||
|
||||
request.onupgradeneeded = (event) => {
|
||||
const db = event.target.result;
|
||||
|
||||
// Models store
|
||||
if (!db.objectStoreNames.contains(this.storeName)) {
|
||||
db.createObjectStore(this.storeName);
|
||||
}
|
||||
|
||||
// Metadata store
|
||||
if (!db.objectStoreNames.contains(this.metadataStoreName)) {
|
||||
const metaStore = db.createObjectStore(this.metadataStoreName);
|
||||
metaStore.createIndex('accessedAt', 'accessedAt');
|
||||
metaStore.createIndex('size', 'size');
|
||||
}
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
async get(key) {
|
||||
await this.init();
|
||||
|
||||
return new Promise((resolve, reject) => {
|
||||
const transaction = this.db.transaction([this.storeName], 'readonly');
|
||||
const store = transaction.objectStore(this.storeName);
|
||||
const request = store.get(key);
|
||||
|
||||
request.onerror = () => reject(request.error);
|
||||
request.onsuccess = () => {
|
||||
if (request.result) {
|
||||
this._updateAccessTime(key);
|
||||
}
|
||||
resolve(request.result || null);
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
async set(key, value, metadata = {}) {
|
||||
await this.init();
|
||||
|
||||
const fullMetadata = {
|
||||
key,
|
||||
size: value.length || value.byteLength,
|
||||
hash: await this._computeHash(value),
|
||||
createdAt: new Date().toISOString(),
|
||||
accessedAt: new Date().toISOString(),
|
||||
accessCount: 1,
|
||||
cacheVersion: CACHE_VERSION,
|
||||
...metadata,
|
||||
};
|
||||
|
||||
return new Promise((resolve, reject) => {
|
||||
const transaction = this.db.transaction(
|
||||
[this.storeName, this.metadataStoreName],
|
||||
'readwrite'
|
||||
);
|
||||
|
||||
const modelStore = transaction.objectStore(this.storeName);
|
||||
const metaStore = transaction.objectStore(this.metadataStoreName);
|
||||
|
||||
modelStore.put(value, key);
|
||||
metaStore.put(fullMetadata, key);
|
||||
|
||||
transaction.oncomplete = () => resolve(fullMetadata);
|
||||
transaction.onerror = () => reject(transaction.error);
|
||||
});
|
||||
}
|
||||
|
||||
async _computeHash(data) {
|
||||
if (typeof crypto !== 'undefined' && crypto.subtle) {
|
||||
const buffer = data instanceof ArrayBuffer ? data : data.buffer;
|
||||
const hashBuffer = await crypto.subtle.digest('SHA-256', buffer);
|
||||
const hashArray = Array.from(new Uint8Array(hashBuffer));
|
||||
const hashHex = hashArray.map(b => b.toString(16).padStart(2, '0')).join('');
|
||||
return `sha256:${hashHex}`;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
async delete(key) {
|
||||
await this.init();
|
||||
|
||||
return new Promise((resolve, reject) => {
|
||||
const transaction = this.db.transaction(
|
||||
[this.storeName, this.metadataStoreName],
|
||||
'readwrite'
|
||||
);
|
||||
|
||||
transaction.objectStore(this.storeName).delete(key);
|
||||
transaction.objectStore(this.metadataStoreName).delete(key);
|
||||
|
||||
transaction.oncomplete = () => resolve(true);
|
||||
transaction.onerror = () => reject(transaction.error);
|
||||
});
|
||||
}
|
||||
|
||||
async has(key) {
|
||||
const value = await this.get(key);
|
||||
return value !== null;
|
||||
}
|
||||
|
||||
async list() {
|
||||
await this.init();
|
||||
|
||||
return new Promise((resolve, reject) => {
|
||||
const transaction = this.db.transaction([this.storeName], 'readonly');
|
||||
const store = transaction.objectStore(this.storeName);
|
||||
const request = store.getAllKeys();
|
||||
|
||||
request.onerror = () => reject(request.error);
|
||||
request.onsuccess = () => resolve(request.result);
|
||||
});
|
||||
}
|
||||
|
||||
async getMetadata(key) {
|
||||
await this.init();
|
||||
|
||||
return new Promise((resolve, reject) => {
|
||||
const transaction = this.db.transaction([this.metadataStoreName], 'readonly');
|
||||
const store = transaction.objectStore(this.metadataStoreName);
|
||||
const request = store.get(key);
|
||||
|
||||
request.onerror = () => reject(request.error);
|
||||
request.onsuccess = () => resolve(request.result || null);
|
||||
});
|
||||
}
|
||||
|
||||
async _updateAccessTime(key) {
|
||||
const metadata = await this.getMetadata(key);
|
||||
if (!metadata) return;
|
||||
|
||||
metadata.accessedAt = new Date().toISOString();
|
||||
metadata.accessCount = (metadata.accessCount || 0) + 1;
|
||||
|
||||
const transaction = this.db.transaction([this.metadataStoreName], 'readwrite');
|
||||
transaction.objectStore(this.metadataStoreName).put(metadata, key);
|
||||
}
|
||||
|
||||
async clear() {
|
||||
await this.init();
|
||||
|
||||
return new Promise((resolve, reject) => {
|
||||
const transaction = this.db.transaction(
|
||||
[this.storeName, this.metadataStoreName],
|
||||
'readwrite'
|
||||
);
|
||||
|
||||
transaction.objectStore(this.storeName).clear();
|
||||
transaction.objectStore(this.metadataStoreName).clear();
|
||||
|
||||
transaction.oncomplete = () => resolve();
|
||||
transaction.onerror = () => reject(transaction.error);
|
||||
});
|
||||
}
|
||||
|
||||
async getSize() {
|
||||
await this.init();
|
||||
|
||||
return new Promise((resolve, reject) => {
|
||||
const transaction = this.db.transaction([this.metadataStoreName], 'readonly');
|
||||
const store = transaction.objectStore(this.metadataStoreName);
|
||||
const request = store.getAll();
|
||||
|
||||
request.onerror = () => reject(request.error);
|
||||
request.onsuccess = () => {
|
||||
const totalSize = request.result.reduce((sum, meta) => sum + (meta.size || 0), 0);
|
||||
resolve(totalSize);
|
||||
};
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================
|
||||
// MODEL LOADER
|
||||
// ============================================
|
||||
|
||||
/**
|
||||
* ModelLoader - Smart model loading with caching
|
||||
*/
|
||||
export class ModelLoader extends EventEmitter {
|
||||
/**
|
||||
* Create a new ModelLoader
|
||||
* @param {object} options - Configuration options
|
||||
*/
|
||||
constructor(options = {}) {
|
||||
super();
|
||||
|
||||
this.id = `loader-${randomBytes(6).toString('hex')}`;
|
||||
|
||||
// Create registry if not provided
|
||||
this.registry = options.registry || new ModelRegistry({
|
||||
registryPath: options.registryPath,
|
||||
});
|
||||
|
||||
// Create distribution manager if not provided
|
||||
this.distribution = options.distribution || new DistributionManager({
|
||||
gcsBucket: options.gcsBucket,
|
||||
gcsProjectId: options.gcsProjectId,
|
||||
cdnBaseUrl: options.cdnBaseUrl,
|
||||
ipfsGateway: options.ipfsGateway,
|
||||
});
|
||||
|
||||
// Cache configuration
|
||||
this.cacheDir = options.cacheDir || DEFAULT_CACHE_DIR;
|
||||
this.maxCacheSize = options.maxCacheSize || MAX_CACHE_SIZE_BYTES;
|
||||
|
||||
// Initialize cache storage based on environment
|
||||
this.cache = this._createCacheStorage(options);
|
||||
|
||||
// Loaded models (in-memory)
|
||||
this.loadedModels = new Map();
|
||||
|
||||
// Loading promises (prevent duplicate loads)
|
||||
this.loadingPromises = new Map();
|
||||
|
||||
// Lazy load queue
|
||||
this.lazyLoadQueue = [];
|
||||
this.lazyLoadActive = false;
|
||||
|
||||
// Stats
|
||||
this.stats = {
|
||||
cacheHits: 0,
|
||||
cacheMisses: 0,
|
||||
downloads: 0,
|
||||
validationErrors: 0,
|
||||
lazyLoads: 0,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Create appropriate cache storage for environment
|
||||
* @private
|
||||
*/
|
||||
_createCacheStorage(options) {
|
||||
// Browser environment
|
||||
if (typeof window !== 'undefined' && typeof indexedDB !== 'undefined') {
|
||||
return new IndexedDBCache(options.dbName || 'ruvector-models');
|
||||
}
|
||||
|
||||
// Node.js environment
|
||||
return new FileSystemCache(this.cacheDir);
|
||||
}
|
||||
|
||||
/**
|
||||
* Initialize the loader
|
||||
*/
|
||||
async initialize() {
|
||||
// Initialize cache
|
||||
if (this.cache.init) {
|
||||
await this.cache.init();
|
||||
}
|
||||
|
||||
// Load registry if path provided
|
||||
if (this.registry.registryPath) {
|
||||
try {
|
||||
await this.registry.load();
|
||||
} catch (error) {
|
||||
this.emit('warning', { message: 'Failed to load registry', error });
|
||||
}
|
||||
}
|
||||
|
||||
this.emit('initialized', { loaderId: this.id });
|
||||
|
||||
return this;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get cache key for a model
|
||||
* @private
|
||||
*/
|
||||
_getCacheKey(name, version) {
|
||||
return `${name}@${version}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Load a model
|
||||
* @param {string} name - Model name
|
||||
* @param {string} version - Version (default: latest)
|
||||
* @param {object} options - Load options
|
||||
* @returns {Promise<Buffer|Uint8Array>}
|
||||
*/
|
||||
async load(name, version = 'latest', options = {}) {
|
||||
const key = this._getCacheKey(name, version);
|
||||
|
||||
// Return cached in-memory model
|
||||
if (this.loadedModels.has(key) && !options.forceReload) {
|
||||
this.stats.cacheHits++;
|
||||
return this.loadedModels.get(key);
|
||||
}
|
||||
|
||||
// Return existing loading promise
|
||||
if (this.loadingPromises.has(key)) {
|
||||
return this.loadingPromises.get(key);
|
||||
}
|
||||
|
||||
// Start loading
|
||||
const loadPromise = this._loadModel(name, version, options);
|
||||
this.loadingPromises.set(key, loadPromise);
|
||||
|
||||
try {
|
||||
const model = await loadPromise;
|
||||
this.loadedModels.set(key, model);
|
||||
return model;
|
||||
} finally {
|
||||
this.loadingPromises.delete(key);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Internal model loading logic
|
||||
* @private
|
||||
*/
|
||||
async _loadModel(name, version, options = {}) {
|
||||
const { onProgress, skipCache = false, skipValidation = false } = options;
|
||||
|
||||
// Get metadata from registry
|
||||
let metadata = this.registry.get(name, version);
|
||||
|
||||
if (!metadata) {
|
||||
// Try to fetch from remote registry
|
||||
this.emit('warning', { message: `Model ${name}@${version} not in local registry` });
|
||||
throw new Error(`Model not found: ${name}@${version}`);
|
||||
}
|
||||
|
||||
const resolvedVersion = metadata.version;
|
||||
const key = this._getCacheKey(name, resolvedVersion);
|
||||
|
||||
// Check cache first (unless skipped)
|
||||
if (!skipCache) {
|
||||
const cached = await this.cache.get(key);
|
||||
if (cached) {
|
||||
// Validate cached data
|
||||
if (!skipValidation && metadata.hash) {
|
||||
const isValid = this.distribution.verifyIntegrity(cached, metadata.hash);
|
||||
if (isValid) {
|
||||
this.stats.cacheHits++;
|
||||
this.emit('cache_hit', { name, version: resolvedVersion });
|
||||
return cached;
|
||||
} else {
|
||||
this.stats.validationErrors++;
|
||||
this.emit('cache_invalid', { name, version: resolvedVersion });
|
||||
await this.cache.delete(key);
|
||||
}
|
||||
} else {
|
||||
this.stats.cacheHits++;
|
||||
return cached;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
this.stats.cacheMisses++;
|
||||
|
||||
// Download model
|
||||
this.emit('download_start', { name, version: resolvedVersion });
|
||||
|
||||
const data = await this.distribution.download(metadata, {
|
||||
onProgress: (progress) => {
|
||||
this.emit('progress', { name, version: resolvedVersion, ...progress });
|
||||
if (onProgress) onProgress(progress);
|
||||
},
|
||||
});
|
||||
|
||||
// Validate downloaded data
|
||||
if (!skipValidation) {
|
||||
const validation = this.distribution.verifyModel(data, metadata);
|
||||
if (!validation.valid) {
|
||||
this.stats.validationErrors++;
|
||||
throw new Error(`Model validation failed: ${JSON.stringify(validation.checks)}`);
|
||||
}
|
||||
}
|
||||
|
||||
// Store in cache
|
||||
await this.cache.set(key, data, {
|
||||
modelName: name,
|
||||
version: resolvedVersion,
|
||||
format: metadata.format,
|
||||
});
|
||||
|
||||
this.stats.downloads++;
|
||||
this.emit('loaded', { name, version: resolvedVersion, size: data.length });
|
||||
|
||||
// Cleanup cache if needed
|
||||
await this._cleanupCacheIfNeeded();
|
||||
|
||||
return data;
|
||||
}
|
||||
|
||||
/**
|
||||
* Lazy load a model (load in background)
|
||||
* @param {string} name - Model name
|
||||
* @param {string} version - Version
|
||||
* @param {object} options - Load options
|
||||
*/
|
||||
async lazyLoad(name, version = 'latest', options = {}) {
|
||||
const key = this._getCacheKey(name, version);
|
||||
|
||||
// Already loaded or loading
|
||||
if (this.loadedModels.has(key) || this.loadingPromises.has(key)) {
|
||||
return;
|
||||
}
|
||||
|
||||
// Check cache
|
||||
const cached = await this.cache.has(key);
|
||||
if (cached) {
|
||||
return; // Already in cache
|
||||
}
|
||||
|
||||
// Add to queue
|
||||
this.lazyLoadQueue.push({ name, version, options });
|
||||
this.stats.lazyLoads++;
|
||||
|
||||
this.emit('lazy_queued', { name, version, queueLength: this.lazyLoadQueue.length });
|
||||
|
||||
// Start processing if not active
|
||||
if (!this.lazyLoadActive) {
|
||||
this._processLazyLoadQueue();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Process lazy load queue
|
||||
* @private
|
||||
*/
|
||||
async _processLazyLoadQueue() {
|
||||
if (this.lazyLoadActive || this.lazyLoadQueue.length === 0) return;
|
||||
|
||||
this.lazyLoadActive = true;
|
||||
|
||||
while (this.lazyLoadQueue.length > 0) {
|
||||
const { name, version, options } = this.lazyLoadQueue.shift();
|
||||
|
||||
try {
|
||||
await this.load(name, version, {
|
||||
...options,
|
||||
lazy: true,
|
||||
});
|
||||
} catch (error) {
|
||||
this.emit('lazy_error', { name, version, error: error.message });
|
||||
}
|
||||
|
||||
// Small delay between lazy loads
|
||||
await new Promise(resolve => setTimeout(resolve, 100));
|
||||
}
|
||||
|
||||
this.lazyLoadActive = false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Preload multiple models
|
||||
* @param {Array<{name: string, version?: string}>} models - Models to preload
|
||||
*/
|
||||
async preload(models) {
|
||||
const results = await Promise.allSettled(
|
||||
models.map(({ name, version }) => this.load(name, version || 'latest'))
|
||||
);
|
||||
|
||||
return {
|
||||
total: models.length,
|
||||
loaded: results.filter(r => r.status === 'fulfilled').length,
|
||||
failed: results.filter(r => r.status === 'rejected').length,
|
||||
results,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if a model is loaded in memory
|
||||
* @param {string} name - Model name
|
||||
* @param {string} version - Version
|
||||
* @returns {boolean}
|
||||
*/
|
||||
isLoaded(name, version = 'latest') {
|
||||
const metadata = this.registry.get(name, version);
|
||||
if (!metadata) return false;
|
||||
|
||||
const key = this._getCacheKey(name, metadata.version);
|
||||
return this.loadedModels.has(key);
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if a model is cached on disk
|
||||
* @param {string} name - Model name
|
||||
* @param {string} version - Version
|
||||
* @returns {Promise<boolean>}
|
||||
*/
|
||||
async isCached(name, version = 'latest') {
|
||||
const metadata = this.registry.get(name, version);
|
||||
if (!metadata) return false;
|
||||
|
||||
const key = this._getCacheKey(name, metadata.version);
|
||||
return this.cache.has(key);
|
||||
}
|
||||
|
||||
/**
|
||||
* Unload a model from memory
|
||||
* @param {string} name - Model name
|
||||
* @param {string} version - Version
|
||||
*/
|
||||
unload(name, version = 'latest') {
|
||||
const metadata = this.registry.get(name, version);
|
||||
if (!metadata) return false;
|
||||
|
||||
const key = this._getCacheKey(name, metadata.version);
|
||||
return this.loadedModels.delete(key);
|
||||
}
|
||||
|
||||
/**
|
||||
* Unload all models from memory
|
||||
*/
|
||||
unloadAll() {
|
||||
const count = this.loadedModels.size;
|
||||
this.loadedModels.clear();
|
||||
return count;
|
||||
}
|
||||
|
||||
/**
|
||||
* Remove a model from cache
|
||||
* @param {string} name - Model name
|
||||
* @param {string} version - Version
|
||||
*/
|
||||
async removeFromCache(name, version = 'latest') {
|
||||
const metadata = this.registry.get(name, version);
|
||||
if (!metadata) return false;
|
||||
|
||||
const key = this._getCacheKey(name, metadata.version);
|
||||
return this.cache.delete(key);
|
||||
}
|
||||
|
||||
/**
|
||||
* Clear all cached models
|
||||
*/
|
||||
async clearCache() {
|
||||
await this.cache.clear();
|
||||
this.emit('cache_cleared');
|
||||
}
|
||||
|
||||
/**
|
||||
* Cleanup cache if over size limit
|
||||
* @private
|
||||
*/
|
||||
async _cleanupCacheIfNeeded() {
|
||||
const currentSize = await this.cache.getSize();
|
||||
const threshold = this.maxCacheSize * CACHE_CLEANUP_THRESHOLD;
|
||||
|
||||
if (currentSize < threshold) return;
|
||||
|
||||
this.emit('cache_cleanup_start', { currentSize, maxSize: this.maxCacheSize });
|
||||
|
||||
// Get entries sorted by last access time
|
||||
let entries;
|
||||
if (this.cache.getEntriesWithMetadata) {
|
||||
entries = await this.cache.getEntriesWithMetadata();
|
||||
} else {
|
||||
const keys = await this.cache.list();
|
||||
entries = [];
|
||||
for (const key of keys) {
|
||||
const meta = await this.cache.getMetadata(key);
|
||||
if (meta) entries.push(meta);
|
||||
}
|
||||
}
|
||||
|
||||
// Sort by access time (oldest first)
|
||||
entries.sort((a, b) =>
|
||||
new Date(a.accessedAt).getTime() - new Date(b.accessedAt).getTime()
|
||||
);
|
||||
|
||||
// Remove oldest entries until under 80% capacity
|
||||
const targetSize = this.maxCacheSize * 0.8;
|
||||
let removedSize = 0;
|
||||
let removedCount = 0;
|
||||
|
||||
for (const entry of entries) {
|
||||
if (currentSize - removedSize <= targetSize) break;
|
||||
|
||||
await this.cache.delete(entry.key);
|
||||
removedSize += entry.size;
|
||||
removedCount++;
|
||||
}
|
||||
|
||||
this.emit('cache_cleanup_complete', {
|
||||
removedCount,
|
||||
removedSize,
|
||||
newSize: currentSize - removedSize,
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Get cache statistics
|
||||
* @returns {Promise<object>}
|
||||
*/
|
||||
async getCacheStats() {
|
||||
const size = await this.cache.getSize();
|
||||
const keys = await this.cache.list();
|
||||
|
||||
return {
|
||||
entries: keys.length,
|
||||
sizeBytes: size,
|
||||
sizeMB: Math.round(size / (1024 * 1024) * 100) / 100,
|
||||
maxSizeBytes: this.maxCacheSize,
|
||||
usagePercent: Math.round((size / this.maxCacheSize) * 100),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Get loader statistics
|
||||
* @returns {object}
|
||||
*/
|
||||
getStats() {
|
||||
return {
|
||||
...this.stats,
|
||||
loadedModels: this.loadedModels.size,
|
||||
pendingLoads: this.loadingPromises.size,
|
||||
lazyQueueLength: this.lazyLoadQueue.length,
|
||||
hitRate: this.stats.cacheHits + this.stats.cacheMisses > 0
|
||||
? Math.round(
|
||||
(this.stats.cacheHits / (this.stats.cacheHits + this.stats.cacheMisses)) * 100
|
||||
)
|
||||
: 0,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Get model from registry (without loading)
|
||||
* @param {string} name - Model name
|
||||
* @param {string} version - Version
|
||||
* @returns {object|null}
|
||||
*/
|
||||
getModelInfo(name, version = 'latest') {
|
||||
return this.registry.get(name, version);
|
||||
}
|
||||
|
||||
/**
|
||||
* Search for models
|
||||
* @param {object} criteria - Search criteria
|
||||
* @returns {Array}
|
||||
*/
|
||||
searchModels(criteria) {
|
||||
return this.registry.search(criteria);
|
||||
}
|
||||
|
||||
/**
|
||||
* List all available models
|
||||
* @returns {string[]}
|
||||
*/
|
||||
listModels() {
|
||||
return this.registry.listModels();
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================
|
||||
// EXPORTS
|
||||
// ============================================
|
||||
|
||||
export { FileSystemCache, IndexedDBCache, CacheStorage };
|
||||
|
||||
export default ModelLoader;
|
||||
1245
vendor/ruvector/examples/edge-net/pkg/models/model-optimizer.js
vendored
Normal file
1245
vendor/ruvector/examples/edge-net/pkg/models/model-optimizer.js
vendored
Normal file
File diff suppressed because it is too large
Load Diff
696
vendor/ruvector/examples/edge-net/pkg/models/model-registry.js
vendored
Normal file
696
vendor/ruvector/examples/edge-net/pkg/models/model-registry.js
vendored
Normal file
@@ -0,0 +1,696 @@
|
||||
/**
|
||||
* @ruvector/edge-net Model Registry
|
||||
*
|
||||
* Manages model metadata, versions, dependencies, and discovery
|
||||
* for the distributed model distribution infrastructure.
|
||||
*
|
||||
* @module @ruvector/edge-net/models/model-registry
|
||||
*/
|
||||
|
||||
import { EventEmitter } from 'events';
|
||||
import { createHash, randomBytes } from 'crypto';
|
||||
import { promises as fs } from 'fs';
|
||||
import path from 'path';
|
||||
|
||||
// ============================================
|
||||
// SEMVER UTILITIES
|
||||
// ============================================
|
||||
|
||||
/**
|
||||
* Parse a semver version string
|
||||
* @param {string} version - Version string (e.g., "1.2.3", "1.0.0-beta.1")
|
||||
* @returns {object} Parsed version object
|
||||
*/
|
||||
export function parseSemver(version) {
|
||||
const match = String(version).match(
|
||||
/^(\d+)\.(\d+)\.(\d+)(?:-([a-zA-Z0-9.-]+))?(?:\+([a-zA-Z0-9.-]+))?$/
|
||||
);
|
||||
|
||||
if (!match) {
|
||||
throw new Error(`Invalid semver: ${version}`);
|
||||
}
|
||||
|
||||
return {
|
||||
major: parseInt(match[1], 10),
|
||||
minor: parseInt(match[2], 10),
|
||||
patch: parseInt(match[3], 10),
|
||||
prerelease: match[4] || null,
|
||||
build: match[5] || null,
|
||||
raw: version,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Compare two semver versions
|
||||
* @param {string} a - First version
|
||||
* @param {string} b - Second version
|
||||
* @returns {number} -1 if a < b, 0 if equal, 1 if a > b
|
||||
*/
|
||||
export function compareSemver(a, b) {
|
||||
const va = parseSemver(a);
|
||||
const vb = parseSemver(b);
|
||||
|
||||
if (va.major !== vb.major) return va.major - vb.major;
|
||||
if (va.minor !== vb.minor) return va.minor - vb.minor;
|
||||
if (va.patch !== vb.patch) return va.patch - vb.patch;
|
||||
|
||||
// Prerelease versions have lower precedence
|
||||
if (va.prerelease && !vb.prerelease) return -1;
|
||||
if (!va.prerelease && vb.prerelease) return 1;
|
||||
if (va.prerelease && vb.prerelease) {
|
||||
return va.prerelease.localeCompare(vb.prerelease);
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if version satisfies a version range
|
||||
* Supports: "1.0.0", "^1.0.0", "~1.0.0", ">=1.0.0", "1.x", "*"
|
||||
* @param {string} version - Version to check
|
||||
* @param {string} range - Version range
|
||||
* @returns {boolean}
|
||||
*/
|
||||
export function satisfiesSemver(version, range) {
|
||||
const v = parseSemver(version);
|
||||
|
||||
// Exact match
|
||||
if (range === version) return true;
|
||||
|
||||
// Wildcard
|
||||
if (range === '*' || range === 'latest') return true;
|
||||
|
||||
// X-range: 1.x, 1.2.x
|
||||
const xMatch = range.match(/^(\d+)(?:\.(\d+))?\.x$/);
|
||||
if (xMatch) {
|
||||
const major = parseInt(xMatch[1], 10);
|
||||
const minor = xMatch[2] ? parseInt(xMatch[2], 10) : null;
|
||||
if (v.major !== major) return false;
|
||||
if (minor !== null && v.minor !== minor) return false;
|
||||
return true;
|
||||
}
|
||||
|
||||
// Caret range: ^1.0.0 (compatible with)
|
||||
if (range.startsWith('^')) {
|
||||
const r = parseSemver(range.slice(1));
|
||||
if (v.major !== r.major) return false;
|
||||
if (v.major === 0) {
|
||||
if (v.minor !== r.minor) return false;
|
||||
return v.patch >= r.patch;
|
||||
}
|
||||
return compareSemver(version, range.slice(1)) >= 0;
|
||||
}
|
||||
|
||||
// Tilde range: ~1.0.0 (approximately equivalent)
|
||||
if (range.startsWith('~')) {
|
||||
const r = parseSemver(range.slice(1));
|
||||
if (v.major !== r.major) return false;
|
||||
if (v.minor !== r.minor) return false;
|
||||
return v.patch >= r.patch;
|
||||
}
|
||||
|
||||
// Comparison ranges: >=1.0.0, >1.0.0, <=1.0.0, <1.0.0
|
||||
if (range.startsWith('>=')) {
|
||||
return compareSemver(version, range.slice(2)) >= 0;
|
||||
}
|
||||
if (range.startsWith('>')) {
|
||||
return compareSemver(version, range.slice(1)) > 0;
|
||||
}
|
||||
if (range.startsWith('<=')) {
|
||||
return compareSemver(version, range.slice(2)) <= 0;
|
||||
}
|
||||
if (range.startsWith('<')) {
|
||||
return compareSemver(version, range.slice(1)) < 0;
|
||||
}
|
||||
|
||||
// Fallback to exact match
|
||||
return compareSemver(version, range) === 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the latest version from a list
|
||||
* @param {string[]} versions - List of version strings
|
||||
* @returns {string} Latest version
|
||||
*/
|
||||
export function getLatestVersion(versions) {
|
||||
if (!versions || versions.length === 0) return null;
|
||||
return versions.sort((a, b) => compareSemver(b, a))[0];
|
||||
}
|
||||
|
||||
// ============================================
|
||||
// MODEL METADATA
|
||||
// ============================================
|
||||
|
||||
/**
|
||||
* Model metadata structure
|
||||
* @typedef {object} ModelMetadata
|
||||
* @property {string} name - Model identifier (e.g., "phi-1.5-int4")
|
||||
* @property {string} version - Semantic version
|
||||
* @property {number} size - Model size in bytes
|
||||
* @property {string} hash - SHA256 hash for integrity
|
||||
* @property {string} format - Model format (onnx, safetensors, gguf)
|
||||
* @property {string[]} capabilities - Model capabilities
|
||||
* @property {object} sources - Download sources (gcs, ipfs, cdn)
|
||||
* @property {object} dependencies - Base models and adapters
|
||||
* @property {object} quantization - Quantization details
|
||||
* @property {object} metadata - Additional metadata
|
||||
*/
|
||||
|
||||
/**
|
||||
* Create a model metadata object
|
||||
* @param {object} options - Model options
|
||||
* @returns {ModelMetadata}
|
||||
*/
|
||||
export function createModelMetadata(options) {
|
||||
const {
|
||||
name,
|
||||
version = '1.0.0',
|
||||
size = 0,
|
||||
hash = '',
|
||||
format = 'onnx',
|
||||
capabilities = [],
|
||||
sources = {},
|
||||
dependencies = {},
|
||||
quantization = null,
|
||||
metadata = {},
|
||||
} = options;
|
||||
|
||||
if (!name) {
|
||||
throw new Error('Model name is required');
|
||||
}
|
||||
|
||||
// Validate version
|
||||
parseSemver(version);
|
||||
|
||||
return {
|
||||
name,
|
||||
version,
|
||||
size,
|
||||
hash,
|
||||
format,
|
||||
capabilities: Array.isArray(capabilities) ? capabilities : [capabilities],
|
||||
sources: {
|
||||
gcs: sources.gcs || null,
|
||||
ipfs: sources.ipfs || null,
|
||||
cdn: sources.cdn || null,
|
||||
...sources,
|
||||
},
|
||||
dependencies: {
|
||||
base: dependencies.base || null,
|
||||
adapters: dependencies.adapters || [],
|
||||
...dependencies,
|
||||
},
|
||||
quantization: quantization ? {
|
||||
type: quantization.type || 'int4',
|
||||
bits: quantization.bits || 4,
|
||||
blockSize: quantization.blockSize || 32,
|
||||
symmetric: quantization.symmetric ?? true,
|
||||
} : null,
|
||||
metadata: {
|
||||
createdAt: metadata.createdAt || new Date().toISOString(),
|
||||
updatedAt: metadata.updatedAt || new Date().toISOString(),
|
||||
author: metadata.author || 'RuVector',
|
||||
license: metadata.license || 'Apache-2.0',
|
||||
description: metadata.description || '',
|
||||
tags: metadata.tags || [],
|
||||
...metadata,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
// ============================================
|
||||
// MODEL REGISTRY
|
||||
// ============================================
|
||||
|
||||
/**
|
||||
* ModelRegistry - Manages model metadata, versions, and dependencies
|
||||
*/
|
||||
export class ModelRegistry extends EventEmitter {
|
||||
/**
|
||||
* Create a new ModelRegistry
|
||||
* @param {object} options - Registry options
|
||||
*/
|
||||
constructor(options = {}) {
|
||||
super();
|
||||
|
||||
this.id = `registry-${randomBytes(6).toString('hex')}`;
|
||||
this.registryPath = options.registryPath || null;
|
||||
|
||||
// Model storage: { modelName: { version: ModelMetadata } }
|
||||
this.models = new Map();
|
||||
|
||||
// Dependency graph
|
||||
this.dependencies = new Map();
|
||||
|
||||
// Search index
|
||||
this.searchIndex = {
|
||||
byCapability: new Map(),
|
||||
byFormat: new Map(),
|
||||
byTag: new Map(),
|
||||
};
|
||||
|
||||
// Stats
|
||||
this.stats = {
|
||||
totalModels: 0,
|
||||
totalVersions: 0,
|
||||
totalSize: 0,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Register a new model or version
|
||||
* @param {object} modelData - Model metadata
|
||||
* @returns {ModelMetadata}
|
||||
*/
|
||||
register(modelData) {
|
||||
const metadata = createModelMetadata(modelData);
|
||||
const { name, version } = metadata;
|
||||
|
||||
// Get or create model entry
|
||||
if (!this.models.has(name)) {
|
||||
this.models.set(name, new Map());
|
||||
this.stats.totalModels++;
|
||||
}
|
||||
|
||||
const versions = this.models.get(name);
|
||||
|
||||
// Check if version exists
|
||||
if (versions.has(version)) {
|
||||
this.emit('warning', {
|
||||
type: 'version_exists',
|
||||
model: name,
|
||||
version,
|
||||
});
|
||||
}
|
||||
|
||||
// Store metadata
|
||||
versions.set(version, metadata);
|
||||
this.stats.totalVersions++;
|
||||
this.stats.totalSize += metadata.size;
|
||||
|
||||
// Update search index
|
||||
this._indexModel(metadata);
|
||||
|
||||
// Update dependency graph
|
||||
this._updateDependencies(metadata);
|
||||
|
||||
this.emit('registered', { name, version, metadata });
|
||||
|
||||
return metadata;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get model metadata
|
||||
* @param {string} name - Model name
|
||||
* @param {string} version - Version (default: latest)
|
||||
* @returns {ModelMetadata|null}
|
||||
*/
|
||||
get(name, version = 'latest') {
|
||||
const versions = this.models.get(name);
|
||||
if (!versions) return null;
|
||||
|
||||
if (version === 'latest') {
|
||||
const latest = getLatestVersion([...versions.keys()]);
|
||||
return latest ? versions.get(latest) : null;
|
||||
}
|
||||
|
||||
// Check for exact match first
|
||||
if (versions.has(version)) {
|
||||
return versions.get(version);
|
||||
}
|
||||
|
||||
// Try to find matching version in range
|
||||
for (const [v, metadata] of versions) {
|
||||
if (satisfiesSemver(v, version)) {
|
||||
return metadata;
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* List all versions of a model
|
||||
* @param {string} name - Model name
|
||||
* @returns {string[]}
|
||||
*/
|
||||
listVersions(name) {
|
||||
const versions = this.models.get(name);
|
||||
if (!versions) return [];
|
||||
|
||||
return [...versions.keys()].sort((a, b) => compareSemver(b, a));
|
||||
}
|
||||
|
||||
/**
|
||||
* List all registered models
|
||||
* @returns {string[]}
|
||||
*/
|
||||
listModels() {
|
||||
return [...this.models.keys()];
|
||||
}
|
||||
|
||||
/**
|
||||
* Search for models
|
||||
* @param {object} criteria - Search criteria
|
||||
* @returns {ModelMetadata[]}
|
||||
*/
|
||||
search(criteria = {}) {
|
||||
const {
|
||||
name = null,
|
||||
capability = null,
|
||||
format = null,
|
||||
tag = null,
|
||||
minVersion = null,
|
||||
maxVersion = null,
|
||||
maxSize = null,
|
||||
query = null,
|
||||
} = criteria;
|
||||
|
||||
let results = [];
|
||||
|
||||
// Start with all models or filtered by name
|
||||
if (name) {
|
||||
const versions = this.models.get(name);
|
||||
if (versions) {
|
||||
results = [...versions.values()];
|
||||
}
|
||||
} else {
|
||||
// Collect all model versions
|
||||
for (const versions of this.models.values()) {
|
||||
results.push(...versions.values());
|
||||
}
|
||||
}
|
||||
|
||||
// Filter by capability
|
||||
if (capability) {
|
||||
results = results.filter(m =>
|
||||
m.capabilities.includes(capability)
|
||||
);
|
||||
}
|
||||
|
||||
// Filter by format
|
||||
if (format) {
|
||||
results = results.filter(m => m.format === format);
|
||||
}
|
||||
|
||||
// Filter by tag
|
||||
if (tag) {
|
||||
results = results.filter(m =>
|
||||
m.metadata.tags && m.metadata.tags.includes(tag)
|
||||
);
|
||||
}
|
||||
|
||||
// Filter by version range
|
||||
if (minVersion) {
|
||||
results = results.filter(m =>
|
||||
compareSemver(m.version, minVersion) >= 0
|
||||
);
|
||||
}
|
||||
|
||||
if (maxVersion) {
|
||||
results = results.filter(m =>
|
||||
compareSemver(m.version, maxVersion) <= 0
|
||||
);
|
||||
}
|
||||
|
||||
// Filter by size
|
||||
if (maxSize) {
|
||||
results = results.filter(m => m.size <= maxSize);
|
||||
}
|
||||
|
||||
// Text search
|
||||
if (query) {
|
||||
const q = query.toLowerCase();
|
||||
results = results.filter(m =>
|
||||
m.name.toLowerCase().includes(q) ||
|
||||
m.metadata.description?.toLowerCase().includes(q) ||
|
||||
m.metadata.tags?.some(t => t.toLowerCase().includes(q))
|
||||
);
|
||||
}
|
||||
|
||||
return results;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get models by capability
|
||||
* @param {string} capability - Capability to search for
|
||||
* @returns {ModelMetadata[]}
|
||||
*/
|
||||
getByCapability(capability) {
|
||||
const models = this.searchIndex.byCapability.get(capability);
|
||||
if (!models) return [];
|
||||
|
||||
return models.map(key => {
|
||||
const [name, version] = key.split('@');
|
||||
return this.get(name, version);
|
||||
}).filter(Boolean);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all dependencies for a model
|
||||
* @param {string} name - Model name
|
||||
* @param {string} version - Version
|
||||
* @param {boolean} recursive - Include transitive dependencies
|
||||
* @returns {ModelMetadata[]}
|
||||
*/
|
||||
getDependencies(name, version = 'latest', recursive = true) {
|
||||
const model = this.get(name, version);
|
||||
if (!model) return [];
|
||||
|
||||
const deps = [];
|
||||
const visited = new Set();
|
||||
const queue = [model];
|
||||
|
||||
while (queue.length > 0) {
|
||||
const current = queue.shift();
|
||||
const key = `${current.name}@${current.version}`;
|
||||
|
||||
if (visited.has(key)) continue;
|
||||
visited.add(key);
|
||||
|
||||
// Add base model
|
||||
if (current.dependencies.base) {
|
||||
const [baseName, baseVersion] = current.dependencies.base.split('@');
|
||||
const baseDep = this.get(baseName, baseVersion || 'latest');
|
||||
if (baseDep) {
|
||||
deps.push(baseDep);
|
||||
if (recursive) queue.push(baseDep);
|
||||
}
|
||||
}
|
||||
|
||||
// Add adapters
|
||||
if (current.dependencies.adapters) {
|
||||
for (const adapter of current.dependencies.adapters) {
|
||||
const [adapterName, adapterVersion] = adapter.split('@');
|
||||
const adapterDep = this.get(adapterName, adapterVersion || 'latest');
|
||||
if (adapterDep) {
|
||||
deps.push(adapterDep);
|
||||
if (recursive) queue.push(adapterDep);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return deps;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get dependents (models that depend on this one)
|
||||
* @param {string} name - Model name
|
||||
* @param {string} version - Version
|
||||
* @returns {ModelMetadata[]}
|
||||
*/
|
||||
getDependents(name, version = 'latest') {
|
||||
const key = version === 'latest'
|
||||
? name
|
||||
: `${name}@${version}`;
|
||||
|
||||
const dependents = [];
|
||||
|
||||
for (const [depKey, dependencies] of this.dependencies) {
|
||||
if (dependencies.includes(key) || dependencies.includes(name)) {
|
||||
const [modelName, modelVersion] = depKey.split('@');
|
||||
const model = this.get(modelName, modelVersion);
|
||||
if (model) dependents.push(model);
|
||||
}
|
||||
}
|
||||
|
||||
return dependents;
|
||||
}
|
||||
|
||||
/**
|
||||
* Compute hash for a model file
|
||||
* @param {Buffer|Uint8Array} data - Model data
|
||||
* @returns {string} SHA256 hash
|
||||
*/
|
||||
static computeHash(data) {
|
||||
return `sha256:${createHash('sha256').update(data).digest('hex')}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Verify model integrity
|
||||
* @param {string} name - Model name
|
||||
* @param {string} version - Version
|
||||
* @param {Buffer|Uint8Array} data - Model data
|
||||
* @returns {boolean}
|
||||
*/
|
||||
verify(name, version, data) {
|
||||
const model = this.get(name, version);
|
||||
if (!model) return false;
|
||||
|
||||
const computedHash = ModelRegistry.computeHash(data);
|
||||
return model.hash === computedHash;
|
||||
}
|
||||
|
||||
/**
|
||||
* Update search index for a model
|
||||
* @private
|
||||
*/
|
||||
_indexModel(metadata) {
|
||||
const key = `${metadata.name}@${metadata.version}`;
|
||||
|
||||
// Index by capability
|
||||
for (const cap of metadata.capabilities) {
|
||||
if (!this.searchIndex.byCapability.has(cap)) {
|
||||
this.searchIndex.byCapability.set(cap, []);
|
||||
}
|
||||
this.searchIndex.byCapability.get(cap).push(key);
|
||||
}
|
||||
|
||||
// Index by format
|
||||
if (!this.searchIndex.byFormat.has(metadata.format)) {
|
||||
this.searchIndex.byFormat.set(metadata.format, []);
|
||||
}
|
||||
this.searchIndex.byFormat.get(metadata.format).push(key);
|
||||
|
||||
// Index by tags
|
||||
if (metadata.metadata.tags) {
|
||||
for (const tag of metadata.metadata.tags) {
|
||||
if (!this.searchIndex.byTag.has(tag)) {
|
||||
this.searchIndex.byTag.set(tag, []);
|
||||
}
|
||||
this.searchIndex.byTag.get(tag).push(key);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Update dependency graph
|
||||
* @private
|
||||
*/
|
||||
_updateDependencies(metadata) {
|
||||
const key = `${metadata.name}@${metadata.version}`;
|
||||
const deps = [];
|
||||
|
||||
if (metadata.dependencies.base) {
|
||||
deps.push(metadata.dependencies.base);
|
||||
}
|
||||
|
||||
if (metadata.dependencies.adapters) {
|
||||
deps.push(...metadata.dependencies.adapters);
|
||||
}
|
||||
|
||||
if (deps.length > 0) {
|
||||
this.dependencies.set(key, deps);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Export registry to JSON
|
||||
* @returns {object}
|
||||
*/
|
||||
export() {
|
||||
const models = {};
|
||||
|
||||
for (const [name, versions] of this.models) {
|
||||
models[name] = {};
|
||||
for (const [version, metadata] of versions) {
|
||||
models[name][version] = metadata;
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
version: '1.0.0',
|
||||
generatedAt: new Date().toISOString(),
|
||||
stats: this.stats,
|
||||
models,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Import registry from JSON
|
||||
* @param {object} data - Registry data
|
||||
*/
|
||||
import(data) {
|
||||
if (!data.models) return;
|
||||
|
||||
for (const [name, versions] of Object.entries(data.models)) {
|
||||
for (const [version, metadata] of Object.entries(versions)) {
|
||||
this.register({
|
||||
...metadata,
|
||||
name,
|
||||
version,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Save registry to file
|
||||
* @param {string} filePath - File path
|
||||
*/
|
||||
async save(filePath = null) {
|
||||
const targetPath = filePath || this.registryPath;
|
||||
if (!targetPath) {
|
||||
throw new Error('No registry path specified');
|
||||
}
|
||||
|
||||
const data = JSON.stringify(this.export(), null, 2);
|
||||
await fs.writeFile(targetPath, data, 'utf-8');
|
||||
|
||||
this.emit('saved', { path: targetPath });
|
||||
}
|
||||
|
||||
/**
|
||||
* Load registry from file
|
||||
* @param {string} filePath - File path
|
||||
*/
|
||||
async load(filePath = null) {
|
||||
const targetPath = filePath || this.registryPath;
|
||||
if (!targetPath) {
|
||||
throw new Error('No registry path specified');
|
||||
}
|
||||
|
||||
try {
|
||||
const data = await fs.readFile(targetPath, 'utf-8');
|
||||
this.import(JSON.parse(data));
|
||||
this.emit('loaded', { path: targetPath });
|
||||
} catch (error) {
|
||||
if (error.code === 'ENOENT') {
|
||||
this.emit('warning', { message: 'Registry file not found, starting fresh' });
|
||||
} else {
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get registry statistics
|
||||
* @returns {object}
|
||||
*/
|
||||
getStats() {
|
||||
return {
|
||||
...this.stats,
|
||||
capabilities: this.searchIndex.byCapability.size,
|
||||
formats: this.searchIndex.byFormat.size,
|
||||
tags: this.searchIndex.byTag.size,
|
||||
dependencyEdges: this.dependencies.size,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================
|
||||
// DEFAULT EXPORT
|
||||
// ============================================
|
||||
|
||||
export default ModelRegistry;
|
||||
548
vendor/ruvector/examples/edge-net/pkg/models/model-utils.js
vendored
Normal file
548
vendor/ruvector/examples/edge-net/pkg/models/model-utils.js
vendored
Normal file
@@ -0,0 +1,548 @@
|
||||
/**
|
||||
* @ruvector/edge-net Model Utilities
|
||||
*
|
||||
* Helper functions for model management, optimization, and deployment.
|
||||
*
|
||||
* @module @ruvector/edge-net/models/utils
|
||||
*/
|
||||
|
||||
import { createHash, randomBytes } from 'crypto';
|
||||
import { existsSync, readFileSync, writeFileSync, mkdirSync, statSync, createReadStream } from 'fs';
|
||||
import { join, dirname } from 'path';
|
||||
import { homedir } from 'os';
|
||||
import { pipeline } from 'stream/promises';
|
||||
import { fileURLToPath } from 'url';
|
||||
|
||||
const __filename = fileURLToPath(import.meta.url);
|
||||
const __dirname = dirname(__filename);
|
||||
|
||||
// ============================================
|
||||
// CONFIGURATION
|
||||
// ============================================
|
||||
|
||||
export const DEFAULT_CACHE_DIR = process.env.ONNX_CACHE_DIR ||
|
||||
join(homedir(), '.ruvector', 'models', 'onnx');
|
||||
|
||||
export const REGISTRY_PATH = join(__dirname, 'registry.json');
|
||||
|
||||
export const GCS_CONFIG = {
|
||||
bucket: process.env.GCS_MODEL_BUCKET || 'ruvector-models',
|
||||
projectId: process.env.GCS_PROJECT_ID || 'ruvector',
|
||||
};
|
||||
|
||||
export const IPFS_CONFIG = {
|
||||
gateway: process.env.IPFS_GATEWAY || 'https://ipfs.io/ipfs',
|
||||
pinataApiKey: process.env.PINATA_API_KEY,
|
||||
pinataSecret: process.env.PINATA_SECRET,
|
||||
};
|
||||
|
||||
// ============================================
|
||||
// REGISTRY MANAGEMENT
|
||||
// ============================================
|
||||
|
||||
/**
|
||||
* Load the model registry
|
||||
* @returns {Object} Registry object
|
||||
*/
|
||||
export function loadRegistry() {
|
||||
try {
|
||||
if (existsSync(REGISTRY_PATH)) {
|
||||
return JSON.parse(readFileSync(REGISTRY_PATH, 'utf-8'));
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('[Registry] Failed to load:', error.message);
|
||||
}
|
||||
return { version: '1.0.0', models: {}, profiles: {}, adapters: {} };
|
||||
}
|
||||
|
||||
/**
|
||||
* Save the model registry
|
||||
* @param {Object} registry - Registry object to save
|
||||
*/
|
||||
export function saveRegistry(registry) {
|
||||
registry.updated = new Date().toISOString();
|
||||
writeFileSync(REGISTRY_PATH, JSON.stringify(registry, null, 2));
|
||||
}
|
||||
|
||||
/**
|
||||
* Get a model from the registry
|
||||
* @param {string} modelId - Model identifier
|
||||
* @returns {Object|null} Model metadata or null
|
||||
*/
|
||||
export function getModel(modelId) {
|
||||
const registry = loadRegistry();
|
||||
return registry.models[modelId] || null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get a deployment profile
|
||||
* @param {string} profileId - Profile identifier
|
||||
* @returns {Object|null} Profile configuration or null
|
||||
*/
|
||||
export function getProfile(profileId) {
|
||||
const registry = loadRegistry();
|
||||
return registry.profiles[profileId] || null;
|
||||
}
|
||||
|
||||
// ============================================
|
||||
// FILE UTILITIES
|
||||
// ============================================
|
||||
|
||||
/**
|
||||
* Format bytes to human-readable size
|
||||
* @param {number} bytes - Size in bytes
|
||||
* @returns {string} Formatted size string
|
||||
*/
|
||||
export function formatSize(bytes) {
|
||||
const units = ['B', 'KB', 'MB', 'GB', 'TB'];
|
||||
let size = bytes;
|
||||
let unitIndex = 0;
|
||||
while (size >= 1024 && unitIndex < units.length - 1) {
|
||||
size /= 1024;
|
||||
unitIndex++;
|
||||
}
|
||||
return `${size.toFixed(1)}${units[unitIndex]}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse size string to bytes
|
||||
* @param {string} sizeStr - Size string like "100MB"
|
||||
* @returns {number} Size in bytes
|
||||
*/
|
||||
export function parseSize(sizeStr) {
|
||||
const units = { 'B': 1, 'KB': 1024, 'MB': 1024**2, 'GB': 1024**3, 'TB': 1024**4 };
|
||||
const match = sizeStr.match(/^([\d.]+)\s*(B|KB|MB|GB|TB)?$/i);
|
||||
if (!match) return 0;
|
||||
const value = parseFloat(match[1]);
|
||||
const unit = (match[2] || 'B').toUpperCase();
|
||||
return value * (units[unit] || 1);
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculate SHA256 hash of a file
|
||||
* @param {string} filePath - Path to file
|
||||
* @returns {Promise<string>} Hex-encoded hash
|
||||
*/
|
||||
export async function hashFile(filePath) {
|
||||
const hash = createHash('sha256');
|
||||
const stream = createReadStream(filePath);
|
||||
|
||||
return new Promise((resolve, reject) => {
|
||||
stream.on('data', (data) => hash.update(data));
|
||||
stream.on('end', () => resolve(hash.digest('hex')));
|
||||
stream.on('error', reject);
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculate SHA256 hash of a buffer
|
||||
* @param {Buffer} buffer - Data buffer
|
||||
* @returns {string} Hex-encoded hash
|
||||
*/
|
||||
export function hashBuffer(buffer) {
|
||||
return createHash('sha256').update(buffer).digest('hex');
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the cache directory for a model
|
||||
* @param {string} modelId - HuggingFace model ID
|
||||
* @returns {string} Cache directory path
|
||||
*/
|
||||
export function getModelCacheDir(modelId) {
|
||||
return join(DEFAULT_CACHE_DIR, modelId.replace(/\//g, '--'));
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if a model is cached locally
|
||||
* @param {string} modelId - Model identifier
|
||||
* @returns {boolean} True if cached
|
||||
*/
|
||||
export function isModelCached(modelId) {
|
||||
const model = getModel(modelId);
|
||||
if (!model) return false;
|
||||
const cacheDir = getModelCacheDir(model.huggingface);
|
||||
return existsSync(cacheDir);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get cached model size
|
||||
* @param {string} modelId - Model identifier
|
||||
* @returns {number} Size in bytes or 0
|
||||
*/
|
||||
export function getCachedModelSize(modelId) {
|
||||
const model = getModel(modelId);
|
||||
if (!model) return 0;
|
||||
const cacheDir = getModelCacheDir(model.huggingface);
|
||||
if (!existsSync(cacheDir)) return 0;
|
||||
return getDirectorySize(cacheDir);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get directory size recursively
|
||||
* @param {string} dir - Directory path
|
||||
* @returns {number} Total size in bytes
|
||||
*/
|
||||
export function getDirectorySize(dir) {
|
||||
let size = 0;
|
||||
try {
|
||||
const { readdirSync } = require('fs');
|
||||
const entries = readdirSync(dir, { withFileTypes: true });
|
||||
for (const entry of entries) {
|
||||
const fullPath = join(dir, entry.name);
|
||||
if (entry.isDirectory()) {
|
||||
size += getDirectorySize(fullPath);
|
||||
} else {
|
||||
size += statSync(fullPath).size;
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
// Ignore errors
|
||||
}
|
||||
return size;
|
||||
}
|
||||
|
||||
// ============================================
|
||||
// MODEL OPTIMIZATION
|
||||
// ============================================
|
||||
|
||||
/**
|
||||
* Quantization configurations
|
||||
*/
|
||||
export const QUANTIZATION_CONFIGS = {
|
||||
int4: {
|
||||
bits: 4,
|
||||
blockSize: 32,
|
||||
expectedReduction: 0.25, // 4x smaller
|
||||
description: 'Aggressive quantization, some quality loss',
|
||||
},
|
||||
int8: {
|
||||
bits: 8,
|
||||
blockSize: 128,
|
||||
expectedReduction: 0.5, // 2x smaller
|
||||
description: 'Balanced quantization, minimal quality loss',
|
||||
},
|
||||
fp16: {
|
||||
bits: 16,
|
||||
blockSize: null,
|
||||
expectedReduction: 0.5, // 2x smaller than fp32
|
||||
description: 'Half precision, no quality loss',
|
||||
},
|
||||
fp32: {
|
||||
bits: 32,
|
||||
blockSize: null,
|
||||
expectedReduction: 1.0, // No change
|
||||
description: 'Full precision, original quality',
|
||||
},
|
||||
};
|
||||
|
||||
/**
|
||||
* Estimate quantized model size
|
||||
* @param {string} modelId - Model identifier
|
||||
* @param {string} quantType - Quantization type
|
||||
* @returns {number} Estimated size in bytes
|
||||
*/
|
||||
export function estimateQuantizedSize(modelId, quantType) {
|
||||
const model = getModel(modelId);
|
||||
if (!model) return 0;
|
||||
|
||||
const originalSize = parseSize(model.size);
|
||||
const config = QUANTIZATION_CONFIGS[quantType] || QUANTIZATION_CONFIGS.fp32;
|
||||
|
||||
return Math.floor(originalSize * config.expectedReduction);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get recommended quantization for a device profile
|
||||
* @param {Object} deviceProfile - Device capabilities
|
||||
* @returns {string} Recommended quantization type
|
||||
*/
|
||||
export function getRecommendedQuantization(deviceProfile) {
|
||||
const { memory, isEdge, requiresSpeed } = deviceProfile;
|
||||
|
||||
if (memory < 512 * 1024 * 1024) { // < 512MB
|
||||
return 'int4';
|
||||
} else if (memory < 2 * 1024 * 1024 * 1024 || isEdge) { // < 2GB or edge
|
||||
return 'int8';
|
||||
} else if (requiresSpeed) {
|
||||
return 'fp16';
|
||||
}
|
||||
return 'fp32';
|
||||
}
|
||||
|
||||
// ============================================
|
||||
// DOWNLOAD UTILITIES
|
||||
// ============================================
|
||||
|
||||
/**
|
||||
* Download progress callback type
|
||||
* @callback ProgressCallback
|
||||
* @param {Object} progress - Progress information
|
||||
* @param {number} progress.loaded - Bytes loaded
|
||||
* @param {number} progress.total - Total bytes
|
||||
* @param {string} progress.file - Current file name
|
||||
*/
|
||||
|
||||
/**
|
||||
* Download a file with progress reporting
|
||||
* @param {string} url - URL to download
|
||||
* @param {string} destPath - Destination path
|
||||
* @param {ProgressCallback} [onProgress] - Progress callback
|
||||
* @returns {Promise<string>} Downloaded file path
|
||||
*/
|
||||
export async function downloadFile(url, destPath, onProgress) {
|
||||
const destDir = dirname(destPath);
|
||||
if (!existsSync(destDir)) {
|
||||
mkdirSync(destDir, { recursive: true });
|
||||
}
|
||||
|
||||
const response = await fetch(url);
|
||||
if (!response.ok) {
|
||||
throw new Error(`HTTP ${response.status}: ${response.statusText}`);
|
||||
}
|
||||
|
||||
const totalSize = parseInt(response.headers.get('content-length') || '0', 10);
|
||||
let loadedSize = 0;
|
||||
|
||||
const { createWriteStream } = await import('fs');
|
||||
const fileStream = createWriteStream(destPath);
|
||||
const reader = response.body.getReader();
|
||||
|
||||
try {
|
||||
while (true) {
|
||||
const { done, value } = await reader.read();
|
||||
if (done) break;
|
||||
|
||||
fileStream.write(value);
|
||||
loadedSize += value.length;
|
||||
|
||||
if (onProgress) {
|
||||
onProgress({
|
||||
loaded: loadedSize,
|
||||
total: totalSize,
|
||||
file: destPath,
|
||||
});
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
fileStream.end();
|
||||
}
|
||||
|
||||
return destPath;
|
||||
}
|
||||
|
||||
// ============================================
|
||||
// IPFS UTILITIES
|
||||
// ============================================
|
||||
|
||||
/**
|
||||
* Pin a file to IPFS via Pinata
|
||||
* @param {string} filePath - Path to file to pin
|
||||
* @param {Object} metadata - Metadata for the pin
|
||||
* @returns {Promise<string>} IPFS CID
|
||||
*/
|
||||
export async function pinToIPFS(filePath, metadata = {}) {
|
||||
if (!IPFS_CONFIG.pinataApiKey || !IPFS_CONFIG.pinataSecret) {
|
||||
throw new Error('Pinata API credentials not configured');
|
||||
}
|
||||
|
||||
const FormData = (await import('form-data')).default;
|
||||
const form = new FormData();
|
||||
|
||||
form.append('file', createReadStream(filePath));
|
||||
form.append('pinataMetadata', JSON.stringify({
|
||||
name: metadata.name || filePath,
|
||||
keyvalues: metadata,
|
||||
}));
|
||||
|
||||
const response = await fetch('https://api.pinata.cloud/pinning/pinFileToIPFS', {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'pinata_api_key': IPFS_CONFIG.pinataApiKey,
|
||||
'pinata_secret_api_key': IPFS_CONFIG.pinataSecret,
|
||||
},
|
||||
body: form,
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
throw new Error(`Pinata error: ${response.statusText}`);
|
||||
}
|
||||
|
||||
const result = await response.json();
|
||||
return result.IpfsHash;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get IPFS gateway URL for a CID
|
||||
* @param {string} cid - IPFS CID
|
||||
* @returns {string} Gateway URL
|
||||
*/
|
||||
export function getIPFSUrl(cid) {
|
||||
return `${IPFS_CONFIG.gateway}/${cid}`;
|
||||
}
|
||||
|
||||
// ============================================
|
||||
// GCS UTILITIES
|
||||
// ============================================
|
||||
|
||||
/**
|
||||
* Generate GCS URL for a model
|
||||
* @param {string} modelId - Model identifier
|
||||
* @param {string} fileName - File name
|
||||
* @returns {string} GCS URL
|
||||
*/
|
||||
export function getGCSUrl(modelId, fileName) {
|
||||
return `https://storage.googleapis.com/${GCS_CONFIG.bucket}/${modelId}/${fileName}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if a model exists in GCS
|
||||
* @param {string} modelId - Model identifier
|
||||
* @param {string} fileName - File name
|
||||
* @returns {Promise<boolean>} True if exists
|
||||
*/
|
||||
export async function checkGCSExists(modelId, fileName) {
|
||||
const url = getGCSUrl(modelId, fileName);
|
||||
try {
|
||||
const response = await fetch(url, { method: 'HEAD' });
|
||||
return response.ok;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================
|
||||
// ADAPTER UTILITIES
|
||||
// ============================================
|
||||
|
||||
/**
|
||||
* MicroLoRA adapter configuration
|
||||
*/
|
||||
export const LORA_DEFAULTS = {
|
||||
rank: 8,
|
||||
alpha: 16,
|
||||
dropout: 0.1,
|
||||
targetModules: ['q_proj', 'v_proj'],
|
||||
};
|
||||
|
||||
/**
|
||||
* Create adapter metadata
|
||||
* @param {string} name - Adapter name
|
||||
* @param {string} baseModel - Base model identifier
|
||||
* @param {Object} options - Training options
|
||||
* @returns {Object} Adapter metadata
|
||||
*/
|
||||
export function createAdapterMetadata(name, baseModel, options = {}) {
|
||||
return {
|
||||
id: `${name}-${randomBytes(4).toString('hex')}`,
|
||||
name,
|
||||
baseModel,
|
||||
rank: options.rank || LORA_DEFAULTS.rank,
|
||||
alpha: options.alpha || LORA_DEFAULTS.alpha,
|
||||
targetModules: options.targetModules || LORA_DEFAULTS.targetModules,
|
||||
created: new Date().toISOString(),
|
||||
size: null, // Set after training
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Get adapter save path
|
||||
* @param {string} adapterName - Adapter name
|
||||
* @returns {string} Save path
|
||||
*/
|
||||
export function getAdapterPath(adapterName) {
|
||||
return join(DEFAULT_CACHE_DIR, 'adapters', adapterName);
|
||||
}
|
||||
|
||||
// ============================================
|
||||
// BENCHMARK UTILITIES
|
||||
// ============================================
|
||||
|
||||
/**
|
||||
* Create a benchmark result object
|
||||
* @param {string} modelId - Model identifier
|
||||
* @param {number[]} times - Latency measurements in ms
|
||||
* @returns {Object} Benchmark results
|
||||
*/
|
||||
export function createBenchmarkResult(modelId, times) {
|
||||
times.sort((a, b) => a - b);
|
||||
|
||||
return {
|
||||
model: modelId,
|
||||
timestamp: new Date().toISOString(),
|
||||
iterations: times.length,
|
||||
stats: {
|
||||
avg: times.reduce((a, b) => a + b, 0) / times.length,
|
||||
median: times[Math.floor(times.length / 2)],
|
||||
p95: times[Math.floor(times.length * 0.95)],
|
||||
p99: times[Math.floor(times.length * 0.99)],
|
||||
min: times[0],
|
||||
max: times[times.length - 1],
|
||||
stddev: calculateStdDev(times),
|
||||
},
|
||||
rawTimes: times,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculate standard deviation
|
||||
* @param {number[]} values - Array of values
|
||||
* @returns {number} Standard deviation
|
||||
*/
|
||||
function calculateStdDev(values) {
|
||||
const mean = values.reduce((a, b) => a + b, 0) / values.length;
|
||||
const squareDiffs = values.map(v => Math.pow(v - mean, 2));
|
||||
const avgSquareDiff = squareDiffs.reduce((a, b) => a + b, 0) / squareDiffs.length;
|
||||
return Math.sqrt(avgSquareDiff);
|
||||
}
|
||||
|
||||
// ============================================
|
||||
// EXPORTS
|
||||
// ============================================
|
||||
|
||||
export default {
|
||||
// Configuration
|
||||
DEFAULT_CACHE_DIR,
|
||||
REGISTRY_PATH,
|
||||
GCS_CONFIG,
|
||||
IPFS_CONFIG,
|
||||
QUANTIZATION_CONFIGS,
|
||||
LORA_DEFAULTS,
|
||||
|
||||
// Registry
|
||||
loadRegistry,
|
||||
saveRegistry,
|
||||
getModel,
|
||||
getProfile,
|
||||
|
||||
// Files
|
||||
formatSize,
|
||||
parseSize,
|
||||
hashFile,
|
||||
hashBuffer,
|
||||
getModelCacheDir,
|
||||
isModelCached,
|
||||
getCachedModelSize,
|
||||
getDirectorySize,
|
||||
|
||||
// Optimization
|
||||
estimateQuantizedSize,
|
||||
getRecommendedQuantization,
|
||||
|
||||
// Download
|
||||
downloadFile,
|
||||
|
||||
// IPFS
|
||||
pinToIPFS,
|
||||
getIPFSUrl,
|
||||
|
||||
// GCS
|
||||
getGCSUrl,
|
||||
checkGCSExists,
|
||||
|
||||
// Adapters
|
||||
createAdapterMetadata,
|
||||
getAdapterPath,
|
||||
|
||||
// Benchmarks
|
||||
createBenchmarkResult,
|
||||
};
|
||||
914
vendor/ruvector/examples/edge-net/pkg/models/models-cli.js
vendored
Executable file
914
vendor/ruvector/examples/edge-net/pkg/models/models-cli.js
vendored
Executable file
@@ -0,0 +1,914 @@
|
||||
#!/usr/bin/env node
|
||||
/**
|
||||
* @ruvector/edge-net Models CLI
|
||||
*
|
||||
* CLI tool for managing ONNX models in the edge-net ecosystem.
|
||||
* Supports listing, downloading, optimizing, and uploading models.
|
||||
*
|
||||
* @module @ruvector/edge-net/models/cli
|
||||
*/
|
||||
|
||||
import { Command } from 'commander';
|
||||
import { createWriteStream, existsSync, mkdirSync, readFileSync, writeFileSync, statSync, unlinkSync, readdirSync } from 'fs';
|
||||
import { join, basename, dirname } from 'path';
|
||||
import { homedir, cpus, totalmem } from 'os';
|
||||
import { pipeline } from 'stream/promises';
|
||||
import { createHash } from 'crypto';
|
||||
import { EventEmitter } from 'events';
|
||||
import { fileURLToPath } from 'url';
|
||||
|
||||
const __filename = fileURLToPath(import.meta.url);
|
||||
const __dirname = dirname(__filename);
|
||||
|
||||
// ============================================
|
||||
// CONFIGURATION
|
||||
// ============================================
|
||||
|
||||
const DEFAULT_CACHE_DIR = process.env.ONNX_CACHE_DIR ||
|
||||
join(homedir(), '.ruvector', 'models', 'onnx');
|
||||
|
||||
const GCS_BUCKET = process.env.GCS_MODEL_BUCKET || 'ruvector-models';
|
||||
const GCS_BASE_URL = `https://storage.googleapis.com/${GCS_BUCKET}`;
|
||||
const IPFS_GATEWAY = process.env.IPFS_GATEWAY || 'https://ipfs.io/ipfs';
|
||||
|
||||
const REGISTRY_PATH = join(__dirname, 'registry.json');
|
||||
|
||||
// ============================================
|
||||
// MODEL REGISTRY
|
||||
// ============================================
|
||||
|
||||
/**
|
||||
* Load model registry from disk
|
||||
*/
|
||||
function loadRegistry() {
|
||||
try {
|
||||
if (existsSync(REGISTRY_PATH)) {
|
||||
return JSON.parse(readFileSync(REGISTRY_PATH, 'utf-8'));
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('[Registry] Failed to load registry:', error.message);
|
||||
}
|
||||
return getDefaultRegistry();
|
||||
}
|
||||
|
||||
/**
|
||||
* Save model registry to disk
|
||||
*/
|
||||
function saveRegistry(registry) {
|
||||
try {
|
||||
writeFileSync(REGISTRY_PATH, JSON.stringify(registry, null, 2));
|
||||
console.log('[Registry] Saved to:', REGISTRY_PATH);
|
||||
} catch (error) {
|
||||
console.error('[Registry] Failed to save:', error.message);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Default registry with known models
|
||||
*/
|
||||
function getDefaultRegistry() {
|
||||
return {
|
||||
version: '1.0.0',
|
||||
updated: new Date().toISOString(),
|
||||
models: {
|
||||
// Embedding Models
|
||||
'minilm-l6': {
|
||||
name: 'MiniLM-L6-v2',
|
||||
type: 'embedding',
|
||||
huggingface: 'Xenova/all-MiniLM-L6-v2',
|
||||
dimensions: 384,
|
||||
size: '22MB',
|
||||
tier: 1,
|
||||
quantized: ['int8', 'fp16'],
|
||||
description: 'Fast, good quality embeddings for edge',
|
||||
},
|
||||
'e5-small': {
|
||||
name: 'E5-Small-v2',
|
||||
type: 'embedding',
|
||||
huggingface: 'Xenova/e5-small-v2',
|
||||
dimensions: 384,
|
||||
size: '28MB',
|
||||
tier: 1,
|
||||
quantized: ['int8', 'fp16'],
|
||||
description: 'Microsoft E5 - excellent retrieval',
|
||||
},
|
||||
'bge-small': {
|
||||
name: 'BGE-Small-EN-v1.5',
|
||||
type: 'embedding',
|
||||
huggingface: 'Xenova/bge-small-en-v1.5',
|
||||
dimensions: 384,
|
||||
size: '33MB',
|
||||
tier: 2,
|
||||
quantized: ['int8', 'fp16'],
|
||||
description: 'Best for retrieval tasks',
|
||||
},
|
||||
'gte-small': {
|
||||
name: 'GTE-Small',
|
||||
type: 'embedding',
|
||||
huggingface: 'Xenova/gte-small',
|
||||
dimensions: 384,
|
||||
size: '67MB',
|
||||
tier: 2,
|
||||
quantized: ['int8', 'fp16'],
|
||||
description: 'High quality embeddings',
|
||||
},
|
||||
'gte-base': {
|
||||
name: 'GTE-Base',
|
||||
type: 'embedding',
|
||||
huggingface: 'Xenova/gte-base',
|
||||
dimensions: 768,
|
||||
size: '100MB',
|
||||
tier: 3,
|
||||
quantized: ['int8', 'fp16'],
|
||||
description: 'Higher quality, 768d',
|
||||
},
|
||||
// Generation Models
|
||||
'distilgpt2': {
|
||||
name: 'DistilGPT2',
|
||||
type: 'generation',
|
||||
huggingface: 'Xenova/distilgpt2',
|
||||
size: '82MB',
|
||||
tier: 1,
|
||||
quantized: ['int8', 'int4', 'fp16'],
|
||||
capabilities: ['general', 'completion'],
|
||||
description: 'Fast text generation',
|
||||
},
|
||||
'tinystories': {
|
||||
name: 'TinyStories-33M',
|
||||
type: 'generation',
|
||||
huggingface: 'Xenova/TinyStories-33M',
|
||||
size: '65MB',
|
||||
tier: 1,
|
||||
quantized: ['int8', 'int4'],
|
||||
capabilities: ['stories', 'creative'],
|
||||
description: 'Ultra-small for stories',
|
||||
},
|
||||
'phi-1.5': {
|
||||
name: 'Phi-1.5',
|
||||
type: 'generation',
|
||||
huggingface: 'Xenova/phi-1_5',
|
||||
size: '280MB',
|
||||
tier: 2,
|
||||
quantized: ['int8', 'int4', 'fp16'],
|
||||
capabilities: ['code', 'reasoning', 'math'],
|
||||
description: 'Microsoft Phi-1.5 - code & reasoning',
|
||||
},
|
||||
'starcoder-tiny': {
|
||||
name: 'TinyStarCoder-Py',
|
||||
type: 'generation',
|
||||
huggingface: 'Xenova/tiny_starcoder_py',
|
||||
size: '40MB',
|
||||
tier: 1,
|
||||
quantized: ['int8', 'int4'],
|
||||
capabilities: ['code', 'python'],
|
||||
description: 'Ultra-small Python code model',
|
||||
},
|
||||
'qwen-0.5b': {
|
||||
name: 'Qwen-1.5-0.5B',
|
||||
type: 'generation',
|
||||
huggingface: 'Xenova/Qwen1.5-0.5B',
|
||||
size: '430MB',
|
||||
tier: 3,
|
||||
quantized: ['int8', 'int4', 'fp16'],
|
||||
capabilities: ['multilingual', 'general', 'code'],
|
||||
description: 'Qwen 0.5B - multilingual small model',
|
||||
},
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
// ============================================
|
||||
// UTILITIES
|
||||
// ============================================
|
||||
|
||||
/**
|
||||
* Format bytes to human-readable size
|
||||
*/
|
||||
function formatSize(bytes) {
|
||||
const units = ['B', 'KB', 'MB', 'GB'];
|
||||
let size = bytes;
|
||||
let unitIndex = 0;
|
||||
while (size >= 1024 && unitIndex < units.length - 1) {
|
||||
size /= 1024;
|
||||
unitIndex++;
|
||||
}
|
||||
return `${size.toFixed(1)}${units[unitIndex]}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculate SHA256 hash of a file
|
||||
*/
|
||||
async function hashFile(filePath) {
|
||||
const { createReadStream } = await import('fs');
|
||||
const hash = createHash('sha256');
|
||||
const stream = createReadStream(filePath);
|
||||
|
||||
return new Promise((resolve, reject) => {
|
||||
stream.on('data', (data) => hash.update(data));
|
||||
stream.on('end', () => resolve(hash.digest('hex')));
|
||||
stream.on('error', reject);
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Download file with progress
|
||||
*/
|
||||
async function downloadFile(url, destPath, options = {}) {
|
||||
const { showProgress = true } = options;
|
||||
|
||||
// Ensure directory exists
|
||||
const destDir = dirname(destPath);
|
||||
if (!existsSync(destDir)) {
|
||||
mkdirSync(destDir, { recursive: true });
|
||||
}
|
||||
|
||||
const response = await fetch(url);
|
||||
if (!response.ok) {
|
||||
throw new Error(`HTTP ${response.status}: ${response.statusText}`);
|
||||
}
|
||||
|
||||
const totalSize = parseInt(response.headers.get('content-length') || '0', 10);
|
||||
let downloadedSize = 0;
|
||||
|
||||
const fileStream = createWriteStream(destPath);
|
||||
const reader = response.body.getReader();
|
||||
|
||||
try {
|
||||
while (true) {
|
||||
const { done, value } = await reader.read();
|
||||
if (done) break;
|
||||
|
||||
fileStream.write(value);
|
||||
downloadedSize += value.length;
|
||||
|
||||
if (showProgress && totalSize > 0) {
|
||||
const progress = ((downloadedSize / totalSize) * 100).toFixed(1);
|
||||
process.stdout.write(`\r Downloading: ${progress}% (${formatSize(downloadedSize)}/${formatSize(totalSize)})`);
|
||||
}
|
||||
}
|
||||
if (showProgress) console.log('');
|
||||
} finally {
|
||||
fileStream.end();
|
||||
}
|
||||
|
||||
return destPath;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get cache directory for a model
|
||||
*/
|
||||
function getModelCacheDir(modelId) {
|
||||
return join(DEFAULT_CACHE_DIR, modelId.replace(/\//g, '--'));
|
||||
}
|
||||
|
||||
// ============================================
|
||||
// COMMANDS
|
||||
// ============================================
|
||||
|
||||
/**
|
||||
* List available models
|
||||
*/
|
||||
async function listModels(options) {
|
||||
const registry = loadRegistry();
|
||||
const { type, tier, cached } = options;
|
||||
|
||||
console.log('\n=== Edge-Net Model Registry ===\n');
|
||||
console.log(`Registry Version: ${registry.version}`);
|
||||
console.log(`Last Updated: ${registry.updated}\n`);
|
||||
|
||||
const models = Object.entries(registry.models)
|
||||
.filter(([_, m]) => !type || m.type === type)
|
||||
.filter(([_, m]) => !tier || m.tier === parseInt(tier))
|
||||
.sort((a, b) => a[1].tier - b[1].tier);
|
||||
|
||||
if (cached) {
|
||||
// Only show cached models
|
||||
for (const [id, model] of models) {
|
||||
const cacheDir = getModelCacheDir(model.huggingface);
|
||||
if (existsSync(cacheDir)) {
|
||||
printModelInfo(id, model, true);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// Group by type
|
||||
const embedding = models.filter(([_, m]) => m.type === 'embedding');
|
||||
const generation = models.filter(([_, m]) => m.type === 'generation');
|
||||
|
||||
if (embedding.length > 0) {
|
||||
console.log('EMBEDDING MODELS:');
|
||||
console.log('-'.repeat(60));
|
||||
for (const [id, model] of embedding) {
|
||||
const isCached = existsSync(getModelCacheDir(model.huggingface));
|
||||
printModelInfo(id, model, isCached);
|
||||
}
|
||||
console.log('');
|
||||
}
|
||||
|
||||
if (generation.length > 0) {
|
||||
console.log('GENERATION MODELS:');
|
||||
console.log('-'.repeat(60));
|
||||
for (const [id, model] of generation) {
|
||||
const isCached = existsSync(getModelCacheDir(model.huggingface));
|
||||
printModelInfo(id, model, isCached);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
console.log('\nUse "models-cli download <model>" to download a model');
|
||||
console.log('Use "models-cli optimize <model> --quantize int4" to optimize\n');
|
||||
}
|
||||
|
||||
function printModelInfo(id, model, isCached) {
|
||||
const cachedIcon = isCached ? '[CACHED]' : '';
|
||||
const tierIcon = ['', '[T1]', '[T2]', '[T3]', '[T4]'][model.tier] || '';
|
||||
console.log(` ${id.padEnd(20)} ${model.size.padEnd(8)} ${tierIcon.padEnd(5)} ${cachedIcon}`);
|
||||
console.log(` ${model.description}`);
|
||||
if (model.capabilities) {
|
||||
console.log(` Capabilities: ${model.capabilities.join(', ')}`);
|
||||
}
|
||||
if (model.quantized) {
|
||||
console.log(` Quantized: ${model.quantized.join(', ')}`);
|
||||
}
|
||||
console.log('');
|
||||
}
|
||||
|
||||
/**
|
||||
* Download a model
|
||||
*/
|
||||
async function downloadModel(modelId, options) {
|
||||
const registry = loadRegistry();
|
||||
const model = registry.models[modelId];
|
||||
|
||||
if (!model) {
|
||||
console.error(`Error: Model "${modelId}" not found in registry`);
|
||||
console.error('Use "models-cli list" to see available models');
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
console.log(`\nDownloading model: ${model.name}`);
|
||||
console.log(` Source: ${model.huggingface}`);
|
||||
console.log(` Size: ~${model.size}`);
|
||||
console.log(` Type: ${model.type}`);
|
||||
|
||||
const cacheDir = getModelCacheDir(model.huggingface);
|
||||
|
||||
if (existsSync(cacheDir) && !options.force) {
|
||||
console.log(`\nModel already cached at: ${cacheDir}`);
|
||||
console.log('Use --force to re-download');
|
||||
return;
|
||||
}
|
||||
|
||||
// Use transformers.js to download
|
||||
try {
|
||||
console.log('\nInitializing download via transformers.js...');
|
||||
|
||||
const { pipeline, env } = await import('@xenova/transformers');
|
||||
env.cacheDir = DEFAULT_CACHE_DIR;
|
||||
env.allowRemoteModels = true;
|
||||
|
||||
const pipelineType = model.type === 'embedding' ? 'feature-extraction' : 'text-generation';
|
||||
|
||||
console.log(`Loading ${pipelineType} pipeline...`);
|
||||
const pipe = await pipeline(pipelineType, model.huggingface, {
|
||||
quantized: options.quantize !== 'fp32',
|
||||
progress_callback: (progress) => {
|
||||
if (progress.status === 'downloading') {
|
||||
const pct = ((progress.loaded / progress.total) * 100).toFixed(1);
|
||||
process.stdout.write(`\r ${progress.file}: ${pct}%`);
|
||||
}
|
||||
},
|
||||
});
|
||||
|
||||
console.log('\n\nModel downloaded successfully!');
|
||||
console.log(`Cache location: ${cacheDir}`);
|
||||
|
||||
// Verify download
|
||||
if (options.verify) {
|
||||
console.log('\nVerifying model...');
|
||||
// Quick inference test
|
||||
if (model.type === 'embedding') {
|
||||
const result = await pipe('test embedding');
|
||||
console.log(` Embedding dimensions: ${result.data.length}`);
|
||||
} else {
|
||||
const result = await pipe('Hello', { max_new_tokens: 5 });
|
||||
console.log(` Generation test passed`);
|
||||
}
|
||||
console.log('Verification complete!');
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('\nDownload failed:', error.message);
|
||||
if (error.message.includes('transformers')) {
|
||||
console.error('Make sure @xenova/transformers is installed: npm install @xenova/transformers');
|
||||
}
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Optimize a model for edge deployment
|
||||
*/
|
||||
async function optimizeModel(modelId, options) {
|
||||
const registry = loadRegistry();
|
||||
const model = registry.models[modelId];
|
||||
|
||||
if (!model) {
|
||||
console.error(`Error: Model "${modelId}" not found`);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const cacheDir = getModelCacheDir(model.huggingface);
|
||||
if (!existsSync(cacheDir)) {
|
||||
console.error(`Error: Model not cached. Run "models-cli download ${modelId}" first`);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
console.log(`\nOptimizing model: ${model.name}`);
|
||||
console.log(` Quantization: ${options.quantize || 'int8'}`);
|
||||
console.log(` Pruning: ${options.prune || 'none'}`);
|
||||
|
||||
const outputDir = options.output || join(cacheDir, 'optimized');
|
||||
if (!existsSync(outputDir)) {
|
||||
mkdirSync(outputDir, { recursive: true });
|
||||
}
|
||||
|
||||
// Find ONNX files
|
||||
const onnxFiles = findOnnxFiles(cacheDir);
|
||||
if (onnxFiles.length === 0) {
|
||||
console.error('No ONNX files found in model cache');
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
console.log(`\nFound ${onnxFiles.length} ONNX file(s) to optimize`);
|
||||
|
||||
for (const onnxFile of onnxFiles) {
|
||||
const fileName = basename(onnxFile);
|
||||
const outputPath = join(outputDir, fileName.replace('.onnx', `_${options.quantize || 'int8'}.onnx`));
|
||||
|
||||
console.log(`\nProcessing: ${fileName}`);
|
||||
const originalSize = statSync(onnxFile).size;
|
||||
|
||||
try {
|
||||
// For now, we'll simulate optimization
|
||||
// In production, this would use onnxruntime-tools or similar
|
||||
await simulateOptimization(onnxFile, outputPath, options);
|
||||
|
||||
if (existsSync(outputPath)) {
|
||||
const optimizedSize = statSync(outputPath).size;
|
||||
const reduction = ((1 - optimizedSize / originalSize) * 100).toFixed(1);
|
||||
console.log(` Original: ${formatSize(originalSize)}`);
|
||||
console.log(` Optimized: ${formatSize(optimizedSize)} (${reduction}% reduction)`);
|
||||
}
|
||||
} catch (error) {
|
||||
console.error(` Optimization failed: ${error.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
console.log(`\nOptimized models saved to: ${outputDir}`);
|
||||
}
|
||||
|
||||
function findOnnxFiles(dir) {
|
||||
const files = [];
|
||||
try {
|
||||
const entries = readdirSync(dir, { withFileTypes: true });
|
||||
for (const entry of entries) {
|
||||
const fullPath = join(dir, entry.name);
|
||||
if (entry.isDirectory()) {
|
||||
files.push(...findOnnxFiles(fullPath));
|
||||
} else if (entry.name.endsWith('.onnx')) {
|
||||
files.push(fullPath);
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
// Ignore read errors
|
||||
}
|
||||
return files;
|
||||
}
|
||||
|
||||
async function simulateOptimization(inputPath, outputPath, options) {
|
||||
// This is a placeholder for actual ONNX optimization
|
||||
// In production, you would use:
|
||||
// - onnxruntime-tools for quantization
|
||||
// - onnx-simplifier for graph optimization
|
||||
// - Custom pruning algorithms
|
||||
|
||||
const { copyFileSync } = await import('fs');
|
||||
|
||||
console.log(` Quantizing with ${options.quantize || 'int8'}...`);
|
||||
|
||||
// For demonstration, copy the file
|
||||
// Real implementation would run ONNX optimization
|
||||
copyFileSync(inputPath, outputPath);
|
||||
|
||||
console.log(' Note: Full quantization requires onnxruntime-tools');
|
||||
console.log(' Install with: pip install onnxruntime-tools');
|
||||
}
|
||||
|
||||
/**
|
||||
* Upload model to registry (GCS + optional IPFS)
|
||||
*/
|
||||
async function uploadModel(modelId, options) {
|
||||
const registry = loadRegistry();
|
||||
const model = registry.models[modelId];
|
||||
|
||||
if (!model) {
|
||||
console.error(`Error: Model "${modelId}" not found`);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const cacheDir = getModelCacheDir(model.huggingface);
|
||||
if (!existsSync(cacheDir)) {
|
||||
console.error(`Error: Model not cached. Download first.`);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
console.log(`\nUploading model: ${model.name}`);
|
||||
|
||||
// Find optimized or original ONNX files
|
||||
const optimizedDir = join(cacheDir, 'optimized');
|
||||
const sourceDir = existsSync(optimizedDir) ? optimizedDir : cacheDir;
|
||||
const onnxFiles = findOnnxFiles(sourceDir);
|
||||
|
||||
if (onnxFiles.length === 0) {
|
||||
console.error('No ONNX files found');
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
console.log(`Found ${onnxFiles.length} file(s) to upload`);
|
||||
|
||||
const uploads = [];
|
||||
|
||||
for (const filePath of onnxFiles) {
|
||||
const fileName = basename(filePath);
|
||||
const hash = await hashFile(filePath);
|
||||
const size = statSync(filePath).size;
|
||||
|
||||
console.log(`\nFile: ${fileName}`);
|
||||
console.log(` Size: ${formatSize(size)}`);
|
||||
console.log(` SHA256: ${hash.substring(0, 16)}...`);
|
||||
|
||||
// GCS upload (would require gcloud auth)
|
||||
const gcsUrl = `${GCS_BASE_URL}/${modelId}/${fileName}`;
|
||||
console.log(` GCS URL: ${gcsUrl}`);
|
||||
|
||||
uploads.push({
|
||||
file: fileName,
|
||||
size,
|
||||
hash,
|
||||
gcs: gcsUrl,
|
||||
});
|
||||
|
||||
// Optional IPFS upload
|
||||
if (options.ipfs) {
|
||||
console.log(' IPFS: Pinning...');
|
||||
// In production, this would use ipfs-http-client or Pinata API
|
||||
const ipfsCid = `bafybeig${hash.substring(0, 48)}`;
|
||||
console.log(` IPFS CID: ${ipfsCid}`);
|
||||
uploads[uploads.length - 1].ipfs = `${IPFS_GATEWAY}/${ipfsCid}`;
|
||||
}
|
||||
}
|
||||
|
||||
// Update registry
|
||||
if (!model.artifacts) model.artifacts = {};
|
||||
model.artifacts[options.quantize || 'original'] = uploads;
|
||||
model.lastUpload = new Date().toISOString();
|
||||
|
||||
saveRegistry(registry);
|
||||
|
||||
console.log('\nUpload metadata saved to registry');
|
||||
console.log('Note: Actual GCS upload requires `gcloud auth` and gsutil');
|
||||
console.log('Run: gsutil -m cp -r <files> gs://ruvector-models/<model>/');
|
||||
}
|
||||
|
||||
/**
|
||||
* Train a MicroLoRA adapter
|
||||
*/
|
||||
async function trainAdapter(adapterName, options) {
|
||||
console.log(`\nTraining MicroLoRA adapter: ${adapterName}`);
|
||||
console.log(` Base model: ${options.base || 'phi-1.5'}`);
|
||||
console.log(` Dataset: ${options.dataset || 'custom'}`);
|
||||
console.log(` Rank: ${options.rank || 8}`);
|
||||
console.log(` Epochs: ${options.epochs || 3}`);
|
||||
|
||||
const registry = loadRegistry();
|
||||
const baseModel = registry.models[options.base || 'phi-1.5'];
|
||||
|
||||
if (!baseModel) {
|
||||
console.error(`Error: Base model "${options.base}" not found`);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
console.log('\nMicroLoRA Training Configuration:');
|
||||
console.log(` Base: ${baseModel.huggingface}`);
|
||||
console.log(` LoRA Rank (r): ${options.rank || 8}`);
|
||||
console.log(` Alpha: ${(options.rank || 8) * 2}`);
|
||||
console.log(` Target modules: q_proj, v_proj`);
|
||||
|
||||
// Simulate training progress
|
||||
console.log('\nTraining progress:');
|
||||
for (let epoch = 1; epoch <= (options.epochs || 3); epoch++) {
|
||||
console.log(` Epoch ${epoch}/${options.epochs || 3}:`);
|
||||
for (let step = 0; step <= 100; step += 20) {
|
||||
await new Promise(r => setTimeout(r, 100));
|
||||
process.stdout.write(`\r Step ${step}/100 - Loss: ${(2.5 - epoch * 0.3 - step * 0.01).toFixed(4)}`);
|
||||
}
|
||||
console.log('');
|
||||
}
|
||||
|
||||
const adapterPath = options.output || join(DEFAULT_CACHE_DIR, 'adapters', adapterName);
|
||||
if (!existsSync(dirname(adapterPath))) {
|
||||
mkdirSync(dirname(adapterPath), { recursive: true });
|
||||
}
|
||||
|
||||
// Save adapter metadata
|
||||
const adapterMeta = {
|
||||
name: adapterName,
|
||||
baseModel: options.base || 'phi-1.5',
|
||||
rank: options.rank || 8,
|
||||
trained: new Date().toISOString(),
|
||||
size: '~2MB', // MicroLoRA adapters are small
|
||||
};
|
||||
|
||||
writeFileSync(join(adapterPath, 'adapter_config.json'), JSON.stringify(adapterMeta, null, 2));
|
||||
|
||||
console.log(`\nAdapter saved to: ${adapterPath}`);
|
||||
console.log('Note: Full LoRA training requires PyTorch and PEFT library');
|
||||
}
|
||||
|
||||
/**
|
||||
* Benchmark model performance
|
||||
*/
|
||||
async function benchmarkModel(modelId, options) {
|
||||
const registry = loadRegistry();
|
||||
const model = registry.models[modelId];
|
||||
|
||||
if (!model) {
|
||||
console.error(`Error: Model "${modelId}" not found`);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
console.log(`\n=== Benchmarking: ${model.name} ===\n`);
|
||||
|
||||
const iterations = options.iterations || 10;
|
||||
const warmup = options.warmup || 2;
|
||||
|
||||
console.log('System Information:');
|
||||
console.log(` CPU: ${cpus()[0].model}`);
|
||||
console.log(` Cores: ${cpus().length}`);
|
||||
console.log(` Memory: ${formatSize(totalmem())}`);
|
||||
console.log('');
|
||||
|
||||
try {
|
||||
const { pipeline, env } = await import('@xenova/transformers');
|
||||
env.cacheDir = DEFAULT_CACHE_DIR;
|
||||
|
||||
const pipelineType = model.type === 'embedding' ? 'feature-extraction' : 'text-generation';
|
||||
|
||||
console.log('Loading model...');
|
||||
const pipe = await pipeline(pipelineType, model.huggingface, {
|
||||
quantized: true,
|
||||
});
|
||||
|
||||
// Warmup
|
||||
console.log(`\nWarmup (${warmup} iterations)...`);
|
||||
for (let i = 0; i < warmup; i++) {
|
||||
if (model.type === 'embedding') {
|
||||
await pipe('warmup text');
|
||||
} else {
|
||||
await pipe('Hello', { max_new_tokens: 5 });
|
||||
}
|
||||
}
|
||||
|
||||
// Benchmark
|
||||
console.log(`\nBenchmarking (${iterations} iterations)...`);
|
||||
const times = [];
|
||||
|
||||
for (let i = 0; i < iterations; i++) {
|
||||
const start = performance.now();
|
||||
|
||||
if (model.type === 'embedding') {
|
||||
await pipe('The quick brown fox jumps over the lazy dog.');
|
||||
} else {
|
||||
await pipe('Once upon a time', { max_new_tokens: 20 });
|
||||
}
|
||||
|
||||
const elapsed = performance.now() - start;
|
||||
times.push(elapsed);
|
||||
process.stdout.write(`\r Iteration ${i + 1}/${iterations}: ${elapsed.toFixed(1)}ms`);
|
||||
}
|
||||
|
||||
console.log('\n');
|
||||
|
||||
// Calculate statistics
|
||||
times.sort((a, b) => a - b);
|
||||
const avg = times.reduce((a, b) => a + b, 0) / times.length;
|
||||
const median = times[Math.floor(times.length / 2)];
|
||||
const p95 = times[Math.floor(times.length * 0.95)];
|
||||
const min = times[0];
|
||||
const max = times[times.length - 1];
|
||||
|
||||
console.log('Results:');
|
||||
console.log(` Average: ${avg.toFixed(2)}ms`);
|
||||
console.log(` Median: ${median.toFixed(2)}ms`);
|
||||
console.log(` P95: ${p95.toFixed(2)}ms`);
|
||||
console.log(` Min: ${min.toFixed(2)}ms`);
|
||||
console.log(` Max: ${max.toFixed(2)}ms`);
|
||||
|
||||
if (model.type === 'embedding') {
|
||||
console.log(` Throughput: ${(1000 / avg).toFixed(1)} embeddings/sec`);
|
||||
} else {
|
||||
console.log(` Throughput: ${(1000 / avg * 20).toFixed(1)} tokens/sec`);
|
||||
}
|
||||
|
||||
// Save results
|
||||
if (options.output) {
|
||||
const results = {
|
||||
model: modelId,
|
||||
timestamp: new Date().toISOString(),
|
||||
system: {
|
||||
cpu: cpus()[0].model,
|
||||
cores: cpus().length,
|
||||
memory: totalmem(),
|
||||
},
|
||||
config: {
|
||||
iterations,
|
||||
warmup,
|
||||
quantized: true,
|
||||
},
|
||||
results: { avg, median, p95, min, max },
|
||||
};
|
||||
writeFileSync(options.output, JSON.stringify(results, null, 2));
|
||||
console.log(`\nResults saved to: ${options.output}`);
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('\nBenchmark failed:', error.message);
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Manage local cache
|
||||
*/
|
||||
async function manageCache(action, options) {
|
||||
console.log(`\n=== Model Cache Management ===\n`);
|
||||
console.log(`Cache directory: ${DEFAULT_CACHE_DIR}\n`);
|
||||
|
||||
if (!existsSync(DEFAULT_CACHE_DIR)) {
|
||||
console.log('Cache directory does not exist.');
|
||||
if (action === 'init') {
|
||||
mkdirSync(DEFAULT_CACHE_DIR, { recursive: true });
|
||||
console.log('Created cache directory.');
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
switch (action) {
|
||||
case 'list':
|
||||
case undefined:
|
||||
listCacheContents();
|
||||
break;
|
||||
case 'clean':
|
||||
cleanCache(options);
|
||||
break;
|
||||
case 'size':
|
||||
showCacheSize();
|
||||
break;
|
||||
case 'init':
|
||||
console.log('Cache directory exists.');
|
||||
break;
|
||||
default:
|
||||
console.error(`Unknown action: ${action}`);
|
||||
}
|
||||
}
|
||||
|
||||
function listCacheContents() {
|
||||
const entries = readdirSync(DEFAULT_CACHE_DIR, { withFileTypes: true });
|
||||
const models = entries.filter(e => e.isDirectory());
|
||||
|
||||
if (models.length === 0) {
|
||||
console.log('No cached models found.');
|
||||
return;
|
||||
}
|
||||
|
||||
console.log('Cached Models:');
|
||||
for (const model of models) {
|
||||
const modelPath = join(DEFAULT_CACHE_DIR, model.name);
|
||||
const size = getDirectorySize(modelPath);
|
||||
console.log(` ${model.name.replace('--', '/')}`);
|
||||
console.log(` Size: ${formatSize(size)}`);
|
||||
}
|
||||
}
|
||||
|
||||
function getDirectorySize(dir) {
|
||||
let size = 0;
|
||||
try {
|
||||
const entries = readdirSync(dir, { withFileTypes: true });
|
||||
for (const entry of entries) {
|
||||
const fullPath = join(dir, entry.name);
|
||||
if (entry.isDirectory()) {
|
||||
size += getDirectorySize(fullPath);
|
||||
} else {
|
||||
size += statSync(fullPath).size;
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
// Ignore errors
|
||||
}
|
||||
return size;
|
||||
}
|
||||
|
||||
function showCacheSize() {
|
||||
const totalSize = getDirectorySize(DEFAULT_CACHE_DIR);
|
||||
console.log(`Total cache size: ${formatSize(totalSize)}`);
|
||||
}
|
||||
|
||||
function cleanCache(options) {
|
||||
if (!options.force) {
|
||||
console.log('This will delete all cached models.');
|
||||
console.log('Use --force to confirm.');
|
||||
return;
|
||||
}
|
||||
|
||||
const entries = readdirSync(DEFAULT_CACHE_DIR, { withFileTypes: true });
|
||||
let cleaned = 0;
|
||||
|
||||
for (const entry of entries) {
|
||||
if (entry.isDirectory()) {
|
||||
const modelPath = join(DEFAULT_CACHE_DIR, entry.name);
|
||||
const { rmSync } = require('fs');
|
||||
rmSync(modelPath, { recursive: true });
|
||||
console.log(` Removed: ${entry.name}`);
|
||||
cleaned++;
|
||||
}
|
||||
}
|
||||
|
||||
console.log(`\nCleaned ${cleaned} cached model(s).`);
|
||||
}
|
||||
|
||||
// ============================================
|
||||
// CLI SETUP
|
||||
// ============================================
|
||||
|
||||
const program = new Command();
|
||||
|
||||
program
|
||||
.name('models-cli')
|
||||
.description('Edge-Net Models CLI - Manage ONNX models for edge deployment')
|
||||
.version('1.0.0');
|
||||
|
||||
program
|
||||
.command('list')
|
||||
.description('List available models')
|
||||
.option('-t, --type <type>', 'Filter by type (embedding, generation)')
|
||||
.option('--tier <tier>', 'Filter by tier (1-4)')
|
||||
.option('--cached', 'Show only cached models')
|
||||
.action(listModels);
|
||||
|
||||
program
|
||||
.command('download <model>')
|
||||
.description('Download a model from HuggingFace')
|
||||
.option('-f, --force', 'Force re-download')
|
||||
.option('-q, --quantize <type>', 'Quantization type (int4, int8, fp16, fp32)', 'int8')
|
||||
.option('--verify', 'Verify model after download')
|
||||
.action(downloadModel);
|
||||
|
||||
program
|
||||
.command('optimize <model>')
|
||||
.description('Optimize a model for edge deployment')
|
||||
.option('-q, --quantize <type>', 'Quantization type (int4, int8, fp16)', 'int8')
|
||||
.option('-p, --prune <sparsity>', 'Pruning sparsity (0-1)')
|
||||
.option('-o, --output <path>', 'Output directory')
|
||||
.action(optimizeModel);
|
||||
|
||||
program
|
||||
.command('upload <model>')
|
||||
.description('Upload optimized model to registry (GCS + IPFS)')
|
||||
.option('--ipfs', 'Also pin to IPFS')
|
||||
.option('-q, --quantize <type>', 'Quantization variant to upload')
|
||||
.action(uploadModel);
|
||||
|
||||
program
|
||||
.command('train <adapter>')
|
||||
.description('Train a MicroLoRA adapter')
|
||||
.option('-b, --base <model>', 'Base model to adapt', 'phi-1.5')
|
||||
.option('-d, --dataset <path>', 'Training dataset path')
|
||||
.option('-r, --rank <rank>', 'LoRA rank', '8')
|
||||
.option('-e, --epochs <epochs>', 'Training epochs', '3')
|
||||
.option('-o, --output <path>', 'Output path for adapter')
|
||||
.action(trainAdapter);
|
||||
|
||||
program
|
||||
.command('benchmark <model>')
|
||||
.description('Run performance benchmarks')
|
||||
.option('-i, --iterations <n>', 'Number of iterations', '10')
|
||||
.option('-w, --warmup <n>', 'Warmup iterations', '2')
|
||||
.option('-o, --output <path>', 'Save results to JSON file')
|
||||
.action(benchmarkModel);
|
||||
|
||||
program
|
||||
.command('cache [action]')
|
||||
.description('Manage local model cache (list, clean, size, init)')
|
||||
.option('-f, --force', 'Force action without confirmation')
|
||||
.action(manageCache);
|
||||
|
||||
// Parse and execute
|
||||
program.parse();
|
||||
214
vendor/ruvector/examples/edge-net/pkg/models/registry.json
vendored
Normal file
214
vendor/ruvector/examples/edge-net/pkg/models/registry.json
vendored
Normal file
@@ -0,0 +1,214 @@
|
||||
{
|
||||
"version": "1.0.0",
|
||||
"updated": "2026-01-03T00:00:00.000Z",
|
||||
"gcs_bucket": "ruvector-models",
|
||||
"ipfs_gateway": "https://ipfs.io/ipfs",
|
||||
"models": {
|
||||
"minilm-l6": {
|
||||
"name": "MiniLM-L6-v2",
|
||||
"type": "embedding",
|
||||
"huggingface": "Xenova/all-MiniLM-L6-v2",
|
||||
"dimensions": 384,
|
||||
"size": "22MB",
|
||||
"tier": 1,
|
||||
"quantized": ["int8", "fp16"],
|
||||
"description": "Fast, good quality embeddings for edge deployment",
|
||||
"recommended_for": ["edge-minimal", "low-memory"],
|
||||
"artifacts": {}
|
||||
},
|
||||
"e5-small": {
|
||||
"name": "E5-Small-v2",
|
||||
"type": "embedding",
|
||||
"huggingface": "Xenova/e5-small-v2",
|
||||
"dimensions": 384,
|
||||
"size": "28MB",
|
||||
"tier": 1,
|
||||
"quantized": ["int8", "fp16"],
|
||||
"description": "Microsoft E5 - excellent for retrieval tasks",
|
||||
"recommended_for": ["retrieval", "semantic-search"],
|
||||
"artifacts": {}
|
||||
},
|
||||
"bge-small": {
|
||||
"name": "BGE-Small-EN-v1.5",
|
||||
"type": "embedding",
|
||||
"huggingface": "Xenova/bge-small-en-v1.5",
|
||||
"dimensions": 384,
|
||||
"size": "33MB",
|
||||
"tier": 2,
|
||||
"quantized": ["int8", "fp16"],
|
||||
"description": "BAAI BGE - best for retrieval and ranking",
|
||||
"recommended_for": ["retrieval", "reranking"],
|
||||
"artifacts": {}
|
||||
},
|
||||
"gte-small": {
|
||||
"name": "GTE-Small",
|
||||
"type": "embedding",
|
||||
"huggingface": "Xenova/gte-small",
|
||||
"dimensions": 384,
|
||||
"size": "67MB",
|
||||
"tier": 2,
|
||||
"quantized": ["int8", "fp16"],
|
||||
"description": "General Text Embeddings - high quality",
|
||||
"recommended_for": ["general", "quality"],
|
||||
"artifacts": {}
|
||||
},
|
||||
"gte-base": {
|
||||
"name": "GTE-Base",
|
||||
"type": "embedding",
|
||||
"huggingface": "Xenova/gte-base",
|
||||
"dimensions": 768,
|
||||
"size": "100MB",
|
||||
"tier": 3,
|
||||
"quantized": ["int8", "fp16"],
|
||||
"description": "GTE Base - 768 dimensions for higher quality",
|
||||
"recommended_for": ["cloud", "high-quality"],
|
||||
"artifacts": {}
|
||||
},
|
||||
"multilingual-e5": {
|
||||
"name": "Multilingual-E5-Small",
|
||||
"type": "embedding",
|
||||
"huggingface": "Xenova/multilingual-e5-small",
|
||||
"dimensions": 384,
|
||||
"size": "118MB",
|
||||
"tier": 3,
|
||||
"quantized": ["int8", "fp16"],
|
||||
"description": "Supports 100+ languages",
|
||||
"recommended_for": ["multilingual", "international"],
|
||||
"artifacts": {}
|
||||
},
|
||||
"distilgpt2": {
|
||||
"name": "DistilGPT2",
|
||||
"type": "generation",
|
||||
"huggingface": "Xenova/distilgpt2",
|
||||
"size": "82MB",
|
||||
"tier": 1,
|
||||
"quantized": ["int8", "int4", "fp16"],
|
||||
"capabilities": ["general", "completion"],
|
||||
"description": "Fast distilled GPT-2 for text generation",
|
||||
"recommended_for": ["edge", "fast-inference"],
|
||||
"artifacts": {}
|
||||
},
|
||||
"tinystories": {
|
||||
"name": "TinyStories-33M",
|
||||
"type": "generation",
|
||||
"huggingface": "Xenova/TinyStories-33M",
|
||||
"size": "65MB",
|
||||
"tier": 1,
|
||||
"quantized": ["int8", "int4"],
|
||||
"capabilities": ["stories", "creative"],
|
||||
"description": "Ultra-small model trained on children's stories",
|
||||
"recommended_for": ["creative", "stories", "minimal"],
|
||||
"artifacts": {}
|
||||
},
|
||||
"starcoder-tiny": {
|
||||
"name": "TinyStarCoder-Py",
|
||||
"type": "generation",
|
||||
"huggingface": "Xenova/tiny_starcoder_py",
|
||||
"size": "40MB",
|
||||
"tier": 1,
|
||||
"quantized": ["int8", "int4"],
|
||||
"capabilities": ["code", "python"],
|
||||
"description": "Ultra-small Python code generation",
|
||||
"recommended_for": ["code", "python", "edge"],
|
||||
"artifacts": {}
|
||||
},
|
||||
"phi-1.5": {
|
||||
"name": "Phi-1.5",
|
||||
"type": "generation",
|
||||
"huggingface": "Xenova/phi-1_5",
|
||||
"size": "280MB",
|
||||
"tier": 2,
|
||||
"quantized": ["int8", "int4", "fp16"],
|
||||
"capabilities": ["code", "reasoning", "math"],
|
||||
"description": "Microsoft Phi-1.5 - excellent code and reasoning",
|
||||
"recommended_for": ["code", "reasoning", "balanced"],
|
||||
"artifacts": {}
|
||||
},
|
||||
"codegen-350m": {
|
||||
"name": "CodeGen-350M-Mono",
|
||||
"type": "generation",
|
||||
"huggingface": "Xenova/codegen-350M-mono",
|
||||
"size": "320MB",
|
||||
"tier": 2,
|
||||
"quantized": ["int8", "int4", "fp16"],
|
||||
"capabilities": ["code", "python"],
|
||||
"description": "Salesforce CodeGen - Python specialist",
|
||||
"recommended_for": ["code", "python"],
|
||||
"artifacts": {}
|
||||
},
|
||||
"qwen-0.5b": {
|
||||
"name": "Qwen-1.5-0.5B",
|
||||
"type": "generation",
|
||||
"huggingface": "Xenova/Qwen1.5-0.5B",
|
||||
"size": "430MB",
|
||||
"tier": 3,
|
||||
"quantized": ["int8", "int4", "fp16"],
|
||||
"capabilities": ["multilingual", "general", "code"],
|
||||
"description": "Alibaba Qwen 0.5B - multilingual capabilities",
|
||||
"recommended_for": ["multilingual", "general"],
|
||||
"artifacts": {}
|
||||
},
|
||||
"phi-2": {
|
||||
"name": "Phi-2",
|
||||
"type": "generation",
|
||||
"huggingface": "Xenova/phi-2",
|
||||
"size": "550MB",
|
||||
"tier": 3,
|
||||
"quantized": ["int8", "int4", "fp16"],
|
||||
"capabilities": ["code", "reasoning", "math", "general"],
|
||||
"description": "Microsoft Phi-2 - advanced reasoning model",
|
||||
"recommended_for": ["reasoning", "code", "quality"],
|
||||
"artifacts": {}
|
||||
},
|
||||
"gemma-2b": {
|
||||
"name": "Gemma-2B-IT",
|
||||
"type": "generation",
|
||||
"huggingface": "Xenova/gemma-2b-it",
|
||||
"size": "1.1GB",
|
||||
"tier": 4,
|
||||
"quantized": ["int8", "int4", "fp16"],
|
||||
"capabilities": ["instruction", "general", "code", "reasoning"],
|
||||
"description": "Google Gemma 2B instruction-tuned",
|
||||
"recommended_for": ["cloud", "high-quality", "instruction"],
|
||||
"artifacts": {}
|
||||
}
|
||||
},
|
||||
"profiles": {
|
||||
"edge-minimal": {
|
||||
"description": "Minimal footprint for constrained edge devices",
|
||||
"embedding": "minilm-l6",
|
||||
"generation": "tinystories",
|
||||
"total_size": "~87MB",
|
||||
"quantization": "int4"
|
||||
},
|
||||
"edge-balanced": {
|
||||
"description": "Best quality/size ratio for edge deployment",
|
||||
"embedding": "e5-small",
|
||||
"generation": "phi-1.5",
|
||||
"total_size": "~308MB",
|
||||
"quantization": "int8"
|
||||
},
|
||||
"edge-code": {
|
||||
"description": "Optimized for code generation tasks",
|
||||
"embedding": "bge-small",
|
||||
"generation": "starcoder-tiny",
|
||||
"total_size": "~73MB",
|
||||
"quantization": "int8"
|
||||
},
|
||||
"edge-full": {
|
||||
"description": "Maximum quality on edge devices",
|
||||
"embedding": "gte-base",
|
||||
"generation": "phi-2",
|
||||
"total_size": "~650MB",
|
||||
"quantization": "int8"
|
||||
},
|
||||
"cloud-optimal": {
|
||||
"description": "Best quality for cloud/server deployment",
|
||||
"embedding": "gte-base",
|
||||
"generation": "gemma-2b",
|
||||
"total_size": "~1.2GB",
|
||||
"quantization": "fp16"
|
||||
}
|
||||
},
|
||||
"adapters": {}
|
||||
}
|
||||
1418
vendor/ruvector/examples/edge-net/pkg/models/training-utils.js
vendored
Normal file
1418
vendor/ruvector/examples/edge-net/pkg/models/training-utils.js
vendored
Normal file
File diff suppressed because it is too large
Load Diff
1025
vendor/ruvector/examples/edge-net/pkg/models/wasm-core.js
vendored
Normal file
1025
vendor/ruvector/examples/edge-net/pkg/models/wasm-core.js
vendored
Normal file
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user