Squashed 'vendor/ruvector/' content from commit b64c2172
git-subtree-dir: vendor/ruvector git-subtree-split: b64c21726f2bb37286d9ee36a7869fef60cc6900
This commit is contained in:
ruv
256
crates/ruvector-postgres/examples/sparse_example.sql
Normal file
256
crates/ruvector-postgres/examples/sparse_example.sql
Normal file
@@ -0,0 +1,256 @@
|
||||
-- Sparse Vectors Example Usage
|
||||
-- This file demonstrates the sparse vector functionality
|
||||
|
||||
-- ============================================================================
|
||||
-- Setup
|
||||
-- ============================================================================
|
||||
|
||||
-- Create extension (assuming already installed)
|
||||
-- CREATE EXTENSION IF NOT EXISTS ruvector_postgres;
|
||||
|
||||
-- Create sample tables
|
||||
CREATE TABLE IF NOT EXISTS sparse_documents (
|
||||
id SERIAL PRIMARY KEY,
|
||||
title TEXT,
|
||||
content TEXT,
|
||||
sparse_embedding sparsevec,
|
||||
created_at TIMESTAMP DEFAULT NOW()
|
||||
);
|
||||
|
||||
-- ============================================================================
|
||||
-- Inserting Data
|
||||
-- ============================================================================
|
||||
|
||||
-- Method 1: String format
|
||||
INSERT INTO sparse_documents (title, content, sparse_embedding) VALUES
|
||||
('Machine Learning Basics',
|
||||
'Introduction to neural networks and deep learning',
|
||||
'{1024:0.5, 2048:0.3, 4096:0.8, 8192:0.2}'::sparsevec),
|
||||
|
||||
('Natural Language Processing',
|
||||
'Text processing and language models',
|
||||
'{1024:0.3, 3072:0.7, 4096:0.4, 9216:0.6}'::sparsevec),
|
||||
|
||||
('Computer Vision',
|
||||
'Image recognition and object detection',
|
||||
'{2048:0.9, 5120:0.4, 6144:0.5, 7168:0.3}'::sparsevec);
|
||||
|
||||
-- Method 2: Array construction
|
||||
INSERT INTO sparse_documents (title, content, sparse_embedding) VALUES
|
||||
('Reinforcement Learning',
|
||||
'Q-learning and policy gradients',
|
||||
ruvector_to_sparse(
|
||||
ARRAY[1024, 4096, 10240]::int[],
|
||||
ARRAY[0.6, 0.8, 0.4]::real[],
|
||||
30000
|
||||
));
|
||||
|
||||
-- Method 3: Convert from dense
|
||||
INSERT INTO sparse_documents (title, sparse_embedding)
|
||||
SELECT 'From Dense Vector',
|
||||
ruvector_dense_to_sparse(
|
||||
ARRAY[0, 0.5, 0, 0.3, 0, 0, 0.8, 0, 0, 0.2]::real[]
|
||||
);
|
||||
|
||||
-- ============================================================================
|
||||
-- Basic Queries
|
||||
-- ============================================================================
|
||||
|
||||
-- View all documents with sparse vectors
|
||||
SELECT id, title,
|
||||
ruvector_sparse_nnz(sparse_embedding) as num_nonzero,
|
||||
ruvector_sparse_dim(sparse_embedding) as dimension,
|
||||
ruvector_sparse_norm(sparse_embedding) as l2_norm
|
||||
FROM sparse_documents;
|
||||
|
||||
-- ============================================================================
|
||||
-- Similarity Search
|
||||
-- ============================================================================
|
||||
|
||||
-- Define a query vector
|
||||
WITH query AS (
|
||||
SELECT '{1024:0.5, 2048:0.3, 4096:0.8}'::sparsevec AS query_vec
|
||||
)
|
||||
-- Search by dot product (inner product)
|
||||
SELECT d.id, d.title,
|
||||
ruvector_sparse_dot(d.sparse_embedding, q.query_vec) AS dot_product,
|
||||
ruvector_sparse_cosine(d.sparse_embedding, q.query_vec) AS cosine_sim,
|
||||
ruvector_sparse_euclidean(d.sparse_embedding, q.query_vec) AS euclidean_dist
|
||||
FROM sparse_documents d, query q
|
||||
ORDER BY dot_product DESC
|
||||
LIMIT 5;
|
||||
|
||||
-- Find documents with high cosine similarity
|
||||
WITH query AS (
|
||||
SELECT '{1024:0.5, 4096:0.8}'::sparsevec AS query_vec
|
||||
)
|
||||
SELECT id, title,
|
||||
ruvector_sparse_cosine(sparse_embedding, query_vec) AS similarity
|
||||
FROM sparse_documents, query
|
||||
WHERE ruvector_sparse_cosine(sparse_embedding, query_vec) > 0.3
|
||||
ORDER BY similarity DESC;
|
||||
|
||||
-- ============================================================================
|
||||
-- Sparsification Operations
|
||||
-- ============================================================================
|
||||
|
||||
-- Keep only top-k elements
|
||||
SELECT id, title,
|
||||
sparse_embedding AS original,
|
||||
ruvector_sparse_top_k(sparse_embedding, 2) AS top_2_elements
|
||||
FROM sparse_documents
|
||||
LIMIT 3;
|
||||
|
||||
-- Prune small values
|
||||
SELECT id, title,
|
||||
sparse_embedding AS original,
|
||||
ruvector_sparse_prune(sparse_embedding, 0.4) AS pruned
|
||||
FROM sparse_documents
|
||||
LIMIT 3;
|
||||
|
||||
-- ============================================================================
|
||||
-- BM25 Text Search Example
|
||||
-- ============================================================================
|
||||
|
||||
-- Create BM25-specific table
|
||||
CREATE TABLE IF NOT EXISTS bm25_articles (
|
||||
id SERIAL PRIMARY KEY,
|
||||
title TEXT,
|
||||
content TEXT,
|
||||
term_frequencies sparsevec, -- TF values
|
||||
doc_length REAL
|
||||
);
|
||||
|
||||
-- Insert sample documents with term frequencies
|
||||
INSERT INTO bm25_articles (title, content, term_frequencies, doc_length) VALUES
|
||||
('AI Research Paper',
|
||||
'Deep learning models for natural language processing',
|
||||
'{100:2.0, 200:1.0, 300:3.0, 400:1.0}'::sparsevec, -- TF values
|
||||
7.0),
|
||||
|
||||
('Machine Learning Tutorial',
|
||||
'Introduction to supervised and unsupervised learning',
|
||||
'{100:1.0, 250:2.0, 300:1.0, 500:2.0}'::sparsevec,
|
||||
6.0),
|
||||
|
||||
('Data Science Guide',
|
||||
'Statistical analysis and data visualization techniques',
|
||||
'{150:1.0, 250:1.0, 350:2.0, 450:1.0}'::sparsevec,
|
||||
6.0);
|
||||
|
||||
-- BM25 search
|
||||
WITH
|
||||
query AS (
|
||||
-- Query with IDF weights (normally computed from corpus)
|
||||
SELECT '{100:1.5, 300:2.0, 400:1.2}'::sparsevec AS query_idf
|
||||
),
|
||||
collection_stats AS (
|
||||
SELECT AVG(doc_length) AS avg_doc_len
|
||||
FROM bm25_articles
|
||||
)
|
||||
SELECT a.id, a.title,
|
||||
ruvector_sparse_bm25(
|
||||
q.query_idf,
|
||||
a.term_frequencies,
|
||||
a.doc_length,
|
||||
cs.avg_doc_len,
|
||||
1.2, -- k1 parameter
|
||||
0.75 -- b parameter
|
||||
) AS bm25_score
|
||||
FROM bm25_articles a, query q, collection_stats cs
|
||||
ORDER BY bm25_score DESC
|
||||
LIMIT 5;
|
||||
|
||||
-- ============================================================================
|
||||
-- Hybrid Search (Dense + Sparse)
|
||||
-- ============================================================================
|
||||
|
||||
-- Create hybrid table (requires vector extension)
|
||||
-- Uncomment if you have dense vector support
|
||||
/*
|
||||
CREATE TABLE IF NOT EXISTS hybrid_documents (
|
||||
id SERIAL PRIMARY KEY,
|
||||
title TEXT,
|
||||
dense_embedding vector(768),
|
||||
sparse_embedding sparsevec
|
||||
);
|
||||
|
||||
-- Hybrid search combining both signals
|
||||
WITH query AS (
|
||||
SELECT
|
||||
random_vector(768) AS query_dense, -- Replace with actual query
|
||||
'{1024:0.5, 2048:0.3}'::sparsevec AS query_sparse
|
||||
)
|
||||
SELECT id, title,
|
||||
0.7 * (1 - (dense_embedding <=> query_dense)) + -- Dense similarity
|
||||
0.3 * ruvector_sparse_dot(sparse_embedding, query_sparse) AS hybrid_score
|
||||
FROM hybrid_documents, query
|
||||
ORDER BY hybrid_score DESC
|
||||
LIMIT 10;
|
||||
*/
|
||||
|
||||
-- ============================================================================
|
||||
-- Utility Operations
|
||||
-- ============================================================================
|
||||
|
||||
-- Convert sparse to dense
|
||||
SELECT id, title,
|
||||
ruvector_sparse_to_dense(sparse_embedding) AS dense_array
|
||||
FROM sparse_documents
|
||||
LIMIT 3;
|
||||
|
||||
-- Get vector statistics
|
||||
SELECT
|
||||
COUNT(*) as num_documents,
|
||||
AVG(ruvector_sparse_nnz(sparse_embedding)) AS avg_nonzero,
|
||||
MIN(ruvector_sparse_nnz(sparse_embedding)) AS min_nonzero,
|
||||
MAX(ruvector_sparse_nnz(sparse_embedding)) AS max_nonzero,
|
||||
AVG(ruvector_sparse_norm(sparse_embedding)) AS avg_norm
|
||||
FROM sparse_documents;
|
||||
|
||||
-- Find documents with similar sparsity
|
||||
WITH target AS (
|
||||
SELECT sparse_embedding, ruvector_sparse_nnz(sparse_embedding) AS target_nnz
|
||||
FROM sparse_documents
|
||||
WHERE id = 1
|
||||
)
|
||||
SELECT d.id, d.title,
|
||||
ruvector_sparse_nnz(d.sparse_embedding) AS doc_nnz,
|
||||
ABS(ruvector_sparse_nnz(d.sparse_embedding) - t.target_nnz) AS nnz_diff
|
||||
FROM sparse_documents d, target t
|
||||
WHERE d.id != 1
|
||||
ORDER BY nnz_diff
|
||||
LIMIT 5;
|
||||
|
||||
-- ============================================================================
|
||||
-- Performance Analysis
|
||||
-- ============================================================================
|
||||
|
||||
-- Check storage size
|
||||
SELECT id, title,
|
||||
pg_column_size(sparse_embedding) AS sparse_bytes,
|
||||
ruvector_sparse_nnz(sparse_embedding) AS num_nonzero,
|
||||
pg_column_size(sparse_embedding)::float /
|
||||
GREATEST(ruvector_sparse_nnz(sparse_embedding), 1) AS bytes_per_element
|
||||
FROM sparse_documents
|
||||
ORDER BY sparse_bytes DESC;
|
||||
|
||||
-- Batch similarity computation
|
||||
EXPLAIN ANALYZE
|
||||
WITH queries AS (
|
||||
SELECT generate_series(1, 3) AS query_id,
|
||||
'{1024:0.5, 2048:0.3}'::sparsevec AS query_vec
|
||||
)
|
||||
SELECT q.query_id, d.id, d.title,
|
||||
ruvector_sparse_dot(d.sparse_embedding, q.query_vec) AS score
|
||||
FROM sparse_documents d
|
||||
CROSS JOIN queries q
|
||||
ORDER BY q.query_id, score DESC;
|
||||
|
||||
-- ============================================================================
|
||||
-- Cleanup (optional)
|
||||
-- ============================================================================
|
||||
|
||||
-- DROP TABLE IF EXISTS sparse_documents CASCADE;
|
||||
-- DROP TABLE IF EXISTS bm25_articles CASCADE;
|
||||
-- DROP TABLE IF EXISTS hybrid_documents CASCADE;
|
||||
Reference in New Issue
Block a user