Pixeltable and pgvector

import pixeltable as pxt
from pixeltable.functions.document import document_splitter
from pixeltable.functions.huggingface import sentence_transformer

TableModel = pxt.model_base()
embed = sentence_transformer.using(
    model_id='sentence-transformers/all-MiniLM-L6-v2'
)

class Docs(TableModel, name='docs'):
    document: pxt.Document
    title: pxt.String

class Chunks(
    TableModel,
    name='chunks',
    base=Docs,
    iterator=document_splitter(
        Docs.document, separators='sentence,token_limit', limit=512
    ),
):
    __indexes__ = [pxt.EmbeddingIndex(text, embedding=embed)]

if __name__ == '__main__':
    # After: pxt schema update app.py search
    docs = pxt.get_table('search.docs')
    docs.insert([{'document': 'papers/ml.pdf', 'title': 'Research Paper'}])
    chunks = pxt.get_table('search.chunks')
    sim = chunks.text.similarity(string='machine learning algorithms')
    chunks.order_by(sim, asc=False).limit(10).select(chunks.text, chunks.title)

-- github.com/pgvector/pgvector
CREATE EXTENSION IF NOT EXISTS vector;

CREATE TABLE items (
  id bigserial PRIMARY KEY,
  title text,
  embedding vector(384)
);

CREATE INDEX ON items USING hnsw (embedding vector_l2_ops);

-- You still chunk the PDF and call an embedder in application code.
INSERT INTO items (title, embedding) VALUES (
  'Research Paper',
  '[0.1, 0.2, ...]'
);

SELECT id, title
FROM items
ORDER BY embedding <-> '[0.11, 0.19, ...]'
LIMIT 10;