Pixeltable vs Cloudflare

import pixeltable as pxt
from pixeltable.functions.document import document_splitter
from pixeltable.functions.huggingface import sentence_transformer

TableModel = pxt.model_base()
embed = sentence_transformer.using(model_id='sentence-transformers/all-MiniLM-L6-v2')

class Docs(TableModel, name='docs'):
    document: pxt.Document
    title: pxt.String

class Chunks(
    TableModel,
    name='chunks',
    base=Docs,
    iterator=document_splitter(Docs.document, separators='sentence', limit=512),
):
    __indexes__ = [pxt.EmbeddingIndex(text, embedding=embed)]

# pxt schema update app.py search
docs = pxt.get_table('search.docs')
docs.insert([{'document': 'specs.pdf', 'title': 'System Specs'}])

# Query vector similarity in-engine
chunks = pxt.get_table('search.chunks')
sim = chunks.text.similarity(string='hardware requirements')
results = chunks.order_by(sim, asc=False).limit(3).select(chunks.text, chunks.title)

// Cloudflare Worker (index.ts)
export interface Env {
  DB: D1Database;
  VECTORS: VectorizeIndex;
  AI: Ai;
}

export default {
  async fetch(request: Request, env: Env): Promise<Response> {
    const { title, text } = await request.json();
    const docId = crypto.randomUUID();
    
    // 1. Insert into Cloudflare D1
    await env.DB.prepare(
      'INSERT INTO docs (id, title, text) VALUES (?, ?, ?)'
    ).bind(docId, title, text).run();

    // 2. Generate embedding via Workers AI
    const { data } = await env.AI.run('@cf/baai/bge-small-en-v1.5', {
      text: [text]
    });

    // 3. Insert vector into Cloudflare Vectorize
    await env.VECTORS.upsert([
      { id: docId, values: data[0], metadata: { title } }
    ]);

    return Response.json({ success: true, id: docId });
    // Must manage: chunking by hand, retry on Workers AI limits, consistency between D1 and Vectorize
  }
};

# Upgrade embedder in app.py:
new_embed = sentence_transformer.using(model_id='BAAI/bge-large-en-v1.5')

class Chunks(
    TableModel,
    name='chunks',
    base=Docs,
    iterator=document_splitter(Docs.document, separators='sentence', limit=512),
):
    __indexes__ = [pxt.EmbeddingIndex(text, embedding=new_embed)]

# Run: pxt schema update app.py search
# Pixeltable calculates missing embeddings and updates the index incrementally.

// Must write and execute a custom migration Worker:
async function backfillEmbeddings(env: Env) {
  let offset = 0;
  const limit = 50;
  
  while (true) {
    const { results } = await env.DB.prepare(
      'SELECT id, text, title FROM docs LIMIT ? OFFSET ?'
    ).bind(limit, offset).all();
    
    if (!results || results.length === 0) break;
    
    for (const doc of results) {
      const { data } = await env.AI.run('@cf/baai/bge-large-en-v1.5', {
        text: [doc.text as string]
      });
      await env.VECTORS.upsert([
        { id: doc.id as string, values: data[0], metadata: { title: doc.title } }
      ]);
    }
    offset += limit;
  }
}
// Risk: Worker CPU execution timeouts (50ms on free tier), rate limit limits, no rollback