Iterate on Your DataNot Your Infrastructure

1import pixeltable as pxt
2from pixeltable.functions import openai, image as pxt_image
3from pixeltable.functions.anthropic import messages, invoke_tools
4from pixeltable.functions.document import document_splitter
5from pixeltable.functions.huggingface import sentence_transformer, clip
6from pixeltable.functions.video import extract_audio, frame_iterator
7import config, functions
8
9pxt.create_dir("app", if_exists="ignore")
10sentence_embed = sentence_transformer.using(model_id=config.EMBEDDING_MODEL_ID)
11
12# ── 1. Document Pipeline ─────────────────────────────────────
13documents = pxt.create_table("app.documents", {
14    "document": pxt.Document, "timestamp": pxt.Timestamp,
15})
16chunks = pxt.create_view("app.chunks", documents,
17    iterator=document_splitter(document=documents.document,
18        separators="page, sentence", metadata="title, heading, page"),
19)
20chunks.add_embedding_index("text", string_embed=sentence_embed)
21
22@pxt.query
23def search_documents(query_text: str):
24    sim = chunks.text.similarity(query_text)
25    return chunks.where(sim > 0.5).order_by(sim, asc=False).limit(20)
26
27# ── 2. Image Pipeline ────────────────────────────────────────
28images = pxt.create_table("app.images", {
29    "image": pxt.Image, "timestamp": pxt.Timestamp,
30})
31images.add_computed_column(
32    thumbnail=pxt_image.b64_encode(pxt_image.thumbnail(images.image, size=(320, 320)))
33)
34images.add_embedding_index("image",
35    embedding=clip.using(model_id=config.CLIP_MODEL_ID))
36
37# ── 3. Video Pipeline ────────────────────────────────────────
38videos = pxt.create_table("app.videos", {
39    "video": pxt.Video, "timestamp": pxt.Timestamp,
40})
41video_frames = pxt.create_view("app.video_frames", videos,
42    iterator=frame_iterator(video=videos.video, keyframes_only=True))
43video_frames.add_embedding_index("frame",
44    embedding=clip.using(model_id=config.CLIP_MODEL_ID))
45videos.add_computed_column(audio=extract_audio(videos.video, format="mp3"))
46# audio → Whisper transcription → sentence splitting → embedding (chained views)
47
48# ── 4. Chat History ──────────────────────────────────────────
49chat_history = pxt.create_table("app.chat_history", {
50    "role": pxt.String, "content": pxt.String,
51    "conversation_id": pxt.String, "timestamp": pxt.Timestamp,
52})
53chat_history.add_embedding_index("content", string_embed=sentence_embed)
54
55# ── 5. Agent Pipeline (8-step tool-calling workflow) ─────────
56tools = pxt.tools(functions.web_search, search_video_transcripts)
57
58agent = pxt.create_table("app.agent", {
59    "prompt": pxt.String, "timestamp": pxt.Timestamp,
60    "initial_system_prompt": pxt.String,
61    "final_system_prompt": pxt.String,
62    "max_tokens": pxt.Int, "temperature": pxt.Float,
63})
64
65# Step 1: Initial LLM call with tool selection
66agent.add_computed_column(
67    initial_response=messages(
68        model=config.CLAUDE_MODEL_ID,
69        messages=[{"role": "user", "content": agent.prompt}],
70        tools=tools, tool_choice=tools.choice(required=True),
71    )
72)
73# Step 2: Execute selected tools
74agent.add_computed_column(tool_output=invoke_tools(tools, agent.initial_response))
75# Step 3: Parallel RAG context retrieval
76agent.add_computed_column(doc_context=search_documents(agent.prompt))
77agent.add_computed_column(image_context=search_images(agent.prompt))
78agent.add_computed_column(video_frame_context=search_video_frames(agent.prompt))
79agent.add_computed_column(chat_memory_context=search_chat_history(agent.prompt))
80# Steps 4-6: Assemble multimodal context + final messages
81agent.add_computed_column(
82    multimodal_context=functions.assemble_context(
83        agent.prompt, agent.tool_output, agent.doc_context, agent.chat_memory_context,
84    )
85)
86agent.add_computed_column(
87    final_messages=functions.assemble_final_messages(
88        agent.history_context, agent.multimodal_context,
89        image_context=agent.image_context, video_frame_context=agent.video_frame_context,
90    )
91)
92# Step 7: Final LLM reasoning
93agent.add_computed_column(
94    final_response=messages(model=config.CLAUDE_MODEL_ID, messages=agent.final_messages)
95)
96# Step 8: Extract answer
97agent.add_computed_column(answer=agent.final_response.content[0].text)