ceeae77e7d
Swap the SQLite backing store for petalbrain Postgres + pgvector, modeled on vault-mcp. All second-brain relational tables now live in the `second_brain` schema (owned by the lovebug role); embeddings are written to the shared public.embeddings table. Locked design decisions (per Travis): - DB: existing petalbrain Postgres, second_brain schema, lovebug role. - Connection: containerized homelab-postgres:5432, plain psycopg_pool (min=1/max=10), no PgBouncer. - ORM stays SQLAlchemy; int autoincrement PKs + naive UTC DateTime. - Embeddings: reuse shared public.embeddings keyed by (source_schema='second_brain', source_table='extractions', source_id, model='nomic-embed-text'). Summaries only for this round. - Pipeline: chunk_text → Ollama nomic-embed-text → delete-before-insert upsert, with graceful degradation (no DB / no Ollama → log + skip). - Alembic stands up second-brain's own schema; public.embeddings stays out-of-band. - File-based wiki compiler is unchanged. No SQLite data import — starting clean. This commit is the scaffolding only; `alembic upgrade head` and a smoke test of the embedding path are the next checkpoint.
118 lines
3.9 KiB
Python
118 lines
3.9 KiB
Python
"""v1 second_brain: sources, extractions, wiki_pages
|
|
|
|
Revision ID: 4a1e2f6c9d10
|
|
Revises:
|
|
Create Date: 2026-05-24 00:00:00.000000
|
|
|
|
Emits the v1 second_brain schema: the relational backbone of the
|
|
extraction pipeline. Enum types (`source_type`, `source_status`) live
|
|
inside the schema so they don't collide with anything else in the
|
|
petalbrain cluster.
|
|
|
|
env.py creates the second_brain schema and pins alembic_version to it
|
|
before this migration runs. Below we re-issue CREATE SCHEMA IF NOT
|
|
EXISTS defensively and SET search_path so unqualified names resolve.
|
|
|
|
Vector indexing lives in public.embeddings — managed out-of-band, not
|
|
by this migration tree.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from collections.abc import Sequence
|
|
|
|
from alembic import op
|
|
|
|
revision: str = "4a1e2f6c9d10"
|
|
down_revision: str | Sequence[str] | None = None
|
|
branch_labels: str | Sequence[str] | None = None
|
|
depends_on: str | Sequence[str] | None = None
|
|
|
|
|
|
def upgrade() -> None:
|
|
"""Create the v1 second_brain schema."""
|
|
op.execute("CREATE SCHEMA IF NOT EXISTS second_brain")
|
|
op.execute("SET search_path TO second_brain")
|
|
|
|
op.execute(
|
|
"CREATE TYPE source_type AS ENUM ('video', 'article')"
|
|
)
|
|
op.execute(
|
|
"""
|
|
CREATE TYPE source_status AS ENUM (
|
|
'pending', 'pulled', 'transcribed', 'analyzed',
|
|
'accepted', 'published', 'failed'
|
|
)
|
|
"""
|
|
)
|
|
|
|
op.execute(
|
|
"""
|
|
CREATE TABLE sources (
|
|
id SERIAL PRIMARY KEY,
|
|
url VARCHAR(2000) NOT NULL UNIQUE,
|
|
title VARCHAR(500),
|
|
source_type source_type NOT NULL DEFAULT 'video',
|
|
domain VARCHAR(50) NOT NULL DEFAULT 'development',
|
|
focus TEXT,
|
|
status source_status NOT NULL DEFAULT 'pending',
|
|
media_path VARCHAR(1000),
|
|
transcript_path VARCHAR(1000),
|
|
transcript_text TEXT,
|
|
published_at TIMESTAMP,
|
|
duration_seconds INTEGER,
|
|
author VARCHAR(200),
|
|
ingested_at TIMESTAMP NOT NULL DEFAULT (now() AT TIME ZONE 'UTC'),
|
|
updated_at TIMESTAMP,
|
|
error_message TEXT
|
|
)
|
|
"""
|
|
)
|
|
op.execute("CREATE INDEX idx_sources_status ON sources(status)")
|
|
op.execute("CREATE INDEX idx_sources_domain ON sources(domain)")
|
|
|
|
op.execute(
|
|
"""
|
|
CREATE TABLE extractions (
|
|
id SERIAL PRIMARY KEY,
|
|
source_id INTEGER NOT NULL UNIQUE
|
|
REFERENCES sources(id) ON DELETE CASCADE,
|
|
summary TEXT,
|
|
key_points JSON,
|
|
entities JSON,
|
|
claims JSON,
|
|
open_questions JSON,
|
|
contradictions JSON,
|
|
raw_response TEXT,
|
|
input_tokens INTEGER,
|
|
output_tokens INTEGER,
|
|
created_at TIMESTAMP NOT NULL DEFAULT (now() AT TIME ZONE 'UTC')
|
|
)
|
|
"""
|
|
)
|
|
|
|
op.execute(
|
|
"""
|
|
CREATE TABLE wiki_pages (
|
|
id SERIAL PRIMARY KEY,
|
|
vault_path VARCHAR(500) NOT NULL UNIQUE,
|
|
domain VARCHAR(50) NOT NULL,
|
|
title VARCHAR(300) NOT NULL,
|
|
git_sha_before VARCHAR(40),
|
|
git_sha_after VARCHAR(40),
|
|
created_at TIMESTAMP NOT NULL DEFAULT (now() AT TIME ZONE 'UTC'),
|
|
updated_at TIMESTAMP
|
|
)
|
|
"""
|
|
)
|
|
|
|
|
|
def downgrade() -> None:
|
|
"""Drop the v1 second_brain tables and enums (schema is left intact)."""
|
|
op.execute("SET search_path TO second_brain")
|
|
op.execute("DROP TABLE IF EXISTS wiki_pages")
|
|
op.execute("DROP TABLE IF EXISTS extractions")
|
|
op.execute("DROP TABLE IF EXISTS sources")
|
|
op.execute("DROP TYPE IF EXISTS source_status")
|
|
op.execute("DROP TYPE IF EXISTS source_type")
|