From ceeae77e7d683ca939a6b9920a54431c536f19ce Mon Sep 17 00:00:00 2001 From: Travis Herbranson Date: Sun, 24 May 2026 22:46:48 -0400 Subject: [PATCH 01/13] postgres migration: schema, models, embeddings, alembic MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Swap the SQLite backing store for petalbrain Postgres + pgvector, modeled on vault-mcp. All second-brain relational tables now live in the `second_brain` schema (owned by the lovebug role); embeddings are written to the shared public.embeddings table. Locked design decisions (per Travis): - DB: existing petalbrain Postgres, second_brain schema, lovebug role. - Connection: containerized homelab-postgres:5432, plain psycopg_pool (min=1/max=10), no PgBouncer. - ORM stays SQLAlchemy; int autoincrement PKs + naive UTC DateTime. - Embeddings: reuse shared public.embeddings keyed by (source_schema='second_brain', source_table='extractions', source_id, model='nomic-embed-text'). Summaries only for this round. - Pipeline: chunk_text → Ollama nomic-embed-text → delete-before-insert upsert, with graceful degradation (no DB / no Ollama → log + skip). - Alembic stands up second-brain's own schema; public.embeddings stays out-of-band. - File-based wiki compiler is unchanged. No SQLite data import — starting clean. This commit is the scaffolding only; `alembic upgrade head` and a smoke test of the embedding path are the next checkpoint. --- alembic.ini | 52 ++++ alembic/env.py | 108 ++++++++ alembic/script.py.mako | 27 ++ ...f6c9d10_v1_second_brain_pipeline_tables.py | 117 ++++++++ config/settings.toml | 33 ++- config/settings.toml.example | 69 +++++ pyproject.toml | 10 + src/second_brain/config.py | 60 ++++- src/second_brain/database.py | 117 ++++++-- src/second_brain/embeddings/__init__.py | 251 ++++++++++++++++++ src/second_brain/main.py | 21 +- src/second_brain/models.py | 36 ++- src/second_brain/scheduler/runner.py | 13 +- 13 files changed, 867 insertions(+), 47 deletions(-) create mode 100644 alembic.ini create mode 100644 alembic/env.py create mode 100644 alembic/script.py.mako create mode 100644 alembic/versions/4a1e2f6c9d10_v1_second_brain_pipeline_tables.py create mode 100644 config/settings.toml.example create mode 100644 src/second_brain/embeddings/__init__.py diff --git a/alembic.ini b/alembic.ini new file mode 100644 index 0000000..bbc8a8e --- /dev/null +++ b/alembic.ini @@ -0,0 +1,52 @@ +# Alembic config for the second_brain schema migrations. +# +# Mirrors vault-mcp/alembic.ini. The sqlalchemy.url placeholder lets the +# `alembic` CLI parse this file without env vars at import time; env.py +# overrides it from SECOND_BRAIN_DATABASE_URL / HERBYLAB_DATABASE_URL at +# runtime. + +[alembic] +script_location = %(here)s/alembic + +prepend_sys_path = . + +path_separator = os + +sqlalchemy.url = postgresql+psycopg://placeholder:placeholder@localhost/placeholder + + +[post_write_hooks] + +[loggers] +keys = root,sqlalchemy,alembic + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = WARNING +handlers = console +qualname = + +[logger_sqlalchemy] +level = WARNING +handlers = +qualname = sqlalchemy.engine + +[logger_alembic] +level = INFO +handlers = +qualname = alembic + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(levelname)-5.5s [%(name)s] %(message)s +datefmt = %H:%M:%S diff --git a/alembic/env.py b/alembic/env.py new file mode 100644 index 0000000..fb460c7 --- /dev/null +++ b/alembic/env.py @@ -0,0 +1,108 @@ +"""Alembic environment for second-brain — second_brain schema. + +Mirrors vault-mcp/alembic/env.py. Reads SECOND_BRAIN_DATABASE_URL (with +HERBYLAB_DATABASE_URL as a fallback) so the same migration tree works +across local dev, CI, and production. + +The connection bootstraps the `second_brain` schema and pins alembic's +own version table to that schema; migration scripts themselves set +search_path before issuing unqualified DDL. + +The bootstrap-then-commit pattern below is the fix for the "autobegin +trap": running any further statement before context.begin_transaction() +would autobegin a new transaction, causing alembic's begin_transaction() +to nest as a savepoint that silently rolls back when the connection +closes. Do not regress this. +""" + +from __future__ import annotations + +import os +from logging.config import fileConfig + +from dotenv import load_dotenv +from sqlalchemy import engine_from_config, pool + +from alembic import context + +# Make the project's `src/` importable so `second_brain.models` resolves +# without a `pip install -e .`. +import sys +from pathlib import Path + +_PROJECT_ROOT = Path(__file__).resolve().parent.parent +sys.path.insert(0, str(_PROJECT_ROOT / "src")) + +from second_brain.models import Base, SCHEMA # noqa: E402 + +load_dotenv() + +config = context.config + +if config.config_file_name is not None: + fileConfig(config.config_file_name) + + +def _resolve_db_url() -> str | None: + """Pick the first usable URL out of the two homelab conventions.""" + for var in ("SECOND_BRAIN_DATABASE_URL", "HERBYLAB_DATABASE_URL"): + v = os.environ.get(var) + if v: + return v + return None + + +_db_url = _resolve_db_url() +if _db_url: + config.set_main_option("sqlalchemy.url", _db_url) + +target_metadata = Base.metadata + + +def run_migrations_offline() -> None: + url = config.get_main_option("sqlalchemy.url") + context.configure( + url=url, + target_metadata=target_metadata, + literal_binds=True, + dialect_opts={"paramstyle": "named"}, + version_table_schema=SCHEMA, + include_schemas=True, + ) + + with context.begin_transaction(): + context.run_migrations() + + +def run_migrations_online() -> None: + connectable = engine_from_config( + config.get_section(config.config_ini_section, {}), + prefix="sqlalchemy.", + poolclass=pool.NullPool, + ) + + with connectable.connect() as connection: + # Bootstrap the schema before alembic looks for alembic_version. + # Commit explicitly so the schema persists; otherwise the implicit + # transaction is rolled back when the connection closes. Avoid + # running any further statement here — it would autobegin a new + # transaction and cause alembic's `begin_transaction()` to nest as + # a savepoint, which silently rolls back when the connection closes. + # The migration script sets search_path itself. + connection.exec_driver_sql(f"CREATE SCHEMA IF NOT EXISTS {SCHEMA}") + connection.commit() + context.configure( + connection=connection, + target_metadata=target_metadata, + version_table_schema=SCHEMA, + include_schemas=True, + ) + + with context.begin_transaction(): + context.run_migrations() + + +if context.is_offline_mode(): + run_migrations_offline() +else: + run_migrations_online() diff --git a/alembic/script.py.mako b/alembic/script.py.mako new file mode 100644 index 0000000..891119b --- /dev/null +++ b/alembic/script.py.mako @@ -0,0 +1,27 @@ +"""${message} + +Revision ID: ${up_revision} +Revises: ${down_revision | comma,n} +Create Date: ${create_date} + +""" +from __future__ import annotations + +from collections.abc import Sequence + +from alembic import op +import sqlalchemy as sa +${imports if imports else ""} + +revision: str = ${repr(up_revision)} +down_revision: str | Sequence[str] | None = ${repr(down_revision)} +branch_labels: str | Sequence[str] | None = ${repr(branch_labels)} +depends_on: str | Sequence[str] | None = ${repr(depends_on)} + + +def upgrade() -> None: + ${upgrades if upgrades else "pass"} + + +def downgrade() -> None: + ${downgrades if downgrades else "pass"} diff --git a/alembic/versions/4a1e2f6c9d10_v1_second_brain_pipeline_tables.py b/alembic/versions/4a1e2f6c9d10_v1_second_brain_pipeline_tables.py new file mode 100644 index 0000000..da2d4eb --- /dev/null +++ b/alembic/versions/4a1e2f6c9d10_v1_second_brain_pipeline_tables.py @@ -0,0 +1,117 @@ +"""v1 second_brain: sources, extractions, wiki_pages + +Revision ID: 4a1e2f6c9d10 +Revises: +Create Date: 2026-05-24 00:00:00.000000 + +Emits the v1 second_brain schema: the relational backbone of the +extraction pipeline. Enum types (`source_type`, `source_status`) live +inside the schema so they don't collide with anything else in the +petalbrain cluster. + +env.py creates the second_brain schema and pins alembic_version to it +before this migration runs. Below we re-issue CREATE SCHEMA IF NOT +EXISTS defensively and SET search_path so unqualified names resolve. + +Vector indexing lives in public.embeddings — managed out-of-band, not +by this migration tree. +""" + +from __future__ import annotations + +from collections.abc import Sequence + +from alembic import op + +revision: str = "4a1e2f6c9d10" +down_revision: str | Sequence[str] | None = None +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + + +def upgrade() -> None: + """Create the v1 second_brain schema.""" + op.execute("CREATE SCHEMA IF NOT EXISTS second_brain") + op.execute("SET search_path TO second_brain") + + op.execute( + "CREATE TYPE source_type AS ENUM ('video', 'article')" + ) + op.execute( + """ + CREATE TYPE source_status AS ENUM ( + 'pending', 'pulled', 'transcribed', 'analyzed', + 'accepted', 'published', 'failed' + ) + """ + ) + + op.execute( + """ + CREATE TABLE sources ( + id SERIAL PRIMARY KEY, + url VARCHAR(2000) NOT NULL UNIQUE, + title VARCHAR(500), + source_type source_type NOT NULL DEFAULT 'video', + domain VARCHAR(50) NOT NULL DEFAULT 'development', + focus TEXT, + status source_status NOT NULL DEFAULT 'pending', + media_path VARCHAR(1000), + transcript_path VARCHAR(1000), + transcript_text TEXT, + published_at TIMESTAMP, + duration_seconds INTEGER, + author VARCHAR(200), + ingested_at TIMESTAMP NOT NULL DEFAULT (now() AT TIME ZONE 'UTC'), + updated_at TIMESTAMP, + error_message TEXT + ) + """ + ) + op.execute("CREATE INDEX idx_sources_status ON sources(status)") + op.execute("CREATE INDEX idx_sources_domain ON sources(domain)") + + op.execute( + """ + CREATE TABLE extractions ( + id SERIAL PRIMARY KEY, + source_id INTEGER NOT NULL UNIQUE + REFERENCES sources(id) ON DELETE CASCADE, + summary TEXT, + key_points JSON, + entities JSON, + claims JSON, + open_questions JSON, + contradictions JSON, + raw_response TEXT, + input_tokens INTEGER, + output_tokens INTEGER, + created_at TIMESTAMP NOT NULL DEFAULT (now() AT TIME ZONE 'UTC') + ) + """ + ) + + op.execute( + """ + CREATE TABLE wiki_pages ( + id SERIAL PRIMARY KEY, + vault_path VARCHAR(500) NOT NULL UNIQUE, + domain VARCHAR(50) NOT NULL, + title VARCHAR(300) NOT NULL, + git_sha_before VARCHAR(40), + git_sha_after VARCHAR(40), + created_at TIMESTAMP NOT NULL DEFAULT (now() AT TIME ZONE 'UTC'), + updated_at TIMESTAMP + ) + """ + ) + + +def downgrade() -> None: + """Drop the v1 second_brain tables and enums (schema is left intact).""" + op.execute("SET search_path TO second_brain") + op.execute("DROP TABLE IF EXISTS wiki_pages") + op.execute("DROP TABLE IF EXISTS extractions") + op.execute("DROP TABLE IF EXISTS sources") + op.execute("DROP TYPE IF EXISTS source_status") + op.execute("DROP TYPE IF EXISTS source_type") diff --git a/config/settings.toml b/config/settings.toml index 84fcc23..08d4657 100644 --- a/config/settings.toml +++ b/config/settings.toml @@ -2,10 +2,9 @@ # Copy this file and edit paths to match your setup. # Override the config file path with SECOND_BRAIN_CONFIG env var. -# Path to the SQLite database -db_path = "~/.local/share/second-brain/second_brain.db" - -# Path to your Obsidian vault (git-managed directory) +# Path to your Obsidian vault (git-managed directory). The wiki compiler +# writes markdown into this directory and commits inside it. We do NOT +# dual-write into petalbrain.wiki — the file-based vault stays canonical. vault_path = "~/Documents/second-brain-vault" # Directory where downloaded media files are stored @@ -21,6 +20,21 @@ whisper_model = "small" # Active knowledge domains domains = ["development", "content", "business", "homelab"] +[database] +# Postgres + pgvector lives on the petalbrain instance. +# +# Preferred: leave `url` empty and export SECOND_BRAIN_DATABASE_URL (or the +# shared HERBYLAB_DATABASE_URL) in the environment so the password never +# lands in this checked-in file. +# +# Containerised second-brain runs should target homelab-postgres on the +# homelab docker network: +# postgresql+psycopg://lovebug:@homelab-postgres:5432/petalbrain +# +# Host-side dev runs can point at the published port instead: +# postgresql+psycopg://lovebug:@127.0.0.1:5433/petalbrain +url = "" + [extractor] # "cli" → runs `claude -p` under your Max OAuth (no per-token billing). # "api" → uses Anthropic SDK, requires ANTHROPIC_API_KEY. @@ -29,6 +43,17 @@ model = "claude-sonnet-4-6" cli_binary = "claude" timeout_sec = 600 +[embeddings] +# Best-effort: extraction summaries are chunked (512 tokens / 64 overlap) +# and embedded via Ollama's nomic-embed-text (768-d) into the shared +# public.embeddings table. Failures (no DB / no Ollama) log a warning and +# the pipeline keeps moving — the relational + file vault stay canonical. +enabled = true +# Containerized default; for host-side runs set OLLAMA_URL=http://127.0.0.1:11434 +ollama_url = "http://ollama:11434" +model = "nomic-embed-text" +embed_field = "summary" + [scheduler] # Time window for overnight processing (24h format, may cross midnight) window_start = "22:00" diff --git a/config/settings.toml.example b/config/settings.toml.example new file mode 100644 index 0000000..08d4657 --- /dev/null +++ b/config/settings.toml.example @@ -0,0 +1,69 @@ +# second-brain runtime configuration +# Copy this file and edit paths to match your setup. +# Override the config file path with SECOND_BRAIN_CONFIG env var. + +# Path to your Obsidian vault (git-managed directory). The wiki compiler +# writes markdown into this directory and commits inside it. We do NOT +# dual-write into petalbrain.wiki — the file-based vault stays canonical. +vault_path = "~/Documents/second-brain-vault" + +# Directory where downloaded media files are stored +media_dir = "~/.local/share/second-brain/media" + +# Directory where Whisper SRT transcripts are stored +subtitles_dir = "~/.local/share/second-brain/subtitles" + +# Whisper model size: tiny | base | small | medium | large +# small is a good balance of speed and accuracy +whisper_model = "small" + +# Active knowledge domains +domains = ["development", "content", "business", "homelab"] + +[database] +# Postgres + pgvector lives on the petalbrain instance. +# +# Preferred: leave `url` empty and export SECOND_BRAIN_DATABASE_URL (or the +# shared HERBYLAB_DATABASE_URL) in the environment so the password never +# lands in this checked-in file. +# +# Containerised second-brain runs should target homelab-postgres on the +# homelab docker network: +# postgresql+psycopg://lovebug:@homelab-postgres:5432/petalbrain +# +# Host-side dev runs can point at the published port instead: +# postgresql+psycopg://lovebug:@127.0.0.1:5433/petalbrain +url = "" + +[extractor] +# "cli" → runs `claude -p` under your Max OAuth (no per-token billing). +# "api" → uses Anthropic SDK, requires ANTHROPIC_API_KEY. +backend = "cli" +model = "claude-sonnet-4-6" +cli_binary = "claude" +timeout_sec = 600 + +[embeddings] +# Best-effort: extraction summaries are chunked (512 tokens / 64 overlap) +# and embedded via Ollama's nomic-embed-text (768-d) into the shared +# public.embeddings table. Failures (no DB / no Ollama) log a warning and +# the pipeline keeps moving — the relational + file vault stay canonical. +enabled = true +# Containerized default; for host-side runs set OLLAMA_URL=http://127.0.0.1:11434 +ollama_url = "http://ollama:11434" +model = "nomic-embed-text" +embed_field = "summary" + +[scheduler] +# Time window for overnight processing (24h format, may cross midnight) +window_start = "22:00" +window_end = "06:00" + +# CLI backend: cap calls per rolling hour (Max sub has 5h-window message caps). +max_calls_per_hour = 30 + +# Minimum seconds between extraction calls (rate-limit smoother) +min_gap_seconds = 120 + +# Legacy API-backend budget (only enforced when extractor.backend = "api"). +max_tokens_per_hour = 100_000 diff --git a/pyproject.toml b/pyproject.toml index 83c0d8f..c927da2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,10 +4,15 @@ version = "0.1.0" description = "Personal knowledge system: video/article extraction pipeline + LLM-maintained wiki" requires-python = ">=3.12" dependencies = [ + "alembic>=1.13.0", "click>=8.1.0", + "embedding-chunking", "fastapi>=0.115.0", + "httpx>=0.28.0", "jinja2>=3.1.0", "openai-whisper", + "psycopg[binary]>=3.2", + "psycopg-pool>=3.2", "pydantic>=2.0.0", "python-dotenv>=1.0.0", "sqlalchemy>=2.0.0", @@ -23,6 +28,11 @@ dependencies = [ # Max OAuth subscription and has no Python-side Anthropic dependency. api = ["anthropic>=0.40.0"] +[tool.uv.sources] +# Shared chunker that vault-mcp / ob1-enricher / ob1-reflector also pin. +# Same source so the 512/64 chunking stays in lockstep across services. +embedding-chunking = { git = "https://gitea.plantbasedsoutherner.com/petal-power/embedding-chunking.git", tag = "v0.1.0" } + [project.scripts] second-brain = "second_brain.main:cli" diff --git a/src/second_brain/config.py b/src/second_brain/config.py index 5d8a738..242f3c0 100644 --- a/src/second_brain/config.py +++ b/src/second_brain/config.py @@ -3,6 +3,11 @@ Configuration loading for second-brain. Settings are read from config/settings.toml (or a path set via SECOND_BRAIN_CONFIG). All values have sensible defaults so the system works out of the box. + +The relational store is Postgres (petalbrain DB, `second_brain` schema). Connection +details are resolved by `second_brain.database` — see that module for the URL +resolution order. The vault is still a file-based Obsidian directory; this +project does NOT dual-write to petalbrain.wiki. """ from __future__ import annotations @@ -21,12 +26,17 @@ except ModuleNotFoundError: # --------------------------------------------------------------------------- _DEFAULTS: dict[str, Any] = { - "db_path": "~/.local/share/second-brain/second_brain.db", "vault_path": "~/Documents/second-brain-vault", "media_dir": "~/.local/share/second-brain/media", "subtitles_dir": "~/.local/share/second-brain/subtitles", "whisper_model": "small", "domains": ["development", "content", "business", "homelab"], + "database": { + # Empty by default — the runtime expects SECOND_BRAIN_DATABASE_URL + # (or HERBYLAB_DATABASE_URL) in the environment. Fill this only if + # you prefer keeping the URL inside settings.toml. + "url": "", + }, "extractor": { # "cli" runs `claude -p` under the Max OAuth subscription (default). # "api" uses the Anthropic SDK and requires ANTHROPIC_API_KEY. @@ -35,6 +45,19 @@ _DEFAULTS: dict[str, Any] = { "cli_binary": "claude", "timeout_sec": 600, }, + "embeddings": { + # Best-effort embedding of extraction summaries into public.embeddings. + # If disabled, or if Ollama / the DB is unreachable, the pipeline + # logs a warning and keeps going — file/relational data stays the + # source of truth. + "enabled": True, + # Default targets the `ollama` service on the homelab docker network + # (same convention vault-mcp uses). For host-side runs, set + # OLLAMA_URL=http://127.0.0.1:11434 in the env instead. + "ollama_url": "http://ollama:11434", + "model": "nomic-embed-text", + "embed_field": "summary", + }, "scheduler": { "window_start": "22:00", "window_end": "06:00", @@ -49,6 +72,13 @@ _DEFAULTS: dict[str, Any] = { DOMAINS = ["development", "content", "business", "homelab"] +def _merge(default: dict, override: dict) -> dict: + """Shallow merge — override wins per top-level key.""" + out = dict(default) + out.update(override or {}) + return out + + class Config: """Holds resolved configuration values.""" @@ -57,10 +87,6 @@ class Config: # --- paths --- - @property - def db_path(self) -> Path: - return Path(self._raw.get("db_path", _DEFAULTS["db_path"])).expanduser() - @property def vault_path(self) -> Path: return Path(self._raw.get("vault_path", _DEFAULTS["vault_path"])).expanduser() @@ -89,17 +115,23 @@ class Config: def anthropic_api_key(self) -> str | None: return os.getenv("ANTHROPIC_API_KEY") - # --- extractor --- + # --- structured sections --- + + @property + def database(self) -> dict[str, Any]: + return _merge(_DEFAULTS["database"], self._raw.get("database", {})) @property def extractor(self) -> dict[str, Any]: - return {**_DEFAULTS["extractor"], **self._raw.get("extractor", {})} + return _merge(_DEFAULTS["extractor"], self._raw.get("extractor", {})) - # --- scheduler --- + @property + def embeddings(self) -> dict[str, Any]: + return _merge(_DEFAULTS["embeddings"], self._raw.get("embeddings", {})) @property def scheduler(self) -> dict[str, Any]: - return {**_DEFAULTS["scheduler"], **self._raw.get("scheduler", {})} + return _merge(_DEFAULTS["scheduler"], self._raw.get("scheduler", {})) # --- prompts dir --- @@ -112,7 +144,7 @@ class Config: def ensure_dirs(self) -> None: """Create runtime directories if they don't exist.""" - for d in (self.db_path.parent, self.media_dir, self.subtitles_dir): + for d in (self.media_dir, self.subtitles_dir): d.mkdir(parents=True, exist_ok=True) @@ -150,4 +182,10 @@ def load_config(path: Path | None = None) -> Config: return _config_instance -__all__ = ["Config", "DOMAINS", "load_config"] +def reset_config_singleton() -> None: + """Test hook — clear the cached Config so a re-read picks up new settings.""" + global _config_instance + _config_instance = None + + +__all__ = ["Config", "DOMAINS", "load_config", "reset_config_singleton"] diff --git a/src/second_brain/database.py b/src/second_brain/database.py index 7c91f55..71050d0 100644 --- a/src/second_brain/database.py +++ b/src/second_brain/database.py @@ -1,43 +1,105 @@ """ Database engine and session management for second-brain. -Ported and adapted from xtract/database.py. -Uses a singleton pattern: call get_database() everywhere. +Postgres + pgvector backend on the petalbrain instance. + +Resolution order for the connection URL: +1. constructor arg passed to `get_database()` +2. env var `SECOND_BRAIN_DATABASE_URL` +3. env var `HERBYLAB_DATABASE_URL` (shared homelab convention) +4. `[database].url` in settings.toml + +The URL is expected to use SQLAlchemy's `postgresql+psycopg://` driver +(psycopg v3). Plain `postgresql://` URLs are accepted and rewritten to use +psycopg; this matches the vault-mcp convention where one URL feeds both +Alembic (`+psycopg`) and raw psycopg (no prefix). + +Connection mode: containerized (homelab-postgres:5432 on the homelab docker +network). For local-host dev runs, point at 127.0.0.1:5433 instead. """ from __future__ import annotations +import os from contextlib import contextmanager -from pathlib import Path from typing import Generator -from sqlalchemy import create_engine +from sqlalchemy import create_engine, text +from sqlalchemy.engine import Engine from sqlalchemy.orm import Session, sessionmaker -from second_brain.models import Base +from second_brain.models import SCHEMA, Base + + +def _normalize_url(url: str) -> str: + """Ensure the URL uses SQLAlchemy's psycopg-v3 driver.""" + if url.startswith("postgresql+psycopg://"): + return url + if url.startswith("postgresql://"): + return url.replace("postgresql://", "postgresql+psycopg://", 1) + return url + + +def _resolve_url(explicit: str | None) -> str: + if explicit: + return _normalize_url(explicit) + for var in ("SECOND_BRAIN_DATABASE_URL", "HERBYLAB_DATABASE_URL"): + v = os.environ.get(var) + if v: + return _normalize_url(v) + + # Fall back to settings.toml [database].url + try: + from second_brain.config import load_config + + cfg = load_config() + url = cfg.database.get("url") + if url: + return _normalize_url(url) + except Exception: + pass + + raise RuntimeError( + "No Postgres connection URL found. Set SECOND_BRAIN_DATABASE_URL " + "(or HERBYLAB_DATABASE_URL), or fill `[database].url` in " + "config/settings.toml." + ) class Database: - """Manages the SQLAlchemy engine and session factory.""" + """Manages the SQLAlchemy engine and session factory for second-brain.""" - def __init__(self, db_path: Path) -> None: - self.db_path = db_path - self.db_path.parent.mkdir(parents=True, exist_ok=True) + def __init__(self, url: str | None = None) -> None: + self.url = _resolve_url(url) - self.engine = create_engine( - f"sqlite:///{self.db_path}", + # Pool sizing matches vault-mcp (min=1/max=10). No PgBouncer; a normal + # QueuePool is fine for the CLI + scheduler + web workers we run. + self.engine: Engine = create_engine( + self.url, echo=False, - connect_args={"check_same_thread": False}, + pool_size=int(os.environ.get("SECOND_BRAIN_DB_POOL_MIN", "1")), + max_overflow=int(os.environ.get("SECOND_BRAIN_DB_POOL_MAX", "9")), + pool_pre_ping=True, + future=True, ) self.SessionLocal = sessionmaker( autocommit=False, autoflush=False, bind=self.engine, + future=True, ) def init_db(self) -> None: - """Create all tables (idempotent).""" + """Create the schema and tables if missing. + + Schema creation is idempotent and only relevant for dev/test setups + where Alembic hasn't been applied yet. In production, prefer + `alembic upgrade head` — this method just defensively ensures the + schema exists before `create_all` issues unqualified DDL. + """ + with self.engine.begin() as conn: + conn.execute(text(f"CREATE SCHEMA IF NOT EXISTS {SCHEMA}")) Base.metadata.create_all(bind=self.engine) def get_session(self) -> Session: @@ -57,6 +119,9 @@ class Database: finally: sess.close() + def close(self) -> None: + self.engine.dispose() + # --------------------------------------------------------------------------- # Singleton @@ -65,17 +130,25 @@ class Database: _db_instance: Database | None = None -def get_database(db_path: Path | None = None) -> Database: - """Return the singleton Database, creating and initialising it on first call.""" +def get_database(url: str | None = None) -> Database: + """Return the singleton Database, creating it lazily on first call. + + Schema/tables are *not* auto-created here — run `alembic upgrade head` + once during setup. Tests may explicitly call `Database(url).init_db()` + against a scratch database. + """ global _db_instance if _db_instance is None: - if db_path is None: - from second_brain.config import load_config - - db_path = load_config().db_path - _db_instance = Database(db_path) - _db_instance.init_db() + _db_instance = Database(url) return _db_instance -__all__ = ["Database", "get_database"] +def reset_database_singleton() -> None: + """Test hook — drop the cached singleton so a new URL can take effect.""" + global _db_instance + if _db_instance is not None: + _db_instance.close() + _db_instance = None + + +__all__ = ["Database", "get_database", "reset_database_singleton"] diff --git a/src/second_brain/embeddings/__init__.py b/src/second_brain/embeddings/__init__.py new file mode 100644 index 0000000..61b43b9 --- /dev/null +++ b/src/second_brain/embeddings/__init__.py @@ -0,0 +1,251 @@ +"""Embed second-brain extractions into the shared public.embeddings table. + +This is a near-verbatim copy of the vault-mcp recipe (vault_mcp.core.embeddings), +adapted for our `source_schema='second_brain' / source_table='extractions'` +key tuple and our embed payload (the LLM-generated summary). + +Behaviour: +- Reuses the shared `embedding_chunking` library for 512-token chunking with + 64-token overlap. +- Calls Ollama's /api/embeddings with `num_ctx=8192` to embed nomic-embed-text + vectors (768-d). +- Writes vector literals as `%s::vector` to avoid taking a hard dependency on + pgvector's Python adapter. +- Delete-before-insert on the (schema, table, source_id, embedding_model) + tuple so a re-embed of an extraction that previously had N chunks but now + has M str | None: + """Resolve a raw psycopg DSN (no SQLAlchemy `+psycopg` prefix).""" + for var in ("SECOND_BRAIN_DATABASE_URL", "HERBYLAB_DATABASE_URL"): + v = os.environ.get(var) + if v: + return v.replace("postgresql+psycopg://", "postgresql://", 1) + + # Fallback: pull from settings.toml so a host-side `process` works even + # when the operator forgot to export the env var. + try: + from second_brain.config import load_config + + url = load_config().database.get("url") + if url: + return url.replace("postgresql+psycopg://", "postgresql://", 1) + except Exception: + pass + return None + + +def _get_pool() -> ConnectionPool | None: + """Lazy-init a small connection pool. Returns None if no DB URL is configured.""" + global _pool + if _pool is not None: + return _pool + dsn = _database_url() + if not dsn: + return None + _pool = ConnectionPool(dsn, min_size=1, max_size=4, open=True, timeout=10) + return _pool + + +def close_pool() -> None: + """Close the embedding-side pool. Safe to call multiple times.""" + global _pool + if _pool is not None: + _pool.close() + _pool = None + + +# --------------------------------------------------------------------------- +# Embedding model client +# --------------------------------------------------------------------------- + + +def _ollama_url() -> str: + """Env var takes precedence over settings.toml so containerized runs can + point at a different Ollama instance without rebuilding the image.""" + env = os.environ.get("OLLAMA_URL") + if env: + return env + try: + from second_brain.config import load_config + + return load_config().embeddings.get("ollama_url", "http://ollama:11434") + except Exception: + return "http://ollama:11434" + + +def _embedding_model() -> str: + env = os.environ.get("EMBEDDING_MODEL") + if env: + return env + try: + from second_brain.config import load_config + + return load_config().embeddings.get("model", "nomic-embed-text") + except Exception: + return "nomic-embed-text" + + +def embed_text(text: str) -> list[float]: + """Call Ollama's embeddings endpoint. Returns the model-native vector.""" + resp = httpx.post( + f"{_ollama_url()}/api/embeddings", + json={ + "model": _embedding_model(), + "prompt": text, + # nomic-embed-text supports 8192-token context. Ollama defaults + # the runner to 2048, which 500s on long summaries. + "options": {"num_ctx": 8192}, + }, + timeout=120.0, + ) + resp.raise_for_status() + data = resp.json() + vec = data.get("embedding") + if not vec or not isinstance(vec, list): + raise RuntimeError( + f"Ollama returned no embedding for model {_embedding_model()}" + ) + return vec + + +def _vector_literal(vec: list[float]) -> str: + """pgvector accepts text-literal vectors of form '[v1,v2,...]'.""" + return "[" + ",".join(repr(float(v)) for v in vec) + "]" + + +# --------------------------------------------------------------------------- +# Public API +# --------------------------------------------------------------------------- + + +def embed_extraction(extraction_id: int, summary: str | None) -> int | None: + """Embed an extraction summary into public.embeddings. + + Args: + extraction_id: PK of the row in second_brain.extractions. + summary: the LLM-generated summary string; we embed this and + nothing else for the current round (key_points / claims + are not embedded yet). + + Returns the number of chunk rows written, or None on skip/failure. + Best-effort: failures are logged, not raised. + """ + if not summary or not summary.strip(): + logger.debug("embed_extraction: skip — extraction %s has no summary", extraction_id) + return None + + pool = _get_pool() + if pool is None: + logger.info( + "embed_extraction: skip — no SECOND_BRAIN_DATABASE_URL configured" + ) + return None + + model = _embedding_model() + + try: + chunks: list[Chunk] = chunk_text(summary) + chunk_embeddings: list[tuple[Chunk, list[float]]] = [ + (c, embed_text(c.text)) for c in chunks + ] + except Exception as exc: + logger.warning( + "embed_extraction: chunk/embed failed for extraction %s: %s", + extraction_id, + exc, + ) + return None + + try: + with pool.connection() as conn: + n = _upsert_embeddings( + conn, extraction_id, chunk_embeddings, model + ) + conn.commit() + logger.info( + "embed_extraction: extraction=%s chunks=%d model=%s", + extraction_id, + n, + model, + ) + return n + except Exception as exc: + logger.warning( + "embed_extraction: DB write failed for extraction %s: %s", + extraction_id, + exc, + ) + return None + + +def _upsert_embeddings( + conn: Any, + extraction_id: int, + chunk_embeddings: list[tuple[Chunk, list[float]]], + model: str, +) -> int: + """Replace all public.embeddings rows for this (extraction, model) pair.""" + with conn.cursor() as cur: + cur.execute( + """ + DELETE FROM public.embeddings + WHERE source_schema = %s + AND source_table = %s + AND source_id = %s + AND embedding_model = %s + """, + (SOURCE_SCHEMA, SOURCE_TABLE, extraction_id, model), + ) + for chunk, vec in chunk_embeddings: + cur.execute( + """ + INSERT INTO public.embeddings + (source_schema, source_table, source_id, chunk_index, + chunk_text, chunk_token_count, embedding, embedding_model) + VALUES (%s, %s, %s, %s, %s, %s, %s::vector, %s) + """, + ( + SOURCE_SCHEMA, + SOURCE_TABLE, + extraction_id, + chunk.index, + chunk.text, + chunk.token_count, + _vector_literal(vec), + model, + ), + ) + return len(chunk_embeddings) + + +__all__ = ["embed_extraction", "embed_text", "close_pool"] diff --git a/src/second_brain/main.py b/src/second_brain/main.py index 065e98d..7a5dab2 100644 --- a/src/second_brain/main.py +++ b/src/second_brain/main.py @@ -54,7 +54,7 @@ def add(url: str, domain: str, focus: Optional[str], title: Optional[str]) -> No """Queue a source URL for processing.""" config = load_config() config.ensure_dirs() - db = get_database(config.db_path) + db = get_database() source_type = SourceType.ARTICLE if _is_article(url) else SourceType.VIDEO @@ -115,7 +115,7 @@ def process( config = load_config() config.ensure_dirs() - db = get_database(config.db_path) + db = get_database() yt_adapter = YouTubeAdapter(config) art_adapter = ArticleAdapter(config) @@ -199,19 +199,32 @@ def process( existing_ext.raw_response = getattr(result, "_raw_response", None) existing_ext.input_tokens = getattr(result, "_input_tokens", None) existing_ext.output_tokens = getattr(result, "_output_tokens", None) + ext_row = existing_ext else: - extraction = Extraction( + ext_row = Extraction( source_id=source.id, raw_response=getattr(result, "_raw_response", None), input_tokens=getattr(result, "_input_tokens", None), output_tokens=getattr(result, "_output_tokens", None), **result.to_db_dict(), ) - sess.add(extraction) + sess.add(ext_row) source.status = SourceStatus.ANALYZED source.updated_at = utcnow() + # Flush so the new extraction gets its PK before we kick + # off embedding (we key public.embeddings rows by it). + sess.flush() + extraction_id = ext_row.id + extraction_summary = ext_row.summary click.echo(" ✓ analyzed") + + if config.embeddings.get("enabled", True): + from second_brain.embeddings import embed_extraction + + n = embed_extraction(extraction_id, extraction_summary) + if n is not None: + click.echo(f" ✓ embedded ({n} chunk(s))") except Exception as exc: source.error_message = str(exc) source.status = SourceStatus.FAILED diff --git a/src/second_brain/models.py b/src/second_brain/models.py index dd4d6f7..0454350 100644 --- a/src/second_brain/models.py +++ b/src/second_brain/models.py @@ -1,6 +1,10 @@ """ SQLAlchemy ORM models for second-brain. +All tables live in the Postgres `second_brain` schema (in the petalbrain DB). +PK style is int autoincrement (SERIAL in Postgres); timestamps are naive UTC +(`timestamp without time zone`) per the locked design decisions in CLAUDE.md. + Tables: - Source — a queued/processed URL (video or article) - Extraction — structured LLM output for a source @@ -24,14 +28,19 @@ from sqlalchemy import ( ForeignKey, Integer, JSON, + MetaData, String, Text, ) from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column, relationship +SCHEMA = "second_brain" + class Base(DeclarativeBase): - """Shared declarative base.""" + """Shared declarative base. All tables live in the `second_brain` schema.""" + + metadata = MetaData(schema=SCHEMA) # --------------------------------------------------------------------------- @@ -54,6 +63,22 @@ class SourceStatus(enum.Enum): FAILED = "failed" +# Native Postgres enum types, scoped to the second_brain schema so they +# don't collide with anything else in the cluster. +_SourceTypeEnum = Enum( + SourceType, + name="source_type", + schema=SCHEMA, + values_callable=lambda et: [e.value for e in et], +) +_SourceStatusEnum = Enum( + SourceStatus, + name="source_status", + schema=SCHEMA, + values_callable=lambda et: [e.value for e in et], +) + + # --------------------------------------------------------------------------- # Models # --------------------------------------------------------------------------- @@ -69,7 +94,7 @@ class Source(Base): url: Mapped[str] = mapped_column(String(2000), unique=True, nullable=False) title: Mapped[Optional[str]] = mapped_column(String(500), nullable=True) source_type: Mapped[SourceType] = mapped_column( - Enum(SourceType), nullable=False, default=SourceType.VIDEO + _SourceTypeEnum, nullable=False, default=SourceType.VIDEO ) # Domain and focus @@ -78,7 +103,7 @@ class Source(Base): # Pipeline status status: Mapped[SourceStatus] = mapped_column( - Enum(SourceStatus), nullable=False, default=SourceStatus.PENDING + _SourceStatusEnum, nullable=False, default=SourceStatus.PENDING ) # File paths (set after pull/transcribe steps) @@ -115,7 +140,9 @@ class Extraction(Base): id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True) source_id: Mapped[int] = mapped_column( - ForeignKey("sources.id", ondelete="CASCADE"), nullable=False, unique=True + ForeignKey(f"{SCHEMA}.sources.id", ondelete="CASCADE"), + nullable=False, + unique=True, ) # Core fields (mirrors the extraction schema contract) @@ -167,6 +194,7 @@ class WikiPage(Base): __all__ = [ "Base", + "SCHEMA", "Source", "SourceStatus", "SourceType", diff --git a/src/second_brain/scheduler/runner.py b/src/second_brain/scheduler/runner.py index ee63aca..6541692 100644 --- a/src/second_brain/scheduler/runner.py +++ b/src/second_brain/scheduler/runner.py @@ -127,18 +127,27 @@ class Scheduler: existing.raw_response = getattr(result, "_raw_response", None) existing.input_tokens = getattr(result, "_input_tokens", None) existing.output_tokens = getattr(result, "_output_tokens", None) + ext_row = existing else: - sess.add(Extraction( + ext_row = Extraction( source_id=source.id, raw_response=getattr(result, "_raw_response", None), input_tokens=getattr(result, "_input_tokens", None), output_tokens=getattr(result, "_output_tokens", None), **fields, - )) + ) + sess.add(ext_row) source.status = SourceStatus.ANALYZED source.updated_at = utcnow() + # Flush so the embedding step can key on the new PK. + sess.flush() + if self.config.embeddings.get("enabled", True): + from second_brain.embeddings import embed_extraction + + embed_extraction(ext_row.id, ext_row.summary) + calls_this_hour += 1 used = (getattr(result, "_input_tokens", 0) or 0) + ( getattr(result, "_output_tokens", 0) or 0 From 2e07d7a8fe8e84dec2e35a56f14ccf4ec280e8f9 Mon Sep 17 00:00:00 2001 From: Travis Herbranson Date: Sun, 24 May 2026 22:49:06 -0400 Subject: [PATCH 02/13] alembic: skip CREATE SCHEMA when present + add embedding smoke test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The runtime role (lovebug) doesn't have CREATE on the petalbrain database even though it owns the second_brain schema, so a bare `CREATE SCHEMA IF NOT EXISTS` errors out with permission denied. Gate the bootstrap on a pg_namespace lookup so we only attempt the create when the schema is genuinely missing — operators bootstrap it once as postgres superuser, alembic just respects it afterward. The smoke test exercises the full Postgres + embedding path against a live DB + Ollama (autoskipped otherwise): writes an extraction, embeds the summary, asserts public.embeddings has the expected row count, and re-embeds to verify the delete-before-insert idempotency. --- alembic/env.py | 15 +- ...f6c9d10_v1_second_brain_pipeline_tables.py | 5 +- tests/test_smoke_embedding.py | 148 ++++++++++++++++++ uv.lock | 114 ++++++++++++++ 4 files changed, 279 insertions(+), 3 deletions(-) create mode 100644 tests/test_smoke_embedding.py diff --git a/alembic/env.py b/alembic/env.py index fb460c7..09c8a84 100644 --- a/alembic/env.py +++ b/alembic/env.py @@ -83,13 +83,24 @@ def run_migrations_online() -> None: with connectable.connect() as connection: # Bootstrap the schema before alembic looks for alembic_version. - # Commit explicitly so the schema persists; otherwise the implicit + # The runtime role (lovebug) does NOT have CREATE on the petalbrain + # database — bare `CREATE SCHEMA IF NOT EXISTS` would fail with + # permission denied even though the schema already exists. So we + # only attempt to create it when the schema is genuinely missing + # (operator must have bootstrapped it once as postgres superuser: + # `CREATE SCHEMA second_brain AUTHORIZATION lovebug`). + # Commit explicitly so any DDL persists; otherwise the implicit # transaction is rolled back when the connection closes. Avoid # running any further statement here — it would autobegin a new # transaction and cause alembic's `begin_transaction()` to nest as # a savepoint, which silently rolls back when the connection closes. # The migration script sets search_path itself. - connection.exec_driver_sql(f"CREATE SCHEMA IF NOT EXISTS {SCHEMA}") + row = connection.exec_driver_sql( + "SELECT 1 FROM pg_namespace WHERE nspname = %s", + (SCHEMA,), + ).fetchone() + if row is None: + connection.exec_driver_sql(f"CREATE SCHEMA {SCHEMA}") connection.commit() context.configure( connection=connection, diff --git a/alembic/versions/4a1e2f6c9d10_v1_second_brain_pipeline_tables.py b/alembic/versions/4a1e2f6c9d10_v1_second_brain_pipeline_tables.py index da2d4eb..9fb2072 100644 --- a/alembic/versions/4a1e2f6c9d10_v1_second_brain_pipeline_tables.py +++ b/alembic/versions/4a1e2f6c9d10_v1_second_brain_pipeline_tables.py @@ -31,7 +31,10 @@ depends_on: str | Sequence[str] | None = None def upgrade() -> None: """Create the v1 second_brain schema.""" - op.execute("CREATE SCHEMA IF NOT EXISTS second_brain") + # The schema is bootstrapped by env.py (only if missing); skip the + # IF NOT EXISTS form here because lovebug doesn't have CREATE on the + # petalbrain database and the bare statement errors out even when the + # schema already exists. Just SET search_path and emit DDL inside it. op.execute("SET search_path TO second_brain") op.execute( diff --git a/tests/test_smoke_embedding.py b/tests/test_smoke_embedding.py new file mode 100644 index 0000000..caf18a2 --- /dev/null +++ b/tests/test_smoke_embedding.py @@ -0,0 +1,148 @@ +"""Smoke test for the Postgres + embedding pipeline. + +Runs only when SECOND_BRAIN_DATABASE_URL (or HERBYLAB_DATABASE_URL) is set +*and* an Ollama instance with nomic-embed-text is reachable. Otherwise the +test is skipped — there is no in-memory fixture for pgvector, and we +explicitly want the smoke check to exercise the real wire format. + +It covers: + 1. SQLAlchemy can write a Source + Extraction into the `second_brain` + schema. + 2. The embeddings module chunks the summary, calls Ollama, and writes + vector rows into public.embeddings keyed correctly. + 3. Re-running embedding is idempotent (delete-before-insert wins). +""" + +from __future__ import annotations + +import os +import socket +import uuid + +import httpx +import psycopg +import pytest + + +def _ollama_reachable() -> bool: + url = os.environ.get("OLLAMA_URL", "http://127.0.0.1:11434") + try: + r = httpx.get(f"{url}/api/tags", timeout=2.0) + if r.status_code != 200: + return False + models = [m["name"] for m in r.json().get("models", [])] + return any(m.startswith("nomic-embed-text") for m in models) + except (httpx.HTTPError, socket.error): + return False + + +def _db_reachable() -> bool: + url = os.environ.get("SECOND_BRAIN_DATABASE_URL") or os.environ.get( + "HERBYLAB_DATABASE_URL" + ) + if not url: + return False + raw = url.replace("postgresql+psycopg://", "postgresql://", 1) + try: + with psycopg.connect(raw, connect_timeout=2) as conn: # noqa: F841 + return True + except Exception: + return False + + +pytestmark = pytest.mark.skipif( + not (_db_reachable() and _ollama_reachable()), + reason="Smoke test needs SECOND_BRAIN_DATABASE_URL + reachable Ollama (nomic-embed-text)", +) + + +def test_extraction_round_trip_with_embedding(): + from second_brain.database import Database, reset_database_singleton + from second_brain.embeddings import close_pool, embed_extraction + from second_brain.models import ( + Extraction, + Source, + SourceStatus, + SourceType, + utcnow, + ) + + reset_database_singleton() + db = Database() + + # Unique URL so re-runs don't collide on the unique constraint. + unique = uuid.uuid4().hex[:8] + url = f"https://example.test/smoke/{unique}" + + summary_text = ( + "Smoke test extraction summary. " + "The second-brain pipeline embeds this string into Postgres. " + "Re-running the embed should be idempotent — delete-before-insert " + "keys on (source_schema, source_table, source_id, embedding_model)." + ) + + with db.session() as sess: + src = Source( + url=url, + title=f"smoke-test-{unique}", + domain="development", + source_type=SourceType.ARTICLE, + status=SourceStatus.ANALYZED, + ingested_at=utcnow(), + ) + sess.add(src) + sess.flush() + ext = Extraction(source_id=src.id, summary=summary_text) + sess.add(ext) + sess.flush() + extraction_id = ext.id + source_id = src.id + + try: + n1 = embed_extraction(extraction_id, summary_text) + assert n1 is not None and n1 >= 1, f"first embed should write rows, got {n1}" + + # Idempotency: re-embed should produce the same chunk count and not + # accumulate orphan rows. + n2 = embed_extraction(extraction_id, summary_text) + assert n2 == n1, f"re-embed should match first count: {n1} vs {n2}" + + raw_url = ( + os.environ.get("SECOND_BRAIN_DATABASE_URL") + or os.environ.get("HERBYLAB_DATABASE_URL") + ).replace("postgresql+psycopg://", "postgresql://", 1) + with psycopg.connect(raw_url) as conn, conn.cursor() as cur: + cur.execute( + """ + SELECT COUNT(*) + FROM public.embeddings + WHERE source_schema = 'second_brain' + AND source_table = 'extractions' + AND source_id = %s + AND embedding_model = 'nomic-embed-text' + """, + (extraction_id,), + ) + (count,) = cur.fetchone() + assert count == n1, f"public.embeddings should hold {n1} rows, has {count}" + finally: + # Tidy up so the smoke test stays leak-free. + with db.session() as sess: + sess.query(Source).filter(Source.id == source_id).delete() + raw_url = ( + os.environ.get("SECOND_BRAIN_DATABASE_URL") + or os.environ.get("HERBYLAB_DATABASE_URL") + ).replace("postgresql+psycopg://", "postgresql://", 1) + with psycopg.connect(raw_url) as conn, conn.cursor() as cur: + cur.execute( + """ + DELETE FROM public.embeddings + WHERE source_schema = 'second_brain' + AND source_table = 'extractions' + AND source_id = %s + """, + (extraction_id,), + ) + conn.commit() + close_pool() + reset_database_singleton() diff --git a/uv.lock b/uv.lock index 57fc794..8e17809 100644 --- a/uv.lock +++ b/uv.lock @@ -2,6 +2,20 @@ version = 1 revision = 3 requires-python = ">=3.12" +[[package]] +name = "alembic" +version = "1.18.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "mako" }, + { name = "sqlalchemy" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/94/13/8b084e0f2efb0275a1d534838844926f798bd766566b1375174e2448cd31/alembic-1.18.4.tar.gz", hash = "sha256:cb6e1fd84b6174ab8dbb2329f86d631ba9559dd78df550b57804d607672cedbc", size = 2056725, upload-time = "2026-02-10T16:00:47.195Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d2/29/6533c317b74f707ea28f8d633734dbda2119bbadfc61b2f3640ba835d0f7/alembic-1.18.4-py3-none-any.whl", hash = "sha256:a5ed4adcf6d8a4cb575f3d759f071b03cd6e5c7618eb796cb52497be25bfe19a", size = 263893, upload-time = "2026-02-10T16:00:49.997Z" }, +] + [[package]] name = "annotated-doc" version = "0.0.4" @@ -277,6 +291,14 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/a7/5f/ed01f9a3cdffbd5a008556fc7b2a08ddb1cc6ace7effa7340604b1d16699/docstring_parser-0.18.0-py3-none-any.whl", hash = "sha256:b3fcbed555c47d8479be0796ef7e19c2670d428d72e96da63f3a40122860374b", size = 22484, upload-time = "2026-04-14T04:09:18.638Z" }, ] +[[package]] +name = "embedding-chunking" +version = "0.1.0" +source = { git = "https://gitea.plantbasedsoutherner.com/petal-power/embedding-chunking.git?tag=v0.1.0#176c04cd6c2232b77d6a3ebcfacbfc64311fd96c" } +dependencies = [ + { name = "tiktoken" }, +] + [[package]] name = "fastapi" version = "0.136.3" @@ -674,6 +696,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/6a/bd/6e2b76a6c5dee10397db9c929f0c5066766ec1036046f0335b7ca7ca08b8/lxml_html_clean-0.4.5-py3-none-any.whl", hash = "sha256:c76fcadd1e5bfb9b8bafc2200d51e4e78eb0dad67f56881c21dfb6484c7e7746", size = 14573, upload-time = "2026-05-20T12:17:52.215Z" }, ] +[[package]] +name = "mako" +version = "1.3.12" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "markupsafe" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/00/62/791b31e69ae182791ec67f04850f2f062716bbd205483d63a215f3e062d3/mako-1.3.12.tar.gz", hash = "sha256:9f778e93289bd410bb35daadeb4fc66d95a746f0b75777b942088b7fd7af550a", size = 400219, upload-time = "2026-04-28T19:01:08.512Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/bc/b1/a0ec7a5a9db730a08daef1fdfb8090435b82465abbf758a596f0ea88727e/mako-1.3.12-py3-none-any.whl", hash = "sha256:8f61569480282dbf557145ce441e4ba888be453c30989f879f0d652e39f53ea9", size = 78521, upload-time = "2026-04-28T19:01:10.393Z" }, +] + [[package]] name = "markupsafe" version = "3.0.3" @@ -1020,6 +1054,76 @@ dependencies = [ ] sdist = { url = "https://files.pythonhosted.org/packages/35/8e/d36f8880bcf18ec026a55807d02fe4c7357da9f25aebd92f85178000c0dc/openai_whisper-20250625.tar.gz", hash = "sha256:37a91a3921809d9f44748ffc73c0a55c9f366c85a3ef5c2ae0cc09540432eb96", size = 803191, upload-time = "2025-06-26T01:06:13.34Z" } +[[package]] +name = "psycopg" +version = "3.3.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions", marker = "python_full_version < '3.13'" }, + { name = "tzdata", marker = "sys_platform == 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/db/2f/cb91e5502ec9de1de6f1b76cfbf69531932725361168bb06963620c77e2e/psycopg-3.3.4.tar.gz", hash = "sha256:e21207764952cff81b6b8bdacad9a3939f2793367fdac2987b3aac36a651b5bc", size = 165799, upload-time = "2026-05-01T23:31:55.179Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5c/e0/7b3dee031daae7743609ce3c746565d4a3ed7c2c186479eb48e34e838c64/psycopg-3.3.4-py3-none-any.whl", hash = "sha256:b6bbc25ccf05c8fad3b061d9db2ef0909a555171b84b07f29458a447253d679a", size = 213001, upload-time = "2026-05-01T23:20:50.816Z" }, +] + +[package.optional-dependencies] +binary = [ + { name = "psycopg-binary", marker = "implementation_name != 'pypy'" }, +] + +[[package]] +name = "psycopg-binary" +version = "3.3.4" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/95/7d/03818e13ba7f36de93573c93ee3482006d3dfa8b0f8d28df511bad0a1a92/psycopg_binary-3.3.4-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:5ab28a2a7649df3b72e6b674b4c190e448e8e77cf496a65bd846472048de2089", size = 4591122, upload-time = "2026-05-01T23:27:56.162Z" }, + { url = "https://files.pythonhosted.org/packages/a5/b9/11b341edf8d54e2694726b273fe9652b254d989f4f63e3ac6816ad6b55f4/psycopg_binary-3.3.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:6402a9d8146cf4b3974ded3fd28a971e83dc6a0333eb7822524a3aa20b546578", size = 4669943, upload-time = "2026-05-01T23:28:04.522Z" }, + { url = "https://files.pythonhosted.org/packages/8b/18/4665bacd65e7865b4372fcd8abb8b9186ada4b0025f8c2ca691b364a556c/psycopg_binary-3.3.4-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:580ae30a5f95ccd90008ec697d3ed6a4a2047a516407ad904283fa42086936e9", size = 5469697, upload-time = "2026-05-01T23:28:11.337Z" }, + { url = "https://files.pythonhosted.org/packages/7c/b1/b83136c6e510593d9b0c759ba5384337bc4ad82d19fda675adc4b2703c84/psycopg_binary-3.3.4-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:e7510c37550f91a187e3660a8cc50d4b760f8c3b8b2f89ebc5698cd2c7f2c85d", size = 5152995, upload-time = "2026-05-01T23:28:20.529Z" }, + { url = "https://files.pythonhosted.org/packages/67/8d/a9821e2a648afe6091989929982a3b0f00b2631a859cb81379728f08fb75/psycopg_binary-3.3.4-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:77df19583501ea288eaf15ac0fe7ad01e6d8091a91d5c41df5c718f307d8e31b", size = 6738180, upload-time = "2026-05-01T23:28:30.654Z" }, + { url = "https://files.pythonhosted.org/packages/7e/58/2e349e8d23905dc2317b80ac65f48fb6f821a4777a4e994a60da91c4850f/psycopg_binary-3.3.4-cp312-cp312-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:018fbed325936da502feb546642c982dcc4b9ffdea32dfef78dbf3b7f7ad4070", size = 4978828, upload-time = "2026-05-01T23:28:37.277Z" }, + { url = "https://files.pythonhosted.org/packages/45/48/57b00d03b4721878326122a1f1e6b0a90b85bcaec56b5b2f8ea6cfa45235/psycopg_binary-3.3.4-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:17a21953a9e5ff3a16dab692625a3676e2f101db5e40072f39dbee2250194d68", size = 4509757, upload-time = "2026-05-01T23:28:43.078Z" }, + { url = "https://files.pythonhosted.org/packages/25/37/33b47d8c007df69aec500df5889767c4d313748e8e9e27a2fef8a6dabcee/psycopg_binary-3.3.4-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:eb05ee1c2b817d27c537333224c9e83c7afb86fe7296ba970990068baf819b16", size = 4190546, upload-time = "2026-05-01T23:28:50.016Z" }, + { url = "https://files.pythonhosted.org/packages/ca/c6/32b0835dbc2122617902b649d76a91c1e75406e76bf3d595b0c3bb5ffad6/psycopg_binary-3.3.4-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:773d573e11f437ce0bdb95b7c18dc58390494f96d43f8b45b9760436114f7652", size = 3926197, upload-time = "2026-05-01T23:28:55.55Z" }, + { url = "https://files.pythonhosted.org/packages/cd/68/d190ef0c0c5b16ded07831dabc8ddd412f4cdab07ec6e30ed38d9bda0e1f/psycopg_binary-3.3.4-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:71e55ccbdfae79a2ed9c6369c3008a3025817ff9d7e27b32a2d84e2a4267e66e", size = 4236627, upload-time = "2026-05-01T23:29:05.336Z" }, + { url = "https://files.pythonhosted.org/packages/25/8f/81dcbc2e8454b74d14881275ea45f00791052dac531a9fa8be1730d1685b/psycopg_binary-3.3.4-cp312-cp312-win_amd64.whl", hash = "sha256:494ca54901be8cf9eb7e02c25b731f2317c378efa44f43e8f9bd0e1184ae7be4", size = 3560782, upload-time = "2026-05-01T23:29:11.967Z" }, + { url = "https://files.pythonhosted.org/packages/09/43/13e9c406fbbf354580476e248a16b64802a376873ebe6339e30bb655572d/psycopg_binary-3.3.4-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:fbd1d4ed566895ad2d3bf4ddfd8bae90026930ddf29df3b9d91d32c8c47866a7", size = 4590377, upload-time = "2026-05-01T23:29:18.782Z" }, + { url = "https://files.pythonhosted.org/packages/22/be/2923cd7c3683e7afdecf4f10796a18de02f5c5ddc0969aa2ad0a8cdd3bbd/psycopg_binary-3.3.4-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:75a9067e236f9b9ae3535b66fe99bddb33d39c0de10112e49b9ab11eee53dc31", size = 4669023, upload-time = "2026-05-01T23:29:25.884Z" }, + { url = "https://files.pythonhosted.org/packages/96/a0/2c913d6fe13d6a8bd13597d36739bf47af063ad9399e402cfecab16f3c1e/psycopg_binary-3.3.4-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:b56b603ebcea8aa10b46228b8410ba7f13e7c2ee54389d4d9be0927fd8ce2a70", size = 5467423, upload-time = "2026-05-01T23:29:33.416Z" }, + { url = "https://files.pythonhosted.org/packages/e7/38/205d10bc1ad0df4a21c5c51659126bd3ea0ef98fcad1e852f78c249bb9c3/psycopg_binary-3.3.4-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:c677c4ad433cb7150c8cd304a0769ae3bcfbe5ea0676eb53faa7b1443b16d0d3", size = 5151137, upload-time = "2026-05-01T23:29:42.013Z" }, + { url = "https://files.pythonhosted.org/packages/36/fc/f0381ddcd45eff3bb70dbca6823a996048d7f507b2ec3fc92c6fabc0fe87/psycopg_binary-3.3.4-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:26df2717e59c0473e4465a97dfb1b7afebaa479277870fd5784d1436470db47c", size = 6736671, upload-time = "2026-05-01T23:29:51.626Z" }, + { url = "https://files.pythonhosted.org/packages/95/40/fa545ae152c24327651e5624e4902121e808270be36c10b12e9939be09bc/psycopg_binary-3.3.4-cp313-cp313-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:1dc1f79fd16bb1f3f4421417a514607539f17804d95c7ed617265369d1981cae", size = 4979601, upload-time = "2026-05-01T23:29:56.961Z" }, + { url = "https://files.pythonhosted.org/packages/86/e4/2f8a47ee97f90cd2b933d0463081d35631ff419de2b8c984a5f369857de0/psycopg_binary-3.3.4-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:136f199a407b5348b9b857c504aff60c77622a28482e7195839ce1b51238c4cc", size = 4510513, upload-time = "2026-05-01T23:30:07.243Z" }, + { url = "https://files.pythonhosted.org/packages/0e/0e/94e842ff4a7f98ed162580ca2e8b8864b28c1e0350f2443f8ee47f821167/psycopg_binary-3.3.4-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:b6f5a29e9c775b9f12a1a717aa7a2c80f9e1db6f27ba44a5b59c80ac61d2ffcf", size = 4187243, upload-time = "2026-05-01T23:30:15.352Z" }, + { url = "https://files.pythonhosted.org/packages/d0/83/fc6c174b672e29b7de996ea77b6cbddf46c891751c3355f6974292baa6b4/psycopg_binary-3.3.4-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:ee17a2cf4943cde261adfad1bbc5bf38d6b3776d7afff74c7cabcbeaeb08c260", size = 3927347, upload-time = "2026-05-01T23:30:21.186Z" }, + { url = "https://files.pythonhosted.org/packages/e9/65/768364d4a97a15b1a7f47ba52688c1686f22941d8332a8398cefc468e25f/psycopg_binary-3.3.4-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:5c4ab71be17bdca30cb34c34c4e1496e2f5d6f20c199c12bad226070b22ef9bf", size = 4236393, upload-time = "2026-05-01T23:30:26.211Z" }, + { url = "https://files.pythonhosted.org/packages/bd/3b/218efbc9e645becd80cdf651acda05f85cfe546b7a9c0458c7cbc8fe1f74/psycopg_binary-3.3.4-cp313-cp313-win_amd64.whl", hash = "sha256:dbfdb9b6cc79f31104a7b162a2b921b765fcc62af6c00540a167a8de47e4ed38", size = 3564592, upload-time = "2026-05-01T23:30:31.764Z" }, + { url = "https://files.pythonhosted.org/packages/48/a6/828c9185701dab71b234c2a76c38a08b098ebfec5020716b4e93807492b5/psycopg_binary-3.3.4-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:28b7398fdd19db3232c884fb24550bdfe951221f510e195e233299e4c9b78f97", size = 4607292, upload-time = "2026-05-01T23:30:38.962Z" }, + { url = "https://files.pythonhosted.org/packages/92/58/5b40dbc9d839045c9dae956960e4fb6d20bcabe6c59a2aa34fc3a371913f/psycopg_binary-3.3.4-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:1fbaa292a3c8bb61b45df1ad3da1908ccee7cb889db9425e3557d9e34e2a4829", size = 4687023, upload-time = "2026-05-01T23:30:47.227Z" }, + { url = "https://files.pythonhosted.org/packages/85/a9/793f0ac107a9003b48441d0d1f9f616d96e0f37458dd8dc12528ceff55fb/psycopg_binary-3.3.4-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:94596f9e7633ee3f6440711d43bb70aa31cc0a46a900ab8b4201a366ace5c9e7", size = 5486985, upload-time = "2026-05-01T23:30:55.517Z" }, + { url = "https://files.pythonhosted.org/packages/8f/26/42e8533497e2592334f68ec529cf5f840f7fa4e99575a4bb61aa184dbfbf/psycopg_binary-3.3.4-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:8c0056529e68dbe9184cd4019a1f3d8f3a4ead2f6fc7a5afcf27d3314edd1277", size = 5168745, upload-time = "2026-05-01T23:31:01.904Z" }, + { url = "https://files.pythonhosted.org/packages/15/af/b7151776cc08d5935d45c833ec818a9beb417cf7c08239af1aafbdae78ee/psycopg_binary-3.3.4-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2c09aad7051326e7603c14e50636db9c01f78272dc54b3accff03d46370461e6", size = 6761486, upload-time = "2026-05-01T23:31:14.511Z" }, + { url = "https://files.pythonhosted.org/packages/d0/ed/c92533b9124712d592cbf1cd6c76da933a2e0acea81dfe1fbe7e735f0cff/psycopg_binary-3.3.4-cp314-cp314-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:514404ed543efd620c85602b747df2a23cf1241b4067199e1a66f2d2757aaa41", size = 4997427, upload-time = "2026-05-01T23:31:20.901Z" }, + { url = "https://files.pythonhosted.org/packages/a2/23/ccadfd0de416aa188356daa199453af24087b042e296088706d190ae0295/psycopg_binary-3.3.4-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:46893c26858be12cc49ca4226ed6a60b4bfccadd946b3bebb783a60b38788228", size = 4533549, upload-time = "2026-05-01T23:31:26.204Z" }, + { url = "https://files.pythonhosted.org/packages/fd/a0/c8f43cee36386f7bc891ab41a9d31ea07cf9826038e732da79f26b1e5f34/psycopg_binary-3.3.4-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:df1d567fc430f6df15c9fcf67d87685fc49bdb325adc0db5af1adfb2f44eb5c9", size = 4210256, upload-time = "2026-05-01T23:31:33.884Z" }, + { url = "https://files.pythonhosted.org/packages/4e/2c/c1547871be3790676e8868b38655496422f94f0978dfb66b74bdba2f1676/psycopg_binary-3.3.4-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:6b9016b1714da4dd5ecaaa75b82098aa5a0b87854ce9b092e21c27c4ae23e014", size = 3946204, upload-time = "2026-05-01T23:31:39.626Z" }, + { url = "https://files.pythonhosted.org/packages/c4/b1/f6670f00fa7ea601584623f6c11602ab92117d83eaff885e0210f6de7418/psycopg_binary-3.3.4-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:47c656a8a7ba6eb0cff1801a4caaa9c8bdc12d03080e273aff1c8ac39971a77e", size = 4255811, upload-time = "2026-05-01T23:31:44.986Z" }, + { url = "https://files.pythonhosted.org/packages/eb/e6/5fff07a70d1f945ed90ae131c3bd76cab32beff7c58c6db15ad5820b6d1f/psycopg_binary-3.3.4-cp314-cp314-win_amd64.whl", hash = "sha256:c37e024c07308cd06cf3ec51bfd0e7f6157585a4d84d1bce4a7f5f7913719bf8", size = 3666849, upload-time = "2026-05-01T23:31:51.165Z" }, +] + +[[package]] +name = "psycopg-pool" +version = "3.3.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/90/82/7a23d26039827ecd4ebe93905651029ddd307c5182ad59296dfb6f67b528/psycopg_pool-3.3.1.tar.gz", hash = "sha256:b10b10b7a175d5cc1592147dc5b7eec8a9e0834eb3ed2c4a92c858e2f51eb63c", size = 31661, upload-time = "2026-05-01T23:31:59.809Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/37/ed/89c2c620af0e1660354cd8aabf9f5b21f911597ce22acb37c805d6c86bc8/psycopg_pool-3.3.1-py3-none-any.whl", hash = "sha256:2af5b432941c4c9ad5c87b3fa410aec910ec8f7c122855897983a06c45f2e4b5", size = 40023, upload-time = "2026-05-01T23:31:53.136Z" }, +] + [[package]] name = "pydantic" version = "2.13.4" @@ -1294,10 +1398,15 @@ name = "second-brain" version = "0.1.0" source = { editable = "." } dependencies = [ + { name = "alembic" }, { name = "click" }, + { name = "embedding-chunking" }, { name = "fastapi" }, + { name = "httpx" }, { name = "jinja2" }, { name = "openai-whisper" }, + { name = "psycopg", extra = ["binary"] }, + { name = "psycopg-pool" }, { name = "pydantic" }, { name = "python-dotenv" }, { name = "sqlalchemy" }, @@ -1314,11 +1423,16 @@ api = [ [package.metadata] requires-dist = [ + { name = "alembic", specifier = ">=1.13.0" }, { name = "anthropic", marker = "extra == 'api'", specifier = ">=0.40.0" }, { name = "click", specifier = ">=8.1.0" }, + { name = "embedding-chunking", git = "https://gitea.plantbasedsoutherner.com/petal-power/embedding-chunking.git?tag=v0.1.0" }, { name = "fastapi", specifier = ">=0.115.0" }, + { name = "httpx", specifier = ">=0.28.0" }, { name = "jinja2", specifier = ">=3.1.0" }, { name = "openai-whisper" }, + { name = "psycopg", extras = ["binary"], specifier = ">=3.2" }, + { name = "psycopg-pool", specifier = ">=3.2" }, { name = "pydantic", specifier = ">=2.0.0" }, { name = "python-dotenv", specifier = ">=1.0.0" }, { name = "sqlalchemy", specifier = ">=2.0.0" }, From a2d46a7c6e0464279cd19c95d35d3bbeb817a37c Mon Sep 17 00:00:00 2001 From: Travis Herbranson Date: Sun, 24 May 2026 22:52:08 -0400 Subject: [PATCH 03/13] =?UTF-8?q?gauntlet:=20fix=20lint=20=E2=80=94=20drop?= =?UTF-8?q?=20unused=20imports=20+=20reorder=20models.py=20+=20add=20dev?= =?UTF-8?q?=20deps?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Ruff autofix removed pre-existing F401 unused imports across adapters, context, compiler, extractor, scheduler, web, and main. Also reordered models.py so its SQLAlchemy imports sit at the top of the file (E402 was triggered by the inline `utcnow` helper definition). Added a [dependency-groups] dev block (pytest, pytest-cov, ruff) so the zero-check test gauntlet — which runs `pytest --cov --cov-report=term-missing` — can resolve its plugins without a manual `uv pip install pytest-cov`. --- pyproject.toml | 7 + src/second_brain/adapters/article.py | 1 - src/second_brain/adapters/youtube.py | 1 - src/second_brain/compiler/wiki.py | 1 - src/second_brain/context/assembler.py | 2 - src/second_brain/extractor/engine.py | 2 +- src/second_brain/extractor/schema.py | 3 +- src/second_brain/main.py | 2 - src/second_brain/models.py | 13 +- src/second_brain/scheduler/runner.py | 1 - src/second_brain/web/app.py | 7 +- uv.lock | 189 ++++++++++++++++++++++++++ 12 files changed, 208 insertions(+), 21 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index c927da2..2603013 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -28,6 +28,13 @@ dependencies = [ # Max OAuth subscription and has no Python-side Anthropic dependency. api = ["anthropic>=0.40.0"] +[dependency-groups] +dev = [ + "pytest>=8.0", + "pytest-cov>=5.0", + "ruff>=0.8", +] + [tool.uv.sources] # Shared chunker that vault-mcp / ob1-enricher / ob1-reflector also pin. # Same source so the 512/64 chunking stays in lockstep across services. diff --git a/src/second_brain/adapters/article.py b/src/second_brain/adapters/article.py index e9eea88..c203a96 100644 --- a/src/second_brain/adapters/article.py +++ b/src/second_brain/adapters/article.py @@ -8,7 +8,6 @@ Phase 1: basic fetch + text extraction. from __future__ import annotations from datetime import datetime -from typing import Optional from urllib.parse import urlparse import trafilatura diff --git a/src/second_brain/adapters/youtube.py b/src/second_brain/adapters/youtube.py index 4fbb6fc..5dfdc7a 100644 --- a/src/second_brain/adapters/youtube.py +++ b/src/second_brain/adapters/youtube.py @@ -7,7 +7,6 @@ Adapted to work with the second-brain Source model and Config. from __future__ import annotations -import json from datetime import datetime from pathlib import Path from typing import Optional diff --git a/src/second_brain/compiler/wiki.py b/src/second_brain/compiler/wiki.py index e27e3ee..6e907b3 100644 --- a/src/second_brain/compiler/wiki.py +++ b/src/second_brain/compiler/wiki.py @@ -15,7 +15,6 @@ from __future__ import annotations import re import subprocess -from datetime import datetime from pathlib import Path from typing import Optional diff --git a/src/second_brain/context/assembler.py b/src/second_brain/context/assembler.py index 959ef12..785a73f 100644 --- a/src/second_brain/context/assembler.py +++ b/src/second_brain/context/assembler.py @@ -9,8 +9,6 @@ Future phases can layer in wiki context, related sources, etc. from __future__ import annotations -from pathlib import Path -from typing import Optional from second_brain.config import Config from second_brain.models import Source diff --git a/src/second_brain/extractor/engine.py b/src/second_brain/extractor/engine.py index 13a2bd1..d868ff9 100644 --- a/src/second_brain/extractor/engine.py +++ b/src/second_brain/extractor/engine.py @@ -18,7 +18,7 @@ import os from typing import Optional from second_brain.extractor.schema import ExtractionResult, LLMExtraction, SourceMeta -from second_brain.llm.claude_cli import ClaudeCLI, ClaudeCLIError, DEFAULT_MODEL +from second_brain.llm.claude_cli import ClaudeCLI, DEFAULT_MODEL from second_brain.models import Source diff --git a/src/second_brain/extractor/schema.py b/src/second_brain/extractor/schema.py index b26280c..2ea994d 100644 --- a/src/second_brain/extractor/schema.py +++ b/src/second_brain/extractor/schema.py @@ -7,10 +7,9 @@ stored in the Extraction model as JSON columns. from __future__ import annotations -from datetime import datetime from typing import Optional -from pydantic import BaseModel, Field, HttpUrl +from pydantic import BaseModel, Field class LLMExtraction(BaseModel): diff --git a/src/second_brain/main.py b/src/second_brain/main.py index 7a5dab2..44a360b 100644 --- a/src/second_brain/main.py +++ b/src/second_brain/main.py @@ -11,8 +11,6 @@ Commands: from __future__ import annotations -import sys -from pathlib import Path from typing import Optional from urllib.parse import urlparse diff --git a/src/second_brain/models.py b/src/second_brain/models.py index 0454350..7519e53 100644 --- a/src/second_brain/models.py +++ b/src/second_brain/models.py @@ -17,23 +17,24 @@ import enum from datetime import datetime, timezone from typing import Optional - -def utcnow() -> datetime: - """Naive UTC timestamp. Replaces `datetime.utcnow()` (deprecated in 3.12).""" - return datetime.now(timezone.utc).replace(tzinfo=None) - from sqlalchemy import ( + JSON, DateTime, Enum, ForeignKey, Integer, - JSON, MetaData, String, Text, ) from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column, relationship + +def utcnow() -> datetime: + """Naive UTC timestamp. Replaces `datetime.utcnow()` (deprecated in 3.12).""" + return datetime.now(timezone.utc).replace(tzinfo=None) + + SCHEMA = "second_brain" diff --git a/src/second_brain/scheduler/runner.py b/src/second_brain/scheduler/runner.py index 6541692..c3fc205 100644 --- a/src/second_brain/scheduler/runner.py +++ b/src/second_brain/scheduler/runner.py @@ -21,7 +21,6 @@ from __future__ import annotations import time from datetime import datetime, time as dtime -from typing import Optional from second_brain.config import Config from second_brain.database import get_database diff --git a/src/second_brain/web/app.py b/src/second_brain/web/app.py index 5e697f9..4ad1c21 100644 --- a/src/second_brain/web/app.py +++ b/src/second_brain/web/app.py @@ -14,15 +14,14 @@ from __future__ import annotations from pathlib import Path from typing import List, Optional -from fastapi import FastAPI, Form, HTTPException, Request +from fastapi import FastAPI, HTTPException, Request from fastapi.responses import HTMLResponse from fastapi.templating import Jinja2Templates from pydantic import BaseModel -from sqlalchemy.orm import Session -from second_brain.config import DOMAINS, load_config +from second_brain.config import DOMAINS from second_brain.database import get_database -from second_brain.models import Extraction, Source, SourceStatus, SourceType +from second_brain.models import Extraction, Source, SourceStatus # --------------------------------------------------------------------------- # App setup diff --git a/uv.lock b/uv.lock index 8e17809..b99bf72 100644 --- a/uv.lock +++ b/uv.lock @@ -192,6 +192,90 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/8e/ca/6a667ccbe649856dcd3458bab80b016681b274399d6211187c6ab969fc50/courlan-1.3.2-py3-none-any.whl", hash = "sha256:d0dab52cf5b5b1000ee2839fbc2837e93b2514d3cb5bb61ae158a55b7a04c6be", size = 33848, upload-time = "2024-10-29T16:40:18.325Z" }, ] +[[package]] +name = "coverage" +version = "7.14.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/23/7f/d0720730a397a999ffc0fd3f5bebef347338e3a47b727da66fbb228e2ff2/coverage-7.14.0.tar.gz", hash = "sha256:057a6af2f160a85384cde4ab36f0d2777bae1057bae255f95413cdd382aa5c74", size = 919489, upload-time = "2026-05-10T18:02:31.397Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/09/1e/2f996b2c8415cbb6f54b0f5ec1ee850c96d7911961afb4fc05f4a89d8c58/coverage-7.14.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:7ffd19fc8aed057fd686a17a4935eef5f9859d69208f96310e893e64b9b6ccf5", size = 219967, upload-time = "2026-05-10T18:00:13.756Z" }, + { url = "https://files.pythonhosted.org/packages/34/23/35c7aea1274aef7525bdd2dc92f710bdde6d11652239d71d1ec450067939/coverage-7.14.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:829994cfe1aeb773ca27bf246d4badc1e764893e3bfb98fff820fcecd1ca4662", size = 220329, upload-time = "2026-05-10T18:00:15.264Z" }, + { url = "https://files.pythonhosted.org/packages/75/cf/a8f4b43a16e194b0261257ad28ded5853ec052570afef4a84e1d81189f3b/coverage-7.14.0-cp312-cp312-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:b4f07cf7edcb7ec39431a5074d7ea83b29a9f71fcfc494f0f40af4e65180420f", size = 251839, upload-time = "2026-05-10T18:00:17.16Z" }, + { url = "https://files.pythonhosted.org/packages/69/ff/6699e7b71e60d3049eb2bdcbc95ee3f35707b2b0e48f32e9e63d3ce30c08/coverage-7.14.0-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:ca3d9cf2c32b521bd9518385608787fa86f38daf993695307531822c3430ed67", size = 254576, upload-time = "2026-05-10T18:00:18.829Z" }, + { url = "https://files.pythonhosted.org/packages/22/ec/c936d495fcd67f48f03a9c4ad3297ff80d1f222a5df3980f15b34c186c21/coverage-7.14.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:92af52828e7f29d827346b0294e5a0853fa206db77db0395b282918d41e28db9", size = 255690, upload-time = "2026-05-10T18:00:20.648Z" }, + { url = "https://files.pythonhosted.org/packages/5c/42/5af63f636cc62a4a2b1b3ba9146f6ee6f53a35a50d5cefc54d5670f60999/coverage-7.14.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:7b2bb6c9d7e769360d0f20a0f219603fd64f0c8f97de17ab25853261602be0fb", size = 257949, upload-time = "2026-05-10T18:00:22.28Z" }, + { url = "https://files.pythonhosted.org/packages/26/d3/a225317bd2012132a27e1176d51660b826f99bb975876463c44ea0d7ee5a/coverage-7.14.0-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:1c9ed6ef99f88fb8c14aa8e2bf8eb0fe55fa2edfea68f8675d78741df1a5ac0e", size = 252242, upload-time = "2026-05-10T18:00:24.076Z" }, + { url = "https://files.pythonhosted.org/packages/f1/7f/9e65495298c3ea414742998539c37d048b5e81cc818fb1828cc6b51d10bf/coverage-7.14.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:8231ade007f37959fbf58acc677f26b922c02eda6f0428ea307da0fd39681bf3", size = 253608, upload-time = "2026-05-10T18:00:25.588Z" }, + { url = "https://files.pythonhosted.org/packages/94/46/1522b524a35bdad22b2b8c4f9d32d0a104b524726ec380b2db68db1746f5/coverage-7.14.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:d8b013632cc1ce1d09dbe4f32667b4d320ec2f54fc326ebeffcd0b0bcc2bb6c4", size = 251753, upload-time = "2026-05-10T18:00:27.104Z" }, + { url = "https://files.pythonhosted.org/packages/f3/e9/cdf00d38817742c541ade405e115a3f7bf36e6f2a8b99d4f209861b85a2d/coverage-7.14.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:1733198802d71ec4c524f322e2867ee05c62e9e75df86bdca545407a221827d1", size = 255823, upload-time = "2026-05-10T18:00:29.038Z" }, + { url = "https://files.pythonhosted.org/packages/38/fc/5e7877cf5f902d08a17ff1c532511476d87e1bea355bd5028cb97f902e79/coverage-7.14.0-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:72a305291fa8ee01332f1aaf38b348ca34097f6aa0b0ef627eef2837e57bbba5", size = 251323, upload-time = "2026-05-10T18:00:30.647Z" }, + { url = "https://files.pythonhosted.org/packages/18/9d/50f05a72dff8487464fdd4178dda5daed642a060e60afb644e3d45123559/coverage-7.14.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:fcaba850dd317c65423a9d63d88f9573c53b00354d6dd95724576cc98a131595", size = 253197, upload-time = "2026-05-10T18:00:32.211Z" }, + { url = "https://files.pythonhosted.org/packages/00/3f/6f61ffe6439df266c3cf60f5c99cfaa21103d0210d706a42fc6c30683ff8/coverage-7.14.0-cp312-cp312-win32.whl", hash = "sha256:5ac83957a80d0701310e96d8bec68cdcf4f90a7674b7d13f15a344315b41ab27", size = 222515, upload-time = "2026-05-10T18:00:33.717Z" }, + { url = "https://files.pythonhosted.org/packages/85/19/93853133df2cb371083285ef6a93982a0173e7a233b0f61373ba9fd30eb2/coverage-7.14.0-cp312-cp312-win_amd64.whl", hash = "sha256:70390b0da32cb90b501953716302906e8bcce087cb283e70d8c97729f22e92b2", size = 223324, upload-time = "2026-05-10T18:00:35.172Z" }, + { url = "https://files.pythonhosted.org/packages/74/18/9f7fe62f659f24b7a82a0be56bf94c1bd0a89e0ae7ab4c668f6e82404294/coverage-7.14.0-cp312-cp312-win_arm64.whl", hash = "sha256:91b993743d959b8be85b4abf9d5478216a69329c321efe5be0433c1a841d691d", size = 221944, upload-time = "2026-05-10T18:00:37.014Z" }, + { url = "https://files.pythonhosted.org/packages/6b/76/b7c66ee3c66e1b0f9d894c8125983aa0c03fb2336f2fd16559f9c966157f/coverage-7.14.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:f2bbb8254370eb4c628ff3d6fa8a7f74ddc40565394d4f7ab791d1fe568e37ef", size = 219990, upload-time = "2026-05-10T18:00:38.887Z" }, + { url = "https://files.pythonhosted.org/packages/b3/af/e567cbad5ba69c013a50146dfa886dc7193361fda77521f51274ff620e1b/coverage-7.14.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:23b81107f46d3f21d0cbce30664fcec0f5d9f585638a67081750f99738f6bf66", size = 220365, upload-time = "2026-05-10T18:00:40.864Z" }, + { url = "https://files.pythonhosted.org/packages/44/6f/9ad575d505b4d805b254febc8a5b338a2efe278f8786e56ff1cb8413f9c3/coverage-7.14.0-cp313-cp313-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:22a7e06a5f11a757cdfe79018e9095f9f69ae283c5cd8123774c788deec8717b", size = 251363, upload-time = "2026-05-10T18:00:42.489Z" }, + { url = "https://files.pythonhosted.org/packages/6f/5f/b5370068b2f57787454592ed7dcd1002f0f1703b7db1fa30f6a325a4ca6e/coverage-7.14.0-cp313-cp313-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:9d1aa57a1dc8e05bdc42e81c5d671d849577aeedf279f4c449d6d286f9ed88ca", size = 253961, upload-time = "2026-05-10T18:00:44.079Z" }, + { url = "https://files.pythonhosted.org/packages/29/1e/51adf17738976e8f2b85ddef7b7aa12a0838b056c92f175941d8862767c1/coverage-7.14.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:90c1a51bcfddf645b3bb7ec333d9e94393a8e94f55642380fa8a9a5a9e636cb7", size = 255193, upload-time = "2026-05-10T18:00:45.623Z" }, + { url = "https://files.pythonhosted.org/packages/9e/7b/5bfd7ac1df3b881c2ac7a5cbc99c7609e6296c402f5ef587cd81c6f355b3/coverage-7.14.0-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:a841fae2fadcae4f438d43b6ccc4aac2ad609f47cdb6cfdce60cbb3fe5ca7bc2", size = 257326, upload-time = "2026-05-10T18:00:47.173Z" }, + { url = "https://files.pythonhosted.org/packages/7d/38/1d37d316b174fad3843a1d76dbdfe4398771c9ecd0515935dd9ece9cd627/coverage-7.14.0-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c79d2319cabef1fe8e86df73371126931550804738f78ad7d31e3aad85a67367", size = 251582, upload-time = "2026-05-10T18:00:49.152Z" }, + { url = "https://files.pythonhosted.org/packages/34/46/746704f95980ba220214e1a41e18cec5aea80a898eaa53c51bf2d645ff36/coverage-7.14.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:1b23b0c6f0b1db6ad769b7050c8b641c0bf215ded26c1816955b17b7f26edfa9", size = 253325, upload-time = "2026-05-10T18:00:51.252Z" }, + { url = "https://files.pythonhosted.org/packages/e1/b9/bbe87206d9687b192352f893797825b5f5b15ecd3aa9c68fbff0c074d77b/coverage-7.14.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:55d3089079ce181a4566b1065ab28d2575eb76d8ac8f81f4fcda2bf037fee087", size = 251291, upload-time = "2026-05-10T18:00:52.816Z" }, + { url = "https://files.pythonhosted.org/packages/46/57/b8cdb12ac0d73ef0243218bd5e22c9df8f92edab8018213a86aec67c5324/coverage-7.14.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:49c005cba1e2f9677fb2845dcdf9a2e72a52a17d63e8231aaaae35d9f50215ef", size = 255448, upload-time = "2026-05-10T18:00:54.548Z" }, + { url = "https://files.pythonhosted.org/packages/1f/d4/5002019538b2036ce3c84340f54d2fd5100d55b0a6b0894eee56128d03c7/coverage-7.14.0-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:9117377b823daa28aa8635fbb08cda1cd6be3d7143257345459559aeef852d52", size = 251110, upload-time = "2026-05-10T18:00:56.122Z" }, + { url = "https://files.pythonhosted.org/packages/37/53/20c5009477660f084e6ed60bc02a91894b8e234e617e86ecfd9aaf78e27b/coverage-7.14.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:7b79d646cf46d5cf9a9f40281d4441df5849e445726e369006d2b117710b33fe", size = 252885, upload-time = "2026-05-10T18:00:57.967Z" }, + { url = "https://files.pythonhosted.org/packages/ae/ab/3cf6427ac9c1f1db747dbb1ce71dde47984876d4c2cfd018a3fef0a78d4d/coverage-7.14.0-cp313-cp313-win32.whl", hash = "sha256:fb609b3658479e33f9516d46f1a89dbb9b6c261366e3a11844a96ec487533dae", size = 222539, upload-time = "2026-05-10T18:00:59.581Z" }, + { url = "https://files.pythonhosted.org/packages/8f/b8/9228523e80321c2cb4880d1f589bc0171f2f71432c35118ad04dc01decce/coverage-7.14.0-cp313-cp313-win_amd64.whl", hash = "sha256:0773d8329cf32b6fd222e4b52622c61fe8d503eb966cfc8d3c3c10c96266d50e", size = 223344, upload-time = "2026-05-10T18:01:01.531Z" }, + { url = "https://files.pythonhosted.org/packages/a3/99/118daa192f95e3a6cb2740100fbf8797cda1734b4134ef0b5d501a7fa8f3/coverage-7.14.0-cp313-cp313-win_arm64.whl", hash = "sha256:b4e26a0f1b696faf283bffe5b8569e44e336c582439df5d53281ab89ee0cba96", size = 221966, upload-time = "2026-05-10T18:01:03.16Z" }, + { url = "https://files.pythonhosted.org/packages/e6/f1/a46cc0c013be170216253184a32366d7cbdb9252feaec866b05c2d12a894/coverage-7.14.0-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:953f521ca9445300397e65fda3dca58b2dbd68fee983777420b57ac3c77e9f90", size = 220679, upload-time = "2026-05-10T18:01:05.058Z" }, + { url = "https://files.pythonhosted.org/packages/64/8c/9c30a3d311a34177fa432995be7fbfc64477d8bac5630bd38055b1c9b424/coverage-7.14.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:98af83fd65ae24b1fdd03aaead967a9f523bcd2f1aab2d4f3ffda65bb568a6f1", size = 221033, upload-time = "2026-05-10T18:01:07.002Z" }, + { url = "https://files.pythonhosted.org/packages/9a/cd/3fb5e06c3badefd0c1b47e2044fdca67f8220a4ec2e7fcfb476aa0a67c6c/coverage-7.14.0-cp313-cp313t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:668b92e6958c4db7cf92e81caac328dfbbdbb215db2850ad28f0cbe1eea0bfbd", size = 262333, upload-time = "2026-05-10T18:01:08.903Z" }, + { url = "https://files.pythonhosted.org/packages/a8/e6/fbc322325c7294d3e22c1ad6b79e45d0806b25228c8e5842aed6d8169aa7/coverage-7.14.0-cp313-cp313t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:9fbd898551762dea00d3fef2b1c4f99afd2c6a3ff952ea07d60a9bd5ed4f34bc", size = 264410, upload-time = "2026-05-10T18:01:10.531Z" }, + { url = "https://files.pythonhosted.org/packages/08/92/c497b264bec1673c47cc77e26f760fcda4654cabf1f39546d1a23a3b8c35/coverage-7.14.0-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:68af363c07ecd8d4b7d4043d85cb376d7d227eceb54e5323ee45da73dbd3e426", size = 266836, upload-time = "2026-05-10T18:01:12.19Z" }, + { url = "https://files.pythonhosted.org/packages/78/fc/045da320987f401af5d2815d351e8aa799aec859f60e29f445e3089eeedb/coverage-7.14.0-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:6e57054a583da8ac55edf24117ea4c9133032cfc4cf72aa2d48c1e5d4b52f899", size = 267974, upload-time = "2026-05-10T18:01:13.926Z" }, + { url = "https://files.pythonhosted.org/packages/1b/ae/227b1e379497fb7a4fc3286e620f80c8a1e7cec66d45695a01639eb1af65/coverage-7.14.0-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:cc3499459bbcdd51a65b64c35ab7ed2764eaf3cba826e0df3f1d7fe2e102b70b", size = 261578, upload-time = "2026-05-10T18:01:15.564Z" }, + { url = "https://files.pythonhosted.org/packages/a0/f5/3570342900f2acea31d33ff1590c5d8bac1a8e1a2e1c6d34a5d5e61de681/coverage-7.14.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:45899ec2138a4346ed34d601dedf5076fb74edf2d1dd9dc76a78e82397edee90", size = 264394, upload-time = "2026-05-10T18:01:17.607Z" }, + { url = "https://files.pythonhosted.org/packages/16/29/de1bbc01c935b28f89b1dc3db85b011c055e843a8e5e3b83141c3f80af7f/coverage-7.14.0-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:8767486808c436f05b23ab98eb963fb29185e32a9357a166971685cb3459900f", size = 262022, upload-time = "2026-05-10T18:01:19.304Z" }, + { url = "https://files.pythonhosted.org/packages/35/95/f53890b0bf2fc10ab168e05d38869215e73ca24c4cb521c3bb0eb62fe16b/coverage-7.14.0-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:a3b5ddfd6aa7ddad53ee3edb231e88a2151507a43229b7d71b953916deca127d", size = 265732, upload-time = "2026-05-10T18:01:21.494Z" }, + { url = "https://files.pythonhosted.org/packages/ed/ea/c919e259081dd2bdf0e43b87209709ba7ec2e4117c2a7f5185379c43463c/coverage-7.14.0-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:63df0fe568e698e1045792399f8ab6da3a6c2dce3182813fb92afa2641087b47", size = 260921, upload-time = "2026-05-10T18:01:23.533Z" }, + { url = "https://files.pythonhosted.org/packages/1a/2c/c2831889705a81dc5d1c6ca12e4d8e9b95dfc146d153488a6c0ea685d28e/coverage-7.14.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:827d6397dbd95144939b18f89edf31f63e1f99633e8d5f32f22ba8bdda567477", size = 263109, upload-time = "2026-05-10T18:01:25.165Z" }, + { url = "https://files.pythonhosted.org/packages/5a/a9/2fcae5003cac3d63fe344d2166243c2756935f48420863c5272b240d550b/coverage-7.14.0-cp313-cp313t-win32.whl", hash = "sha256:7bf43e000d24012599b879791cff41589af90674722421ef11b11a5431920bab", size = 223212, upload-time = "2026-05-10T18:01:27.157Z" }, + { url = "https://files.pythonhosted.org/packages/3f/bb/18e94d7b14b9b398164197114a587a04ab7c9fdbe1d237eef57311c5e883/coverage-7.14.0-cp313-cp313t-win_amd64.whl", hash = "sha256:3f5549365af25d770e06b1f8f5682d9a5637d06eb494db91c6fa75d3950cc917", size = 224272, upload-time = "2026-05-10T18:01:29.107Z" }, + { url = "https://files.pythonhosted.org/packages/db/56/4f14fad782b035c81c4ffd09159e7103d42bb1d93ac8496d04b90a11b7da/coverage-7.14.0-cp313-cp313t-win_arm64.whl", hash = "sha256:6d160217ec6fe890f16ad3a9531761589443749e448f91986c972714fad361c8", size = 222530, upload-time = "2026-05-10T18:01:31.151Z" }, + { url = "https://files.pythonhosted.org/packages/1c/18/b9a6586d73992807c26f9a5f274131be3d76b56b18a82b9392e2a25d2e45/coverage-7.14.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:9aed9fa983514ca032790f3fe0d1c0e42ca7e16b42432af1706b50a9a46bef5d", size = 220036, upload-time = "2026-05-10T18:01:33.057Z" }, + { url = "https://files.pythonhosted.org/packages/f3/9b/4165a1d56ddc302a0e2d518fd9d412a4fd0b57562618c78c5f21c57194f5/coverage-7.14.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:ba3b8390db29296dbbf49e91b6fe08f990743a90c8f447ba4c2ffc29670dfa63", size = 220368, upload-time = "2026-05-10T18:01:34.705Z" }, + { url = "https://files.pythonhosted.org/packages/69/aa/c12e52a5ba148d9995229d557e3be6e554fe469addc0e9241b2f0956d8ea/coverage-7.14.0-cp314-cp314-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:3a5d8e876dfa2f102e970b183863d6dedd023d3c0eeca1fe7a9787bc5f28b212", size = 251417, upload-time = "2026-05-10T18:01:36.949Z" }, + { url = "https://files.pythonhosted.org/packages/d7/51/ec641c26e6dca1b25a7d2035ba6ecb7c884ef1a100a9e42fbe4ce4405139/coverage-7.14.0-cp314-cp314-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:5ebb8f4614a3787d567e610bbfdf96a4798dd69a1afb1bd8ad228d4111fe6ff3", size = 253924, upload-time = "2026-05-10T18:01:38.985Z" }, + { url = "https://files.pythonhosted.org/packages/33/c4/59c3de0bd1b538824173fd518fed51c1ce740ca5ed68e74545983f4053a9/coverage-7.14.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6b9bf47223dd8db3d4c4b2e443b02bace480d428f0822c3f991600448a176c97", size = 255269, upload-time = "2026-05-10T18:01:40.957Z" }, + { url = "https://files.pythonhosted.org/packages/7b/a9/36dfa153a62040296f6e7febfdb20a5720622f6ef5a81a41e8237b9a5344/coverage-7.14.0-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:3485a836550b303d006d57cc06e3d5afaabc642c77050b7c985a97b13e3776b8", size = 257583, upload-time = "2026-05-10T18:01:42.607Z" }, + { url = "https://files.pythonhosted.org/packages/26/7b/cc2c048d4114d9ab1c2409e9ee365e5ae10736df6dffcfc9444effa6c708/coverage-7.14.0-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:3e7e88110bae996d199d1693ca8ec3fd52441d426401ae963437598667b4c5eb", size = 251434, upload-time = "2026-05-10T18:01:44.537Z" }, + { url = "https://files.pythonhosted.org/packages/ee/df/6770eaa576e604575e9a78055313250faef5faa84bd6f71a39fece519c43/coverage-7.14.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:15228a6800ce7bdf1b74800595e56db7138cecb338fdbf044806e10dcf182dfe", size = 253280, upload-time = "2026-05-10T18:01:46.175Z" }, + { url = "https://files.pythonhosted.org/packages/ad/9e/1c0264514a3f98259a6d64765a397b2c8373e3ba59ee722a4802d3ec0c61/coverage-7.14.0-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:9d26ac7f5398bafc5b57421ad994e8a4749e8a7a0e62d05ec7d53014d5963bfa", size = 251241, upload-time = "2026-05-10T18:01:48.732Z" }, + { url = "https://files.pythonhosted.org/packages/64/16/4efdf3e3c4079cdbf0ece56a2fea872df9e8a3e15a13a0af4400e1075944/coverage-7.14.0-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:2fb73254ff43c911c967a899e1359bc5049b4b115d6e8fbdde4937d0a2246cd5", size = 255516, upload-time = "2026-05-10T18:01:50.819Z" }, + { url = "https://files.pythonhosted.org/packages/93/69/b1de96346603881b3d1bc8d6447c83200e1c9700ffbaff926ba01ff5724c/coverage-7.14.0-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:454a380af72c6adada298ed270d38c7a391288198dbfb8467f786f588751a90c", size = 251059, upload-time = "2026-05-10T18:01:52.773Z" }, + { url = "https://files.pythonhosted.org/packages/a4/66/2881853e0363a5e0a724d1103e53650795367471b6afb234f8b49e713bc6/coverage-7.14.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:65c86fb646d2bd2972e96bd1a8b45817ed907cee68655d6295fe7ec031d04cca", size = 252716, upload-time = "2026-05-10T18:01:54.506Z" }, + { url = "https://files.pythonhosted.org/packages/55/5c/0d3305d002c41dcde873dbe456491e663dc55152ca526b630b5c47efd62f/coverage-7.14.0-cp314-cp314-win32.whl", hash = "sha256:6a6516b02a6101398e19a3f44820f69bab2590697f7def4331f668b14adaf828", size = 222788, upload-time = "2026-05-10T18:01:56.487Z" }, + { url = "https://files.pythonhosted.org/packages/f9/58/6e1b8f52fdc3184b47dc5037f5070d83a3d11042db1594b02d2a44d786c8/coverage-7.14.0-cp314-cp314-win_amd64.whl", hash = "sha256:45e0f79d8351fa76e256716df91eab12890d32678b9590df7ae1042e4bd4cf5d", size = 223600, upload-time = "2026-05-10T18:01:58.497Z" }, + { url = "https://files.pythonhosted.org/packages/00/70/a18c408e674bc26281cadaedc7351f929bd2094e191e4b15271c30b084cc/coverage-7.14.0-cp314-cp314-win_arm64.whl", hash = "sha256:4b899594a8b2d81e5cc064a0d7f9cac2081fed91049456cae7676787e41549c9", size = 222168, upload-time = "2026-05-10T18:02:00.411Z" }, + { url = "https://files.pythonhosted.org/packages/3d/89/2681f071d238b62aff8dfc2ab44fc24cfdb38d1c01f391a80522ff5d3a16/coverage-7.14.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:f580f8c80acd94ac72e863efe2cab791d8c38d153e0b463b92dfa000d5c84cd1", size = 220766, upload-time = "2026-05-10T18:02:02.313Z" }, + { url = "https://files.pythonhosted.org/packages/bd/c7/c987babafd9207ffa1995e1ef1f9b26762cf4963aa768a66b6f0501e4616/coverage-7.14.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:a2bd259c442cd43c49b30fbafc51776eb19ea396faf159d26a83e6a0a5f13b0c", size = 221035, upload-time = "2026-05-10T18:02:04.017Z" }, + { url = "https://files.pythonhosted.org/packages/5a/e9/d6a5ac3b333088143d6fc877d398a9a674dc03124a2f776e131f03864823/coverage-7.14.0-cp314-cp314t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:a706b908dfa85538863504c624b237a3cc34232bf403c057414ebfdb3b4d9f84", size = 262405, upload-time = "2026-05-10T18:02:05.915Z" }, + { url = "https://files.pythonhosted.org/packages/38/b1/e70838d29a7c08e22d44398a46db90815bbcbf28de06992bd9210d1a8d8e/coverage-7.14.0-cp314-cp314t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:7333cd944ee4393b9b3d3c1b598c936d4fc8d70573a4c7dacfec5590dd50e436", size = 264530, upload-time = "2026-05-10T18:02:07.582Z" }, + { url = "https://files.pythonhosted.org/packages/6b/73/5c31ef97763288d03d9995152b96d5475b527c63d91c84b01caea894b83a/coverage-7.14.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0f162bc9a15b82d947b02651b0c7e1609d6f7a8735ca330cfadec8481dd97d5a", size = 266932, upload-time = "2026-05-10T18:02:09.401Z" }, + { url = "https://files.pythonhosted.org/packages/e1/76/dd56d80f29c5f05b4d76f7e7c6d47cafacae017189c75c5759d24f9ff0cc/coverage-7.14.0-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:362cb78e01a5dc82009d88004cf60f2e6b6d6fcbfdec05b05af73b0abf40118f", size = 268062, upload-time = "2026-05-10T18:02:11.399Z" }, + { url = "https://files.pythonhosted.org/packages/6e/c7/27ba85cd5b95614f159ff93ebff1901584a8d192e2e5e24c4943a7453f59/coverage-7.14.0-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:acebd068fca5512c3a6fde9c045f901613478781a73f0e82b307b214daef23fb", size = 261504, upload-time = "2026-05-10T18:02:13.257Z" }, + { url = "https://files.pythonhosted.org/packages/13/2e/e8149f60ab5d5684c6eee881bdf34b127115cddbb958b196768dd9d63473/coverage-7.14.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:29fe3da551dface75deb2ccbf87b6b66e2e7ef38f6d89050b428be94afff3490", size = 264398, upload-time = "2026-05-10T18:02:15.063Z" }, + { url = "https://files.pythonhosted.org/packages/d9/7f/1261b025285323225f4b4abffa5a643649dfd67e25ddca7ebcbdea3b7cb3/coverage-7.14.0-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:b4cc4fce8672fffcb09b0eafc167b396b3ba53c4a7230f54b7aaffbf6c835fa9", size = 262000, upload-time = "2026-05-10T18:02:16.756Z" }, + { url = "https://files.pythonhosted.org/packages/d3/dc/829c54f60b9d08389439c00f813c752781c496fc5788c78d8006db4b4f2b/coverage-7.14.0-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:5d4a51aad8ba8bdcd2b8bd8f03d4aca19693fa2327a3470e4718a25b03481020", size = 265732, upload-time = "2026-05-10T18:02:18.817Z" }, + { url = "https://files.pythonhosted.org/packages/ed/b0/70bd1419941652fa062689cba9c3eeafb8f5e6fbb890bce41c3bdda5dbd6/coverage-7.14.0-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:9f323af3e1e4f68b60b7b247e37b8515563a61375518fa59de1af48ba28a3db6", size = 260847, upload-time = "2026-05-10T18:02:20.528Z" }, + { url = "https://files.pythonhosted.org/packages/f2/73/be40b2390656c654d35ea0015ea7ba3d945769cf80790ad5e0bb2d56d2ba/coverage-7.14.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:1a0abc7342ea9711c469dd8b821c6c311e6bc6aac1442e5fbd6b27fae0a8f3db", size = 263166, upload-time = "2026-05-10T18:02:22.337Z" }, + { url = "https://files.pythonhosted.org/packages/29/55/4a643f712fcf7cf2881f8ec1e0ccb7b164aff3108f69b51801246c8799f2/coverage-7.14.0-cp314-cp314t-win32.whl", hash = "sha256:a9f864ef57b7172e2db87a096642dd51e179e085ab6b2c371c29e885f65c8fb2", size = 223573, upload-time = "2026-05-10T18:02:24.11Z" }, + { url = "https://files.pythonhosted.org/packages/27/96/3acae5da0953be042c0b4dea6d6789d2f080701c77b88e44d5bd41b9219b/coverage-7.14.0-cp314-cp314t-win_amd64.whl", hash = "sha256:29943e552fdc08e082eb51400fb2f58e118a83b5542bd06531214e084399b644", size = 224680, upload-time = "2026-05-10T18:02:25.896Z" }, + { url = "https://files.pythonhosted.org/packages/93/3d/6ab5d2dd8325d838737c6f8d83d62eb6230e0d70b87b51b57bbfd08fa767/coverage-7.14.0-cp314-cp314t-win_arm64.whl", hash = "sha256:742a73ea621953b012f2c4c2219b512180dd84489acf5b1596b0aafc55b9100b", size = 222703, upload-time = "2026-05-10T18:02:27.822Z" }, + { url = "https://files.pythonhosted.org/packages/61/e8/cb8e80d6f9f55b99588625062822bf946cf03ed06315df4bd8397f5632a1/coverage-7.14.0-py3-none-any.whl", hash = "sha256:8de5b61163aee3d05c8a2beab6f47913df7981dad1baf82c414d99158c286ab1", size = 211764, upload-time = "2026-05-10T18:02:29.538Z" }, +] + [[package]] name = "cuda-bindings" version = "13.2.0" @@ -479,6 +563,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/94/16/70255075a9859a0e3adb789b68ceb0e210dec03934245fd98d248226572f/idna-3.16-py3-none-any.whl", hash = "sha256:cc246e3a3f89580c3a951b5ad298ca4638078b2cdd4f115654332b5c26daded5", size = 74165, upload-time = "2026-05-22T00:16:16.698Z" }, ] +[[package]] +name = "iniconfig" +version = "2.3.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/72/34/14ca021ce8e5dfedc35312d08ba8bf51fdd999c576889fc2c24cb97f4f10/iniconfig-2.3.0.tar.gz", hash = "sha256:c76315c77db068650d49c5b56314774a7804df16fee4402c1f19d6d15d8c4730", size = 20503, upload-time = "2025-10-18T21:55:43.219Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cb/b1/3846dd7f199d53cb17f49cba7e651e9ce294d8497c8c150530ed11865bb8/iniconfig-2.3.0-py3-none-any.whl", hash = "sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12", size = 7484, upload-time = "2025-10-18T21:55:41.639Z" }, +] + [[package]] name = "jinja2" version = "3.1.6" @@ -1054,6 +1147,24 @@ dependencies = [ ] sdist = { url = "https://files.pythonhosted.org/packages/35/8e/d36f8880bcf18ec026a55807d02fe4c7357da9f25aebd92f85178000c0dc/openai_whisper-20250625.tar.gz", hash = "sha256:37a91a3921809d9f44748ffc73c0a55c9f366c85a3ef5c2ae0cc09540432eb96", size = 803191, upload-time = "2025-06-26T01:06:13.34Z" } +[[package]] +name = "packaging" +version = "26.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d7/f1/e7a6dd94a8d4a5626c03e4e99c87f241ba9e350cd9e6d75123f992427270/packaging-26.2.tar.gz", hash = "sha256:ff452ff5a3e828ce110190feff1178bb1f2ea2281fa2075aadb987c2fb221661", size = 228134, upload-time = "2026-04-24T20:15:23.917Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/df/b2/87e62e8c3e2f4b32e5fe99e0b86d576da1312593b39f47d8ceef365e95ed/packaging-26.2-py3-none-any.whl", hash = "sha256:5fc45236b9446107ff2415ce77c807cee2862cb6fac22b8a73826d0693b0980e", size = 100195, upload-time = "2026-04-24T20:15:22.081Z" }, +] + +[[package]] +name = "pluggy" +version = "1.6.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f9/e2/3e91f31a7d2b083fe6ef3fa267035b518369d9511ffab804f839851d2779/pluggy-1.6.0.tar.gz", hash = "sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3", size = 69412, upload-time = "2025-05-15T12:30:07.975Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538, upload-time = "2025-05-15T12:30:06.134Z" }, +] + [[package]] name = "psycopg" version = "3.3.4" @@ -1214,6 +1325,45 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/fa/c3/7c8b240552251faf6b3a957db200fcfbbcec36763c050428b601e0c9b83b/pydantic_core-2.46.4-graalpy312-graalpy250_312_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:00c603d540afdd6b80eb39f078f33ebd46211f02f33e34a32d9f053bba711de0", size = 2147590, upload-time = "2026-05-06T13:39:29.883Z" }, ] +[[package]] +name = "pygments" +version = "2.20.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/c3/b2/bc9c9196916376152d655522fdcebac55e66de6603a76a02bca1b6414f6c/pygments-2.20.0.tar.gz", hash = "sha256:6757cd03768053ff99f3039c1a36d6c0aa0b263438fcab17520b30a303a82b5f", size = 4955991, upload-time = "2026-03-29T13:29:33.898Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f4/7e/a72dd26f3b0f4f2bf1dd8923c85f7ceb43172af56d63c7383eb62b332364/pygments-2.20.0-py3-none-any.whl", hash = "sha256:81a9e26dd42fd28a23a2d169d86d7ac03b46e2f8b59ed4698fb4785f946d0176", size = 1231151, upload-time = "2026-03-29T13:29:30.038Z" }, +] + +[[package]] +name = "pytest" +version = "9.0.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "iniconfig" }, + { name = "packaging" }, + { name = "pluggy" }, + { name = "pygments" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/7d/0d/549bd94f1a0a402dc8cf64563a117c0f3765662e2e668477624baeec44d5/pytest-9.0.3.tar.gz", hash = "sha256:b86ada508af81d19edeb213c681b1d48246c1a91d304c6c81a427674c17eb91c", size = 1572165, upload-time = "2026-04-07T17:16:18.027Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d4/24/a372aaf5c9b7208e7112038812994107bc65a84cd00e0354a88c2c77a617/pytest-9.0.3-py3-none-any.whl", hash = "sha256:2c5efc453d45394fdd706ade797c0a81091eccd1d6e4bccfcd476e2b8e0ab5d9", size = 375249, upload-time = "2026-04-07T17:16:16.13Z" }, +] + +[[package]] +name = "pytest-cov" +version = "7.1.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "coverage" }, + { name = "pluggy" }, + { name = "pytest" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b1/51/a849f96e117386044471c8ec2bd6cfebacda285da9525c9106aeb28da671/pytest_cov-7.1.0.tar.gz", hash = "sha256:30674f2b5f6351aa09702a9c8c364f6a01c27aae0c1366ae8016160d1efc56b2", size = 55592, upload-time = "2026-03-21T20:11:16.284Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9d/7a/d968e294073affff457b041c2be9868a40c1c71f4a35fcc1e45e5493067b/pytest_cov-7.1.0-py3-none-any.whl", hash = "sha256:a0461110b7865f9a271aa1b51e516c9a95de9d696734a2f71e3e78f46e1d4678", size = 22876, upload-time = "2026-03-21T20:11:14.438Z" }, +] + [[package]] name = "python-dateutil" version = "2.9.0.post0" @@ -1393,6 +1543,31 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/a0/f4/c67b0b3f1b9245e8d266f0f112c500d50e5b4e83cb6f3b71b6528104182a/requests-2.34.2-py3-none-any.whl", hash = "sha256:2a0d60c172f83ac6ab31e4554906c0f3b3588d37b5cb939b1c061f4907e278e0", size = 73075, upload-time = "2026-05-14T19:25:26.443Z" }, ] +[[package]] +name = "ruff" +version = "0.15.14" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/dc/8a/8bce2894573e9dae6ff4d77fe34ad727d79b9e6238ad288c5638990d90f6/ruff-0.15.14.tar.gz", hash = "sha256:48e866b165be4a9bdbf310f7d3c9a07edef2fe8cd63ffeb4e00bb590506ebf9f", size = 4700910, upload-time = "2026-05-21T14:34:55.177Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b9/c8/74a92c6ff9fcfb4f1f947126d3ebee8389276e161ecc85de5bda7cda51bd/ruff-0.15.14-py3-none-linux_armv6l.whl", hash = "sha256:8dd2db9416e487c8d4b01fa7056bb02c4d05969d4f8d17a08c229c2f4ff3c108", size = 10739177, upload-time = "2026-05-21T14:34:37.332Z" }, + { url = "https://files.pythonhosted.org/packages/45/91/254a35c20acc38a7223c9d2d594af12e794432464f2cdeb52af1dc4a892d/ruff-0.15.14-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:be4ff55af755bd71a00ab3dc6bd7ffc467bd76e0df6881e286c2e3d23e8fb43b", size = 11144969, upload-time = "2026-05-21T14:34:43.978Z" }, + { url = "https://files.pythonhosted.org/packages/56/9e/d13e40f83b8d0a94430e6778ce1d94a43b38cf2efe63278bdd2b4c65abbf/ruff-0.15.14-py3-none-macosx_11_0_arm64.whl", hash = "sha256:48d5909d7d06276ce7dde6d32bfa4b0d4cb2651145cd8ee4b440722cbc77832f", size = 10478207, upload-time = "2026-05-21T14:34:48.378Z" }, + { url = "https://files.pythonhosted.org/packages/8d/f1/b15a7839fa4f332f8acec78e20564f26bb2d866e3d21710b877fd0263000/ruff-0.15.14-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ca8cbfa94c4f90984a67561978602746d4cd27103568f745fa90eee3f0d4107d", size = 10818459, upload-time = "2026-05-21T14:34:22.318Z" }, + { url = "https://files.pythonhosted.org/packages/45/33/53d651177f84f94b400a0e27f8824eeada3dddc9d5ee8aeb048f4352a520/ruff-0.15.14-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:9a6bbc0333f1ab053423bcbf6226477d266ca7cec7738c4c8e3f55647803f3c4", size = 10541800, upload-time = "2026-05-21T14:34:20.209Z" }, + { url = "https://files.pythonhosted.org/packages/b8/a6/868f87e0bf9786ed24b5d0d0ad8676b8a94fd1912f42cddf9cfc7857818a/ruff-0.15.14-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8a24a4f7605d7003a6674d4387651effd939dead3fddd0f36561eb77a9a2e542", size = 11342149, upload-time = "2026-05-21T14:34:46.365Z" }, + { url = "https://files.pythonhosted.org/packages/a7/8b/38cd5c19faffdcc05a408d2b78edccc69492ab9720eadb49ea15ef80d768/ruff-0.15.14-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:049b5326e53ed80978f2fc041a280603f69dd6b0c95464342a2bb4572d9d9e2f", size = 12212563, upload-time = "2026-05-21T14:34:28.579Z" }, + { url = "https://files.pythonhosted.org/packages/3e/4d/a3c5b874a556d5731e3e657aaf04311bb76f0a5c3ec220ed43051be6b64b/ruff-0.15.14-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d4ed42e6696c8dfa5f06728e6441993901f548eb92d73bc472cb5a38d1395fbf", size = 11493299, upload-time = "2026-05-21T14:34:41.836Z" }, + { url = "https://files.pythonhosted.org/packages/1e/c0/56472c251d09858a53e51efbd485b09e1995d8731668b76d52e5dd6ee0f1/ruff-0.15.14-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:715c543cf450c4888251f91c52f1942a800541d9bddd7ac060aa4e6b77ae7cba", size = 11455931, upload-time = "2026-05-21T14:34:57.276Z" }, + { url = "https://files.pythonhosted.org/packages/2c/4a/e2e7b4d8dbf233d4eace59c75bc3435fa6d8bd3bae82d351d4e4300c0fd1/ruff-0.15.14-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:72ebab6013ec887d439d8b7593737a0a4ffb06d45d209d4e4bf2e92813082d3f", size = 11400794, upload-time = "2026-05-21T14:34:39.773Z" }, + { url = "https://files.pythonhosted.org/packages/97/c7/83c0539fe34c3e09136204d1e75d6052492364e0b3cb05e9465423f567d7/ruff-0.15.14-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:49072d36abdbe97a8dd7f480afe9c675699c0c495d4c84076e2c1203c4550581", size = 10804759, upload-time = "2026-05-21T14:34:31.045Z" }, + { url = "https://files.pythonhosted.org/packages/86/a6/18f2bfc095a2ab4a78745644e428205532ce6653a5d0fa8501572891534d/ruff-0.15.14-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:958522aee105068640c2c2ceae08f413ae44d922f52a1374ac13d6a96032fc93", size = 10539517, upload-time = "2026-05-21T14:34:53.064Z" }, + { url = "https://files.pythonhosted.org/packages/54/3a/5a8b3b69c654d4e4bf1d246ac5b49cbcdac6eaab6905925f8915f31e3b80/ruff-0.15.14-py3-none-musllinux_1_2_i686.whl", hash = "sha256:f3707da619a143a2e8830e2abab8224478d69ace2d28cb6c20543ae97c36bf61", size = 11065169, upload-time = "2026-05-21T14:34:24.484Z" }, + { url = "https://files.pythonhosted.org/packages/ed/c5/8864e4e7925b836ea354b31d57641ec03830564e281a8b6f061f8c3e0ec1/ruff-0.15.14-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:bb01d645694e3ec0102105d07ef2d53703970407d59c04e59d3ba0b7a1d53553", size = 11560214, upload-time = "2026-05-21T14:34:50.975Z" }, + { url = "https://files.pythonhosted.org/packages/36/38/012bf76752e1f89ed50b77b99532d90f3a3e287bc7918e1fc0948ac866ac/ruff-0.15.14-py3-none-win32.whl", hash = "sha256:6d0c1ad2a0ab718d39b6d8fd2217981ce4d625cd96a720095f798fb47d8b13e6", size = 10805548, upload-time = "2026-05-21T14:34:33.453Z" }, + { url = "https://files.pythonhosted.org/packages/d1/b7/4ea2c170f10ad760fff2a5250beb18897719dc8b52b53a24cddbb9dd3f19/ruff-0.15.14-py3-none-win_amd64.whl", hash = "sha256:802342981e056db3851a7836e5b070f8f15f67d4a685ae2a6160939d364b2902", size = 11939523, upload-time = "2026-05-21T14:34:18.077Z" }, + { url = "https://files.pythonhosted.org/packages/62/d5/bc97ff895ec35cf3925d4bd60f3b39d822f377a446906ec9bcc87405e59b/ruff-0.15.14-py3-none-win_arm64.whl", hash = "sha256:ff47b90a9ef6a40c9e2f3b479c1fb78531adf055b94c1eba0a7ba04b31951826", size = 11208607, upload-time = "2026-05-21T14:34:26.525Z" }, +] + [[package]] name = "second-brain" version = "0.1.0" @@ -1421,6 +1596,13 @@ api = [ { name = "anthropic" }, ] +[package.dev-dependencies] +dev = [ + { name = "pytest" }, + { name = "pytest-cov" }, + { name = "ruff" }, +] + [package.metadata] requires-dist = [ { name = "alembic", specifier = ">=1.13.0" }, @@ -1443,6 +1625,13 @@ requires-dist = [ ] provides-extras = ["api"] +[package.metadata.requires-dev] +dev = [ + { name = "pytest", specifier = ">=8.0" }, + { name = "pytest-cov", specifier = ">=5.0" }, + { name = "ruff", specifier = ">=0.8" }, +] + [[package]] name = "setuptools" version = "81.0.0" From 6c1445c8edb22ee95947b09c5117098394822875 Mon Sep 17 00:00:00 2001 From: Travis Herbranson Date: Sun, 24 May 2026 22:57:31 -0400 Subject: [PATCH 04/13] docs: update CLAUDE.md + migration plan to reflect shipped Postgres design MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CLAUDE.md updates: - new src/second_brain/embeddings/ entry in the project layout - setup section now lists the postgres-superuser bootstrap (CREATE SCHEMA AUTHORIZATION lovebug) and the alembic step - env var table now covers SECOND_BRAIN_DATABASE_URL, OLLAMA_URL, EMBEDDING_MODEL, and the pool sizing knobs - conventions section gained five new gotchas (the lovebug CREATE gap, public.embeddings ownership, best-effort embedding, etc.) postgres-migration-planning.md flipped from "planning questions" to a "what shipped" runbook — locked decisions table, operator setup steps, known gotchas, and the next-iteration backlog. --- CLAUDE.md | 41 +++++-- postgres-migration-planning.md | 178 +++++++++++++++++++------------ reviews/a-review-2026-05-24-2.md | 30 ++++++ reviews/a-review-2026-05-24-3.md | 65 +++++++++++ 4 files changed, 239 insertions(+), 75 deletions(-) create mode 100644 reviews/a-review-2026-05-24-2.md create mode 100644 reviews/a-review-2026-05-24-3.md diff --git a/CLAUDE.md b/CLAUDE.md index 51a1adc..4e30706 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -22,18 +22,36 @@ pending → pulled → transcribed → analyzed → accepted → published src/second_brain/ ├── adapters/ # Pull adapters (youtube.py, article.py) ├── context/ # Context assembler +├── embeddings/ # Chunk + nomic-embed-text + upsert into public.embeddings ├── extractor/ # LLM extraction engine + Pydantic schema ├── llm/ # claude_cli.py — thin subprocess wrapper around `claude -p` ├── compiler/ # Wiki compiler (reads vault, writes pages, git commits) ├── scheduler/ # Rate-limit smoother (call-count for CLI, token-budget for API) └── web/ # FastAPI + Jinja2 + HTMX UI + +alembic/ # Schema migrations for the second_brain Postgres schema ``` ## Setup ```bash cp config/settings.toml.example config/settings.toml -# Edit settings.toml with your paths +# Edit settings.toml — at minimum, point the paths and set SECOND_BRAIN_DATABASE_URL. + +# Connection string format. Containerised (default): +# postgresql+psycopg://lovebug:@homelab-postgres:5432/petalbrain +# Host-side dev runs: +# postgresql+psycopg://lovebug:@127.0.0.1:5433/petalbrain +export SECOND_BRAIN_DATABASE_URL=... + +# One-time, before first migration: bootstrap the schema as postgres +# superuser. lovebug doesn't have CREATE on petalbrain, so this can't be +# done from the app side — same pattern as wiki/trellis/ob1. +# CREATE SCHEMA second_brain AUTHORIZATION lovebug; +# GRANT ALL PRIVILEGES ON SCHEMA second_brain TO lovebug; + +uv sync +uv run alembic upgrade head uv run second-brain serve ``` @@ -58,6 +76,10 @@ uv run second-brain serve - `ANTHROPIC_API_KEY` — only required when `extractor.backend = "api"`. The default `cli` backend uses your Max OAuth and does not need it. - `SECOND_BRAIN_CONFIG` — optional override for the settings file path. +- `SECOND_BRAIN_DATABASE_URL` — Postgres connection string. Falls back to `HERBYLAB_DATABASE_URL` if unset. Required at runtime; both `second_brain.database` and `second_brain.embeddings` read it. Use the SQLAlchemy `postgresql+psycopg://` form; the embeddings module strips the `+psycopg` prefix internally for raw psycopg. +- `OLLAMA_URL` — overrides `[embeddings].ollama_url`. Default `http://ollama:11434` (homelab docker network); for host-side runs use `http://127.0.0.1:11434`. +- `EMBEDDING_MODEL` — overrides `[embeddings].model`. Default `nomic-embed-text` (768-d). +- `SECOND_BRAIN_DB_POOL_MIN` / `SECOND_BRAIN_DB_POOL_MAX` — SQLAlchemy pool sizing knobs (defaults 1 / 9 overflow). ## Domains @@ -68,24 +90,31 @@ uv run second-brain serve ## Key files -- `config/settings.toml` — vault path, DB path, scheduler window, `[extractor]` backend config +- `config/settings.toml` — vault path, `[database]`, `[extractor]`, `[embeddings]`, `[scheduler]` - `prompts/.md` — per-domain extraction prompt templates - `src/second_brain/extractor/schema.py` — `LLMExtraction` (LLM-only fields, drives the `--json-schema` contract) + `ExtractionResult` (LLMExtraction + SourceMeta filled from the DB) - `src/second_brain/llm/claude_cli.py` — CLI subprocess wrapper -- `src/second_brain/models.py` — SQLAlchemy models + `utcnow()` helper (replaces deprecated `datetime.utcnow`) -- `postgres-migration-planning.md` — open questions blocking the SQLite → Postgres+pgvector migration +- `src/second_brain/models.py` — SQLAlchemy models (all tables live in the `second_brain` schema) + `utcnow()` helper +- `src/second_brain/database.py` — Postgres engine + URL resolution +- `src/second_brain/embeddings/__init__.py` — chunk → Ollama → upsert into `public.embeddings`; best-effort with graceful degradation +- `alembic/` + `alembic.ini` — schema migrations for the `second_brain` schema (Alembic does **not** manage `public.embeddings`) +- `postgres-migration-planning.md` — original planning doc; superseded by this file once the migration shipped ## Conventions / gotchas -- **Naive UTC timestamps everywhere.** Use `from second_brain.models import utcnow` rather than `datetime.utcnow()` (deprecated in 3.12) or `datetime.now()` (local time). +- **Naive UTC timestamps everywhere.** Use `from second_brain.models import utcnow` rather than `datetime.utcnow()` (deprecated in 3.12) or `datetime.now()` (local time). Postgres columns are `timestamp without time zone` — do not switch to `timestamptz`. +- **PK style.** Int autoincrement (`SERIAL`) — locked for compatibility with the existing schema shape. Do not switch to `BIGINT GENERATED ALWAYS AS IDENTITY` or UUID. - **Re-query inside the session.** SQLAlchemy ORM attribute access after the session closes raises `DetachedInstanceError`. The `process` command and the web accept/reject endpoints both collect IDs first, then re-query inside the per-iteration / per-request session block, and pre-compute any dict views needed for templates *inside* the `with db.session()` block. - **Starlette `TemplateResponse` signature.** Starlette ≥ 0.36 requires `templates.TemplateResponse(request, name, context)`. The old `(name, context)` form raises `TypeError: unhashable type: 'dict'`. Do not put `"request": request` into the context dict — it's auto-injected now. - **Vault git repo auto-init.** The wiki compiler calls `_ensure_git_repo()` on every run, which `git init`s the vault and creates an empty seed commit if no `.git` is present. `_git_commit()` also checks `git diff --cached --quiet` before committing to avoid empty commits. - **Scheduler throttling shape.** The CLI backend is throttled by `max_calls_per_hour` (Max subscription limits are call-shaped, not token-shaped). The API backend uses `max_tokens_per_hour`. Both share `min_gap_seconds`. The scheduler targets sources in `TRANSCRIBED` state — earlier code mistakenly filtered on `ANALYZED`, making it a no-op. +- **Alembic + lovebug privilege gap.** `lovebug` owns the `second_brain` schema but does **not** have `CREATE` on the `petalbrain` database. `alembic/env.py` therefore guards `CREATE SCHEMA` behind a `pg_namespace` lookup, and the v1 migration intentionally drops the `IF NOT EXISTS` form (a bare `CREATE SCHEMA IF NOT EXISTS` raises permission denied even when the schema is already present). The schema is bootstrapped once, out-of-band, by `postgres` superuser. +- **public.embeddings is shared, not ours.** Alembic only manages the `second_brain` schema. The vector table itself is owned by `postgres` and shared across vault-mcp / ob1 / second-brain — we just INSERT into it. Rows are keyed by `(source_schema='second_brain', source_table='extractions', source_id, embedding_model='nomic-embed-text')`. +- **Embedding is best-effort.** The pipeline writes the relational `extractions` row first and `sess.flush()`es; only then calls `embed_extraction()`. Any failure (no DB URL, no Ollama, embedding 500) logs at WARNING and returns None — the relational row and file vault stay canonical. +- **File-based wiki, not petalbrain.wiki.** The wiki compiler still writes markdown into the configured `vault_path` and commits via git. We do **not** dual-write into `petalbrain.wiki` — that's vault-mcp's territory and intentionally out of scope. ## In-flight work -- **Postgres + pgvector migration.** SQLite is the current backing store. Travis wants to move to his existing Postgres + pgvector instance and use the shared `/projects/shared/python/embedding_chunking` module for vector indexing. Planning questions are enumerated in `postgres-migration-planning.md` at the project root; no migration work has started yet. - **Vault location.** Recommended `/opt/projects/second-brain-vault/` as the production location (peer to the project dir). Not yet confirmed. ## Validation diff --git a/postgres-migration-planning.md b/postgres-migration-planning.md index 99cbac7..0748454 100644 --- a/postgres-migration-planning.md +++ b/postgres-migration-planning.md @@ -1,78 +1,118 @@ -# Postgres + pgvector migration — planning questions +# Postgres + pgvector migration — implementation notes -Status: awaiting input from Travis before scoping the migration. -Trigger: user pivoted away from SQLite — wants to use the existing Postgres + pgvector instance, plus the shared `/projects/shared/python/embedding_chunking` module. +Status: **shipped** (2026-05-24). Originally a planning checklist; rewritten +to capture the implemented design and the decisions that landed. Kept in the +repo so the next iteration has the "why" without grepping commits. --- -## 1. Connection / secrets +## What shipped -- [ ] Host, port, database name, user -- [ ] Where does the connection string live? Options: - - env var (e.g. `SECOND_BRAIN_DB_URL`) - - `config/settings.toml` (would need a `.local.toml` override so the password doesn't get committed) - - `.pgpass` - - other -- [ ] Is the Postgres instance on `herbydev`, the NAS, or elsewhere? Tailscale-only or LAN-reachable? -- [ ] Driver preference: - - `psycopg` v3 sync (default suggestion — pipeline + scheduler are sync today) - - `asyncpg` (only worth it if we also rewrite the web layer to async) - -## 2. Schema layout - -- [ ] Same Postgres **instance** as `vault-mcp`'s `petalbrain.wiki`? Yes / No -- [ ] If yes — same **database** with a dedicated schema (e.g. `second_brain.sources`, `second_brain.extractions`)? Or its own database entirely? -- [ ] Naming conventions to match: - - snake_case tables — assumed yes - - `created_at` / `updated_at` as `timestamptz` (vs current naive UTC) — preference? - - PK type: `bigserial` / `bigint identity` / UUID? -- [ ] Should second-brain integrate with `vault-mcp` at all? - - Option A: standalone — second-brain owns its tables, the wiki compiler still writes Obsidian markdown files - - Option B: dual-write — published extractions land in `petalbrain.wiki` via `vault-mcp` so they show up in the unified wiki - - Option C: replace the file-based compiler entirely with `vault-mcp` writes - -## 3. Embeddings - -- [ ] Public API of `/projects/shared/python/embedding_chunking`: - - function signatures - - what it returns (raw text chunks? chunks + vectors? chunker only, embeddings done separately?) - - embedding model + vector dimension (drives the `vector(N)` column type) - - any batching / rate-limit behavior we need to respect -- [ ] Where should embeddings attach? - - (a) on `Extraction` rows — semantic search over summaries / key_points / claims - - (b) on chunked transcripts — chunk-level retrieval for "find the part of the video that says X" - - (c) both (separate tables: `extraction_embeddings` + `transcript_chunks`) -- [ ] pgvector index strategy: - - HNSW — better recall, slower build, recommended default for personal-scale corpus - - IVFFlat — faster build, needs periodic `ANALYZE` + a meaningful `lists` value -- [ ] Distance metric: cosine (typical), L2, or inner product? - -## 4. Migration tooling - -- [ ] Schema is small and not yet in production. Cleanest path: - - SQLAlchemy `Base.metadata.create_all()` against Postgres - - one-off import script to move existing SQLite rows over (if any are worth keeping) -- [ ] Alternative: stand up Alembic now while the cost is low. Pays off the first time you `ALTER` a table in anger; costs ~1 session of setup. -- [ ] Travis's call — defaulting to `create_all` + import script unless you say otherwise. - -## 5. Pooling - -- [ ] Three consumers share the DB: CLI (`process`, `compile`), the FastAPI web server, the scheduler. -- [ ] Default plan: SQLAlchemy `QueuePool` with `pool_size=5, max_overflow=5`. -- [ ] Is PgBouncer already in front of this instance? If yes — switch to `NullPool` on our side and let PgBouncer pool. - -## 6. Other open items (carryover from prior session) - -- [ ] Vault location decision: I recommended `/opt/projects/second-brain-vault/` as a peer dir. Not yet confirmed. -- [ ] NAS mount: is it mounted on `herbydev`, and at what path? Relevant if media (downloaded videos, transcripts) should live there rather than on the herbydev SSD. +- Backing store switched from SQLite to the existing **petalbrain** Postgres + instance (pgvector/pg17), connecting over the homelab docker network at + `homelab-postgres:5432`. +- Relational tables (`sources`, `extractions`, `wiki_pages`) live in a new + `second_brain` schema owned by the **`lovebug`** role — no dedicated service + role. +- Embeddings reuse the **shared `public.embeddings` table** (HNSW vector index + already in place). Rows are keyed by + `(source_schema='second_brain', source_table='extractions', source_id, embedding_model='nomic-embed-text')`. + For this round we embed **extraction summaries only** (not transcript + chunks, not key_points / claims). +- Embedding pipeline mirrors `vault_mcp.core.embeddings`: shared + `embedding_chunking` for 512/64-token chunking → Ollama + `nomic-embed-text` (768-d, `num_ctx=8192`) → delete-before-insert upsert + with vector text literals (`%s::vector`). +- **Graceful degradation** is the rule: missing DB URL, Ollama down, or any + DB error logs a warning and returns `None`. The relational row and the + file-based vault stay canonical. +- **Alembic** stands up the `second_brain` schema; `public.embeddings` is + managed out-of-band and intentionally outside the migration tree. +- File-based wiki compiler is unchanged. No dual-write to `petalbrain.wiki`. +- **No SQLite import** — we started clean on Postgres. --- -## What I will produce once these are answered +## Locked decisions (from the planning round) -1. Updated `src/second_brain/database.py` with a Postgres engine + URL resolution -2. Schema migration: new tables for embeddings (and chunks if we go with option (c)) -3. Wiring of `embedding_chunking` into the extractor (or into a post-extraction step in the scheduler) -4. `config/settings.toml.example` updates with the new `[database]` and `[embeddings]` blocks -5. README / CLAUDE.md updates noting the Postgres dependency -6. One-shot SQLite → Postgres import script (only if there are rows worth keeping) +| Question | Decision | +|---|---| +| DB instance | Existing petalbrain (homelab-postgres) | +| Schema | New `second_brain` schema (owned by `lovebug`) | +| Connection | Containerised: `homelab-postgres:5432` (NOT host `127.0.0.1:5433`) | +| Pooling | `psycopg_pool.ConnectionPool` min=1/max=10 — no PgBouncer in front | +| Driver | psycopg v3, SQLAlchemy uses `postgresql+psycopg://` | +| PK style | Int autoincrement (`SERIAL`) — kept | +| Timestamps | Naive UTC `timestamp without time zone` — kept | +| Embedding scope | Extraction summaries only (Phase 1) | +| Embedding model | `nomic-embed-text` (768-d), Ollama via `OLLAMA_URL` | +| Migration tool | Alembic, second-brain owns its own tree | +| `public.embeddings` | NOT managed by our Alembic — shared with vault-mcp / ob1 | +| File-based wiki | Kept intact; no dual-write into petalbrain.wiki | +| Old SQLite data | Not imported — clean start | + +--- + +## Setup (operator runbook) + +```bash +# 1. One-time, as postgres superuser, bootstrap the schema. lovebug doesn't +# have CREATE on petalbrain, so this can't be done from the app side. +docker exec homelab-postgres psql -U postgres -d petalbrain -c \ + "CREATE SCHEMA second_brain AUTHORIZATION lovebug; + GRANT ALL PRIVILEGES ON SCHEMA second_brain TO lovebug;" + +# 2. Point the project at the DB (env wins over settings.toml). +export SECOND_BRAIN_DATABASE_URL=\ +"postgresql+psycopg://lovebug:@homelab-postgres:5432/petalbrain" + +# 3. Install deps and run the migration. +uv sync +uv run alembic upgrade head +``` + +The `lovebug` password lives in `/opt/backups/postgres-consolidation/credentials.env` +(0600, host-only). + +--- + +## Key files (post-migration) + +- `src/second_brain/models.py` — schema-qualified ORM, `__table_args__` → + `second_brain`; native Postgres enum types are scoped to the schema too. +- `src/second_brain/database.py` — psycopg-v3 engine + URL resolution + (constructor arg → `SECOND_BRAIN_DATABASE_URL` → `HERBYLAB_DATABASE_URL` + → `[database].url`). +- `src/second_brain/embeddings/__init__.py` — near-verbatim port of + `vault_mcp/core/embeddings.py` adapted to our key tuple. +- `alembic.ini` + `alembic/env.py` — schema-pinned Alembic; the env.py guards + `CREATE SCHEMA` behind `pg_namespace` because `lovebug` lacks `CREATE` on + the database. +- `alembic/versions/4a1e2f6c9d10_v1_second_brain_pipeline_tables.py` — v1 + migration: enum types + `sources` + `extractions` + `wiki_pages`. +- `tests/test_smoke_embedding.py` — end-to-end smoke test exercising the full + Postgres + Ollama path. Auto-skips when either is unreachable. + +--- + +## Known gotchas + +1. **`CREATE SCHEMA IF NOT EXISTS` errors as `lovebug`** even when the schema + exists. `pg_namespace` lookup in `alembic/env.py` is the workaround. +2. **`+psycopg` URL prefix.** SQLAlchemy needs it; raw psycopg doesn't. + `database.py` adds it if missing; the embeddings module strips it. +3. **Pool conflict.** `database.py` opens an SQLAlchemy pool; `embeddings/__init__.py` + opens an independent psycopg pool. Both share the same DSN; both are small + (min=1) so they don't strain the cluster. +4. **`anthropic-skills:zero-check` needs `pytest-cov`.** Added to the new + `[dependency-groups].dev` block so `uv sync` resolves it. + +--- + +## Next iterations (open, not blocked) + +- Embed transcript chunks (or at least key_points / claims), not just summaries. +- Promote vector retrieval into the web UI (semantic search across extractions). +- Add a periodic backfill script that re-embeds extractions whose + `embedding_model` no longer matches the current default. +- Decide on the production vault path (`/opt/projects/second-brain-vault/`?). diff --git a/reviews/a-review-2026-05-24-2.md b/reviews/a-review-2026-05-24-2.md new file mode 100644 index 0000000..de7b7bb --- /dev/null +++ b/reviews/a-review-2026-05-24-2.md @@ -0,0 +1,30 @@ +# a-review — 2026-05-24 + +**Model:** gemini +**Diff base:** `HEAD~1` (12 files changed, 208 insertions(+), 21 deletions(-)) +**Session reference:** none +**Context:** CLAUDE.md + session notes + expanded diff + +--- + +Based on the provided context, here is my code review. + +### [INFO] Project Dependencies + +* **File:** `pyproject.toml` +* **Finding:** The change introduces a `[dependency-groups].dev` section to manage development-specific dependencies like `pytest`, `pytest-cov`, and `ruff`. +* **Reasoning:** This is a positive change that follows standard Python project structure. It cleanly separates the project's runtime dependencies from the tools required for testing and linting, which improves dependency management and clarifies the project's setup for new developers. + +### [INFO] Code Cleanup + +* **Files:** `src/second_brain/adapters/article.py`, `src/second_brain/adapters/youtube.py`, `src/second_brain/compiler/wiki.py`, `src/second_brain/context/assembler.py`, `src/second_brain/extractor/engine.py`, `src/second_brain/extractor/schema.py`, `src/second_brain/main.py`, `src/second_brain/scheduler/runner.py`, `src/second_brain/web/app.py` +* **Finding:** Multiple unused imports (`Optional`, `datetime`, `json`, `Path`, etc.) were removed from several files. +* **Reasoning:** This is a beneficial code hygiene practice. Removing unused imports reduces clutter, makes the code easier to read, and helps linters and static analysis tools run more effectively. It has no impact on runtime behavior but improves maintainability. + +### Summary + +The submitted changes are of high quality. They consist of two main improvements: +1. Properly defining development dependencies in `pyproject.toml`. +2. Cleaning up unused imports across the codebase, likely as a result of running a linter like the newly added `ruff`. + +I found no security vulnerabilities, bugs, or convention violations. The changes are safe and improve the project's overall maintainability and structure. The implementation aligns with the goal of setting up a better development and testing environment. diff --git a/reviews/a-review-2026-05-24-3.md b/reviews/a-review-2026-05-24-3.md new file mode 100644 index 0000000..b182b35 --- /dev/null +++ b/reviews/a-review-2026-05-24-3.md @@ -0,0 +1,65 @@ +# a-review — 2026-05-24 + +**Model:** gemini +**Diff base:** `4696d7f` (24 files changed, 1496 insertions(+), 144 deletions(-)) +**Session reference:** none +**Context:** CLAUDE.md + session notes + expanded diff + +--- + +Excellent, the changes look solid overall. This is a substantial and well-executed migration from SQLite to Postgres, with the bonus of a new embedding feature. The code quality is high, particularly the database and transaction management logic. + +I have a few findings, mostly related to documentation and test robustness. + +### [WARNING] Documentation is now out of sync with the codebase + +* **Location:** `CLAUDE.md` +* **Issue:** The project's main documentation file, `CLAUDE.md`, has not been updated to reflect these significant changes. + 1. It still states the "Postgres + pgvector migration" is "In-flight work" and refers to `postgres-migration-planning.md`. This change appears to complete that migration. + 2. The "Project layout" and "Key files" sections are missing the new `alembic/` directory for migrations and the new `src/second_brain/embeddings/` module. + 3. The configuration sections do not mention the new `[database]` or `[embeddings]` tables in `config/settings.toml`. +* **Impact:** New developers (or future you!) will be confused by the discrepancy between the documentation and the actual state of the code. This makes maintenance harder and increases the risk of mistakes. +* **Recommendation:** After a large feature lands, it's crucial to update the project's core documentation. `CLAUDE.md` should be updated to reflect that the project now uses Postgres, describe the new embedding pipeline, and document the new configuration options. The `postgres-migration-planning.md` file should likely be archived or deleted. + +### [INFO] Test fragility due to hardcoded string literal + +* **Location:** `tests/test_smoke_embedding.py`, line 111 +* **Issue:** The test query hardcodes the embedding model name: `AND embedding_model = 'nomic-embed-text'`. The application code correctly reads the model from the configuration. If the default model in `config/settings.toml` were to change in the future, this test would fail even if the application logic is correct. +* **Impact:** This creates a tight coupling between the test and a specific configuration value, making the test suite more brittle. +* **Recommendation:** The test should load the application `Config` and use the configured embedding model name in its verification query. This ensures the test is checking that the system works with *whatever* model is currently configured. + +```python +# tests/test_smoke_embedding.py + +# at the top +from second_brain.config import load_config + +# in the test function +config = load_config() +embedding_model = config.embeddings.get("model", "nomic-embed-text") + +# ... later in the raw SQL query +cur.execute( + """ + SELECT COUNT(*) ... + WHERE ... + AND embedding_model = %s + """, + (extraction_id, embedding_model), +) +``` + +### [INFO] Unused imports removed + +* **Location:** Multiple files, including `src/second_brain/adapters/article.py`, `src/second_brain/compiler/wiki.py`, `src/second_brain/extractor/schema.py`, etc. +* **Issue:** The diff removes several unused imports. While cleaning this up is good, the fact they were present suggests a linter (like `ruff`, which is in the dev dependencies) may not be running automatically as part of the development or pre-commit workflow. +* **Impact:** This is a minor code hygiene issue, but consistent linting helps catch small issues early and maintains code quality across the project. +* **Recommendation:** Consider setting up a pre-commit hook to run `ruff check --fix` to automate the removal of unused imports and fix other simple linting issues. + +## Summary + +Overall, this is a high-quality contribution. The migration to Postgres is thoughtful and robust, correctly using Alembic for schema management and a connection pool for performance. The implementation of the embedding pipeline is well-contained, idempotent, and gracefully handles potential failures. The addition of a comprehensive smoke test is a fantastic practice that significantly increases confidence in the new functionality. + +The primary area for improvement is in process: the project's documentation needs to be treated as part of the code and updated in the same change that implements new features or makes significant architectural modifications. + +Aside from the documentation gap, the code is clean, correct, and demonstrates a strong understanding of the technologies involved. Great work. From f44ac2ff7c454b3b98a3cd79ed7d8c7f108cbb74 Mon Sep 17 00:00:00 2001 From: Travis Herbranson Date: Sun, 24 May 2026 22:58:30 -0400 Subject: [PATCH 05/13] test: smoke test reads embedding model from config, not hardcoded literal a-review (gemini, full migration diff) flagged that the smoke test had the embedding model name baked into the SQL count query, decoupling it from the application's configured value. Read it back from `config.embeddings.model` (env override still wins) so the test stays valid if the default ever moves off nomic-embed-text. --- tests/test_smoke_embedding.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/tests/test_smoke_embedding.py b/tests/test_smoke_embedding.py index caf18a2..e2f9d0c 100644 --- a/tests/test_smoke_embedding.py +++ b/tests/test_smoke_embedding.py @@ -98,6 +98,15 @@ def test_extraction_round_trip_with_embedding(): extraction_id = ext.id source_id = src.id + # Read the configured model so the test doesn't break if the default + # is changed in settings.toml or via EMBEDDING_MODEL. + from second_brain.config import load_config + + embedding_model = os.environ.get( + "EMBEDDING_MODEL", + load_config().embeddings.get("model", "nomic-embed-text"), + ) + try: n1 = embed_extraction(extraction_id, summary_text) assert n1 is not None and n1 >= 1, f"first embed should write rows, got {n1}" @@ -119,9 +128,9 @@ def test_extraction_round_trip_with_embedding(): WHERE source_schema = 'second_brain' AND source_table = 'extractions' AND source_id = %s - AND embedding_model = 'nomic-embed-text' + AND embedding_model = %s """, - (extraction_id,), + (extraction_id, embedding_model), ) (count,) = cur.fetchone() assert count == n1, f"public.embeddings should hold {n1} rows, has {count}" From 0d9ebc9cc7760ccfceb4b454e794f96ff32d626d Mon Sep 17 00:00:00 2001 From: Travis Herbranson Date: Sun, 24 May 2026 23:33:13 -0400 Subject: [PATCH 06/13] embeddings: register pool close() at interpreter exit MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Without this, every `second-brain process` run prints couldn't stop thread 'pool-1-worker-N' within 5.0 seconds during interpreter shutdown — psycopg_pool's background workers don't get a clean stop signal before Python's threading shutdown deadline. Registering close_pool via atexit the first time we open the pool fixes it without changing any API. `close_pool` is already idempotent, so the explicit teardown path in the smoke test (which calls it directly) and the atexit path coexist safely. --- src/second_brain/embeddings/__init__.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/second_brain/embeddings/__init__.py b/src/second_brain/embeddings/__init__.py index 61b43b9..c11ea5d 100644 --- a/src/second_brain/embeddings/__init__.py +++ b/src/second_brain/embeddings/__init__.py @@ -23,6 +23,7 @@ index, not authoritative state. from __future__ import annotations +import atexit import logging import os from typing import Any @@ -73,6 +74,11 @@ def _get_pool() -> ConnectionPool | None: if not dsn: return None _pool = ConnectionPool(dsn, min_size=1, max_size=4, open=True, timeout=10) + # Make sure the pool's worker threads get a clean shutdown when the + # interpreter exits; without this psycopg_pool prints + # "couldn't stop thread pool-1-worker-N within 5.0 seconds" on every + # `second-brain process` run. + atexit.register(close_pool) return _pool From 09efbb5965d29993e1e8a6faac275fe10588ce51 Mon Sep 17 00:00:00 2001 From: Travis Herbranson Date: Mon, 25 May 2026 08:11:53 -0400 Subject: [PATCH 07/13] settings: v2 migration + ORM + settings_store helper MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Introduces second_brain.pipeline_settings — the single-row config row the upcoming web dashboard edits and the workers read at the start of each run. Pinned to id=1 by a CHECK constraint so upserts-by-PK keep the table singleton, and the migration seeds the row with the table's column defaults via INSERT ... ON CONFLICT DO NOTHING. Two consumer groups carved out: - transcription_* fields persist now; future tower-side worker reads them. - extraction_* fields will be read by the existing scheduler in the next commit, which is the actual behavior change Travis cares about today. The settings_store helper centralises get/update + form-parsing (time-of-day, int-or-none) and active-window math so the routes and the scheduler don't reimplement them. --- .../7b3e8a5c1f29_v2_pipeline_settings.py | 70 ++++++++++ src/second_brain/models.py | 63 ++++++++- src/second_brain/settings_store.py | 121 ++++++++++++++++++ 3 files changed, 253 insertions(+), 1 deletion(-) create mode 100644 alembic/versions/7b3e8a5c1f29_v2_pipeline_settings.py create mode 100644 src/second_brain/settings_store.py diff --git a/alembic/versions/7b3e8a5c1f29_v2_pipeline_settings.py b/alembic/versions/7b3e8a5c1f29_v2_pipeline_settings.py new file mode 100644 index 0000000..ee2ea63 --- /dev/null +++ b/alembic/versions/7b3e8a5c1f29_v2_pipeline_settings.py @@ -0,0 +1,70 @@ +"""v2 second_brain: pipeline_settings table + +Revision ID: 7b3e8a5c1f29 +Revises: 4a1e2f6c9d10 +Create Date: 2026-05-25 00:00:00.000000 + +Adds a single-row `pipeline_settings` table that the web dashboard edits +and the workers (current dev-side scheduler + future tower-side transcribe +worker) read at the start of each run. The transcription_* fields persist +now but are only consumed once the transcribe worker exists — the +extraction_* fields are wired into the existing scheduler in this commit. + +Single-row enforcement: PK is fixed at `id=1` with a CHECK constraint, so +INSERT-as-upsert-by-PK keeps the table free of stale rows. + +Schema/permission notes mirror the v1 migration: lovebug lacks CREATE on +the petalbrain database, so env.py has already bootstrapped the schema — +this migration just SETs search_path and emits DDL inside it. +""" + +from __future__ import annotations + +from collections.abc import Sequence + +from alembic import op + +revision: str = "7b3e8a5c1f29" +down_revision: str | Sequence[str] | None = "4a1e2f6c9d10" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + + +def upgrade() -> None: + op.execute("SET search_path TO second_brain") + + op.execute( + """ + CREATE TABLE pipeline_settings ( + id INTEGER PRIMARY KEY DEFAULT 1, + transcription_enabled BOOLEAN NOT NULL DEFAULT TRUE, + transcription_active_hours_start TIME, + transcription_active_hours_end TIME, + transcription_max_concurrent_gpu_jobs INTEGER NOT NULL DEFAULT 1, + transcription_max_video_length_seconds INTEGER, + transcription_max_items_per_run INTEGER NOT NULL DEFAULT 10, + extraction_enabled BOOLEAN NOT NULL DEFAULT TRUE, + extraction_active_hours_start TIME, + extraction_active_hours_end TIME, + extraction_max_items_per_run INTEGER NOT NULL DEFAULT 20, + updated_at TIMESTAMP NOT NULL + DEFAULT (now() AT TIME ZONE 'UTC'), + CONSTRAINT pipeline_settings_singleton CHECK (id = 1) + ) + """ + ) + + # Seed the singleton row with the table's column defaults. Idempotent + # via ON CONFLICT so re-running this migration on a partly-applied DB + # (or a manual cherry-pick) is safe. + op.execute( + """ + INSERT INTO pipeline_settings (id) VALUES (1) + ON CONFLICT (id) DO NOTHING + """ + ) + + +def downgrade() -> None: + op.execute("SET search_path TO second_brain") + op.execute("DROP TABLE IF EXISTS pipeline_settings") diff --git a/src/second_brain/models.py b/src/second_brain/models.py index 7519e53..2467c7c 100644 --- a/src/second_brain/models.py +++ b/src/second_brain/models.py @@ -14,11 +14,12 @@ Tables: from __future__ import annotations import enum -from datetime import datetime, timezone +from datetime import datetime, time, timezone from typing import Optional from sqlalchemy import ( JSON, + Boolean, DateTime, Enum, ForeignKey, @@ -26,6 +27,7 @@ from sqlalchemy import ( MetaData, String, Text, + Time, ) from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column, relationship @@ -193,6 +195,64 @@ class WikiPage(Base): return f"WikiPage(id={self.id}, vault_path={self.vault_path!r})" +class PipelineSettings(Base): + """Single-row config the web dashboard edits and workers read. + + The transcription_* fields persist now and will be honored by the + tower-side transcribe worker once it exists; the extraction_* fields + are already wired into the dev-side scheduler. The row is pinned to + `id=1` by a DB-level CHECK so upserts-by-PK keep the table singleton. + """ + + __tablename__ = "pipeline_settings" + + id: Mapped[int] = mapped_column(Integer, primary_key=True, default=1) + + # Transcription (consumed by the tower-side worker — not yet built). + transcription_enabled: Mapped[bool] = mapped_column( + Boolean, nullable=False, default=True + ) + transcription_active_hours_start: Mapped[Optional[time]] = mapped_column( + Time, nullable=True + ) + transcription_active_hours_end: Mapped[Optional[time]] = mapped_column( + Time, nullable=True + ) + transcription_max_concurrent_gpu_jobs: Mapped[int] = mapped_column( + Integer, nullable=False, default=1 + ) + transcription_max_video_length_seconds: Mapped[Optional[int]] = mapped_column( + Integer, nullable=True + ) + transcription_max_items_per_run: Mapped[int] = mapped_column( + Integer, nullable=False, default=10 + ) + + # Extraction (consumed by the dev-side scheduler today). + extraction_enabled: Mapped[bool] = mapped_column( + Boolean, nullable=False, default=True + ) + extraction_active_hours_start: Mapped[Optional[time]] = mapped_column( + Time, nullable=True + ) + extraction_active_hours_end: Mapped[Optional[time]] = mapped_column( + Time, nullable=True + ) + extraction_max_items_per_run: Mapped[int] = mapped_column( + Integer, nullable=False, default=20 + ) + + updated_at: Mapped[datetime] = mapped_column( + DateTime, default=utcnow, onupdate=utcnow + ) + + def __repr__(self) -> str: + return ( + f"PipelineSettings(transcription_enabled={self.transcription_enabled}, " + f"extraction_enabled={self.extraction_enabled})" + ) + + __all__ = [ "Base", "SCHEMA", @@ -200,6 +260,7 @@ __all__ = [ "SourceStatus", "SourceType", "Extraction", + "PipelineSettings", "WikiPage", "utcnow", ] diff --git a/src/second_brain/settings_store.py b/src/second_brain/settings_store.py new file mode 100644 index 0000000..877cb59 --- /dev/null +++ b/src/second_brain/settings_store.py @@ -0,0 +1,121 @@ +"""Helpers for the singleton `pipeline_settings` row. + +The web dashboard edits this row; workers read it at the start of each +run to decide whether to do anything and how much. DB values override +the static defaults in `settings.toml`. + +All functions take a SQLAlchemy `Session` so the caller owns the +transaction boundary — matches the rest of the codebase, which keeps +session lifecycle inside `with db.session()` blocks. +""" + +from __future__ import annotations + +from datetime import time +from typing import Any + +from sqlalchemy.orm import Session + +from second_brain.models import PipelineSettings, utcnow + + +# Field names allowed in `update_settings`. Anything else is rejected so a +# malicious form post can't toggle id/updated_at/etc. +_EDITABLE_FIELDS: tuple[str, ...] = ( + "transcription_enabled", + "transcription_active_hours_start", + "transcription_active_hours_end", + "transcription_max_concurrent_gpu_jobs", + "transcription_max_video_length_seconds", + "transcription_max_items_per_run", + "extraction_enabled", + "extraction_active_hours_start", + "extraction_active_hours_end", + "extraction_max_items_per_run", +) + + +def get_settings(sess: Session) -> PipelineSettings: + """Return the singleton settings row, creating it on first call. + + The v2 migration seeds the row with all DB-side defaults, so this + "create on miss" path only kicks in when somebody bootstrapped the + schema without applying the migration (e.g. an unconfigured test + using `Base.metadata.create_all`). + """ + row = sess.get(PipelineSettings, 1) + if row is None: + row = PipelineSettings(id=1) + sess.add(row) + sess.flush() + return row + + +def update_settings(sess: Session, **fields: Any) -> PipelineSettings: + """Apply a partial update to the singleton row. + + Unknown keys raise `KeyError`. Empty-string values for nullable fields + are normalised to None so the HTML form's blank inputs (active hours, + video-length cap) round-trip as NULL rather than as the strings "". + """ + for k in fields: + if k not in _EDITABLE_FIELDS: + raise KeyError(f"Unknown pipeline_settings field: {k!r}") + + row = get_settings(sess) + for k, v in fields.items(): + if v == "": + v = None + setattr(row, k, v) + row.updated_at = utcnow() + return row + + +def parse_time_or_none(raw: str | None) -> time | None: + """Parse an HTML value (HH:MM) into a `time`. + + Empty / missing values map to None — `update_settings` will store + NULL, which workers interpret as "no window, always active". + """ + if raw is None or raw == "": + return None + parts = raw.split(":") + if len(parts) < 2: + raise ValueError(f"Could not parse time: {raw!r}") + h, m = int(parts[0]), int(parts[1]) + return time(h, m) + + +def parse_int_or_none(raw: str | None) -> int | None: + if raw is None or raw == "": + return None + return int(raw) + + +def in_active_window(now_t: time, start: time | None, end: time | None) -> bool: + """Return True if `now_t` lies in the [start, end] window. + + Either bound being None means "no constraint on that side", which + matches the dashboard's "leave blank = always active" semantics. + Handles overnight windows (start > end) by treating them as the + complement that crosses midnight. + """ + if start is None and end is None: + return True + if start is None: + return now_t <= end # type: ignore[operator] + if end is None: + return now_t >= start + if start <= end: + return start <= now_t <= end + # crosses midnight + return now_t >= start or now_t <= end + + +__all__ = [ + "get_settings", + "update_settings", + "parse_time_or_none", + "parse_int_or_none", + "in_active_window", +] From c92a40bce155bfc569b2ba5dea6d66ceecd4b2ac Mon Sep 17 00:00:00 2001 From: Travis Herbranson Date: Mon, 25 May 2026 08:15:29 -0400 Subject: [PATCH 08/13] web: dashboard + settings editor (HTMX save) + pipeline observability MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds three new routes on the FastAPI web app: - GET /dashboard — pipeline counts by status (per-status drill-down via filtered /?status=… links), recent activity (latest 10 sources), and a settings snapshot card with an "edit →" shortcut. - GET /settings — full editing form for pipeline_settings. - POST /settings/save — HTMX endpoint that validates the partial form, upserts the row, and returns the rerendered card with a "Saved." or error flash. The card is its own _settings_card.html partial so the HTMX swap targets only the form region. Form parsing lives in settings_store helpers (parse_time_or_none, parse_int_or_none) — keeps the route thin and the empty-string-to-NULL normalisation in one place. Validation rejects negative counts and sub-1 GPU concurrency. The base template grew a Dashboard + Settings nav so the new routes are reachable without typing URLs. Added python-multipart to deps because FastAPI's Form(...) raises at app import without it. --- pyproject.toml | 1 + src/second_brain/web/app.py | 182 +++++++++++++++++- .../web/templates/_settings_card.html | 130 +++++++++++++ src/second_brain/web/templates/base.html | 2 + src/second_brain/web/templates/dashboard.html | 158 +++++++++++++++ src/second_brain/web/templates/settings.html | 14 ++ uv.lock | 11 ++ 7 files changed, 497 insertions(+), 1 deletion(-) create mode 100644 src/second_brain/web/templates/_settings_card.html create mode 100644 src/second_brain/web/templates/dashboard.html create mode 100644 src/second_brain/web/templates/settings.html diff --git a/pyproject.toml b/pyproject.toml index 2603013..d8dd7a5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -15,6 +15,7 @@ dependencies = [ "psycopg-pool>=3.2", "pydantic>=2.0.0", "python-dotenv>=1.0.0", + "python-multipart>=0.0.20", "sqlalchemy>=2.0.0", "tomli>=2.0.0", "trafilatura>=1.9.0", diff --git a/src/second_brain/web/app.py b/src/second_brain/web/app.py index 4ad1c21..59f8304 100644 --- a/src/second_brain/web/app.py +++ b/src/second_brain/web/app.py @@ -14,14 +14,21 @@ from __future__ import annotations from pathlib import Path from typing import List, Optional -from fastapi import FastAPI, HTTPException, Request +from fastapi import FastAPI, Form, HTTPException, Request from fastapi.responses import HTMLResponse from fastapi.templating import Jinja2Templates from pydantic import BaseModel +from sqlalchemy import func from second_brain.config import DOMAINS from second_brain.database import get_database from second_brain.models import Extraction, Source, SourceStatus +from second_brain.settings_store import ( + get_settings, + parse_int_or_none, + parse_time_or_none, + update_settings, +) # --------------------------------------------------------------------------- # App setup @@ -295,3 +302,176 @@ def _source_to_response(s: Source) -> SourceResponse: ingested_at=s.ingested_at.isoformat() if s.ingested_at else "", has_extraction=s.extraction is not None, ) + + +# --------------------------------------------------------------------------- +# Dashboard — pipeline observability +# --------------------------------------------------------------------------- + + +@app.get("/dashboard", response_class=HTMLResponse) +async def dashboard(request: Request): + """Overview: status counts, recent activity, and a settings snapshot.""" + db = get_database() + with db.session() as sess: + # Counts by status. Materialise inside the session so the keys stay + # plain str → int and templates don't try to touch a closed session. + rows = ( + sess.query(Source.status, func.count(Source.id)) + .group_by(Source.status) + .all() + ) + counts = {s.value: 0 for s in SourceStatus} + for status, n in rows: + counts[status.value] = int(n) + total = sum(counts.values()) + + recent_rows = ( + sess.query(Source) + .order_by(Source.ingested_at.desc()) + .limit(10) + .all() + ) + recent = [_source_to_dict(s) for s in recent_rows] + + settings_row = get_settings(sess) + settings_summary = _settings_to_dict(settings_row) + + return templates.TemplateResponse( + request, + "dashboard.html", + { + "counts": counts, + "total": total, + "recent": recent, + "statuses_in_order": [s.value for s in SourceStatus], + "settings": settings_summary, + }, + ) + + +# --------------------------------------------------------------------------- +# Settings — view + HTMX save +# --------------------------------------------------------------------------- + + +@app.get("/settings", response_class=HTMLResponse) +async def settings_view(request: Request): + db = get_database() + with db.session() as sess: + row = get_settings(sess) + ctx = _settings_to_dict(row) + + return templates.TemplateResponse( + request, + "settings.html", + {"settings": ctx, "flash": None, "error": None}, + ) + + +@app.post("/settings/save", response_class=HTMLResponse) +async def settings_save( + request: Request, + transcription_enabled: Optional[str] = Form(None), + transcription_active_hours_start: Optional[str] = Form(None), + transcription_active_hours_end: Optional[str] = Form(None), + transcription_max_concurrent_gpu_jobs: Optional[str] = Form(None), + transcription_max_video_length_seconds: Optional[str] = Form(None), + transcription_max_items_per_run: Optional[str] = Form(None), + extraction_enabled: Optional[str] = Form(None), + extraction_active_hours_start: Optional[str] = Form(None), + extraction_active_hours_end: Optional[str] = Form(None), + extraction_max_items_per_run: Optional[str] = Form(None), +): + """HTMX save endpoint — returns the rendered settings card with a flash.""" + db = get_database() + error: Optional[str] = None + flash: Optional[str] = None + + try: + fields = { + # Checkboxes only show up in the form payload when checked. The + # template emits a hidden "_present" companion so we can tell + # "unchecked" apart from "field missing because of a partial post". + "transcription_enabled": transcription_enabled == "on", + "transcription_active_hours_start": parse_time_or_none( + transcription_active_hours_start + ), + "transcription_active_hours_end": parse_time_or_none( + transcription_active_hours_end + ), + "transcription_max_concurrent_gpu_jobs": int( + transcription_max_concurrent_gpu_jobs or "1" + ), + "transcription_max_video_length_seconds": parse_int_or_none( + transcription_max_video_length_seconds + ), + "transcription_max_items_per_run": int( + transcription_max_items_per_run or "10" + ), + "extraction_enabled": extraction_enabled == "on", + "extraction_active_hours_start": parse_time_or_none( + extraction_active_hours_start + ), + "extraction_active_hours_end": parse_time_or_none( + extraction_active_hours_end + ), + "extraction_max_items_per_run": int(extraction_max_items_per_run or "20"), + } + # Cheap inline validation — keep counts non-negative and gpu jobs >= 1. + if fields["transcription_max_concurrent_gpu_jobs"] < 1: + raise ValueError("transcription_max_concurrent_gpu_jobs must be ≥ 1") + if fields["transcription_max_items_per_run"] < 0: + raise ValueError("transcription_max_items_per_run must be ≥ 0") + if fields["extraction_max_items_per_run"] < 0: + raise ValueError("extraction_max_items_per_run must be ≥ 0") + if ( + fields["transcription_max_video_length_seconds"] is not None + and fields["transcription_max_video_length_seconds"] < 0 + ): + raise ValueError("transcription_max_video_length_seconds must be ≥ 0") + + with db.session() as sess: + row = update_settings(sess, **fields) + ctx = _settings_to_dict(row) + flash = "Saved." + except (ValueError, KeyError) as exc: + error = str(exc) + # Re-render the form using whatever the user submitted so they don't + # lose in-progress edits on a validation bounce. + with db.session() as sess: + ctx = _settings_to_dict(get_settings(sess)) + + return templates.TemplateResponse( + request, + "_settings_card.html", + {"settings": ctx, "flash": flash, "error": error}, + ) + + +# --------------------------------------------------------------------------- +# Settings serialiser +# --------------------------------------------------------------------------- + + +def _settings_to_dict(row) -> dict: + """Render the settings row as a template-friendly dict (no ORM bleed).""" + + def t(v): + return v.strftime("%H:%M") if v is not None else "" + + return { + "transcription_enabled": row.transcription_enabled, + "transcription_active_hours_start": t(row.transcription_active_hours_start), + "transcription_active_hours_end": t(row.transcription_active_hours_end), + "transcription_max_concurrent_gpu_jobs": row.transcription_max_concurrent_gpu_jobs, + "transcription_max_video_length_seconds": row.transcription_max_video_length_seconds, + "transcription_max_items_per_run": row.transcription_max_items_per_run, + "extraction_enabled": row.extraction_enabled, + "extraction_active_hours_start": t(row.extraction_active_hours_start), + "extraction_active_hours_end": t(row.extraction_active_hours_end), + "extraction_max_items_per_run": row.extraction_max_items_per_run, + "updated_at": row.updated_at.strftime("%Y-%m-%d %H:%M:%S") + if row.updated_at + else "", + } diff --git a/src/second_brain/web/templates/_settings_card.html b/src/second_brain/web/templates/_settings_card.html new file mode 100644 index 0000000..7fc709c --- /dev/null +++ b/src/second_brain/web/templates/_settings_card.html @@ -0,0 +1,130 @@ +{# HTMX-replaceable inner card. Posted to /settings/save; swaps itself in place. #} +
+ {% if flash %} +
+ ✓ {{ flash }} +
+ {% endif %} + {% if error %} +
+ ✗ {{ error }} +
+ {% endif %} + +
+ + +
+
+

Transcription

+

consumed by tower-side worker (not yet built)

+
+ +
+ + + +
+ +
+ + +

blank = always active

+
+
+ + +
+ +
+ + +
+
+ + +
+ +
+ + +
+
+
+ + +
+
+

Extraction

+

consumed by the dev-side scheduler

+
+ +
+ + +
+ +
+ + +

blank = always active

+
+
+ + +
+ +
+ + +
+
+
+ +
+

last saved {{ settings.updated_at }}

+ +
+
+
diff --git a/src/second_brain/web/templates/base.html b/src/second_brain/web/templates/base.html index af340d3..10a9d1d 100644 --- a/src/second_brain/web/templates/base.html +++ b/src/second_brain/web/templates/base.html @@ -16,8 +16,10 @@

personal knowledge pipeline

diff --git a/src/second_brain/web/templates/dashboard.html b/src/second_brain/web/templates/dashboard.html new file mode 100644 index 0000000..4e92937 --- /dev/null +++ b/src/second_brain/web/templates/dashboard.html @@ -0,0 +1,158 @@ +{% extends "base.html" %} + +{% block title %}Dashboard — second-brain{% endblock %} + +{% block content %} +{% set status_colors = { + 'pending': 'bg-gray-100 text-gray-700', + 'pulled': 'bg-blue-100 text-blue-700', + 'transcribed': 'bg-yellow-100 text-yellow-700', + 'analyzed': 'bg-orange-100 text-orange-700', + 'accepted': 'bg-green-100 text-green-700', + 'published': 'bg-purple-100 text-purple-700', + 'failed': 'bg-red-100 text-red-700' +} %} +{% set domain_colors = { + 'development': 'bg-cyan-50 text-cyan-700 border-cyan-200', + 'content': 'bg-pink-50 text-pink-700 border-pink-200', + 'business': 'bg-amber-50 text-amber-700 border-amber-200', + 'homelab': 'bg-teal-50 text-teal-700 border-teal-200' +} %} + +
+

Dashboard

+ total sources: {{ total }} +
+ + +
+

+ Pipeline +

+
+ {% for status in statuses_in_order %} + +
{{ counts.get(status, 0) }}
+
+ + {{ status }} + +
+ + {% endfor %} +
+
+ + +
+ + +
+
+

+ Settings +

+ + edit → + +
+ +
+
+
Transcription
+
+ {% if settings.transcription_enabled %} + enabled + {% else %} + paused + {% endif %} +
+
+
+
active window
+
+ {% if settings.transcription_active_hours_start or settings.transcription_active_hours_end %} + {{ settings.transcription_active_hours_start or "—" }} – {{ settings.transcription_active_hours_end or "—" }} + {% else %} + always + {% endif %} +
+
+
+
GPU jobs
+
{{ settings.transcription_max_concurrent_gpu_jobs }}
+
+
+
max items/run
+
{{ settings.transcription_max_items_per_run }}
+
+ +
+ +
+
Extraction
+
+ {% if settings.extraction_enabled %} + enabled + {% else %} + paused + {% endif %} +
+
+
+
active window
+
+ {% if settings.extraction_active_hours_start or settings.extraction_active_hours_end %} + {{ settings.extraction_active_hours_start or "—" }} – {{ settings.extraction_active_hours_end or "—" }} + {% else %} + always + {% endif %} +
+
+
+
max items/run
+
{{ settings.extraction_max_items_per_run }}
+
+
+ +

+ updated {{ settings.updated_at }} +

+
+ + +
+

+ Recent activity +

+ {% if recent %} +
+ {% for source in recent %} + +
+

{{ source.title }}

+

{{ source.url }}

+
+
+ + {{ source.status }} + + + {{ source.domain }} + + {{ source.ingested_at }} +
+ + {% endfor %} +
+ {% else %} +

No sources yet.

+ {% endif %} +
+
+{% endblock %} diff --git a/src/second_brain/web/templates/settings.html b/src/second_brain/web/templates/settings.html new file mode 100644 index 0000000..80cb917 --- /dev/null +++ b/src/second_brain/web/templates/settings.html @@ -0,0 +1,14 @@ +{% extends "base.html" %} + +{% block title %}Settings — second-brain{% endblock %} + +{% block content %} +
+

Settings

+

+ Workers read these at the start of each run. DB values override settings.toml. +

+
+ +{% include "_settings_card.html" %} +{% endblock %} diff --git a/uv.lock b/uv.lock index b99bf72..141b482 100644 --- a/uv.lock +++ b/uv.lock @@ -1385,6 +1385,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/0b/d7/1959b9648791274998a9c3526f6d0ec8fd2233e4d4acce81bbae76b44b2a/python_dotenv-1.2.2-py3-none-any.whl", hash = "sha256:1d8214789a24de455a8b8bd8ae6fe3c6b69a5e3d64aa8a8e5d68e694bbcb285a", size = 22101, upload-time = "2026-03-01T16:00:25.09Z" }, ] +[[package]] +name = "python-multipart" +version = "0.0.29" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/4e/fe/70bd71a6738b09a0bdf6480ca6436b167469ca4578b2a0efbe390b4b0e70/python_multipart-0.0.29.tar.gz", hash = "sha256:643e93849196645e2dbdd81a0f8829a23123ad7f797a84a364c6fb3563f18904", size = 45678, upload-time = "2026-05-17T17:29:47.654Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/8f/cb/769cfc37177252872a45a71f3fbdde9d51b471a3f3c14bfe95dde3407386/python_multipart-0.0.29-py3-none-any.whl", hash = "sha256:2ddcc971cef266225f54f552d8fa10bcfbb1f14446caec199060daac59ff2d69", size = 29640, upload-time = "2026-05-17T17:29:45.69Z" }, +] + [[package]] name = "pytz" version = "2026.2" @@ -1584,6 +1593,7 @@ dependencies = [ { name = "psycopg-pool" }, { name = "pydantic" }, { name = "python-dotenv" }, + { name = "python-multipart" }, { name = "sqlalchemy" }, { name = "tomli" }, { name = "trafilatura" }, @@ -1617,6 +1627,7 @@ requires-dist = [ { name = "psycopg-pool", specifier = ">=3.2" }, { name = "pydantic", specifier = ">=2.0.0" }, { name = "python-dotenv", specifier = ">=1.0.0" }, + { name = "python-multipart", specifier = ">=0.0.20" }, { name = "sqlalchemy", specifier = ">=2.0.0" }, { name = "tomli", specifier = ">=2.0.0" }, { name = "trafilatura", specifier = ">=1.9.0" }, From 7eafdf3e03065dea38b0ba46e38b9b1343e59dff Mon Sep 17 00:00:00 2001 From: Travis Herbranson Date: Mon, 25 May 2026 08:17:44 -0400 Subject: [PATCH 09/13] scheduler: honor pipeline_settings (extraction_enabled, window, max_items) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The dev-side scheduler now snapshots the singleton pipeline_settings row at the top of run() and applies three DB-driven gates: - extraction_enabled = false → skip the entire run with a single log line. Lets Travis pause extraction from the dashboard without touching the config file or restarting anything. - extraction_active_hours_start/_end → fast-fail if outside the window AND (when set) override the static settings.toml [scheduler].window_* bounds so the inner loop also respects the dashboard's choice. Either side being NULL means "no constraint on that side" — matches the form's "leave blank = always active" semantics. - extraction_max_items_per_run → hard cap on processed items per invocation. 0/null means unlimited (existing behavior). Snapshot semantics are deliberate: a mid-run toggle doesn't half-apply, same way max_calls_per_hour is tracked locally. Workers re-read on the next invocation. --- src/second_brain/scheduler/runner.py | 43 +++++++++++++++++++++++++++- 1 file changed, 42 insertions(+), 1 deletion(-) diff --git a/src/second_brain/scheduler/runner.py b/src/second_brain/scheduler/runner.py index c3fc205..988da5f 100644 --- a/src/second_brain/scheduler/runner.py +++ b/src/second_brain/scheduler/runner.py @@ -25,6 +25,7 @@ from datetime import datetime, time as dtime from second_brain.config import Config from second_brain.database import get_database from second_brain.models import Source, SourceStatus, utcnow +from second_brain.settings_store import get_settings, in_active_window class Scheduler: @@ -51,6 +52,38 @@ class Scheduler: from second_brain.models import Extraction db = get_database() + + # Pull the DB-side runtime knobs once at the top of the run. The + # dashboard edits these; settings.toml is the static fallback. The + # values are snapshotted for the duration of this run so a mid-run + # toggle doesn't half-apply (consistent with the way max_calls is + # tracked locally). + with db.session() as sess: + settings = get_settings(sess) + extraction_enabled = settings.extraction_enabled + db_window = ( + settings.extraction_active_hours_start, + settings.extraction_active_hours_end, + ) + max_items = int(settings.extraction_max_items_per_run or 0) + + if not extraction_enabled: + print("[Scheduler] extraction_enabled=false in pipeline_settings. Skipping run.") + return + + # DB window (if set) overrides the static settings.toml window. If + # only one side is set, treat the other as unbounded (no constraint). + if db_window[0] is not None or db_window[1] is not None: + self.window_start = db_window[0] or dtime(0, 0) + self.window_end = db_window[1] or dtime(23, 59) + + if not in_active_window(datetime.now().time(), db_window[0], db_window[1]): + print( + f"[Scheduler] outside extraction active window " + f"{_fmt_opt(db_window[0])}–{_fmt_opt(db_window[1])}. Skipping run." + ) + return + assembler = ContextAssembler(self.config) ex_cfg = self.config.extractor engine = ExtractionEngine( @@ -74,10 +107,14 @@ class Scheduler: print( f"[Scheduler] backend={self.backend} window " f"{_fmt(self.window_start)}–{_fmt(self.window_end)} cap {cap_desc} " - f"gap {self.min_gap_seconds}s" + f"gap {self.min_gap_seconds}s max_items={max_items or '∞'}" ) while self._in_window(): + # DB-side hard cap on items per run (0 = unlimited). + if max_items and processed >= max_items: + print(f"[Scheduler] reached extraction_max_items_per_run={max_items}. Done.") + break # Reset hourly buckets elapsed = time.time() - hour_start if elapsed >= 3600: @@ -191,4 +228,8 @@ def _fmt(t: dtime) -> str: return t.strftime("%H:%M") +def _fmt_opt(t: dtime | None) -> str: + return t.strftime("%H:%M") if t is not None else "—" + + __all__ = ["Scheduler"] From 33ae5a6f7dda09d686ba3fac671488f6cdfdc214 Mon Sep 17 00:00:00 2001 From: Travis Herbranson Date: Mon, 25 May 2026 10:11:37 -0400 Subject: [PATCH 10/13] pipeline split: tower transcribe worker + claim queue + faster-whisper MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Splits pull+transcribe (now tower-side, eager) from extract+embed (stays on the dev scheduler). Three machine-coordination pieces land together because they reference each other: - v3 migration adds sources.claimed_by + claimed_at — observability + stale-claim recovery columns. The actual race-safety primitive is `SELECT ... FOR UPDATE SKIP LOCKED` in the new claim helper, so two machines can poll the queue without doubling work. - src/second_brain/claim.py owns the claim dance (claim_next_source, release_claim, reap_stale_claims). Both stage gates filter by source_type so the tower never grabs articles and the dev side never grabs videos. - src/second_brain/transcribe.py wraps faster-whisper (lazy-imported so it stays out of the dev install). resolve_settings() reads env > [whisper] block > defaults, falling back to int8 on cpu / float16 on cuda when compute_type is unspecified. Default model large-v3. - src/second_brain/scheduler/transcribe_worker.py is the long-running poll loop. Reads pipeline_settings every iteration so the dashboard's enable/window/max-items/max-video-length take effect within one cycle. Reaps stale claims at startup. SIGTERM-clean. DB-unreachable backs off with a log line; never crash-loops. - adapters/youtube.py drops the torch-whisper transcribe path; pull stays. Removes openai-whisper from the default deps and gates faster-whisper behind a new `tower` extra (uv sync --extra tower). - main.py: new `second-brain transcribe-worker` (--once for ad-hoc). `process` now article-only on the pull side but still picks up TRANSCRIBED of any source_type for the extract step. Live-verified: migration applies clean, transcribe-worker --once honors transcription_enabled=false gate. --- .../c8f1d9e34b7a_v3_source_claim_columns.py | 50 ++ pyproject.toml | 6 +- src/second_brain/adapters/youtube.py | 109 +--- src/second_brain/claim.py | 154 +++++ src/second_brain/config.py | 25 +- src/second_brain/main.py | 121 +++- src/second_brain/models.py | 4 + .../scheduler/transcribe_worker.py | 312 +++++++++ src/second_brain/transcribe.py | 171 +++++ uv.lock | 611 +++++++----------- 10 files changed, 1064 insertions(+), 499 deletions(-) create mode 100644 alembic/versions/c8f1d9e34b7a_v3_source_claim_columns.py create mode 100644 src/second_brain/claim.py create mode 100644 src/second_brain/scheduler/transcribe_worker.py create mode 100644 src/second_brain/transcribe.py diff --git a/alembic/versions/c8f1d9e34b7a_v3_source_claim_columns.py b/alembic/versions/c8f1d9e34b7a_v3_source_claim_columns.py new file mode 100644 index 0000000..32d5434 --- /dev/null +++ b/alembic/versions/c8f1d9e34b7a_v3_source_claim_columns.py @@ -0,0 +1,50 @@ +"""v3 second_brain: sources.claimed_by / claimed_at + +Revision ID: c8f1d9e34b7a +Revises: 7b3e8a5c1f29 +Create Date: 2026-05-25 12:30:00.000000 + +Adds a lightweight claim mechanism so the tower and dev sides never grab +the same row off the queue. The claim flow used by both workers is: + + BEGIN; + SELECT id, status, source_type + FROM second_brain.sources + WHERE + AND claimed_by IS NULL + ORDER BY ingested_at ASC + LIMIT 1 + FOR UPDATE SKIP LOCKED; + UPDATE second_brain.sources + SET claimed_by = :worker, claimed_at = now() AT TIME ZONE 'UTC' + WHERE id = :id; + COMMIT; + +`FOR UPDATE SKIP LOCKED` is the actual race-safety primitive — the column +pair is observability (who has it, since when, for stale-claim recovery). +A partial index would help once we have a real queue depth; deliberately +not added now per Travis's "no gold-plating". +""" + +from __future__ import annotations + +from collections.abc import Sequence + +from alembic import op + +revision: str = "c8f1d9e34b7a" +down_revision: str | Sequence[str] | None = "7b3e8a5c1f29" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + + +def upgrade() -> None: + op.execute("SET search_path TO second_brain") + op.execute("ALTER TABLE sources ADD COLUMN claimed_by VARCHAR(100)") + op.execute("ALTER TABLE sources ADD COLUMN claimed_at TIMESTAMP") + + +def downgrade() -> None: + op.execute("SET search_path TO second_brain") + op.execute("ALTER TABLE sources DROP COLUMN IF EXISTS claimed_at") + op.execute("ALTER TABLE sources DROP COLUMN IF EXISTS claimed_by") diff --git a/pyproject.toml b/pyproject.toml index d8dd7a5..6a94e96 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,7 +10,6 @@ dependencies = [ "fastapi>=0.115.0", "httpx>=0.28.0", "jinja2>=3.1.0", - "openai-whisper", "psycopg[binary]>=3.2", "psycopg-pool>=3.2", "pydantic>=2.0.0", @@ -29,6 +28,11 @@ dependencies = [ # Max OAuth subscription and has no Python-side Anthropic dependency. api = ["anthropic>=0.40.0"] +# Tower-only: faster-whisper (CTranslate2) for the transcribe worker. +# Pulls in CUDA wheels — install with `uv sync --extra tower` on the +# tower box, leave it off on dev. CPU/int8 path works on this same wheel. +tower = ["faster-whisper>=1.0.0"] + [dependency-groups] dev = [ "pytest>=8.0", diff --git a/src/second_brain/adapters/youtube.py b/src/second_brain/adapters/youtube.py index 5dfdc7a..8866b24 100644 --- a/src/second_brain/adapters/youtube.py +++ b/src/second_brain/adapters/youtube.py @@ -1,8 +1,12 @@ """ -YouTube pull + Whisper transcribe adapter. +YouTube pull adapter (yt-dlp only). -Ported from xtract/main.py (VideoDownloader + SubtitleExtractor classes). -Adapted to work with the second-brain Source model and Config. +Transcription used to live here on a torch-whisper backend. It moved to +`second_brain.transcribe` (faster-whisper) once the pipeline split so the +tower owns transcription and the dev side never imports the heavy model. + +This module is now safe to import from any process — yt-dlp is small, and +no GPU / whisper dependency is loaded at import time. """ from __future__ import annotations @@ -11,7 +15,6 @@ from datetime import datetime from pathlib import Path from typing import Optional -import whisper import yt_dlp from second_brain.config import Config @@ -84,84 +87,18 @@ class VideoDownloader: } -class SubtitleExtractor: - """Transcribe a video file to an SRT subtitle file using OpenAI Whisper. - - Ported from xtract SubtitleExtractor — identical core logic, - writes SRTs to config.subtitles_dir. - """ - - def __init__(self, config: Config) -> None: - self.config = config - self._model = None - - def _load_model(self) -> None: - if self._model is None: - print(f"[Whisper] Loading {self.config.whisper_model} model…") - self._model = whisper.load_model(self.config.whisper_model) - print("[Whisper] Model ready") - - def extract(self, media_file: Path) -> Optional[Path]: - """Transcribe *media_file* and return the path to the SRT file.""" - self.config.subtitles_dir.mkdir(parents=True, exist_ok=True) - srt_file = self.config.subtitles_dir / f"{media_file.stem}.srt" - - if srt_file.exists(): - print(f"[Whisper] Already transcribed: {srt_file.name}") - return srt_file - - self._load_model() - print(f"[Whisper] Transcribing: {media_file.name}") - - try: - result = self._model.transcribe( - str(media_file), - verbose=False, - language=None, - ) - self._write_srt(result, srt_file) - print(f"[Whisper] Saved: {srt_file.name}") - return srt_file - except Exception as exc: - print(f"[Whisper] Error transcribing {media_file.name}: {exc}") - return None - - def extract_text(self, media_file: Path) -> Optional[str]: - """Return the plain-text transcript (no SRT formatting).""" - self._load_model() - try: - result = self._model.transcribe(str(media_file), verbose=False) - return " ".join(seg["text"].strip() for seg in result["segments"]) - except Exception as exc: - print(f"[Whisper] Error: {exc}") - return None - - @staticmethod - def _write_srt(result: dict, output_file: Path) -> None: - with open(output_file, "w", encoding="utf-8") as fh: - for i, seg in enumerate(result["segments"], start=1): - start = _fmt_timestamp(seg["start"]) - end = _fmt_timestamp(seg["end"]) - fh.write(f"{i}\n{start} --> {end}\n{seg['text'].strip()}\n\n") - - @staticmethod - def read_srt(srt_file: Path) -> str: - """Read an SRT file and return its raw text content.""" - return srt_file.read_text(encoding="utf-8") - - # --------------------------------------------------------------------------- # High-level adapter # --------------------------------------------------------------------------- class YouTubeAdapter: - """Orchestrates pull + transcribe for a YouTube Source record.""" + """yt-dlp pull side of a video Source. Transcription is handled by + `second_brain.transcribe.Transcriber` on the tower.""" def __init__(self, config: Config) -> None: self.config = config self.downloader = VideoDownloader(config) - self.transcriber = SubtitleExtractor(config) def pull(self, source: Source) -> bool: """Download the video. Updates source in-place, returns success.""" @@ -186,38 +123,12 @@ class YouTubeAdapter: source.status = SourceStatus.FAILED return False - def transcribe(self, source: Source) -> bool: - """Transcribe the downloaded video. Updates source in-place.""" - if not source.media_path: - source.error_message = "No media_path; run pull first" - return False - - media_file = Path(source.media_path) - srt_file = self.transcriber.extract(media_file) - if srt_file is None: - source.error_message = "Whisper transcription failed" - source.status = SourceStatus.FAILED - return False - - source.transcript_path = str(srt_file) - source.transcript_text = SubtitleExtractor.read_srt(srt_file) - source.status = SourceStatus.TRANSCRIBED - return True - # --------------------------------------------------------------------------- # Helpers # --------------------------------------------------------------------------- -def _fmt_timestamp(seconds: float) -> str: - h = int(seconds // 3600) - m = int((seconds % 3600) // 60) - s = int(seconds % 60) - ms = int((seconds % 1) * 1000) - return f"{h:02d}:{m:02d}:{s:02d},{ms:03d}" - - def _parse_upload_date(upload_date: Optional[str]) -> Optional[datetime]: """Parse yt-dlp's YYYYMMDD upload_date string.""" if not upload_date or len(upload_date) != 8: @@ -228,4 +139,4 @@ def _parse_upload_date(upload_date: Optional[str]) -> Optional[datetime]: return None -__all__ = ["VideoDownloader", "SubtitleExtractor", "YouTubeAdapter"] +__all__ = ["VideoDownloader", "YouTubeAdapter"] diff --git a/src/second_brain/claim.py b/src/second_brain/claim.py new file mode 100644 index 0000000..6a791d3 --- /dev/null +++ b/src/second_brain/claim.py @@ -0,0 +1,154 @@ +"""Queue-claim helpers shared by the tower transcribe worker, the dev-side +process command, and the dev-side extract scheduler. + +The race-safety primitive is `SELECT ... FOR UPDATE SKIP LOCKED`. The +`claimed_by` / `claimed_at` columns are observability + stale-claim +recovery only — Postgres row-locks are what actually keep two machines +from grabbing the same row. + +Usage pattern (worker side, simplified):: + + with db.session() as sess: + source_id = claim_next_source( + sess, + worker_id="tower", + statuses=[SourceStatus.PENDING, SourceStatus.PULLED], + source_type=SourceType.VIDEO, + ) + # commit drops the row lock — the claimed_by/at columns persist. + + if source_id is None: + # nothing to do; sleep and retry + return + + with db.session() as sess: + source = sess.get(Source, source_id) + try: + ...do work, mutate status... + finally: + release_claim(sess, source_id) + +The claim is *released* once the row reaches a terminal status for this +stage (TRANSCRIBED, FAILED, etc.) so a follower stage can pick it up. If +a worker dies mid-claim, `reap_stale_claims` (called at worker startup) +clears claims older than the configured grace window. +""" + +from __future__ import annotations + +from datetime import timedelta +from typing import Iterable, Optional + +from sqlalchemy import text +from sqlalchemy.orm import Session + +from second_brain.models import Source, SourceStatus, SourceType, utcnow + + +def claim_next_source( + sess: Session, + *, + worker_id: str, + statuses: Iterable[SourceStatus], + source_type: Optional[SourceType] = None, +) -> Optional[int]: + """Atomically claim the oldest unclaimed source matching the filters. + + Returns the source's id (so the caller can re-fetch it in a fresh + session if it likes), or None if no row matched. The caller's + session-level COMMIT drops the row lock — the claim survives via the + `claimed_by` / `claimed_at` columns. + """ + if not statuses: + return None + + status_values = [s.value for s in statuses] + params: dict = { + "statuses": status_values, + "worker": worker_id, + "now": utcnow(), + } + type_clause = "" + if source_type is not None: + type_clause = "AND source_type = :source_type" + params["source_type"] = source_type.value + + # Raw SQL because SQLAlchemy ORM doesn't expose SKIP LOCKED cleanly + # across all minor versions and we want the exact lock semantics + # documented in the module header. `status::text` lets us bind a + # parameter list against the native enum cleanly. + pick = sess.execute( + text( + f""" + SELECT id + FROM second_brain.sources + WHERE status::text = ANY(:statuses) + {type_clause} + AND claimed_by IS NULL + ORDER BY ingested_at ASC + LIMIT 1 + FOR UPDATE SKIP LOCKED + """ + ), + params, + ).first() + + if pick is None: + return None + + source_id = int(pick[0]) + sess.execute( + text( + """ + UPDATE second_brain.sources + SET claimed_by = :worker, + claimed_at = :now + WHERE id = :id + """ + ), + {"worker": worker_id, "now": params["now"], "id": source_id}, + ) + return source_id + + +def release_claim(sess: Session, source_id: int) -> None: + """Drop the claim on a source so a follower stage can pick it up. + + The work-state columns (`status`, `transcript_text`, …) are the + caller's responsibility — this just nulls the claim pair. + """ + src = sess.get(Source, source_id) + if src is None: + return + src.claimed_by = None + src.claimed_at = None + + +def reap_stale_claims( + sess: Session, + *, + older_than: timedelta = timedelta(minutes=30), +) -> int: + """Clear claims older than the grace window — recovery for crashed workers. + + Returns the number of rows reaped. Safe to call at worker startup; + pairs with the worker setting a short poll interval so a stuck + transcribe doesn't dam the queue forever. + """ + cutoff = utcnow() - older_than + result = sess.execute( + text( + """ + UPDATE second_brain.sources + SET claimed_by = NULL, + claimed_at = NULL + WHERE claimed_by IS NOT NULL + AND claimed_at < :cutoff + """ + ), + {"cutoff": cutoff}, + ) + return result.rowcount or 0 + + +__all__ = ["claim_next_source", "release_claim", "reap_stale_claims"] diff --git a/src/second_brain/config.py b/src/second_brain/config.py index 242f3c0..1298515 100644 --- a/src/second_brain/config.py +++ b/src/second_brain/config.py @@ -29,8 +29,14 @@ _DEFAULTS: dict[str, Any] = { "vault_path": "~/Documents/second-brain-vault", "media_dir": "~/.local/share/second-brain/media", "subtitles_dir": "~/.local/share/second-brain/subtitles", - "whisper_model": "small", "domains": ["development", "content", "business", "homelab"], + "whisper": { + # Tower default: faster-whisper large-v3 on cuda/float16. The + # transcribe helper picks int8 automatically if the device is cpu. + "whisper_model": "large-v3", + "whisper_device": "cuda", + "whisper_compute_type": "", + }, "database": { # Empty by default — the runtime expects SECOND_BRAIN_DATABASE_URL # (or HERBYLAB_DATABASE_URL) in the environment. Fill this only if @@ -101,11 +107,24 @@ class Config: self._raw.get("subtitles_dir", _DEFAULTS["subtitles_dir"]) ).expanduser() - # --- extraction --- + # --- transcription --- + + @property + def whisper(self) -> dict[str, Any]: + """Merged [whisper] block. Env vars in `transcribe.resolve_settings` + still win over these for the tower worker.""" + return _merge(_DEFAULTS["whisper"], self._raw.get("whisper", {})) @property def whisper_model(self) -> str: - return self._raw.get("whisper_model", _DEFAULTS["whisper_model"]) + """Backwards-compat shortcut — older callers (and settings.toml + in the wild) reference whisper_model at the top level.""" + return ( + self._raw.get("whisper_model") + or self.whisper.get("whisper_model", "large-v3") + ) + + # --- extraction --- @property def domains(self) -> list[str]: diff --git a/src/second_brain/main.py b/src/second_brain/main.py index 44a360b..2e96bee 100644 --- a/src/second_brain/main.py +++ b/src/second_brain/main.py @@ -104,8 +104,13 @@ def process( skip_transcribe: bool, skip_extract: bool, ) -> None: - """Run the pipeline on pending sources.""" - from second_brain.adapters.youtube import YouTubeAdapter + """Run the dev-side pipeline: article pulls + extract/embed. + + Video pull + transcribe have moved to the tower-side + `transcribe-worker` daemon (faster-whisper on the 3080); this command + is article-only on the pull side, but still picks up TRANSCRIBED rows + of any source_type for the extract/embed step. + """ from second_brain.adapters.article import ArticleAdapter from second_brain.context.assembler import ContextAssembler from second_brain.extractor.engine import ExtractionEngine @@ -115,7 +120,6 @@ def process( config.ensure_dirs() db = get_database() - yt_adapter = YouTubeAdapter(config) art_adapter = ArticleAdapter(config) assembler = ContextAssembler(config) @@ -135,52 +139,61 @@ def process( click.echo("[process] Extraction step will be skipped.") skip_extract = True + # Dev side: pull only ARTICLE sources (videos belong to the tower + # worker) — but pick up TRANSCRIBED of any type for extract/embed. with db.session() as sess: - query = sess.query(Source.id).filter( - Source.status.in_([ - SourceStatus.PENDING, - SourceStatus.PULLED, - SourceStatus.TRANSCRIBED, - ]) - ) + pull_ids = [] + if not skip_pull: + pull_ids = [ + row[0] + for row in sess.query(Source.id) + .filter( + Source.status == SourceStatus.PENDING, + Source.source_type == SourceType.ARTICLE, + ) + .all() + ] + extract_ids = [ + row[0] + for row in sess.query(Source.id) + .filter(Source.status == SourceStatus.TRANSCRIBED) + .all() + ] + # Keep order stable (pulls first so freshly-transcribed articles + # roll into extract in the same invocation) and de-duplicate. + source_ids: list[int] = [] + for sid in pull_ids + extract_ids: + if sid not in source_ids: + source_ids.append(sid) if limit: - query = query.limit(limit) - source_ids = [row[0] for row in query.all()] + source_ids = source_ids[:limit] if not source_ids: click.echo("[process] No sources to process.") return click.echo(f"[process] Processing {len(source_ids)} source(s)…") + if skip_transcribe: + click.echo("[process] (--skip-transcribe is now a no-op — transcription " + "runs on the tower worker.)") for source_id in source_ids: with db.session() as sess: source = sess.get(Source, source_id) click.echo(f"\n [{source.id}] {source.url[:80]}") - # Pull step - if not skip_pull and source.status == SourceStatus.PENDING: - click.echo(" → pull") - if source.source_type == SourceType.VIDEO: - ok = yt_adapter.pull(source) - else: - ok = art_adapter.pull(source) + # Pull step — articles only on the dev side. + if ( + not skip_pull + and source.status == SourceStatus.PENDING + and source.source_type == SourceType.ARTICLE + ): + click.echo(" → pull (article)") + ok = art_adapter.pull(source) if not ok: click.echo(f" ✗ pull failed: {source.error_message}") continue - # Transcribe step (videos only — articles come out transcribed) - if ( - not skip_transcribe - and source.status == SourceStatus.PULLED - and source.source_type == SourceType.VIDEO - ): - click.echo(" → transcribe") - ok = yt_adapter.transcribe(source) - if not ok: - click.echo(f" ✗ transcribe failed: {source.error_message}") - continue - # Extract step if not skip_extract and source.status == SourceStatus.TRANSCRIBED and engine: click.echo(" → extract") @@ -289,6 +302,52 @@ def schedule(dry_run: bool) -> None: scheduler.run(dry_run=dry_run) +# --------------------------------------------------------------------------- +# transcribe-worker (tower-side daemon) +# --------------------------------------------------------------------------- + + +@cli.command("transcribe-worker") +@click.option("--once", is_flag=True, default=False, + help="Process at most one source then exit (useful for ad-hoc runs / tests).") +@click.option("--poll-interval", default=None, type=int, + help="Seconds between empty-queue polls (default 15).") +@click.option("--worker-id", default=None, + help="Identifier stamped into sources.claimed_by (default tower:).") +def transcribe_worker( + once: bool, + poll_interval: Optional[int], + worker_id: Optional[str], +) -> None: + """Long-running tower-side worker: claim → yt-dlp pull → faster-whisper. + + Reads pipeline_settings each iteration so the dashboard's enable + toggle / active-hours window / max-items-per-run / max-video-length + all take effect within one poll cycle. + """ + import logging + + from second_brain.scheduler.transcribe_worker import ( + DEFAULT_POLL_INTERVAL_SECONDS, + TranscribeWorker, + ) + + logging.basicConfig( + level=logging.INFO, + format="%(asctime)s %(levelname)s %(name)s: %(message)s", + ) + + config = load_config() + config.ensure_dirs() + worker = TranscribeWorker( + config, + worker_id=worker_id, + poll_interval=poll_interval or DEFAULT_POLL_INTERVAL_SECONDS, + ) + processed = worker.run(once=once) + click.echo(f"[transcribe-worker] exiting; processed={processed}") + + # --------------------------------------------------------------------------- # Helpers # --------------------------------------------------------------------------- diff --git a/src/second_brain/models.py b/src/second_brain/models.py index 2467c7c..daa42b1 100644 --- a/src/second_brain/models.py +++ b/src/second_brain/models.py @@ -124,6 +124,10 @@ class Source(Base): updated_at: Mapped[Optional[datetime]] = mapped_column(DateTime, nullable=True) error_message: Mapped[Optional[str]] = mapped_column(Text, nullable=True) + # Claim mechanism — see `second_brain.claim` for the queue dance. + claimed_by: Mapped[Optional[str]] = mapped_column(String(100), nullable=True) + claimed_at: Mapped[Optional[datetime]] = mapped_column(DateTime, nullable=True) + # Relationships extraction: Mapped[Optional["Extraction"]] = relationship( "Extraction", back_populates="source", uselist=False, cascade="all, delete-orphan" diff --git a/src/second_brain/scheduler/transcribe_worker.py b/src/second_brain/scheduler/transcribe_worker.py new file mode 100644 index 0000000..57a2527 --- /dev/null +++ b/src/second_brain/scheduler/transcribe_worker.py @@ -0,0 +1,312 @@ +"""Tower-side transcribe worker — long-running poll loop. + +Runs on the EndeavourOS tower (RTX 3080) as a systemd service. Reaches +petalbrain over WireGuard at db.wg.herbylab.dev:5432. + +Loop body: + 1. Read pipeline_settings. + - transcription_enabled = false → sleep, retry. + - outside transcription_active_hours → sleep, retry. + - transcription_max_concurrent_gpu_jobs is consulted (default 1 = + this worker is the single in-flight job; future scale-out can fan + a per-job lock here). + 2. Claim the oldest unclaimed VIDEO source in {PENDING, PULLED} via + `SELECT ... FOR UPDATE SKIP LOCKED`. + 3. If PENDING → yt-dlp pull → PULLED (claim retained). + 4. If PULLED → faster-whisper transcribe → TRANSCRIBED (claim released). + 5. Honor transcription_max_video_length_seconds: skip & mark FAILED + with a clear error_message if the metadata's duration exceeds it. + +Graceful degradation: + - DB unreachable → log at WARNING, sleep, retry. Never crash-loop. + - Whisper failure on one file → mark that source FAILED, release claim, + continue with the next one. + - SIGTERM (systemd stop) → finish the current iteration, exit cleanly. +""" + +from __future__ import annotations + +import logging +import os +import signal +import socket +import time +from datetime import datetime +from pathlib import Path + +from second_brain.claim import ( + claim_next_source, + reap_stale_claims, + release_claim, +) +from second_brain.config import Config +from second_brain.database import get_database +from second_brain.models import Source, SourceStatus, SourceType, utcnow +from second_brain.settings_store import get_settings, in_active_window + +logger = logging.getLogger(__name__) + + +# Default loop pace — short so a dropped-in video starts transcribing +# right away. The worker also re-reads pipeline_settings every iteration +# so a dashboard toggle takes effect within ~POLL_INTERVAL_SECONDS. +DEFAULT_POLL_INTERVAL_SECONDS = 15 +DEFAULT_DB_BACKOFF_SECONDS = 30 + + +class TranscribeWorker: + """The tower's transcribe poll loop.""" + + def __init__( + self, + config: Config, + *, + worker_id: str | None = None, + poll_interval: int = DEFAULT_POLL_INTERVAL_SECONDS, + db_backoff: int = DEFAULT_DB_BACKOFF_SECONDS, + ) -> None: + self.config = config + self.worker_id = ( + worker_id + or os.environ.get("SECOND_BRAIN_WORKER_ID") + or f"tower:{socket.gethostname()}" + ) + self.poll_interval = poll_interval + self.db_backoff = db_backoff + self._stop = False + + # SIGTERM/SIGINT → clean shutdown after the current iteration. + signal.signal(signal.SIGTERM, self._on_signal) + signal.signal(signal.SIGINT, self._on_signal) + + def _on_signal(self, signum, frame): # noqa: ARG002 — frame required by signal API + logger.info("worker received signal %s — stopping after current iteration", signum) + self._stop = True + + # ------------------------------------------------------------------ + # Top-level run loop + # ------------------------------------------------------------------ + + def run(self, *, once: bool = False, max_iterations: int | None = None) -> int: + """Drive the loop. Returns the count of sources processed in this run. + + `once=True` runs a single iteration regardless of queue depth — + useful for ad-hoc CLI invocation. `max_iterations` caps loops so + tests don't run forever. + """ + processed = 0 + iterations = 0 + startup_reaped = False + + while not self._stop: + iterations += 1 + if max_iterations is not None and iterations > max_iterations: + break + + try: + db = get_database() + except Exception as exc: + logger.warning("DB not configured: %s — sleeping %ds", exc, self.db_backoff) + if once: + return processed + self._sleep(self.db_backoff) + continue + + try: + with db.session() as sess: + if not startup_reaped: + reaped = reap_stale_claims(sess) + if reaped: + logger.info("reaped %d stale claim(s) on startup", reaped) + startup_reaped = True + + settings = get_settings(sess) + enabled = settings.transcription_enabled + window = ( + settings.transcription_active_hours_start, + settings.transcription_active_hours_end, + ) + max_items = int(settings.transcription_max_items_per_run or 0) + max_video_length = settings.transcription_max_video_length_seconds + except Exception as exc: + logger.warning("DB unreachable: %s — sleeping %ds", exc, self.db_backoff) + if once: + return processed + self._sleep(self.db_backoff) + continue + + if not enabled: + logger.info("transcription_enabled=false — sleeping") + if once: + return processed + self._sleep(self.poll_interval) + continue + + if not in_active_window(datetime.now().time(), window[0], window[1]): + logger.info("outside transcription active window — sleeping") + if once: + return processed + self._sleep(self.poll_interval) + continue + + if max_items and processed >= max_items: + logger.info("reached transcription_max_items_per_run=%d — exiting", max_items) + break + + handled = self._process_one(max_video_length) + if handled is None: + # Nothing to claim — sleep and re-poll. + if once: + return processed + self._sleep(self.poll_interval) + continue + + processed += 1 + if once: + return processed + + logger.info("worker stopping (processed=%d, iterations=%d)", processed, iterations) + return processed + + # ------------------------------------------------------------------ + # Per-source step + # ------------------------------------------------------------------ + + def _process_one(self, max_video_length: int | None) -> int | None: + """Try to claim and advance one video source. Returns the source id + on success, or None if nothing was claimable. + """ + db = get_database() + + # Step 1: claim. Short transaction, FOR UPDATE SKIP LOCKED inside. + with db.session() as sess: + source_id = claim_next_source( + sess, + worker_id=self.worker_id, + statuses=[SourceStatus.PENDING, SourceStatus.PULLED], + source_type=SourceType.VIDEO, + ) + + if source_id is None: + return None + + # Step 2: do the work. Fresh session because we crossed a commit. + try: + with db.session() as sess: + source = sess.get(Source, source_id) + if source is None: + return None + logger.info("[%s] starting %s id=%d url=%s", + self.worker_id, source.status.value, source.id, source.url) + self._advance(source, max_video_length, sess) + except Exception as exc: + logger.exception("unexpected error on source %s: %s", source_id, exc) + with db.session() as sess: + src = sess.get(Source, source_id) + if src is not None: + src.status = SourceStatus.FAILED + src.error_message = f"transcribe worker crashed: {exc}" + src.updated_at = utcnow() + release_claim(sess, source_id) + return source_id + + def _advance( + self, + source: Source, + max_video_length: int | None, + sess, + ) -> None: + """Advance a single claimed source through pull → transcribe.""" + from second_brain.adapters.youtube import YouTubeAdapter + from second_brain.transcribe import Transcriber, resolve_settings, write_srt + + # Length guard — relies on yt-dlp's metadata for PENDING sources; + # for already-PULLED sources we trust whatever the prior pull + # recorded on the row. + if source.status == SourceStatus.PENDING: + yt = YouTubeAdapter(self.config) + try: + meta = yt.downloader.extract_metadata(source.url) + except Exception as exc: + logger.warning("metadata fetch failed for source %s: %s", source.id, exc) + meta = {} + duration = meta.get("duration_seconds") + if ( + max_video_length is not None + and duration is not None + and duration > max_video_length + ): + source.error_message = ( + f"video length {duration}s exceeds " + f"transcription_max_video_length_seconds={max_video_length}" + ) + source.status = SourceStatus.FAILED + source.updated_at = utcnow() + release_claim(sess, source.id) + logger.info("skipping source %s — too long (%ss)", source.id, duration) + return + + # Pre-fill the metadata so we don't re-hit yt-dlp during pull. + if not source.title: + source.title = meta.get("title") + if not source.author: + source.author = meta.get("author") + if not source.duration_seconds: + source.duration_seconds = duration + if not source.published_at: + source.published_at = meta.get("published_at") + + ok = yt.pull(source) + if not ok: + source.updated_at = utcnow() + release_claim(sess, source.id) + return + + # Transcribe. + if source.status == SourceStatus.PULLED: + if not source.media_path: + source.error_message = "no media_path after pull" + source.status = SourceStatus.FAILED + source.updated_at = utcnow() + release_claim(sess, source.id) + return + + wh = resolve_settings(self.config.whisper) + transcriber = Transcriber( + model=wh["model"], + device=wh["device"], + compute_type=wh["compute_type"], + ) + media_file = Path(source.media_path) + srt_path = self.config.subtitles_dir / f"{media_file.stem}.srt" + + try: + text, segments = transcriber.transcribe(media_file) + except Exception as exc: + source.error_message = f"transcribe failed: {exc}" + source.status = SourceStatus.FAILED + source.updated_at = utcnow() + release_claim(sess, source.id) + logger.exception("transcribe failed for source %s", source.id) + return + + write_srt(segments, srt_path) + source.transcript_path = str(srt_path) + source.transcript_text = text + source.status = SourceStatus.TRANSCRIBED + source.updated_at = utcnow() + release_claim(sess, source.id) + logger.info("transcribed source %s — %d chars", source.id, len(text)) + + # ------------------------------------------------------------------ + # Helpers + # ------------------------------------------------------------------ + + def _sleep(self, seconds: int) -> None: + """Sleep that wakes early on SIGTERM.""" + end = time.time() + seconds + while time.time() < end and not self._stop: + time.sleep(1) + + +__all__ = ["TranscribeWorker", "DEFAULT_POLL_INTERVAL_SECONDS"] diff --git a/src/second_brain/transcribe.py b/src/second_brain/transcribe.py new file mode 100644 index 0000000..8c7f201 --- /dev/null +++ b/src/second_brain/transcribe.py @@ -0,0 +1,171 @@ +"""Audio/video transcription via faster-whisper (CTranslate2 backend). + +Lazy-imports `faster_whisper` so the dev side — which doesn't transcribe +anything — doesn't need the heavyweight CUDA wheels installed. + +Config keys (read from `[whisper]` in settings.toml, env-overrides win): +- `whisper_model` : the model name. Default "large-v3". +- `whisper_device` : "cuda" | "cpu" | "auto". Default "cuda". +- `whisper_compute_type` : "float16" | "int8" | "int8_float16" | … + Default float16 on cuda, int8 on cpu. + +Outputs an SRT file next to `config.subtitles_dir/{stem}.srt` plus a +plain-text transcript string. The pipeline writes the text into +`sources.transcript_text` and the path into `sources.transcript_path`, +matching the previous torch-whisper contract so downstream extract/embed +doesn't need to change. +""" + +from __future__ import annotations + +import logging +import os +from pathlib import Path +from typing import Iterable, Optional + +logger = logging.getLogger(__name__) + + +# --------------------------------------------------------------------------- +# Config resolution +# --------------------------------------------------------------------------- + + +def _resolve(cfg: dict | None, env_key: str, cfg_key: str, default: str) -> str: + env = os.environ.get(env_key) + if env: + return env + if cfg is not None: + v = cfg.get(cfg_key) + if v: + return v + return default + + +def resolve_settings(config_block: dict | None = None) -> dict[str, str]: + """Resolve whisper runtime settings from env > config block > defaults. + + Pulls model/device/compute_type and picks a sane compute_type per device + when none was specified. Returns plain strings so the caller can stash + them in logs / pass straight to faster-whisper. + """ + model = _resolve(config_block, "WHISPER_MODEL", "whisper_model", "large-v3") + device = _resolve(config_block, "WHISPER_DEVICE", "whisper_device", "cuda") + + explicit_compute = None + if config_block is not None: + explicit_compute = config_block.get("whisper_compute_type") + explicit_compute = os.environ.get("WHISPER_COMPUTE_TYPE", explicit_compute) + + if explicit_compute: + compute_type = explicit_compute + elif device == "cpu": + compute_type = "int8" + else: + compute_type = "float16" + + return {"model": model, "device": device, "compute_type": compute_type} + + +# --------------------------------------------------------------------------- +# Transcriber +# --------------------------------------------------------------------------- + + +class Transcriber: + """faster-whisper wrapper. Model is loaded lazily on first transcribe.""" + + def __init__( + self, + model: str = "large-v3", + device: str = "cuda", + compute_type: str = "float16", + ) -> None: + self.model = model + self.device = device + self.compute_type = compute_type + self._impl = None + + def _load(self) -> None: + if self._impl is not None: + return + try: + from faster_whisper import WhisperModel # type: ignore[import-not-found] + except ImportError as exc: + raise RuntimeError( + "faster-whisper is not installed. On the tower run: " + "uv sync --extra tower" + ) from exc + + logger.info( + "loading faster-whisper model=%s device=%s compute_type=%s", + self.model, + self.device, + self.compute_type, + ) + self._impl = WhisperModel( + self.model, device=self.device, compute_type=self.compute_type + ) + + def transcribe( + self, + media_file: Path, + *, + language: Optional[str] = None, + beam_size: int = 5, + ) -> tuple[str, list[dict]]: + """Transcribe `media_file`. Returns (full_text, segments). + + Each segment dict has start/end (float seconds) and text (str) — + same shape the SRT writer needs. + """ + self._load() + assert self._impl is not None # for the type-checker + + segments_iter, info = self._impl.transcribe( + str(media_file), + language=language, + beam_size=beam_size, + ) + logger.info( + "transcribe start: %s language=%s duration=%.1fs", + media_file.name, + getattr(info, "language", "?"), + getattr(info, "duration", 0.0) or 0.0, + ) + + segments: list[dict] = [] + text_parts: list[str] = [] + for seg in segments_iter: + t = (seg.text or "").strip() + segments.append({"start": float(seg.start), "end": float(seg.end), "text": t}) + if t: + text_parts.append(t) + + return " ".join(text_parts), segments + + +# --------------------------------------------------------------------------- +# SRT writer +# --------------------------------------------------------------------------- + + +def write_srt(segments: Iterable[dict], output: Path) -> None: + """Write a list of {start,end,text} dicts as SRT into `output`.""" + output.parent.mkdir(parents=True, exist_ok=True) + with open(output, "w", encoding="utf-8") as fh: + for i, seg in enumerate(segments, start=1): + start = _fmt_timestamp(seg["start"]) + end = _fmt_timestamp(seg["end"]) + fh.write(f"{i}\n{start} --> {end}\n{seg['text']}\n\n") + + +def _fmt_timestamp(seconds: float) -> str: + h = int(seconds // 3600) + m = int((seconds % 3600) // 60) + s = int(seconds % 60) + ms = int((seconds % 1) * 1000) + return f"{h:02d}:{m:02d}:{s:02d},{ms:03d}" + + +__all__ = ["Transcriber", "resolve_settings", "write_srt"] diff --git a/uv.lock b/uv.lock index 141b482..9b9178b 100644 --- a/uv.lock +++ b/uv.lock @@ -66,6 +66,30 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/da/42/e921fccf5015463e32a3cf6ee7f980a6ed0f395ceeaa45060b61d86486c2/anyio-4.13.0-py3-none-any.whl", hash = "sha256:08b310f9e24a9594186fd75b4f73f4a4152069e3853f1ed8bfbf58369f4ad708", size = 114353, upload-time = "2026-03-24T12:59:08.246Z" }, ] +[[package]] +name = "av" +version = "17.0.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/4e/f0/8c8dca97ae0cf00e8e2a53bb5cb9aca5fd484f585ef3e9b412200aff3ebd/av-17.0.1.tar.gz", hash = "sha256:fbcbd4aa43bca6a8691816283112d1659a27f407bbeb66d1397023691339f5d4", size = 4411938, upload-time = "2026-04-18T17:12:34.29Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/4c/82/e7007dcef7bd2d2c377e2e85977701384f42d19fc808c2ccb3a99eaf58f2/av-17.0.1-cp311-abi3-macosx_11_0_x86_64.whl", hash = "sha256:987f4f46ceae4da6c614dcbd2b8149be9dbf680c3bb7a6841c58af9cff4d9230", size = 23238802, upload-time = "2026-04-18T17:11:51.166Z" }, + { url = "https://files.pythonhosted.org/packages/6b/aa/858b09a08ea6f83f91be44b5a5adad13ae8d9ac8b80fda27e73c24bfb160/av-17.0.1-cp311-abi3-macosx_14_0_arm64.whl", hash = "sha256:d97f54e55b18a74912f479c1978aadd1341d38d892dee95bb5c2f2dccfa72f32", size = 18709338, upload-time = "2026-04-18T17:11:53.286Z" }, + { url = "https://files.pythonhosted.org/packages/a8/8b/8de3fd21c4b0b74d44337421abeab0e71462337fb6a28fff888e0c356cbd/av-17.0.1-cp311-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:e6eee84afa48d0e9321047cd3e4facd44b401493f6bdc753e2e1d1e7c9e6d13e", size = 34007351, upload-time = "2026-04-18T17:11:56.116Z" }, + { url = "https://files.pythonhosted.org/packages/02/28/167b291356c2cc315a2d62a95b0ceace72b5b0bf547de30b89313110f032/av-17.0.1-cp311-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:c58c71bffd9383908c85695ac61d3184c668accb04a5bd1b262e0fb8d09f60a5", size = 36345295, upload-time = "2026-04-18T17:11:59.125Z" }, + { url = "https://files.pythonhosted.org/packages/04/fa/aae56f2ff2c204c408641e1120f5ca5ce9c3390cf5362245c6f1158704b5/av-17.0.1-cp311-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:42d6745d30a410ec9b22aef79a52a7ab5a001eb8f5adfd952946606a30983318", size = 35183754, upload-time = "2026-04-18T17:12:01.697Z" }, + { url = "https://files.pythonhosted.org/packages/ba/bd/776046f27093aef80155a204ca7d82a887ae4ee72ba4ef8411b46ea7898c/av-17.0.1-cp311-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:3ed6bcd7021fe55832f95b8ef78dd01a4cb21faf3cd71f1e1bf4f20bf100b278", size = 37430809, upload-time = "2026-04-18T17:12:04.231Z" }, + { url = "https://files.pythonhosted.org/packages/d9/d5/3261bd2c6b7f6c0aa8379fc970d1ecf496330990b992ad28607785074268/av-17.0.1-cp311-abi3-win_amd64.whl", hash = "sha256:9af524e8632a54032e361d6b88895bd3e7c6212ca560de60f5ccc525323c764c", size = 28889649, upload-time = "2026-04-18T17:12:07.04Z" }, + { url = "https://files.pythonhosted.org/packages/98/39/381104e427a0c7231d2ec0d25d538d58fc20fc0458846b95860d3ef8073b/av-17.0.1-cp311-abi3-win_arm64.whl", hash = "sha256:50e58a473d65ea29b645e45c9fd8518a6783737135683ecc40571a91592bdfe4", size = 21918412, upload-time = "2026-04-18T17:12:09.312Z" }, + { url = "https://files.pythonhosted.org/packages/c7/8c/bb1498f031abb6157b30b7fc2379359176953821b6ba59fbd89dbb56f61f/av-17.0.1-cp314-cp314t-macosx_11_0_x86_64.whl", hash = "sha256:1d33871742d1e71562db3c8e752cacc5a62766d7efc3ae408bff1c3e26ebb46e", size = 23484157, upload-time = "2026-04-18T17:12:11.67Z" }, + { url = "https://files.pythonhosted.org/packages/1a/58/dedaef187b797243cd5762722e376c69c5ad95ab23db44127f09afc2cd66/av-17.0.1-cp314-cp314t-macosx_14_0_arm64.whl", hash = "sha256:1229e879f4b6431bc00f69d7f8891fe9a683b0a6e0e009e6c98eb7e449f0383d", size = 18920872, upload-time = "2026-04-18T17:12:14.826Z" }, + { url = "https://files.pythonhosted.org/packages/9b/26/5c550231651d6285e6a5c4f6f4a0e67459bfe2b622a7c9352be8cca8c819/av-17.0.1-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:4744837f4116964280bcc72285e3cdd51361e98a696205aadd924203440ef511", size = 37471077, upload-time = "2026-04-18T17:12:17.349Z" }, + { url = "https://files.pythonhosted.org/packages/59/e4/9807b89a9d775c6f015677996c48bce48aaff70b5d95885adf39e59832a2/av-17.0.1-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:3d0a7d45d9599bf9df9f8249827113d4f36df1cd6b5356227b997f0552dbc98e", size = 39566981, upload-time = "2026-04-18T17:12:19.942Z" }, + { url = "https://files.pythonhosted.org/packages/5c/72/a22a657abc3de652f5b4f46cbbebdf7cba629752112791b81f05d340991d/av-17.0.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:9acd0b6a6e02af2b37f63d97a03ee2c47936d58e82425c3cd075a95245937c59", size = 38397369, upload-time = "2026-04-18T17:12:22.909Z" }, + { url = "https://files.pythonhosted.org/packages/ae/b2/f4e83e41c1e3c186f34b7df506779d0cd7e40499e2e19519c7ece148cd20/av-17.0.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:3d3a36204cb1f1e7691e6446afa8d6b7097b09946dae732c71c5d05ce09e506e", size = 40582445, upload-time = "2026-04-18T17:12:26.285Z" }, + { url = "https://files.pythonhosted.org/packages/c8/59/8676188b72eed09d48ce6cfaf0f22b0bb9f3cfd74d388ee2b7fdf960536d/av-17.0.1-cp314-cp314t-win_amd64.whl", hash = "sha256:b87b98afe971cde123953073bc9c95ab0b7efd2ecc082dd2dbd11f9d9abf190e", size = 29217136, upload-time = "2026-04-18T17:12:29.189Z" }, + { url = "https://files.pythonhosted.org/packages/5f/af/0a6e1d2a845988039f6c197fa7269b5e9abbe17354fb41cc9d75bb260fcb/av-17.0.1-cp314-cp314t-win_arm64.whl", hash = "sha256:a87a42c36e29f75e7dff7281944f2a6876a2c8875e225ccbf6c1ae62748b4caa", size = 22072676, upload-time = "2026-04-18T17:12:31.836Z" }, +] + [[package]] name = "babel" version = "2.18.0" @@ -277,69 +301,35 @@ wheels = [ ] [[package]] -name = "cuda-bindings" -version = "13.2.0" +name = "ctranslate2" +version = "4.7.2" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "cuda-pathfinder" }, + { name = "numpy" }, + { name = "pyyaml" }, + { name = "setuptools" }, ] wheels = [ - { url = "https://files.pythonhosted.org/packages/52/c8/b2589d68acf7e3d63e2be330b84bc25712e97ed799affbca7edd7eae25d6/cuda_bindings-13.2.0-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e865447abfb83d6a98ad5130ed3c70b1fc295ae3eeee39fd07b4ddb0671b6788", size = 5722404, upload-time = "2026-03-11T00:12:44.041Z" }, - { url = "https://files.pythonhosted.org/packages/1f/92/f899f7bbb5617bb65ec52a6eac1e9a1447a86b916c4194f8a5001b8cde0c/cuda_bindings-13.2.0-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:46d8776a55d6d5da9dd6e9858fba2efcda2abe6743871dee47dd06eb8cb6d955", size = 6320619, upload-time = "2026-03-11T00:12:45.939Z" }, - { url = "https://files.pythonhosted.org/packages/df/93/eef988860a3ca985f82c4f3174fc0cdd94e07331ba9a92e8e064c260337f/cuda_bindings-13.2.0-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6629ca2df6f795b784752409bcaedbd22a7a651b74b56a165ebc0c9dcbd504d0", size = 5614610, upload-time = "2026-03-11T00:12:50.337Z" }, - { url = "https://files.pythonhosted.org/packages/18/23/6db3aba46864aee357ab2415135b3fe3da7e9f1fa0221fa2a86a5968099c/cuda_bindings-13.2.0-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7dca0da053d3b4cc4869eff49c61c03f3c5dbaa0bcd712317a358d5b8f3f385d", size = 6149914, upload-time = "2026-03-11T00:12:52.374Z" }, - { url = "https://files.pythonhosted.org/packages/c0/87/87a014f045b77c6de5c8527b0757fe644417b184e5367db977236a141602/cuda_bindings-13.2.0-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a6464b30f46692d6c7f65d4a0e0450d81dd29de3afc1bb515653973d01c2cd6e", size = 5685673, upload-time = "2026-03-11T00:12:56.371Z" }, - { url = "https://files.pythonhosted.org/packages/ee/5e/c0fe77a73aaefd3fff25ffaccaac69c5a63eafdf8b9a4c476626ef0ac703/cuda_bindings-13.2.0-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f4af9f3e1be603fa12d5ad6cfca7844c9d230befa9792b5abdf7dd79979c3626", size = 6191386, upload-time = "2026-03-11T00:12:58.965Z" }, - { url = "https://files.pythonhosted.org/packages/5f/58/ed2c3b39c8dd5f96aa7a4abef0d47a73932c7a988e30f5fa428f00ed0da1/cuda_bindings-13.2.0-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:df850a1ff8ce1b3385257b08e47b70e959932f5f432d0a4e46a355962b4e4771", size = 5507469, upload-time = "2026-03-11T00:13:04.063Z" }, - { url = "https://files.pythonhosted.org/packages/1f/01/0c941b112ceeb21439b05895eace78ca1aa2eaaf695c8521a068fd9b4c00/cuda_bindings-13.2.0-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e8a16384c6494e5485f39314b0b4afb04bee48d49edb16d5d8593fd35bbd231b", size = 6059693, upload-time = "2026-03-11T00:13:06.003Z" }, -] - -[[package]] -name = "cuda-pathfinder" -version = "1.5.4" -source = { registry = "https://pypi.org/simple" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/11/d0/c177e29701cf1d3008d7d2b16b5fc626592ce13bd535f8795c5f57187e0e/cuda_pathfinder-1.5.4-py3-none-any.whl", hash = "sha256:9563d3175ce1828531acf4b94e1c1c7d67208c347ca002493e2654878b26f4b7", size = 51657, upload-time = "2026-04-27T22:42:07.712Z" }, -] - -[[package]] -name = "cuda-toolkit" -version = "13.0.2" -source = { registry = "https://pypi.org/simple" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/57/b2/453099f5f3b698d7d0eab38916aac44c7f76229f451709e2eb9db6615dcd/cuda_toolkit-13.0.2-py2.py3-none-any.whl", hash = "sha256:b198824cf2f54003f50d64ada3a0f184b42ca0846c1c94192fa269ecd97a66eb", size = 2364, upload-time = "2025-12-19T23:24:07.328Z" }, -] - -[package.optional-dependencies] -cudart = [ - { name = "nvidia-cuda-runtime", marker = "sys_platform == 'linux' or sys_platform == 'win32'" }, -] -cufft = [ - { name = "nvidia-cufft", marker = "sys_platform == 'linux' or sys_platform == 'win32'" }, -] -cufile = [ - { name = "nvidia-cufile", marker = "sys_platform == 'linux'" }, -] -cupti = [ - { name = "nvidia-cuda-cupti", marker = "sys_platform == 'linux' or sys_platform == 'win32'" }, -] -curand = [ - { name = "nvidia-curand", marker = "sys_platform == 'linux' or sys_platform == 'win32'" }, -] -cusolver = [ - { name = "nvidia-cusolver", marker = "sys_platform == 'linux' or sys_platform == 'win32'" }, -] -cusparse = [ - { name = "nvidia-cusparse", marker = "sys_platform == 'linux' or sys_platform == 'win32'" }, -] -nvjitlink = [ - { name = "nvidia-nvjitlink", marker = "sys_platform == 'linux' or sys_platform == 'win32'" }, -] -nvrtc = [ - { name = "nvidia-cuda-nvrtc", marker = "sys_platform == 'linux' or sys_platform == 'win32'" }, -] -nvtx = [ - { name = "nvidia-nvtx", marker = "sys_platform == 'linux' or sys_platform == 'win32'" }, + { url = "https://files.pythonhosted.org/packages/68/7a/b584ea42131f0bc418fa75851f3e75fea254a22fc883e456fcf6e4c403fc/ctranslate2-4.7.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:e8818f9211246063ae19b5efa8785519de85f87a19ffe0b270eb8d36a3d79afc", size = 1259514, upload-time = "2026-05-19T06:41:34.347Z" }, + { url = "https://files.pythonhosted.org/packages/64/72/2df1d7cedec58efcc691163d810303d559468aff61e1159b487ac8d94ac1/ctranslate2-4.7.2-cp312-cp312-macosx_11_0_x86_64.whl", hash = "sha256:3df7d2123238da1b1608b691533cfc46b11e5cfda7a27598584d8866701289a9", size = 11922285, upload-time = "2026-05-19T06:41:36.587Z" }, + { url = "https://files.pythonhosted.org/packages/b3/11/1414218b3bca90cbbe687e834d6f0bb2b9901232f7bcce63c05e619e0083/ctranslate2-4.7.2-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:84e11aa6ed8e68eb800ec5aa019830828127e1e9c0c52fe8a816a4f5a28a78a5", size = 16874916, upload-time = "2026-05-19T06:41:39.988Z" }, + { url = "https://files.pythonhosted.org/packages/39/9f/80cebd1ec68c102a846563deb1cae4be7be22db4a72410d8c1a958d482ad/ctranslate2-4.7.2-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e55d07df5298c08dc218c80d492482b732b7f0077a017602fb6fec9601c4021d", size = 39018505, upload-time = "2026-05-19T06:41:45.239Z" }, + { url = "https://files.pythonhosted.org/packages/fe/d0/5439a33c3844d68393f622ba8cc51d5c9b589a4d652490dbd3a4b5d8bc9b/ctranslate2-4.7.2-cp312-cp312-win_amd64.whl", hash = "sha256:f03298e83c427db8f0f4cabb5b9b28680586dae05112651fb39794cd08f849a5", size = 18845969, upload-time = "2026-05-19T06:41:49.388Z" }, + { url = "https://files.pythonhosted.org/packages/3f/36/6dcfe766d4b329131c227acb982d03ceb2f5dd1509bc036a2fbb0197ebc7/ctranslate2-4.7.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:9152e575b226c57e677d3fecc2f2ce0825c828845ac922986382b8c37fc39740", size = 1259476, upload-time = "2026-05-19T06:41:52.082Z" }, + { url = "https://files.pythonhosted.org/packages/7c/74/e89361f9f84179984ca3be47bf53f2ccfb2bb0dc3c35005457bf574bf2bd/ctranslate2-4.7.2-cp313-cp313-macosx_11_0_x86_64.whl", hash = "sha256:eec0521ae9e790147561394257c7a95886f3d8ff28842ffd094248154a928fee", size = 11922163, upload-time = "2026-05-19T06:41:54.337Z" }, + { url = "https://files.pythonhosted.org/packages/11/a5/1000f55ccc1c62acb6ec798ae6c773e175fd08cbac3651e8a59c74c8014f/ctranslate2-4.7.2-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3bd9d5842716ddaed629315fb2f87139ab8e8c62364ea487fbc77c63cca49a23", size = 16875787, upload-time = "2026-05-19T06:41:58.039Z" }, + { url = "https://files.pythonhosted.org/packages/b9/e2/cd18090c3282bcbd2492ee129e721a25ffd3e12d2ece4a19a99091c1e87a/ctranslate2-4.7.2-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c4d1ec8d6c9aaecb049d562bbd4358e6320b3638dcaa28ba786b2eaefd559ae6", size = 39018460, upload-time = "2026-05-19T06:42:02.672Z" }, + { url = "https://files.pythonhosted.org/packages/b4/a2/909ce8b121c0148762e49dfda71bcec9b57c502dcb68e879ba5d84370d75/ctranslate2-4.7.2-cp313-cp313-win_amd64.whl", hash = "sha256:e329af76bbcd0c488f8a191fe6c988c3ca9fe09d3086f8469702dc906a8b4066", size = 18845965, upload-time = "2026-05-19T06:42:06.312Z" }, + { url = "https://files.pythonhosted.org/packages/16/f6/9aeee3b30883a4b6d2c5f2e7b8f2f5316d89eabe6eed5a62815805f1eae2/ctranslate2-4.7.2-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:031ce4eea0589f7e7c8b9a2ab66951d565044ed15c64f2e33af7e1ec5858dde8", size = 1260313, upload-time = "2026-05-19T06:42:09.055Z" }, + { url = "https://files.pythonhosted.org/packages/71/ed/96827f638a79baee9efe84d675a4b6e3c4c9337648b5edf7892a17c0a5d2/ctranslate2-4.7.2-cp314-cp314-macosx_11_0_x86_64.whl", hash = "sha256:8d7ede5e85acefb1b0ed18e3aa026e10a39eb4262229a058211200e2838abf02", size = 11922645, upload-time = "2026-05-19T06:42:11.41Z" }, + { url = "https://files.pythonhosted.org/packages/0d/b8/e42557c86a29ace602a3c0676b8ae6cfa12e52fe05fb74aa9af37d813b99/ctranslate2-4.7.2-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:05127c6f04de072fba6e6e6e012f7c9265cd65cb1ad19420b861d23fb1e62e90", size = 16864569, upload-time = "2026-05-19T06:42:14.664Z" }, + { url = "https://files.pythonhosted.org/packages/50/95/39dd1f7eb4af9dd73c4b708c053a5cf170e3ff957d8950eb51f00a54184d/ctranslate2-4.7.2-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:54392ef0f1353c8b339640ca1a342761258ea3c58e744e91562d225dc5f40ec4", size = 38984169, upload-time = "2026-05-19T06:42:19.644Z" }, + { url = "https://files.pythonhosted.org/packages/d8/2d/7fd9576f654f88c867fe212c53803d487f86a04efde1994f16597439bddc/ctranslate2-4.7.2-cp314-cp314-win_amd64.whl", hash = "sha256:ea8f491f719f4e9ef38349ab2c37d7001a0e49881ee9f2193df3ddbf3aca1d4b", size = 19103583, upload-time = "2026-05-19T06:42:23.573Z" }, + { url = "https://files.pythonhosted.org/packages/1c/96/d0fc93b0451f1fa2257744769dc85e873c4c824fe9b6364d6b2b8d034c13/ctranslate2-4.7.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:2496caa4bbbaac10d81d071fa8d47ebd6a713f879e80a560797cb59767ad2a54", size = 1282921, upload-time = "2026-05-19T06:42:25.923Z" }, + { url = "https://files.pythonhosted.org/packages/4b/77/8b1a5290c4bd28a3ee3f9e4f72fedd08ccae96d7860b8b52fae0d0a34522/ctranslate2-4.7.2-cp314-cp314t-macosx_11_0_x86_64.whl", hash = "sha256:b0a2065b998d14e9322a0daaa5b16f3afb23c878d443c2eb37ad1129eee80c30", size = 11943705, upload-time = "2026-05-19T06:42:28.039Z" }, + { url = "https://files.pythonhosted.org/packages/9f/e9/6ea19da08c5d1103ddfc616ce3ff2ada28d6f0c03745cc07dbed4c51d20f/ctranslate2-4.7.2-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f81af66bb0074b5c8d48eb5574cae2d8fc88c1af0c7377ce5ce043fef75c4317", size = 16852294, upload-time = "2026-05-19T06:42:31.234Z" }, + { url = "https://files.pythonhosted.org/packages/87/35/5f557ef7ed82ba1519f6cf5ccc33e2a805f9809d96bdd975a2cb9e925110/ctranslate2-4.7.2-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d1bed2db340b99c1c63993459a775f667b5db35e09894b55c1035adc733ce1ae", size = 38954642, upload-time = "2026-05-19T06:42:35.798Z" }, + { url = "https://files.pythonhosted.org/packages/3e/58/0b6d03d2da81f4f6a5f674f1934828ec72a9393d17c9f086dbb7c7c0399f/ctranslate2-4.7.2-cp314-cp314t-win_amd64.whl", hash = "sha256:76f568c6c9641cbc056f5c5c635e6ea7c927ad8ce23321804dcbbee66b2c9e24", size = 19126024, upload-time = "2026-05-19T06:42:39.605Z" }, ] [[package]] @@ -399,6 +389,22 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/e0/82/45359b62a067409bd929ae8a56b8ed13e5a8c8a61194b3c236920999ab83/fastapi-0.136.3-py3-none-any.whl", hash = "sha256:3d2a69bdf04b7e9f3afa292c3bc7a98816bbfafa10bc9b45f3f3700d2f761620", size = 117481, upload-time = "2026-05-23T18:53:16.924Z" }, ] +[[package]] +name = "faster-whisper" +version = "1.2.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "av" }, + { name = "ctranslate2" }, + { name = "huggingface-hub" }, + { name = "onnxruntime" }, + { name = "tokenizers" }, + { name = "tqdm" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/05/99/49ee85903dee060d9f08297b4a342e5e0bcfca2f027a07b4ee0a38ab13f9/faster_whisper-1.2.1-py3-none-any.whl", hash = "sha256:79a66ad50688c0b794dd501dc340a736992a6342f7f95e5811be60b5224a26a7", size = 1118909, upload-time = "2025-10-31T11:35:47.794Z" }, +] + [[package]] name = "filelock" version = "3.29.0" @@ -408,6 +414,14 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/81/47/dd9a212ef6e343a6857485ffe25bba537304f1913bdbed446a23f7f592e1/filelock-3.29.0-py3-none-any.whl", hash = "sha256:96f5f6344709aa1572bbf631c640e4ebeeb519e08da902c39a001882f30ac258", size = 39812, upload-time = "2026-04-19T15:39:08.752Z" }, ] +[[package]] +name = "flatbuffers" +version = "25.12.19" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e8/2d/d2a548598be01649e2d46231d151a6c56d10b964d94043a335ae56ea2d92/flatbuffers-25.12.19-py2.py3-none-any.whl", hash = "sha256:7634f50c427838bb021c2d66a3d1168e9d199b0607e6329399f04846d42e20b4", size = 26661, upload-time = "2025-12-19T23:16:13.622Z" }, +] + [[package]] name = "fsspec" version = "2026.4.0" @@ -481,6 +495,38 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/04/4b/29cac41a4d98d144bf5f6d33995617b185d14b22401f75ca86f384e87ff1/h11-0.16.0-py3-none-any.whl", hash = "sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86", size = 37515, upload-time = "2025-04-24T03:35:24.344Z" }, ] +[[package]] +name = "hf-xet" +version = "1.5.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/74/d8/5c06fc76461418326a7decf8367480c35be11a41fd938633929c60a9ec6b/hf_xet-1.5.0.tar.gz", hash = "sha256:e0fb0a34d9f406eed88233e829a67ec016bec5af19e480eac65a233ea289a948", size = 837196, upload-time = "2026-05-06T06:18:15.583Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/68/9b/6912c99070915a4f28119e3c5b52a9abd1eec0ad5cb293b8c967a0c6f5a2/hf_xet-1.5.0-cp313-cp313t-macosx_10_12_x86_64.whl", hash = "sha256:7d70fe2ce97b9db73b9c9b9c81fe3693640aec83416a966c446afea54acfae3c", size = 4023383, upload-time = "2026-05-06T06:17:53.947Z" }, + { url = "https://files.pythonhosted.org/packages/0f/6d/9563cfde59b5d8128a9c7ec972a087f4c782e4f7bac5a85234edfd5d5e49/hf_xet-1.5.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:73a0dae8c71de3b0633a45c73f4a4a5ed09e94b43441d82981a781d4f12baa42", size = 3792751, upload-time = "2026-05-06T06:17:51.791Z" }, + { url = "https://files.pythonhosted.org/packages/07/a5/ed5a0cf35b49a0571af5a8f53416dad1877a718c021c9937c3a53cb45781/hf_xet-1.5.0-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:a60290ec57e9b71767fba7c3645ddafdd0759974b540441510c629c6db6db24a", size = 4456058, upload-time = "2026-05-06T06:17:40.735Z" }, + { url = "https://files.pythonhosted.org/packages/60/fb/3ae8bf2a7a37a4197d0195d7247fd25b3952e15cb8a599e285dfaa6f52b3/hf_xet-1.5.0-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:e5de0f6deada0dada870bb376a11bcd1f08abf3a968a6d118f33e72d1b1eb480", size = 4250783, upload-time = "2026-05-06T06:17:38.412Z" }, + { url = "https://files.pythonhosted.org/packages/a2/9b/8bae40d4d91525085137196e84eb0ed49cf65b5e96e5c3ecdadd8bd0fac2/hf_xet-1.5.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:c799d49f1a5544a0ef7591c0ee75e0d6b93d6f56dc7a4979f59f7518d2872216", size = 4445594, upload-time = "2026-05-06T06:18:04.219Z" }, + { url = "https://files.pythonhosted.org/packages/13/59/c74efbbd4e8728172b2cc72a2bc014d2947a4b7bdced932fbd3f5da1a4e5/hf_xet-1.5.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:2baea1b0b989e5c152fe81425f7745ddc8901280ba3d97c98d8cdece7b706c60", size = 4663995, upload-time = "2026-05-06T06:18:06.1Z" }, + { url = "https://files.pythonhosted.org/packages/73/32/8e1e0410af64cda9b139d1dcebdc993a8ff9c8c7c0e2696ae356d75ccc0d/hf_xet-1.5.0-cp313-cp313t-win_amd64.whl", hash = "sha256:526345b3ed45f374f6317349df489167606736c876241ba984105afe7fd4839d", size = 3966608, upload-time = "2026-05-06T06:18:19.74Z" }, + { url = "https://files.pythonhosted.org/packages/fc/34/a8febc8f4edbea8b3e21b02ebc8b628679b84ba7e45cde624a7736b51500/hf_xet-1.5.0-cp313-cp313t-win_arm64.whl", hash = "sha256:786d28e2eb8315d5035544b9d137b4a842d600c434bb91bf7d0d953cce906ad4", size = 3796946, upload-time = "2026-05-06T06:18:17.568Z" }, + { url = "https://files.pythonhosted.org/packages/2a/20/8fc8996afe5815fa1a6be8e9e5c02f24500f409d599e905800d498a4e14d/hf_xet-1.5.0-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:872d5601e6deea30d15865ede55d29eac6daf5a534ab417b99b6ef6b076dd96c", size = 4023495, upload-time = "2026-05-06T06:18:01.94Z" }, + { url = "https://files.pythonhosted.org/packages/32/6a/93d84463c00cecb561a7508aa6303e35ee2894294eac14245526924415fe/hf_xet-1.5.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:9929561f5abf4581c8ea79587881dfef6b8abb2a0d8a51915936fc2a614f4e73", size = 3792731, upload-time = "2026-05-06T06:18:00.021Z" }, + { url = "https://files.pythonhosted.org/packages/9d/5a/8ec8e0c863b382d00b3c2e2af6ded6b06371be617144a625903a6d562f4b/hf_xet-1.5.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:f7b7bbae318e583a86fb21e5a4a175d6721d628a2874f4bd022d0e660c32a682", size = 4456738, upload-time = "2026-05-06T06:17:49.574Z" }, + { url = "https://files.pythonhosted.org/packages/c5/ca/f7effa1a67717da2bcc6b6c28f71c6ca648c77acaec4e2c32f40cbe16d85/hf_xet-1.5.0-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:cf7b2dc6f31a4ea754bb50f74cde482dcf5d366d184076d8530b9872787f3761", size = 4251622, upload-time = "2026-05-06T06:17:47.096Z" }, + { url = "https://files.pythonhosted.org/packages/65/f2/19247dba3e231cf77dec59ddfb878f00057635ff773d099c9b59d37812c3/hf_xet-1.5.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:8dbcbab554c9ef158ef2c991545c3e970ddd8cc7acdcd0a78c5a41095dab4ded", size = 4445667, upload-time = "2026-05-06T06:18:11.983Z" }, + { url = "https://files.pythonhosted.org/packages/7f/64/6f116801a3bcfb6f59f5c251f48cadc47ea54026441c4a385079286a94fa/hf_xet-1.5.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:5906bf7718d3636dc13402914736abe723492cb730f744834f5f5b67d3a12702", size = 4664619, upload-time = "2026-05-06T06:18:13.771Z" }, + { url = "https://files.pythonhosted.org/packages/5c/e8/069542d37946ed08669b127e1496fa99e78196d71de8d41eda5e9f1b7a58/hf_xet-1.5.0-cp314-cp314t-win_amd64.whl", hash = "sha256:5f3dc2248fc01cc0a00cd392ab497f1ca373fcbc7e3f2da1f452480b384e839e", size = 3966802, upload-time = "2026-05-06T06:18:28.162Z" }, + { url = "https://files.pythonhosted.org/packages/f9/91/fc6fdec27b14d04e88c386ac0a0129732b53fa23f7c4a78f4b83a039c567/hf_xet-1.5.0-cp314-cp314t-win_arm64.whl", hash = "sha256:b285cea1b5bab46b758772716ba8d6854a1a0310fed1c249d678a8b38601e5a0", size = 3797168, upload-time = "2026-05-06T06:18:26.287Z" }, + { url = "https://files.pythonhosted.org/packages/3d/fb/69ff198a82cae7eb1a69fb84d93b3a3e4816564d76817fe541ddc96874eb/hf_xet-1.5.0-cp37-abi3-macosx_10_12_x86_64.whl", hash = "sha256:dad0dc84e941b8ba3c860659fe1fdc35c049d47cce293f003287757e971a8f56", size = 4030814, upload-time = "2026-05-06T06:17:57.933Z" }, + { url = "https://files.pythonhosted.org/packages/9b/ff/edcc2b40162bef3ff78e14ab637e5f3b89243d6aee72f5949d3bb6a5af83/hf_xet-1.5.0-cp37-abi3-macosx_11_0_arm64.whl", hash = "sha256:fd6e5a9b0fdac4ed03ed45ef79254a655b1aaab514a02202617fbf643f5fdf7a", size = 3798444, upload-time = "2026-05-06T06:17:55.79Z" }, + { url = "https://files.pythonhosted.org/packages/49/4d/103f76b04310e5e57656696cc184690d20c466af0bca3ca88f8c8ea5d4f3/hf_xet-1.5.0-cp37-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:3531b1823a0e6d77d80f9ed15ca0e00f0d115094f8ac033d5cae88f4564cc949", size = 4465986, upload-time = "2026-05-06T06:17:44.886Z" }, + { url = "https://files.pythonhosted.org/packages/c4/a2/546f47f464737b3edbab6f8ddb57f2599b93d2cbb66f06abb475ccb48651/hf_xet-1.5.0-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:9a0ee58cd18d5ea799f7ed11290bbccbe56bdd8b1d97ca74b9cc49a3945d7a3b", size = 4259865, upload-time = "2026-05-06T06:17:42.639Z" }, + { url = "https://files.pythonhosted.org/packages/95/7f/1be593c1f28613be2e196473481cd81bfc5910795e30a34e8f744f6cac4f/hf_xet-1.5.0-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:1e60df5a42e9bed8628b6416af2cba4cba57ae9f02de226a06b020d98e1aab18", size = 4459835, upload-time = "2026-05-06T06:18:08.026Z" }, + { url = "https://files.pythonhosted.org/packages/aa/b2/703569fc881f3284487e68cda7b42179978480da3c438042a6bbbb4a671c/hf_xet-1.5.0-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:4b35549ce62601b84da4ff9b24d970032ace3d4430f52d91bcbb26c901d6c690", size = 4672414, upload-time = "2026-05-06T06:18:09.864Z" }, + { url = "https://files.pythonhosted.org/packages/af/37/1b6def445c567286b50aa3b33828158e135b1be44938dde59f11382a500c/hf_xet-1.5.0-cp37-abi3-win_amd64.whl", hash = "sha256:2806c7c17b4d23f8d88f7c4814f838c3b6150773fe339c20af23e1cfaf2797e4", size = 3977238, upload-time = "2026-05-06T06:18:23.621Z" }, + { url = "https://files.pythonhosted.org/packages/62/94/3b66b148778ee100dcfd69c2ca22b57b41b44d3063ceec934f209e9184ce/hf_xet-1.5.0-cp37-abi3-win_arm64.whl", hash = "sha256:b6c9df403040248c76d808d3e047d64db2d923bae593eb244c41e425cf6cd7be", size = 3806916, upload-time = "2026-05-06T06:18:21.7Z" }, +] + [[package]] name = "htmldate" version = "1.9.4" @@ -554,6 +600,26 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/2a/39/e50c7c3a983047577ee07d2a9e53faf5a69493943ec3f6a384bdc792deb2/httpx-0.28.1-py3-none-any.whl", hash = "sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad", size = 73517, upload-time = "2024-12-06T15:37:21.509Z" }, ] +[[package]] +name = "huggingface-hub" +version = "1.16.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "filelock" }, + { name = "fsspec" }, + { name = "hf-xet", marker = "platform_machine == 'AMD64' or platform_machine == 'aarch64' or platform_machine == 'amd64' or platform_machine == 'arm64' or platform_machine == 'x86_64'" }, + { name = "httpx" }, + { name = "packaging" }, + { name = "pyyaml" }, + { name = "tqdm" }, + { name = "typer" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/48/0f/ed994dbade67a54407c28cab96ef845e0e6d25500be56aca6394f8bfc9dd/huggingface_hub-1.16.1.tar.gz", hash = "sha256:7f1dc4c5ec21aed69be630ad0c3378616be16f3de1a47b141c0e812965d9c832", size = 792534, upload-time = "2026-05-21T18:40:00.908Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/49/79/621a7dbb80c70974f73a597275351ebe03ce5bc65cb5f8f4acb5859252bc/huggingface_hub-1.16.1-py3-none-any.whl", hash = "sha256:64340de934b9ce37857ef85a82de72f5629e8a270f9119eabb12bf495eb53c22", size = 668176, upload-time = "2026-05-21T18:39:58.596Z" }, +] + [[package]] name = "idna" version = "3.16" @@ -668,30 +734,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/f2/ac/52f4e86d1924a7fc05af3aeb34488570eccc39b4af90530dd6acecdf16b5/justext-3.0.2-py2.py3-none-any.whl", hash = "sha256:62b1c562b15c3c6265e121cc070874243a443bfd53060e869393f09d6b6cc9a7", size = 837940, upload-time = "2025-02-25T20:21:44.179Z" }, ] -[[package]] -name = "llvmlite" -version = "0.47.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/01/88/a8952b6d5c21e74cbf158515b779666f692846502623e9e3c39d8e8ba25f/llvmlite-0.47.0.tar.gz", hash = "sha256:62031ce968ec74e95092184d4b0e857e444f8fdff0b8f9213707699570c33ccc", size = 193614, upload-time = "2026-03-31T18:29:53.497Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/fa/48/4b7fe0e34c169fa2f12532916133e0b219d2823b540733651b34fdac509a/llvmlite-0.47.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:306a265f408c259067257a732c8e159284334018b4083a9e35f67d19792b164f", size = 37232769, upload-time = "2026-03-31T18:28:43.735Z" }, - { url = "https://files.pythonhosted.org/packages/e6/4b/e3f2cd17822cf772a4a51a0a8080b0032e6d37b2dbe8cfb724eac4e31c52/llvmlite-0.47.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:5853bf26160857c0c2573415ff4efe01c4c651e59e2c55c2a088740acfee51cd", size = 56275178, upload-time = "2026-03-31T18:28:48.342Z" }, - { url = "https://files.pythonhosted.org/packages/b6/55/a3b4a543185305a9bdf3d9759d53646ed96e55e7dfd43f53e7a421b8fbae/llvmlite-0.47.0-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:003bcf7fa579e14db59c1a1e113f93ab8a06b56a4be31c7f08264d1d4072d077", size = 55128632, upload-time = "2026-03-31T18:28:52.901Z" }, - { url = "https://files.pythonhosted.org/packages/2f/f5/d281ae0f79378a5a91f308ea9fdb9f9cc068fddd09629edc0725a5a8fde1/llvmlite-0.47.0-cp312-cp312-win_amd64.whl", hash = "sha256:f3079f25bdc24cd9d27c4b2b5e68f5f60c4fdb7e8ad5ee2b9b006007558f9df7", size = 38138692, upload-time = "2026-03-31T18:28:57.147Z" }, - { url = "https://files.pythonhosted.org/packages/77/6f/4615353e016799f80fa52ccb270a843c413b22361fadda2589b2922fb9b0/llvmlite-0.47.0-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:a3c6a735d4e1041808434f9d440faa3d78d9b4af2ee64d05a66f351883b6ceec", size = 37232771, upload-time = "2026-03-31T18:29:01.324Z" }, - { url = "https://files.pythonhosted.org/packages/31/b8/69f5565f1a280d032525878a86511eebed0645818492feeb169dfb20ae8e/llvmlite-0.47.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:2699a74321189e812d476a43d6d7f652f51811e7b5aad9d9bba842a1c7927acb", size = 56275178, upload-time = "2026-03-31T18:29:05.748Z" }, - { url = "https://files.pythonhosted.org/packages/d6/da/b32cafcb926fb0ce2aa25553bf32cb8764af31438f40e2481df08884c947/llvmlite-0.47.0-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6c6951e2b29930227963e53ee152441f0e14be92e9d4231852102d986c761e40", size = 55128632, upload-time = "2026-03-31T18:29:11.235Z" }, - { url = "https://files.pythonhosted.org/packages/46/9f/4898b44e4042c60fafcb1162dfb7014f6f15b1ec19bf29cfea6bf26df90d/llvmlite-0.47.0-cp313-cp313-win_amd64.whl", hash = "sha256:c2e9adf8698d813a9a5efb2d4370caf344dbc1e145019851fee6a6f319ba760e", size = 38138695, upload-time = "2026-03-31T18:29:15.43Z" }, - { url = "https://files.pythonhosted.org/packages/1c/d4/33c8af00f0bf6f552d74f3a054f648af2c5bc6bece97972f3bfadce4f5ec/llvmlite-0.47.0-cp314-cp314-macosx_12_0_arm64.whl", hash = "sha256:de966c626c35c9dff5ae7bf12db25637738d0df83fc370cf793bc94d43d92d14", size = 37232773, upload-time = "2026-03-31T18:29:19.453Z" }, - { url = "https://files.pythonhosted.org/packages/64/1d/a760e993e0c0ba6db38d46b9f48f6c7dceb8ac838824997fb9e25f97bc04/llvmlite-0.47.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:ddbccff2aeaff8670368340a158abefc032fe9b3ccf7d9c496639263d00151aa", size = 56275176, upload-time = "2026-03-31T18:29:24.149Z" }, - { url = "https://files.pythonhosted.org/packages/84/3b/e679bc3b29127182a7f4aa2d2e9e5bea42adb93fb840484147d59c236299/llvmlite-0.47.0-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d4a7b778a2e144fc64468fb9bf509ac1226c9813a00b4d7afea5d988c4e22fca", size = 55128631, upload-time = "2026-03-31T18:29:29.536Z" }, - { url = "https://files.pythonhosted.org/packages/be/f7/19e2a09c62809c9e63bbd14ce71fb92c6ff7b7b3045741bb00c781efc3c9/llvmlite-0.47.0-cp314-cp314-win_amd64.whl", hash = "sha256:694e3c2cdc472ed2bd8bd4555ca002eec4310961dd58ef791d508f57b5cc4c94", size = 39153826, upload-time = "2026-03-31T18:29:33.681Z" }, - { url = "https://files.pythonhosted.org/packages/40/a1/581a8c707b5e80efdbbe1dd94527404d33fe50bceb71f39d5a7e11bd57b7/llvmlite-0.47.0-cp314-cp314t-macosx_12_0_arm64.whl", hash = "sha256:92ec8a169a20b473c1c54d4695e371bde36489fc1efa3688e11e99beba0abf9c", size = 37232772, upload-time = "2026-03-31T18:29:37.952Z" }, - { url = "https://files.pythonhosted.org/packages/11/03/16090dd6f74ba2b8b922276047f15962fbeea0a75d5601607edb301ba945/llvmlite-0.47.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:fa1cbd800edd3b20bc141521f7fd45a6185a5b84109aa6855134e81397ffe72b", size = 56275178, upload-time = "2026-03-31T18:29:42.58Z" }, - { url = "https://files.pythonhosted.org/packages/f5/cb/0abf1dd4c5286a95ffe0c1d8c67aec06b515894a0dd2ac97f5e27b82ab0b/llvmlite-0.47.0-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f6725179b89f03b17dabe236ff3422cb8291b4c1bf40af152826dfd34e350ae8", size = 55128632, upload-time = "2026-03-31T18:29:46.939Z" }, - { url = "https://files.pythonhosted.org/packages/4f/79/d3bbab197e86e0ff4f9c07122895b66a3e0d024247fcff7f12c473cb36d9/llvmlite-0.47.0-cp314-cp314t-win_amd64.whl", hash = "sha256:6842cf6f707ec4be3d985a385ad03f72b2d724439e118fcbe99b2929964f0453", size = 39153839, upload-time = "2026-03-31T18:29:51.004Z" }, -] - [[package]] name = "lxml" version = "6.1.1" @@ -801,6 +843,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/bc/b1/a0ec7a5a9db730a08daef1fdfb8090435b82465abbf758a596f0ea88727e/mako-1.3.12-py3-none-any.whl", hash = "sha256:8f61569480282dbf557145ce441e4ba888be453c30989f879f0d652e39f53ea9", size = 78521, upload-time = "2026-04-28T19:01:10.393Z" }, ] +[[package]] +name = "markdown-it-py" +version = "4.2.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "mdurl" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/06/ff/7841249c247aa650a76b9ee4bbaeae59370dc8bfd2f6c01f3630c35eb134/markdown_it_py-4.2.0.tar.gz", hash = "sha256:04a21681d6fbb623de53f6f364d352309d4094dd4194040a10fd51833e418d49", size = 82454, upload-time = "2026-05-07T12:08:28.36Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b3/81/4da04ced5a082363ecfa159c010d200ecbd959ae410c10c0264a38cac0f5/markdown_it_py-4.2.0-py3-none-any.whl", hash = "sha256:9f7ebbcd14fe59494226453aed97c1070d83f8d24b6fc3a3bcf9a38092641c4a", size = 91687, upload-time = "2026-05-07T12:08:27.182Z" }, +] + [[package]] name = "markupsafe" version = "3.0.3" @@ -865,58 +919,12 @@ wheels = [ ] [[package]] -name = "more-itertools" -version = "11.1.0" +name = "mdurl" +version = "0.1.2" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/de/1d/f4da6f02cdffe04d6362210b807146a26044c88d839208aec273bb0d9184/more_itertools-11.1.0.tar.gz", hash = "sha256:48e8f4d9e7e5878571ecf6f2b4e57634f93cd474cc8cfbd2376f2d11b396e30d", size = 145772, upload-time = "2026-05-22T14:14:29.909Z" } +sdist = { url = "https://files.pythonhosted.org/packages/d6/54/cfe61301667036ec958cb99bd3efefba235e65cdeb9c84d24a8293ba1d90/mdurl-0.1.2.tar.gz", hash = "sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba", size = 8729, upload-time = "2022-08-14T12:40:10.846Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/e8/3d/1087453384dbde46a8c7f9356eead2c58be8a7bf156bca40243377c85715/more_itertools-11.1.0-py3-none-any.whl", hash = "sha256:4b65538ae22f6fed0ce4874efd317463a7489796a0939fa66824dd542125a192", size = 72226, upload-time = "2026-05-22T14:14:28.824Z" }, -] - -[[package]] -name = "mpmath" -version = "1.3.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/e0/47/dd32fa426cc72114383ac549964eecb20ecfd886d1e5ccf5340b55b02f57/mpmath-1.3.0.tar.gz", hash = "sha256:7a28eb2a9774d00c7bc92411c19a89209d5da7c4c9a9e227be8330a23a25b91f", size = 508106, upload-time = "2023-03-07T16:47:11.061Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/43/e3/7d92a15f894aa0c9c4b49b8ee9ac9850d6e63b03c9c32c0367a13ae62209/mpmath-1.3.0-py3-none-any.whl", hash = "sha256:a0b2b9fe80bbcd81a6647ff13108738cfb482d481d826cc0e02f5b35e5c88d2c", size = 536198, upload-time = "2023-03-07T16:47:09.197Z" }, -] - -[[package]] -name = "networkx" -version = "3.6.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/6a/51/63fe664f3908c97be9d2e4f1158eb633317598cfa6e1fc14af5383f17512/networkx-3.6.1.tar.gz", hash = "sha256:26b7c357accc0c8cde558ad486283728b65b6a95d85ee1cd66bafab4c8168509", size = 2517025, upload-time = "2025-12-08T17:02:39.908Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/9e/c9/b2622292ea83fbb4ec318f5b9ab867d0a28ab43c5717bb85b0a5f6b3b0a4/networkx-3.6.1-py3-none-any.whl", hash = "sha256:d47fbf302e7d9cbbb9e2555a0d267983d2aa476bac30e90dfbe5669bd57f3762", size = 2068504, upload-time = "2025-12-08T17:02:38.159Z" }, -] - -[[package]] -name = "numba" -version = "0.65.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "llvmlite" }, - { name = "numpy" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/f6/c5/db2ac3685833d626c0dcae6bd2330cd68433e1fd248d15f70998160d3ad7/numba-0.65.1.tar.gz", hash = "sha256:19357146c32fe9ed25059ab915e8465fb13951cf6b0aace3826b76886373ab23", size = 2765600, upload-time = "2026-04-24T02:02:56.551Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/57/bc/76f8f8c5cf9adee47fdb7bbb03be8900f76f902d451d7477cf12b845e1de/numba-0.65.1-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:ac3f1e77c352dd0ea9712732c2d8f9ca507717435eec5b5013bf138ac33c4a08", size = 2681371, upload-time = "2026-04-24T02:02:26.105Z" }, - { url = "https://files.pythonhosted.org/packages/69/47/a415af0283e4db0398104c6d1c11c9861a98dc67a7aa442a7769ed5d6196/numba-0.65.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:52bc6f3ceb8fcaff9b2ae26b4c6b1e9fee39db8d355534c0fe4f39a901246b84", size = 3802467, upload-time = "2026-04-24T02:02:27.712Z" }, - { url = "https://files.pythonhosted.org/packages/46/36/246f73ec99cfeab2f2cb2ce7d4218766cc36a2da418901223f4f4da9c813/numba-0.65.1-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:90ca10b3463bae0bd70589726fe3c77d01d6b5fc86bee54bcdf9fb6b47c28977", size = 3502628, upload-time = "2026-04-24T02:02:29.763Z" }, - { url = "https://files.pythonhosted.org/packages/db/9e/3c679b2ee078425b9e99a91e44f8d132a6830d8ccce5227bc5e9181aeed8/numba-0.65.1-cp312-cp312-win_amd64.whl", hash = "sha256:5971c632be2a2351500431f46213821dba8d02b18a9f7d02fd36bd2743e41a6a", size = 2750611, upload-time = "2026-04-24T02:02:31.477Z" }, - { url = "https://files.pythonhosted.org/packages/79/37/14a4579049c1eb673afd0de0cb4842982acd55b9ce2643e763db858bcea0/numba-0.65.1-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:1735c15c1134a5108b4d6a5c77fc0947924ea066a738dc09a52008c13df9cad3", size = 2681344, upload-time = "2026-04-24T02:02:33.65Z" }, - { url = "https://files.pythonhosted.org/packages/a0/22/b8d873f6466b20aa563fc9b33acd48dec89a07803ddaa2f1c8ca1cd33126/numba-0.65.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:c09f49117ef255e1f1c6dad0c7a1ed39868243862a73be5706793241a3755f1b", size = 3810619, upload-time = "2026-04-24T02:02:36.041Z" }, - { url = "https://files.pythonhosted.org/packages/62/08/e16a8b5d9a018962ebb5c66be662317cde32b9f5dab08441f90bed5522fb/numba-0.65.1-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:594a8680b3fadac99e97e489b1fd89007177e5336713745c3b769528c635a464", size = 3509783, upload-time = "2026-04-24T02:02:38.245Z" }, - { url = "https://files.pythonhosted.org/packages/fd/a5/03c970d57f4c1741354837353ce39fb5206952ae1dba8922d29c86f64805/numba-0.65.1-cp313-cp313-win_amd64.whl", hash = "sha256:85be74c0d036842699a30058f82fb88fc5ffdc59f7615cab5792ea92914c9b62", size = 2750534, upload-time = "2026-04-24T02:02:39.903Z" }, - { url = "https://files.pythonhosted.org/packages/4f/2e/8aed9b726d9ba5f11ad287645fd479e88278db3060a25cb1225d730eb2b7/numba-0.65.1-cp314-cp314-macosx_12_0_arm64.whl", hash = "sha256:33f5eb68eb1c843511615d14663ce60258525d6a4c65ab040e2c2b0c4cf17450", size = 2681554, upload-time = "2026-04-24T02:02:41.812Z" }, - { url = "https://files.pythonhosted.org/packages/87/96/f3eb235fafa82a34e2ab5dd7dc9ffff998ebf5f0bbc23fa56a96aeb44da6/numba-0.65.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:71e73029bf53a62cc6afcf96be4bd942290d8b4c55f0a454fb536158115790f7", size = 3779602, upload-time = "2026-04-24T02:02:43.726Z" }, - { url = "https://files.pythonhosted.org/packages/09/90/b0f09b48752d23640b8284f22aa597737e8adaddc7fbfacc4708b7f73a4c/numba-0.65.1-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3a07635e0be926b9bdbffb09137c230fb13f6ec0e564914ba937cee12ce3eb35", size = 3479532, upload-time = "2026-04-24T02:02:45.427Z" }, - { url = "https://files.pythonhosted.org/packages/56/46/3f7fc04fb853559e74b210e0b62c19974ec844cefec611f9e535f4da3761/numba-0.65.1-cp314-cp314-win_amd64.whl", hash = "sha256:2a20fcdabdefbdacf88d85caf70c3b18c4bcb7ebb8f82e6a19486383dd26ab63", size = 2752637, upload-time = "2026-04-24T02:02:47.664Z" }, - { url = "https://files.pythonhosted.org/packages/81/7b/c1a341a9067367778f4152a5f01061cf281fb09582c92c510ec4918cabf6/numba-0.65.1-cp314-cp314t-macosx_12_0_arm64.whl", hash = "sha256:548dd4b3a4508d5062768d1514b2cd7b015f9a25ec7af651c50dee243965e652", size = 2684600, upload-time = "2026-04-24T02:02:49.653Z" }, - { url = "https://files.pythonhosted.org/packages/03/36/98ddbcf3e4f04a6dd07e1c67249955920579ba4af6bb6868e3088f4ed282/numba-0.65.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:78abc28feff2c2ff8307fff3975b6438352759c9acb797ecd6b1fb6e7e39e31d", size = 3817198, upload-time = "2026-04-24T02:02:51.266Z" }, - { url = "https://files.pythonhosted.org/packages/a3/83/0dad21057ece5a835599f5d24099b091703995e23dbbf894f259e91c010b/numba-0.65.1-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ee7676cb389555805f9b9a1840cbcd1ea6c8bd5376ab6918e3a29c5ea1dbda20", size = 3533862, upload-time = "2026-04-24T02:02:52.987Z" }, - { url = "https://files.pythonhosted.org/packages/32/36/8be7118ffd4c8440881046eac3d0982cc5ab42909508cf5d67024d62a2e4/numba-0.65.1-cp314-cp314t-win_amd64.whl", hash = "sha256:20609346e3bd75204950dcbbfe383a8d7dbf4902f442aedbf00f97fef4aa8f38", size = 2758237, upload-time = "2026-04-24T02:02:54.612Z" }, + { url = "https://files.pythonhosted.org/packages/b3/38/89ba8ad64ae25be8de66a6d463314cf1eb366222074cfda9ee839c56a4b4/mdurl-0.1.2-py3-none-any.whl", hash = "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8", size = 9979, upload-time = "2022-08-14T12:40:09.779Z" }, ] [[package]] @@ -981,171 +989,36 @@ wheels = [ ] [[package]] -name = "nvidia-cublas" -version = "13.1.1.3" +name = "onnxruntime" +version = "1.26.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "nvidia-cuda-nvrtc" }, -] -wheels = [ - { url = "https://files.pythonhosted.org/packages/a7/a1/0bd24ee8c8d03adac032fd2909426a00c88f8c57961b1277ded97f91119f/nvidia_cublas-13.1.1.3-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:b7a210458267ac818974c53038fbec2e969d5c99f305ab15c72522fa9f001dd5", size = 542848918, upload-time = "2026-04-08T18:46:22.985Z" }, - { url = "https://files.pythonhosted.org/packages/3b/cd/154ca20c38269e05eff77c1464e6c1da89f50a6390b565e9d82e06bc11e1/nvidia_cublas-13.1.1.3-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:37936a16db8fe4ac1f065c2139360608a543a09275cb1a1af612e08cfa065436", size = 423138758, upload-time = "2026-04-08T18:46:58.655Z" }, -] - -[[package]] -name = "nvidia-cuda-cupti" -version = "13.0.85" -source = { registry = "https://pypi.org/simple" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/2a/2a/80353b103fc20ce05ef51e928daed4b6015db4aaa9162ed0997090fe2250/nvidia_cuda_cupti-13.0.85-py3-none-manylinux_2_25_aarch64.whl", hash = "sha256:796bd679890ee55fb14a94629b698b6db54bcfd833d391d5e94017dd9d7d3151", size = 10310827, upload-time = "2025-09-04T08:26:42.012Z" }, - { url = "https://files.pythonhosted.org/packages/33/6d/737d164b4837a9bbd202f5ae3078975f0525a55730fe871d8ed4e3b952b0/nvidia_cuda_cupti-13.0.85-py3-none-manylinux_2_25_x86_64.whl", hash = "sha256:4eb01c08e859bf924d222250d2e8f8b8ff6d3db4721288cf35d14252a4d933c8", size = 10715597, upload-time = "2025-09-04T08:26:51.312Z" }, -] - -[[package]] -name = "nvidia-cuda-nvrtc" -version = "13.0.88" -source = { registry = "https://pypi.org/simple" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/c3/68/483a78f5e8f31b08fb1bb671559968c0ca3a065ac7acabfc7cee55214fd6/nvidia_cuda_nvrtc-13.0.88-py3-none-manylinux2010_x86_64.manylinux_2_12_x86_64.whl", hash = "sha256:ad9b6d2ead2435f11cbb6868809d2adeeee302e9bb94bcf0539c7a40d80e8575", size = 90215200, upload-time = "2025-09-04T08:28:44.204Z" }, - { url = "https://files.pythonhosted.org/packages/b7/dc/6bb80850e0b7edd6588d560758f17e0550893a1feaf436807d64d2da040f/nvidia_cuda_nvrtc-13.0.88-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:d27f20a0ca67a4bb34268a5e951033496c5b74870b868bacd046b1b8e0c3267b", size = 43015449, upload-time = "2025-09-04T08:28:20.239Z" }, -] - -[[package]] -name = "nvidia-cuda-runtime" -version = "13.0.96" -source = { registry = "https://pypi.org/simple" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/87/4f/17d7b9b8e285199c58ce28e31b5c5bbaa4d8271af06a89b6405258245de2/nvidia_cuda_runtime-13.0.96-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:ef9bcbe90493a2b9d810e43d249adb3d02e98dd30200d86607d8d02687c43f55", size = 2261060, upload-time = "2025-10-09T08:55:15.78Z" }, - { url = "https://files.pythonhosted.org/packages/2e/24/d1558f3b68b1d26e706813b1d10aa1d785e4698c425af8db8edc3dced472/nvidia_cuda_runtime-13.0.96-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:7f82250d7782aa23b6cfe765ecc7db554bd3c2870c43f3d1821f1d18aebf0548", size = 2243632, upload-time = "2025-10-09T08:55:36.117Z" }, -] - -[[package]] -name = "nvidia-cudnn-cu13" -version = "9.20.0.48" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "nvidia-cublas" }, -] -wheels = [ - { url = "https://files.pythonhosted.org/packages/56/c5/83384d846b2fd17c44bd499b36c75a45ed4f095fbbb2252294e89cea5c5c/nvidia_cudnn_cu13-9.20.0.48-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:e31454ae00094b0c55319d9d15b6fa2fc50a9e1c0f5c8c80fb75258234e731e1", size = 444574296, upload-time = "2026-03-09T19:28:27.751Z" }, - { url = "https://files.pythonhosted.org/packages/6e/5e/edb9c0ae051602c3ccaffe424256463636d639e27d7f302dde9975ef9e7a/nvidia_cudnn_cu13-9.20.0.48-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:0c45dd8eeb50b603f07995b1b300c62ffe6a1980482b82b3bcf94a4ca9d49304", size = 366173588, upload-time = "2026-03-09T19:29:34.474Z" }, -] - -[[package]] -name = "nvidia-cufft" -version = "12.0.0.61" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "nvidia-nvjitlink" }, -] -wheels = [ - { url = "https://files.pythonhosted.org/packages/8b/ae/f417a75c0259e85c1d2f83ca4e960289a5f814ed0cea74d18c353d3e989d/nvidia_cufft-12.0.0.61-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:2708c852ef8cd89d1d2068bdbece0aa188813a0c934db3779b9b1faa8442e5f5", size = 214053554, upload-time = "2025-09-04T08:31:38.196Z" }, - { url = "https://files.pythonhosted.org/packages/a8/2f/7b57e29836ea8714f81e9898409196f47d772d5ddedddf1592eadb8ab743/nvidia_cufft-12.0.0.61-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:6c44f692dce8fd5ffd3e3df134b6cdb9c2f72d99cf40b62c32dde45eea9ddad3", size = 214085489, upload-time = "2025-09-04T08:31:56.044Z" }, -] - -[[package]] -name = "nvidia-cufile" -version = "1.15.1.6" -source = { registry = "https://pypi.org/simple" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/3f/70/4f193de89a48b71714e74602ee14d04e4019ad36a5a9f20c425776e72cd6/nvidia_cufile-1.15.1.6-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:08a3ecefae5a01c7f5117351c64f17c7c62efa5fffdbe24fc7d298da19cd0b44", size = 1223672, upload-time = "2025-09-04T08:32:22.779Z" }, - { url = "https://files.pythonhosted.org/packages/ab/73/cc4a14c9813a8a0d509417cf5f4bdaba76e924d58beb9864f5a7baceefbf/nvidia_cufile-1.15.1.6-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:bdc0deedc61f548bddf7733bdc216456c2fdb101d020e1ab4b88d232d5e2f6d1", size = 1136992, upload-time = "2025-09-04T08:32:14.119Z" }, -] - -[[package]] -name = "nvidia-curand" -version = "10.4.0.35" -source = { registry = "https://pypi.org/simple" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/1e/72/7c2ae24fb6b63a32e6ae5d241cc65263ea18d08802aaae087d9f013335a2/nvidia_curand-10.4.0.35-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:133df5a7509c3e292aaa2b477afd0194f06ce4ea24d714d616ff36439cee349a", size = 61962106, upload-time = "2025-08-04T10:21:41.128Z" }, - { url = "https://files.pythonhosted.org/packages/a5/9f/be0a41ca4a4917abf5cb9ae0daff1a6060cc5de950aec0396de9f3b52bc5/nvidia_curand-10.4.0.35-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:1aee33a5da6e1db083fe2b90082def8915f30f3248d5896bcec36a579d941bfc", size = 59544258, upload-time = "2025-08-04T10:22:03.992Z" }, -] - -[[package]] -name = "nvidia-cusolver" -version = "12.0.4.66" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "nvidia-cublas" }, - { name = "nvidia-cusparse" }, - { name = "nvidia-nvjitlink" }, -] -wheels = [ - { url = "https://files.pythonhosted.org/packages/c8/c3/b30c9e935fc01e3da443ec0116ed1b2a009bb867f5324d3f2d7e533e776b/nvidia_cusolver-12.0.4.66-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:02c2457eaa9e39de20f880f4bd8820e6a1cfb9f9a34f820eb12a155aa5bc92d2", size = 223467760, upload-time = "2025-09-04T08:33:04.222Z" }, - { url = "https://files.pythonhosted.org/packages/5f/67/cba3777620cdacb99102da4042883709c41c709f4b6323c10781a9c3aa34/nvidia_cusolver-12.0.4.66-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:0a759da5dea5c0ea10fd307de75cdeb59e7ea4fcb8add0924859b944babf1112", size = 200941980, upload-time = "2025-09-04T08:33:22.767Z" }, -] - -[[package]] -name = "nvidia-cusparse" -version = "12.6.3.3" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "nvidia-nvjitlink" }, -] -wheels = [ - { url = "https://files.pythonhosted.org/packages/f8/94/5c26f33738ae35276672f12615a64bd008ed5be6d1ebcb23579285d960a9/nvidia_cusparse-12.6.3.3-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:80bcc4662f23f1054ee334a15c72b8940402975e0eab63178fc7e670aa59472c", size = 162155568, upload-time = "2025-09-04T08:33:42.864Z" }, - { url = "https://files.pythonhosted.org/packages/fa/18/623c77619c31d62efd55302939756966f3ecc8d724a14dab2b75f1508850/nvidia_cusparse-12.6.3.3-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:2b3c89c88d01ee0e477cb7f82ef60a11a4bcd57b6b87c33f789350b59759360b", size = 145942937, upload-time = "2025-09-04T08:33:58.029Z" }, -] - -[[package]] -name = "nvidia-cusparselt-cu13" -version = "0.8.1" -source = { registry = "https://pypi.org/simple" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/46/e1/cdc1797eadf82d3a9a575a19b33fdc871a97edbec42c00b5b5e914f4aff4/nvidia_cusparselt_cu13-0.8.1-py3-none-manylinux2014_aarch64.whl", hash = "sha256:4dca476c50bf4780d46cd0bfbd82e2bc10a08e4fef7950917ce8d7578d22a23f", size = 221051344, upload-time = "2025-09-05T18:49:51.289Z" }, - { url = "https://files.pythonhosted.org/packages/34/7d/2661f2fb3ac4302f3a246f5fc030213ac60c1fe0bce84f9783dbd831dbb7/nvidia_cusparselt_cu13-0.8.1-py3-none-manylinux2014_x86_64.whl", hash = "sha256:786ce87568c303fadb5afcc7102d454cd3040d75f6f8626f5db460d1871f4dd0", size = 170148586, upload-time = "2025-09-05T18:50:50.248Z" }, -] - -[[package]] -name = "nvidia-nccl-cu13" -version = "2.29.7" -source = { registry = "https://pypi.org/simple" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/72/0d/daf50d44177ee0cbc7ff0a0c91eb5ff676c82be42f9a970bc7597f440c3a/nvidia_nccl_cu13-2.29.7-py3-none-manylinux_2_18_aarch64.whl", hash = "sha256:674a12383e3c38a1bcccae7d4f3633b37852230b6047883cb2f4c2d1b36d9bf5", size = 206014712, upload-time = "2026-03-03T05:34:20.843Z" }, - { url = "https://files.pythonhosted.org/packages/67/f4/58e4e91b6919367c7aafb8e36fce9aad1a3047e536bf7e2fd560927d3a4c/nvidia_nccl_cu13-2.29.7-py3-none-manylinux_2_18_x86_64.whl", hash = "sha256:edd81538446786ec3b73972543e53bb43bcaf0bfc8ef76cb679fcc390ffe136d", size = 205976000, upload-time = "2026-03-03T05:36:24.472Z" }, -] - -[[package]] -name = "nvidia-nvjitlink" -version = "13.0.88" -source = { registry = "https://pypi.org/simple" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/56/7a/123e033aaff487c77107195fa5a2b8686795ca537935a24efae476c41f05/nvidia_nvjitlink-13.0.88-py3-none-manylinux2010_x86_64.manylinux_2_12_x86_64.whl", hash = "sha256:13a74f429e23b921c1109976abefacc69835f2f433ebd323d3946e11d804e47b", size = 40713933, upload-time = "2025-09-04T08:35:43.553Z" }, - { url = "https://files.pythonhosted.org/packages/ab/2c/93c5250e64df4f894f1cbb397c6fd71f79813f9fd79d7cd61de3f97b3c2d/nvidia_nvjitlink-13.0.88-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:e931536ccc7d467a98ba1d8b89ff7fa7f1fa3b13f2b0069118cd7f47bff07d0c", size = 38768748, upload-time = "2025-09-04T08:35:20.008Z" }, -] - -[[package]] -name = "nvidia-nvshmem-cu13" -version = "3.4.5" -source = { registry = "https://pypi.org/simple" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/dc/0f/05cc9c720236dcd2db9c1ab97fff629e96821be2e63103569da0c9b72f19/nvidia_nvshmem_cu13-3.4.5-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:6dc2a197f38e5d0376ad52cd1a2a3617d3cdc150fd5966f4aee9bcebb1d68fe9", size = 60215947, upload-time = "2025-09-06T00:32:20.022Z" }, - { url = "https://files.pythonhosted.org/packages/3c/35/a9bf80a609e74e3b000fef598933235c908fcefcef9026042b8e6dfde2a9/nvidia_nvshmem_cu13-3.4.5-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:290f0a2ee94c9f3687a02502f3b9299a9f9fe826e6d0287ee18482e78d495b80", size = 60412546, upload-time = "2025-09-06T00:32:41.564Z" }, -] - -[[package]] -name = "nvidia-nvtx" -version = "13.0.85" -source = { registry = "https://pypi.org/simple" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/c2/f3/d86c845465a2723ad7e1e5c36dcd75ddb82898b3f53be47ebd429fb2fa5d/nvidia_nvtx-13.0.85-py3-none-manylinux1_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:4936d1d6780fbe68db454f5e72a42ff64d1fd6397df9f363ae786930fd5c1cd4", size = 148047, upload-time = "2025-09-04T08:29:01.761Z" }, - { url = "https://files.pythonhosted.org/packages/a8/64/3708a90d1ebe202ffdeb7185f878a3c84d15c2b2c31858da2ce0583e2def/nvidia_nvtx-13.0.85-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:cb7780edb6b14107373c835bf8b72e7a178bac7367e23da7acb108f973f157a6", size = 148878, upload-time = "2025-09-04T08:28:53.627Z" }, -] - -[[package]] -name = "openai-whisper" -version = "20250625" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "more-itertools" }, - { name = "numba" }, + { name = "flatbuffers" }, { name = "numpy" }, - { name = "tiktoken" }, - { name = "torch" }, - { name = "tqdm" }, - { name = "triton", marker = "(platform_machine == 'x86_64' and sys_platform == 'linux') or sys_platform == 'linux2'" }, + { name = "packaging" }, + { name = "protobuf" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/81/b1/d111b1df656761f980d9e298a60039a9cb66036b1d039e777537743d0ac3/onnxruntime-1.26.0-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:05b028781b322ad74b57ce5b50aa5280bb1fe96ceec334628ade681e0b24c1ac", size = 18016624, upload-time = "2026-05-12T00:41:01.735Z" }, + { url = "https://files.pythonhosted.org/packages/f6/a0/3f9d896a0385a36bd04345d6d0b802821a5782adde562e7e135f6bb71c73/onnxruntime-1.26.0-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:91f2bb870a4b9224eba0a6728c1fa7a9e552b8e59e1083c51fbbc3d013f2b5c0", size = 16052692, upload-time = "2026-05-08T19:07:13.829Z" }, + { url = "https://files.pythonhosted.org/packages/7c/43/2a4e04f8dbeffad19bbcced4bcd4289bf478921518437404d6b92bdf213b/onnxruntime-1.26.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9b6dd70599005bd1bf29779f04a91978b92b5e719c11a20068a8f8e535f725b6", size = 18185439, upload-time = "2026-05-08T19:07:36.299Z" }, + { url = "https://files.pythonhosted.org/packages/44/fc/026d0a7162b9c2153dac292baea9e027c42304dc1d9dc6f8ff5b4cfbaedd/onnxruntime-1.26.0-cp312-cp312-win_amd64.whl", hash = "sha256:a26374dc7fbcaae593601086b242120e13f2310558df0991da6dd8b8fac00414", size = 13026427, upload-time = "2026-05-08T19:08:03.503Z" }, + { url = "https://files.pythonhosted.org/packages/3e/27/1dcf88e45e4c69db5f7b106f2dacc3801ba98994e082ca03e1dfdf7bfe57/onnxruntime-1.26.0-cp312-cp312-win_arm64.whl", hash = "sha256:54a8053410fd31fd66469bd754fcfe8a4df9f7eb44756b4b5479bf50c842d948", size = 12796647, upload-time = "2026-05-08T19:07:52.108Z" }, + { url = "https://files.pythonhosted.org/packages/cf/a2/c801242685e0ce48a4ca51dfafbb588765e0446397e123be53ba5598f3f5/onnxruntime-1.26.0-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:ccce19c5f771b8268902f77d9fed9e88f9499465d6780808faa6611a789d33f0", size = 18016563, upload-time = "2026-05-08T19:07:28.081Z" }, + { url = "https://files.pythonhosted.org/packages/e2/64/0492c0b1db04e29b2630c87cfa36f9d6872b1ca8614b90c5cad58fac7d76/onnxruntime-1.26.0-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:bdbed8cf3b672b66acb032f33a253bc27f42bce6ece48ae3fab4fa483a5e96e0", size = 16052634, upload-time = "2026-05-08T19:07:16.885Z" }, + { url = "https://files.pythonhosted.org/packages/3d/26/4d09ddc755a84fc8d5e192991626b0e0680e8f6c5d58f4f1d05c42bc48cf/onnxruntime-1.26.0-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c07af6fc6d5557835f2b6ee7a96d8b3235d0c57a8e230efdedaee106a8a3cbc6", size = 18185632, upload-time = "2026-05-08T19:07:38.756Z" }, + { url = "https://files.pythonhosted.org/packages/77/89/3e52249aa08fa301e217ecba07b5246a8338fa2b401e109326e3fc5be0f9/onnxruntime-1.26.0-cp313-cp313-win_amd64.whl", hash = "sha256:61bec80655efa460591c2bc655392d57d2650ce85533a6b9b3b7a790d7ea7916", size = 13026751, upload-time = "2026-05-08T19:08:06.2Z" }, + { url = "https://files.pythonhosted.org/packages/06/b3/c1c8782b14af6797c303de132d6eef26a9fb80dfacd3750ce57911d11c6b/onnxruntime-1.26.0-cp313-cp313-win_arm64.whl", hash = "sha256:a6677545ff451e3539a02746d2f207d8c5baa4a0a818886bb9d6a6eb9511ee89", size = 12796807, upload-time = "2026-05-08T19:07:54.879Z" }, + { url = "https://files.pythonhosted.org/packages/c3/f5/47b0676408abec652c14b84d7173e389837832d850c24f87184277313e8d/onnxruntime-1.26.0-cp313-cp313t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5e016edc15d3c19f36807e1c6b10be5b27807688c32720f91b5ae480a95215d0", size = 16057265, upload-time = "2026-05-08T19:07:19.603Z" }, + { url = "https://files.pythonhosted.org/packages/3b/45/33ab6deeef010ca844c877dd618cebc079590bbe52d2a3678e7223b1b908/onnxruntime-1.26.0-cp313-cp313t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f5fc48a91a046a6a5c9b147f83fb41d65d24d24923373b222cdd248f0f4f4aac", size = 18197590, upload-time = "2026-05-08T19:07:41.422Z" }, + { url = "https://files.pythonhosted.org/packages/40/89/17546c1c20f6bfc3ae41c22152378a26edfea918af3129e2139dcd7c99f3/onnxruntime-1.26.0-cp314-cp314-macosx_14_0_arm64.whl", hash = "sha256:33a791f31432a3af1a96db5e54818b37aba5e5eefc2e6af5794c10a9118a9993", size = 18019724, upload-time = "2026-05-08T19:07:30.723Z" }, + { url = "https://files.pythonhosted.org/packages/bb/24/89457a35f6af29538a76647f2c18c3a28277e6c19234c847e7b4b7c19860/onnxruntime-1.26.0-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e90c00732c4553618103149d93f688e8c3063017938f8983e21a71d9f3b6d22e", size = 16054821, upload-time = "2026-05-08T19:07:22.348Z" }, + { url = "https://files.pythonhosted.org/packages/12/f9/15b2e1815cf570d238e0135529f80d2dce64e8e8818a1489cae83823c5c6/onnxruntime-1.26.0-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:01498e80ba8988428d08c2d51b1338f89e3de2a93e6ffe555f79c68f26a5c06b", size = 18185815, upload-time = "2026-05-08T19:07:44.179Z" }, + { url = "https://files.pythonhosted.org/packages/d7/65/2e11055faf015e4b07f45b513fa49b391baf2e19d92d77d73ebee13c1004/onnxruntime-1.26.0-cp314-cp314-win_amd64.whl", hash = "sha256:7ead61450d8405167c87dd3a31d8da1d576b490a57dab1aa8b82a7da6825f5aa", size = 13349887, upload-time = "2026-05-08T19:08:08.671Z" }, + { url = "https://files.pythonhosted.org/packages/19/e4/0f9d1a5718b1781c610c1e354765a3820597081754277a6a9a2b50705702/onnxruntime-1.26.0-cp314-cp314-win_arm64.whl", hash = "sha256:31d71a53490e46910877d0902b5ad99c69a5955e5c7ea6c82863519410e1ba7c", size = 13140121, upload-time = "2026-05-08T19:07:57.804Z" }, + { url = "https://files.pythonhosted.org/packages/1c/42/3b8e635f067d06d9f45bede470b8d539d101a4166c272213158dfd08b6ce/onnxruntime-1.26.0-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d7b6d258fb78fdfcf049795bcfaa74dcb90ae7baa277afd21e6fd28b83f2c496", size = 16057240, upload-time = "2026-05-08T19:07:25.163Z" }, + { url = "https://files.pythonhosted.org/packages/93/99/f2be40a31b908d96b861ae0ce98582fa376c18a7f816b9d5eb4cd6aa0a4c/onnxruntime-1.26.0-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4eefd386a45202aefb7a5132b94f32df9d506c9edcc7faf2fc60d65183f4b183", size = 18197382, upload-time = "2026-05-08T19:07:46.965Z" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/35/8e/d36f8880bcf18ec026a55807d02fe4c7357da9f25aebd92f85178000c0dc/openai_whisper-20250625.tar.gz", hash = "sha256:37a91a3921809d9f44748ffc73c0a55c9f366c85a3ef5c2ae0cc09540432eb96", size = 803191, upload-time = "2025-06-26T01:06:13.34Z" } [[package]] name = "packaging" @@ -1165,6 +1038,21 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538, upload-time = "2025-05-15T12:30:06.134Z" }, ] +[[package]] +name = "protobuf" +version = "7.35.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/60/fd/5b1491d9e4b586d621c54f4c36b888714164b6875f8d6afa3f9072906a51/protobuf-7.35.0.tar.gz", hash = "sha256:a2efd84605f41e559f1881b0912b44099d0a2ac9bf46b3474823f10fb393b0e6", size = 458677, upload-time = "2026-05-19T23:02:29.197Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/83/ee/93d06e358a4aa32280b00e722d3ea0a1f25fc3cc5778d80581c9cca2c10e/protobuf-7.35.0-cp310-abi3-macosx_10_9_universal2.whl", hash = "sha256:66be6c513931c794fa92c080ffee41671390da3d79da219cf9c0c0907f035dda", size = 433225, upload-time = "2026-05-19T23:02:19.884Z" }, + { url = "https://files.pythonhosted.org/packages/8b/39/1c76c2da93f3c507e958e0aecee2391cc44d4625de6c728bbc555195b5a8/protobuf-7.35.0-cp310-abi3-manylinux2014_aarch64.whl", hash = "sha256:fcbe42a4ac09d3ec9c987ddfcd956afd0b15f1ff613bd8371bde9405ffd5c8e5", size = 328847, upload-time = "2026-05-19T23:02:22.3Z" }, + { url = "https://files.pythonhosted.org/packages/91/1a/39f7ce90a238c1a987a4d81ec26379e02ca0aff367de68e4a1fa474215b9/protobuf-7.35.0-cp310-abi3-manylinux2014_s390x.whl", hash = "sha256:4cbf5cc286130e06a6c9bbefac442431173906dfcc979712183d4adcc01b37ee", size = 344030, upload-time = "2026-05-19T23:02:23.591Z" }, + { url = "https://files.pythonhosted.org/packages/70/5b/6baf9008817964454055ff3fe65f1de0b5f1e26c80c82f7fb108b7cd4ea3/protobuf-7.35.0-cp310-abi3-manylinux2014_x86_64.whl", hash = "sha256:6c0f98f10c8a05ea30f8993dfef2de093d27b490fdae78bb60c8343795d55011", size = 327130, upload-time = "2026-05-19T23:02:24.637Z" }, + { url = "https://files.pythonhosted.org/packages/8e/e5/e46adb0badc388bfb84877a5f9f026aff63f60e611016cf64dbe77e05446/protobuf-7.35.0-cp310-abi3-win32.whl", hash = "sha256:4c4617b83ade0e279d1d2bfe04025a1adb87f9ed657de038620dc0ff959357f6", size = 428946, upload-time = "2026-05-19T23:02:25.741Z" }, + { url = "https://files.pythonhosted.org/packages/a7/ab/547fbd9e16d879dd13c167478f8ae0a83a428008ca07a5e06acdc23ad473/protobuf-7.35.0-cp310-abi3-win_amd64.whl", hash = "sha256:f05bcadf9a2a6b8dda047007075135fb7d08c73d9177aabc067e1be46881a201", size = 439996, upload-time = "2026-05-19T23:02:26.808Z" }, + { url = "https://files.pythonhosted.org/packages/b8/ef/50433d346c56657a70d27f156c7b349ac59a068b01de4eb796e747eecc43/protobuf-7.35.0-py3-none-any.whl", hash = "sha256:c13f325cf242bad135c350629eeb5d54b24228eb472fb3e2e9ebbd4c5dc20ca0", size = 171659, upload-time = "2026-05-19T23:02:27.842Z" }, +] + [[package]] name = "psycopg" version = "3.3.4" @@ -1552,6 +1440,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/a0/f4/c67b0b3f1b9245e8d266f0f112c500d50e5b4e83cb6f3b71b6528104182a/requests-2.34.2-py3-none-any.whl", hash = "sha256:2a0d60c172f83ac6ab31e4554906c0f3b3588d37b5cb939b1c061f4907e278e0", size = 73075, upload-time = "2026-05-14T19:25:26.443Z" }, ] +[[package]] +name = "rich" +version = "15.0.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "markdown-it-py" }, + { name = "pygments" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c0/8f/0722ca900cc807c13a6a0c696dacf35430f72e0ec571c4275d2371fca3e9/rich-15.0.0.tar.gz", hash = "sha256:edd07a4824c6b40189fb7ac9bc4c52536e9780fbbfbddf6f1e2502c31b068c36", size = 230680, upload-time = "2026-04-12T08:24:00.75Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/82/3b/64d4899d73f91ba49a8c18a8ff3f0ea8f1c1d75481760df8c68ef5235bf5/rich-15.0.0-py3-none-any.whl", hash = "sha256:33bd4ef74232fb73fe9279a257718407f169c09b78a87ad3d296f548e27de0bb", size = 310654, upload-time = "2026-04-12T08:24:02.83Z" }, +] + [[package]] name = "ruff" version = "0.15.14" @@ -1588,7 +1489,6 @@ dependencies = [ { name = "fastapi" }, { name = "httpx" }, { name = "jinja2" }, - { name = "openai-whisper" }, { name = "psycopg", extra = ["binary"] }, { name = "psycopg-pool" }, { name = "pydantic" }, @@ -1605,6 +1505,9 @@ dependencies = [ api = [ { name = "anthropic" }, ] +tower = [ + { name = "faster-whisper" }, +] [package.dev-dependencies] dev = [ @@ -1620,9 +1523,9 @@ requires-dist = [ { name = "click", specifier = ">=8.1.0" }, { name = "embedding-chunking", git = "https://gitea.plantbasedsoutherner.com/petal-power/embedding-chunking.git?tag=v0.1.0" }, { name = "fastapi", specifier = ">=0.115.0" }, + { name = "faster-whisper", marker = "extra == 'tower'", specifier = ">=1.0.0" }, { name = "httpx", specifier = ">=0.28.0" }, { name = "jinja2", specifier = ">=3.1.0" }, - { name = "openai-whisper" }, { name = "psycopg", extras = ["binary"], specifier = ">=3.2" }, { name = "psycopg-pool", specifier = ">=3.2" }, { name = "pydantic", specifier = ">=2.0.0" }, @@ -1634,7 +1537,7 @@ requires-dist = [ { name = "uvicorn", extras = ["standard"], specifier = ">=0.30.0" }, { name = "yt-dlp", specifier = ">=2024.1.0" }, ] -provides-extras = ["api"] +provides-extras = ["api", "tower"] [package.metadata.requires-dev] dev = [ @@ -1652,6 +1555,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/e1/e3/c164c88b2e5ce7b24d667b9bd83589cf4f3520d97cad01534cd3c4f55fdb/setuptools-81.0.0-py3-none-any.whl", hash = "sha256:fdd925d5c5d9f62e4b74b30d6dd7828ce236fd6ed998a08d81de62ce5a6310d6", size = 1062021, upload-time = "2026-02-06T21:10:37.175Z" }, ] +[[package]] +name = "shellingham" +version = "1.5.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/58/15/8b3609fd3830ef7b27b655beb4b4e9c62313a4e8da8c676e142cc210d58e/shellingham-1.5.4.tar.gz", hash = "sha256:8dbca0739d487e5bd35ab3ca4b36e11c4078f3a234bfce294b0a0291363404de", size = 10310, upload-time = "2023-10-24T04:13:40.426Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e0/f9/0595336914c5619e5f28a1fb793285925a8cd4b432c9da0a987836c7f822/shellingham-1.5.4-py2.py3-none-any.whl", hash = "sha256:7ecfff8f2fd72616f7481040475a65b2bf8af90a56c89140852d1120324e8686", size = 9755, upload-time = "2023-10-24T04:13:38.866Z" }, +] + [[package]] name = "six" version = "1.17.0" @@ -1724,18 +1636,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/93/79/920b8e0a8b20f793e8d64855095cb8febabf6175b8550b6f7a547d813891/starlette-1.1.0-py3-none-any.whl", hash = "sha256:7f0dfd38e428aad5cb6f9f667f0ca1d2d8ca3f3385dccac8305f79ec98458382", size = 72899, upload-time = "2026-05-23T16:55:39.201Z" }, ] -[[package]] -name = "sympy" -version = "1.14.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "mpmath" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/83/d3/803453b36afefb7c2bb238361cd4ae6125a569b4db67cd9e79846ba2d68c/sympy-1.14.0.tar.gz", hash = "sha256:d3d3fe8df1e5a0b42f0e7bdf50541697dbe7d23746e894990c030e2b05e72517", size = 7793921, upload-time = "2025-04-27T18:05:01.611Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/a2/09/77d55d46fd61b4a135c444fc97158ef34a095e5681d0a6c10b75bf356191/sympy-1.14.0-py3-none-any.whl", hash = "sha256:e091cc3e99d2141a0ba2847328f5479b05d94a6635cb96148ccb3f34671bd8f5", size = 6299353, upload-time = "2025-04-27T18:04:59.103Z" }, -] - [[package]] name = "tiktoken" version = "0.13.0" @@ -1792,6 +1692,33 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/9e/90/39a85a4b63c84213e78b3c17d22e1bf45328acf8ebb33ef93be30d0a3911/tld-0.13.2-py2.py3-none-any.whl", hash = "sha256:9b8fdbdb880e7ba65b216a4937f2c94c49a7226723783d5838fc958ac76f4e0c", size = 296743, upload-time = "2026-03-06T23:50:32.465Z" }, ] +[[package]] +name = "tokenizers" +version = "0.23.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "huggingface-hub" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c1/60/21f715d9faba5f5407ff759472ade058ec4a507ad62bcea47cb847239a73/tokenizers-0.23.1.tar.gz", hash = "sha256:1feeeadf865a7915adc25445dea30e9933e593c31bb96c277cee36de227c8bfa", size = 365748, upload-time = "2026-04-27T14:43:25.606Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/87/39/b87a87d5bb9470610b80a2d31df42fcffeaf35118b8b97952b2aff598cc7/tokenizers-0.23.1-cp310-abi3-macosx_10_12_x86_64.whl", hash = "sha256:e03d6ffcbe0d56ee9c1ccd070e70a13fa750727c0277e138152acbc0252c2224", size = 3146732, upload-time = "2026-04-27T14:43:15.427Z" }, + { url = "https://files.pythonhosted.org/packages/e2/6a/068ed9f6e444c9d7e9d55ce134181325700f3d7f30410721bdc8f848d727/tokenizers-0.23.1-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:e0948bbb1ac1d7cdfc9fb6d62c596e3b7550036ad60ecd654a66ad273326324e", size = 3054954, upload-time = "2026-04-27T14:43:13.745Z" }, + { url = "https://files.pythonhosted.org/packages/6c/36/e006edf031154cba92b8416057d92c3abe3635e4c4b0aa0b5b9bb39dde70/tokenizers-0.23.1-cp310-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1bf13402aff9bc533c89cb849ec3b412dc3fbeacc9744840e423d7bf3f7dc0e3", size = 3374081, upload-time = "2026-04-27T14:43:01.241Z" }, + { url = "https://files.pythonhosted.org/packages/a2/ef/7735d226f9c7f874a6bee5e3f27fb25ecabdf207d37b8cf45286d0795893/tokenizers-0.23.1-cp310-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:f836ca703b89ae07919a309f9651f7a88fd5a33d5f718ba5ad0870ec0256bad6", size = 3247641, upload-time = "2026-04-27T14:43:03.856Z" }, + { url = "https://files.pythonhosted.org/packages/b9/d9/24827036f6e21297bfffda0768e58eb6096a4f411e932964a01707857931/tokenizers-0.23.1-cp310-abi3-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ae848657742035523fdf261773630cb819a26995fcd3d9ecae0c1daf6e5a4959", size = 3585624, upload-time = "2026-04-27T14:43:10.664Z" }, + { url = "https://files.pythonhosted.org/packages/0c/9a/22f3582b3a4f49358293a5206e25317621ee4526bfe9cdaa0f07a12e770e/tokenizers-0.23.1-cp310-abi3-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:53b09e85775d5187941e7bab30e941b4134ab4a7dd8c68e783d231fb7ca27c51", size = 3844062, upload-time = "2026-04-27T14:43:05.643Z" }, + { url = "https://files.pythonhosted.org/packages/7e/65/b8f8814eef95800f20721384136d9a1d22241d50b2874357cb70542c392f/tokenizers-0.23.1-cp310-abi3-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ea5a0ce170074329faaa8ea3f6400ecde604b6678192688533af80980daae71a", size = 3460098, upload-time = "2026-04-27T14:43:08.854Z" }, + { url = "https://files.pythonhosted.org/packages/0d/d5/1353e5f677ec27c2494fb6a6725e82d56c985f53e90ec511369e7e4f02c6/tokenizers-0.23.1-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5075b405006415ea148a992d093699c66eb01952bf59f4d5727089a98bda45a4", size = 3346235, upload-time = "2026-04-27T14:43:12.377Z" }, + { url = "https://files.pythonhosted.org/packages/71/89/39b6b8fc073fb6d413d0147aa333dc7eff7be65639ac9d19930a0b21bf33/tokenizers-0.23.1-cp310-abi3-manylinux_2_31_riscv64.whl", hash = "sha256:56f3a77de629917652f876294dc9fe6bad4a0c43bc229dc72e59bb23a0f4729a", size = 3426398, upload-time = "2026-04-27T14:43:07.264Z" }, + { url = "https://files.pythonhosted.org/packages/0f/80/127c854da64827e5b79264ce524993a90dddcb320e5cd42412c5c02f9e8a/tokenizers-0.23.1-cp310-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:9d10a6d957ef01896dc274e890eee27d41bd0e74ef31e60616f0fc311345184e", size = 9823279, upload-time = "2026-04-27T14:43:17.222Z" }, + { url = "https://files.pythonhosted.org/packages/fe/ba/44c2502feb1a058f096ddfb4e0996ef3225a01a388e1a9b094e91689fe93/tokenizers-0.23.1-cp310-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:1974288a609c343774f1b897c8b482c791ab17b75ab5c8c2b1737565c1d82288", size = 9644986, upload-time = "2026-04-27T14:43:19.45Z" }, + { url = "https://files.pythonhosted.org/packages/9e/c1/464019a9fb059870bfe4eebb4ba12208f3042035e258bf5e782906bd3847/tokenizers-0.23.1-cp310-abi3-musllinux_1_2_i686.whl", hash = "sha256:120468fb4c24faf0543c835a4fabafa4deb3f20a035c9b6e83d0b553a97615d4", size = 9976181, upload-time = "2026-04-27T14:43:21.463Z" }, + { url = "https://files.pythonhosted.org/packages/79/94/3ac1432bda31626071e9b6a12709b97ae05131c804b94c8f3ac622c5da32/tokenizers-0.23.1-cp310-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:e3d8f40ea6268047de7046906326abed5134f27d4e8447b23763afe5808c8a96", size = 10113853, upload-time = "2026-04-27T14:43:23.617Z" }, + { url = "https://files.pythonhosted.org/packages/6a/dd/631b21433c771b1382535326f0eca80b9c9cee2e64961dd993bc9ac4669e/tokenizers-0.23.1-cp310-abi3-win32.whl", hash = "sha256:93120a930b919416da7cd10a2f606ac9919cc69cacae7980fa2140e277660948", size = 2536263, upload-time = "2026-04-27T14:43:29.888Z" }, + { url = "https://files.pythonhosted.org/packages/97/c9/2553f72aaf65a2797d4229e37fa7fbe38ffbf3e32912d31bdd78b3323e59/tokenizers-0.23.1-cp310-abi3-win_amd64.whl", hash = "sha256:e7bfaf995c1bdbbd21d13539decb6650967013759318627d85daeb7881af16b7", size = 2798223, upload-time = "2026-04-27T14:43:28.51Z" }, + { url = "https://files.pythonhosted.org/packages/cd/2b/2be299bab55fc595e3d38567edb1a87f86e594842968fa9515a07bdcf422/tokenizers-0.23.1-cp310-abi3-win_arm64.whl", hash = "sha256:a26197957d8e4425dfba746315f3c425ea00cfa8367c5fbc4ec73447893dcea9", size = 2664127, upload-time = "2026-04-27T14:43:26.949Z" }, +] + [[package]] name = "tomli" version = "2.4.1" @@ -1837,50 +1764,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/7b/61/cceae43728b7de99d9b847560c262873a1f6c98202171fd5ed62640b494b/tomli-2.4.1-py3-none-any.whl", hash = "sha256:0d85819802132122da43cb86656f8d1f8c6587d54ae7dcaf30e90533028b49fe", size = 14583, upload-time = "2026-03-25T20:22:03.012Z" }, ] -[[package]] -name = "torch" -version = "2.12.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "cuda-bindings", marker = "sys_platform == 'linux'" }, - { name = "cuda-toolkit", extra = ["cudart", "cufft", "cufile", "cupti", "curand", "cusolver", "cusparse", "nvjitlink", "nvrtc", "nvtx"], marker = "sys_platform == 'linux'" }, - { name = "filelock" }, - { name = "fsspec" }, - { name = "jinja2" }, - { name = "networkx" }, - { name = "nvidia-cublas", marker = "sys_platform == 'linux'" }, - { name = "nvidia-cudnn-cu13", marker = "sys_platform == 'linux'" }, - { name = "nvidia-cusparselt-cu13", marker = "sys_platform == 'linux'" }, - { name = "nvidia-nccl-cu13", marker = "sys_platform == 'linux'" }, - { name = "nvidia-nvshmem-cu13", marker = "sys_platform == 'linux'" }, - { name = "setuptools" }, - { name = "sympy" }, - { name = "triton", marker = "sys_platform == 'linux'" }, - { name = "typing-extensions" }, -] -wheels = [ - { url = "https://files.pythonhosted.org/packages/ef/bb/285d643f254731294c9b595a007eac39db4600a98682d7bca688f42ca164/torch-2.12.0-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:b41339df93d491435e790ff8bcbae1c0ce777175889bfd1281d119862793e6a2", size = 88010197, upload-time = "2026-05-13T14:55:35.414Z" }, - { url = "https://files.pythonhosted.org/packages/79/81/76debf1db1343bd929bbb5d74c89fb437c2ed88eb144712557e7bd3eea45/torch-2.12.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:8fbef9f108a863e7722a73740998967e3b074742a834fc5be3a535a2befa7057", size = 426376751, upload-time = "2026-05-13T14:55:03.353Z" }, - { url = "https://files.pythonhosted.org/packages/de/f0/80026028b603c4650ff270fc3785bdef4bd6738765a9cc5a0f5a637d65a2/torch-2.12.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:4b4f64c2c2b11f7510d93dd6412b87025ff6eddd6bb61c3b5a3d892ea20c4756", size = 532261691, upload-time = "2026-05-13T14:52:54.453Z" }, - { url = "https://files.pythonhosted.org/packages/b9/c2/64b06cbb7830fb3cd9be13e1158b31a3f36b68e6a209105ee3c9d9480be0/torch-2.12.0-cp312-cp312-win_amd64.whl", hash = "sha256:8b958caff4a14d3a3b0b2dfc6a378f64dda9728a9dad28c08a0db9ce4dafb549", size = 122988114, upload-time = "2026-05-13T14:54:42.153Z" }, - { url = "https://files.pythonhosted.org/packages/86/ca/01896c80ba921676aa45886b2c5b8d774912de2a1f719de48169c6f755cd/torch-2.12.0-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:90dd587a5f61bfe1307148b581e2084fc5bc4a06e2b90a20e9a36b81087ff16b", size = 88009511, upload-time = "2026-05-13T14:54:47.411Z" }, - { url = "https://files.pythonhosted.org/packages/a5/04/52bdaf4787eab6ac7d7f5851dff934e4def0bc8ead9c8fd2b69b3e529699/torch-2.12.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:864392c73b7654f4d2b3ae712f607937d0dbb1101c4555fbb41848106b297f39", size = 426383231, upload-time = "2026-05-13T14:53:32.129Z" }, - { url = "https://files.pythonhosted.org/packages/49/8a/94bdecd13f5aaa90d45920b89789d9fe7c6f4af8c3cdd7ce01fcb59908fc/torch-2.12.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:5d6b560dfa7d56291c07d615c3bb73e8d9943d9b6d87f76cd0d9d570c4797fa6", size = 532269288, upload-time = "2026-05-13T14:53:49.423Z" }, - { url = "https://files.pythonhosted.org/packages/3e/2f/bdbaaa267de519ef1b73054bf590d8c93c37a266c9a4e24a01bd38b6918f/torch-2.12.0-cp313-cp313-win_amd64.whl", hash = "sha256:3fee918902090ade827643e758e98363278815de583c75d111fdd665ebffde9f", size = 122987706, upload-time = "2026-05-13T14:54:00.335Z" }, - { url = "https://files.pythonhosted.org/packages/9b/ad/e95e822f3538171e22640a7fbe839a1fdb666600bf6487025de2ff03b11a/torch-2.12.0-cp313-cp313t-macosx_14_0_arm64.whl", hash = "sha256:10ee1448a9f304d3b987eb4656f664ba6e4d7b410ca7a5a7c642199777a2cf88", size = 88319556, upload-time = "2026-05-13T14:54:05.574Z" }, - { url = "https://files.pythonhosted.org/packages/b7/07/055d06d985b445d67422d25b033c11cf55bbb81785d4c4e68e28bca5820e/torch-2.12.0-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:af68dbf403439cae9ceaeaaf92f8352b460787dcd27b92aa05c40dd4a19c0f1e", size = 426397656, upload-time = "2026-05-13T14:52:38.84Z" }, - { url = "https://files.pythonhosted.org/packages/43/94/b0b4fdc3014122e0a7302fb90086d352aa48f2576f0b252561ebb38c01a8/torch-2.12.0-cp313-cp313t-manylinux_2_28_x86_64.whl", hash = "sha256:a6a2eebb237d3b1d9ad3b378e86d9b9e0782afdea8b1e0eba6a13646b9b49c07", size = 532183124, upload-time = "2026-05-13T14:53:16.178Z" }, - { url = "https://files.pythonhosted.org/packages/d8/c8/052405e6ad05d3237bfe5a4df78f917773956f8e17813a2d44c059068b74/torch-2.12.0-cp313-cp313t-win_amd64.whl", hash = "sha256:2140e373e9a51a3e22ef62e8d14366d0b470d18f0adf19fdc757368077133a34", size = 123232462, upload-time = "2026-05-13T14:52:27.26Z" }, - { url = "https://files.pythonhosted.org/packages/67/dc/ac069f8d6e8be701535921141055293b0d4819d3d7f224a4612cf157c7f9/torch-2.12.0-cp314-cp314-macosx_14_0_arm64.whl", hash = "sha256:f7dfae4a519197dfa050e98d8e36378a0fb5899625a875c2b54445005a2e404e", size = 88027282, upload-time = "2026-05-13T14:53:05.258Z" }, - { url = "https://files.pythonhosted.org/packages/33/c3/1c1eb00e34555b536dddf792676026a988d710ed36981aa00499b36b0620/torch-2.12.0-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:891c769072637c74e9a5a77a3bc782894696d8ffec83b938df8536dee7f0ba78", size = 426386961, upload-time = "2026-05-13T14:51:28.406Z" }, - { url = "https://files.pythonhosted.org/packages/cd/d4/7e730dba0c7032a4154dc9056b76cf9625515e030e269cfbf8098fcfee7d/torch-2.12.0-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:e2ad3eb85d39c3cab62dfa93ed5a73516e6a53c6713cb97d004004fe089f0f1f", size = 532272265, upload-time = "2026-05-13T14:51:59.308Z" }, - { url = "https://files.pythonhosted.org/packages/f1/b4/92c80d1bbfee1c0036c06d1d2155a3065bd2423134c83bf8a47e65cd6b9b/torch-2.12.0-cp314-cp314-win_amd64.whl", hash = "sha256:c66696857e987efb8bc1777a37357ec4f60ab5e8af6250b83d6034437fa2d8f3", size = 122987138, upload-time = "2026-05-13T14:51:45.942Z" }, - { url = "https://files.pythonhosted.org/packages/7b/78/2e12b37ce50a19a037d7bc62d652a5a8f27385a7b05859d6bc9204f20cfe/torch-2.12.0-cp314-cp314t-macosx_14_0_arm64.whl", hash = "sha256:b4556715c8572758625d62b6e0ae3b1f76c440221913a6fb5e100f321fb4fb02", size = 88320100, upload-time = "2026-05-13T14:51:39.955Z" }, - { url = "https://files.pythonhosted.org/packages/56/5e/83c450ec7b0bb40a7b74611c1b5440f9260e33c54c90d556fd4a1f0fd955/torch-2.12.0-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:a43ac605a5e13116c72b64c359644cce0229f213dde48d2ae0ae5eb5becf7feb", size = 426391871, upload-time = "2026-05-13T14:52:14.989Z" }, - { url = "https://files.pythonhosted.org/packages/c9/e9/1a0b575d98d0afedd8f157d23fa3d2759421483660448e60d0a4b10b6daa/torch-2.12.0-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:6a7512adfdd7f6732e40de1c620831e3c75b39b98cef60b11d0c5f0a76473ec5", size = 532192241, upload-time = "2026-05-13T14:51:07.795Z" }, - { url = "https://files.pythonhosted.org/packages/88/21/afadd25ecd81b3cea1e11c73cf1ab41a983a50271548c3ec7ec3b9efc3e9/torch-2.12.0-cp314-cp314t-win_amd64.whl", hash = "sha256:5f96b63f8287f66a005dd1b5a6abba2920f11156c5e5c4d815f3e2050fd1aa16", size = 123231092, upload-time = "2026-05-13T14:51:18.854Z" }, -] - [[package]] name = "tqdm" version = "4.67.3" @@ -1912,20 +1795,18 @@ wheels = [ ] [[package]] -name = "triton" -version = "3.7.0" +name = "typer" +version = "0.25.1" source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "annotated-doc" }, + { name = "click" }, + { name = "rich" }, + { name = "shellingham" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e4/51/9aed62104cea109b820bbd6c14245af756112017d309da813ef107d42e7e/typer-0.25.1.tar.gz", hash = "sha256:9616eb8853a09ffeabab1698952f33c6f29ffdbceb4eaeecf571880e8d7664cc", size = 122276, upload-time = "2026-04-30T19:32:16.964Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/f7/13/ec05adfcd87311d532ba61e3af143e8be59fcd26675884c4682841406a20/triton-3.7.0-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a4bf49b00a7a377a68a6da603a876e797614e6455a80e9021669c476a953ad9a", size = 188505104, upload-time = "2026-05-07T19:05:09.843Z" }, - { url = "https://files.pythonhosted.org/packages/62/7b/468a576e35beef1426e0828e28e9ba9e65f5474d496f16ee126c15646324/triton-3.7.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8f111161d49bf903c0eaedde3962353a3d841c08a836839b7cc1025b8426efcf", size = 201457567, upload-time = "2026-05-07T18:46:13.505Z" }, - { url = "https://files.pythonhosted.org/packages/01/e1/a59a583de59b8f62c495d67c80ee3ea97d09e91ac80c4c6e76456ed8d8ac/triton-3.7.0-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:abdf6beaa89b1bcfb9a43cd990536ce66091a997841a4814b260b7bee4c88c3c", size = 188503209, upload-time = "2026-05-07T19:05:17.935Z" }, - { url = "https://files.pythonhosted.org/packages/30/b1/b7507bb9815d403927c8dd51d4158ed2e11751a92dbc118a044f247b6848/triton-3.7.0-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a35d7afe3f3f058e7ec49fcce09794049e0ffc5c59019ac25ec3413741b8c4e7", size = 201453566, upload-time = "2026-05-07T18:46:20.427Z" }, - { url = "https://files.pythonhosted.org/packages/a6/8f/0bea7a6a0c989315c9135a1d7fb37e41905cfb3a17cbc1f10044ebd4cc3a/triton-3.7.0-cp313-cp313t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cc1d61c172d257db80ddf42595131fb196ad2e9bdd751e90fe2ef13531734e8b", size = 188612899, upload-time = "2026-05-07T19:05:24.955Z" }, - { url = "https://files.pythonhosted.org/packages/e1/02/d96f57828d0912aec733b9bc7e0e7dbfd2c6f079a8fa433ac25cb93d1a30/triton-3.7.0-cp313-cp313t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:70fb9bbdc9f400afc54bbf6eb2670af28829a6ae3996863317964783141daf56", size = 201553816, upload-time = "2026-05-07T18:46:27.49Z" }, - { url = "https://files.pythonhosted.org/packages/40/fb/82a802dac4689f2a2fb2e69302e6a138eecc3e175bbe976ba3cfc717683a/triton-3.7.0-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c4a44a8476d0d3571eac4e4d1048e1ff75aad81a09ff4602ccfc56c6dea1672e", size = 188507879, upload-time = "2026-05-07T19:05:32.209Z" }, - { url = "https://files.pythonhosted.org/packages/8f/af/9904ec6d3c93d9b24e5ec360445bbdf758b7f00bfbeedb89cb0eb64eb8bb/triton-3.7.0-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b9b85e72968a9d8bba5ddb24e9b64aaabaf48affb042f2755cb7cfa92b7531ce", size = 201460637, upload-time = "2026-05-07T18:46:34.749Z" }, - { url = "https://files.pythonhosted.org/packages/a1/f9/4835a8ea746b88727d8899f4e3ccce4f9cacb38abfc3bb0a638266c53111/triton-3.7.0-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:18a160de426fd99f92b0baf509045360afbd3bfaa0b4a5171dde800ec9f09684", size = 188608706, upload-time = "2026-05-07T19:05:39.218Z" }, - { url = "https://files.pythonhosted.org/packages/c1/68/fa86e5a39608000f645535b2c124920126327ab731f8c4fafd5b07ff8d4b/triton-3.7.0-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ce061073102714b725f3660ec6939d94a1da7984b3aa99c921417cae273672f5", size = 201546766, upload-time = "2026-05-07T18:46:42.088Z" }, + { url = "https://files.pythonhosted.org/packages/3f/f9/2b3ff4e56e5fa7debfaf9eb135d0da96f3e9a1d5b27222223c7296336e5f/typer-0.25.1-py3-none-any.whl", hash = "sha256:75caa44ed46a03fb2dab8808753ffacdbfea88495e74c85a28c5eefcf5f39c89", size = 58409, upload-time = "2026-04-30T19:32:18.271Z" }, ] [[package]] From 913d4f0335141270f2a4460a7bc62419bb1a7f07 Mon Sep 17 00:00:00 2001 From: Travis Herbranson Date: Mon, 25 May 2026 10:20:46 -0400 Subject: [PATCH 11/13] deploy/tower + claim race + transcribe smoke tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit deploy/tower/: - second-brain-transcribe.service — systemd unit. User=herbyadmin, Type=simple, After=/Wants= wg-quick@wg-lan.service so the WG tunnel must come up first. Restart=always with a StartLimitBurst guard. - second-brain-transcribe.env.example — env file template documenting the SECOND_BRAIN_DATABASE_URL form for db.wg.herbylab.dev (10.99.0.1) and the optional WHISPER_* overrides. - README.md — EndeavourOS install steps (nvidia/cuda/cudnn, ffmpeg, uv + tower extra, model pre-warm), WG topology reference, validation checklist for what to confirm once the tunnel is live, and a follow-ups section flagging the local-disk → NAS media migration as out-of-scope-for-this-round. Tests: - tests/test_claim.py — live-DB race test. Two threads call claim_next_source against a single PULLED video row; SKIP LOCKED must give exactly one of them the row, the other gets None. Also asserts the claimed_by/at columns land + release nulls them. Auto-skips when no SECOND_BRAIN_DATABASE_URL is set. - tests/test_transcribe.py — pure-Python coverage of resolve_settings (cpu→int8, cuda→float16, env-over-block) and write_srt; plus a CPU smoke test that synthesizes a numpy audio array and runs the `tiny` model on cpu/int8 (auto-skipped when faster-whisper isn't installed, i.e. on the dev side without --extra tower). --- deploy/tower/README.md | 167 ++++++++++++++++++ .../tower/second-brain-transcribe.env.example | 24 +++ deploy/tower/second-brain-transcribe.service | 46 +++++ tests/test_claim.py | 107 +++++++++++ tests/test_transcribe.py | 142 +++++++++++++++ 5 files changed, 486 insertions(+) create mode 100644 deploy/tower/README.md create mode 100644 deploy/tower/second-brain-transcribe.env.example create mode 100644 deploy/tower/second-brain-transcribe.service create mode 100644 tests/test_claim.py create mode 100644 tests/test_transcribe.py diff --git a/deploy/tower/README.md b/deploy/tower/README.md new file mode 100644 index 0000000..9b391e7 --- /dev/null +++ b/deploy/tower/README.md @@ -0,0 +1,167 @@ +# Tower-side transcribe worker — deploy notes + +Target host: **EndeavourOS tower** (Arch base), RTX 3080. Runs the +`second-brain transcribe-worker` daemon under systemd. Reaches the +petalbrain Postgres over WireGuard at `db.wg.herbylab.dev:5432`. + +This worker only does **pull + transcribe** for VIDEO sources. Articles +and all extraction/embedding stay on the dev side. + +--- + +## 1. System packages + +```bash +# Build / runtime tooling. +sudo pacman -S --needed base-devel git ffmpeg python uv + +# NVIDIA — driver, CUDA, cuDNN. faster-whisper / CTranslate2 needs cuDNN +# at runtime; missing-symbol errors at first transcribe are usually a +# cuDNN install gap. Use whichever driver/cuda channel matches your +# Arch derivative; on stock EndeavourOS the AUR/community packages work: +sudo pacman -S --needed nvidia nvidia-utils cuda cudnn + +# Confirm the GPU is visible. +nvidia-smi +``` + +yt-dlp comes in via the Python project (`uv sync` step below), so the +distro yt-dlp is *optional*. Leave it off the host unless you want it on +your CLI PATH. + +--- + +## 2. Project checkout + venv + +```bash +sudo mkdir -p /opt/projects/homelab +sudo chown herbyadmin:herbyadmin /opt/projects/homelab +cd /opt/projects/homelab +git clone gitea-lovebug:petal-power/second-brain.git +cd second-brain + +# Install the runtime deps + the `tower` extra (faster-whisper / CTranslate2). +# The dev side does NOT install --extra tower; only the tower needs it. +uv sync --extra tower +``` + +`faster-whisper` will pull a couple hundred MB of CUDA wheels. First-run +model download (~3 GB for large-v3) lands under `~/.cache/huggingface/` +the first time the worker transcribes; pre-warm it if you want: + +```bash +uv run python -c "from faster_whisper import WhisperModel; WhisperModel('large-v3', device='cuda', compute_type='float16')" +``` + +--- + +## 3. WireGuard + +The worker assumes the tunnel is already configured and that +`wg-quick@wg-lan.service` brings it up at boot. Topology: + +| Peer | WG IP | +|------|-------| +| hub (herbydev) | `10.99.0.1` | +| tower | `10.99.0.2` | + +`db.wg.herbylab.dev` resolves to `10.99.0.1`. Confirm connectivity from +the tower: + +```bash +ping -c2 db.wg.herbylab.dev +nc -vz db.wg.herbylab.dev 5432 +``` + +Tunnel setup itself is **out of scope for this README** — that's +configured separately as part of Travis's infra. + +--- + +## 4. EnvironmentFile + +```bash +sudo install -m 0600 -o root -g root \ + deploy/tower/second-brain-transcribe.env.example \ + /etc/default/second-brain-transcribe +sudo $EDITOR /etc/default/second-brain-transcribe # paste the lovebug password +``` + +The lovebug password lives in `/opt/backups/postgres-consolidation/credentials.env` +on the dev host. Copy it across through whatever channel you trust +(personal Bitwarden, scp over WG, etc.) — do not commit it. + +--- + +## 5. systemd unit + +```bash +sudo install -m 0644 deploy/tower/second-brain-transcribe.service \ + /etc/systemd/system/second-brain-transcribe.service +sudo systemctl daemon-reload +sudo systemctl enable --now second-brain-transcribe.service +``` + +The unit declares `After=` + `Wants=` on `wg-quick@wg-lan.service`, so +the worker starts after the tunnel comes up at boot. If the tunnel +disappears mid-run the worker keeps polling and logs DB-unreachable +warnings; it doesn't crash-loop. + +### Day-to-day commands + +```bash +# What's it doing right now? +systemctl status second-brain-transcribe.service +journalctl -u second-brain-transcribe.service -f + +# Pause without uninstalling — toggle from the dashboard at +# https://brain.herbylab.dev/settings (or whichever host). +# Or stop the unit: +sudo systemctl stop second-brain-transcribe.service + +# Pick up code updates: +cd /opt/projects/homelab/second-brain +git pull +uv sync --extra tower +sudo systemctl restart second-brain-transcribe.service +``` + +--- + +## 6. Validation — what to confirm once WG is live + +These can only be verified on the tower itself: + +1. **WG reachability.** `nc -vz db.wg.herbylab.dev 5432` succeeds. +2. **DB auth.** `uv run alembic current` returns the latest revision id. +3. **GPU large-v3 path.** With a short test video queued via + `second-brain add ` on the dev side, run `uv run second-brain transcribe-worker --once` + on the tower. Watch `journalctl` — first run pulls the model from + HuggingFace (slow), subsequent runs are fast. The source row should + end up in `TRANSCRIBED` with `transcript_text` populated and + `claimed_by` cleared. +4. **Race safety smoke test.** Run two `--once` invocations in parallel + against an empty queue, then with one PENDING video. Only one should + claim it; the other should see no work. +5. **Settings round-trip.** Flip `transcription_enabled` off on the + dashboard; within one poll cycle (~15s) the worker logs + `transcription_enabled=false — sleeping`. Flip back on, the next + PENDING video gets picked up. + +--- + +## Follow-ups (out of scope for this round) + +- **Media + subtitles directory currently lives on the tower's local + disk** (under `~/.local/share/second-brain/`). Move to a NAS mount + later so the dev side can compile/inspect the SRTs without a + cross-machine fetch. Open question on the right mount point + cred + flow — see the project's `postgres-migration-planning.md` for the + related vault-location decision Travis still owes. +- **Tower-side observability.** No metrics endpoint yet; journald is + the only signal. A `/api/worker-status` heartbeat the dashboard could + read would be nice once two workers exist. +- **GPU concurrency > 1.** `transcription_max_concurrent_gpu_jobs` + exists in `pipeline_settings` but the worker only runs one job at a + time. Scale-out would need a per-job semaphore (or just running N + worker instances with distinct `SECOND_BRAIN_WORKER_ID`s). diff --git a/deploy/tower/second-brain-transcribe.env.example b/deploy/tower/second-brain-transcribe.env.example new file mode 100644 index 0000000..2371c8c --- /dev/null +++ b/deploy/tower/second-brain-transcribe.env.example @@ -0,0 +1,24 @@ +# /etc/default/second-brain-transcribe +# +# Copy to /etc/default/second-brain-transcribe, chmod 0600, chown root:root +# (systemd reads it before dropping to User=herbyadmin so a non-readable +# permissions mode is fine — the herbyadmin user never touches this file). + +# REQUIRED — petalbrain Postgres reached over the WireGuard tunnel. +# Hub-side hostname is db.wg.herbylab.dev (10.99.0.1); the tower is 10.99.0.2. +# No TLS — WireGuard already encrypts, and lovebug doesn't require SSL. +SECOND_BRAIN_DATABASE_URL=postgresql+psycopg://lovebug:REPLACE_ME@db.wg.herbylab.dev:5432/petalbrain + +# OPTIONAL — only set if the tower has a non-standard whisper layout. +# Defaults from settings.toml: model=large-v3, device=cuda, compute_type=float16. +# WHISPER_MODEL=large-v3 +# WHISPER_DEVICE=cuda +# WHISPER_COMPUTE_TYPE=float16 + +# OPTIONAL — identifier stamped into sources.claimed_by (default tower:). +# Useful if you ever run multiple tower workers and want to tell them apart. +# SECOND_BRAIN_WORKER_ID=tower-main + +# OPTIONAL — bump worker log verbosity. INFO is the default. +# PYTHONUNBUFFERED=1 +# SECOND_BRAIN_LOG_LEVEL=DEBUG diff --git a/deploy/tower/second-brain-transcribe.service b/deploy/tower/second-brain-transcribe.service new file mode 100644 index 0000000..33805d9 --- /dev/null +++ b/deploy/tower/second-brain-transcribe.service @@ -0,0 +1,46 @@ +[Unit] +Description=second-brain transcribe worker (tower-side, faster-whisper on GPU) +Documentation=https://gitea.plantbasedsoutherner.com/petal-power/second-brain +# The worker reaches petalbrain over WireGuard, so the tunnel must be up first. +# `Wants=` is the soft form — if the tunnel disappears later the worker logs +# DB-unreachable warnings and retries, it doesn't crash-loop. +After=network-online.target wg-quick@wg-lan.service +Wants=network-online.target wg-quick@wg-lan.service + +[Service] +Type=simple +User=herbyadmin +Group=herbyadmin + +# Adjust to wherever you `git clone`d the repo on the tower. +WorkingDirectory=/opt/projects/homelab/second-brain + +# Env (SECOND_BRAIN_DATABASE_URL, optional knobs). 0600, owned by herbyadmin. +EnvironmentFile=/etc/default/second-brain-transcribe + +# `uv run` resolves the project venv. Keep the worker in the foreground so +# systemd owns the lifecycle; the worker handles SIGTERM/SIGINT cleanly and +# exits after the current iteration finishes. +ExecStart=/usr/bin/uv run second-brain transcribe-worker + +Restart=always +RestartSec=10 + +# Crash loop guard — systemd will give up after this if the unit keeps +# dying. The worker itself logs+backs-off on DB outages so this only +# trips on genuine failure. +StartLimitIntervalSec=300 +StartLimitBurst=5 + +# A few sandboxing nice-to-haves. Loose on purpose — the worker needs +# /dev/nvidia* for CUDA and writes to user-owned media/subtitles dirs. +ProtectSystem=full +ProtectHome=no +NoNewPrivileges=true + +# Stdout/stderr → journald. +StandardOutput=journal +StandardError=journal + +[Install] +WantedBy=multi-user.target diff --git a/tests/test_claim.py b/tests/test_claim.py new file mode 100644 index 0000000..195516b --- /dev/null +++ b/tests/test_claim.py @@ -0,0 +1,107 @@ +"""Live-DB test: claim_next_source is race-safe against concurrent callers. + +Skips when no DB URL is configured. This is the integration-level guard +for the SELECT ... FOR UPDATE SKIP LOCKED dance — unit-testing the SQL +without Postgres would prove nothing. +""" + +from __future__ import annotations + +import os +import threading +import uuid + +import psycopg +import pytest + + +def _db_reachable() -> bool: + url = os.environ.get("SECOND_BRAIN_DATABASE_URL") or os.environ.get( + "HERBYLAB_DATABASE_URL" + ) + if not url: + return False + raw = url.replace("postgresql+psycopg://", "postgresql://", 1) + try: + with psycopg.connect(raw, connect_timeout=2): + return True + except Exception: + return False + + +pytestmark = pytest.mark.skipif( + not _db_reachable(), + reason="claim race test needs SECOND_BRAIN_DATABASE_URL pointed at a real petalbrain", +) + + +def test_two_workers_never_grab_the_same_row(): + from second_brain.claim import claim_next_source, release_claim + from second_brain.database import Database, reset_database_singleton + from second_brain.models import Source, SourceStatus, SourceType, utcnow + + reset_database_singleton() + db = Database() + + # Seed a single TRANSCRIBED-stage candidate. + unique = uuid.uuid4().hex[:8] + url = f"https://example.test/claim-race/{unique}" + with db.session() as sess: + src = Source( + url=url, + title=f"claim-race-{unique}", + domain="development", + source_type=SourceType.VIDEO, + status=SourceStatus.PULLED, + ingested_at=utcnow(), + ) + sess.add(src) + sess.flush() + source_id = src.id + + results: list[int | None] = [] + lock = threading.Lock() + + def worker(name: str): + with db.session() as sess: + claimed = claim_next_source( + sess, + worker_id=name, + statuses=[SourceStatus.PENDING, SourceStatus.PULLED], + source_type=SourceType.VIDEO, + ) + with lock: + results.append(claimed) + + t1 = threading.Thread(target=worker, args=("worker-A",)) + t2 = threading.Thread(target=worker, args=("worker-B",)) + t1.start() + t2.start() + t1.join() + t2.join() + + try: + # Exactly one worker should have grabbed the row; the other gets None. + # (SKIP LOCKED guarantees this even if both transactions overlap.) + non_none = [r for r in results if r is not None] + assert non_none == [source_id], ( + f"expected exactly one claimant of {source_id}, got results={results}" + ) + + # Confirm claim columns landed. + with db.session() as sess: + row = sess.get(Source, source_id) + assert row.claimed_by in {"worker-A", "worker-B"} + assert row.claimed_at is not None + + # Releasing the claim should null both columns. + with db.session() as sess: + release_claim(sess, source_id) + with db.session() as sess: + row = sess.get(Source, source_id) + assert row.claimed_by is None + assert row.claimed_at is None + finally: + with db.session() as sess: + sess.query(Source).filter(Source.id == source_id).delete() + reset_database_singleton() diff --git a/tests/test_transcribe.py b/tests/test_transcribe.py new file mode 100644 index 0000000..4ce4050 --- /dev/null +++ b/tests/test_transcribe.py @@ -0,0 +1,142 @@ +"""Unit + integration coverage for second_brain.transcribe. + +Unit-level: resolve_settings + write_srt are pure-Python and run anywhere. + +Integration: an end-to-end CPU/int8 transcribe of a synthesized audio +clip — auto-skipped when ffmpeg or faster-whisper is missing (the dev +side doesn't install faster-whisper by default). Only the *tower* with +GPU large-v3 can validate the production hot path; this test proves the +wiring + the SRT contract + the lazy-import path on cheap CPU. +""" + +from __future__ import annotations + +import os +import shutil +import subprocess +import tempfile +from pathlib import Path + +import pytest + + +# --------------------------------------------------------------------------- +# Pure helpers +# --------------------------------------------------------------------------- + + +def test_resolve_settings_picks_int8_for_cpu_by_default(): + from second_brain.transcribe import resolve_settings + + out = resolve_settings({"whisper_device": "cpu"}) + assert out["device"] == "cpu" + assert out["compute_type"] == "int8" + assert out["model"] == "large-v3" + + +def test_resolve_settings_picks_float16_for_cuda_by_default(): + from second_brain.transcribe import resolve_settings + + out = resolve_settings({"whisper_device": "cuda"}) + assert out["device"] == "cuda" + assert out["compute_type"] == "float16" + + +def test_resolve_settings_env_overrides_block(monkeypatch): + from second_brain.transcribe import resolve_settings + + monkeypatch.setenv("WHISPER_MODEL", "small") + monkeypatch.setenv("WHISPER_DEVICE", "cpu") + monkeypatch.setenv("WHISPER_COMPUTE_TYPE", "int8_float16") + out = resolve_settings({"whisper_device": "cuda"}) + assert out["model"] == "small" + assert out["device"] == "cpu" + assert out["compute_type"] == "int8_float16" + + +def test_write_srt_roundtrip(tmp_path): + from second_brain.transcribe import write_srt + + segs = [ + {"start": 0.0, "end": 1.5, "text": "hello"}, + {"start": 1.5, "end": 3.25, "text": "world"}, + ] + out = tmp_path / "x.srt" + write_srt(segs, out) + body = out.read_text() + assert "1\n00:00:00,000 --> 00:00:01,500\nhello" in body + assert "2\n00:00:01,500 --> 00:00:03,250\nworld" in body + + +# --------------------------------------------------------------------------- +# Integration: synthesized audio → CPU/int8 faster-whisper +# --------------------------------------------------------------------------- + + +def _have_faster_whisper() -> bool: + try: + import faster_whisper # type: ignore[import-not-found] # noqa: F401 + except ImportError: + return False + return True + + +def _have_numpy() -> bool: + try: + import numpy # noqa: F401 + except ImportError: + return False + return True + + +@pytest.mark.skipif( + not (_have_faster_whisper() and _have_numpy()), + reason="needs faster-whisper (uv sync --extra tower) + numpy to exercise CPU path", +) +def test_cpu_transcribe_smoke(tmp_path): + """Hand faster-whisper a synthesized audio array on CPU/int8 with `tiny`. + + Pure-tone input gives empty / minimal recognition — the point is the + wiring: model loads, the segments iterator drains, an SRT is writable. + Using `tiny` to keep this under ~30s coldcache. Skipped on dev where + faster-whisper isn't installed; only the tower's `uv sync --extra + tower` brings it in. + + Bypassing ffmpeg by passing a numpy array — faster-whisper.transcribe + accepts (path | file-like | np.ndarray). This exercises the same code + path the worker uses, only with zero external binaries. + """ + import numpy as np + + from second_brain.transcribe import write_srt + + # Build 2s of 440Hz mono at 16kHz, the model's native sample rate. + sr = 16000 + duration = 2.0 + t = np.arange(0, int(sr * duration)) / sr + audio = (0.2 * np.sin(2 * np.pi * 440 * t)).astype(np.float32) + + # Load `tiny` to keep this cheap; we're not asserting transcription + # quality, just that the pipeline executes and returns the documented + # shape. + from faster_whisper import WhisperModel # type: ignore[import-not-found] + + model = WhisperModel("tiny", device="cpu", compute_type="int8") + segments_iter, _ = model.transcribe(audio, language="en", beam_size=1) + + # Materialize to mirror what Transcriber.transcribe does internally. + segments: list[dict] = [] + text_parts: list[str] = [] + for seg in segments_iter: + s = (seg.text or "").strip() + segments.append({"start": float(seg.start), "end": float(seg.end), "text": s}) + if s: + text_parts.append(s) + text = " ".join(text_parts) + + assert isinstance(text, str) + assert isinstance(segments, list) + + srt = tmp_path / "out.srt" + write_srt(segments, srt) + assert srt.exists() From f9feb0b393bacd650c7760b6f6f26d761f81c55c Mon Sep 17 00:00:00 2001 From: Travis Herbranson Date: Mon, 25 May 2026 10:34:54 -0400 Subject: [PATCH 12/13] =?UTF-8?q?gauntlet:=20fix=20lint=20=E2=80=94=20drop?= =?UTF-8?q?=20unused=20subprocess/shutil/tempfile/Path=20imports?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Leftover ffmpeg-era imports from the earlier draft of the CPU transcribe smoke test; the final numpy-array-bypass version doesn't need them. Ruff autofix. --- tests/test_transcribe.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/tests/test_transcribe.py b/tests/test_transcribe.py index 4ce4050..d75a624 100644 --- a/tests/test_transcribe.py +++ b/tests/test_transcribe.py @@ -11,11 +11,6 @@ wiring + the SRT contract + the lazy-import path on cheap CPU. from __future__ import annotations -import os -import shutil -import subprocess -import tempfile -from pathlib import Path import pytest From 597c83649cee31cb1b2e9c939c99f349e3657573 Mon Sep 17 00:00:00 2001 From: Travis Herbranson Date: Mon, 25 May 2026 11:48:37 -0400 Subject: [PATCH 13/13] deploy/web: containerized FastAPI UI on the homelab docker network MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Dockerfile, compose.yml, env template, and runbook for the second-brain web container. Targets herbys-dev (10.0.21.207); Traefik (file-provider on 10.0.11.20) reaches the published host port at 10.0.21.207:8080. Image: - python:3.12-slim + the official uv binary copied from the upstream image, plus apt-installed git + ca-certificates so the Gitea VCS pin for embedding-chunking resolves at build time. - uv sync --frozen --no-dev --no-install-project, source copy, then a second uv sync to install the project itself. Two-step so the lock install layer caches independently of source edits. - No ffmpeg / claude CLI / faster-whisper — web role doesn't need any of them. Extraction runs on the dev host's CLI; transcription on the tower. - Drops to uid 1000 (`app`) before CMD. Uvicorn binds 0.0.0.0:8000 inside the container, with --proxy-headers + --forwarded-allow-ips=* so Traefik's X-Forwarded-* survive. compose.yml: - Joins the existing external `homelab` bridge network so the container reaches homelab-postgres:5432 and ollama:11434 by service DNS. - Publishes the uvicorn port at 10.0.21.207:8080 (LAN IP bound, not 0.0.0.0) for Traefik on the separate VM to reach. NO traefik.* labels — file-provider Traefik can't read them. - env_file: .env (0600, gitignored) — SECOND_BRAIN_DATABASE_URL points at homelab-postgres:5432 (containerised), NOT the host's 127.0.0.1:5433 port-map. - restart: unless-stopped. README.md: - Build + bring-up commands. - SECURITY note: no SSO / no CSRF / mutating endpoints — Traefik route must be LAN-only for now (Travis's call). - The two infra steps Travis applies himself, with ready-to-paste snippets: - Knot DNS: brain.herbylab.dev → 10.0.11.20. - Traefik dynamic config: file-provider router + service block. - Verification checklist for both before-and-after-DNS states. Live-verified on herbys-dev: container Up, dashboard returns 200 with real status counts from petalbrain, settings save round-trip works, no tracebacks in logs. --- deploy/web/.env.example | 20 +++++ deploy/web/.gitignore | 2 + deploy/web/Dockerfile | 63 ++++++++++++++++ deploy/web/README.md | 159 ++++++++++++++++++++++++++++++++++++++++ deploy/web/compose.yml | 41 +++++++++++ 5 files changed, 285 insertions(+) create mode 100644 deploy/web/.env.example create mode 100644 deploy/web/.gitignore create mode 100644 deploy/web/Dockerfile create mode 100644 deploy/web/README.md create mode 100644 deploy/web/compose.yml diff --git a/deploy/web/.env.example b/deploy/web/.env.example new file mode 100644 index 0000000..fb6729a --- /dev/null +++ b/deploy/web/.env.example @@ -0,0 +1,20 @@ +# deploy/web/.env — runtime secrets for the second-brain-web container. +# +# Copy this to deploy/web/.env, fill in REPLACE_ME, chmod 0600. The real +# .env is gitignored (see deploy/web/.gitignore). Source of truth for +# the lovebug password: /opt/backups/postgres-consolidation/credentials.env +# on herbys-dev (mode 0600, host-only). + +# REQUIRED. Containerised connection — the web container reaches Postgres +# by docker-network DNS name, NOT via the host's 127.0.0.1:5433 publish. +SECOND_BRAIN_DATABASE_URL=postgresql+psycopg://lovebug:REPLACE_ME@homelab-postgres:5432/petalbrain + +# OPTIONAL. The dashboard does not currently embed search queries, but +# the embedding module reads this lazily. Default targets the homelab +# `ollama` container. +OLLAMA_URL=http://ollama:11434 +EMBEDDING_MODEL=nomic-embed-text + +# OPTIONAL. Pool sizing — single-instance web doesn't need much. +# SECOND_BRAIN_DB_POOL_MIN=1 +# SECOND_BRAIN_DB_POOL_MAX=4 diff --git a/deploy/web/.gitignore b/deploy/web/.gitignore new file mode 100644 index 0000000..efdaaaa --- /dev/null +++ b/deploy/web/.gitignore @@ -0,0 +1,2 @@ +# Real .env carries the lovebug Postgres password. Never commit. +.env diff --git a/deploy/web/Dockerfile b/deploy/web/Dockerfile new file mode 100644 index 0000000..ed51e1e --- /dev/null +++ b/deploy/web/Dockerfile @@ -0,0 +1,63 @@ +# second-brain web container — review/dashboard UI only. +# +# Slim by design: this image runs `uvicorn` against second_brain.web.app. +# It does NOT need ffmpeg, the claude CLI, faster-whisper, or any GPU +# tooling. Extraction runs on the dev host's CLI; transcription runs on +# the tower. The web role is just "FastAPI + psycopg + Jinja templates". + +FROM python:3.12-slim AS base + +ENV PYTHONUNBUFFERED=1 \ + PIP_DISABLE_PIP_VERSION_CHECK=1 \ + UV_LINK_MODE=copy \ + UV_PROJECT_ENVIRONMENT=/app/.venv + +# uv is fetched as a static binary from its official image so we don't +# pull a full python stack to install it. +COPY --from=ghcr.io/astral-sh/uv:latest /uv /uvx /usr/local/bin/ + +# `git` is needed at *build* time because pyproject.toml pins +# embedding-chunking to a Gitea VCS source. `ca-certificates` so the +# https handshake validates. Slim image to clean up apt afterwards. +RUN apt-get update \ + && apt-get install -y --no-install-recommends git ca-certificates \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /app + +# Project metadata first so the layer cache survives source edits. +COPY pyproject.toml uv.lock ./ + +# `--frozen` enforces lockfile parity. `--no-dev` excludes pytest/ruff. +# `--no-install-project` so we don't try to install the (not-yet-copied) +# source tree in this layer. +RUN uv sync --frozen --no-dev --no-install-project + +# Source + runtime config. config/settings.toml.example is committed; a +# real settings.toml gets bind-mounted in by compose at runtime. +COPY src/ ./src/ +COPY config/ ./config/ +COPY alembic/ ./alembic/ +COPY alembic.ini ./alembic.ini + +# Now install the project itself into the venv (puts the `second-brain` +# console script + the second_brain package on PATH). +RUN uv sync --frozen --no-dev + +# Drop privileges. The image doesn't write to the host bind-mounts; the +# settings.toml mount is read-only. +RUN useradd --uid 1000 --create-home --shell /sbin/nologin app \ + && chown -R app:app /app +USER app + +EXPOSE 8000 + +# Bind to all interfaces inside the container — the host port publish +# decides who can reach it from outside. `--proxy-headers` lets Traefik +# (off-box on 10.0.11.20) forward X-Forwarded-* correctly. +CMD ["/app/.venv/bin/uvicorn", \ + "second_brain.web.app:app", \ + "--host", "0.0.0.0", \ + "--port", "8000", \ + "--proxy-headers", \ + "--forwarded-allow-ips=*"] diff --git a/deploy/web/README.md b/deploy/web/README.md new file mode 100644 index 0000000..eef62a5 --- /dev/null +++ b/deploy/web/README.md @@ -0,0 +1,159 @@ +# second-brain web app — deploy notes + +Target host: **herbys-dev** (10.0.21.207). Runs the FastAPI + Jinja UI +under `docker compose`, on the existing `homelab` bridge network so it +can reach `homelab-postgres:5432` and `ollama:11434` by service DNS. + +Traefik (file-provider VM at **10.0.11.20**) handles TLS + the public +hostname `brain.herbylab.dev`. The compose file does NOT carry +`traefik.*` labels — file-provider Traefik can't see them. + +--- + +## ⚠ Security note (no SSO) + +Travis opted **not** to put this behind Authentik for the initial roll-out. +The web app has **no authentication, no CSRF, no rate limiting**, and it +exposes mutating endpoints (accept/reject sources, edit pipeline settings). + +→ The Traefik route should be reachable **from LAN / trusted networks +only**. If you ever publish this past Cloudflare for off-LAN access, gate +it behind the Authentik forward-auth middleware first, or add an +`auth_required` decorator to the FastAPI app. + +--- + +## 1. One-time setup on herbys-dev + +```bash +cd /opt/projects/homelab/second-brain/deploy/web + +# Real env file — paste the lovebug password. +cp .env.example .env +chmod 0600 .env +$EDITOR .env +# SECOND_BRAIN_DATABASE_URL must use host=homelab-postgres (NOT 127.0.0.1). +# Password lives at /opt/backups/postgres-consolidation/credentials.env. + +# Build + start. Compose will join the existing external `homelab` net. +docker compose up -d --build + +# Confirm it's healthy. +docker compose ps +docker compose logs --tail 50 +curl -fsS http://10.0.21.207:8080/dashboard | head +``` + +Day-to-day: + +```bash +docker compose logs -f # tail +docker compose restart # bounce (config reload) +docker compose pull && docker compose up -d # if image were registry-hosted +git pull && docker compose up -d --build # build-on-host update path +``` + +--- + +## 2. INFRA STEPS — Travis applies these manually + +The compose stack is *up* once `docker compose up -d --build` returns. +The **public URL** at `https://brain.herbylab.dev` only resolves after +the two steps below land on their respective infra. They live on +machines outside this repo's authority (Knot on the DNS box; Traefik on +10.0.11.20), so this README documents them rather than executing them. + +### 2a. Knot DNS — A record for `brain.herbylab.dev` + +Add this to the `herbylab.dev` zone file on the Knot host (file path / +include layout matches the existing per-service A records — pick the +spot used for the other `*.herbylab.dev` services): + +```zone +brain.herbylab.dev. 300 IN A 10.0.11.20 +``` + +300s TTL matches the existing convention. Reload Knot: + +```bash +sudo knotc reload +dig +short brain.herbylab.dev @ # should return 10.0.11.20 +``` + +### 2b. Traefik file-provider router on 10.0.11.20 + +Add the snippet below to the Traefik dynamic config (same file the rest +of the `*.herbylab.dev` services live in — file-provider config picks +up changes without a restart): + +```yaml +http: + routers: + second-brain: + rule: "Host(`brain.herbylab.dev`)" + entryPoints: + - websecure + tls: + certResolver: cloudflare + service: second-brain + # No middleware on day one. If/when SSO lands, prepend the + # authentik forward-auth chain here (see other routers for the + # shape) and remove the LAN restriction. + + services: + second-brain: + loadBalancer: + servers: + - url: "http://10.0.21.207:8080" + passHostHeader: true +``` + +Verify on the Traefik host: + +```bash +# Watch for config-reload events: +docker logs traefik --tail 20 -f +# Or hit the API if it's exposed: +curl -fsS http://localhost:8080/api/http/routers/second-brain@file +``` + +Once both 2a and 2b are in place, `https://brain.herbylab.dev/dashboard` +should resolve through Cloudflare → Traefik → 10.0.21.207:8080 → the +`second-brain-web` container. + +--- + +## 3. Verification checklist + +After `docker compose up -d --build`: + +| Check | Command | Expect | +|---|---|---| +| Container up | `docker compose ps` | `second-brain-web` is `Up`, not restarting | +| Startup log clean | `docker compose logs --tail 50` | uvicorn `Started server process` + `Application startup complete`, no tracebacks | +| Local-host reachable | `curl -fsS http://10.0.21.207:8080/dashboard \| head -1` | `` or similar | +| DB reachable | `curl -fsS http://10.0.21.207:8080/dashboard` | dashboard renders status counts (proves the homelab-net path to Postgres) | +| Settings round-trip | open `/settings` in a browser, save | green "Saved." flash + new `updated_at` | + +After Travis's 2a + 2b infra steps: + +| Check | Command | Expect | +|---|---|---| +| DNS | `dig +short brain.herbylab.dev` | `10.0.11.20` | +| TLS | `curl -fsS https://brain.herbylab.dev/dashboard \| head -1` | dashboard HTML | +| Cert | `openssl s_client -connect brain.herbylab.dev:443 -servername brain.herbylab.dev <<<''` | Cloudflare-issued cert | + +--- + +## 4. Follow-ups (not built this round) + +- **SSO via Authentik.** Add the existing forward-auth middleware to the + Traefik router once we want this off-LAN. Authentik lives at + 10.0.21.207:9000 (same host as this container — different port). +- **CSRF.** Settings save is an HTMX POST with no token; trivial to + forge from a same-origin browser. Acceptable while LAN-restricted. +- **Image registry.** Currently build-on-host. Push to Gitea's container + registry (`gitea.plantbasedsoutherner.com/petal-power/second-brain-web`) + when there's a second box that needs to pull the image. +- **Dev-side extraction systemd timer.** Travis is running extraction + manually for now; the timer/cron path is deferred. diff --git a/deploy/web/compose.yml b/deploy/web/compose.yml new file mode 100644 index 0000000..ffb1a5f --- /dev/null +++ b/deploy/web/compose.yml @@ -0,0 +1,41 @@ +# second-brain web — compose file for herbys-dev (10.0.21.207). +# +# House style: file is named `compose.yml`, no bare `docker run`. +# Bring up with: docker compose up -d --build +# Tear down with: docker compose down +# +# The Traefik VM lives on a different host (10.0.11.20) and uses the +# file provider, NOT docker label discovery. We therefore publish the +# uvicorn port back to a fixed host:port that Traefik's static service +# definition can point at. There are deliberately NO traefik.* labels +# in this file — they'd be silently ignored. +# +# The DB connection is the **containerised** one: `homelab-postgres:5432` +# on the homelab bridge network. Do NOT switch this to 127.0.0.1:5433 +# from the container — that loops back inside the container, not the host. + +services: + second-brain-web: + build: + context: ../.. + dockerfile: deploy/web/Dockerfile + image: second-brain-web:latest + container_name: second-brain-web + restart: unless-stopped + env_file: + - .env + environment: + SECOND_BRAIN_CONFIG: /app/config/settings.toml + volumes: + - ../../config/settings.toml:/app/config/settings.toml:ro + networks: + - homelab + # Bind to the herbys-dev LAN IP only so this isn't accidentally + # exposed on other interfaces. Traefik (10.0.11.20) reaches the + # service at http://10.0.21.207:8080 via the LAN. + ports: + - "10.0.21.207:8080:8000" + +networks: + homelab: + external: true