From 913d4f0335141270f2a4460a7bc62419bb1a7f07 Mon Sep 17 00:00:00 2001 From: Travis Herbranson Date: Mon, 25 May 2026 10:20:46 -0400 Subject: [PATCH] deploy/tower + claim race + transcribe smoke tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit deploy/tower/: - second-brain-transcribe.service — systemd unit. User=herbyadmin, Type=simple, After=/Wants= wg-quick@wg-lan.service so the WG tunnel must come up first. Restart=always with a StartLimitBurst guard. - second-brain-transcribe.env.example — env file template documenting the SECOND_BRAIN_DATABASE_URL form for db.wg.herbylab.dev (10.99.0.1) and the optional WHISPER_* overrides. - README.md — EndeavourOS install steps (nvidia/cuda/cudnn, ffmpeg, uv + tower extra, model pre-warm), WG topology reference, validation checklist for what to confirm once the tunnel is live, and a follow-ups section flagging the local-disk → NAS media migration as out-of-scope-for-this-round. Tests: - tests/test_claim.py — live-DB race test. Two threads call claim_next_source against a single PULLED video row; SKIP LOCKED must give exactly one of them the row, the other gets None. Also asserts the claimed_by/at columns land + release nulls them. Auto-skips when no SECOND_BRAIN_DATABASE_URL is set. - tests/test_transcribe.py — pure-Python coverage of resolve_settings (cpu→int8, cuda→float16, env-over-block) and write_srt; plus a CPU smoke test that synthesizes a numpy audio array and runs the `tiny` model on cpu/int8 (auto-skipped when faster-whisper isn't installed, i.e. on the dev side without --extra tower). --- deploy/tower/README.md | 167 ++++++++++++++++++ .../tower/second-brain-transcribe.env.example | 24 +++ deploy/tower/second-brain-transcribe.service | 46 +++++ tests/test_claim.py | 107 +++++++++++ tests/test_transcribe.py | 142 +++++++++++++++ 5 files changed, 486 insertions(+) create mode 100644 deploy/tower/README.md create mode 100644 deploy/tower/second-brain-transcribe.env.example create mode 100644 deploy/tower/second-brain-transcribe.service create mode 100644 tests/test_claim.py create mode 100644 tests/test_transcribe.py diff --git a/deploy/tower/README.md b/deploy/tower/README.md new file mode 100644 index 0000000..9b391e7 --- /dev/null +++ b/deploy/tower/README.md @@ -0,0 +1,167 @@ +# Tower-side transcribe worker — deploy notes + +Target host: **EndeavourOS tower** (Arch base), RTX 3080. Runs the +`second-brain transcribe-worker` daemon under systemd. Reaches the +petalbrain Postgres over WireGuard at `db.wg.herbylab.dev:5432`. + +This worker only does **pull + transcribe** for VIDEO sources. Articles +and all extraction/embedding stay on the dev side. + +--- + +## 1. System packages + +```bash +# Build / runtime tooling. +sudo pacman -S --needed base-devel git ffmpeg python uv + +# NVIDIA — driver, CUDA, cuDNN. faster-whisper / CTranslate2 needs cuDNN +# at runtime; missing-symbol errors at first transcribe are usually a +# cuDNN install gap. Use whichever driver/cuda channel matches your +# Arch derivative; on stock EndeavourOS the AUR/community packages work: +sudo pacman -S --needed nvidia nvidia-utils cuda cudnn + +# Confirm the GPU is visible. +nvidia-smi +``` + +yt-dlp comes in via the Python project (`uv sync` step below), so the +distro yt-dlp is *optional*. Leave it off the host unless you want it on +your CLI PATH. + +--- + +## 2. Project checkout + venv + +```bash +sudo mkdir -p /opt/projects/homelab +sudo chown herbyadmin:herbyadmin /opt/projects/homelab +cd /opt/projects/homelab +git clone gitea-lovebug:petal-power/second-brain.git +cd second-brain + +# Install the runtime deps + the `tower` extra (faster-whisper / CTranslate2). +# The dev side does NOT install --extra tower; only the tower needs it. +uv sync --extra tower +``` + +`faster-whisper` will pull a couple hundred MB of CUDA wheels. First-run +model download (~3 GB for large-v3) lands under `~/.cache/huggingface/` +the first time the worker transcribes; pre-warm it if you want: + +```bash +uv run python -c "from faster_whisper import WhisperModel; WhisperModel('large-v3', device='cuda', compute_type='float16')" +``` + +--- + +## 3. WireGuard + +The worker assumes the tunnel is already configured and that +`wg-quick@wg-lan.service` brings it up at boot. Topology: + +| Peer | WG IP | +|------|-------| +| hub (herbydev) | `10.99.0.1` | +| tower | `10.99.0.2` | + +`db.wg.herbylab.dev` resolves to `10.99.0.1`. Confirm connectivity from +the tower: + +```bash +ping -c2 db.wg.herbylab.dev +nc -vz db.wg.herbylab.dev 5432 +``` + +Tunnel setup itself is **out of scope for this README** — that's +configured separately as part of Travis's infra. + +--- + +## 4. EnvironmentFile + +```bash +sudo install -m 0600 -o root -g root \ + deploy/tower/second-brain-transcribe.env.example \ + /etc/default/second-brain-transcribe +sudo $EDITOR /etc/default/second-brain-transcribe # paste the lovebug password +``` + +The lovebug password lives in `/opt/backups/postgres-consolidation/credentials.env` +on the dev host. Copy it across through whatever channel you trust +(personal Bitwarden, scp over WG, etc.) — do not commit it. + +--- + +## 5. systemd unit + +```bash +sudo install -m 0644 deploy/tower/second-brain-transcribe.service \ + /etc/systemd/system/second-brain-transcribe.service +sudo systemctl daemon-reload +sudo systemctl enable --now second-brain-transcribe.service +``` + +The unit declares `After=` + `Wants=` on `wg-quick@wg-lan.service`, so +the worker starts after the tunnel comes up at boot. If the tunnel +disappears mid-run the worker keeps polling and logs DB-unreachable +warnings; it doesn't crash-loop. + +### Day-to-day commands + +```bash +# What's it doing right now? +systemctl status second-brain-transcribe.service +journalctl -u second-brain-transcribe.service -f + +# Pause without uninstalling — toggle from the dashboard at +# https://brain.herbylab.dev/settings (or whichever host). +# Or stop the unit: +sudo systemctl stop second-brain-transcribe.service + +# Pick up code updates: +cd /opt/projects/homelab/second-brain +git pull +uv sync --extra tower +sudo systemctl restart second-brain-transcribe.service +``` + +--- + +## 6. Validation — what to confirm once WG is live + +These can only be verified on the tower itself: + +1. **WG reachability.** `nc -vz db.wg.herbylab.dev 5432` succeeds. +2. **DB auth.** `uv run alembic current` returns the latest revision id. +3. **GPU large-v3 path.** With a short test video queued via + `second-brain add ` on the dev side, run `uv run second-brain transcribe-worker --once` + on the tower. Watch `journalctl` — first run pulls the model from + HuggingFace (slow), subsequent runs are fast. The source row should + end up in `TRANSCRIBED` with `transcript_text` populated and + `claimed_by` cleared. +4. **Race safety smoke test.** Run two `--once` invocations in parallel + against an empty queue, then with one PENDING video. Only one should + claim it; the other should see no work. +5. **Settings round-trip.** Flip `transcription_enabled` off on the + dashboard; within one poll cycle (~15s) the worker logs + `transcription_enabled=false — sleeping`. Flip back on, the next + PENDING video gets picked up. + +--- + +## Follow-ups (out of scope for this round) + +- **Media + subtitles directory currently lives on the tower's local + disk** (under `~/.local/share/second-brain/`). Move to a NAS mount + later so the dev side can compile/inspect the SRTs without a + cross-machine fetch. Open question on the right mount point + cred + flow — see the project's `postgres-migration-planning.md` for the + related vault-location decision Travis still owes. +- **Tower-side observability.** No metrics endpoint yet; journald is + the only signal. A `/api/worker-status` heartbeat the dashboard could + read would be nice once two workers exist. +- **GPU concurrency > 1.** `transcription_max_concurrent_gpu_jobs` + exists in `pipeline_settings` but the worker only runs one job at a + time. Scale-out would need a per-job semaphore (or just running N + worker instances with distinct `SECOND_BRAIN_WORKER_ID`s). diff --git a/deploy/tower/second-brain-transcribe.env.example b/deploy/tower/second-brain-transcribe.env.example new file mode 100644 index 0000000..2371c8c --- /dev/null +++ b/deploy/tower/second-brain-transcribe.env.example @@ -0,0 +1,24 @@ +# /etc/default/second-brain-transcribe +# +# Copy to /etc/default/second-brain-transcribe, chmod 0600, chown root:root +# (systemd reads it before dropping to User=herbyadmin so a non-readable +# permissions mode is fine — the herbyadmin user never touches this file). + +# REQUIRED — petalbrain Postgres reached over the WireGuard tunnel. +# Hub-side hostname is db.wg.herbylab.dev (10.99.0.1); the tower is 10.99.0.2. +# No TLS — WireGuard already encrypts, and lovebug doesn't require SSL. +SECOND_BRAIN_DATABASE_URL=postgresql+psycopg://lovebug:REPLACE_ME@db.wg.herbylab.dev:5432/petalbrain + +# OPTIONAL — only set if the tower has a non-standard whisper layout. +# Defaults from settings.toml: model=large-v3, device=cuda, compute_type=float16. +# WHISPER_MODEL=large-v3 +# WHISPER_DEVICE=cuda +# WHISPER_COMPUTE_TYPE=float16 + +# OPTIONAL — identifier stamped into sources.claimed_by (default tower:). +# Useful if you ever run multiple tower workers and want to tell them apart. +# SECOND_BRAIN_WORKER_ID=tower-main + +# OPTIONAL — bump worker log verbosity. INFO is the default. +# PYTHONUNBUFFERED=1 +# SECOND_BRAIN_LOG_LEVEL=DEBUG diff --git a/deploy/tower/second-brain-transcribe.service b/deploy/tower/second-brain-transcribe.service new file mode 100644 index 0000000..33805d9 --- /dev/null +++ b/deploy/tower/second-brain-transcribe.service @@ -0,0 +1,46 @@ +[Unit] +Description=second-brain transcribe worker (tower-side, faster-whisper on GPU) +Documentation=https://gitea.plantbasedsoutherner.com/petal-power/second-brain +# The worker reaches petalbrain over WireGuard, so the tunnel must be up first. +# `Wants=` is the soft form — if the tunnel disappears later the worker logs +# DB-unreachable warnings and retries, it doesn't crash-loop. +After=network-online.target wg-quick@wg-lan.service +Wants=network-online.target wg-quick@wg-lan.service + +[Service] +Type=simple +User=herbyadmin +Group=herbyadmin + +# Adjust to wherever you `git clone`d the repo on the tower. +WorkingDirectory=/opt/projects/homelab/second-brain + +# Env (SECOND_BRAIN_DATABASE_URL, optional knobs). 0600, owned by herbyadmin. +EnvironmentFile=/etc/default/second-brain-transcribe + +# `uv run` resolves the project venv. Keep the worker in the foreground so +# systemd owns the lifecycle; the worker handles SIGTERM/SIGINT cleanly and +# exits after the current iteration finishes. +ExecStart=/usr/bin/uv run second-brain transcribe-worker + +Restart=always +RestartSec=10 + +# Crash loop guard — systemd will give up after this if the unit keeps +# dying. The worker itself logs+backs-off on DB outages so this only +# trips on genuine failure. +StartLimitIntervalSec=300 +StartLimitBurst=5 + +# A few sandboxing nice-to-haves. Loose on purpose — the worker needs +# /dev/nvidia* for CUDA and writes to user-owned media/subtitles dirs. +ProtectSystem=full +ProtectHome=no +NoNewPrivileges=true + +# Stdout/stderr → journald. +StandardOutput=journal +StandardError=journal + +[Install] +WantedBy=multi-user.target diff --git a/tests/test_claim.py b/tests/test_claim.py new file mode 100644 index 0000000..195516b --- /dev/null +++ b/tests/test_claim.py @@ -0,0 +1,107 @@ +"""Live-DB test: claim_next_source is race-safe against concurrent callers. + +Skips when no DB URL is configured. This is the integration-level guard +for the SELECT ... FOR UPDATE SKIP LOCKED dance — unit-testing the SQL +without Postgres would prove nothing. +""" + +from __future__ import annotations + +import os +import threading +import uuid + +import psycopg +import pytest + + +def _db_reachable() -> bool: + url = os.environ.get("SECOND_BRAIN_DATABASE_URL") or os.environ.get( + "HERBYLAB_DATABASE_URL" + ) + if not url: + return False + raw = url.replace("postgresql+psycopg://", "postgresql://", 1) + try: + with psycopg.connect(raw, connect_timeout=2): + return True + except Exception: + return False + + +pytestmark = pytest.mark.skipif( + not _db_reachable(), + reason="claim race test needs SECOND_BRAIN_DATABASE_URL pointed at a real petalbrain", +) + + +def test_two_workers_never_grab_the_same_row(): + from second_brain.claim import claim_next_source, release_claim + from second_brain.database import Database, reset_database_singleton + from second_brain.models import Source, SourceStatus, SourceType, utcnow + + reset_database_singleton() + db = Database() + + # Seed a single TRANSCRIBED-stage candidate. + unique = uuid.uuid4().hex[:8] + url = f"https://example.test/claim-race/{unique}" + with db.session() as sess: + src = Source( + url=url, + title=f"claim-race-{unique}", + domain="development", + source_type=SourceType.VIDEO, + status=SourceStatus.PULLED, + ingested_at=utcnow(), + ) + sess.add(src) + sess.flush() + source_id = src.id + + results: list[int | None] = [] + lock = threading.Lock() + + def worker(name: str): + with db.session() as sess: + claimed = claim_next_source( + sess, + worker_id=name, + statuses=[SourceStatus.PENDING, SourceStatus.PULLED], + source_type=SourceType.VIDEO, + ) + with lock: + results.append(claimed) + + t1 = threading.Thread(target=worker, args=("worker-A",)) + t2 = threading.Thread(target=worker, args=("worker-B",)) + t1.start() + t2.start() + t1.join() + t2.join() + + try: + # Exactly one worker should have grabbed the row; the other gets None. + # (SKIP LOCKED guarantees this even if both transactions overlap.) + non_none = [r for r in results if r is not None] + assert non_none == [source_id], ( + f"expected exactly one claimant of {source_id}, got results={results}" + ) + + # Confirm claim columns landed. + with db.session() as sess: + row = sess.get(Source, source_id) + assert row.claimed_by in {"worker-A", "worker-B"} + assert row.claimed_at is not None + + # Releasing the claim should null both columns. + with db.session() as sess: + release_claim(sess, source_id) + with db.session() as sess: + row = sess.get(Source, source_id) + assert row.claimed_by is None + assert row.claimed_at is None + finally: + with db.session() as sess: + sess.query(Source).filter(Source.id == source_id).delete() + reset_database_singleton() diff --git a/tests/test_transcribe.py b/tests/test_transcribe.py new file mode 100644 index 0000000..4ce4050 --- /dev/null +++ b/tests/test_transcribe.py @@ -0,0 +1,142 @@ +"""Unit + integration coverage for second_brain.transcribe. + +Unit-level: resolve_settings + write_srt are pure-Python and run anywhere. + +Integration: an end-to-end CPU/int8 transcribe of a synthesized audio +clip — auto-skipped when ffmpeg or faster-whisper is missing (the dev +side doesn't install faster-whisper by default). Only the *tower* with +GPU large-v3 can validate the production hot path; this test proves the +wiring + the SRT contract + the lazy-import path on cheap CPU. +""" + +from __future__ import annotations + +import os +import shutil +import subprocess +import tempfile +from pathlib import Path + +import pytest + + +# --------------------------------------------------------------------------- +# Pure helpers +# --------------------------------------------------------------------------- + + +def test_resolve_settings_picks_int8_for_cpu_by_default(): + from second_brain.transcribe import resolve_settings + + out = resolve_settings({"whisper_device": "cpu"}) + assert out["device"] == "cpu" + assert out["compute_type"] == "int8" + assert out["model"] == "large-v3" + + +def test_resolve_settings_picks_float16_for_cuda_by_default(): + from second_brain.transcribe import resolve_settings + + out = resolve_settings({"whisper_device": "cuda"}) + assert out["device"] == "cuda" + assert out["compute_type"] == "float16" + + +def test_resolve_settings_env_overrides_block(monkeypatch): + from second_brain.transcribe import resolve_settings + + monkeypatch.setenv("WHISPER_MODEL", "small") + monkeypatch.setenv("WHISPER_DEVICE", "cpu") + monkeypatch.setenv("WHISPER_COMPUTE_TYPE", "int8_float16") + out = resolve_settings({"whisper_device": "cuda"}) + assert out["model"] == "small" + assert out["device"] == "cpu" + assert out["compute_type"] == "int8_float16" + + +def test_write_srt_roundtrip(tmp_path): + from second_brain.transcribe import write_srt + + segs = [ + {"start": 0.0, "end": 1.5, "text": "hello"}, + {"start": 1.5, "end": 3.25, "text": "world"}, + ] + out = tmp_path / "x.srt" + write_srt(segs, out) + body = out.read_text() + assert "1\n00:00:00,000 --> 00:00:01,500\nhello" in body + assert "2\n00:00:01,500 --> 00:00:03,250\nworld" in body + + +# --------------------------------------------------------------------------- +# Integration: synthesized audio → CPU/int8 faster-whisper +# --------------------------------------------------------------------------- + + +def _have_faster_whisper() -> bool: + try: + import faster_whisper # type: ignore[import-not-found] # noqa: F401 + except ImportError: + return False + return True + + +def _have_numpy() -> bool: + try: + import numpy # noqa: F401 + except ImportError: + return False + return True + + +@pytest.mark.skipif( + not (_have_faster_whisper() and _have_numpy()), + reason="needs faster-whisper (uv sync --extra tower) + numpy to exercise CPU path", +) +def test_cpu_transcribe_smoke(tmp_path): + """Hand faster-whisper a synthesized audio array on CPU/int8 with `tiny`. + + Pure-tone input gives empty / minimal recognition — the point is the + wiring: model loads, the segments iterator drains, an SRT is writable. + Using `tiny` to keep this under ~30s coldcache. Skipped on dev where + faster-whisper isn't installed; only the tower's `uv sync --extra + tower` brings it in. + + Bypassing ffmpeg by passing a numpy array — faster-whisper.transcribe + accepts (path | file-like | np.ndarray). This exercises the same code + path the worker uses, only with zero external binaries. + """ + import numpy as np + + from second_brain.transcribe import write_srt + + # Build 2s of 440Hz mono at 16kHz, the model's native sample rate. + sr = 16000 + duration = 2.0 + t = np.arange(0, int(sr * duration)) / sr + audio = (0.2 * np.sin(2 * np.pi * 440 * t)).astype(np.float32) + + # Load `tiny` to keep this cheap; we're not asserting transcription + # quality, just that the pipeline executes and returns the documented + # shape. + from faster_whisper import WhisperModel # type: ignore[import-not-found] + + model = WhisperModel("tiny", device="cpu", compute_type="int8") + segments_iter, _ = model.transcribe(audio, language="en", beam_size=1) + + # Materialize to mirror what Transcriber.transcribe does internally. + segments: list[dict] = [] + text_parts: list[str] = [] + for seg in segments_iter: + s = (seg.text or "").strip() + segments.append({"start": float(seg.start), "end": float(seg.end), "text": s}) + if s: + text_parts.append(s) + text = " ".join(text_parts) + + assert isinstance(text, str) + assert isinstance(segments, list) + + srt = tmp_path / "out.srt" + write_srt(segments, srt) + assert srt.exists()