33ae5a6f7d
Splits pull+transcribe (now tower-side, eager) from extract+embed (stays on the dev scheduler). Three machine-coordination pieces land together because they reference each other: - v3 migration adds sources.claimed_by + claimed_at — observability + stale-claim recovery columns. The actual race-safety primitive is `SELECT ... FOR UPDATE SKIP LOCKED` in the new claim helper, so two machines can poll the queue without doubling work. - src/second_brain/claim.py owns the claim dance (claim_next_source, release_claim, reap_stale_claims). Both stage gates filter by source_type so the tower never grabs articles and the dev side never grabs videos. - src/second_brain/transcribe.py wraps faster-whisper (lazy-imported so it stays out of the dev install). resolve_settings() reads env > [whisper] block > defaults, falling back to int8 on cpu / float16 on cuda when compute_type is unspecified. Default model large-v3. - src/second_brain/scheduler/transcribe_worker.py is the long-running poll loop. Reads pipeline_settings every iteration so the dashboard's enable/window/max-items/max-video-length take effect within one cycle. Reaps stale claims at startup. SIGTERM-clean. DB-unreachable backs off with a log line; never crash-loops. - adapters/youtube.py drops the torch-whisper transcribe path; pull stays. Removes openai-whisper from the default deps and gates faster-whisper behind a new `tower` extra (uv sync --extra tower). - main.py: new `second-brain transcribe-worker` (--once for ad-hoc). `process` now article-only on the pull side but still picks up TRANSCRIBED of any source_type for the extract step. Live-verified: migration applies clean, transcribe-worker --once honors transcription_enabled=false gate.
57 lines
1.6 KiB
TOML
57 lines
1.6 KiB
TOML
[project]
|
|
name = "second-brain"
|
|
version = "0.1.0"
|
|
description = "Personal knowledge system: video/article extraction pipeline + LLM-maintained wiki"
|
|
requires-python = ">=3.12"
|
|
dependencies = [
|
|
"alembic>=1.13.0",
|
|
"click>=8.1.0",
|
|
"embedding-chunking",
|
|
"fastapi>=0.115.0",
|
|
"httpx>=0.28.0",
|
|
"jinja2>=3.1.0",
|
|
"psycopg[binary]>=3.2",
|
|
"psycopg-pool>=3.2",
|
|
"pydantic>=2.0.0",
|
|
"python-dotenv>=1.0.0",
|
|
"python-multipart>=0.0.20",
|
|
"sqlalchemy>=2.0.0",
|
|
"tomli>=2.0.0",
|
|
"trafilatura>=1.9.0",
|
|
"uvicorn[standard]>=0.30.0",
|
|
"yt-dlp>=2024.1.0",
|
|
]
|
|
|
|
[project.optional-dependencies]
|
|
# Only required when extractor.backend = "api" in settings.toml.
|
|
# The default "cli" backend shells out to the Claude Code CLI under your
|
|
# Max OAuth subscription and has no Python-side Anthropic dependency.
|
|
api = ["anthropic>=0.40.0"]
|
|
|
|
# Tower-only: faster-whisper (CTranslate2) for the transcribe worker.
|
|
# Pulls in CUDA wheels — install with `uv sync --extra tower` on the
|
|
# tower box, leave it off on dev. CPU/int8 path works on this same wheel.
|
|
tower = ["faster-whisper>=1.0.0"]
|
|
|
|
[dependency-groups]
|
|
dev = [
|
|
"pytest>=8.0",
|
|
"pytest-cov>=5.0",
|
|
"ruff>=0.8",
|
|
]
|
|
|
|
[tool.uv.sources]
|
|
# Shared chunker that vault-mcp / ob1-enricher / ob1-reflector also pin.
|
|
# Same source so the 512/64 chunking stays in lockstep across services.
|
|
embedding-chunking = { git = "https://gitea.plantbasedsoutherner.com/petal-power/embedding-chunking.git", tag = "v0.1.0" }
|
|
|
|
[project.scripts]
|
|
second-brain = "second_brain.main:cli"
|
|
|
|
[build-system]
|
|
requires = ["hatchling"]
|
|
build-backend = "hatchling.build"
|
|
|
|
[tool.hatch.build.targets.wheel]
|
|
packages = ["src/second_brain"]
|