settings: v2 migration + ORM + settings_store helper
Introduces second_brain.pipeline_settings — the single-row config row the upcoming web dashboard edits and the workers read at the start of each run. Pinned to id=1 by a CHECK constraint so upserts-by-PK keep the table singleton, and the migration seeds the row with the table's column defaults via INSERT ... ON CONFLICT DO NOTHING. Two consumer groups carved out: - transcription_* fields persist now; future tower-side worker reads them. - extraction_* fields will be read by the existing scheduler in the next commit, which is the actual behavior change Travis cares about today. The settings_store helper centralises get/update + form-parsing (time-of-day, int-or-none) and active-window math so the routes and the scheduler don't reimplement them.
This commit is contained in:
parent
0d9ebc9cc7
commit
09efbb5965
70
alembic/versions/7b3e8a5c1f29_v2_pipeline_settings.py
Normal file
70
alembic/versions/7b3e8a5c1f29_v2_pipeline_settings.py
Normal file
@ -0,0 +1,70 @@
|
|||||||
|
"""v2 second_brain: pipeline_settings table
|
||||||
|
|
||||||
|
Revision ID: 7b3e8a5c1f29
|
||||||
|
Revises: 4a1e2f6c9d10
|
||||||
|
Create Date: 2026-05-25 00:00:00.000000
|
||||||
|
|
||||||
|
Adds a single-row `pipeline_settings` table that the web dashboard edits
|
||||||
|
and the workers (current dev-side scheduler + future tower-side transcribe
|
||||||
|
worker) read at the start of each run. The transcription_* fields persist
|
||||||
|
now but are only consumed once the transcribe worker exists — the
|
||||||
|
extraction_* fields are wired into the existing scheduler in this commit.
|
||||||
|
|
||||||
|
Single-row enforcement: PK is fixed at `id=1` with a CHECK constraint, so
|
||||||
|
INSERT-as-upsert-by-PK keeps the table free of stale rows.
|
||||||
|
|
||||||
|
Schema/permission notes mirror the v1 migration: lovebug lacks CREATE on
|
||||||
|
the petalbrain database, so env.py has already bootstrapped the schema —
|
||||||
|
this migration just SETs search_path and emits DDL inside it.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from collections.abc import Sequence
|
||||||
|
|
||||||
|
from alembic import op
|
||||||
|
|
||||||
|
revision: str = "7b3e8a5c1f29"
|
||||||
|
down_revision: str | Sequence[str] | None = "4a1e2f6c9d10"
|
||||||
|
branch_labels: str | Sequence[str] | None = None
|
||||||
|
depends_on: str | Sequence[str] | None = None
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade() -> None:
|
||||||
|
op.execute("SET search_path TO second_brain")
|
||||||
|
|
||||||
|
op.execute(
|
||||||
|
"""
|
||||||
|
CREATE TABLE pipeline_settings (
|
||||||
|
id INTEGER PRIMARY KEY DEFAULT 1,
|
||||||
|
transcription_enabled BOOLEAN NOT NULL DEFAULT TRUE,
|
||||||
|
transcription_active_hours_start TIME,
|
||||||
|
transcription_active_hours_end TIME,
|
||||||
|
transcription_max_concurrent_gpu_jobs INTEGER NOT NULL DEFAULT 1,
|
||||||
|
transcription_max_video_length_seconds INTEGER,
|
||||||
|
transcription_max_items_per_run INTEGER NOT NULL DEFAULT 10,
|
||||||
|
extraction_enabled BOOLEAN NOT NULL DEFAULT TRUE,
|
||||||
|
extraction_active_hours_start TIME,
|
||||||
|
extraction_active_hours_end TIME,
|
||||||
|
extraction_max_items_per_run INTEGER NOT NULL DEFAULT 20,
|
||||||
|
updated_at TIMESTAMP NOT NULL
|
||||||
|
DEFAULT (now() AT TIME ZONE 'UTC'),
|
||||||
|
CONSTRAINT pipeline_settings_singleton CHECK (id = 1)
|
||||||
|
)
|
||||||
|
"""
|
||||||
|
)
|
||||||
|
|
||||||
|
# Seed the singleton row with the table's column defaults. Idempotent
|
||||||
|
# via ON CONFLICT so re-running this migration on a partly-applied DB
|
||||||
|
# (or a manual cherry-pick) is safe.
|
||||||
|
op.execute(
|
||||||
|
"""
|
||||||
|
INSERT INTO pipeline_settings (id) VALUES (1)
|
||||||
|
ON CONFLICT (id) DO NOTHING
|
||||||
|
"""
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade() -> None:
|
||||||
|
op.execute("SET search_path TO second_brain")
|
||||||
|
op.execute("DROP TABLE IF EXISTS pipeline_settings")
|
||||||
@ -14,11 +14,12 @@ Tables:
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import enum
|
import enum
|
||||||
from datetime import datetime, timezone
|
from datetime import datetime, time, timezone
|
||||||
from typing import Optional
|
from typing import Optional
|
||||||
|
|
||||||
from sqlalchemy import (
|
from sqlalchemy import (
|
||||||
JSON,
|
JSON,
|
||||||
|
Boolean,
|
||||||
DateTime,
|
DateTime,
|
||||||
Enum,
|
Enum,
|
||||||
ForeignKey,
|
ForeignKey,
|
||||||
@ -26,6 +27,7 @@ from sqlalchemy import (
|
|||||||
MetaData,
|
MetaData,
|
||||||
String,
|
String,
|
||||||
Text,
|
Text,
|
||||||
|
Time,
|
||||||
)
|
)
|
||||||
from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column, relationship
|
from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column, relationship
|
||||||
|
|
||||||
@ -193,6 +195,64 @@ class WikiPage(Base):
|
|||||||
return f"WikiPage(id={self.id}, vault_path={self.vault_path!r})"
|
return f"WikiPage(id={self.id}, vault_path={self.vault_path!r})"
|
||||||
|
|
||||||
|
|
||||||
|
class PipelineSettings(Base):
|
||||||
|
"""Single-row config the web dashboard edits and workers read.
|
||||||
|
|
||||||
|
The transcription_* fields persist now and will be honored by the
|
||||||
|
tower-side transcribe worker once it exists; the extraction_* fields
|
||||||
|
are already wired into the dev-side scheduler. The row is pinned to
|
||||||
|
`id=1` by a DB-level CHECK so upserts-by-PK keep the table singleton.
|
||||||
|
"""
|
||||||
|
|
||||||
|
__tablename__ = "pipeline_settings"
|
||||||
|
|
||||||
|
id: Mapped[int] = mapped_column(Integer, primary_key=True, default=1)
|
||||||
|
|
||||||
|
# Transcription (consumed by the tower-side worker — not yet built).
|
||||||
|
transcription_enabled: Mapped[bool] = mapped_column(
|
||||||
|
Boolean, nullable=False, default=True
|
||||||
|
)
|
||||||
|
transcription_active_hours_start: Mapped[Optional[time]] = mapped_column(
|
||||||
|
Time, nullable=True
|
||||||
|
)
|
||||||
|
transcription_active_hours_end: Mapped[Optional[time]] = mapped_column(
|
||||||
|
Time, nullable=True
|
||||||
|
)
|
||||||
|
transcription_max_concurrent_gpu_jobs: Mapped[int] = mapped_column(
|
||||||
|
Integer, nullable=False, default=1
|
||||||
|
)
|
||||||
|
transcription_max_video_length_seconds: Mapped[Optional[int]] = mapped_column(
|
||||||
|
Integer, nullable=True
|
||||||
|
)
|
||||||
|
transcription_max_items_per_run: Mapped[int] = mapped_column(
|
||||||
|
Integer, nullable=False, default=10
|
||||||
|
)
|
||||||
|
|
||||||
|
# Extraction (consumed by the dev-side scheduler today).
|
||||||
|
extraction_enabled: Mapped[bool] = mapped_column(
|
||||||
|
Boolean, nullable=False, default=True
|
||||||
|
)
|
||||||
|
extraction_active_hours_start: Mapped[Optional[time]] = mapped_column(
|
||||||
|
Time, nullable=True
|
||||||
|
)
|
||||||
|
extraction_active_hours_end: Mapped[Optional[time]] = mapped_column(
|
||||||
|
Time, nullable=True
|
||||||
|
)
|
||||||
|
extraction_max_items_per_run: Mapped[int] = mapped_column(
|
||||||
|
Integer, nullable=False, default=20
|
||||||
|
)
|
||||||
|
|
||||||
|
updated_at: Mapped[datetime] = mapped_column(
|
||||||
|
DateTime, default=utcnow, onupdate=utcnow
|
||||||
|
)
|
||||||
|
|
||||||
|
def __repr__(self) -> str:
|
||||||
|
return (
|
||||||
|
f"PipelineSettings(transcription_enabled={self.transcription_enabled}, "
|
||||||
|
f"extraction_enabled={self.extraction_enabled})"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
__all__ = [
|
__all__ = [
|
||||||
"Base",
|
"Base",
|
||||||
"SCHEMA",
|
"SCHEMA",
|
||||||
@ -200,6 +260,7 @@ __all__ = [
|
|||||||
"SourceStatus",
|
"SourceStatus",
|
||||||
"SourceType",
|
"SourceType",
|
||||||
"Extraction",
|
"Extraction",
|
||||||
|
"PipelineSettings",
|
||||||
"WikiPage",
|
"WikiPage",
|
||||||
"utcnow",
|
"utcnow",
|
||||||
]
|
]
|
||||||
|
|||||||
121
src/second_brain/settings_store.py
Normal file
121
src/second_brain/settings_store.py
Normal file
@ -0,0 +1,121 @@
|
|||||||
|
"""Helpers for the singleton `pipeline_settings` row.
|
||||||
|
|
||||||
|
The web dashboard edits this row; workers read it at the start of each
|
||||||
|
run to decide whether to do anything and how much. DB values override
|
||||||
|
the static defaults in `settings.toml`.
|
||||||
|
|
||||||
|
All functions take a SQLAlchemy `Session` so the caller owns the
|
||||||
|
transaction boundary — matches the rest of the codebase, which keeps
|
||||||
|
session lifecycle inside `with db.session()` blocks.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from datetime import time
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
|
from second_brain.models import PipelineSettings, utcnow
|
||||||
|
|
||||||
|
|
||||||
|
# Field names allowed in `update_settings`. Anything else is rejected so a
|
||||||
|
# malicious form post can't toggle id/updated_at/etc.
|
||||||
|
_EDITABLE_FIELDS: tuple[str, ...] = (
|
||||||
|
"transcription_enabled",
|
||||||
|
"transcription_active_hours_start",
|
||||||
|
"transcription_active_hours_end",
|
||||||
|
"transcription_max_concurrent_gpu_jobs",
|
||||||
|
"transcription_max_video_length_seconds",
|
||||||
|
"transcription_max_items_per_run",
|
||||||
|
"extraction_enabled",
|
||||||
|
"extraction_active_hours_start",
|
||||||
|
"extraction_active_hours_end",
|
||||||
|
"extraction_max_items_per_run",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def get_settings(sess: Session) -> PipelineSettings:
|
||||||
|
"""Return the singleton settings row, creating it on first call.
|
||||||
|
|
||||||
|
The v2 migration seeds the row with all DB-side defaults, so this
|
||||||
|
"create on miss" path only kicks in when somebody bootstrapped the
|
||||||
|
schema without applying the migration (e.g. an unconfigured test
|
||||||
|
using `Base.metadata.create_all`).
|
||||||
|
"""
|
||||||
|
row = sess.get(PipelineSettings, 1)
|
||||||
|
if row is None:
|
||||||
|
row = PipelineSettings(id=1)
|
||||||
|
sess.add(row)
|
||||||
|
sess.flush()
|
||||||
|
return row
|
||||||
|
|
||||||
|
|
||||||
|
def update_settings(sess: Session, **fields: Any) -> PipelineSettings:
|
||||||
|
"""Apply a partial update to the singleton row.
|
||||||
|
|
||||||
|
Unknown keys raise `KeyError`. Empty-string values for nullable fields
|
||||||
|
are normalised to None so the HTML form's blank inputs (active hours,
|
||||||
|
video-length cap) round-trip as NULL rather than as the strings "".
|
||||||
|
"""
|
||||||
|
for k in fields:
|
||||||
|
if k not in _EDITABLE_FIELDS:
|
||||||
|
raise KeyError(f"Unknown pipeline_settings field: {k!r}")
|
||||||
|
|
||||||
|
row = get_settings(sess)
|
||||||
|
for k, v in fields.items():
|
||||||
|
if v == "":
|
||||||
|
v = None
|
||||||
|
setattr(row, k, v)
|
||||||
|
row.updated_at = utcnow()
|
||||||
|
return row
|
||||||
|
|
||||||
|
|
||||||
|
def parse_time_or_none(raw: str | None) -> time | None:
|
||||||
|
"""Parse an HTML <input type=time> value (HH:MM) into a `time`.
|
||||||
|
|
||||||
|
Empty / missing values map to None — `update_settings` will store
|
||||||
|
NULL, which workers interpret as "no window, always active".
|
||||||
|
"""
|
||||||
|
if raw is None or raw == "":
|
||||||
|
return None
|
||||||
|
parts = raw.split(":")
|
||||||
|
if len(parts) < 2:
|
||||||
|
raise ValueError(f"Could not parse time: {raw!r}")
|
||||||
|
h, m = int(parts[0]), int(parts[1])
|
||||||
|
return time(h, m)
|
||||||
|
|
||||||
|
|
||||||
|
def parse_int_or_none(raw: str | None) -> int | None:
|
||||||
|
if raw is None or raw == "":
|
||||||
|
return None
|
||||||
|
return int(raw)
|
||||||
|
|
||||||
|
|
||||||
|
def in_active_window(now_t: time, start: time | None, end: time | None) -> bool:
|
||||||
|
"""Return True if `now_t` lies in the [start, end] window.
|
||||||
|
|
||||||
|
Either bound being None means "no constraint on that side", which
|
||||||
|
matches the dashboard's "leave blank = always active" semantics.
|
||||||
|
Handles overnight windows (start > end) by treating them as the
|
||||||
|
complement that crosses midnight.
|
||||||
|
"""
|
||||||
|
if start is None and end is None:
|
||||||
|
return True
|
||||||
|
if start is None:
|
||||||
|
return now_t <= end # type: ignore[operator]
|
||||||
|
if end is None:
|
||||||
|
return now_t >= start
|
||||||
|
if start <= end:
|
||||||
|
return start <= now_t <= end
|
||||||
|
# crosses midnight
|
||||||
|
return now_t >= start or now_t <= end
|
||||||
|
|
||||||
|
|
||||||
|
__all__ = [
|
||||||
|
"get_settings",
|
||||||
|
"update_settings",
|
||||||
|
"parse_time_or_none",
|
||||||
|
"parse_int_or_none",
|
||||||
|
"in_active_window",
|
||||||
|
]
|
||||||
Loading…
Reference in New Issue
Block a user