From c92a40bce155bfc569b2ba5dea6d66ceecd4b2ac Mon Sep 17 00:00:00 2001 From: Travis Herbranson Date: Mon, 25 May 2026 08:15:29 -0400 Subject: [PATCH] web: dashboard + settings editor (HTMX save) + pipeline observability MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds three new routes on the FastAPI web app: - GET /dashboard — pipeline counts by status (per-status drill-down via filtered /?status=… links), recent activity (latest 10 sources), and a settings snapshot card with an "edit →" shortcut. - GET /settings — full editing form for pipeline_settings. - POST /settings/save — HTMX endpoint that validates the partial form, upserts the row, and returns the rerendered card with a "Saved." or error flash. The card is its own _settings_card.html partial so the HTMX swap targets only the form region. Form parsing lives in settings_store helpers (parse_time_or_none, parse_int_or_none) — keeps the route thin and the empty-string-to-NULL normalisation in one place. Validation rejects negative counts and sub-1 GPU concurrency. The base template grew a Dashboard + Settings nav so the new routes are reachable without typing URLs. Added python-multipart to deps because FastAPI's Form(...) raises at app import without it. --- pyproject.toml | 1 + src/second_brain/web/app.py | 182 +++++++++++++++++- .../web/templates/_settings_card.html | 130 +++++++++++++ src/second_brain/web/templates/base.html | 2 + src/second_brain/web/templates/dashboard.html | 158 +++++++++++++++ src/second_brain/web/templates/settings.html | 14 ++ uv.lock | 11 ++ 7 files changed, 497 insertions(+), 1 deletion(-) create mode 100644 src/second_brain/web/templates/_settings_card.html create mode 100644 src/second_brain/web/templates/dashboard.html create mode 100644 src/second_brain/web/templates/settings.html diff --git a/pyproject.toml b/pyproject.toml index 2603013..d8dd7a5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -15,6 +15,7 @@ dependencies = [ "psycopg-pool>=3.2", "pydantic>=2.0.0", "python-dotenv>=1.0.0", + "python-multipart>=0.0.20", "sqlalchemy>=2.0.0", "tomli>=2.0.0", "trafilatura>=1.9.0", diff --git a/src/second_brain/web/app.py b/src/second_brain/web/app.py index 4ad1c21..59f8304 100644 --- a/src/second_brain/web/app.py +++ b/src/second_brain/web/app.py @@ -14,14 +14,21 @@ from __future__ import annotations from pathlib import Path from typing import List, Optional -from fastapi import FastAPI, HTTPException, Request +from fastapi import FastAPI, Form, HTTPException, Request from fastapi.responses import HTMLResponse from fastapi.templating import Jinja2Templates from pydantic import BaseModel +from sqlalchemy import func from second_brain.config import DOMAINS from second_brain.database import get_database from second_brain.models import Extraction, Source, SourceStatus +from second_brain.settings_store import ( + get_settings, + parse_int_or_none, + parse_time_or_none, + update_settings, +) # --------------------------------------------------------------------------- # App setup @@ -295,3 +302,176 @@ def _source_to_response(s: Source) -> SourceResponse: ingested_at=s.ingested_at.isoformat() if s.ingested_at else "", has_extraction=s.extraction is not None, ) + + +# --------------------------------------------------------------------------- +# Dashboard — pipeline observability +# --------------------------------------------------------------------------- + + +@app.get("/dashboard", response_class=HTMLResponse) +async def dashboard(request: Request): + """Overview: status counts, recent activity, and a settings snapshot.""" + db = get_database() + with db.session() as sess: + # Counts by status. Materialise inside the session so the keys stay + # plain str → int and templates don't try to touch a closed session. + rows = ( + sess.query(Source.status, func.count(Source.id)) + .group_by(Source.status) + .all() + ) + counts = {s.value: 0 for s in SourceStatus} + for status, n in rows: + counts[status.value] = int(n) + total = sum(counts.values()) + + recent_rows = ( + sess.query(Source) + .order_by(Source.ingested_at.desc()) + .limit(10) + .all() + ) + recent = [_source_to_dict(s) for s in recent_rows] + + settings_row = get_settings(sess) + settings_summary = _settings_to_dict(settings_row) + + return templates.TemplateResponse( + request, + "dashboard.html", + { + "counts": counts, + "total": total, + "recent": recent, + "statuses_in_order": [s.value for s in SourceStatus], + "settings": settings_summary, + }, + ) + + +# --------------------------------------------------------------------------- +# Settings — view + HTMX save +# --------------------------------------------------------------------------- + + +@app.get("/settings", response_class=HTMLResponse) +async def settings_view(request: Request): + db = get_database() + with db.session() as sess: + row = get_settings(sess) + ctx = _settings_to_dict(row) + + return templates.TemplateResponse( + request, + "settings.html", + {"settings": ctx, "flash": None, "error": None}, + ) + + +@app.post("/settings/save", response_class=HTMLResponse) +async def settings_save( + request: Request, + transcription_enabled: Optional[str] = Form(None), + transcription_active_hours_start: Optional[str] = Form(None), + transcription_active_hours_end: Optional[str] = Form(None), + transcription_max_concurrent_gpu_jobs: Optional[str] = Form(None), + transcription_max_video_length_seconds: Optional[str] = Form(None), + transcription_max_items_per_run: Optional[str] = Form(None), + extraction_enabled: Optional[str] = Form(None), + extraction_active_hours_start: Optional[str] = Form(None), + extraction_active_hours_end: Optional[str] = Form(None), + extraction_max_items_per_run: Optional[str] = Form(None), +): + """HTMX save endpoint — returns the rendered settings card with a flash.""" + db = get_database() + error: Optional[str] = None + flash: Optional[str] = None + + try: + fields = { + # Checkboxes only show up in the form payload when checked. The + # template emits a hidden "_present" companion so we can tell + # "unchecked" apart from "field missing because of a partial post". + "transcription_enabled": transcription_enabled == "on", + "transcription_active_hours_start": parse_time_or_none( + transcription_active_hours_start + ), + "transcription_active_hours_end": parse_time_or_none( + transcription_active_hours_end + ), + "transcription_max_concurrent_gpu_jobs": int( + transcription_max_concurrent_gpu_jobs or "1" + ), + "transcription_max_video_length_seconds": parse_int_or_none( + transcription_max_video_length_seconds + ), + "transcription_max_items_per_run": int( + transcription_max_items_per_run or "10" + ), + "extraction_enabled": extraction_enabled == "on", + "extraction_active_hours_start": parse_time_or_none( + extraction_active_hours_start + ), + "extraction_active_hours_end": parse_time_or_none( + extraction_active_hours_end + ), + "extraction_max_items_per_run": int(extraction_max_items_per_run or "20"), + } + # Cheap inline validation — keep counts non-negative and gpu jobs >= 1. + if fields["transcription_max_concurrent_gpu_jobs"] < 1: + raise ValueError("transcription_max_concurrent_gpu_jobs must be ≥ 1") + if fields["transcription_max_items_per_run"] < 0: + raise ValueError("transcription_max_items_per_run must be ≥ 0") + if fields["extraction_max_items_per_run"] < 0: + raise ValueError("extraction_max_items_per_run must be ≥ 0") + if ( + fields["transcription_max_video_length_seconds"] is not None + and fields["transcription_max_video_length_seconds"] < 0 + ): + raise ValueError("transcription_max_video_length_seconds must be ≥ 0") + + with db.session() as sess: + row = update_settings(sess, **fields) + ctx = _settings_to_dict(row) + flash = "Saved." + except (ValueError, KeyError) as exc: + error = str(exc) + # Re-render the form using whatever the user submitted so they don't + # lose in-progress edits on a validation bounce. + with db.session() as sess: + ctx = _settings_to_dict(get_settings(sess)) + + return templates.TemplateResponse( + request, + "_settings_card.html", + {"settings": ctx, "flash": flash, "error": error}, + ) + + +# --------------------------------------------------------------------------- +# Settings serialiser +# --------------------------------------------------------------------------- + + +def _settings_to_dict(row) -> dict: + """Render the settings row as a template-friendly dict (no ORM bleed).""" + + def t(v): + return v.strftime("%H:%M") if v is not None else "" + + return { + "transcription_enabled": row.transcription_enabled, + "transcription_active_hours_start": t(row.transcription_active_hours_start), + "transcription_active_hours_end": t(row.transcription_active_hours_end), + "transcription_max_concurrent_gpu_jobs": row.transcription_max_concurrent_gpu_jobs, + "transcription_max_video_length_seconds": row.transcription_max_video_length_seconds, + "transcription_max_items_per_run": row.transcription_max_items_per_run, + "extraction_enabled": row.extraction_enabled, + "extraction_active_hours_start": t(row.extraction_active_hours_start), + "extraction_active_hours_end": t(row.extraction_active_hours_end), + "extraction_max_items_per_run": row.extraction_max_items_per_run, + "updated_at": row.updated_at.strftime("%Y-%m-%d %H:%M:%S") + if row.updated_at + else "", + } diff --git a/src/second_brain/web/templates/_settings_card.html b/src/second_brain/web/templates/_settings_card.html new file mode 100644 index 0000000..7fc709c --- /dev/null +++ b/src/second_brain/web/templates/_settings_card.html @@ -0,0 +1,130 @@ +{# HTMX-replaceable inner card. Posted to /settings/save; swaps itself in place. #} +
+ {% if flash %} +
+ ✓ {{ flash }} +
+ {% endif %} + {% if error %} +
+ ✗ {{ error }} +
+ {% endif %} + +
+ + +
+
+

Transcription

+

consumed by tower-side worker (not yet built)

+
+ +
+ + + +
+ +
+ + +

blank = always active

+
+
+ + +
+ +
+ + +
+
+ + +
+ +
+ + +
+
+
+ + +
+
+

Extraction

+

consumed by the dev-side scheduler

+
+ +
+ + +
+ +
+ + +

blank = always active

+
+
+ + +
+ +
+ + +
+
+
+ +
+

last saved {{ settings.updated_at }}

+ +
+
+
diff --git a/src/second_brain/web/templates/base.html b/src/second_brain/web/templates/base.html index af340d3..10a9d1d 100644 --- a/src/second_brain/web/templates/base.html +++ b/src/second_brain/web/templates/base.html @@ -16,8 +16,10 @@

personal knowledge pipeline

diff --git a/src/second_brain/web/templates/dashboard.html b/src/second_brain/web/templates/dashboard.html new file mode 100644 index 0000000..4e92937 --- /dev/null +++ b/src/second_brain/web/templates/dashboard.html @@ -0,0 +1,158 @@ +{% extends "base.html" %} + +{% block title %}Dashboard — second-brain{% endblock %} + +{% block content %} +{% set status_colors = { + 'pending': 'bg-gray-100 text-gray-700', + 'pulled': 'bg-blue-100 text-blue-700', + 'transcribed': 'bg-yellow-100 text-yellow-700', + 'analyzed': 'bg-orange-100 text-orange-700', + 'accepted': 'bg-green-100 text-green-700', + 'published': 'bg-purple-100 text-purple-700', + 'failed': 'bg-red-100 text-red-700' +} %} +{% set domain_colors = { + 'development': 'bg-cyan-50 text-cyan-700 border-cyan-200', + 'content': 'bg-pink-50 text-pink-700 border-pink-200', + 'business': 'bg-amber-50 text-amber-700 border-amber-200', + 'homelab': 'bg-teal-50 text-teal-700 border-teal-200' +} %} + +
+

Dashboard

+ total sources: {{ total }} +
+ + +
+

+ Pipeline +

+
+ {% for status in statuses_in_order %} + +
{{ counts.get(status, 0) }}
+
+ + {{ status }} + +
+ + {% endfor %} +
+
+ + +
+ + +
+
+

+ Settings +

+ + edit → + +
+ +
+
+
Transcription
+
+ {% if settings.transcription_enabled %} + enabled + {% else %} + paused + {% endif %} +
+
+
+
active window
+
+ {% if settings.transcription_active_hours_start or settings.transcription_active_hours_end %} + {{ settings.transcription_active_hours_start or "—" }} – {{ settings.transcription_active_hours_end or "—" }} + {% else %} + always + {% endif %} +
+
+
+
GPU jobs
+
{{ settings.transcription_max_concurrent_gpu_jobs }}
+
+
+
max items/run
+
{{ settings.transcription_max_items_per_run }}
+
+ +
+ +
+
Extraction
+
+ {% if settings.extraction_enabled %} + enabled + {% else %} + paused + {% endif %} +
+
+
+
active window
+
+ {% if settings.extraction_active_hours_start or settings.extraction_active_hours_end %} + {{ settings.extraction_active_hours_start or "—" }} – {{ settings.extraction_active_hours_end or "—" }} + {% else %} + always + {% endif %} +
+
+
+
max items/run
+
{{ settings.extraction_max_items_per_run }}
+
+
+ +

+ updated {{ settings.updated_at }} +

+
+ + +
+

+ Recent activity +

+ {% if recent %} +
+ {% for source in recent %} + +
+

{{ source.title }}

+

{{ source.url }}

+
+
+ + {{ source.status }} + + + {{ source.domain }} + + {{ source.ingested_at }} +
+ + {% endfor %} +
+ {% else %} +

No sources yet.

+ {% endif %} +
+
+{% endblock %} diff --git a/src/second_brain/web/templates/settings.html b/src/second_brain/web/templates/settings.html new file mode 100644 index 0000000..80cb917 --- /dev/null +++ b/src/second_brain/web/templates/settings.html @@ -0,0 +1,14 @@ +{% extends "base.html" %} + +{% block title %}Settings — second-brain{% endblock %} + +{% block content %} +
+

Settings

+

+ Workers read these at the start of each run. DB values override settings.toml. +

+
+ +{% include "_settings_card.html" %} +{% endblock %} diff --git a/uv.lock b/uv.lock index b99bf72..141b482 100644 --- a/uv.lock +++ b/uv.lock @@ -1385,6 +1385,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/0b/d7/1959b9648791274998a9c3526f6d0ec8fd2233e4d4acce81bbae76b44b2a/python_dotenv-1.2.2-py3-none-any.whl", hash = "sha256:1d8214789a24de455a8b8bd8ae6fe3c6b69a5e3d64aa8a8e5d68e694bbcb285a", size = 22101, upload-time = "2026-03-01T16:00:25.09Z" }, ] +[[package]] +name = "python-multipart" +version = "0.0.29" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/4e/fe/70bd71a6738b09a0bdf6480ca6436b167469ca4578b2a0efbe390b4b0e70/python_multipart-0.0.29.tar.gz", hash = "sha256:643e93849196645e2dbdd81a0f8829a23123ad7f797a84a364c6fb3563f18904", size = 45678, upload-time = "2026-05-17T17:29:47.654Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/8f/cb/769cfc37177252872a45a71f3fbdde9d51b471a3f3c14bfe95dde3407386/python_multipart-0.0.29-py3-none-any.whl", hash = "sha256:2ddcc971cef266225f54f552d8fa10bcfbb1f14446caec199060daac59ff2d69", size = 29640, upload-time = "2026-05-17T17:29:45.69Z" }, +] + [[package]] name = "pytz" version = "2026.2" @@ -1584,6 +1593,7 @@ dependencies = [ { name = "psycopg-pool" }, { name = "pydantic" }, { name = "python-dotenv" }, + { name = "python-multipart" }, { name = "sqlalchemy" }, { name = "tomli" }, { name = "trafilatura" }, @@ -1617,6 +1627,7 @@ requires-dist = [ { name = "psycopg-pool", specifier = ">=3.2" }, { name = "pydantic", specifier = ">=2.0.0" }, { name = "python-dotenv", specifier = ">=1.0.0" }, + { name = "python-multipart", specifier = ">=0.0.20" }, { name = "sqlalchemy", specifier = ">=2.0.0" }, { name = "tomli", specifier = ">=2.0.0" }, { name = "trafilatura", specifier = ">=1.9.0" },