From 956bf8d0c3d537a23cc932bad04bd4a3265b5ee5 Mon Sep 17 00:00:00 2001 From: Travis Herbranson Date: Mon, 25 May 2026 12:59:02 -0400 Subject: [PATCH] web: add-to-queue form on the dashboard (HTMX, parity with CLI add) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Extracts the `second-brain add` CLI's queueing logic into a service module (sources_service.add_source) so the CLI and the new web POST route share the same validation, dedupe, and source_type heuristic — no behaviour drift between the two entry points. UI: - The dashboard body (Pipeline counts + Settings snapshot + Recent activity) moves into _dashboard_body.html, wrapped in `
`. HTMX targets that id for swap. - A new "Add to queue" section sits at the top of the partial: URL (required, type=url), Domain (select matching DOMAINS), Title and Focus (both optional, match the CLI flags). hx-post=/sources/add, hx-target=#dashboard-body, hx-swap=outerHTML — same pattern as the settings save form. Route: - POST /sources/add calls sources_service.add_source inside a single transaction, then re-renders _dashboard_body.html so the pipeline counts and recent-activity list update in place. Flash slots above the form report: ✓ Queued : on a fresh add ✓ Already queued (id=…, status=…) on a dupe (matches CLI text) ✗ on bad URL / unknown domain CLI: - `second-brain add` now delegates to the same service. Field values for the success echo are captured inside the session block so a post-commit detached-instance access can't fail. Bad input exits 2 with a clear stderr message instead of raising. Live-verified through the web container on herbys-dev: dashboard renders the form, happy add lands a row, dup-detect matches the CLI phrasing, two validation errors surface as red flashes, swap target id survives across swaps, no tracebacks in uvicorn logs. --- src/second_brain/main.py | 55 ++--- src/second_brain/sources_service.py | 112 +++++++++ src/second_brain/web/app.py | 114 +++++++--- .../web/templates/_dashboard_body.html | 215 ++++++++++++++++++ src/second_brain/web/templates/dashboard.html | 148 +----------- 5 files changed, 431 insertions(+), 213 deletions(-) create mode 100644 src/second_brain/sources_service.py create mode 100644 src/second_brain/web/templates/_dashboard_body.html diff --git a/src/second_brain/main.py b/src/second_brain/main.py index 2e96bee..6b91e47 100644 --- a/src/second_brain/main.py +++ b/src/second_brain/main.py @@ -11,8 +11,8 @@ Commands: from __future__ import annotations +import sys from typing import Optional -from urllib.parse import urlparse import click @@ -50,32 +50,33 @@ def cli() -> None: @click.option("--title", "-t", default=None, help="Override title.") def add(url: str, domain: str, focus: Optional[str], title: Optional[str]) -> None: """Queue a source URL for processing.""" + from second_brain.sources_service import add_source + config = load_config() config.ensure_dirs() db = get_database() - source_type = SourceType.ARTICLE if _is_article(url) else SourceType.VIDEO - - with db.session() as sess: - existing = sess.query(Source).filter_by(url=url).first() - if existing: - click.echo( - f"[add] Already queued (id={existing.id}, status={existing.status.value})" + try: + with db.session() as sess: + result = add_source( + sess, url=url, domain=domain, focus=focus, title=title ) - return + # Capture display fields inside the session — `result.source` + # detaches on commit and accessing it later would error. + source_id = result.source.id + source_url = result.source.url + source_type_value = result.source.source_type.value + existing_status = result.source.status.value + added = result.added + except ValueError as exc: + click.echo(f"[add] Error: {exc}", err=True) + sys.exit(2) - source = Source( - url=url, - title=title, - domain=domain, - focus=focus, - source_type=source_type, - status=SourceStatus.PENDING, - ingested_at=utcnow(), - ) - sess.add(source) + if not added: + click.echo(f"[add] Already queued (id={source_id}, status={existing_status})") + return - click.echo(f"[add] Queued {source_type.value}: {url}") + click.echo(f"[add] Queued {source_type_value}: {source_url}") click.echo(f" domain={domain}" + (f" focus={focus}" if focus else "")) @@ -348,19 +349,5 @@ def transcribe_worker( click.echo(f"[transcribe-worker] exiting; processed={processed}") -# --------------------------------------------------------------------------- -# Helpers -# --------------------------------------------------------------------------- - - -def _is_article(url: str) -> bool: - parsed = urlparse(url) - video_hosts = { - "youtube.com", "www.youtube.com", "youtu.be", - "vimeo.com", "www.vimeo.com", - } - return parsed.netloc not in video_hosts - - if __name__ == "__main__": cli() diff --git a/src/second_brain/sources_service.py b/src/second_brain/sources_service.py new file mode 100644 index 0000000..15c7dd9 --- /dev/null +++ b/src/second_brain/sources_service.py @@ -0,0 +1,112 @@ +"""Service-layer helpers for the `sources` table. + +Single source of truth for "queue a URL" — both the `second-brain add` +CLI and the web POST /sources/add route call into here, so they can't +drift on validation, dedupe, or default semantics. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Optional +from urllib.parse import urlparse + +from sqlalchemy.orm import Session + +from second_brain.config import DOMAINS +from second_brain.models import Source, SourceStatus, SourceType, utcnow + + +_VIDEO_HOSTS = { + "youtube.com", + "www.youtube.com", + "youtu.be", + "vimeo.com", + "www.vimeo.com", +} + + +def is_article_url(url: str) -> bool: + """Match the CLI's `_is_article` heuristic: anything not on a known + video host is treated as an article.""" + parsed = urlparse(url) + return parsed.netloc not in _VIDEO_HOSTS + + +def validate_url(url: str) -> str: + """Normalize and validate the input URL. Raises ValueError on bad shape.""" + if url is None: + raise ValueError("url is required") + url = url.strip() + if not url: + raise ValueError("url is required") + parsed = urlparse(url) + if parsed.scheme not in ("http", "https"): + raise ValueError("url must use http or https") + if not parsed.netloc: + raise ValueError("url is missing a host") + return url + + +@dataclass +class AddResult: + """Outcome of `add_source`. `added=False` means the URL was already queued.""" + + source: Source + added: bool + + +def add_source( + sess: Session, + *, + url: str, + domain: str = "development", + focus: Optional[str] = None, + title: Optional[str] = None, +) -> AddResult: + """Queue a single source URL. + + Mirrors the CLI add path exactly: + - normalize + validate URL + - reject unknown domains + - if the URL is already queued, return the existing row with added=False + - otherwise insert a PENDING row and return it with added=True + + Does NOT open or commit a session — caller owns the transaction + boundary (matches the rest of the codebase). For the CLI, that's + `db.session()`; for the web route, the request handler's + `with db.session()` block. + """ + url = validate_url(url) + + if domain not in DOMAINS: + raise ValueError( + f"unknown domain {domain!r}; valid: {', '.join(DOMAINS)}" + ) + + title = (title or "").strip() or None + focus = (focus or "").strip() or None + + existing = sess.query(Source).filter_by(url=url).first() + if existing is not None: + return AddResult(source=existing, added=False) + + source_type = ( + SourceType.ARTICLE if is_article_url(url) else SourceType.VIDEO + ) + + source = Source( + url=url, + title=title, + domain=domain, + focus=focus, + source_type=source_type, + status=SourceStatus.PENDING, + ingested_at=utcnow(), + ) + sess.add(source) + sess.flush() + return AddResult(source=source, added=True) + + +__all__ = ["AddResult", "add_source", "is_article_url", "validate_url"] diff --git a/src/second_brain/web/app.py b/src/second_brain/web/app.py index 59f8304..880c0fb 100644 --- a/src/second_brain/web/app.py +++ b/src/second_brain/web/app.py @@ -29,6 +29,7 @@ from second_brain.settings_store import ( parse_time_or_none, update_settings, ) +from second_brain.sources_service import add_source # --------------------------------------------------------------------------- # App setup @@ -309,45 +310,94 @@ def _source_to_response(s: Source) -> SourceResponse: # --------------------------------------------------------------------------- +def _build_dashboard_context( + sess, + *, + add_flash: Optional[str] = None, + add_error: Optional[str] = None, +) -> dict: + """Collect everything the dashboard partial needs from one session. + + Materialise inside the session so the keys stay plain str → int / + detached dicts; templates never touch a closed session. + """ + rows = ( + sess.query(Source.status, func.count(Source.id)) + .group_by(Source.status) + .all() + ) + counts = {s.value: 0 for s in SourceStatus} + for status, n in rows: + counts[status.value] = int(n) + total = sum(counts.values()) + + recent_rows = ( + sess.query(Source).order_by(Source.ingested_at.desc()).limit(10).all() + ) + recent = [_source_to_dict(s) for s in recent_rows] + + settings_summary = _settings_to_dict(get_settings(sess)) + + return { + "counts": counts, + "total": total, + "recent": recent, + "statuses_in_order": [s.value for s in SourceStatus], + "settings": settings_summary, + "domains": DOMAINS, + "add_flash": add_flash, + "add_error": add_error, + } + + @app.get("/dashboard", response_class=HTMLResponse) async def dashboard(request: Request): """Overview: status counts, recent activity, and a settings snapshot.""" db = get_database() with db.session() as sess: - # Counts by status. Materialise inside the session so the keys stay - # plain str → int and templates don't try to touch a closed session. - rows = ( - sess.query(Source.status, func.count(Source.id)) - .group_by(Source.status) - .all() + ctx = _build_dashboard_context(sess) + + return templates.TemplateResponse(request, "dashboard.html", ctx) + + +@app.post("/sources/add", response_class=HTMLResponse) +async def sources_add( + request: Request, + url: str = Form(...), + domain: str = Form("development"), + title: Optional[str] = Form(None), + focus: Optional[str] = Form(None), +): + """HTMX endpoint — queue a source, re-render the dashboard body.""" + db = get_database() + add_flash: Optional[str] = None + add_error: Optional[str] = None + + try: + with db.session() as sess: + result = add_source( + sess, url=url, domain=domain, focus=focus, title=title + ) + # Snapshot the message inside the session — `result.source` + # detaches on commit. + if result.added: + add_flash = ( + f"Queued {result.source.source_type.value}: {result.source.url}" + ) + else: + add_flash = ( + f"Already queued (id={result.source.id}, " + f"status={result.source.status.value})" + ) + except ValueError as exc: + add_error = str(exc) + + with db.session() as sess: + ctx = _build_dashboard_context( + sess, add_flash=add_flash, add_error=add_error ) - counts = {s.value: 0 for s in SourceStatus} - for status, n in rows: - counts[status.value] = int(n) - total = sum(counts.values()) - recent_rows = ( - sess.query(Source) - .order_by(Source.ingested_at.desc()) - .limit(10) - .all() - ) - recent = [_source_to_dict(s) for s in recent_rows] - - settings_row = get_settings(sess) - settings_summary = _settings_to_dict(settings_row) - - return templates.TemplateResponse( - request, - "dashboard.html", - { - "counts": counts, - "total": total, - "recent": recent, - "statuses_in_order": [s.value for s in SourceStatus], - "settings": settings_summary, - }, - ) + return templates.TemplateResponse(request, "_dashboard_body.html", ctx) # --------------------------------------------------------------------------- diff --git a/src/second_brain/web/templates/_dashboard_body.html b/src/second_brain/web/templates/_dashboard_body.html new file mode 100644 index 0000000..f1d34ec --- /dev/null +++ b/src/second_brain/web/templates/_dashboard_body.html @@ -0,0 +1,215 @@ +{# HTMX-replaceable dashboard body. POST /sources/add re-renders this + partial; hx-target="#dashboard-body" + hx-swap="outerHTML" keeps the + wrapping div in place for the next swap. #} +{% set status_colors = { + 'pending': 'bg-gray-100 text-gray-700', + 'pulled': 'bg-blue-100 text-blue-700', + 'transcribed': 'bg-yellow-100 text-yellow-700', + 'analyzed': 'bg-orange-100 text-orange-700', + 'accepted': 'bg-green-100 text-green-700', + 'published': 'bg-purple-100 text-purple-700', + 'failed': 'bg-red-100 text-red-700' +} %} +{% set domain_colors = { + 'development': 'bg-cyan-50 text-cyan-700 border-cyan-200', + 'content': 'bg-pink-50 text-pink-700 border-pink-200', + 'business': 'bg-amber-50 text-amber-700 border-amber-200', + 'homelab': 'bg-teal-50 text-teal-700 border-teal-200' +} %} +
+ + +
+
+

+ Add to queue +

+

video on a known host (YouTube / Vimeo) → tower; otherwise treated as an article

+
+ + {% if add_flash %} +
+ ✓ {{ add_flash }} +
+ {% endif %} + {% if add_error %} +
+ ✗ {{ add_error }} +
+ {% endif %} + +
+ +
+ + +
+ +
+ + +
+ +
+ + +
+ +
+ +
+ +
+ + +
+
+
+ + +
+

+ Pipeline +

+
+ {% for status in statuses_in_order %} + +
{{ counts.get(status, 0) }}
+
+ + {{ status }} + +
+ + {% endfor %} +
+
+ + +
+ + +
+
+

+ Settings +

+ + edit → + +
+ +
+
+
Transcription
+
+ {% if settings.transcription_enabled %} + enabled + {% else %} + paused + {% endif %} +
+
+
+
active window
+
+ {% if settings.transcription_active_hours_start or settings.transcription_active_hours_end %} + {{ settings.transcription_active_hours_start or "—" }} – {{ settings.transcription_active_hours_end or "—" }} + {% else %} + always + {% endif %} +
+
+
+
GPU jobs
+
{{ settings.transcription_max_concurrent_gpu_jobs }}
+
+
+
max items/run
+
{{ settings.transcription_max_items_per_run }}
+
+ +
+ +
+
Extraction
+
+ {% if settings.extraction_enabled %} + enabled + {% else %} + paused + {% endif %} +
+
+
+
active window
+
+ {% if settings.extraction_active_hours_start or settings.extraction_active_hours_end %} + {{ settings.extraction_active_hours_start or "—" }} – {{ settings.extraction_active_hours_end or "—" }} + {% else %} + always + {% endif %} +
+
+
+
max items/run
+
{{ settings.extraction_max_items_per_run }}
+
+
+ +

+ updated {{ settings.updated_at }} +

+
+ + +
+

+ Recent activity +

+ {% if recent %} +
+ {% for source in recent %} + +
+

{{ source.title }}

+

{{ source.url }}

+
+
+ + {{ source.status }} + + + {{ source.domain }} + + {{ source.ingested_at }} +
+ + {% endfor %} +
+ {% else %} +

No sources yet.

+ {% endif %} +
+
+
diff --git a/src/second_brain/web/templates/dashboard.html b/src/second_brain/web/templates/dashboard.html index 4e92937..d362e00 100644 --- a/src/second_brain/web/templates/dashboard.html +++ b/src/second_brain/web/templates/dashboard.html @@ -3,156 +3,10 @@ {% block title %}Dashboard — second-brain{% endblock %} {% block content %} -{% set status_colors = { - 'pending': 'bg-gray-100 text-gray-700', - 'pulled': 'bg-blue-100 text-blue-700', - 'transcribed': 'bg-yellow-100 text-yellow-700', - 'analyzed': 'bg-orange-100 text-orange-700', - 'accepted': 'bg-green-100 text-green-700', - 'published': 'bg-purple-100 text-purple-700', - 'failed': 'bg-red-100 text-red-700' -} %} -{% set domain_colors = { - 'development': 'bg-cyan-50 text-cyan-700 border-cyan-200', - 'content': 'bg-pink-50 text-pink-700 border-pink-200', - 'business': 'bg-amber-50 text-amber-700 border-amber-200', - 'homelab': 'bg-teal-50 text-teal-700 border-teal-200' -} %} -

Dashboard

total sources: {{ total }}
- -
-

- Pipeline -

-
- {% for status in statuses_in_order %} - -
{{ counts.get(status, 0) }}
-
- - {{ status }} - -
- - {% endfor %} -
-
- - -
- - -
-
-

- Settings -

- - edit → - -
- -
-
-
Transcription
-
- {% if settings.transcription_enabled %} - enabled - {% else %} - paused - {% endif %} -
-
-
-
active window
-
- {% if settings.transcription_active_hours_start or settings.transcription_active_hours_end %} - {{ settings.transcription_active_hours_start or "—" }} – {{ settings.transcription_active_hours_end or "—" }} - {% else %} - always - {% endif %} -
-
-
-
GPU jobs
-
{{ settings.transcription_max_concurrent_gpu_jobs }}
-
-
-
max items/run
-
{{ settings.transcription_max_items_per_run }}
-
- -
- -
-
Extraction
-
- {% if settings.extraction_enabled %} - enabled - {% else %} - paused - {% endif %} -
-
-
-
active window
-
- {% if settings.extraction_active_hours_start or settings.extraction_active_hours_end %} - {{ settings.extraction_active_hours_start or "—" }} – {{ settings.extraction_active_hours_end or "—" }} - {% else %} - always - {% endif %} -
-
-
-
max items/run
-
{{ settings.extraction_max_items_per_run }}
-
-
- -

- updated {{ settings.updated_at }} -

-
- - -
-

- Recent activity -

- {% if recent %} -
- {% for source in recent %} - -
-

{{ source.title }}

-

{{ source.url }}

-
-
- - {{ source.status }} - - - {{ source.domain }} - - {{ source.ingested_at }} -
- - {% endfor %} -
- {% else %} -

No sources yet.

- {% endif %} -
-
+{% include "_dashboard_body.html" %} {% endblock %}