second-brain/src/second_brain/web/app.py
Travis Herbranson 5131f3ccf6 web+cli: edit-mode for source metadata + retry/re-run actions
Adds per-record edit mode and retry to the source detail page, plus a
CLI `second-brain retry <id>` for parity.

Edit mode exposes domain (select), title, focus on /sources/{id}.
URL stays read-only — it's the UNIQUE dedupe key and the embeddings
identity tuple. Save goes through a new service helper
`update_source_metadata` so the web route stays a thin wrapper.

Retry exposes two modes, both available on any record:
  - full: status -> PENDING (re-pull + re-process)
  - extract: status -> TRANSCRIBED (re-extract only, requires an
    existing transcript; offered when source_type=video and a
    transcript is present, to avoid the expensive re-download path)

Both modes clear claimed_by/claimed_at and error_message so the queue
dance picks it up cleanly.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
2026-05-25 16:08:26 -04:00

680 lines
23 KiB
Python

"""
FastAPI web interface for second-brain.
Ported and extended from xtract/webapp.py.
New additions:
- Domain and status filter params on the source list
- Source detail shows the full ExtractionResult
- Accept / reject HTMX actions
- Queue view (pending + in-progress sources)
"""
from __future__ import annotations
from pathlib import Path
from typing import List, Optional
from fastapi import FastAPI, Form, HTTPException, Request
from fastapi.responses import HTMLResponse
from fastapi.templating import Jinja2Templates
from pydantic import BaseModel
from sqlalchemy import func
from second_brain.config import DOMAINS
from second_brain.database import get_database
from second_brain.models import Extraction, Source, SourceStatus
from second_brain.settings_store import (
get_settings,
parse_int_or_none,
parse_time_or_none,
update_settings,
)
from second_brain.sources_service import (
DEFAULT_PLAYLIST_MAX_ITEMS,
RETRY_EXTRACT,
RETRY_FULL,
RETRY_MODES,
add_playlist,
add_source,
is_youtube_playlist_url,
reset_source_for_retry,
update_source_metadata,
)
# ---------------------------------------------------------------------------
# App setup
# ---------------------------------------------------------------------------
app = FastAPI(
title="second-brain",
description="Personal knowledge extraction pipeline",
version="0.1.0",
)
_templates_dir = Path(__file__).parent / "templates"
templates = Jinja2Templates(directory=str(_templates_dir))
# ---------------------------------------------------------------------------
# Pydantic response models
# ---------------------------------------------------------------------------
class SourceResponse(BaseModel):
id: int
url: str
title: Optional[str]
domain: str
status: str
source_type: str
focus: Optional[str]
ingested_at: str
has_extraction: bool
class Config:
from_attributes = True
class ExtractionResponse(BaseModel):
summary: Optional[str]
key_points: Optional[list]
entities: Optional[list]
claims: Optional[list]
open_questions: Optional[list]
contradictions: Optional[list]
class Config:
from_attributes = True
# ---------------------------------------------------------------------------
# HTML routes
# ---------------------------------------------------------------------------
@app.get("/", response_class=HTMLResponse)
async def index(
request: Request,
domain: Optional[str] = None,
status: Optional[str] = None,
):
"""Main source list with optional domain/status filters."""
db = get_database()
with db.session() as sess:
q = sess.query(Source)
if domain and domain in DOMAINS:
q = q.filter(Source.domain == domain)
if status:
try:
q = q.filter(Source.status == SourceStatus[status.upper()])
except KeyError:
pass
sources = q.order_by(Source.ingested_at.desc()).all()
source_list = [_source_to_dict(s) for s in sources]
return templates.TemplateResponse(
request,
"index.html",
{
"sources": source_list,
"domains": DOMAINS,
"statuses": [s.value for s in SourceStatus],
"selected_domain": domain or "",
"selected_status": status or "",
},
)
def _render_source_detail(
request: Request,
source_id: int,
*,
edit: bool = False,
flash: Optional[str] = None,
error: Optional[str] = None,
form_overrides: Optional[dict] = None,
) -> HTMLResponse:
"""Single render path for /sources/{id} — used by GET + every HTMX
mutation endpoint so each action gets a consistent re-render.
`form_overrides` lets a failed edit-save bounce show the user's
in-progress values instead of the row's stored values.
"""
db = get_database()
with db.session() as sess:
source = sess.query(Source).filter(Source.id == source_id).first()
if not source:
raise HTTPException(status_code=404, detail="Source not found")
source_data = _source_to_dict(source)
extraction_data = (
_extraction_to_dict(source.extraction) if source.extraction else None
)
if form_overrides:
source_data = {**source_data, **form_overrides}
return templates.TemplateResponse(
request,
"source_detail.html",
{
"source": source_data,
"extraction": extraction_data,
"edit_mode": edit,
"flash": flash,
"error": error,
"domains": DOMAINS,
},
)
@app.get("/sources/{source_id}", response_class=HTMLResponse)
async def source_detail(request: Request, source_id: int, edit: bool = False):
"""Source detail page with extraction results and accept/reject actions.
`?edit=1` opens the metadata editor inline. Editing exposes domain,
title, and focus — URL stays read-only since it's the UNIQUE dedupe
key for the queue and the embeddings identity tuple.
"""
return _render_source_detail(request, source_id, edit=edit)
_IN_FLIGHT_STATUSES = [
SourceStatus.PENDING,
SourceStatus.PULLED,
SourceStatus.TRANSCRIBED,
]
def _build_queue_context(
sess,
*,
add_flash: Optional[str] = None,
add_error: Optional[str] = None,
) -> dict:
"""Context for queue.html / _queue_body.html (in-flight source list + form)."""
sources = (
sess.query(Source)
.filter(Source.status.in_(_IN_FLIGHT_STATUSES))
.order_by(Source.ingested_at.asc())
.all()
)
return {
"sources": [_source_to_dict(s) for s in sources],
"domains": DOMAINS,
"add_flash": add_flash,
"add_error": add_error,
}
@app.get("/queue", response_class=HTMLResponse)
async def queue_view(request: Request):
"""Queue view: pending and in-flight sources, with an add-to-queue form."""
db = get_database()
with db.session() as sess:
ctx = _build_queue_context(sess)
return templates.TemplateResponse(request, "queue.html", ctx)
# ---------------------------------------------------------------------------
# HTMX action endpoints
# ---------------------------------------------------------------------------
@app.post("/sources/{source_id}/accept")
async def accept_source(request: Request, source_id: int):
"""Mark a source as accepted (ready for wiki compilation)."""
db = get_database()
with db.session() as sess:
source = sess.query(Source).filter(Source.id == source_id).first()
if not source:
raise HTTPException(status_code=404, detail="Source not found")
source.status = SourceStatus.ACCEPTED
return _render_source_detail(
request,
source_id,
flash="Accepted — will be included in next wiki compile.",
)
@app.post("/sources/{source_id}/reject")
async def reject_source(request: Request, source_id: int):
"""Mark a source as failed/rejected (excluded from wiki)."""
db = get_database()
with db.session() as sess:
source = sess.query(Source).filter(Source.id == source_id).first()
if not source:
raise HTTPException(status_code=404, detail="Source not found")
source.status = SourceStatus.FAILED
return _render_source_detail(request, source_id, flash="Rejected.")
@app.post("/sources/{source_id}/edit", response_class=HTMLResponse)
async def edit_source_metadata(
request: Request,
source_id: int,
domain: str = Form(...),
title: Optional[str] = Form(None),
focus: Optional[str] = Form(None),
):
"""HTMX edit-mode save. Updates domain/title/focus only — URL is
intentionally not editable (see service-layer comment)."""
db = get_database()
try:
with db.session() as sess:
update_source_metadata(
sess,
source_id=source_id,
domain=domain,
title=title,
focus=focus,
)
except LookupError as exc:
raise HTTPException(status_code=404, detail=str(exc)) from exc
except ValueError as exc:
# Bounce back to edit mode with the user's in-progress values so
# they don't have to retype them.
return _render_source_detail(
request,
source_id,
edit=True,
error=str(exc),
form_overrides={
"domain": domain,
"title": title or "",
"focus": focus or "",
},
)
return _render_source_detail(request, source_id, flash="Saved.")
@app.post("/sources/{source_id}/retry", response_class=HTMLResponse)
async def retry_source(
request: Request,
source_id: int,
mode: str = Form(RETRY_FULL),
):
"""HTMX retry/re-run. `mode=full` → PENDING (re-pull + re-process).
`mode=extract` → TRANSCRIBED (re-extract only; requires an existing
transcript). Always clears the claim pair and error_message."""
if mode not in RETRY_MODES:
raise HTTPException(status_code=400, detail=f"unknown mode {mode!r}")
db = get_database()
try:
with db.session() as sess:
source = reset_source_for_retry(
sess, source_id=source_id, mode=mode
)
new_status = source.status.value
except LookupError as exc:
raise HTTPException(status_code=404, detail=str(exc)) from exc
except ValueError as exc:
return _render_source_detail(request, source_id, error=str(exc))
if mode == RETRY_EXTRACT:
flash = f"Re-queued for extraction (status → {new_status})."
else:
flash = f"Re-queued from scratch (status → {new_status})."
return _render_source_detail(request, source_id, flash=flash)
# ---------------------------------------------------------------------------
# JSON API endpoints
# ---------------------------------------------------------------------------
@app.get("/api/sources", response_model=List[SourceResponse])
async def api_sources(
domain: Optional[str] = None,
status: Optional[str] = None,
):
db = get_database()
with db.session() as sess:
q = sess.query(Source)
if domain:
q = q.filter(Source.domain == domain)
if status:
try:
q = q.filter(Source.status == SourceStatus[status.upper()])
except KeyError:
pass
sources = q.order_by(Source.ingested_at.desc()).all()
return [_source_to_response(s) for s in sources]
@app.get("/api/sources/{source_id}/extraction", response_model=ExtractionResponse)
async def api_extraction(source_id: int):
db = get_database()
with db.session() as sess:
extraction = sess.query(Extraction).filter_by(source_id=source_id).first()
if not extraction:
raise HTTPException(status_code=404, detail="No extraction for this source")
return ExtractionResponse(
summary=extraction.summary,
key_points=extraction.key_points,
entities=extraction.entities,
claims=extraction.claims,
open_questions=extraction.open_questions,
contradictions=extraction.contradictions,
)
# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------
def _source_to_dict(s: Source) -> dict:
return {
"id": s.id,
"url": s.url,
"title": s.title or s.url,
"domain": s.domain,
"status": s.status.value,
"source_type": s.source_type.value,
"focus": s.focus,
"ingested_at": s.ingested_at.strftime("%Y-%m-%d %H:%M") if s.ingested_at else "",
"has_extraction": s.extraction is not None,
"has_transcript": bool((s.transcript_text or "").strip()),
"error_message": s.error_message,
}
def _extraction_to_dict(e: Optional[Extraction]) -> Optional[dict]:
if e is None:
return None
return {
"summary": e.summary,
"key_points": e.key_points or [],
"entities": e.entities or [],
"claims": e.claims or [],
"open_questions": e.open_questions or [],
"contradictions": e.contradictions or [],
"input_tokens": e.input_tokens,
"output_tokens": e.output_tokens,
}
def _playlist_flash(pr) -> str:
"""Human-readable summary for a playlist fan-out. Matches the CLI tone."""
if pr.expanded == 0:
return "Playlist returned no entries"
parts = [f"Queued {pr.added} video{'' if pr.added == 1 else 's'}"]
extras = []
if pr.duplicates:
extras.append(f"{pr.duplicates} duplicate{'' if pr.duplicates == 1 else 's'} skipped")
if pr.failed:
extras.append(f"{pr.failed} failed")
if extras:
parts.append(f"({', '.join(extras)})")
return " ".join(parts)
def _source_to_response(s: Source) -> SourceResponse:
return SourceResponse(
id=s.id,
url=s.url,
title=s.title,
domain=s.domain,
status=s.status.value,
source_type=s.source_type.value,
focus=s.focus,
ingested_at=s.ingested_at.isoformat() if s.ingested_at else "",
has_extraction=s.extraction is not None,
)
# ---------------------------------------------------------------------------
# Dashboard — pipeline observability
# ---------------------------------------------------------------------------
def _build_dashboard_context(
sess,
*,
add_flash: Optional[str] = None,
add_error: Optional[str] = None,
) -> dict:
"""Collect everything the dashboard partial needs from one session.
Materialise inside the session so the keys stay plain str → int /
detached dicts; templates never touch a closed session.
"""
rows = (
sess.query(Source.status, func.count(Source.id))
.group_by(Source.status)
.all()
)
counts = {s.value: 0 for s in SourceStatus}
for status, n in rows:
counts[status.value] = int(n)
total = sum(counts.values())
recent_rows = (
sess.query(Source).order_by(Source.ingested_at.desc()).limit(10).all()
)
recent = [_source_to_dict(s) for s in recent_rows]
settings_summary = _settings_to_dict(get_settings(sess))
return {
"counts": counts,
"total": total,
"recent": recent,
"statuses_in_order": [s.value for s in SourceStatus],
"settings": settings_summary,
"domains": DOMAINS,
"add_flash": add_flash,
"add_error": add_error,
}
@app.get("/dashboard", response_class=HTMLResponse)
async def dashboard(request: Request):
"""Overview: status counts, recent activity, and a settings snapshot."""
db = get_database()
with db.session() as sess:
ctx = _build_dashboard_context(sess)
return templates.TemplateResponse(request, "dashboard.html", ctx)
@app.post("/sources/add", response_class=HTMLResponse)
async def sources_add(
request: Request,
url: str = Form(...),
domain: str = Form("development"),
title: Optional[str] = Form(None),
focus: Optional[str] = Form(None),
):
"""HTMX endpoint — queue a source, re-render the dashboard body."""
db = get_database()
add_flash: Optional[str] = None
add_error: Optional[str] = None
try:
with db.session() as sess:
if is_youtube_playlist_url(url):
pr = add_playlist(
sess,
url=url,
domain=domain,
focus=focus,
max_items=DEFAULT_PLAYLIST_MAX_ITEMS,
)
add_flash = _playlist_flash(pr)
else:
result = add_source(
sess, url=url, domain=domain, focus=focus, title=title
)
# Snapshot the message inside the session — `result.source`
# detaches on commit.
if result.added:
add_flash = (
f"Queued {result.source.source_type.value}: "
f"{result.source.url}"
)
else:
add_flash = (
f"Already queued (id={result.source.id}, "
f"status={result.source.status.value})"
)
except ValueError as exc:
add_error = str(exc)
# Dispatch the response partial based on which page submitted the form.
# HTMX always sends HX-Current-URL with the request URL of the user's
# current page; we parse the path and pick the matching body partial.
# Same /sources/add endpoint, same service call — only the rendered
# region differs, so each page refreshes its own list/counts inline.
from urllib.parse import urlparse as _urlparse
hx_url = request.headers.get("HX-Current-URL") or request.headers.get(
"Referer", ""
)
source_path = _urlparse(hx_url).path if hx_url else ""
with db.session() as sess:
if source_path.rstrip("/") == "/queue":
ctx = _build_queue_context(
sess, add_flash=add_flash, add_error=add_error
)
template_name = "_queue_body.html"
else:
ctx = _build_dashboard_context(
sess, add_flash=add_flash, add_error=add_error
)
template_name = "_dashboard_body.html"
return templates.TemplateResponse(request, template_name, ctx)
# ---------------------------------------------------------------------------
# Settings — view + HTMX save
# ---------------------------------------------------------------------------
@app.get("/settings", response_class=HTMLResponse)
async def settings_view(request: Request):
db = get_database()
with db.session() as sess:
row = get_settings(sess)
ctx = _settings_to_dict(row)
return templates.TemplateResponse(
request,
"settings.html",
{"settings": ctx, "flash": None, "error": None},
)
@app.post("/settings/save", response_class=HTMLResponse)
async def settings_save(
request: Request,
transcription_enabled: Optional[str] = Form(None),
transcription_active_hours_start: Optional[str] = Form(None),
transcription_active_hours_end: Optional[str] = Form(None),
transcription_max_concurrent_gpu_jobs: Optional[str] = Form(None),
transcription_max_video_length_seconds: Optional[str] = Form(None),
transcription_max_items_per_run: Optional[str] = Form(None),
extraction_enabled: Optional[str] = Form(None),
extraction_active_hours_start: Optional[str] = Form(None),
extraction_active_hours_end: Optional[str] = Form(None),
extraction_max_items_per_run: Optional[str] = Form(None),
):
"""HTMX save endpoint — returns the rendered settings card with a flash."""
db = get_database()
error: Optional[str] = None
flash: Optional[str] = None
try:
fields = {
# Checkboxes only show up in the form payload when checked. The
# template emits a hidden "_present" companion so we can tell
# "unchecked" apart from "field missing because of a partial post".
"transcription_enabled": transcription_enabled == "on",
"transcription_active_hours_start": parse_time_or_none(
transcription_active_hours_start
),
"transcription_active_hours_end": parse_time_or_none(
transcription_active_hours_end
),
"transcription_max_concurrent_gpu_jobs": int(
transcription_max_concurrent_gpu_jobs or "1"
),
"transcription_max_video_length_seconds": parse_int_or_none(
transcription_max_video_length_seconds
),
"transcription_max_items_per_run": int(
transcription_max_items_per_run or "10"
),
"extraction_enabled": extraction_enabled == "on",
"extraction_active_hours_start": parse_time_or_none(
extraction_active_hours_start
),
"extraction_active_hours_end": parse_time_or_none(
extraction_active_hours_end
),
"extraction_max_items_per_run": int(extraction_max_items_per_run or "20"),
}
# Cheap inline validation — keep counts non-negative and gpu jobs >= 1.
if fields["transcription_max_concurrent_gpu_jobs"] < 1:
raise ValueError("transcription_max_concurrent_gpu_jobs must be ≥ 1")
if fields["transcription_max_items_per_run"] < 0:
raise ValueError("transcription_max_items_per_run must be ≥ 0")
if fields["extraction_max_items_per_run"] < 0:
raise ValueError("extraction_max_items_per_run must be ≥ 0")
if (
fields["transcription_max_video_length_seconds"] is not None
and fields["transcription_max_video_length_seconds"] < 0
):
raise ValueError("transcription_max_video_length_seconds must be ≥ 0")
with db.session() as sess:
row = update_settings(sess, **fields)
ctx = _settings_to_dict(row)
flash = "Saved."
except (ValueError, KeyError) as exc:
error = str(exc)
# Re-render the form using whatever the user submitted so they don't
# lose in-progress edits on a validation bounce.
with db.session() as sess:
ctx = _settings_to_dict(get_settings(sess))
return templates.TemplateResponse(
request,
"_settings_card.html",
{"settings": ctx, "flash": flash, "error": error},
)
# ---------------------------------------------------------------------------
# Settings serialiser
# ---------------------------------------------------------------------------
def _settings_to_dict(row) -> dict:
"""Render the settings row as a template-friendly dict (no ORM bleed)."""
def t(v):
return v.strftime("%H:%M") if v is not None else ""
return {
"transcription_enabled": row.transcription_enabled,
"transcription_active_hours_start": t(row.transcription_active_hours_start),
"transcription_active_hours_end": t(row.transcription_active_hours_end),
"transcription_max_concurrent_gpu_jobs": row.transcription_max_concurrent_gpu_jobs,
"transcription_max_video_length_seconds": row.transcription_max_video_length_seconds,
"transcription_max_items_per_run": row.transcription_max_items_per_run,
"extraction_enabled": row.extraction_enabled,
"extraction_active_hours_start": t(row.extraction_active_hours_start),
"extraction_active_hours_end": t(row.extraction_active_hours_end),
"extraction_max_items_per_run": row.extraction_max_items_per_run,
"updated_at": row.updated_at.strftime("%Y-%m-%d %H:%M:%S")
if row.updated_at
else "",
}