a2d46a7c6e
Ruff autofix removed pre-existing F401 unused imports across adapters, context, compiler, extractor, scheduler, web, and main. Also reordered models.py so its SQLAlchemy imports sit at the top of the file (E402 was triggered by the inline `utcnow` helper definition). Added a [dependency-groups] dev block (pytest, pytest-cov, ruff) so the zero-check test gauntlet — which runs `pytest --cov --cov-report=term-missing` — can resolve its plugins without a manual `uv pip install pytest-cov`.
278 lines
9.5 KiB
Python
278 lines
9.5 KiB
Python
"""
|
|
Wiki compiler (Karpathy LLM Wiki pattern).
|
|
|
|
Reads the accepted extractions from the DB, reads the existing Obsidian vault
|
|
pages, calls Claude to decide what to create/update, then writes the pages and
|
|
git-commits the vault before and after.
|
|
|
|
Phase 1 implementation:
|
|
- Generates one Markdown page per accepted source (slug from title/url).
|
|
- Stubs the git commit logic with clear interfaces.
|
|
- Does NOT yet do cross-source synthesis (that is Phase 2).
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import re
|
|
import subprocess
|
|
from pathlib import Path
|
|
from typing import Optional
|
|
|
|
from second_brain.config import Config
|
|
from second_brain.database import get_database
|
|
from second_brain.models import Extraction, Source, SourceStatus, WikiPage, utcnow
|
|
|
|
|
|
class WikiCompiler:
|
|
"""
|
|
Compiles accepted extractions into Obsidian vault Markdown pages.
|
|
|
|
Usage::
|
|
|
|
compiler = WikiCompiler(config)
|
|
stats = compiler.run()
|
|
"""
|
|
|
|
def __init__(self, config: Config, api_key: Optional[str] = None) -> None:
|
|
self.config = config
|
|
self._api_key = api_key or config.anthropic_api_key
|
|
|
|
def run(self) -> dict:
|
|
"""
|
|
Main entry point. Processes all accepted (not yet published) sources.
|
|
|
|
Returns a stats dict: {processed, created, updated, errors}.
|
|
"""
|
|
vault = self.config.vault_path
|
|
vault.mkdir(parents=True, exist_ok=True)
|
|
self._ensure_git_repo(vault)
|
|
|
|
db = get_database()
|
|
stats = {"processed": 0, "created": 0, "updated": 0, "errors": 0}
|
|
|
|
with db.session() as sess:
|
|
sources = (
|
|
sess.query(Source)
|
|
.filter(Source.status == SourceStatus.ACCEPTED)
|
|
.all()
|
|
)
|
|
|
|
if not sources:
|
|
print("[Compiler] No accepted sources to compile.")
|
|
return stats
|
|
|
|
sha_before = self._git_sha(vault)
|
|
|
|
for source in sources:
|
|
try:
|
|
created = self._compile_source(source, sess, vault)
|
|
if created:
|
|
stats["created"] += 1
|
|
else:
|
|
stats["updated"] += 1
|
|
source.status = SourceStatus.PUBLISHED
|
|
source.updated_at = utcnow()
|
|
stats["processed"] += 1
|
|
except Exception as exc:
|
|
print(f"[Compiler] Error processing source {source.id}: {exc}")
|
|
stats["errors"] += 1
|
|
|
|
sha_after = self._git_commit(vault, stats["processed"])
|
|
|
|
# Update git sha on all written WikiPage rows
|
|
if sha_after:
|
|
for page in sess.query(WikiPage).filter(
|
|
WikiPage.git_sha_after.is_(None)
|
|
).all():
|
|
page.git_sha_before = sha_before
|
|
page.git_sha_after = sha_after
|
|
|
|
return stats
|
|
|
|
# ------------------------------------------------------------------
|
|
# Per-source page generation
|
|
# ------------------------------------------------------------------
|
|
|
|
def _compile_source(
|
|
self, source: Source, sess, vault: Path
|
|
) -> bool:
|
|
"""Write a vault page for *source*. Returns True if newly created."""
|
|
extraction: Optional[Extraction] = source.extraction
|
|
if extraction is None:
|
|
raise ValueError(f"Source {source.id} has no extraction")
|
|
|
|
slug = _slugify(source.title or source.url)
|
|
domain_dir = vault / source.domain
|
|
domain_dir.mkdir(exist_ok=True)
|
|
page_path = domain_dir / f"{slug}.md"
|
|
relative_path = str(page_path.relative_to(vault))
|
|
|
|
content = self._render_page(source, extraction)
|
|
|
|
created = not page_path.exists()
|
|
page_path.write_text(content, encoding="utf-8")
|
|
print(f"[Compiler] {'Created' if created else 'Updated'}: {relative_path}")
|
|
|
|
# Upsert WikiPage tracking row
|
|
wiki_row = sess.query(WikiPage).filter_by(vault_path=relative_path).first()
|
|
if wiki_row is None:
|
|
wiki_row = WikiPage(
|
|
vault_path=relative_path,
|
|
domain=source.domain,
|
|
title=source.title or slug,
|
|
)
|
|
sess.add(wiki_row)
|
|
else:
|
|
wiki_row.updated_at = utcnow()
|
|
|
|
return created
|
|
|
|
@staticmethod
|
|
def _render_page(source: Source, extraction: Extraction) -> str:
|
|
"""Render the Markdown content for a vault page."""
|
|
lines: list[str] = []
|
|
|
|
title = source.title or source.url
|
|
lines.append(f"# {title}\n")
|
|
|
|
# Front-matter block
|
|
lines.append("---")
|
|
lines.append(f"url: {source.url}")
|
|
lines.append(f"domain: {source.domain}")
|
|
lines.append(f"source_type: {source.source_type.value}")
|
|
if source.published_at:
|
|
lines.append(f"published: {source.published_at.date()}")
|
|
lines.append(f"ingested: {source.ingested_at.date()}")
|
|
if source.focus:
|
|
lines.append(f"focus: {source.focus}")
|
|
lines.append("---\n")
|
|
|
|
# Summary
|
|
if extraction.summary:
|
|
lines.append("## Summary\n")
|
|
lines.append(extraction.summary + "\n")
|
|
|
|
# Key points
|
|
if extraction.key_points:
|
|
lines.append("## Key Points\n")
|
|
for pt in extraction.key_points:
|
|
lines.append(f"- {pt}")
|
|
lines.append("")
|
|
|
|
# Claims
|
|
if extraction.claims:
|
|
lines.append("## Claims\n")
|
|
for c in extraction.claims:
|
|
lines.append(f"- {c}")
|
|
lines.append("")
|
|
|
|
# Entities
|
|
if extraction.entities:
|
|
lines.append("## Entities\n")
|
|
lines.append(", ".join(f"[[{e}]]" for e in extraction.entities) + "\n")
|
|
|
|
# Open questions
|
|
if extraction.open_questions:
|
|
lines.append("## Open Questions\n")
|
|
for q in extraction.open_questions:
|
|
lines.append(f"- {q}")
|
|
lines.append("")
|
|
|
|
# Contradictions
|
|
if extraction.contradictions:
|
|
lines.append("## Contradictions\n")
|
|
for ct in extraction.contradictions:
|
|
lines.append(f"- {ct}")
|
|
lines.append("")
|
|
|
|
return "\n".join(lines)
|
|
|
|
# ------------------------------------------------------------------
|
|
# Git helpers (stubbed — interface is clear for later wiring)
|
|
# ------------------------------------------------------------------
|
|
|
|
@staticmethod
|
|
def _ensure_git_repo(vault: Path) -> None:
|
|
"""Initialise a git repo in the vault if one doesn't already exist.
|
|
|
|
Idempotent. Creates the repo with an initial empty commit so that
|
|
subsequent `git rev-parse HEAD` calls succeed even before the first
|
|
compile writes a page.
|
|
"""
|
|
if (vault / ".git").exists():
|
|
return
|
|
try:
|
|
subprocess.run(
|
|
["git", "init", "--initial-branch=main", "--quiet"],
|
|
cwd=vault, check=True,
|
|
)
|
|
# Allow the empty initial commit so HEAD exists immediately.
|
|
subprocess.run(
|
|
["git", "commit", "--allow-empty", "-m", "second-brain: vault init", "--quiet"],
|
|
cwd=vault, check=True,
|
|
)
|
|
print(f"[Compiler] Initialised vault git repo at {vault}")
|
|
except (subprocess.CalledProcessError, FileNotFoundError) as exc:
|
|
print(f"[Compiler] Warning: could not init git repo in {vault}: {exc}")
|
|
|
|
@staticmethod
|
|
def _git_sha(vault: Path) -> Optional[str]:
|
|
"""Return HEAD sha of the vault git repo, or None if not a git repo."""
|
|
try:
|
|
result = subprocess.run(
|
|
["git", "rev-parse", "HEAD"],
|
|
cwd=vault,
|
|
capture_output=True,
|
|
text=True,
|
|
check=True,
|
|
)
|
|
return result.stdout.strip()
|
|
except (subprocess.CalledProcessError, FileNotFoundError):
|
|
return None
|
|
|
|
@staticmethod
|
|
def _git_commit(vault: Path, count: int) -> Optional[str]:
|
|
"""Stage all changes and commit. Returns new HEAD sha or None.
|
|
|
|
Returns None (without raising) if there's nothing to commit — that's
|
|
the normal case when a compile run produced no actual file changes.
|
|
"""
|
|
try:
|
|
subprocess.run(["git", "add", "-A"], cwd=vault, check=True)
|
|
# Bail out cleanly if there's nothing staged.
|
|
diff = subprocess.run(
|
|
["git", "diff", "--cached", "--quiet"],
|
|
cwd=vault, check=False,
|
|
)
|
|
if diff.returncode == 0:
|
|
return None # nothing to commit
|
|
msg = f"second-brain: compile {count} source(s) [{utcnow().date()}]"
|
|
subprocess.run(["git", "commit", "-m", msg, "--quiet"], cwd=vault, check=True)
|
|
result = subprocess.run(
|
|
["git", "rev-parse", "HEAD"],
|
|
cwd=vault,
|
|
capture_output=True,
|
|
text=True,
|
|
check=True,
|
|
)
|
|
return result.stdout.strip()
|
|
except (subprocess.CalledProcessError, FileNotFoundError):
|
|
return None
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Helpers
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def _slugify(text: str) -> str:
|
|
"""Convert a title to a filesystem-safe slug."""
|
|
text = text.lower()
|
|
text = re.sub(r"[^\w\s-]", "", text)
|
|
text = re.sub(r"[\s_-]+", "-", text)
|
|
text = text.strip("-")
|
|
return text[:100]
|
|
|
|
|
|
__all__ = ["WikiCompiler"]
|