""" Wiki compiler (Karpathy LLM Wiki pattern). Reads the accepted extractions from the DB, reads the existing Obsidian vault pages, calls Claude to decide what to create/update, then writes the pages and git-commits the vault before and after. Phase 1 implementation: - Generates one Markdown page per accepted source (slug from title/url). - Stubs the git commit logic with clear interfaces. - Does NOT yet do cross-source synthesis (that is Phase 2). """ from __future__ import annotations import re import subprocess from pathlib import Path from typing import Optional from second_brain.config import Config from second_brain.database import get_database from second_brain.models import Extraction, Source, SourceStatus, WikiPage, utcnow class WikiCompiler: """ Compiles accepted extractions into Obsidian vault Markdown pages. Usage:: compiler = WikiCompiler(config) stats = compiler.run() """ def __init__(self, config: Config, api_key: Optional[str] = None) -> None: self.config = config self._api_key = api_key or config.anthropic_api_key def run(self) -> dict: """ Main entry point. Processes all accepted (not yet published) sources. Returns a stats dict: {processed, created, updated, errors}. """ vault = self.config.vault_path vault.mkdir(parents=True, exist_ok=True) self._ensure_git_repo(vault) db = get_database() stats = {"processed": 0, "created": 0, "updated": 0, "errors": 0} with db.session() as sess: sources = ( sess.query(Source) .filter(Source.status == SourceStatus.ACCEPTED) .all() ) if not sources: print("[Compiler] No accepted sources to compile.") return stats sha_before = self._git_sha(vault) for source in sources: try: created = self._compile_source(source, sess, vault) if created: stats["created"] += 1 else: stats["updated"] += 1 source.status = SourceStatus.PUBLISHED source.updated_at = utcnow() stats["processed"] += 1 except Exception as exc: print(f"[Compiler] Error processing source {source.id}: {exc}") stats["errors"] += 1 sha_after = self._git_commit(vault, stats["processed"]) # Update git sha on all written WikiPage rows if sha_after: for page in sess.query(WikiPage).filter( WikiPage.git_sha_after.is_(None) ).all(): page.git_sha_before = sha_before page.git_sha_after = sha_after return stats # ------------------------------------------------------------------ # Per-source page generation # ------------------------------------------------------------------ def _compile_source( self, source: Source, sess, vault: Path ) -> bool: """Write a vault page for *source*. Returns True if newly created.""" extraction: Optional[Extraction] = source.extraction if extraction is None: raise ValueError(f"Source {source.id} has no extraction") slug = _slugify(source.title or source.url) domain_dir = vault / source.domain domain_dir.mkdir(exist_ok=True) page_path = domain_dir / f"{slug}.md" relative_path = str(page_path.relative_to(vault)) content = self._render_page(source, extraction) created = not page_path.exists() page_path.write_text(content, encoding="utf-8") print(f"[Compiler] {'Created' if created else 'Updated'}: {relative_path}") # Upsert WikiPage tracking row wiki_row = sess.query(WikiPage).filter_by(vault_path=relative_path).first() if wiki_row is None: wiki_row = WikiPage( vault_path=relative_path, domain=source.domain, title=source.title or slug, ) sess.add(wiki_row) else: wiki_row.updated_at = utcnow() return created @staticmethod def _render_page(source: Source, extraction: Extraction) -> str: """Render the Markdown content for a vault page.""" lines: list[str] = [] title = source.title or source.url lines.append(f"# {title}\n") # Front-matter block lines.append("---") lines.append(f"url: {source.url}") lines.append(f"domain: {source.domain}") lines.append(f"source_type: {source.source_type.value}") if source.published_at: lines.append(f"published: {source.published_at.date()}") lines.append(f"ingested: {source.ingested_at.date()}") if source.focus: lines.append(f"focus: {source.focus}") lines.append("---\n") # Summary if extraction.summary: lines.append("## Summary\n") lines.append(extraction.summary + "\n") # Key points if extraction.key_points: lines.append("## Key Points\n") for pt in extraction.key_points: lines.append(f"- {pt}") lines.append("") # Claims if extraction.claims: lines.append("## Claims\n") for c in extraction.claims: lines.append(f"- {c}") lines.append("") # Entities if extraction.entities: lines.append("## Entities\n") lines.append(", ".join(f"[[{e}]]" for e in extraction.entities) + "\n") # Open questions if extraction.open_questions: lines.append("## Open Questions\n") for q in extraction.open_questions: lines.append(f"- {q}") lines.append("") # Contradictions if extraction.contradictions: lines.append("## Contradictions\n") for ct in extraction.contradictions: lines.append(f"- {ct}") lines.append("") return "\n".join(lines) # ------------------------------------------------------------------ # Git helpers (stubbed — interface is clear for later wiring) # ------------------------------------------------------------------ @staticmethod def _ensure_git_repo(vault: Path) -> None: """Initialise a git repo in the vault if one doesn't already exist. Idempotent. Creates the repo with an initial empty commit so that subsequent `git rev-parse HEAD` calls succeed even before the first compile writes a page. """ if (vault / ".git").exists(): return try: subprocess.run( ["git", "init", "--initial-branch=main", "--quiet"], cwd=vault, check=True, ) # Allow the empty initial commit so HEAD exists immediately. subprocess.run( ["git", "commit", "--allow-empty", "-m", "second-brain: vault init", "--quiet"], cwd=vault, check=True, ) print(f"[Compiler] Initialised vault git repo at {vault}") except (subprocess.CalledProcessError, FileNotFoundError) as exc: print(f"[Compiler] Warning: could not init git repo in {vault}: {exc}") @staticmethod def _git_sha(vault: Path) -> Optional[str]: """Return HEAD sha of the vault git repo, or None if not a git repo.""" try: result = subprocess.run( ["git", "rev-parse", "HEAD"], cwd=vault, capture_output=True, text=True, check=True, ) return result.stdout.strip() except (subprocess.CalledProcessError, FileNotFoundError): return None @staticmethod def _git_commit(vault: Path, count: int) -> Optional[str]: """Stage all changes and commit. Returns new HEAD sha or None. Returns None (without raising) if there's nothing to commit — that's the normal case when a compile run produced no actual file changes. """ try: subprocess.run(["git", "add", "-A"], cwd=vault, check=True) # Bail out cleanly if there's nothing staged. diff = subprocess.run( ["git", "diff", "--cached", "--quiet"], cwd=vault, check=False, ) if diff.returncode == 0: return None # nothing to commit msg = f"second-brain: compile {count} source(s) [{utcnow().date()}]" subprocess.run(["git", "commit", "-m", msg, "--quiet"], cwd=vault, check=True) result = subprocess.run( ["git", "rev-parse", "HEAD"], cwd=vault, capture_output=True, text=True, check=True, ) return result.stdout.strip() except (subprocess.CalledProcessError, FileNotFoundError): return None # --------------------------------------------------------------------------- # Helpers # --------------------------------------------------------------------------- def _slugify(text: str) -> str: """Convert a title to a filesystem-safe slug.""" text = text.lower() text = re.sub(r"[^\w\s-]", "", text) text = re.sub(r"[\s_-]+", "-", text) text = text.strip("-") return text[:100] __all__ = ["WikiCompiler"]