From 34a268d8dc0ab0d726003954fecdb745e900fdca Mon Sep 17 00:00:00 2001 From: Travis Herbranson Date: Fri, 8 May 2026 17:21:13 -0400 Subject: [PATCH] migration: copy 62 notes from pbs-projects and homelab-projects Two-layer structure: Sources (raw notes) + Wiki (compile output) Four domains: Dev (40), Venture (3), Homelab (23), Reference (0) Includes CLAUDE.md spec, index pages at all levels, compile log Co-Authored-By: Lovebug --- .gitignore | 3 + CLAUDE.md | 229 ++++++++ Sources/Dev/a-review.md | 189 +++++++ .../Dev/ai-memory-architecture-research.md | 143 +++++ Sources/Dev/ansible-container-deploy.md | 212 ++++++++ Sources/Dev/cloudflare-cache-rules.md | 86 +++ Sources/Dev/cloudflare-cache-strategy.md | 107 ++++ Sources/Dev/cloudflare-cache-warmer.md | 81 +++ Sources/Dev/content-hub-database-schema.md | 231 ++++++++ .../Dev/content-hub-phase5-architecture.md | 363 +++++++++++++ Sources/Dev/content-hub-phase5-planning.md | 195 +++++++ Sources/Dev/email-obsidian-git-pipeline.md | 146 +++++ Sources/Dev/email-to-obsidian-automation.md | 208 +++++++ Sources/Dev/github-deploy-refactor.md | 65 +++ Sources/Dev/herbys-dev-setup.md | 241 +++++++++ Sources/Dev/index.md | 23 + .../instagram-automation-content-hub-plan.md | 148 +++++ .../Dev/instagram-automation-session-notes.md | 165 ++++++ Sources/Dev/instagram-reel-sync-phase4.md | 223 ++++++++ .../mailerlite-classic-to-new-migration.md | 95 ++++ Sources/Dev/pbs-api-db-connection-fix.md | 94 ++++ .../pbs-membership-loe1-phase2-save-button.md | 223 ++++++++ Sources/Dev/pbs-membership-loe1-phase3.md | 134 +++++ .../Dev/pbs-membership-loe1-recipe-saving.md | 260 +++++++++ Sources/Dev/pbs-project-tracking.md | 113 ++++ Sources/Dev/pbs-seo-automation.md | 507 ++++++++++++++++++ Sources/Dev/pbs-woocommerce-golive.md | 100 ++++ Sources/Dev/pbs-woocommerce-store.md | 246 +++++++++ Sources/Dev/pbs-youtube-analytics.md | 350 ++++++++++++ Sources/Dev/pbsii-cicd-pipeline.md | 319 +++++++++++ Sources/Dev/pre-commit-framework-migration.md | 190 +++++++ Sources/Dev/production-deploy-march-2026.md | 116 ++++ Sources/Dev/project-scaffolding-skill.md | 127 +++++ Sources/Dev/recipe-cards-seed-packets.md | 129 +++++ Sources/Dev/second-brain.md | 215 ++++++++ Sources/Dev/session-notes-convention.md | 150 ++++++ Sources/Dev/wiki-vault-migration.md | 263 +++++++++ Sources/Dev/wordpress-log-monitor.md | 128 +++++ Sources/Dev/work-index-dashboard.md | 153 ++++++ Sources/Dev/zero-check-pipeline.md | 137 +++++ .../2026-05-06-homelab-mcp-server-stand-up.md | 69 +++ ...enumber-switch-chip-to-bridge-alignment.md | 106 ++++ Sources/Homelab/cli-standardization.md | 75 +++ Sources/Homelab/dns-knot-pihole-end-to-end.md | 117 ++++ Sources/Homelab/editor-stack-upgrade.md | 163 ++++++ Sources/Homelab/env-file-hardening.md | 143 +++++ Sources/Homelab/homelab-dns-decision.md | 116 ++++ Sources/Homelab/homelab-mcp-server.md | 185 +++++++ Sources/Homelab/hunyuan3d-sunnie-pipeline.md | 159 ++++++ Sources/Homelab/index.md | 23 + Sources/Homelab/instantmesh-docker.md | 95 ++++ Sources/Homelab/ob1-deployment.md | 288 ++++++++++ Sources/Homelab/ob1-main-deployment.md | 268 +++++++++ .../pbs-infrastructure-intelligence.md | 481 +++++++++++++++++ Sources/Homelab/pbs-security-hardening.md | 382 +++++++++++++ ...server-stability-and-security-hardening.md | 248 +++++++++ Sources/Homelab/server-stability-mysql-oom.md | 146 +++++ .../Homelab/ssh-ca-with-step-ca-authentik.md | 114 ++++ Sources/Homelab/ssh-login-alerting.md | 111 ++++ .../Homelab/sshkm-local-ssh-key-manager.md | 141 +++++ Sources/Homelab/traefik-deployment.md | 189 +++++++ Sources/Homelab/ufw-docker-outage-fix.md | 126 +++++ Sources/Homelab/wp-cron-supercronic-deploy.md | 142 +++++ Sources/Reference/index.md | 23 + Sources/Venture/editor-stack-upgrade.md | 163 ++++++ Sources/Venture/index.md | 23 + Sources/Venture/offsite-day1-business-dev.md | 129 +++++ Sources/Venture/offsite-full-summary.md | 227 ++++++++ Sources/index.md | 26 + Wiki/Dev/index.md | 19 + Wiki/Homelab/index.md | 19 + Wiki/Reference/index.md | 19 + Wiki/Venture/index.md | 19 + Wiki/index.md | 27 + index.md | 35 ++ log.md | 10 + migrate.py | 268 +++++++++ 77 files changed, 12001 insertions(+) create mode 100644 .gitignore create mode 100644 CLAUDE.md create mode 100644 Sources/Dev/a-review.md create mode 100644 Sources/Dev/ai-memory-architecture-research.md create mode 100644 Sources/Dev/ansible-container-deploy.md create mode 100644 Sources/Dev/cloudflare-cache-rules.md create mode 100644 Sources/Dev/cloudflare-cache-strategy.md create mode 100644 Sources/Dev/cloudflare-cache-warmer.md create mode 100644 Sources/Dev/content-hub-database-schema.md create mode 100644 Sources/Dev/content-hub-phase5-architecture.md create mode 100644 Sources/Dev/content-hub-phase5-planning.md create mode 100644 Sources/Dev/email-obsidian-git-pipeline.md create mode 100644 Sources/Dev/email-to-obsidian-automation.md create mode 100644 Sources/Dev/github-deploy-refactor.md create mode 100644 Sources/Dev/herbys-dev-setup.md create mode 100644 Sources/Dev/index.md create mode 100644 Sources/Dev/instagram-automation-content-hub-plan.md create mode 100644 Sources/Dev/instagram-automation-session-notes.md create mode 100644 Sources/Dev/instagram-reel-sync-phase4.md create mode 100644 Sources/Dev/mailerlite-classic-to-new-migration.md create mode 100644 Sources/Dev/pbs-api-db-connection-fix.md create mode 100644 Sources/Dev/pbs-membership-loe1-phase2-save-button.md create mode 100644 Sources/Dev/pbs-membership-loe1-phase3.md create mode 100644 Sources/Dev/pbs-membership-loe1-recipe-saving.md create mode 100644 Sources/Dev/pbs-project-tracking.md create mode 100644 Sources/Dev/pbs-seo-automation.md create mode 100644 Sources/Dev/pbs-woocommerce-golive.md create mode 100644 Sources/Dev/pbs-woocommerce-store.md create mode 100644 Sources/Dev/pbs-youtube-analytics.md create mode 100644 Sources/Dev/pbsii-cicd-pipeline.md create mode 100644 Sources/Dev/pre-commit-framework-migration.md create mode 100644 Sources/Dev/production-deploy-march-2026.md create mode 100644 Sources/Dev/project-scaffolding-skill.md create mode 100644 Sources/Dev/recipe-cards-seed-packets.md create mode 100644 Sources/Dev/second-brain.md create mode 100644 Sources/Dev/session-notes-convention.md create mode 100644 Sources/Dev/wiki-vault-migration.md create mode 100644 Sources/Dev/wordpress-log-monitor.md create mode 100644 Sources/Dev/work-index-dashboard.md create mode 100644 Sources/Dev/zero-check-pipeline.md create mode 100644 Sources/Homelab/2026-05-06-homelab-mcp-server-stand-up.md create mode 100644 Sources/Homelab/2026-05-08-homelab-vlan-renumber-switch-chip-to-bridge-alignment.md create mode 100644 Sources/Homelab/cli-standardization.md create mode 100644 Sources/Homelab/dns-knot-pihole-end-to-end.md create mode 100644 Sources/Homelab/editor-stack-upgrade.md create mode 100644 Sources/Homelab/env-file-hardening.md create mode 100644 Sources/Homelab/homelab-dns-decision.md create mode 100644 Sources/Homelab/homelab-mcp-server.md create mode 100644 Sources/Homelab/hunyuan3d-sunnie-pipeline.md create mode 100644 Sources/Homelab/index.md create mode 100644 Sources/Homelab/instantmesh-docker.md create mode 100644 Sources/Homelab/ob1-deployment.md create mode 100644 Sources/Homelab/ob1-main-deployment.md create mode 100644 Sources/Homelab/pbs-infrastructure-intelligence.md create mode 100644 Sources/Homelab/pbs-security-hardening.md create mode 100644 Sources/Homelab/server-stability-and-security-hardening.md create mode 100644 Sources/Homelab/server-stability-mysql-oom.md create mode 100644 Sources/Homelab/ssh-ca-with-step-ca-authentik.md create mode 100644 Sources/Homelab/ssh-login-alerting.md create mode 100644 Sources/Homelab/sshkm-local-ssh-key-manager.md create mode 100644 Sources/Homelab/traefik-deployment.md create mode 100644 Sources/Homelab/ufw-docker-outage-fix.md create mode 100644 Sources/Homelab/wp-cron-supercronic-deploy.md create mode 100644 Sources/Reference/index.md create mode 100644 Sources/Venture/editor-stack-upgrade.md create mode 100644 Sources/Venture/index.md create mode 100644 Sources/Venture/offsite-day1-business-dev.md create mode 100644 Sources/Venture/offsite-full-summary.md create mode 100644 Sources/index.md create mode 100644 Wiki/Dev/index.md create mode 100644 Wiki/Homelab/index.md create mode 100644 Wiki/Reference/index.md create mode 100644 Wiki/Venture/index.md create mode 100644 Wiki/index.md create mode 100644 index.md create mode 100644 log.md create mode 100644 migrate.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..3bc56ee --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +.obsidian/ +.trash/ +.DS_Store diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000..feb99f4 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,229 @@ +# CLAUDE.md — Wiki Vault Spec + +This file is the spec for this vault. Any agent operating on this vault +reads this file first. Procedures (compile, lint, migration) live in the +`wiki-maintenance` skill on herbydev — this file is the spec they reference. + +## Purpose + +This vault holds Travis's project plans, session notes, and a compiled +LLM-maintained wiki. It consolidates what was previously split across +pbs-projects and homelab-projects into a single repository with a +two-layer structure. + +## Two-Layer Structure + +### Layer 1: Sources + +Raw project plans and session notes. Written by Travis, Jenny, and LLM +agents via the homelab MCP server. These are the input to the compile loop. + +Location: `Sources//` + +Page types in Sources: +- **project-plan** — what a project is, decisions, phases, tasks +- **session-notes** — what happened in a working session + +### Layer 2: Wiki + +Compiled output. Generated and maintained by the wiki-maintenance compile +skill. Never written to directly by the MCP server or agents outside the +compile loop. + +Location: `Wiki//` + +Page types in Wiki: +- **entity** — recurring people, systems, tools with stable home pages +- **topic-landing** — domain-level catalog and prose overview (index.md) +- **synthesis** — cross-cutting analysis with source provenance + +### Spec Layer + +At the vault root: +- `CLAUDE.md` — this file +- `index.md` — navigation front door +- `log.md` — append-only compile audit trail + +## Four Ownership Domains + +| Domain | Scope | +|--------|-------| +| **Dev** | Coding projects, dev environment, AI/ML, DeFi, tooling | +| **Venture** | PBS business + content: recipes, video, brand, membership, marketing, revenue | +| **Homelab** | Infrastructure, networking, hardware, sysadmin, deployments | +| **Reference** | Cross-domain entities that span multiple domains | + +Cross-domain content: prefer the domain where primary ownership lives. If +genuinely cross-cutting and entity-shaped, put in Reference. If project- +shaped, pick one domain and tag with the others. + +## Page Types + +### Project page (Sources only) + +**Frontmatter:** +```yaml +--- +project: +type: project-plan +status: active # active | paused | completed | archived +path: Sources/ +tags: + - +created: YYYY-MM-DD +updated: YYYY-MM-DD +--- +``` + +**Required sections:** +- `## Goal` — one paragraph, what this project achieves +- `## Locked Decisions` — bulleted, dated as needed +- `## Open Items` — what's unresolved +- `## Phases` or `## Tasks` — work breakdown +- `## Notes` — learnings, context, references + +**Filename:** `.md` + +### Session note (Sources only) + +**Frontmatter:** +```yaml +--- +project: +type: session-notes +status: active +path: Sources/ +tags: + - +created: YYYY-MM-DD +updated: YYYY-MM-DD +--- +``` + +**Required sections:** +- `## Outcome` — what came out of the session +- `## Topics Covered` — what was discussed +- `## Key Learnings` — what was learned +- `## Follow-ons` — checkbox list of next steps + +**Filename:** `YYYY-MM-DD-.md` + +### Entity page (Wiki only) + +**Frontmatter:** +```yaml +--- +type: entity +path: Wiki/ +tags: + - +created: YYYY-MM-DD +updated: YYYY-MM-DD +--- +``` + +**Required sections:** +- **What it is** — one paragraph +- **Current state** — present-tense status +- **Where it lives** — host, repo, URL, path as applicable +- **Related projects** — wikilinks +- **History** — terse, dated notes of significant events + +**Filename:** `.md` + +### Topic landing (both layers) + +Index pages at `//index.md` and at `Sources/index.md`, +`Wiki/index.md`, and the root `index.md`. + +**Frontmatter:** +```yaml +--- +type: topic-landing +path: / +tags: + - landing +updated: YYYY-MM-DD +--- +``` + +**Required sections (Sources landing):** +- **Active Projects** — list with one-line status, links +- **Recent Sessions** — last 10 or last 30 days +- **Completed Projects** — collapsed section + +**Required sections (Wiki landing):** +- **Entities** — list with one-line descriptions +- **Synthesis Pages** — list with topics covered + +### Synthesis page (Wiki only) + +**Frontmatter:** +```yaml +--- +type: synthesis +path: Wiki/ +tags: + - +sources: + - "[[]]" + - "[[]]" +created: YYYY-MM-DD +updated: YYYY-MM-DD +--- +``` + +**Required sections:** +- **Synthesis** — the cross-cutting analysis +- **Sources** — narrative discussion of what each source contributed +- **Open questions** — gaps or contradictions surfaced + +## Tag Conventions + +Tags answer "what is this about." Wikilinks answer "what specific thing." + +**Topics:** `claude-code`, `mcp`, `dispatch`, `tailscale`, `proxmox`, +`docker`, `ansible`, `authentik`, `traefik`, `obsidian`, `n8n`, `pgvector`, +`embeddings`, `terminal`, `automation`, `git`, `cloudflare`, `python`, `go`, +`fish`, `starship` + +**Handles:** `ob1`, `lovebug`, `sunnie`, `wiki`, `pbs` + +New tags require explicit user approval or 3+ page usage. + +## Entity Promotion Rules + +A topic earns an entity page when referenced in 2+ projects or 3+ sessions. +Below threshold, references stay as inline text or tags. + +## Git Protocol + +Two-commit pattern per compile run: +- `pre-compile: ` — captures vault state before writes +- `compile: ` — body includes pages touched/created/promoted + +Lint: `lint: ` if auto-fixes were made. + +MCP writes: `mcp: — ` + +## Navigation + +All browsing happens through index pages. The root `index.md` is the front +door, linking into Sources and Wiki indexes, which link into domain indexes. +No one should need to browse the file tree. + +The compile loop maintains all index pages automatically. + +## What Agents Do and Don't Do + +**Do:** +- Write project plans and session notes to `Sources/` via MCP +- Run compile/lint via the wiki-maintenance skill when invoked +- Maintain indexes and entity pages during compile runs + +**Don't:** +- Write directly to `Wiki/` outside the compile loop +- Delete pages without explicit confirmation +- Add tags silently +- Run autonomously — every operation is human-initiated +- Modify files outside the vault diff --git a/Sources/Dev/a-review.md b/Sources/Dev/a-review.md new file mode 100644 index 0000000..4e28d05 --- /dev/null +++ b/Sources/Dev/a-review.md @@ -0,0 +1,189 @@ +--- +created: 2026-04-27 +path: Sources/Dev +project: a-review +status: active +tags: +- pbs +- claude-code +- code-review +- skill +- gemini +- ollama +type: project-plan +updated: 2026-04-27 +--- + +# a-review — Agent Code Review Skill + +A Claude Code skill that sends code changes to an independent AI agent +for review. Model-agnostic — supports Gemini CLI, Ollama, or any model +that accepts a prompt and returns text. Provides the reviewer with +project context (CLAUDE.md, session notes) alongside the diff and +touched files so it can evaluate intent vs. implementation. + +## Why + +A single agent writing and reviewing its own code has blind spots. The +Gemini CLI proof of concept on the Loom project caught a path traversal +vulnerability, a date formatting bug, and a performance issue — all real, +all missed by the authoring agent. a-review makes that second-opinion +workflow repeatable and model-agnostic. + +This is not a replacement for zero-check. zero-check runs deterministic +validation (lint, SAST, tests) — pass/fail. a-review is subjective +evaluation from a different model — "did you miss something a linter +wouldn't catch?" + +## How It Works + +1. Collect the git diff (default: `HEAD~1`, option for staged changes) +2. Collect the diff with expanded context (e.g., git diff -U10 for 10 lines +of surrounding code) +3. Read CLAUDE.md for project conventions and architecture +4. Read the latest session notes entry for intent/context +5. Build a structured review prompt combining all four inputs +6. Pipe to the configured model backend +7. Save output to `reviews/a-review-YYYY-MM-DD.md` in the project +8. Surface a summary back to the user + +## Context Packet + +The reviewer receives a "briefing" assembled from: + +- **CLAUDE.md** — architecture, key patterns, conventions (the "what + and how" of the codebase) +- **Session notes** — latest session entry (the "what was intended" + for this set of changes) +**Expanded diff** — diff with extra surrounding lines so the +reviewer can see the function each change lives in +- **Diff** — the actual changes to evaluate + +This mirrors how a human code review works: read the PR description, +understand the project conventions, look at the changed files, then +evaluate the diff. + +## Review Prompt Template + +The prompt should instruct the reviewer to: + +- Check for security vulnerabilities (injection, path traversal, + auth bypass, data exposure) +- Check for bugs (logic errors, edge cases, null handling, off-by-one) +- Check for performance issues (unnecessary loops, missing caching, + repeated I/O) +- Check for convention violations (patterns defined in CLAUDE.md) +- Compare intent (from session notes) vs. implementation (from diff) +- Rate each finding by severity: critical, warning, info +- Skip stylistic preferences — focus on correctness and safety + +## Model Configuration + +A config file in the project (e.g., `.a-review.yml`) specifies: + +yaml +model: gemini # gemini | ollama | (future backends) +gemini: + command: gemini + approval_mode: plan # read-only, no file writes + trust_workspace: true +ollama: + model: codellama # or any installed model + host: localhost + port: 11434 + + +The skill reads this config to determine which backend to call and +how to call it. + +## Output Format + +Saved to `reviews/a-review-YYYY-MM-DD.md` (with `-N` suffix if +multiple reviews in one day). Format: + +markdown +# a-review — YYYY-MM-DD + +**Model:** gemini-2.5-pro (or whatever ran) +**Diff:** HEAD~1 (N files, M lines changed) +**Session:** + +## Findings + +### [CRITICAL] + +**File:** +**Line:** + +### [WARNING] +... + +### [INFO] +... + +## Summary + + + +## Phasing + +### Phase 1 — MVP with Gemini CLI +- [ ] Create SKILL.md for Claude Code +- [ ] Build the context packet assembler (CLAUDE.md + session notes + + touched files + diff) +- [ ] Build the review prompt template +- [ ] Implement Gemini CLI backend +- [ ] Implement review output parser and markdown writer +- [ ] Save output to `reviews/` folder +- [ ] Test on Loom project + +### Phase 2 — Ollama backend +- [ ] Add Ollama backend (HTTP API call) +- [ ] Add `.a-review.yml` config file support +- [ ] Test with a local model on herbys-dev + +### Phase 3 — Refinement +- [ ] Tune the review prompt based on real-world results +- [ ] Add option for staged changes vs. HEAD~1 +- [ ] Add collision-safe date suffix for multiple daily reviews +- [ ] Evaluate whether to auto-run after zero-check completes +- [ ] Evaluate whether full file contents are needed for large or +cross-cutting changes + +## Out of Scope + +- Full project review (only diff + touched files) +- Non-code review (content, config-only changes) +- Auto-fixing findings (review only, human decides) +- PR/git integration (this is a local skill, not a CI tool) + +## Open Questions + +- Should the review prompt be customizable per-project, or is one + standard template enough? +- What size diff is too big? Should the skill warn or refuse above + a threshold? +- Should findings be appended to the session notes automatically? + +## Success Criteria + +The skill is working when: +- It catches at least one issue per review that the authoring agent + missed +- The findings are actionable, not noise +- Travis trusts it enough to run it after every session + +## Proof of Concept + +Already validated. On 2026-04-27, Gemini CLI reviewed the Loom project +diff and caught: +- Path traversal vulnerability in MCP write tools +- Date formatting bug (AttributeError on None dates) +- Performance issue (vault re-read on every MCP call) + +All three were real. All three were fixed. + + +...sent from Jenny & Travis +-- +...sent from Jenny & Travis \ No newline at end of file diff --git a/Sources/Dev/ai-memory-architecture-research.md b/Sources/Dev/ai-memory-architecture-research.md new file mode 100644 index 0000000..a2a9e1a --- /dev/null +++ b/Sources/Dev/ai-memory-architecture-research.md @@ -0,0 +1,143 @@ +--- +created: 2026-05-04 +path: Sources/Dev +project: ai-memory-architecture-research +status: active +tags: +- homelab +- ai +- memory +- architecture +- research +type: session-notes +updated: 2026-05-04 +--- + +# AI Memory Architecture Research + +Session exploring the landscape of AI agent harnesses, model memory +mechanisms, and persistent context layers. Started as a question about +two products and turned into the architectural backbone for several +follow-on projects. + +## Outcomes + +- Distinguished harnesses from models as separate layers in the AI +tooling stack. Harnesses are free or cheap; models are where the cost +lives. Swapping harnesses doesn't require migrating data or model +choice. +- Identified OB1 (by Nate B. Jones) as the right adoption target for a +personal AI memory layer, validating the self-hosted +MCP-server-plus-database pattern as the approach to project memory at +scale. +- Mapped the conceptual gap between current production AI memory +(transcript replay plus prefix caching) and the research frontier +(soft prompts, persona embeddings, weight-level continual learning). +Confirmed that frontier-API models will never expose substrate access; +open-weight local models do. +- Reframed the original "build a harness" instinct as actually "build +the memory layer." Most existing harnesses are good; what's missing is +the persistent project memory underneath them. + +## Topics covered + +### Poolside + +Frontier coding-focused AI lab, founded 2023 by Jason Warner +(ex-GitHub CTO) and Eiso Kant. Up to ~$12B valuation after Nvidia's +recent investment. Released the Laguna model family — including Laguna +XS.2, a 33B-total/3B-activated MoE under Apache 2.0 open weights that +runs on consumer hardware via Ollama. Their thesis: code is the best +beachhead to AGI because it forces long-horizon reasoning. + +Relevance: Laguna XS.2 is a candidate model for local agentic coding +workloads on the RTX 3080 and slots cleanly into the Phase 2 OpenClaw +plan. + +### Pi.dev + +A minimal terminal coding agent harness by Mario Zechner (badlogic), +MIT-licensed. Notable design choices: no MCP, no sub-agents, no plan +mode, no built-in todos — extensions instead of features. Supports 15+ +providers including Ollama. Tree-structured sessions with branching +and gist sharing. Same category as Claude Code and Aider, different +philosophy. + +Relevance: Useful as a reference architecture for understanding what a +harness is and isn't. Worth a look as a daily driver for local Ollama +work, separate from sanctioned `claude -p` for scripted Claude usage. + +### OpenBrain (OB1) + +Personal AI memory layer by Nate B. Jones. Self-hosted +Postgres-plus-Supabase stack with MCP server, semantic search via +pgvector, auto-extracted metadata, multiple capture sources (MCP, +REST, Slack webhook, Obsidian import). Provider-agnostic — any +MCP-compatible client can read and write through the same database. + +This is the convergence point of the conversation. Independent +implementation of the same architectural insight reached through +reasoning: AI memory should be a separate, owned, queryable layer that +any AI client can read and write through a standard interface, not a +feature locked inside one vendor's product. + +Adoption is the right call rather than building from scratch. The +technical-project extension layer (project state, decision logs, +session notes ingestion) is the gap to fill, and contributing it back +is plausible. + +## Why this matters + +The cliff-of-projects problem — what happens when there are 100 +projects instead of 5 and Dispatch sessions can't carry coherent +project memory — is solved at the architecture level by this category +of solution. Not by a better harness, not by more discipline, but by a +memory layer that lives alongside whatever tools come and go. + +The research-frontier digression on soft prompts and weight-level +memory established the boundary: production AI memory is RAG-shaped +(text retrieved by vector similarity), not weight-shaped or +activation-shaped. Frontier APIs will never expose the substrate. That +clarifies what is and isn't worth chasing — pursue the practical +memory layer (OB1), recognize the exotic stuff (persona embeddings, +hypernetworks) as research-frontier and not currently buildable on +closed-weight models. + +## Key learnings + +- Harnesses and models are independent layers. Picking a harness and +picking a model are separate decisions. +- "Memory" in current AI is the harness keeping a transcript and +replaying it. The model itself is stateless. Prefix caching is a cost +optimization, not a memory mechanism. +- Frontier API access stops at "send text, get text back." No weight +access, no activation hooks, no soft-prompt injection. Open-weight +local models are where substrate experimentation lives. +- Anthropic actively blocks third-party harnesses from using +subscription OAuth (April 2026 enforcement). API key with workspace +spend caps is the only sanctioned path for Claude access from +non-Anthropic tools. `claude -p` headless mode is the loophole — +invoking the official binary from a script is fine. +- The "build my own harness" instinct was actually about needing a +shared memory layer underneath any harness. Different problem, +different scope, much smaller project. + +## Follow-on projects + +- [ ] Deploy OB1 on homelab — Supabase stack on Proxmox, MCP server +reachable via Tailscale +- [ ] Wire Dispatch pre-compact hook to capture transcripts into OB1 via n8n +- [ ] Extend existing markdown summary n8n flow to also POST captures into OB1 +- [ ] Build technical-project extension for OB1 (project state schema, +decision log) +- [ ] Evaluate Laguna XS.2 on Ollama as local coding model +- [ ] Authelia in front of OB1 web surfaces (Studio UI), GoTrue stays +for application identity + +## References + +- Poolside Laguna: open weights via Ollama +- Pi.dev: github.com/badlogic/pi-mono +- OB1: github.com/NateBJones-Projects/OB1 +- srnichols OpenBrain (alternative implementation): lighter, plain +Postgres + pgvector, no Supabase dependency \ No newline at end of file diff --git a/Sources/Dev/ansible-container-deploy.md b/Sources/Dev/ansible-container-deploy.md new file mode 100644 index 0000000..0ab77b6 --- /dev/null +++ b/Sources/Dev/ansible-container-deploy.md @@ -0,0 +1,212 @@ +--- +created: 2026-04-25 +path: Sources/Dev +project: ansible-container-deploy +status: active +tags: +- pbs +- ansible +- docker +- deployment +- automation +type: project-plan +updated: 2026-04-25 +--- + +# Ansible Container Deploy + +Tag-based per-container Ansible deployment, CLI wizard enhancements, and +GitHub clone integration for the PBS Docker stack. + +## Background + +The wordpress-install repo (pbs-production/) has been refactored from a +loop-based all-or-nothing Docker deployment to a tag-based `import_tasks` +pattern. Each container has its own task wrapper under `tasks/containers/` +that calls `common/deploy.yml` with container-specific variables. The +docker-container repo (pbs-workshop/) houses a CLI wizard that scaffolds +new container files (compose template, task wrapper, env template) but +currently only generates simple-tier containers. + +The tag-based refactor is coded but untested. The wizard needs enhancements +to support medium and complex tier containers. A separate GitHub deploy +refactor (github-deploy-refactor, status: complete) established the +pbsdeploybot machine user pattern for repo cloning — that logic needs to be +absorbed into the dockers role. + +## Repos + +- **wordpress-install (pbs-production/)** — production deployment repo, +tag-based structure in place +- **docker-container (pbs-workshop/)** — tooling repo, CLI wizard and +scaffolding +- **claude-code-scaffolding-skill** — separate project, consumes the +patterns defined here (see Phase 5) + +## Architecture + +### common/deploy.yml Variable Contract + +Required: +- `container_name` (str) — directory name under `files/` and deploy path at +`{{ dockers_path }}//` + +Optional: +- `container_has_env` (bool, default false) — templates +`files//.env.j2` → `.env` +- `container_extra_dirs` (list of str) — additional directories to create +- `container_extra_files` (list of {src, dest, mode}) — static files to copy +- `container_extra_templates` (list of {src, dest, mode}) — Jinja templates +to render + +Global vars (from vars/main.yml and inventory): +- `dockers_path` — base directory (`/opt/docker`) +- `user` — system user for file ownership +- `dockers_lan_name` — Docker network name +- `dockers_domain_name` — primary domain + +### Container Complexity Tiers + +**Simple** (redis, portainer, uptime-kuma): `container_name` only +**Medium** (pbs-api): adds `container_extra_dirs`, `container_extra_files`, +`container_extra_templates` +**Complex** (wordpress): medium plus post-deploy tasks + +### common/github.yml (planned) + +Will sit alongside `common/deploy.yml`. Containers that build from a cloned +repo import `github.yml` first (clone/pull via pbsdeploybot), then +`deploy.yml` (compose up). Uses the same SSH host alias pattern established +in github-deploy-refactor. + +## Phases + +### Phase 1: Validate Tag-Based Deployment + +Test the existing tag-based refactor on staging. + +- [ ] Run `--tags traefik` against staging, verify only traefik deploys +- [ ] Run `--tags wordpress` against staging, verify only wordpress deploys +- [ ] Run `--tags "traefik,wordpress"` to verify multi-tag targeting +- [ ] Run `--skip-tags n8n` to verify exclusion works +- [ ] Run full playbook (no tags) to verify all-container deployment still +works +- [ ] Document any issues or adjustments needed + +### Phase 2: Absorb GitHub Clone Logic + +Migrate the github role's clone logic into the dockers role as +`common/github.yml`. + +- [ ] Create `tasks/common/github.yml` in the dockers role +- [ ] Accept per-invocation vars: `repo_url`, `repo_dest`, `repo_version` +- [ ] Use `become_user: "{{ ansible_user }}"` for SSH config access +- [ ] Default `repo_version` to `main` +- [ ] Update container task wrappers that need repo clones (e.g., pbs-hub) +to import `github.yml` before `deploy.yml` +- [ ] Test on staging with an existing repo-based container +- [ ] Decide whether standalone github role is kept for non-container use +cases or retired entirely + +### Phase 3: Wizard Enhancements + +Extend the CLI wizard to support medium-tier containers and repo-based +builds. + +#### 3a: Dockerfile Support +- [ ] Add wizard prompt: "Does this container need a custom Dockerfile?" +- [ ] If yes, generate a Dockerfile template in the container's files +directory +- [ ] Add Dockerfile to `container_extra_files` in the generated task +wrapper + +#### 3b: Git Repo Support +- [ ] Add wizard prompt: "Is this container built from a git repo?" +- [ ] If yes, prompt for `repo_url` and `repo_version` +- [ ] Generate task wrapper that imports `common/github.yml` before +`common/deploy.yml` + +#### 3c: Extra Files/Dirs/Templates +- [ ] Add wizard prompts for additional directories, static files, and +templates +- [ ] Generate `container_extra_dirs`, `container_extra_files`, +`container_extra_templates` entries in the task wrapper + +#### 3d: Vault Secret References +- [ ] When the wizard detects vars that need secrets (db passwords, API +keys, etc.), generate placeholder references in compose templates using `{{ +vault_ }}` syntax +- [ ] Output a post-scaffold checklist reminding the user which vault vars +need to be manually added to encrypted group vars + +#### 3e: Repo-Context Awareness +- [ ] When the wizard is run inside the production repo, it should read +existing `vars/main.yml`, network configs, and Traefik patterns to inform +prompts and output +- [ ] Wizard should auto-detect available networks, domain name, and base +path from existing config +- [ ] Wizard should append generated vars entries to `vars/main.yml` (or +output them for manual paste) +- [ ] Implementation details deferred to Claude Code — requirement is +stated, approach is flexible + +### Phase 4: Production Rollout + +- [ ] Deploy tag-based structure to production (after staging validation) +- [ ] Test selective deployment on production with a low-risk container +- [ ] Update any runbooks or deployment docs + +### Phase 5: Scaffolding Skill Integration + +Separate project (claude-code-scaffolding-skill), documented here for +dependency tracking. + +Three new scaffold types to add: +- **`ansible-docker-container`** — scaffolds the project structure for a +tag-based Docker container deployment (folder tree, template files, vars +skeleton). The wizard then fills in the details. +- **`ansible-role`** — reusable Galaxy-style role (defaults, handlers, +meta, tasks, templates, Molecule testing) +- **`ansible`** — full playbook project (ansible.cfg, inventory, +group_vars, host_vars, playbooks, roles, requirements.yml, vault setup) + +The scaffold creates project structure; the wizard populates +container-specific content. Scaffold = one-time project start, wizard = +repeated container additions. + +This project provides the reference patterns. The scaffolding skill project +owns implementation. + +## Decisions Made + +- **Wizard vs scaffold split:** Scaffold generates project structure, +wizard fills in container details. Different tools, same conventions. +- **Vault secrets stay manual:** Wizard generates `{{ vault_* }}` +placeholder references and a reminder checklist. Actual vault entries are +hand-added. +- **Context-aware wizard implementation deferred to Claude Code:** +Requirement is that the wizard reads existing repo config when run +in-place. How it does that is an implementation detail. +- **Scaffolding skill: start inside existing skill, separate if it doesn't +fit.** Build assuming merge into claude-code-scaffolding-skill. If +infrastructure types feel forced next to dev tooling types, split into +standalone skill. +- **common/github.yml pattern:** Clone logic absorbed into dockers role, +not kept as a standalone role for container use cases. Matches the +common/deploy.yml pattern. + +## Dependencies + +- github-deploy-refactor (complete) — provides pbsdeploybot auth pattern +- claude-code-scaffolding-skill (separate project) — consumes patterns from +Phase 3+ + +## Open Questions + +- Should the standalone github role be kept for non-container clone use +cases, or is pbsdeploybot + common/github.yml sufficient for everything? +- Final call on scaffolding skill merge vs standalone — deferred until +implementation reveals fit + + +...sent from Jenny & Travis \ No newline at end of file diff --git a/Sources/Dev/cloudflare-cache-rules.md b/Sources/Dev/cloudflare-cache-rules.md new file mode 100644 index 0000000..04f29cc --- /dev/null +++ b/Sources/Dev/cloudflare-cache-rules.md @@ -0,0 +1,86 @@ +--- +created: 2026-03-30 +path: Sources/Dev +project: cloudflare-cache-rules +status: active +tags: +- pbs +- cloudflare +- caching +- production +- wordpress +type: session-notes +updated: 2026-03-30 +--- + +# Cloudflare Cache Rules - Fix & Cleanup - March 30, 2026 + +## Problem + +After production server migration (clone → update → promote), Cloudflare cache broke admin functionality. Jenny's post edits were reverting on save — the editor was serving stale cached versions of wp-admin pages. Both the IP-based bypass rule and the wp-admin path bypass rule were failing. + +## Root Causes Found + +### 1. Cloudflare Cache Rules use "last match wins" +Unlike Page Rules (first match wins), Cache Rules are **stackable** — all matching rules apply, and for conflicting settings, the **last matching rule wins**. Our bypass rules were ordered above the Cache Everything rule, so Cache Everything was always winning. + +### 2. Missing leading slashes on URI paths +The wp-admin bypass rule had `wp-admin` instead of `/wp-admin`, so it never matched actual request paths like `/wp-admin/post.php`. + +### 3. Invalid n8n/Gitea entries in path rules +Full URLs (`https://n8n.plantbasedsoutherner.com`) were in URI Path filters, which only match path portions (like `/wp-admin/`). These entries were doing nothing. n8n and Gitea already have Cloudflare proxy disabled, so cache rules aren't needed for them. + +### 4. Staging URL contamination +473 references to `staging.plantbasedsoutherner.com` were baked into the WordPress database from the server migration — post content, Elementor data, postmeta, GUIDs, and plugin caches. + +## What We Fixed + +### Cache Rule Reordering +Moved Cache Everything to position **1** (top) with bypass rules below it. Since last match wins, the bypass rules now correctly override caching. + +**Current rule order:** +1. Cache Everything (2hr Edge TTL, 1 day Browser TTL, serve stale while revalidating) +2. Bypass Cache - PBS Admins (cookie: `pbs_admin_bypass`) +3. Bypass Cache - WP Admin Paths (`/wp-admin`, `/wp-login.php`) +4. Bypass Cache - WooCommerce (`/cart`, `/checkout`, `/my-account`, `woocommerce` cookie) + +### URI Path Fixes +- Added leading slashes: `/wp-admin`, `/wp-login.php`, `/cart`, `/checkout`, `/my-account` +- Removed invalid full-URL entries for n8n and Gitea + +### Cookie-Based Admin Bypass +- Replaced IP-based bypass with custom `pbs_admin_bypass` cookie +- Confirmed working via Cloudflare Trace and DevTools (`CF-Cache-Status: DYNAMIC`) +- Cookie set manually in DevTools for now — WPCode snippet for auto-set on admin login is next + +### Staging URL Cleanup +- Ran WP-CLI search-replace inside Docker container: + ``` + docker exec -it wordpress wp search-replace 'staging.plantbasedsoutherner.com' 'plantbasedsoutherner.com' --all-tables --allow-root + ``` +- 473 replacements made across wp_posts, wp_postmeta, and GUIDs +- Flushed Redis object cache: `wp cache flush --allow-root` +- Manually cleared stale plugin caches: Jetpack site icon, Astra partials cache +- Deleted `astra_partials_config_cache` from wp_options to force regeneration + +### Debugging Tools Used +- Cloudflare **Trace** (Rules → Trace): Simulates a request and shows which rules match in order — confirmed both bypass and Cache Everything were firing, with Cache Everything winning +- Browser DevTools Network tab: Checked `CF-Cache-Status` response header +- `curl` blocked by Cloudflare bot challenge (Bot Fight Mode) — use DevTools instead + +## Key Learnings + +- **Cloudflare Cache Rules: last match wins** — opposite of Page Rules. Put broad rules first, specific overrides below. +- **URI Path filters only match paths** — never full URLs with hostnames +- **Always include leading slashes** in URI path filters (`/wp-admin` not `wp-admin`) +- **Server cloning contaminates URLs** — always run `wp search-replace` after migrating/cloning WordPress between environments +- **WP-CLI handles serialized data safely** — never do raw SQL find-and-replace on WordPress content (Elementor data can break) +- **`wp-config.php` tip**: Add `define('FS_METHOD', 'direct');` to suppress FTP credentials prompt in Docker + +## Still To Do + +- [ ] WPCode Lite snippet to auto-set `pbs_admin_bypass` cookie on admin login +- [ ] Clean up Rule 2 extra expressions (leftover `tjherbranson` wildcard and `starts_with` conditions) +- [ ] Create scoped Cloudflare API token (Zone.Cache Purge only) +- [ ] Build n8n auto-purge workflow (WordPress publish/update → Cloudflare targeted cache purge) +- [ ] Decide: auto-purge homepage on content updates? \ No newline at end of file diff --git a/Sources/Dev/cloudflare-cache-strategy.md b/Sources/Dev/cloudflare-cache-strategy.md new file mode 100644 index 0000000..07b78ee --- /dev/null +++ b/Sources/Dev/cloudflare-cache-strategy.md @@ -0,0 +1,107 @@ +--- +created: 2026-03-30 +path: Sources/Dev +project: cloudflare-cache-strategy +status: active +tags: +- pbs +- cloudflare +- caching +- n8n +- automation +- production +type: project-plan +updated: 2026-03-30 +--- + +# Cloudflare Cache Strategy - Cookie Bypass + Auto-Purge + +## Problem + +After the production server migration (clone → update → promote), +Cloudflare caching broke backend tools (n8n, Gitea) and WordPress admin +functionality. The existing IP-based cache bypass rule failed — Jenny saved +post edits that reverted on reload due to stale cached pages being served. + +The IP-based approach is inherently fragile: ISP IP changes, rule ordering +conflicts with new exception rules, and doesn't scale. + +## Decision + +Replace IP-based bypass with a **hybrid approach**: custom cookie bypass +for admin users + n8n-driven auto-purge on content publish/update. + +## Plan + +### Phase 1: Cookie Bypass + +- [ ] Define custom cookie name (`pbs_admin_bypass`) and generate a random +secret value +- [ ] Test manually: set cookie in browser DevTools, verify Cloudflare +serves `CF-Cache-Status: BYPASS` +- [ ] Add WPCode Lite PHP snippet that sets the cookie automatically on +admin-role login +- [ ] Replace Cloudflare "Bypass Cache - Admin IPs" rule with new rule: + - **Rule name:** Bypass Cache - PBS Admins + - **Match:** Cookie contains `pbs_admin_bypass` + - **Action:** Bypass cache +- [ ] Ensure rule is ordered above "Cache Everything" rule +- [ ] Delete old IP-based bypass rule +- [ ] Verify Jenny's workflow: edit post → save → reload → changes persist + +### Phase 2: Cloudflare API Token + +- [ ] Create scoped Cloudflare API token with only `Zone.Cache Purge` +permission +- [ ] Store token securely (n8n credentials store) +- [ ] Note Zone ID for API calls + +### Phase 3: n8n Auto-Purge Workflow + +- [ ] Configure WordPress webhook(s) to fire on: + - Post publish/update + - Page publish/update + - WooCommerce product update + - WPRM recipe update +- [ ] Build n8n workflow: + - Webhook node receives WordPress payload (includes post/page URL) + - HTTP Request node calls Cloudflare purge API for the specific URL + - (Optional) Also purge homepage URL — decision pending +- [ ] Test: publish a post → verify cached version updates within seconds +- [ ] Add Google Chat notification on purge (optional) + +## Current Cloudflare Cache Rules (Target State) + +1. Bypass Cache - WP Admin Paths (`/wp-admin`, `/wp-login.php`) +2. Bypass Cache - PBS Admins (cookie: `pbs_admin_bypass`) +3. Bypass Cache - n8n/Gitea (existing exception rules) +4. Cache Everything (2hr Edge TTL, 1 day Browser TTL, serve stale while +revalidating) + +## Key Technical Notes + +- Cloudflare purge API endpoint: `POST +/client/v4/zones/{ZONE_ID}/purge_cache` +- Targeted purge supports up to 30 URLs per call +- n8n and Gitea subdomains have Cloudflare proxy disabled (separate fix) +- Cookie approach avoids Ultimate Member conflict — site members won't have +the custom cookie +- If Ultimate Member membership activates later, this approach still works +(unlike `wordpress_logged_in` cookie) + +## Open Questions + +- [ ] Whether to auto-purge homepage on every content update +- [ ] Whether to add a private URL endpoint as a fallback cookie-setting +method + +## Cloudflare API Purge Example + +```bash +curl -X POST " +https://api.cloudflare.com/client/v4/zones/YOUR_ZONE_ID/purge_cache" \ + -H "Authorization: Bearer YOUR_API_TOKEN" \ + -H "Content-Type: application/json" \ + --data '{"files":["https://plantbasedsoutherner.com/updated-post-slug/"]}' +``` +...sent from Jenny & Travis \ No newline at end of file diff --git a/Sources/Dev/cloudflare-cache-warmer.md b/Sources/Dev/cloudflare-cache-warmer.md new file mode 100644 index 0000000..e255d43 --- /dev/null +++ b/Sources/Dev/cloudflare-cache-warmer.md @@ -0,0 +1,81 @@ +--- +created: 2026-04-20 +path: Sources/Dev +project: cloudflare-cache-warmer +status: active +tags: +- pbs +- website +- n8n +- cloudflare +type: project-plan +updated: 2026-04-20 +--- + +# Cloudflare Cache Warmer — n8n Workflow + +## Problem + +Recipe pages fall out of Cloudflare's edge cache between human visits. When +Googlebot crawls a cold page, it hits WordPress directly — full PHP/plugin +execution, slow response. Google deprioritizes slow pages and marks them +"Crawled - currently not indexed." + +## Solution + +Scheduled n8n workflow that fetches the sitemap daily and hits every recipe +URL to keep Cloudflare's cache warm. Googlebot always gets a fast cached +response. + +## Architecture + +1. **Schedule Trigger** — runs once daily (off-peak, e.g. 4:00 AM ET) +2. **HTTP Request** — fetch ` +https://plantbasedsoutherner.com/sitemap_index.xml` +3. **XML Parse** — extract child sitemap URLs from the index +4. **Loop** — for each child sitemap: + - **HTTP Request** — fetch the child sitemap XML + - **XML Parse** — extract all `` URLs +5. **Loop** — for each URL: + - **HTTP Request (GET)** — hit the URL (this warms the cache) + - **Wait** — small delay (1-2 seconds) between requests to avoid +hammering the server on cold hits +6. **Google Chat Notification** — post summary to `webadmin` space: total +URLs warmed, any errors + +## Design Notes + +- Use the n8n Loop node (not Split In Batches) to serialize requests and +avoid overloading Apache workers +- The HTTP Request to warm cache only needs a GET — no need to process the +response body +- Consider a 1-2 second delay between requests so cold misses don't stack +up concurrent PHP workers +- Edge TTL is currently set to 8 days, so daily warming keeps everything +well within window +- As recipes grow, workflow scales automatically since it reads from the +sitemap + +## Validation + +- After first run, spot-check 3-4 recipe URLs in browser DevTools → Network +→ `cf-cache-status` should be `HIT` +- Monitor Google Search Console "Crawled - currently not indexed" count +over the following 2-4 weeks — should decrease + +## Open Questions + +- [ ] Confirm best time of day for the schedule (4 AM ET?) +- [ ] Decide whether to warm ALL sitemap URLs or filter to just recipe/post +URLs +- [ ] Should errors (timeouts, 5xx) trigger a separate alert or just log in +the summary? + +## Future Enhancements + +- Add a check: if `cf-cache-status` is already `HIT`, skip the URL (reduces +unnecessary requests as traffic grows) +- Track cache hit/miss ratio over time via a simple counter in pbs-hub + + +...sent from Jenny & Travis \ No newline at end of file diff --git a/Sources/Dev/content-hub-database-schema.md b/Sources/Dev/content-hub-database-schema.md new file mode 100644 index 0000000..0129e01 --- /dev/null +++ b/Sources/Dev/content-hub-database-schema.md @@ -0,0 +1,231 @@ +--- +created: 2026-03-20 +path: Sources/Dev +project: content-hub-database-schema +status: active +tags: +- pbs +- flask +- mysql +- n8n +- instagram +- automation +type: project-plan +updated: 2026-03-20 +--- + +# PBS Content Hub — Database Schema & Table Relationships + +## Purpose + +This document defines the database schema and table relationships for the +PBS Content Hub. It supplements the Phase 5 Architecture & Planning +Decisions doc and serves as a handoff for Claude Code implementation. + +--- + +## Three Data Domains Being Unified + +### 1. PBS Video Manager (migrating from local SQLite to MySQL) +- Local app being migrated to the server as part of the Content Hub +- `projects` table stores all project info (name, status, description, etc.) +- Projects are the central concept — a project can be a reel, YouTube +video, or any future content type + +### 2. PBS Recipe App (live on production MySQL) +- Already in MySQL (`pbs_automation` database) +- `pbs_recipes` table — stores WordPress recipe data +- Powers Jenny's admin page and n8n recipe lookups + +### 3. Instagram Reel Workflow (currently in n8n data tables) +- Being migrated from n8n internal storage to MySQL +- Becomes the generic `platform_posts` table to support Instagram now and +YouTube/other platforms later + +--- + +## Relationship Model + +`projects` is the hub that connects everything. It reaches out to both +`pbs_recipes` and `platform_posts`. Neither of those tables needs to +reference back to projects or to each other. + +``` +project_types + id ←─── type_id + \ + projects + / \ + recipe_id ───→ ←─── platform_post_id + pbs_recipes platform_posts +``` + +### Why platform_posts has no FK to projects +The platform table is just platform data. There is no scenario where +platform_posts needs to look up project info. Projects owns the +relationship to both recipes and platform posts. + +### Why pbs_recipes has no FK to projects +Same reasoning. Recipes exist independently in WordPress. Projects link to +them, not the other way around. + +--- + +## Table Definitions + +### `pbs_recipes` (exists in MySQL, no changes) + +| Column | Type | Notes | +|---|---|---| +| post_id | INT, PK | WordPress post ID | +| recipe_title | VARCHAR(255) | | +| recipe_url | VARCHAR(255) | | +| keyword | VARCHAR(100), nullable | Trigger word for automation | +| created_at | TIMESTAMP | | +| updated_at | TIMESTAMP | auto-update | + +CRUD: Yes — existing Jenny admin page migrates into Content Hub. + +### `project_types` (new) + +| Column | Type | Notes | +|---|---|---| +| id | INT, PK, auto increment | | +| name | VARCHAR(50) | reel, youtube, etc. | +| created_at | TIMESTAMP | | + +CRUD: Yes — managed in Settings tab. Allows adding new content types +without code changes. + +### `projects` (new — replaces local SQLite video_projects) + +| Column | Type | Notes | +|---|---|---| +| id | INT, PK, auto increment | | +| title | VARCHAR(255) | Working title | +| type_id | INT, FK → project_types.id | | +| status | VARCHAR(50) | draft, ready, live-matched, live-needs-review. +Kept as VARCHAR for now — will become a lookup table when Trello +integration is built | +| recipe_id | INT, FK → pbs_recipes.post_id, nullable | Linked recipe | +| platform_post_id | INT, FK → platform_posts.id, nullable | Linked +platform post | +| created_at | TIMESTAMP | | +| updated_at | TIMESTAMP | auto-update | + +CRUD: Yes — this is the main workspace in the Content Hub. + +Note: Additional columns from the existing SQLite video_projects schema +(description, file paths, checklist data, etc.) will be added separately. +This doc focuses on the connecting fields. + +### `platform_posts` (new — replaces n8n insta_reel table) + +| Column | Type | Notes | +|---|---|---| +| id | INT, PK, auto increment | Internal ID | +| platform | VARCHAR(50) | instagram, youtube, tiktok, etc. | +| platform_post_id | VARCHAR(50) | Native ID from the platform. VARCHAR +because Instagram IDs overflow JavaScript integer limits | +| url | VARCHAR(255), nullable | Link to the post on the platform | +| published_at | TIMESTAMP, nullable | When it went live | +| created_at | TIMESTAMP | When we recorded it | +| updated_at | TIMESTAMP | auto-update | + +CRUD: Yes — view/manage platform data in the Content Hub. + +--- + +## n8n Lookup Path (Comment Reply Automation) + +When n8n receives a comment on an Instagram reel and needs the keyword + +recipe URL: + +``` +platform_posts (match on platform_post_id = reel ID from webhook) + → projects (WHERE platform_post_id = platform_posts.id) + → pbs_recipes (WHERE post_id = projects.recipe_id) + → return keyword + recipe_url for the reply +``` + +If any join in the chain returns null, n8n triggers the existing error +notification to Travis. + +--- + +## Auto-Match Routine + +Triggered by n8n ping to `POST /api/match/run` after new `platform_posts` +insert. + +``` +For each project in "Ready" status with no platform_post_id: + → Look in platform_posts for unmatched rows (no project pointing at +them) where: + - Matching criteria (post_id via recipe link, keyword, or other +heuristics) + → Single confident match → update projects.platform_post_id, status → +"live-matched" + → Multiple matches → status → "live-needs-review" + → No match → leave as-is +``` + +Manual match UI is the fallback — Jenny selects from unmatched +platform_posts to link to a project. + +--- + +## Error Checking / Healthcheck (Real-Time) + +No separate table needed. The Status Dashboard computes checks on page load +by querying the four tables: + +- Platform posts with no project linked (unmatched) +- Projects in "ready" status with no platform_post_id for X days +- Projects linked to a recipe that has no keyword +- Broken join chains (project → recipe or project → platform_post returns +null) + +Displayed as green/yellow/red indicators per project and per system on the +Status tab. + +--- + +## CRUD Summary + +| Table | CRUD Location | Primary User | +|---|---|---| +| pbs_recipes | Reels tab (migrated admin page) | Jenny | +| project_types | Settings tab | Travis | +| projects | Projects + Reels tabs | Both | +| platform_posts | Status tab / Reels tab | Both | + +--- + +## Existing Tables to Keep (n8n automation logging) + +These tables remain in `pbs_automation` and are not part of the Content Hub +schema, but n8n continues to use them: + +- `instagram_posts_processed` — tracks processed Instagram posts (prevents +duplicate processing) +- `instagram_replies` — comment reply log for debugging/analytics + +--- + +## Migration Notes + +- `pbs_recipes` already exists in MySQL — no migration needed +- `projects` replaces local SQLite `video_projects` — SQLAlchemy model +update + data migration +- `platform_posts` replaces n8n internal `insta_reel` data table — data +migration from n8n to MySQL +- `project_types` is new — seed with initial values (reel, youtube) +- All new tables go in the `pbs_automation` database (or new +`pbs_content_hub` database — TBD, still an open question) + +--- + +*Project: Plant Based Southerner Content Hub* +*Planning Session: March 20, 2026* +*Participants: Travis & Claude* \ No newline at end of file diff --git a/Sources/Dev/content-hub-phase5-architecture.md b/Sources/Dev/content-hub-phase5-architecture.md new file mode 100644 index 0000000..0449fb1 --- /dev/null +++ b/Sources/Dev/content-hub-phase5-architecture.md @@ -0,0 +1,363 @@ +--- +created: 2026-03-19 +path: Sources/Dev +project: content-hub-phase5-architecture +status: active +tags: +- pbs +- flask +- mysql +- n8n +- instagram +- automation +- docker +- traefik +type: project-plan +updated: 2026-03-19 +--- + +# PBS Content Hub Phase 5 — Architecture & Planning Decisions + +## Purpose + +This document captures all architecture and planning decisions made during +the Phase 5 planning session. It serves as a handoff doc for Claude Code to +implement the Content Hub refactor. + +--- + +## Context + +### Phases 1–4 Complete +- ✅ Phase 1 — MySQL schema with single `instagram_posts` table +- ✅ Phase 2 — Reply workflow refactored with error notifications +- ✅ Phase 3 — WordPress → MySQL sync via webhook +- ✅ Phase 4 — Instagram reel publish webhook capturing reel IDs + +### Two Existing Apps Being Merged + +**PBS-API (production, running on Linode):** +- Location: `/opt/docker/pbs-api/app.py` +- Container: `pbs-api` on internal Docker network +- Internal URL: `http://pbs-api:5000` +- External URL: `https://plantbasedsoutherner.com/api/...` (via Traefik) +- Database: MySQL `pbs_automation`, connects as user `pbs_api` +- Auth: API key via `X-API-Key` header on all endpoints (except +`/api/health`), Traefik BasicAuth on `/pbsadmin` routes +- Current endpoints: + - `GET /api/health` — health check (no auth) + - `POST /api/instagram-mapping` — save/update recipe mappings (upsert) + - `GET /api/instagram-mapping/{keyword}` — lookup recipe by keyword + - `GET /api/recipes/all` — bulk fetch all recipes + - `GET /pbsadmin/instagram/recipes` — Jenny's admin interface (inline +keyword editing, search/filter, stats) +- MySQL tables (`pbs_automation`): + - `instagram_recipes` — post_id (PK), recipe_title, recipe_url, keyword, +created_at, updated_at + - `instagram_posts_processed` — instagram_post_id (PK), last_checked, +comment_count + - `instagram_replies` — id (auto PK), instagram_comment_id (unique), +instagram_post_id, comment_text, reply_text, recipe_url, sent_at +- Security: API key stored in `.env` file, Traefik BasicAuth for +`/pbsadmin` with username `jenny` + +**PBS Content Hub (local development, Travis's machine):** +- CLI + Flask GUI using SQLite/SQLAlchemy +- CLI commands: `pbs init`, `pbs convert`, `pbs scan`, `pbs archive`, `pbs +status`, `pbs list`, `pbs shell`, `pbs activate/deactivate` +- Flask GUI: Dashboard, Project Detail (Overview, Checklist, Description +Builder), Library CRUD +- Models: `Project`, `VideoFile`, `ChecklistItem`, `LibraryItem` +- Library uses single `content_blocks` table: `id, name, content, +block_type, sort_order` + +--- + +## Architecture Decisions + +### Decision 1: Single Container (pbs-hub) + +**Decision:** Merge PBS-API and Content Hub into ONE Flask app in ONE +container. + +**Rationale:** +- Two containers is over-engineered for current scale (team of 2 + +assistant, one automation consumer) +- The admin page Jenny uses today belongs in the same UI as the new Content +Hub features +- Avoids maintaining two Flask apps, two Dockerfiles, two deployments +- The code cleanliness concern is solved with Flask Blueprints, not +separate containers +- If a two-container split is ever needed (membership site, public API), +the service layer architecture makes extraction straightforward later + +### Decision 2: Flask Blueprints for Code Organization + +**Decision:** Use Flask Blueprints to separate API routes from web UI +routes. + +**Target project structure:** +``` +pbs-hub/ +├── app.py (creates Flask app, registers blueprints) +├── blueprints/ +│ ├── api/ (all JSON endpoints — n8n, CLI, auto-match) +│ │ ├── routes.py +│ │ └── ... +│ ├── web/ (all HTML pages — Jenny's UI, dashboard, reels) +│ │ ├── routes.py +│ │ └── templates/ +│ └── ... +├── services/ (business logic — match routine, recipe CRUD) +├── models/ (SQLAlchemy models) +└── ... +``` + +### Decision 3: Service Layer Refactor + +**Decision:** Extract all SQLAlchemy logic into a service layer. + +- Flask routes (both API and web) call service functions, never query the +DB directly +- CLI will call the API endpoints over HTTPS (not import service functions) +- This is the key enabler for the CLI → API migration + +**Example pattern:** +```python +# services/project_service.py +def get_all_projects(): + return Project.query.all() + +def create_project(title): + project = Project(title=title) + db.session.add(project) + db.session.commit() + return project + +# blueprints/api/routes.py +@api_bp.route('/api/projects', methods=['GET']) +def api_list_projects(): + projects = get_all_projects() + return jsonify([p.to_dict() for p in projects]) + +# blueprints/web/routes.py +@web_bp.route('/projects') +def list_projects(): + projects = get_all_projects() + return render_template('projects.html', projects=projects) +``` + +### Decision 4: SQLite → MySQL Migration + +**Decision:** Replace SQLite with MySQL. SQLAlchemy abstracts the engine, +so models barely change — mostly a connection string swap + migration. + +- Content Hub connects to the same MySQL instance as the current PBS-API +- Database: `pbs_automation` (existing) or new `pbs_content_hub` database +(TBD) +- MySQL user: `pbs_api` (existing, has access to `pbs_automation.*`) + +### Decision 5: CLI Talks to API Over HTTPS + +**Decision:** The pbsVM CLI will make HTTPS calls to the Content Hub API +instead of direct SQLAlchemy queries. + +- Auth: API key in a local config file or environment variable +- Uses `httpx` (or `requests`) to call API endpoints +- No database credentials on Travis's local machine +- CLI needs network access to push data (file management operations remain +local) + +--- + +## UI & Feature Decisions + +### Decision 6: Recipe-to-Reel Linking Strategy + +**Decision:** Manual selection in Content Hub. + +Jenny selects the recipe from a dropdown when creating a reel record. This +auto-populates the post ID, keyword, and URL. No caption parsing, no +fragile keyword matching. + +**Current process being replaced:** Jenny/assistant emails Travis for the +post ID → Travis runs MySQL query → sends back ID. The Content Hub removes +Travis from this loop entirely. + +### Decision 7: Reel Record Lifecycle (Two-Stage) + +**Decision:** Local-first, then matched to live data. + +- **Stage 1 (Pre-publish):** Jenny creates a reel record in the Content Hub +before anything exists on Instagram. Working title, linked recipe, keyword, +built caption. No Instagram reel ID yet. +- **Stage 2 (Post-publish):** Instagram webhook fires → n8n writes reel ID +to `instagram_posts` → hub reads that table and matches to local record. + +### Decision 8: n8n Stays Decoupled + +**Decision:** n8n is dumb to the hub. + +- n8n writes to `instagram_posts` in MySQL — that's its only job +- n8n does NOT call Content Hub API or know about hub reel records +- The Content Hub reads from `instagram_posts` to display what's live +- The hub owns all matching logic +- **One exception:** n8n sends a simple ping to `POST /api/match/run` after +new inserts to trigger the auto-match routine + +### Decision 9: Auto-Match Routine + +**Decision:** Triggered by n8n ping after new `instagram_posts` insert. + +**Logic:** +``` +For each hub reel record in "Ready" status (no reel ID linked): + → Look in instagram_posts for unmatched rows where: + - post_id matches (recipe link) OR keyword matches + → Single confident match → auto-link, status → "Live — Matched" + → Multiple matches → flag as "Needs Review" + → No match → leave as-is +``` + +Manual match UI is the fallback for anything auto-match can't resolve. + +### Decision 10: Recipe Dropdown Source + +**Decision:** MySQL recipes table (not WordPress REST API). Already synced +via webhook, faster, no external calls during form interactions. + +### Decision 11: Reel Detail Page — Tabbed Layout + +``` +[ Overview ] [ Caption Builder ] +``` + +**Overview tab:** +- Reel Title (free text) +- Recipe dropdown (from MySQL) → auto-fills keyword + URL +- Keyword (editable — this IS the recipe review/confirm step) +- Recipe URL (read-only) +- Status indicator +- Save / Delete + +**Caption Builder tab (separate tab within reel detail):** +- Select library blocks (hashtags, links, CTAs, about snippets) +- Preview assembled caption +- Copy to clipboard + +### Decision 12: Recipe Review UI + +**Decision:** Not a separate screen. Baked into the reel creation flow — +when Jenny selects a recipe from the dropdown, she sees the record and +confirms/edits the keyword right there. + +### Decision 13: Manual Match Correction + +A "Link Reel" button on flagged records. Jenny selects from unmatched +`instagram_posts` rows to connect the dots. + +--- + +## Reel Status States + +1. **Draft** — reel record created, no recipe linked yet +2. **Ready** — recipe linked, keyword confirmed, caption built. Waiting to +be posted +3. **Live — Matched** — Instagram webhook received, reel ID attached, +auto-reply active +4. **Live — Needs Review** — auto-match couldn't confidently link, Jenny +needs to manually match + +--- + +## Reel List View + +Data points per reel (layout TBD — table vs cards, getting Jenny's +feedback): +- Reel title +- Linked recipe name (or "No Recipe" warning) +- Keyword +- Status (color dot + label) +- Date created or last updated + +--- + +## Content Hub Navigation (from earlier design doc) + +``` +[ Dashboard ] [ Projects ] [ Reels ] [ Library ] [ Status ] [ Settings +] +``` + +| Tab | Primary User | Description | +|-----|-------------|-------------| +| Dashboard | Both | Overview of recent projects and reels | +| Projects | Travis | Video project management (existing pbsVM GUI) | +| Reels | Jenny | Reel manager, caption builder, publishing workflow | +| Library | Both | Shared hashtags, links, about blocks, CTAs | +| Status | Both | Automation health, comment/reply counts, healthcheck | +| Settings | Travis | Global config, API keys, default checklist | + +--- + +## Target Automation Flow + +``` +Jenny creates reel record in hub → selects recipe → builds caption + ↓ +Jenny posts reel to Instagram + ↓ +Instagram webhook → n8n → writes reel ID to instagram_posts + ↓ +n8n pings hub (POST /api/match/run) + ↓ +Hub auto-match runs: + - Finds match → links record → status = Live — Matched ✅ + - No confident match → status = Needs Review 🟡 + ↓ +Comment comes in on Instagram → n8n looks up instagram_posts → sends DM + +reply + ↓ +Dashboard shows green across the board 🟢 +``` + +--- + +## Refactor Steps (Implementation Order) + +1. **Extract service layer** from current Content Hub SQLAlchemy code +2. **Restructure into Blueprints** (api/ and web/) +3. **Add API routes** alongside existing GUI routes +4. **Migrate PBS-API endpoints** into the new Blueprint structure +5. **Migrate Jenny's admin page** into the Content Hub web UI +6. **Swap SQLite → MySQL** (connection string + migration) +7. **Refactor CLI** to use `httpx` calls to API instead of direct SQLAlchemy +8. **Build Reels tab** (creation form, recipe dropdown, status lifecycle) +9. **Build Caption Builder tab** within reel detail +10. **Build auto-match routine** (`POST /api/match/run`) +11. **Build manual match UI** (fallback for auto-match failures) +12. **Build Status Dashboard** (per-reel health + system health — details +TBD) +13. **Deploy to staging** → test → deploy to production +14. **Update n8n** to point at new container URL + add match/run ping +15. **Retire PBS-API container** + +--- + +## Open Questions (Next Planning Session) + +- [ ] Reel list layout: table rows vs cards (get Jenny's feedback) +- [ ] Jenny's assistant access: same login or separate user accounts? +- [ ] Status Dashboard UI details: per-reel health indicators + system +health panel +- [ ] Database schema for hub reel records table +- [ ] Define green/yellow/red status thresholds in detail +- [ ] Database strategy: extend `pbs_automation` or create new +`pbs_content_hub` database? + +--- + +*Project: Plant Based Southerner Content Hub* +*Planning Session: March 19, 2026* +*Participants: Travis & Claude* +...sent from Jenny & Travis \ No newline at end of file diff --git a/Sources/Dev/content-hub-phase5-planning.md b/Sources/Dev/content-hub-phase5-planning.md new file mode 100644 index 0000000..6d02c0d --- /dev/null +++ b/Sources/Dev/content-hub-phase5-planning.md @@ -0,0 +1,195 @@ +--- +created: 2026-03-19 +path: Sources/Dev +project: content-hub-phase5-planning +status: active +tags: +- pbs +- flask +- mysql +- n8n +- instagram +- automation +- docker +type: project-plan +updated: 2026-03-19 +--- + +# PBS Content Hub — Phase 5 Planning Decisions + +## Context + +Phases 1–4 of the Instagram Automation & Content Hub master plan are +complete: +- ✅ Phase 1 — MySQL schema with single `instagram_posts` table +- ✅ Phase 2 — Reply workflow refactored with error notifications +- ✅ Phase 3 — WordPress → MySQL sync via webhook +- ✅ Phase 4 — Instagram reel publish webhook capturing reel IDs + +Phase 5 focuses on building the Content Hub UI layer that gives Jenny and +her assistant self-service access to recipe/reel management — removing +Travis as the manual middleman for MySQL lookups. + +--- + +## Key Architecture Decisions + +### 1. Recipe-to-Reel Linking Strategy +**Decision:** Manual selection in Content Hub (Option B) + +Jenny selects the recipe from a dropdown when creating a reel record in the +hub. This auto-populates the post ID, keyword, and URL. No caption parsing, +no fragile keyword matching. + +**Rationale:** Jenny is already in the hub building captions — selecting +the recipe is one extra dropdown click. Most reliable approach with zero +parsing ambiguity. + +### 2. Reel Record Lifecycle (Two-Stage) +**Decision:** Local-first, then matched to live data + +- **Stage 1 (Pre-publish):** Jenny creates a reel record in the Content Hub +before anything exists on Instagram. This is a local planning record — +working title, linked recipe, keyword, built caption. +- **Stage 2 (Post-publish):** After Jenny posts the reel, the Instagram +webhook fires, n8n writes the reel ID to `instagram_posts` in MySQL. The +hub reads from that table and matches it to the local record. + +### 3. n8n Stays Decoupled From the Hub +**Decision:** n8n is dumb to the hub + +- n8n writes to `instagram_posts` in MySQL — that's its job +- n8n does NOT call the Content Hub API or know about hub reel records +- The Content Hub reads from `instagram_posts` to display what's live +- The hub owns the matching logic between its local records and n8n's data + +**Rationale:** Keeps n8n focused on automation. The hub is the human +workflow layer on top. + +### 4. Auto-Match Routine +**Decision:** Triggered by n8n ping after new insert + +After n8n writes a new row to `instagram_posts`, it sends a simple ping to +the hub (`POST /api/match/run`). The hub then runs the matching logic: + +``` +For each hub reel record in "Ready" status (no reel ID linked): + → Look in instagram_posts for unmatched rows where: + - post_id matches (recipe link) OR keyword matches + → Single match → auto-link, status → "Live — Matched" ✅ + → Multiple matches → flag as "Needs Review" 🟡 + → No match → leave as-is +``` + +Jenny's manual match UI is the fallback for anything the auto-matcher can't +confidently link. + +### 5. Recipe Dropdown Source +**Decision:** MySQL recipes table (not WordPress REST API) + +The recipe dropdown in the reel creation form pulls from the local MySQL +recipes table, which is already kept in sync via the WordPress publish +webhook. Faster, no external API calls during form interactions. + +--- + +## UI Structure Decisions + +### Reel Detail Page — Tabbed Layout +``` +[ Overview ] [ Caption Builder ] +``` + +**Overview tab:** +- Reel Title (free text) +- Recipe dropdown (from MySQL) → auto-fills keyword + URL +- Keyword (editable — this is the recipe review/confirm step) +- Recipe URL (read-only) +- Status indicator +- Save / Delete + +**Caption Builder tab:** +- Select library blocks (hashtags, links, CTAs, about snippets) +- Preview assembled caption +- Copy to clipboard + +### Reel List View +Data points shown per reel (layout TBD — getting Jenny's input on table vs +cards): +- Reel title +- Linked recipe name (or "No Recipe" warning) +- Keyword +- Status (color dot + label) +- Date created or last updated + +### Recipe Review UI +**Decision:** Not a separate screen. Baked into the reel creation flow — +when Jenny selects a recipe from the dropdown, she sees the record (title, +URL, keyword) and confirms/edits the keyword right there. + +### Manual Match Correction +A "Link Reel" button on flagged records in the dashboard. Jenny selects +from a list of unmatched `instagram_posts` rows to connect the dots. + +--- + +## Reel Status States (Draft) + +1. **Draft** — reel record created, no recipe linked yet +2. **Ready** — recipe linked, keyword confirmed, caption built. Waiting to +be posted +3. **Live — Matched** — Instagram webhook received, reel ID attached, +auto-reply active +4. **Live — Unmatched / Needs Review** — auto-match couldn't confidently +link + +--- + +## Open Questions (Still To Decide) + +- [ ] Reel list layout: table rows vs cards (get Jenny's feedback) +- [ ] Jenny's assistant access: same login or separate user accounts? +- [ ] Status Dashboard details: per-reel health indicators + system health +panel (next planning session) +- [ ] Database schema for hub reel records table (next planning session) +- [ ] Define green/yellow/red status thresholds in detail + +--- + +## Target Automation Flow (Full Picture) + +``` +Jenny creates reel record in hub → selects recipe → builds caption + ↓ +Jenny posts reel to Instagram + ↓ +Instagram webhook → n8n → writes reel ID to instagram_posts + ↓ +n8n pings hub (POST /api/match/run) + ↓ +Hub auto-match runs: + - Finds match → links record → status = Live — Matched ✅ + - No confident match → status = Needs Review 🟡 + ↓ +Comment comes in on Instagram → n8n looks up instagram_posts → sends DM + +reply + ↓ +Dashboard shows green across the board 🟢 +``` + +--- + +## Next Steps + +- [ ] Define reel status states in detail (green/yellow/red thresholds) +- [ ] Sketch Status Dashboard UI +- [ ] Define database schema for hub reel records +- [ ] Get Jenny's feedback on reel list layout (table vs cards) +- [ ] Address assistant access model +- [ ] Build once requirements are locked + +--- + +*Project: Plant Based Southerner Content Hub* +*Planning Session: March 19, 2026* +*Participants: Travis & Claude* \ No newline at end of file diff --git a/Sources/Dev/email-obsidian-git-pipeline.md b/Sources/Dev/email-obsidian-git-pipeline.md new file mode 100644 index 0000000..cd03d53 --- /dev/null +++ b/Sources/Dev/email-obsidian-git-pipeline.md @@ -0,0 +1,146 @@ +--- +created: 2026-03-24 +path: Sources/Dev +project: email-obsidian-git-pipeline +status: active +tags: +- pbs +- n8n +- automation +- gitea +- obsidian +- gmail +- gdrive +type: session-notes +updated: 2026-03-24 +--- + +# Email to Obsidian Git Pipeline — Session Complete + +## 🎉 What We Built + +A fully automated document pipeline that takes Claude-drafted project emails and files them into Gitea (Obsidian vault) and Google Drive simultaneously. + +``` +Gmail → n8n → Parse Email → Extract Frontmatter + ↓ + Gitea (vault) ✅ + ↓ + GDrive (Jenny) ✅ + ↓ + Google Chat notification ✅ +``` + +--- + +## ✅ Completed This Session + +### Gmail OAuth & Trigger +- Created new OAuth 2.0 client in Google Console for n8n (separate from mail client OAuth) +- Enabled Gmail API and Google Drive API +- Gmail trigger polls every hour for unread emails matching subject filter +- Processed emails get labeled `n8n/processed` and removed from INBOX + +### Email Parsing +- HTML body extraction using `` tag to protect markdown formatting +- HTML entity cleaning and sanitization +- YAML frontmatter validation (checks for `---` prefix) + +### Frontmatter Extraction +- Custom `parseYamlValue()` function handles quoted and unquoted values +- Defensive defaults for missing fields: + - `project` defaults to slugified email subject (strips `[flags]`) + - `path` defaults to `PBS/Inbox` for malformed emails +- Subject flag extraction: `[flag1]` = trigger, `[flag2]` = routing category + +### Email Subject Convention +``` +[n8n] PBS Project - project name +[n8n] PBS Content - content idea +[n8n] Thought - quick capture +``` +- Gmail trigger filter: `subject:[n8n] -label:n8n/processed` +- Switch node routes based on `flag2` downstream + +### Gitea File Creation +- Gitea deployed on staging with SQLite (lightweight, no extra MySQL container) +- Healthcheck temporarily removed to allow initial setup via GUI +- Files written via Gitea REST API using HTTP Request node +- Auth: Header Auth with `token YOUR_TOKEN` format (not Bearer!) +- Internal Docker hostname: `http://gitea:3000/api/v1/` + +### Google Drive Upload +- Google Drive API enabled in Google Console +- Markdown converted to binary using Convert to File node (Move Base64 String to File operation) +- Files uploaded to `pbs-planning` folder +- Same markdown content used (raw text, not base64 like Gitea) + +### Google Chat Notifications +- Centralized notification sub-workflow with space selector +- Payload: `{ title, message, status, space, timestamp }` +- Message format: `🌻 PBS Message: *title*\nmessage\n\n_timestamp_` +- Error Trigger node catches failures, routes to same notification sub-workflow +- Spaces: `webadmin`, `insta`, `automation` + +--- + +## 🔑 Key Learnings + +- **Gitea auth** uses `token YOUR_TOKEN` not `Bearer` in Authorization header +- **GDrive Upload** requires binary data — use Convert to File node before GDrive node +- **Convert to File** operation must be `Move Base64 String to File` not `toText` +- **Google APIs** must be explicitly enabled per project even with OAuth credentials set up +- **n8n Error Trigger** only fires on production executions, not manual test runs +- **Gmail labels** are the underlying mechanism for folders — remove `INBOX` label to move emails +- **Gitea healthcheck** uses API endpoint that only responds after initial setup wizard completes — chicken and egg problem with Traefik! +- **Pinned data** in n8n doesn't always flow correctly through newly added nodes + +--- + +## 🔧 Gitea Setup Notes + +- Running on staging with SQLite database +- Admin user created via CLI: `docker exec -it -u git gitea gitea admin user create` +- Must use `-u git` flag or Gitea complains about running as root +- Initial healthcheck pointed to `/api/v1/version` which 404s before setup — temporarily removed to allow Traefik routing +- After setup complete, healthcheck can be restored + +--- + +## 📋 Next Steps + +- [ ] Restore Gitea healthcheck now that setup is complete +- [ ] Connect Obsidian Git plugin to Gitea repo on homelab +- [ ] Build Switch node routing for `flag2` (PBS Content, Thought, etc.) +- [ ] Migrate existing Google Chat webhook workflows to centralized sub-workflow +- [ ] Add GDrive dynamic subfolder creation based on `path` frontmatter field +- [ ] Deploy Gitea to production after staging validation +- [ ] Explore Quartz for browser-based vault viewing (Jenny access) +- [ ] Update Gmail trigger subject convention from `[PBS Project]` to `[n8n]` prefix + +--- + +## 🏗️ Current Pipeline Architecture + +``` +Gmail (hourly poll) + └── Filter: subject:[n8n] -label:n8n/processed + └── Parse HTML body (extract block) + └── Validate YAML frontmatter + └── Extract: project, path, flag2, emailId, subject + └── Prepare file content + ├── Gitea: base64 encoded → HTTP Request → /api/v1/repos/ + └── GDrive: base64 → Convert to File → GDrive Upload + └── Label email: add n8n/processed, remove INBOX + └── Notify: Google Chat automation space + +Error Trigger (unconnected, auto-fires on error) + └── Set error payload + └── Notify: Google Chat automation space +``` + +--- + +*Last Updated: 2026-03-24* +*Maintained by: Travis* +*Session: Email to Obsidian Git Pipeline* \ No newline at end of file diff --git a/Sources/Dev/email-to-obsidian-automation.md b/Sources/Dev/email-to-obsidian-automation.md new file mode 100644 index 0000000..ae61254 --- /dev/null +++ b/Sources/Dev/email-to-obsidian-automation.md @@ -0,0 +1,208 @@ +--- +created: 2026-03-19 +path: Sources/Dev +project: email-to-obsidian-automation +status: active +tags: +- pbs +- n8n +- automation +- docker +- traefik +- production +type: project-plan +updated: 2026-03-19 +--- + +# Email-to-Obsidian Automation — Project Plan + +## 🎯 Project Goal + +Automate the pipeline from Claude chat session → Obsidian vault. Currently, +Claude drafts project notes as `[PBS Project]` emails to +tjcherb@plantbasedsoutherner.com via Gmail draft or message_compose. Those +emails are sitting unprocessed in Gmail. This project builds an n8n +workflow that watches for those emails, parses the Obsidian-compatible +markdown (including YAML frontmatter), commits the `.md` file to a Gitea +repo, and lets the Obsidian Git plugin sync it down to the homelab vault. + +**Full pipeline:** +Claude drafts email → Gmail → n8n picks it up → parses frontmatter → writes +.md to Gitea → Obsidian Git plugin pulls to homelab vault + +--- + +## 📋 Prerequisites + +- [ ] Linode PBS server running (Docker, Traefik, n8n — already operational) +- [ ] Google Workspace email (tjcherb@plantbasedsoutherner.com — already +operational) +- [ ] Obsidian vault on homelab (already exists) +- [ ] Cloudflare DNS with wildcard (already configured) + +--- + +## 🗓️ Phase 1: Gitea Setup on Linode + +### Goal +Deploy Gitea as a lightweight, self-hosted Git server on the PBS Linode +server. Accessible via Traefik at `git.plantbasedsoutherner.com` (or +similar subdomain). This becomes the central Git remote for the Obsidian +vault and potentially other PBS projects (Ansible playbooks, PBS-API code, +Content Hub, n8n workflow exports, dotfiles). + +### Step 1.1: Add Gitea to Docker Compose +- [ ] Add Gitea service to compose stack (use `gitea/gitea:latest` image) +- [ ] Provision a MySQL database (or SQLite if keeping it lean — decision +point) +- [ ] Mount persistent volume for Gitea data (`/data`) +- [ ] Connect to the Traefik network +- [ ] Add Traefik labels for routing (`git.plantbasedsoutherner.com`) +- [ ] Deploy and verify Gitea web UI is accessible with valid SSL + +### Step 1.2: Configure Gitea +- [ ] Complete initial setup wizard (admin account, settings) +- [ ] Create a `pbs-obsidian-vault` repository +- [ ] Generate an API token or SSH key for n8n to push commits +- [ ] Store credentials in ansible-vault (consistent with existing secrets +management) + +### Step 1.3: Initialize the Vault Repo +- [ ] Clone the empty repo to homelab +- [ ] Copy existing Obsidian vault contents into the repo +- [ ] Initial commit and push to Gitea +- [ ] Verify files visible in Gitea web UI + +### Step 1.4: Obsidian Git Plugin Setup (Homelab) +- [ ] Install the Obsidian Git community plugin +- [ ] Configure remote pointing to Gitea repo +- [ ] Set auto-pull interval (every 10-15 minutes) +- [ ] Enable auto-push for local edits (two-way sync) +- [ ] Test: manually create a file in Gitea web UI → verify it appears in +Obsidian after pull + +### Decision Points +- **Subdomain:** `git.plantbasedsoutherner.com` vs ` +gitea.plantbasedsoutherner.com`? +- **Database:** SQLite (simpler, fine for light use) vs MySQL (consistent +with existing stack)? +- **Ansible:** Add Gitea to existing Ansible playbook now, or manual deploy +first then codify later? + +--- + +## 🗓️ Phase 2: n8n Email-to-Vault Workflow + +### Goal +Build an n8n workflow that watches Gmail for `[PBS Project]` emails, +extracts the markdown content, parses the YAML frontmatter to determine the +file path and name, and commits the `.md` file to the correct location in +the Gitea vault repo. + +### Step 2.1: Gmail Trigger Setup +- [ ] Add a Gmail trigger node in n8n (poll-based or webhook) +- [ ] Filter for emails with subject prefix `[PBS Project]` +- [ ] Extract the email body (HTML — the markdown is wrapped in `` +tags based on current format) +- [ ] Mark or label processed emails to avoid re-processing (e.g., add a +`processed` Gmail label) + +### Step 2.2: Parse Email Content +- [ ] Strip HTML wrapper (`` tags) to get raw markdown +- [ ] Extract YAML frontmatter block (content between `---` delimiters) +- [ ] Parse frontmatter fields: + - `project` → used for filename (e.g., +`instagram-automation-session-notes.md`) + - `path` → target directory in vault (e.g., `PBS/Tech/Sessions/`) + - `type` → informational, no routing logic needed yet + - `status` → informational + - `tags` → informational + - `created` / `updated` → informational +- [ ] Construct full file path: `{path}/{project}.md` +- [ ] Handle edge cases: + - Missing frontmatter → notify Travis via Google Chat, skip processing + - Duplicate project slug → overwrite (update) existing file (desired +behavior for status changes) + +### Step 2.3: Commit to Gitea via API +- [ ] Use Gitea's REST API to create/update the file in the repo + - `POST /api/v1/repos/{owner}/{repo}/contents/{filepath}` for new files + - `PUT` with SHA for updates to existing files +- [ ] Set commit message: `[n8n] Add/Update: {project slug} ({type})` +- [ ] Authenticate using the API token from Phase 1 + +### Step 2.4: Notification & Error Handling +- [ ] On success: Google Chat notification — `✅ Vault updated: {project} → +{path}` +- [ ] On failure (parse error, API error, missing frontmatter): Google Chat +alert with details +- [ ] Wire all dead-end branches to notifications (same pattern as +Instagram workflow) + +### Step 2.5: Backfill Existing Emails +- [ ] Run the workflow manually against existing `[PBS Project]` emails +sitting in Gmail +- [ ] Verify all files land in correct vault paths +- [ ] Spot-check frontmatter parsing on a few files + +--- + +## 🧪 Testing Strategy + +- [ ] Test on staging n8n instance first (consistent with existing workflow) +- [ ] Use pinned execution data for safe testing +- [ ] Send a test email with known frontmatter → verify file appears in +Gitea +- [ ] Verify Obsidian Git plugin picks up the new file on homelab +- [ ] Test update scenario: send email with same project slug → verify file +is overwritten +- [ ] Test error scenario: send email with malformed/missing frontmatter → +verify Google Chat alert + +--- + +## 🔑 Key Architecture Decisions + +- **Gitea on Linode (same server as n8n):** Enables localhost +communication, no external auth tokens needed for n8n → Gitea. Also useful +for other PBS repos beyond just the vault. +- **Gitea REST API (not Git CLI from n8n):** Cleaner than shelling out to +git commands inside the n8n container. API is well-documented and n8n HTTP +Request node handles it easily. +- **Obsidian Git plugin (not cron job):** Simpler setup, two-way sync, only +runs when Obsidian is open (which is when you need the files anyway). Can +add cron later if needed. +- **Overwrite on duplicate slug:** Treat the `project` field as a unique +key. If Claude sends an updated version of a project plan, the new one +replaces the old. Git history preserves all versions. +- **Gmail label for dedup:** Processed emails get labeled so the trigger +doesn't re-process them. + +--- + +## 📊 Success Criteria + +- [ ] Gitea running at chosen subdomain with valid SSL +- [ ] Obsidian vault repo syncing bidirectionally (Gitea ↔ homelab) +- [ ] n8n workflow triggers on new `[PBS Project]` emails +- [ ] Markdown files land in correct vault paths based on frontmatter +- [ ] Existing backlog of Gmail project emails are processed +- [ ] Error handling wired to Google Chat +- [ ] End-to-end test: Claude drafts email → file appears in Obsidian vault +within 15 minutes + +--- + +## 🎯 Future Enhancements (Not in Scope) + +- [ ] Add Gitea to Ansible playbook for reproducible deploys +- [ ] Authelia SSO for Gitea web UI (bundle with Portainer/Uptime Kuma SSO +project) +- [ ] n8n workflow version control — export workflow JSON to Gitea repo +- [ ] PBS-API and Content Hub source code hosted on Gitea +- [ ] Obsidian vault backup strategy (Gitea → offsite mirror) + +--- + +*Last Updated: March 19, 2026* +*Maintained by: Travis* \ No newline at end of file diff --git a/Sources/Dev/github-deploy-refactor.md b/Sources/Dev/github-deploy-refactor.md new file mode 100644 index 0000000..3a296e3 --- /dev/null +++ b/Sources/Dev/github-deploy-refactor.md @@ -0,0 +1,65 @@ +--- +created: 2026-04-13 +path: Sources/Dev +project: github-deploy-refactor +status: complete +tags: +- pbs +- ansible +- deployment +- github +type: project-plan +updated: 2026-04-25 +--- + +# GitHub Deploy Refactor: pbsdeploybot Machine User Pattern + +Replaced the per-repo deploy key approach in the wordpress-install Ansible +project with a centralized machine user (pbsdeploybot) for GitHub SSH access. + +## Background + +The original github role used individual deploy keys per repository, copied +to the target server at deploy time. This created three problems: key sprawl +across repos, complex three-way clone branching logic, and a separate SSH +key file for every repo that needed cloning. + +## What Changed + +### Machine user: pbsdeploybot +- GitHub machine user account with read access to all PBS repos +- Single SSH key pair on each target server, configured in `~/.ssh/config` +- Aliased SSH hosts (e.g., `git-real-projects`) route to the correct key +- No deploy keys needed per-repo — pbsdeploybot is added as a collaborator + +### Ansible github role (wordpress-install) +- Removed per-repo deploy key copy tasks +- Removed three-way clone branching logic +- Simplified to a single `ansible.builtin.git` task with `accept_hostkey: true` +- Runs as `become_user: "{{ ansible_user }}"` so SSH config is available +- Vars are passed per-invocation (repo_url, repo_dest, repo_version) + +### Playbook integration +- github role invoked twice: once for ssh-login-alerter, once for pbs-video-manager +- Both clones run before their dependent application roles +- Tagged so `--tags ssh_login_alert` triggers both the clone and the app role + +### Key details +- SSH key: pre-configured on target servers in `~/.ssh/config` +- Host alias pattern avoids bare `github.com` in SSH config +- `repo_version: master` (not main) for repos using that convention +- Defaults in `roles/github/defaults/main.yml`: repo_version=main, owner/group=ansible_user + +## Current State + +Working in the wordpress-install staging branch. The Ansible playbook run +was blocked by an SSH key issue on the staging server (the key was offered +but rejected). Not yet deployed to production. + +## Future Direction + +The docker-container project will absorb the github clone logic into the +dockers role as `tasks/common/github.yml`, sitting next to `common/deploy.yml`. +Containers that need a repo clone will import it with container-specific vars +before calling deploy.yml. This eliminates the standalone github role for +container deployments while keeping the same pbsdeploybot auth pattern. \ No newline at end of file diff --git a/Sources/Dev/herbys-dev-setup.md b/Sources/Dev/herbys-dev-setup.md new file mode 100644 index 0000000..278cf40 --- /dev/null +++ b/Sources/Dev/herbys-dev-setup.md @@ -0,0 +1,241 @@ +--- +created: 2026-04-22 +path: Sources/Dev +project: herbys-dev-setup +status: active +tags: +- homelab +- proxmox +- ansible +- dev-environment +- backups +- tailscale +type: project-plan +updated: 2026-04-22 +--- + +## herbys-dev — Primary Dev Environment + +Promote `us-test-authy` (VMID 100 on PVE, ZFS pool `vmstore`) from test VM +to the primary development environment. Rename to `herbys-dev`, harden it, +establish real backups, and codify the entire config as an idempotent +Ansible playbook in a new repo. + +The `ladybug` name stays with the Claude Desktop + Dispatch instance +running on the host — not the host itself. + +### Why + +The Manjaro tower pulls ~1500W when active. VM on PVE is accessible +anywhere via Tailscale, persistent across sessions, and cheap to run 24/7. +Keeping the tower asleep unless GPU work (Stable Diffusion, Ollama, Immich +ML) is actually needed. + +Current state has gaps: VM is named for its original test purpose, no +formal backup strategy beyond PVE snapshots, VNC exposed on LAN with +password-only auth, no config-as-code story. + +### Goals + +- Rename to `herbys-dev` everywhere (hostname, Tailscale, DNS, configs) +- Idempotent Ansible playbook that can rebuild the entire dev env from a +fresh Ubuntu VM +- Nightly file-level backups of entire `/home/herbyadmin` with offsite copy +- Weekly Proxmox VM backups with sensible retention +- Tailscale-only VNC access (no LAN exposure) +- SSH available on LAN + Tailscale +- Lightweight disk/health alerting +- Dotfiles repo created and sourced by the Ansible playbook + +### Scope — what this plan covers + +- Rename + DNS + Tailscale migration +- New Ansible repo with idempotent playbook +- Backup strategy (file-level + VM-level) +- Security hardening (VNC, SSH, unattended upgrades) +- Baseline tooling install +- Dotfiles repo creation +- Lightweight monitoring (disk space alert) + +### Out of scope (future projects) + +- Full Prometheus/Grafana observability +- Migration of other VMs to the same Ansible pattern +- Claude Desktop / Dispatch reconfiguration beyond what the rename requires + +### Decisions made + +- **Host name**: `herbys-dev` +- **Ansible repo**: new, separate from `trucktrav/ansible-drive` +- **Playbook must be idempotent** — safe to re-run anytime +- **Secrets**: Ansible Vault for anything sensitive; no plaintext tokens in +repo +- **File-level backup scope**: entire `/home/herbyadmin` with sensible +excludes (`.venv`, `node_modules`, `.cache`, `__pycache__`, build artifacts) +- **Backup frequency**: file-level nightly; VM-level weekly full + daily +incremental (pending retention window decision) +- **VNC**: Tailscale-bound only, remove LAN exposure +- **SSH**: allowed on LAN + Tailscale +- **Dotfiles**: include in this project, playbook pulls and symlinks from +the repo +- **Shell config**: existing setup to be pulled from current machines — not +creating from scratch + +### Open decisions (answer during implementation) + +- Proxmox backup target: shelved 1TB spinner re-added to PVE, or NAS over +NFS/SMB? +- File-level backup tool: Restic (leaning this way for rclone backend) or +Borg? +- Offsite destination: `tjgdrive` Google Drive remote, or different target? +- Backup retention windows: how many daily / weekly / monthly to keep? +- Rename approach: in-place (change hostname, Tailscale name, update DNS) +vs. fresh VM from snapshot with new name and data migration +- Ansible run mode: push from control node, or pull-mode via `ansible-pull` +cron? +- Baseline tooling list beyond the obvious (uv, Go, Node, git, tmux, +Neovim, Claude Code, ripgrep, fd, bat, jq, htop) — anything else to bake in? +- Disk alert delivery: local cron + email, or ping into existing +notification channel? + +### Tasks + +#### Phase 1: Preparation + +- [ ] Take a pre-work Proxmox snapshot of current VM 100 +- [ ] Document current state of `us-test-authy`: installed packages, +running services, `/home` contents, systemd user units, Claude Code config, +VNC config, OpenClaw state +- [ ] Decide rename approach (in-place vs fresh VM) +- [ ] Decide Proxmox backup target (spinner vs NAS) +- [ ] Decide file-level backup tool (Restic vs Borg) +- [ ] Decide offsite backup destination and retention policy + +#### Phase 2: Ansible repo setup + +- [ ] Create new GitHub repo `trucktrav/herbys-dev-ansible` (or similar) +- [ ] Initialize with inventory, playbook skeleton, roles directory +- [ ] Set up Ansible Vault for secrets +- [ ] Write first idempotent task as baseline (e.g., hostname set) +- [ ] Test run against current VM to verify no destructive behavior + +#### Phase 3: Dotfiles repo + +- [ ] Create new GitHub repo `trucktrav/dotfiles` +- [ ] Pull existing shell config from current setup +- [ ] Add git config, tmux config, Neovim config, SSH config template +- [ ] Document what's in there in a README + +#### Phase 4: Ansible roles + +- [ ] Role: `common` — hostname, timezone, locale, unattended-upgrades +- [ ] Role: `baseline-tools` — uv, Go, Node (via fnm or nvm), git, tmux, +Neovim, ripgrep, fd, bat, jq, htop, Claude Code +- [ ] Role: `dotfiles` — clone and symlink from dotfiles repo +- [ ] Role: `ssh-hardening` — key-only auth, sensible sshd_config +- [ ] Role: `tailscale` — ensure Tailscale running with key expiry disabled +- [ ] Role: `vnc` — XFCE + TigerVNC bound to Tailscale interface only, +systemd user unit for persistence +- [ ] Role: `claude-desktop` — install via patrickjaja apt repo, Dispatch + +Cowork service units, loginctl linger +- [ ] Role: `backups` — install backup tool, configure repo, systemd timer +for nightly runs +- [ ] Role: `monitoring` — disk space check cron, alert delivery +- [ ] Role: `firewall` — minimal ufw config (allow SSH LAN + Tailscale, VNC +Tailscale only) + +#### Phase 5: Backups + +- [ ] Configure Proxmox-level backup job (weekly full + daily incremental, +chosen target) +- [ ] Set up file-level backup tool (Restic or Borg) +- [ ] Test restore path end-to-end: delete a file, restore from backup, +verify integrity +- [ ] Document restore procedure in the Ansible repo README + +#### Phase 6: Rename migration + +- [ ] Take fresh snapshot immediately before rename +- [ ] Change Linux hostname to `herbys-dev` +- [ ] Update Tailscale node name +- [ ] Update Pi-hole local DNS record (if one exists) +- [ ] Update `/etc/hosts` on other machines (`truckslaptop`, `manjaro1`, +`pve`) +- [ ] Update SSH client config on other machines +- [ ] Update any hardcoded references in Claude Code configs, systemd +units, existing project `CLAUDE.md` files +- [ ] Update the `zero-check` skill if it references the old hostname +- [ ] Verify Dispatch still pairs correctly after rename +- [ ] Verify OpenClaw gateway still routes correctly over Tailscale + +#### Phase 7: Hardening + +- [ ] Migrate VNC to Tailscale-only binding +- [ ] Confirm no lingering LAN exposure on port scan +- [ ] Enable unattended-upgrades for security patches +- [ ] Verify SSH key-only auth (no password fallback) +- [ ] Review running services, disable anything unused + +#### Phase 8: Validation + +- [ ] Re-run full Ansible playbook against live VM — confirm zero changes +reported (true idempotency) +- [ ] Simulate fresh-rebuild scenario: clone fresh Ubuntu VM, run playbook +end-to-end, verify identical state +- [ ] Trigger a backup manually and verify offsite copy exists +- [ ] Trigger restore test on a throwaway file +- [ ] Trigger disk alert manually (e.g., create a test file to push past +threshold) and verify delivery + +#### Phase 9: Documentation + +- [ ] Write `MACHINE.md` on herbys-dev listing what's installed, where data +lives, what's backed up +- [ ] Update Obsidian memory / project notes: `us-test-authy` → +`herbys-dev`, role change from test to primary dev +- [ ] Update any references in other project plans +(project-scaffolding-skill plan currently says "Manjaro tower is primary +dev" — needs correction) +- [ ] README in Ansible repo: what the playbook does, how to run it, how to +add a new role + +### Testing strategy + +- Every Ansible role must be safe to run twice in a row with zero changes +on the second run +- Backup + restore must be validated before the old `us-test-authy` +snapshot is deleted +- Rename migration must include rollback plan (revert snapshot) in case +Dispatch, OpenClaw, or Tailscale break + +### Success criteria + +- Playbook runs green against a fresh Ubuntu VM and produces an identical +dev environment +- Backups running nightly with verified restore path +- Zero LAN-exposed services beyond SSH +- Host renamed everywhere it matters +- Dotfiles repo populated and Ansible-managed +- Machine documented + +### References + +- Current VM: `us-test-authy`, VMID 100, ZFS pool `vmstore` +- Existing Ansible pattern: `trucktrav/ansible-drive` (for structure +reference only, not extended) +- Related project: `zero-check` skill at `~/.claude/skills/zero-check/` +- Related project: `project-scaffolding-skill` plan (filed separately) +- Key infra facts: Tailscale mesh connects `truckslaptop`, `manjaro1`, +`pve`, current VM; Pi-hole + Unbound at `10.0.11.50` + +### Next session + +1. Answer the open decisions above (backup targets, tool choice, rename +approach) +2. Create the new Ansible repo and initialize skeleton +3. Create dotfiles repo and seed with current config +4. Start with the `common` role and test idempotency before scaling to the +full set + + +...sent from Jenny & Travis \ No newline at end of file diff --git a/Sources/Dev/index.md b/Sources/Dev/index.md new file mode 100644 index 0000000..09df967 --- /dev/null +++ b/Sources/Dev/index.md @@ -0,0 +1,23 @@ +--- +type: topic-landing +path: Sources/Dev +tags: + - landing +updated: 2026-05-08 +--- + +# Dev — Sources + +Project plans and session notes for the Dev domain. + +## Active Projects + +*Updated by compile loop.* + +## Recent Sessions + +*Updated by compile loop.* + +## Completed Projects + +*Updated by compile loop.* diff --git a/Sources/Dev/instagram-automation-content-hub-plan.md b/Sources/Dev/instagram-automation-content-hub-plan.md new file mode 100644 index 0000000..449b053 --- /dev/null +++ b/Sources/Dev/instagram-automation-content-hub-plan.md @@ -0,0 +1,148 @@ +--- +created: 2026-03-15 +path: Sources/Dev +project: instagram-automation-content-hub-plan +status: active +tags: +- pbs +- n8n +- instagram +- automation +- mysql +- flask +- docker +- wordpress +type: project-plan +updated: 2026-03-15 +--- + +# Instagram Automation & Content Hub — Master Plan + +## Background + +This plan consolidates two sessions of work: +- Finishing error checking on the Instagram comment reply workflow +- Redesigning the data pipeline to simplify the reply workflow and automate recipe/reel syncing + +### Key Architecture Decision +The reply workflow will be refactored to use a **single `instagram_posts` table lookup** instead of +the current two-table lookup + merge pattern. The `instagram_posts` table will store the recipe URL +directly, eliminating the need to join with `pbs_recipes` at reply time. + +--- + +## Phase 1 — Foundation + +- [ ] Add phpMyAdmin back to Docker Compose on staging +- [ ] Design final MySQL schema + - `pbs_recipes` — post_id, title, url, keyword (updated) + - `instagram_posts` — reel_id, post_id (FK), url, keyword (new url field) + - Foreign key linking `instagram_posts.post_id` → `pbs_recipes.post_id` +- [ ] Migrate n8n datatables to MySQL +- [ ] Verify PBS-API can query updated schema + +--- + +## Phase 2 — Finish Error Checking on Reply Workflow + +- [ ] Refactor reply workflow to single `instagram_posts` table lookup (remove merge) +- [ ] Add IF node after table lookup → Notify Travis on no record found +- [ ] Add IF node for keyword not matched → Notify Travis +- [ ] Verify subscribe check → Notify Travis (already built) +- [ ] Verify hash check → Notify Travis (already built) +- [ ] Test all error paths on staging +- [ ] Deploy to production + +### Notify Travis Workflow (already built) +- Webhook trigger at: https://n8n.plantbasedsoutherner.com/webhook/notify-pbschat +- Payload format: + { + "title": "Alert title", + "message": "Detail message", + "timestamp": "={{ $now.toFormat('MMM dd, HH:mm:ss') }}" + } + +--- + +## Phase 3 — WordPress → MySQL Sync + +- [ ] Wire WordPress publish webhook (proof of concept built) to n8n workflow +- [ ] n8n workflow inserts/updates `pbs_recipes` table on post publish +- [ ] Handle duplicate detection (re-publish should UPDATE not INSERT) +- [ ] HMAC signature verification on incoming webhook (already built in WPCode Lite) +- [ ] Test with staging WordPress → staging n8n → staging MySQL +- [ ] Deploy to production + +### WordPress Webhook Details (already built) +- WPCode Lite snippet fires on post publish (admin only) +- Payload: post_id, title, url, tags, categories +- HMAC signature via X-PBS-Signature header +- Raw body passed via X-PBS-Body header (base64) for signature verification + +--- + +## Phase 4 — Instagram Reel Publish Webhook + +- [ ] Research Meta API for reel publish webhook event +- [ ] Build n8n workflow to receive reel publish event +- [ ] Insert record into `instagram_posts` on new reel +- [ ] Auto-populate URL by looking up matching `pbs_recipes` record via post_id in caption +- [ ] Trigger match verification check after insert + - If match found → update record, no alert + - If no match found → Notify Travis to manually link + +--- + +## Phase 5 — PBS Content Hub Updates (Planning Required First) + +- [ ] **Planning session required before any code is written** +- [ ] Review current Content Hub architecture and existing features +- [ ] Define full requirements for: + - Recipe review UI — Jenny can view new recipe records, confirm or edit keyword before reel goes live + - Reel/recipe match status dashboard — shows matched vs unmatched reels at a glance + - Manual match correction UI — fix bad or missing matches without touching MySQL directly +- [ ] Design UI/UX mockups +- [ ] Get Jenny feedback on workflow fit +- [ ] Build once requirements are locked + +--- + +## Full Automation Flow (Target State) + +1. Recipe published on WordPress + → WPCode Lite webhook fires + → n8n inserts record into `pbs_recipes` + +2. Jenny reviews PBS Content Hub + → Confirms or edits keyword + → One intentional human checkpoint ✅ + +3. Jenny publishes Instagram reel (with WordPress post_id in caption) + → Meta webhook fires + → n8n inserts record into `instagram_posts` with URL from `pbs_recipes` + +4. System verifies match + → If clean → ready for comments + → If broken → Notify Travis via Google Chat + +5. User comments on reel + → n8n comment reply workflow fires + → Single lookup on `instagram_posts` + → Keyword match → send DM + public reply + → All dead ends → Notify Travis + +--- + +## Notes + +- All development on staging first, then promote to production +- phpMyAdmin for direct MySQL access (Travis/admin only) +- PBS Content Hub is the Jenny-facing interface for data management +- Notify Travis workflow handles ALL alerts (Google Chat) +- n8n datatables to be fully retired once MySQL migration complete + +--- + +*Last Updated: March 15, 2026* +*Maintained by: Travis* +*Project: Plant Based Southerner — Instagram Automation* \ No newline at end of file diff --git a/Sources/Dev/instagram-automation-session-notes.md b/Sources/Dev/instagram-automation-session-notes.md new file mode 100644 index 0000000..2e88588 --- /dev/null +++ b/Sources/Dev/instagram-automation-session-notes.md @@ -0,0 +1,165 @@ +--- +created: 2026-03-16 +path: Sources/Dev +project: instagram-automation-session-notes +status: active +tags: +- pbs +- n8n +- instagram +- mysql +- wordpress +- automation +- docker +type: session-notes +updated: 2026-03-16 +--- + +# Session Notes — MySQL Schema, Error Checking & The Great Lockout of 2026 + +## ✅ Accomplished This Session + +### Error Checking — Instagram Reply Workflow +All four error checks are now wired to Notify Travis → Google Chat: + +- ✅ Subscribe check notification +- ✅ Hash verification failure notification (with ⛔ alert level) +- ✅ Empty table lookup notification (includes Reel ID, commenter, comment text) +- ✅ Keyword not matched notification + +**Key learnings:** +- n8n "Always Output Data" needed on MySQL node so IF node can evaluate empty results +- When "Always Output Data" is on, check `$('node').first().json.id` exists instead of `length > 0` (length returns 1 for empty item) +- Google Chat HTTP Request node must use "Using Fields Below" mode, not raw JSON — literal newlines from expressions break JSON +- Build message strings as single `={{ }}` expression using `+` concatenation to keep `\n` as escaped characters +- `$now.toFormat('MMM dd, HH:mm:ss')` works in both test and live contexts (unlike `$execution.startedAt`) +- Timestamp and title passed from calling workflow, Notify Travis just passes them through + +--- + +### MySQL Schema — New Tables Created + +**`pbs_recipes`:** +```sql +CREATE TABLE IF NOT EXISTS pbs_recipes ( + id BIGINT UNSIGNED NOT NULL AUTO_INCREMENT, + post_id BIGINT UNSIGNED NOT NULL, + title VARCHAR(500) NOT NULL, + url VARCHAR(500) NOT NULL, + keyword VARCHAR(100) NOT NULL, + created_at DATETIME DEFAULT CURRENT_TIMESTAMP, + updated_at DATETIME DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + PRIMARY KEY (id), + UNIQUE KEY (post_id) +); +``` + +**`instagram_posts`:** +```sql +CREATE TABLE IF NOT EXISTS instagram_posts ( + id BIGINT UNSIGNED NOT NULL AUTO_INCREMENT, + reel_id VARCHAR(50) NOT NULL, + post_id BIGINT UNSIGNED NULL, + created_at DATETIME DEFAULT CURRENT_TIMESTAMP, + updated_at DATETIME DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + PRIMARY KEY (id), + UNIQUE KEY (reel_id), + FOREIGN KEY (post_id) REFERENCES pbs_recipes(post_id) +); +``` + +**Key decisions:** +- `post_id` is nullable in `instagram_posts` — NULL means unmatched reel, not an error +- `reel_id` stored as VARCHAR not BIGINT (Instagram ID integer overflow issue) +- Normalized schema — keyword and URL live in `pbs_recipes` only, JOIN at query time +- Staying on shared MySQL instance — at current scale (2k/month, 10-20 replies) separate instance is overkill +- phpMyAdmin added back to Docker Compose for direct MySQL access + +--- + +### WordPress Publish Webhook — Proof of Concept Complete +- WPCode Lite snippet fires on post publish (admin only) +- Payload: post_id, title, url, tags, categories +- HMAC signature verification via X-PBS-Signature header +- Raw body passed via X-PBS-Body header (base64 encoded) for exact signature verification +- n8n Code node verifies signature before processing +- Ready to wire into `pbs_recipes` table insert workflow + +--- + +### Architecture Decision — Reply Workflow Refactor +Current two-table lookup + merge is being replaced with: +- Single `instagram_posts` lookup only +- URL stored directly in `instagram_posts` (populated at reel insert time from `pbs_recipes`) +- Simpler, faster, fewer failure points + +--- + +## 🔧 The Great WordPress Lockout of 2026 + +**What happened:** Wordfence locked out staging WP admin. Unlock email never arrived. + +**What we tried (none of it worked):** +- WP-CLI wf:unblock-ip command (not a registered command) +- Clearing wp_options Wordfence entries in MySQL +- Disabling Wordfence plugin +- Restarting WordPress container +- Deleting wflogs directory + +**Root cause discovered:** +Wordfence WAF runs as a PHP auto_prepend_file — completely independent of the plugin being active. Found in .htaccess and wordfence-waf.php in WordPress root. + +**Actual fix:** Wrong email address on the admin account. Changed it, logged right in. 🤦 + +**Real lesson learned:** Read the error message carefully before spending an hour troubleshooting! 😄 + +**Lockout duration if we hadn't figured it out:** 1 MONTH + +--- + +## 📋 Runbook Addition — WordPress Lockout Recovery + +```bash +# Get into WordPress container +docker exec -it wordpress bash +cd /var/www/html + +# List all users +wp --allow-root user list + +# Reset password by username +wp --allow-root user update admin --user_pass=newpassword + +# Reset password by email +wp --allow-root user update user@email.com --user_pass=newpassword + +# If WAF is blocking login page independently of plugin: +# 1. Remove Wordfence WAF block from .htaccess +# 2. Rename wordfence-waf.php to wordfence-waf.php.disabled +# 3. docker restart wordpress +``` + +--- + +## 🔜 Next Steps + +- [ ] Wire WordPress publish webhook → n8n workflow → insert into `pbs_recipes` +- [ ] Build one-time migration workflow — n8n datatables → MySQL +- [ ] Finish reply workflow refactor — remove Merge node, single `instagram_posts` lookup +- [ ] Fix production WordPress admin email (same issue exists there!) +- [ ] Deploy all changes to production once staging is verified + +--- + +## 📝 Notes + +- All development on staging (staging.plantbasedsoutherner.com) +- n8n datatables to be retired once MySQL migration complete +- Foreign key constraint requires `pbs_recipes` record to exist before `instagram_posts` insert +- Use `WHERE post_id IS NULL` query to find unmatched reels for Content Hub dashboard + +--- + +*Last Updated: March 16, 2026* +*Maintained by: Travis* +*Project: Plant Based Southerner — Instagram Automation* \ No newline at end of file diff --git a/Sources/Dev/instagram-reel-sync-phase4.md b/Sources/Dev/instagram-reel-sync-phase4.md new file mode 100644 index 0000000..f2ea4b6 --- /dev/null +++ b/Sources/Dev/instagram-reel-sync-phase4.md @@ -0,0 +1,223 @@ +--- +created: 2026-03-17 +path: Sources/Dev +project: instagram-reel-sync-phase4 +status: active +tags: +- pbs +- n8n +- instagram +- mysql +- automation +type: project-plan +updated: 2026-03-17 +--- + +# Phase 4 Handoff — Instagram Reel Detection & Sync + +## Context for New Chat Session + +This is a handoff document from the previous chat session. Paste this into the new +chat along with the standard project context to get up to speed quickly. + +--- + +## Current State — What's Already Built + +### MySQL Schema (pbs_automation database) + +**`pbs_recipes`:** +```sql +CREATE TABLE IF NOT EXISTS pbs_recipes ( +    id           BIGINT UNSIGNED NOT NULL AUTO_INCREMENT, +    post_id      BIGINT UNSIGNED NOT NULL, +    recipe_title        VARCHAR(500)    NOT NULL, +    recipe_url          VARCHAR(500)    NOT NULL, +    keyword      VARCHAR(100)    NULL, +    created_at   DATETIME        DEFAULT CURRENT_TIMESTAMP, +    updated_at   DATETIME        DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, +    PRIMARY KEY (id), +    UNIQUE KEY (post_id) +); +``` + +**`instagram_posts`:** +```sql +CREATE TABLE IF NOT EXISTS instagram_posts ( +    id           BIGINT UNSIGNED NOT NULL AUTO_INCREMENT, +    reel_id      VARCHAR(50)     NOT NULL, +    pbs_post_id      BIGINT UNSIGNED NULL, +    instragram_caption  TEXT    NULL, +    created_at   DATETIME        DEFAULT CURRENT_TIMESTAMP, +    updated_at   DATETIME        DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, +    PRIMARY KEY (id), +    UNIQUE KEY (reel_id), +    FOREIGN KEY (post_id) REFERENCES pbs_recipes(post_id) +); +``` + +**Key schema decisions:** +- `post_id` is nullable in `instagram_posts` — NULL means unmatched reel +- `reel_id` is VARCHAR not BIGINT (Instagram ID integer overflow issue) +- Normalized — keyword and URL live in `pbs_recipes` only, JOIN at query time +- Foreign key constraint means `pbs_recipes` record must exist before `instagram_posts` insert + +### WordPress → pbs_recipes Sync (Complete ✅) +- WPCode Lite PHP snippet fires on every post save (admin only) +- Payload: post_id, title, url, tags, categories +- HMAC signature via X-PBS-Signature header (raw body via X-PBS-Body base64) +- n8n workflow verifies signature, upserts into pbs_recipes +- Keyword populated from first WordPress tag if present +- COALESCE logic: only sets keyword if currently NULL (preserves manual edits) + +### Instagram Comment Reply Workflow (Complete ✅) +- Handles incoming Instagram comment webhooks +- Hash verification → subscribe check → PBS message filter +- Looks up reel in `instagram_posts` table +- Joins to `pbs_recipes` for keyword and URL +- Sends DM + public reply on keyword match +- All dead ends notify Travis via Google Chat + +### Notify Travis Workflow (Complete ✅) +- Webhook trigger at: https://n8n.plantbasedsoutherner.com/webhook/notify-pbschat +- Routes all alerts to Google Chat +- Payload format: +```json +{ + "title": "Alert title", + "message": "Detail message", + "timestamp": "={{ $now.toFormat('MMM dd, HH:mm:ss') }}" +} +``` + +--- + +## Phase 4 Goal — Automate Instagram Reel Detection + +### The Problem +When Jenny publishes a new Instagram Reel, someone currently has to manually: +1. Get the WordPress post ID +2. Add it to the reel caption +3. Add the reel ID to the `instagram_posts` table + +### The Solution — Three-Tier Detection System + +**Tier 1 — Comment Triggered (real time)** +When a comment comes in and `instagram_posts` lookup returns empty: +- Instead of just notifying Travis, trigger the "Fetch Reel Data" workflow +- Pass enough context to reply to the original commenter if reel is found +- First commenter may experience a few seconds delay — acceptable + +**Tier 2 — Scheduled Poll (every 12 hours)** +- n8n Schedule Trigger polls Instagram media API +- Catches reels that haven't received a comment yet +- Proactively keeps table up to date + +**Tier 3 — Manual Trigger (Content Hub)** +- "Sync Reels" button in PBS Content Hub Flask app +- Jenny or assistant can trigger on demand after publishing + +### Queue and Retry Pattern (Option B) +When Tier 1 triggers: +1. Comment comes in → no record found → store original comment data +2. Trigger "Fetch Reel Data" workflow with comment context +3. If reel found → insert into `instagram_posts` → retry reply to original commenter +4. If reel not found → Notify Travis with details + +--- + +## Phase 4 Architecture + +### New Workflow — "Fetch Reel Data" + +**Triggers:** +- Called from comment reply workflow (Tier 1) with single reel ID +- Called from Schedule Trigger (Tier 2) with last 20 media items +- Called from Content Hub webhook (Tier 3) manually + +**Core loop logic (Loop Over Items node):** +``` +[Get reels from Instagram API] + → [Code Node: extract post_id from caption] + → [Loop Over Items] + → [MySQL SELECT: pbs_recipes WHERE post_id = extracted_id] + → [IF: record found?] + → TRUE → [MySQL UPSERT: instagram_posts] + → FALSE → [Notify Travis: Bad/missing post ID in caption] + → [After loop: if Tier 1 triggered → retry comment reply] +``` + +**Handoff payload from comment reply workflow (Tier 1):** +```json +{ + "resolved_media_id": "the reel ID", + "comment_text": "original comment text", + "from_username": "commenter username", + "comment_id": "needed to post public reply" +} +``` + +### Instagram Media API Details + +**No webhook exists for new reel publications** — confirmed via Meta docs research. +Polling is the only option. + +**API endpoint:** +``` +GET https://graph.facebook.com/v25.0/{ig-user-id}/media + ?fields=id,caption,media_type,media_product_type,permalink,timestamp + &limit=20 + &access_token={token} +``` + +**Key field:** `media_product_type === "REELS"` — do NOT use `media_type`, +it returns "VIDEO" for both regular videos and Reels. + +**Rate limits:** Generous — 4,800 × account impressions/24hrs. Polling every +12 hours is negligible. + +**Token:** Long-lived user access token (60-day expiry). Build monitoring +for token expiry — biggest operational risk. + +--- + +## Key n8n Patterns Established + +- **Always Output Data** on MySQL nodes so IF nodes can evaluate empty results +- **Check field exists** (`id exists`) not `length > 0` when Always Output Data is on +- **Google Chat HTTP Request** must use "Using Fields Below" not raw JSON +- **Message strings** built as single `={{ }}` expression with `+` concatenation +- **`$now.toFormat()`** works in test and live contexts (not `$execution.startedAt`) +- **MySQL query parameters** — do NOT use `=` prefix, plain comma separated values +- **Loop Over Items** — use explicit `$('Node Name').first().json` not `$json` + inside loops since `$json` only references immediate previous node +- **Set node before loops** — consolidate upstream data needed inside loop + so it's always accessible as `$json.field` +- **`reel_id` always VARCHAR** — Instagram IDs exceed JS safe integer limit + +--- + +## Remaining Work After Phase 4 + +- [ ] Refactor comment reply workflow to split at `Check if we sent` + into separate "Instagram Reply Handler" workflow (cleaner logs) +- [ ] PBS Content Hub — Phase 5 planning session required before building +- [ ] Authelia SSO for admin tools (Portainer, n8n, Uptime Kuma, phpMyAdmin) +- [ ] Deploy all staging changes to production + +--- + +## Infrastructure Quick Reference + +- **Staging:** staging.plantbasedsoutherner.com +- **n8n:** n8n.plantbasedsoutherner.com +- **phpMyAdmin:** phpmyadmin.staging.plantbasedsoutherner.com +- **PBS-API:** internal Docker service at http://pbs-api:5000 +- **MySQL database:** pbs_automation +- **Google Chat alerts:** via Notify Travis workflow webhook + +--- + +*Last Updated: March 17, 2026* +*Maintained by: Travis* +*Project: Plant Based Southerner — Instagram Automation Phase 4* \ No newline at end of file diff --git a/Sources/Dev/mailerlite-classic-to-new-migration.md b/Sources/Dev/mailerlite-classic-to-new-migration.md new file mode 100644 index 0000000..f6bc7fb --- /dev/null +++ b/Sources/Dev/mailerlite-classic-to-new-migration.md @@ -0,0 +1,95 @@ +--- +created: 2026-04-29 +path: Sources/Dev +project: mailerlite-classic-to-new-migration +status: completed +tags: +- mailerlite +- wordpress +- migration +- sunnies +type: session-notes +updated: 2026-04-29 +--- + +# MailerLite Classic to New Migration + +## Outcome + +PBS email infrastructure fully migrated from MailerLite Classic to New +MailerLite. All active PBS subscribers, forms, automations, and the +site-wide popup are now running on the New platform. PBS stays within the +free tier (~120 active subscribers, well under the 500 limit). + +Classic remains active as a shared account with Jenny's writer audience, +which was tabled for a later decision. + +## What Changed + +- Active PBS subscribers (~120) exported from Classic by group and imported +into New, group by group, with automations disabled on import to avoid +retriggering welcome flows. +- Three freebie forms rebuilt as raw HTML embeds on their respective +Gutenberg landing pages. +- Site-wide popup form recreated in New MailerLite. +- Official MailerLite WordPress plugin reconfigured with the New account +API key (the "Classic API" field label was misleading legacy wording — same +plugin works for both platforms). +- All Classic-side PBS forms and automations deleted to prevent any stale +embed or cached page from feeding subscribers into the wrong account. + +## Key Learnings + +- **Account topology was the biggest source of confusion.** Classic was a +shared team account (Jenny + Travis via company email). The migration +created a New account that was also shared. Logging in via different emails +kept showing the same data, which initially looked like duplicate accounts +but was actually one account with multiple users. +- **Migration brings ghost subscribers.** The Classic to New migration tool +pulled 496 pre-2023 inactive subscribers into the New account alongside any +active ones. These had to be cleaned out before importing the active PBS +list, otherwise the free tier would have been blown immediately. +- **Subscriber engagement stats do not migrate.** Per-subscriber open/click +history, signup dates, and group-level performance stats are lost in any +CSV-based import. For a small list this is acceptable; baseline group stats +were noted before the move. +- **The MailerLite WordPress plugin only injects the universal loader +script.** It does not manage popup forms, triggers, or display rules — +those live entirely inside the MailerLite account. This explained why the +popup couldn't be "turned on" from the plugin UI. +- **The universal loader script lives in the page head as plugin output.** +Identifying its source took a view-source hunt. Comments and XPath helped +narrow it down to plugin-injected output rather than WPCode, theme files, +or Astra customizer. +- **Form embed approach is now consistent.** All PBS forms (three freebies ++ popup) use raw HTML embeds rather than plugin shortcodes. This decouples +form delivery from the plugin and would make a future plugin removal +painless. + +## Decisions + +- **Keep the MailerLite plugin for now.** It auto-updates the universal +loader script if MailerLite changes infrastructure. Deactivation is a +future option since all forms are raw HTML embeds and don't depend on +plugin shortcodes. +- **Writer-side audience stays in Classic.** No action taken on the writer +subscribers or PPWC2024 group. Future split decision deferred. +- **Verification approach:** monitor subscriber counts in both Classic and +New for a week. Classic should stay flat (no leak from stale embeds), New +should grow as new signups come in. + +## Future Work + +- **Custom Sunnie-based popup system.** Replace the MailerLite-controlled +popup with a self-hosted modal driven by a 3D animated Sunnie character. +Form submission stays MailerLite (raw HTML embed), but trigger logic, +frequency capping, and visual presentation move into WPCode under PBS +control. Architectural goal: control the experience locally rather than +depending on remote configuration that can't be grepped. +- **3D Sunnie pipeline experiment.** 2D render exists; image-to-3D model +conversion to be explored on local hardware, followed by Blender rigging +and animation. Treated as brand investment rather than feature work — +Sunnie as a recurring on-site presence beyond just the popup. + + +...sent from Jenny & Travis \ No newline at end of file diff --git a/Sources/Dev/pbs-api-db-connection-fix.md b/Sources/Dev/pbs-api-db-connection-fix.md new file mode 100644 index 0000000..ce13ecc --- /dev/null +++ b/Sources/Dev/pbs-api-db-connection-fix.md @@ -0,0 +1,94 @@ +--- +created: 2026-04-15 +path: Sources/Dev +project: pbs-api-db-connection-fix +status: active +tags: +- pbs +- pbs-api +- flask +- mysql +- sqlalchemy +- docker +type: session-notes +updated: 2026-04-15 +--- + +# PBS API — DB Connection Pool Fix + +## Problem + +The pbs-api container was periodically appearing unavailable (content +hub inaccessible). The container was not crashing — it was being +marked unhealthy by Docker, causing Traefik to pull it from rotation. + +Root cause: SQLAlchemy connection pool holding idle MySQL connections +that MySQL had already closed server-side. On next use, the dead +connection produces a broken pipe error. If this occurs during a +healthcheck window, the health endpoint fails 3x and Traefik stops +routing traffic. + +Confirmed via logs: +- `pymysql.err.OperationalError: (2006, "MySQL server has gone away +(BrokenPipeError(32, 'Broken pipe'))")` + +Container state confirmed clean — exit code 0, no OOM kill, restart +policy `unless-stopped`. + +## Fix + +Update `get_engine()` in the database module to add connection pool +resilience for MySQL connections. + +### Change: `get_engine()` function + +Locate the `get_engine()` function (currently uses bare +`create_engine(url, echo=False)` for all DB types) and replace with +the following: + +```python +def get_engine() -> Engine: + global _engine + if _engine is None: + url = get_database_url() + if url.startswith("sqlite"): + from pathlib import Path + db_path = Path(url.replace("sqlite:///", "")) + db_path.parent.mkdir(parents=True, exist_ok=True) + _engine = create_engine(url, echo=False) + else: + _engine = create_engine( + url, + echo=False, + pool_pre_ping=True, # test connection before use, +transparently reconnects dead ones + pool_recycle=1800, # proactively recycle +connections every 30 min + ) + return _engine +``` + +### Why this works + +- `pool_pre_ping=True` — issues a lightweight `SELECT 1` before each +query. If the connection is dead, SQLAlchemy silently replaces it. +This is the self-healing mechanism. +- `pool_recycle=1800` — proactively replaces connections older than 30 +minutes, before MySQL's `wait_timeout` closes them server-side. +- SQLite path is unchanged — pooling does not apply. + +## Outcome + +With `pool_pre_ping` in place, dead connections are detected and +replaced transparently before they can cause a request failure or +health check miss. No container restart required. + +## Notes + +- No changes needed to the `/api/health` endpoint — it is a simple 200 +and does not touch the DB, which is correct. +- Docker's `unless-stopped` restart policy does not auto-restart on +healthcheck failure — this is by design and does not need to change +given the root cause is now addressed. +- Autoheal (docker-autoheal) was considered and deferred — not needed +with pool_pre_ping in place. \ No newline at end of file diff --git a/Sources/Dev/pbs-membership-loe1-phase2-save-button.md b/Sources/Dev/pbs-membership-loe1-phase2-save-button.md new file mode 100644 index 0000000..f3174f2 --- /dev/null +++ b/Sources/Dev/pbs-membership-loe1-phase2-save-button.md @@ -0,0 +1,223 @@ +--- +created: 2026-04-06 +path: Sources/Dev +project: pbs-membership-loe1-phase2-save-button +status: planned +tags: +- pbs +- wordpress +- wprm +- wpcode +- javascript +- membership +- frontend +type: project-plan +updated: 2026-04-06 +--- + +# PBS Membership Platform — LOE 1 Phase 2: Save Button on WordPress + +## Context + +This is Phase 2 of LOE 1 (Recipe Saving) in the PBS Membership Platform. +Phase 1 built the backend API and database. Phase 2 builds the user-facing +save button that lives on WordPress recipe pages and calls the Phase 1 API. + +**Related docs:** +- `pbs-membership-loe1-recipe-saving` — Phase 1 backend foundation +--- +## Goal + +Add a save button to WordPress recipe pages that lets Sunnies bookmark +their favorite recipes with a single click. Logged-out visitors see the +button as a signup funnel. + +### Estimated Time: 4-6 hours +--- +## Scope + +### In scope +- JavaScript widget injected via WPCode Lite +- Save button in two locations on recipe pages +- Logged-in and logged-out behaviors +- Visual feedback via Sunnie icon transformation +- Responsive bottom sheet / modal for signup prompt +- Integration with Phase 1 API endpoints + +### Out of scope +- Saved recipes listing page (Phase 3) +- Collections (Phase 4) +- Pinterest button decisions (deferred pending Jenny's input on traffic) +- Star ratings or other WPRM features +--- +## User Experience + +### Logged-in Sunnie +1. Lands on a recipe page +2. Sees heart icon in brand color (two locations) +3. On page load, JS calls `/api/interactions/check` — if already saved, +heart shows as filled Sunnie +4. Clicks heart → calls `POST /api/interactions` → heart animates from +outline to Sunnie +5. Clicks Sunnie → calls `DELETE /api/interactions` → Sunnie animates back +to outline heart + +### Logged-out visitor +1. Lands on a recipe page +2. Sees heart icon (same as logged in) +3. Clicks heart → bottom sheet / modal slides up +4. Modal message: "Join the Sunnies to save your favorite recipes! Build +your personal PBS cookbook — free to join." +5. Options: "Sign Up" (goes to Ultimate Member registration) or "Maybe +Later" (dismisses, returns to recipe) + +**Key principle:** Never hide the save button. It's a passive conversion +funnel on every recipe page. +--- +## Technical Approach + +### Where the code lives + +WPCode Lite snippet — single JavaScript + CSS file injected site-wide on +recipe pages. No plugin dependency, no theme modification. + +### Detecting recipe pages + +The snippet runs on all pages but only activates when it detects WPRM +elements on the page. If no WPRM buttons exist, the script exits quietly. + +### Button injection + +JavaScript targets WPRM's CSS classes to insert the save button into the +existing button groups. TBD which specific selectors during implementation +— depends on current WPRM markup. + +**Locations:** +- **Top of post** — next to Jump to Recipe / Print Recipe +- **Inside recipe card** — next to Print / Pin buttons + +### Login detection + +Client-side check for WordPress logged-in cookie (name includes +`wordpress_logged_in_`). The JS doesn't validate the cookie — that's the +API's job. It only needs to decide: show save action or show signup prompt. + +### API integration + +All API calls go to pbs-hub endpoints from Phase 1: + +- On load: `GET /api/interactions/check?post_id={id}&action=save` +- On save click: `POST /api/interactions` with `{post_id, action: "save"}` +- On unsave click: `DELETE /api/interactions` with `{post_id, action: +"save"}` + +The post ID comes from WordPress page context (`wp_postmeta` or a JS +variable exposed by WordPress). + +### Visual design + +**Button states:** +- Not saved → outlined heart in PBS brand color +- Saved → filled Sunnie icon (PBS already has this asset) +- Hover → subtle scale or bounce +- Transition → 200-300ms animation from heart to Sunnie + +**Animation ideas to experiment with:** +- Heart shrinks → Sunnie blooms in its place +- Petal confetti burst +- Gentle rotation + scale + +Keep animations fast and subtle. Final choice made during implementation +testing. + +### Signup modal / bottom sheet + +Responsive behavior: +- **Desktop** → centered modal dialog +- **Mobile** → bottom sheet sliding up from bottom + +Same HTML/CSS, different positioning based on viewport. Pure CSS media +queries, no framework. + +**Modal content:** +- Sunnie icon +- Headline: "Save your favorites!" +- Body: "Join the Sunnies to build your personal PBS cookbook. Free to +join." +- Primary button: "Sign Up" → Ultimate Member registration URL +- Secondary: "Maybe Later" → dismiss +- Close X in corner +- Tap outside to dismiss + +### Error handling + +- API call fails → brief toast notification: "Couldn't save right now. Try +again?" +- Network offline → same toast +- Auth expired → treat as logged out, show signup modal +- All errors logged to browser console for debugging +--- +## Phase 2 Checklist + +- [ ] Review current WPRM markup on a live recipe page (identify CSS +selectors) +- [ ] Export Sunnie icon as inline SVG for button use +- [ ] Write JavaScript for login detection +- [ ] Write JavaScript for button injection (both locations) +- [ ] Write JavaScript for API integration (check, save, unsave) +- [ ] Build heart-to-Sunnie animation (CSS/SVG) +- [ ] Build signup modal (HTML + CSS) +- [ ] Implement responsive bottom sheet behavior +- [ ] Write error handling and toast notifications +- [ ] Create WPCode Lite snippet with all code +- [ ] Test: logged-in save flow +- [ ] Test: logged-in unsave flow +- [ ] Test: logged-out modal flow +- [ ] Test: API error states +- [ ] Test: mobile responsiveness (Chrome DevTools + real phone) +- [ ] Test: does not break WPRM print or Pinterest buttons +- [ ] Get Jenny's eyes on the final look +--- +## Key Decisions Made + +| Decision | Choice | Rationale | +|----------|--------|-----------| +| Plugin vs custom | Custom JavaScript via WPCode Lite | Existing favorites +plugins store data in their own tables, not pbs-hub. Would create two +sources of truth and block the future AI layer. | +| Button locations | Both top of post AND inside recipe card | Catches +users wherever their eye lands. Reinforces the save action. | +| Logged-out visibility | Always show | Button becomes a passive conversion +funnel on every recipe page. Hiding it means logged-out users never +discover the feature. | +| Logged-out behavior | Modal / bottom sheet with signup prompt | Less +disruptive than direct redirect. Lets user make the choice without losing +their place on the recipe. | +| Mobile pattern | Responsive bottom sheet | Feels native on mobile where +most traffic comes from. Same code as desktop modal. | +| Visual feedback | Heart → Sunnie icon transformation | Brand moment on +every save. Reinforces Sunnies identity. Differentiates from generic recipe +sites. | +| Animation length | 200-300ms max | Fast enough to feel snappy on repeat +use. | +| Pinterest button | Deferred | Awaiting Jenny's input on whether Pinterest +drives meaningful traffic. | +--- +## Open Questions + +- **WPRM CSS selectors:** Need to inspect current recipe page markup to +identify exact classes/IDs for button containers +- **Sunnie icon asset:** Confirm Jenny has an SVG-ready version, or need to +convert existing asset +- **Post ID exposure:** How does WordPress currently expose the post ID to +JavaScript on the page? (Likely via `wp_localize_script` or a data +attribute) +- **Brand color hex:** Exact brand color for the heart outline state +- **Pinterest decision:** Revisit with Jenny after checking Pinterest +analytics +--- +*Next Step: Inspect a recipe page's HTML to identify WPRM CSS selectors and +confirm button injection points* + + +...sent from Jenny & Travis \ No newline at end of file diff --git a/Sources/Dev/pbs-membership-loe1-phase3.md b/Sources/Dev/pbs-membership-loe1-phase3.md new file mode 100644 index 0000000..400b2f1 --- /dev/null +++ b/Sources/Dev/pbs-membership-loe1-phase3.md @@ -0,0 +1,134 @@ +--- +created: 2026-04-08 +path: Sources/Dev +project: pbs-membership-loe1-phase3 +status: planned +tags: +- pbs-hub +- flask +- membership +- loe1 +- the-garden +- ui-design +type: project-plan +updated: 2026-04-08 +--- + +# PBS Membership LOE 1 Phase 3 - The Garden + +## Overview + +Phase 3 of LOE 1 (Recipe Saving) expanded in scope during planning. +Originally scoped as just "the saved recipes page in pbs-hub," it became +clear that the saved recipes page cannot exist without first building the +member experience container it lives in. Phase 3 now covers building **the +Garden** - the branded member area in pbs-hub - with saved recipes as the +first feature planted inside it. + +This session was dedicated to locking in the member experience skeleton: +URL structure, branding, navigation, visual direction, and page layout. No +code was written. Phases 1 (Backend API) and 2 (Save button) are still in +progress and will ship before Phase 3 work begins. + +## Decisions Locked + +| Decision | Choice | Rationale | +|---|---|---| +| Member area location | `plantbasedsoutherner.com/the-garden/` | +Path-based via Traefik routing to pbs-hub. Easy to flip to subdomain later +if needed. | +| Page title / H1 | "Welcome to the Sunflower Garden" | Jenny recommended +"garden" over "patch" - warmer, more inviting, implies tending. | +| Internal naming | "the garden" for member area, "Sunnies" for community | +Clean separation - the community and the place no longer share a name. | +| Login redirect | Ultimate Member sends authenticated users to +`/the-garden/` | Single entry point to the member experience. | +| Logout flow | pbs-hub calls WP logout, lands on WordPress home | WP +remains the auth source of truth. | +| Visual direction | Option #3 (garden-themed cards, structurally normal +grid) | Ships now. Option #1 (full illustrated scene) banked as +someday-goal when illustrator budget exists. | +| Card style | Seed packets | Built-in visual language (title + +illustration + description). "Coming soon" LOEs become unopened seed +packets - perfect metaphor. | +| Empty state copy | "Your garden is still a seed. Start saving recipes and +watch it grow." | Friendly, on-brand, points new Sunnies at the first +action. | + +## Page Structure (Top to Bottom) + +1. **Header strip** - matches WordPress Astra header for seamless visual +transition. "My Garden" is active nav item. +2. **Welcome hero** - soft green gradient or botanical pattern background. +Large warm heading "Welcome to the Sunflower Garden, [Name]." Sunnie +illustration in corner. Short tagline ("Here's what's growing today" or +similar). +3. **Card grid** - seed packet cards for each LOE. Responsive: 3 across +desktop, 2 tablet, 1 mobile. +4. **"Recently added to your garden" strip** - horizontal row of 3-4 most +recent saves. Empty state for new Sunnies shows the copy above with Sunnie +pointing at the saved recipes card. +5. **Footer** - matches WordPress Astra footer. + +## Card States + +- **Active LOE** (Saved Recipes at launch): full color seed packet, +blooming sunflower illustration, count badge ("12 saved"), clickable. +- **Coming soon LOE**: muted/sepia seed packet, seed or sprout +illustration, "Coming soon" label, not clickable but visible. + +## Phase 3 Sub-Phases (Shippable Milestones) + +- **3a - The Garden Skeleton**: Traefik routing for `/the-garden/`, pbs-hub +Flask app serving the page, WP cookie auth middleware, Astra-matching +header/footer templates, empty card grid, hero with Sunnie, logout flow. +Goal: logged-in Sunnie lands on the garden and sees the shell. +- **3b - Saved Recipes Seed Packet**: First active LOE card wired up. +Queries `user_interactions` + `pbs_recipes`, shows count, card is clickable. +- **3c - Saved Recipes Detail Page**: Click-through destination. List/grid +of saved recipes with thumbnails, titles, save dates, unsave action, +click-through to WP recipe. +- **3d - "Recently Added" Strip + Empty State**: The content area on the +garden home, plus the friendly empty state for new Sunnies. +- **3e - Coming Soon Seed Packets**: Muted cards for Meal Plans, Diversity +Tracker, User Profiles. Cheap to add, high brand value - shows Sunnies +what's coming. + +## Open Questions (To Resolve Before Phase 3a Starts) + +1. **Seed packet illustrations.** Phase 3a minimum requires: one active +"saved recipes" seed packet, one generic "coming soon" seed packet, one +corner Sunnie for the hero. Source TBD - Jenny draws, AI-generated, stock, +or hire out. Needs decision before 3a ships or it becomes a blocker. +2. **Astra header/footer matching strategy.** Options: (a) rebuild Astra +header in Jinja templates by hand (simpler but drifts when WP updates), (b) +WordPress serves header/footer HTML via endpoint pbs-hub fetches (more +robust, adds dependency), (c) iframe (rejected). Needs decision before 3a +template work starts. + +## Relationship to Other Phases + +- **Phase 1 (Backend API)**: planning complete, implementation pending. +Phase 3 depends on the `user_interactions` table and query patterns +established here. +- **Phase 2 (Save Button)**: planning complete, implementation pending. +Custom JS via WPCode Lite. Phase 3 reads the data Phase 2 writes. +- **LOEs 2-4**: their "coming soon" seed packets ship in Phase 3e. Actual +implementation is future work. + +## Someday / Stretch + +- **Visual direction Option #1**: full illustrated garden scene with +clickable hotspots. Requires illustrator budget. Revisit when PBS has +revenue to support it or Jenny has bandwidth to draw it. +- **Language expansion**: "your garden," "what's growing," "tend your +garden" for settings, "new in the garden this week" for content callouts. +Microcopy opportunity across the whole member experience. + +## Next Session + +Resume on Phases 1 and 2 implementation. Phase 3 planning is parked until +those ship. + + +...sent from Jenny & Travis \ No newline at end of file diff --git a/Sources/Dev/pbs-membership-loe1-recipe-saving.md b/Sources/Dev/pbs-membership-loe1-recipe-saving.md new file mode 100644 index 0000000..cf954fc --- /dev/null +++ b/Sources/Dev/pbs-membership-loe1-recipe-saving.md @@ -0,0 +1,260 @@ +--- +created: 2026-04-06 +path: Sources/Dev +project: pbs-membership-loe1-recipe-saving +status: active +tags: +- pbs +- flask +- docker +- mysql +- membership +- automation +type: project-plan +updated: 2026-04-06 +--- + +# PBS Membership Platform — LOE 1: Recipe Saving + +## Vision + +The PBS Membership Platform is a long-term initiative to transform +plantbasedsoutherner.com from a content site into a full lifestyle +platform. The end-state vision includes personalized recipe +recommendations, a whole food diversity tracker, meal planning, a companion +app, and a custom LLM trained on plant-based cooking knowledge. + +Recipe Saving is LOE 1 — the foundational project that establishes the +membership infrastructure all other LOEs build on. +--- +## LOE 1 Phases Overview + +- **Phase 0** — Ultimate Member setup (registration, login, user roles, +basic profile page) +- **Phase 1** — Backend foundation (auth middleware, database, API +endpoints) ← THIS DOC +- **Phase 2** — Save button on WordPress recipe pages (JavaScript widget) +- **Phase 3** — Saved recipes page in pbs-hub (frontend UI) +- **Phase 4** — Collections (named groups, organize saved recipes) +- **Phase 5** — Polish (mobile optimization, loading states, error handling) +--- +## All 4 Lines of Effort + +| LOE | Project | Status | +|-----|---------|--------| +| 1 | Recipe Saving | Active — Phase 1 | +| 2 | Weekly Meal Plan Viewer | Queued | +| 3 | Whole Food Diversity Tracker | Queued | +| 4 | User Profiles & Preferences | Queued | +--- +## Phase 1: Backend Foundation + +### Goal + +Build the API layer and database schema that powers recipe saving. No +frontend — just a working API testable via curl and verifiable in +phpMyAdmin. + +### Estimated Time: 4-6 hours +--- +### Architecture + + +Sunnie browser + → WordPress cookie (set by Ultimate Member login) + → Request to pbs-hub API + → Flask auth middleware validates cookie against wp_users (read-only) + → API endpoint processes request + → Writes to pbs_hub database + → Returns JSON response + + +**Key principle:** WordPress owns identity. pbs-hub owns membership +features. The only bridge is the session cookie and read-only MySQL access. +--- +### Database Design + +#### New database: pbs_hub + +**user_interactions** — generic table for all post-related user actions +(save, like, cooked, planned, rated). Serves LOE 1 now and LOEs 2-4 later. + +sql +CREATE DATABASE pbs_hub; + +CREATE TABLE pbs_hub.user_interactions ( + id INT UNSIGNED AUTO_INCREMENT PRIMARY KEY, + wp_user_id INT UNSIGNED NOT NULL, + wp_post_id BIGINT UNSIGNED NOT NULL, + action VARCHAR(50) NOT NULL, + metadata JSON NULL, + created_at DATETIME DEFAULT CURRENT_TIMESTAMP, + UNIQUE KEY unique_action (wp_user_id, wp_post_id, action) +); + + +**user_preferences** — key-value store for platform-specific user settings +(LOE 4, but created now for completeness). + +sql +CREATE TABLE pbs_hub.user_preferences ( + id INT UNSIGNED AUTO_INCREMENT PRIMARY KEY, + wp_user_id INT UNSIGNED NOT NULL, + pref_key VARCHAR(100) NOT NULL, + pref_value JSON NOT NULL, + updated_at DATETIME DEFAULT CURRENT_TIMESTAMP ON UPDATE +CURRENT_TIMESTAMP, + UNIQUE KEY unique_pref (wp_user_id, pref_key) +); + + +#### MySQL user (scoped permissions) + +sql +CREATE USER 'pbshub_app'@'%' IDENTIFIED BY 'strong_password_here'; + +-- Read-only on WordPress (auth + recipe display) +GRANT SELECT ON wordpress.wp_users TO 'pbshub_app'@'%'; +GRANT SELECT ON wordpress.wp_usermeta TO 'pbshub_app'@'%'; +GRANT SELECT ON wordpress.wp_posts TO 'pbshub_app'@'%'; + +-- Full control on pbs_hub +GRANT ALL ON pbs_hub.* TO 'pbshub_app'@'%'; + +FLUSH PRIVILEGES; + + +**Note:** Also audit existing MySQL users (pbs-api, n8n) and scope their +permissions. Stop using root for application access. +--- +### API Endpoints + +| Method | Path | Description | Auth | Tier | +|--------|------|-------------|------|------| +| POST | /api/interactions | Create interaction (save, like, etc.) | +Required | Free | +| DELETE | /api/interactions | Remove interaction | Required | Free | +| GET | /api/interactions?action=save | List user's interactions by action +type | Required | Free | +| GET | /api/interactions/check?post_id=123&action=save | Check if specific +interaction exists | Required | Free | + +**Request body (POST):** +json +{ + "post_id": 123, + "action": "save", + "metadata": null +} + + +**Response (GET list):** +json +{ + "interactions": [ + { + "post_id": 123, + "action": "save", + "post_title": "Southern Black-Eyed Pea Stew", + "post_slug": "southern-black-eyed-pea-stew", + "saved_at": "2026-04-06T12:00:00" + } + ] +} + + +**Note:** The GET list endpoint joins pbs_hub.user_interactions with +wordpress.wp_posts to return recipe title and slug. Query filters on +`post_status = 'publish'` and `post_type IN ('post', 'wprm_recipe')` to +exclude revisions, drafts, and other post types. +--- +### Auth Middleware + +Flask `before_request` middleware that: +1. Reads `wordpress_logged_in_{hash}` cookie from request +2. Extracts user ID and token from cookie value +3. Validates HMAC against wp_users password hash + WordPress secret keys +4. Sets `g.current_user` (user ID) and `g.user_tier` (role) for the request +5. Returns 401 if missing or invalid + +**WordPress secret keys** needed from wp-config.php: +- LOGGED_IN_KEY +- LOGGED_IN_SALT + +**Security considerations:** +- HTTPS enforced everywhere (Traefik + Cloudflare) +- Read-only MySQL user for WordPress tables +- CSRF protection via Flask-WTF (for future form submissions) +- Cookie is httponly and secure (WordPress default) +--- +### Tier Authorization (Future-Proofed) + +Not needed for Phase 1 (everything is free), but the pattern is established: + +python +TIER_PERMISSIONS = { + "free": ["save", "like"], + "basic": ["save", "like", "planned", "cooked"], + "premium": ["save", "like", "planned", "cooked", "collections", +"ai_suggest"], +} + + +Adding paid tiers later is a code change, not a schema change. +--- +### Phase 1 Checklist + +- [ ] Create pbs_hub database on staging MySQL +- [ ] Create user_interactions table +- [ ] Create user_preferences table +- [ ] Create pbshub_app MySQL user with scoped permissions +- [ ] Audit and scope existing MySQL users (root cleanup) +- [ ] Extract WordPress secret keys for cookie validation +- [ ] Build Flask auth middleware (cookie → user ID) +- [ ] Build POST /api/interactions endpoint +- [ ] Build DELETE /api/interactions endpoint +- [ ] Build GET /api/interactions endpoint (with wp_posts join) +- [ ] Build GET /api/interactions/check endpoint +- [ ] Test all endpoints via curl +- [ ] Verify data in phpMyAdmin +- [ ] Error handling and logging +--- +### Key Decisions Made + +| Decision | Choice | Rationale | +|----------|--------|-----------| +| Backend language | Python / Flask | Travis's strongest language. Python +handles PBS scale easily. Revisit Go/Rust only if a specific bottleneck +appears. | +| Auth approach | WordPress cookie validation | Ultimate Member handles +registration/login (Phase 0). pbs-hub validates the cookie for API access. +Plugin is treated as disposable scaffolding. | +| Database | MySQL (pbs_hub database) | Same server, cross-database +read-only joins to WordPress. No data duplication. | +| Interaction table | Generic user_interactions | Single table serves +saves, likes, cooked, planned, rated across all LOEs. Action type is a +column, not a separate table. | +| Recipe data source | WordPress wp_posts (read-only join) | Title and slug +come from wp_posts directly, filtered by post_status and post_type to +exclude revisions. No dependency on pbs_recipes (Instagram automation stays +independent). | +| Paid tier authorization | Code-level, not schema-level | TIER_PERMISSIONS +dictionary gates actions in the API layer. Database doesn't know tiers +exist. | +| MySQL user | Scoped pbshub_app | Read-only on WordPress tables, full +access on pbs_hub. No more root for applications. | +--- +### Open Questions + +- **WordPress cookie hash suffix:** Need to check wp-config.php for the +exact cookie name (wordpress_logged_in_{hash} where hash is derived from +site URL) +- **WPRM post type:** Confirm whether recipes are stored as 'wprm_recipe' +post type or as regular 'post' with recipe metadata +- **pbs-hub deployment:** Does this extend the existing pbs-hub container +or is Phase 1 a standalone Flask app for testing? +--- +*Next Step: Create pbs_hub database and tables on staging, then build the +auth middleware* + +...sent from Jenny & Travis \ No newline at end of file diff --git a/Sources/Dev/pbs-project-tracking.md b/Sources/Dev/pbs-project-tracking.md new file mode 100644 index 0000000..f5a95f6 --- /dev/null +++ b/Sources/Dev/pbs-project-tracking.md @@ -0,0 +1,113 @@ +--- +created: 2026-04-19 +path: Sources/Dev +project: pbs-project-tracking +status: active +tags: +- pbs +- github +- trello +- project-management +type: project-plan +updated: 2026-04-19 +--- + +# PBS Project Tracking System + +Two-layer project tracking: GitHub Issues for task-level work (linked to +commits), Trello for shared high-level visibility with Jenny. + +## Architecture + +- **GitHub Issues** — per-repo project boards, task-level tracking, linked +to commits via issue numbers +- **Trello** — shared board with Jenny, high-level project phases and +milestones +- **Gitea** — unchanged, continues as Obsidian vault repo and n8n pipeline +endpoint +- **Lovebug** — references GitHub issue numbers in phase-based commits +(e.g. `closes #14`) + +## Phase 1 — Manual Setup (Loose Coupling) + +Goal: Get both tracking layers in place and establish the workflow habit. +No automation between GitHub and Trello — manual sync only. + +### GitHub Issues Setup + +- [ ] Enable GitHub Projects on each active PBS repo +- [ ] Create issue templates for standard task types (feature, bug, +infrastructure) +- [ ] Create labels that map to project areas (e.g. `membership`, +`security`, `content-intelligence`, `infrastructure`) +- [ ] Create milestones for current project phases (e.g. "Membership LOE 1 +Phase 3", "Cloudflare Access Setup") +- [ ] Populate initial issues from existing project plans in the vault +- [ ] Confirm Lovebug workflow includes issue number references in commit +messages + +### Trello Board Setup + +- [ ] Create PBS shared board (or repurpose existing if Jenny has one) +- [ ] Define lists: Backlog, Up Next, In Progress, Done +- [ ] Create cards for each high-level project/phase: + - Membership Platform LOE 1 — Phase 3 (Sunflower Garden) + - Cloudflare Access Setup + - Content Intelligence Platform (YouTube + GSC pipelines) + - Per-Container Ansible Playbooks + - WordPress Log Monitor + - Video Editing queue +- [ ] Add checklists to cards for sub-phase visibility (e.g. Phase 3a–3e +under Membership) +- [ ] Invite Jenny to the board +- [ ] Establish cadence — update Trello cards when starting/finishing a +phase + +## Phase 2 — n8n Automation (Trello Sync) + +Goal: Automate status updates from GitHub to Trello so the shared board +stays current without manual effort. + +### n8n Workflow Design + +- [ ] Set up GitHub webhook trigger in n8n (listen for issue state changes) +- [ ] Map GitHub repo + milestone to corresponding Trello card +- [ ] When all issues in a GitHub milestone close → move Trello card to +Done (or update checklist item) +- [ ] When first issue in a milestone moves to In Progress → move Trello +card to In Progress +- [ ] Add Google Chat notification on Trello card transitions (webadmin +space) +- [ ] Test on a single repo/card pair before expanding + +### Mapping Strategy + +- [ ] Define naming convention so n8n can match GitHub milestones to Trello +cards (e.g. milestone name matches card title) +- [ ] Document the mapping in this project file once established + +## Key Decisions + +- **Per-repo issue boards** (not a single central tracker) — keeps issues +next to the code +- **Loose coupling first** — validate the workflow before automating +- **GitHub over Gitea for project tracking** — keeps CI/CD and build +compute off the production server +- **Gitea unchanged** — continues serving Obsidian vault and n8n pipeline + +## Commit Linking Convention + +Reference issues in commit messages for automatic linking: + +```bash +# Link to an issue +git commit -m "Build card grid component for Sunflower Garden #12" + +# Close an issue from a commit +git commit -m "Wire up save button API, closes #14" +``` + +This turns git history into a traceable project journal. + +-- +...sent from Jenny & Travis \ No newline at end of file diff --git a/Sources/Dev/pbs-seo-automation.md b/Sources/Dev/pbs-seo-automation.md new file mode 100644 index 0000000..4c08808 --- /dev/null +++ b/Sources/Dev/pbs-seo-automation.md @@ -0,0 +1,507 @@ +--- +created: 2026-03-23 +path: Sources/Dev +project: pbs-seo-automation +status: active +tags: +- pbs +- seo +- python +- automation +- n8n +- wordpress +- yoast +- streamlit +- analytics +- google-search-console +type: project-plan +updated: 2026-03-23 +--- + +# PBS SEO Automation Pipeline + +## Project Goal +Build a self-hosted SEO automation pipeline that replaces manual copy-paste +SEO workflows with an automated system: collecting search performance data, +tracking rankings over time, researching keywords, generating optimized +meta content via Claude API, and pushing it back to WordPress — all +orchestrated through n8n and visualized in Streamlit. + +## Why This Matters +PBS is already getting organic search traffic, which means SEO is working +to some degree. But optimizing it is currently a manual, disjointed process +— Yoast shows fields but doesn't help fill them intelligently, and there's +no automation connecting keyword research to content optimization. This +project turns SEO from a chore into a data-driven pipeline that works +across all PBS content types. + +## Content Types Covered +- **Recipes** (live) — highest SEO value, drives organic discovery +- **Blog/editorial** (live) — builds authority, targets informational +queries +- **Cookbook landing pages** (future) — transactional/promotional SEO +- **Merch pages** (future) — Product schema, transactional keywords +- **Membership/classes** (future) — funnel-driven, conversion-focused + +The pipeline is designed to handle all content types from day one, even if +only recipes and blog posts exist today. + +--- + +## Architecture Overview + +``` +Google Search Console API (free) + | + Python Collector (PyCharm + UV) + | + SQLite Database + | + ┌────────────┴────────────┐ + │ │ + n8n Streamlit +(orchestration) (dashboard) + │ + ├─ Claude API (generate meta titles/descriptions) + ├─ WordPress REST API (push meta back to Yoast) + └─ Google Chat (alerts & digests) +``` + +### Shared Infrastructure with YouTube Analytics Project +- Same Streamlit instance (separate pages/tabs) +- Same n8n server for orchestration +- Separate SQLite database (keeps projects independent) +- Same Traefik reverse proxy for dashboard access +- Same Google Cloud project for API credentials + +--- + +## Phase 1: Google Search Console Setup +**Estimated Time:** 1-2 hours +**Goal:** Connect Search Console to PBS site and verify API access + +### Tasks +- [ ] Verify Google Search Console is connected for plantbasedsoutherner.com + - If yes: confirm data is flowing, check how far back data goes + - If no: add property, verify via DNS (Cloudflare), wait for data +collection to begin +- [ ] Enable Google Search Console API in Google Cloud project + - Can reuse the same project created for YouTube Analytics +- [ ] Create service account OR extend existing OAuth credentials with +scope: + `https://www.googleapis.com/auth/webmasters.readonly` +- [ ] Test API access — pull a sample query report to confirm data flows + +### Key Details +- Search Console retains 16 months of historical data +- Data is typically delayed 2-3 days +- API uses `google-api-python-client` (same library as YouTube project) +- Service account auth is simpler for automated/server-side collection (no +browser needed) + +### Deliverable +Working API access to PBS search performance data + +--- + +## Phase 2: Search Data Collector +**Estimated Time:** 3-4 hours +**Goal:** Python script that pulls search performance data into SQLite +**Tools:** PyCharm Professional, UV package manager + +### Tasks +- [ ] Initialize project with UV (`uv init pbs-seo-analytics`) +- [ ] Install dependencies: `google-api-python-client`, `google-auth` +- [ ] Build auth module (service account preferred for server-side) +- [ ] Build search query collector (queries, impressions, clicks, CTR, +position by page) +- [ ] Build page performance collector (aggregate metrics per URL) +- [ ] Build device/country breakdown collector +- [ ] Design and create SQLite schema +- [ ] Implement data ingestion with upsert logic (idempotent runs) +- [ ] Add CLI interface for manual runs and backfill (up to 16 months) +- [ ] Initial backfill of all available historical data + +### SQLite Schema (Initial Design) + +```sql +-- Pages tracked on the site +CREATE TABLE pages ( + url TEXT PRIMARY KEY, + page_type TEXT CHECK(page_type IN ('recipe', 'blog', 'merch', +'cookbook', 'membership', 'landing', 'other')), + title TEXT, + first_seen TEXT, + created_at TEXT DEFAULT CURRENT_TIMESTAMP, + updated_at TEXT DEFAULT CURRENT_TIMESTAMP +); + +-- Daily search performance per query per page +CREATE TABLE search_queries ( + date TEXT NOT NULL, + query TEXT NOT NULL, + page_url TEXT NOT NULL, + clicks INTEGER DEFAULT 0, + impressions INTEGER DEFAULT 0, + ctr REAL DEFAULT 0, + avg_position REAL DEFAULT 0, + device TEXT DEFAULT 'all', + country TEXT DEFAULT 'all', + created_at TEXT DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (date, query, page_url, device, country) +); + +-- Daily aggregate performance per page +CREATE TABLE page_daily_metrics ( + date TEXT NOT NULL, + page_url TEXT NOT NULL, + total_clicks INTEGER DEFAULT 0, + total_impressions INTEGER DEFAULT 0, + avg_ctr REAL DEFAULT 0, + avg_position REAL DEFAULT 0, + created_at TEXT DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (date, page_url) +); + +-- Keyword tracking: queries we want to monitor over time +CREATE TABLE tracked_keywords ( + keyword TEXT PRIMARY KEY, + category TEXT, + target_page_url TEXT, + added_at TEXT DEFAULT CURRENT_TIMESTAMP, + notes TEXT +); + +-- Snapshot of rank position for tracked keywords +CREATE TABLE keyword_rank_history ( + keyword TEXT NOT NULL, + date TEXT NOT NULL, + avg_position REAL, + impressions INTEGER DEFAULT 0, + clicks INTEGER DEFAULT 0, + best_page_url TEXT, + created_at TEXT DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (keyword, date), + FOREIGN KEY (keyword) REFERENCES tracked_keywords(keyword) +); + +-- SEO meta content generated and applied +CREATE TABLE seo_meta_log ( + page_url TEXT NOT NULL, + generated_at TEXT NOT NULL, + meta_title TEXT, + meta_description TEXT, + focus_keyword TEXT, + model_used TEXT DEFAULT 'claude-sonnet', + pushed_to_wordpress INTEGER DEFAULT 0, + pushed_at TEXT, + PRIMARY KEY (page_url, generated_at) +); + +-- Site-level daily summary +CREATE TABLE site_daily_metrics ( + date TEXT PRIMARY KEY, + total_clicks INTEGER DEFAULT 0, + total_impressions INTEGER DEFAULT 0, + avg_ctr REAL DEFAULT 0, + avg_position REAL DEFAULT 0, + unique_queries INTEGER DEFAULT 0, + created_at TEXT DEFAULT CURRENT_TIMESTAMP +); +``` + +### What the Data Tells You +Google Search Console API returns four core metrics per query/page +combination: +- **Clicks** — how many times someone clicked through to your site +- **Impressions** — how many times your page appeared in search results +- **CTR (Click-Through Rate)** — clicks / impressions +- **Average Position** — where you ranked (1 = top of page 1) + +You can slice these by: date, query, page, device (mobile/desktop/tablet), +country, and search type (web/image/video). + +### Deliverable +Python CLI tool that backfills and incrementally collects PBS search data +into SQLite + +--- + +## Phase 3: n8n Orchestration + LLM Meta Generation +**Estimated Time:** 4-5 hours +**Goal:** Automate data collection, generate SEO meta content with Claude, +push to WordPress + +### Tasks +- [ ] Create n8n workflow: daily scheduled trigger → Execute Command → +Python collector +- [ ] Build Claude API integration for meta generation: + - Input: page content, current keywords ranking, content type + - Output: optimized meta title, meta description, focus keyword + - System prompt tuned for PBS brand voice (whole food plant based, +southern, warm, NOT "vegan") +- [ ] Build WordPress REST API integration to push meta back to Yoast +fields: + - `_yoast_wpseo_title` (meta title) + - `_yoast_wpseo_metadesc` (meta description) + - `_yoast_wpseo_focuskw` (focus keyword) +- [ ] Add WPCode snippet to expose Yoast fields via WordPress REST API +(required for write access) +- [ ] Create approval workflow: generate meta → notify Travis/Jenny via +Google Chat → approve/reject → push to WordPress +- [ ] Create weekly SEO digest alert for Google Chat +- [ ] Error handling and failure notifications + +### LLM Meta Generation Flow +``` +n8n detects new/updated post in WordPress + │ + Fetch page content + current search queries ranking for that URL + │ + Send to Claude API with SEO-optimized system prompt + │ + Claude generates: meta title, meta description, focus keyword + │ + Store in seo_meta_log table + │ + Send to Google Chat for approval + │ + On approval: push to WordPress via REST API +``` + +### WordPress Integration Detail +Yoast's REST API is read-only by default. To write meta fields, we need a +small WPCode snippet that registers Yoast fields on the WordPress REST API. +This is a lightweight approach — about 20 lines of PHP via WPCode Lite +(already installed), no additional plugins needed. + +Alternatively, n8n can update post meta directly via the WordPress API +using the `meta` field in a PUT request to `/wp-json/wp/v2/posts/`. + +### Alert Ideas +- **New content alert:** "Jenny published a new recipe. Claude generated +meta — approve?" +- **Weekly digest:** Top gaining keywords, biggest position changes, pages +needing optimization +- **Opportunity alert:** "You're ranking #11 for 'plant based collard +greens' — small push could hit page 1" +- **Cannibalization alert:** Multiple PBS pages competing for the same +keyword + +### Deliverable +Fully automated pipeline: collect → analyze → generate → approve → publish +SEO meta + +--- + +## Phase 4: Streamlit SEO Dashboard +**Estimated Time:** 4-6 hours +**Goal:** Visual SEO analytics dashboard integrated alongside YouTube +analytics + +### Tasks +- [ ] Add SEO pages to existing Streamlit app (or create separate app) +- [ ] Build search performance overview (clicks, impressions, CTR trends) +- [ ] Build keyword rank tracker (position changes over time) +- [ ] Build page-level deep dive (which queries drive traffic to each page) +- [ ] Build content gap analysis view (queries with high impressions but +low CTR) +- [ ] Build content type comparison (recipe SEO vs blog SEO performance) +- [ ] Build "opportunities" view (keywords close to page 1, quick wins) +- [ ] Build meta generation log view (what Claude generated, what was +approved) + +### Dashboard Pages (Initial Concept) +1. **Search Overview** — total clicks/impressions/CTR trend, top queries, +top pages +2. **Keyword Tracker** — track specific keywords over time, position change +alerts +3. **Page Deep Dive** — select a page, see all queries driving traffic, +position trends +4. **Content Gaps** — high impression / low click pages (title/description +need work) +5. **Opportunities** — keywords ranking positions 8-20 (striking distance +of page 1) +6. **Content Type Breakdown** — SEO performance by content type (recipe vs +blog vs merch) +7. **Meta Generation Log** — what Claude generated, approval status, +before/after + +### Deliverable +Live SEO dashboard with actionable insights for content strategy + +--- + +## Phase 5: Competitor Intelligence (Open — Free vs Paid) +**Estimated Time:** TBD based on approach +**Goal:** Understand competitive landscape and find content opportunities + +### Option A: DIY / Free Approach +- **Manual competitor research:** Periodically Google target keywords and +note who ranks +- **Python scraping:** Build a lightweight rank checker that searches +Google for target keywords and records positions (note: Google may +rate-limit or block; use responsibly) +- **Free tools:** Google Trends API for search interest over time, +AnswerThePublic for question-based keyword ideas +- **Search Console mining:** Analyze existing query data to find patterns +and gaps — you'd be surprised how much insight is already in your own data +- **Cost:** $0 +- **Limitation:** No competitor backlink data, no domain authority scores, +limited keyword volume estimates + +### Option B: Budget Paid Tools (~$50-75/month) +- **SERPApi or DataForSEO:** Programmatic access to Google search results + - Track competitor rankings for your target keywords + - Get search volume estimates + - API-friendly, integrates cleanly with Python pipeline +- **Best for:** Automated daily rank tracking beyond what Search Console +provides +- **Cost:** ~$50-75/month depending on query volume + +### Option C: Full SEO Platform (~$99-200+/month) +- **Ahrefs, SEMrush, or Moz:** Comprehensive SEO intelligence + - Competitor keyword analysis (what they rank for that you don't) + - Backlink profiles and domain authority + - Content gap analysis at scale + - Keyword difficulty scores +- **Best for:** When you've outgrown Search Console data and need +competitive intelligence +- **Cost:** $99-200+/month + +### Recommendation +Start with Option A (free). Build the pipeline around Google Search Console +data first. After 1-2 months of collecting data, evaluate what questions +you can't answer with free data alone. That will tell you whether Option B +or C is worth the investment. Many sites PBS's size never need to go past +Option A. + +### Deliverable +Decision on competitive intelligence approach based on data from earlier +phases + +--- + +## Phase 6: Advanced SEO Automation & Iteration +**Estimated Time:** Ongoing +**Goal:** Deepen automation and cross-platform insights + +### Future Ideas +- [ ] Auto-detect new WordPress posts and trigger SEO meta generation +without manual intervention +- [ ] Cross-reference YouTube retention data with recipe page SEO +performance (which videos drive search traffic?) +- [ ] Automated internal linking suggestions (connect related recipes/blog +posts) +- [ ] Schema markup validation and monitoring (ensure WPRM recipe schema +stays healthy) +- [ ] Page speed monitoring integration (Core Web Vitals affect rankings) +- [ ] Seasonal keyword planning (predict trending search terms by season +for recipe content) +- [ ] A/B test meta titles: generate two versions, measure CTR difference +- [ ] Content calendar integration: use keyword gaps to suggest what Jenny +should create next +- [ ] Extend to merch, cookbook, and membership pages as they launch + +--- + +## Prerequisites & Dependencies + +| Requirement | Status | Notes | +|---|---|---| +| Google Search Console verified | Needs check | May already be connected +via Workspace | +| Google Cloud project | Shared | Same project as YouTube Analytics | +| Search Console API enabled | Needed | Free, quota-based | +| OAuth/Service Account credentials | Needed | Can extend existing YouTube +creds | +| Python + UV | Ready | Travis's local dev setup | +| Anthropic API key | Needed | For Claude meta generation | +| WPCode Lite (WordPress) | Ready | Already installed — needed for REST API +Yoast fields | +| n8n | Ready | Already running on Linode | +| Streamlit | Shared | Same instance as YouTube dashboard | + +--- + +## API Quotas & Costs + +| Service | Quota/Cost | Notes | +|---|---|---| +| Google Search Console API | 2000 queries/day (free) | More than enough +for PBS | +| Claude API (Sonnet) | ~$0.003 per meta generation | Pennies per recipe | +| WordPress REST API | Unlimited (self-hosted) | No external cost | +| Google Chat webhooks | Unlimited (free) | Already configured for n8n | + +--- + +## Key Decisions + +| Decision | Choice | Rationale | +|---|---|---| +| Data source | Google Search Console (free) | Actual Google data, not +estimates. 16 months history. Sufficient for PBS scale. | +| Competitor intelligence | Deferred (Phase 5) | Start free, evaluate need +after collecting own data. | +| LLM for meta generation | Claude API (Anthropic) | Consistent with PBS +brand, excellent at structured content, cost-effective. | +| Meta push to WordPress | REST API via WPCode snippet | Lightweight, no +extra plugins, uses existing WPCode Lite install. | +| Dashboard | Streamlit (shared with YouTube) | Single analytics platform +for all PBS data. | +| Approval workflow | Google Chat notification | Keeps human in the loop +before meta goes live. Jenny/Travis approve. | + +--- + +## Sequencing & Priority +1. **Phase 1** (Search Console Setup) → unblocks data collection +2. **Phase 2** (Data Collector) → starts building historical dataset, +enables analysis +3. **Phase 3** (n8n + LLM Meta Generation) → the automation sweet spot — no +more copy-paste +4. **Phase 4** (Streamlit Dashboard) → visualize what's working, find +opportunities +5. **Phase 5** (Competitor Intelligence) → evaluate free vs paid based on +real needs +6. **Phase 6** (Advanced) → cross-platform insights, deeper automation + +--- + +## Relationship to Other PBS Projects +- **YouTube Analytics Pipeline:** Shared Streamlit dashboard, shared Google +Cloud project, parallel development +- **PBS Content Hub (Phase 5):** SEO dashboard could become a Content Hub +tab +- **Instagram Automation:** Cross-platform content performance analysis +(search + social) +- **WordPress-to-MySQL sync:** Trigger SEO meta generation when new recipes +are synced +- **Authelia SSO:** Will protect Streamlit dashboard access +- **Yoast SEO plugin:** Stays installed for technical plumbing (sitemaps, +canonical URLs, Open Graph) — but meta content is now generated and pushed +by the pipeline, not manually entered + +--- + +## Note on Yoast +Yoast stays installed but its role changes. It continues handling: +- XML sitemap generation +- Canonical URL management +- Open Graph / social sharing meta tags +- Basic schema markup (supplementing WPRM's recipe schema) + +What it NO LONGER does: +- You stop manually filling in meta titles/descriptions (the pipeline does +this) +- You ignore the content scoring stoplight (Claude's output is smarter than +Yoast's rules) +- Focus keywords are set by data-driven keyword research, not gut feeling + +Yoast becomes invisible plumbing. The pipeline becomes the brain. + +--- + +*Next Step: Phase 1 — Check if Google Search Console is connected for +plantbasedsoutherner.com* \ No newline at end of file diff --git a/Sources/Dev/pbs-woocommerce-golive.md b/Sources/Dev/pbs-woocommerce-golive.md new file mode 100644 index 0000000..ea9ddae --- /dev/null +++ b/Sources/Dev/pbs-woocommerce-golive.md @@ -0,0 +1,100 @@ +--- +created: 2026-03-22 +path: Sources/Dev +project: pbs-woocommerce-golive +status: completed +tags: +- pbs +- wordpress +- woocommerce +- production +- stripe +- paypal +type: session-notes +updated: 2026-03-22 +--- + +# WooCommerce Go-Live: Payment & Tax Setup + +## Session Summary +Flipped the PBS shop from staging/test mode to fully live production +payments. Configured Stripe and PayPal for live transactions, registered +for Virginia sales tax, set up Stripe Tax for automated tax calculation, +and completed a successful end-to-end test purchase. + +## What Was Completed + +### Stripe - Live Mode +- Disabled test mode in WooCommerce Stripe settings +- Connected live API keys (pk_live / sk_live) from Stripe Dashboard +- Stripe handles Google Pay and Apple Pay natively — no extra config needed + +### PayPal - Live Mode +- Disabled sandbox/test mode +- Connected live PayPal business account +- Disabled Google Pay and Apple Pay in PayPal settings (Stripe handles +those) + +### Payment Responsibility +- Stripe: Credit/debit cards, Google Pay, Apple Pay +- PayPal: PayPal balance and PayPal Credit only +- No duplicate payment method buttons at checkout + +### Virginia Sales Tax Registration +- Registered with Virginia Department of Taxation via iReg +- Virginia Account Number: 10-413414875F-001 +- Filing Frequency: Monthly (ST-1 form) +- Beginning Liability Date: 03/2026 +- First filing due: April 20, 2026 (even if $0 in tax collected) +- Must file every month even with no sales — penalty for missed filings + +### Stripe Tax Configuration +- Stripe Tax plugin handling automated tax calculation +- Virginia registration added to Stripe Dashboard → Tax → Registrations +- Default product tax code: Electronically Supplied Services (1000000000) +- Digital cookbook product code: Electronically Supplied Services +(1000000000) +- Digital products are currently tax-exempt in Virginia (and most US states) +- Dashboard automatic tax enabled for any invoices/payment links created in +Stripe + +### Tax Nexus Notes +- Only registered in Virginia (physical presence = mandatory regardless of +thresholds) +- No need to register in other states currently — digital products are +exempt in most states +- Stripe monitors nexus thresholds and will alert when registration needed +in other states +- Virginia legislature considering HB 900 to tax digital goods — monitor +for changes +- When physical products are added (merch, printed cookbook), revisit +default tax code and nexus obligations + +### Test Transaction +- Completed a real end-to-end purchase of the digital cookbook +- Confirmed: order confirmation email received +- Confirmed: order visible in WooCommerce → Orders +- Confirmed: payment visible in Stripe Dashboard +- Confirmed: digital cookbook download/delivery worked +- First official PBS sale! 🎉 + +## Ongoing Obligations + +- [ ] File VA ST-1 monthly by the 20th (first due April 20, 2026) +- [ ] Keep VA account number and sales tax certificate permanently +- [ ] Keep copies of filed returns for 3 years +- [ ] Monitor Stripe Tax "Needs attention" tab for nexus alerts in other +states +- [ ] Watch for Virginia HB 900 (digital goods tax bill) status +- [ ] When adding physical products: update default tax code to General +Tangible Goods (999999999) + +## Shop Page Updates +- Banner for cookbook promotion: use Container (Row direction) with Heading ++ Text Editor on left, Button widget on right +- Background color: PBS green #3E6B41 +- Check responsive/mobile view after building + +## NAICS Code +- 454110 — Electronic Shopping and Mail-Order Houses (used for VA tax +registration) \ No newline at end of file diff --git a/Sources/Dev/pbs-woocommerce-store.md b/Sources/Dev/pbs-woocommerce-store.md new file mode 100644 index 0000000..66ae192 --- /dev/null +++ b/Sources/Dev/pbs-woocommerce-store.md @@ -0,0 +1,246 @@ +--- +created: 2026-03-20 +path: Sources/Dev +project: pbs-woocommerce-store +status: active +tags: +- pbs +- wordpress +- woocommerce +- ecommerce +- stripe +- automation +- n8n +- staging +- production +type: project-plan +updated: 2026-03-20 +--- + +# WooCommerce Cookbook Store — Project Plan + +## Project Goal +Add a commerce section to plantbasedsoutherner.com to sell Jenny's first cookbook (digital + physical), using a lean WooCommerce setup with free plugins only. This is a **stepping stone** — designed to launch quickly while preserving a clean migration path to a custom-built commerce solution in the future. + +## Key Decisions Made +- **WooCommerce over Shopify** — keeps the Sunnies on-site for a seamless brand experience, no monthly platform fees, full data ownership +- **Lean plugin approach** — WooCommerce core + free official extensions only, no premium plugin dependencies for Phase 1 +- **Stripe as primary payment processor** — free WooCommerce integration, supports credit cards, Apple Pay, Google Pay +- **PayPal as secondary option** — free WooCommerce integration, covers buyers who prefer PayPal +- **Staging first, always** — full deployment and testing on staging before production +- **Future-proof mindset** — all product/order data lives in MySQL, making eventual migration to a custom Flask/Go/Rust backend straightforward + +## Architecture Overview + +``` +Sunnies browse → plantbasedsoutherner.com/shop (WordPress + WooCommerce) + ↓ +Add to cart → still on PBS site + ↓ +Checkout → still on PBS site + ↓ +Payment → Stripe handles card processing behind the scenes + ↓ +Order complete → WooCommerce stores order, triggers n8n webhooks + ↓ +n8n automation → confirmation emails, alerts, digital delivery +``` + +## Long-Term Vision +WooCommerce is the **temporary tenant**. The long-term plan is a custom commerce backend (Flask, Go, or Rust) with direct Stripe API integration. Key principles: +- Product and order data stays in MySQL — portable regardless of frontend +- n8n automations are backend-agnostic (webhook-triggered) +- Traefik routing can point to any service, not WooCommerce-specific +- When ready, swap the engine without rebuilding the infrastructure + +--- + +## Phase 1: WooCommerce Core Setup (Staging) +**Goal:** Get a working cookbook store on staging.plantbasedsoutherner.com + +### 1.1 Install & Configure WooCommerce +- [ ] Install WooCommerce plugin on staging +- [ ] Run the setup wizard (store address, currency, product types) +- [ ] Configure general settings (timezone, currency display, measurements) +- [ ] Set store pages (shop, cart, checkout, my account) +- [ ] Disable WooCommerce features we don't need yet (reviews, coupons, etc.) to keep it lean + +### 1.2 Redis Cache Configuration +- [ ] Configure WooCommerce cart/checkout pages to bypass Redis object cache +- [ ] Add cache exclusion rules for WooCommerce session cookies (`woocommerce_cart_hash`, `woocommerce_items_in_cart`, `wp_woocommerce_session_*`) +- [ ] Test that cart persists across page loads (cache bypass working) +- [ ] Flush Redis cache after WooCommerce installation + +### 1.3 Payment Gateway Setup +- [ ] Create Stripe account for PBS (if not already existing) +- [ ] Install WooCommerce Stripe Gateway (free official plugin) +- [ ] Configure Stripe in **test mode** on staging +- [ ] Install WooCommerce PayPal Payments (free official plugin) +- [ ] Configure PayPal in **sandbox mode** on staging +- [ ] Test both payment methods with test card numbers + +### 1.4 Product Setup — Jenny's Cookbook +- [ ] Create product: Physical cookbook (simple product, requires shipping) +- [ ] Create product: Digital cookbook — PDF (downloadable product) +- [ ] Optional: Create bundled product (physical + digital combo) +- [ ] Upload product images (Jenny to provide) +- [ ] Write product descriptions (Jenny to provide or review) +- [ ] Configure pricing +- [ ] Set up shipping zones and rates for physical book +- [ ] Configure digital download settings (download limits, link expiry) + +### 1.5 Store Design & Integration +- [ ] Create/customize shop page with Elementor to match PBS branding +- [ ] Style cart and checkout pages to match site aesthetic +- [ ] Add "Shop" link to main site navigation +- [ ] Ensure mobile responsiveness (Sunnies likely shopping from Instagram links) +- [ ] Test full purchase flow on mobile and desktop + +--- + +## Phase 2: n8n Order Automation +**Goal:** Automate order processing workflows via n8n + +### 2.1 WooCommerce Webhook Setup +- [ ] Configure WooCommerce webhook for `order.completed` → n8n +- [ ] Configure WooCommerce webhook for `order.created` → n8n (for alerts) +- [ ] Verify webhook payload structure + +### 2.2 n8n Workflow: Order Notifications +- [ ] Build workflow: new order → Google Chat notification to Travis +- [ ] Build workflow: new order → summary notification to Jenny (email or preferred channel) +- [ ] Include order details: product, amount, shipping info (if physical) + +### 2.3 n8n Workflow: Digital Delivery (Optional Enhancement) +- [ ] Evaluate if WooCommerce's built-in digital delivery is sufficient +- [ ] If custom delivery needed: n8n workflow to send personalized delivery email via MailerLite or Google Workspace +- [ ] Track download completions + +### 2.4 n8n Workflow: Order Dashboard Sync (Future) +- [ ] Sync order data to a reporting table in `pbs_automation` MySQL database +- [ ] Enables future custom dashboards without querying WooCommerce directly + +--- + +## Phase 3: Production Deployment +**Goal:** Move tested store to production + +### 3.1 Pre-Production Checklist +- [ ] All staging tests pass (payment, cart, checkout, email, webhooks) +- [ ] Jenny has reviewed and approved product pages +- [ ] Stripe switched from test mode to live mode +- [ ] PayPal switched from sandbox to live mode +- [ ] Shipping rates verified +- [ ] Tax configuration reviewed (may need research for PBS's specific situation) +- [ ] Privacy policy and terms of sale updated on site +- [ ] WooCommerce email templates customized with PBS branding + +### 3.2 Production Deployment +- [ ] Install WooCommerce on production +- [ ] Replicate all staging configuration +- [ ] Apply Redis cache exclusions on production +- [ ] Configure live payment gateways +- [ ] Create products (or export/import from staging) +- [ ] Connect n8n production webhooks +- [ ] Test one real transaction (buy your own cookbook!) + +### 3.3 Launch +- [ ] Add shop to site navigation +- [ ] Jenny announces cookbook launch on Instagram +- [ ] Monitor first orders closely for any issues +- [ ] Verify n8n automations fire correctly + +--- + +## Phase 4: Expand Product Catalog (6+ Months) +**Goal:** Add meal plans, merch, and other digital products as Jenny is ready + +- [ ] Evaluate: WooCommerce extensions vs custom PBS-API endpoints per product type +- [ ] Add new products as needed +- [ ] Consider: Sunnie merch store (t-shirts, kitchen items) +- [ ] Consider: Downloadable meal plans / recipe collections +- [ ] Assess if WooCommerce is still serving us or if it's time to start the custom build + +--- + +## Phase 5: Custom Commerce Platform (Future) +**Goal:** Replace WooCommerce with a custom-built solution + +- [ ] Choose framework (Flask, Go, or Rust based on Travis's preference at that time) +- [ ] Direct Stripe API integration (no plugin middleman) +- [ ] Direct PayPal API integration +- [ ] Migrate product data from WooCommerce MySQL tables +- [ ] Migrate order history +- [ ] Build admin UI for Jenny (product management, order viewing) +- [ ] Retire WooCommerce + +--- + +## Plugin List (Free Only) + +| Plugin | Purpose | Cost | +|---|---|---| +| WooCommerce | Core commerce engine | Free | +| WooCommerce Stripe Gateway | Credit card, Apple Pay, Google Pay | Free | +| WooCommerce PayPal Payments | PayPal checkout option | Free | + +**That's it.** Three free plugins. No premium dependencies for Phase 1. + +--- + +## Infrastructure Considerations + +### Linode Resources +- 2GB should handle initial cookbook sales volume +- Monitor memory usage after WooCommerce install (it adds overhead) +- If needed, upgrade to 4GB — WooCommerce + Redis + n8n + WordPress is a lot for 2GB under load + +### Docker / Traefik +- No changes needed to Traefik config — WooCommerce runs inside the existing WordPress container +- No new containers required for Phase 1 + +### Database +- WooCommerce adds ~20+ tables to the WordPress database +- These are separate from `pbs_automation` — no conflict with existing PBS-API data +- Future migration: WooCommerce tables are well-documented, data extraction is straightforward + +### Security +- Stripe handles PCI compliance (card data never touches our server) +- WooCommerce checkout should be HTTPS-only (already handled by Traefik + Let's Encrypt) +- Consider adding Wordfence WooCommerce-specific rules + +--- + +## Key Risks & Mitigations + +| Risk | Mitigation | +|---|---| +| WooCommerce performance on 2GB Linode | Monitor closely, upgrade if needed | +| Redis cache interfering with cart/checkout | Configure cache exclusions before testing | +| Plugin update breaks something | Test updates on staging first, always | +| Scope creep into paid plugins | Hard rule: free plugins only for Phase 1 | +| Tax complexity | Research early, may need free tax config or manual rates | + +--- + +## Discussion Points Captured +- **WordPress regret is real** — WooCommerce is a pragmatic compromise, not a long-term commitment +- **Custom commerce is realistic** — Flask/Go/Rust + Stripe API is a legitimate future project within Travis's skillset (estimated: a month of weekends) +- **Shop Pay not available on WooCommerce** — not a dealbreaker, Apple Pay + Google Pay + PayPal covers the same need +- **Stripe handles all the scary payment stuff** — PCI compliance, fraud, chargebacks. Our code just says "charge $24.99" and Stripe does the rest +- **Data portability** — all product/order data lives in MySQL and can be migrated when the custom build is ready + +--- + +## Next Steps +1. Deploy WooCommerce on staging +2. Configure Redis cache exclusions +3. Set up Stripe test account +4. Create cookbook products +5. Test full purchase flow + +--- + +*Project: Plant Based Southerner — Commerce* +*Created: March 20, 2026* +*Maintained by: Travis with Claude* \ No newline at end of file diff --git a/Sources/Dev/pbs-youtube-analytics.md b/Sources/Dev/pbs-youtube-analytics.md new file mode 100644 index 0000000..f789655 --- /dev/null +++ b/Sources/Dev/pbs-youtube-analytics.md @@ -0,0 +1,350 @@ +--- +created: 2026-03-23 +path: Sources/Dev +project: pbs-youtube-analytics +status: active +tags: +- pbs +- youtube +- python +- automation +- n8n +- flask +- streamlit +- analytics +type: project-plan +updated: 2026-03-23 +--- + +# PBS YouTube Analytics Pipeline + +## Project Goal +Build a self-hosted YouTube analytics pipeline for the PBS channel that +collects video performance data (with a focus on audience retention), +stores it in SQLite, automates collection via n8n, sends alerts to Google +Chat, and visualizes insights through a Streamlit dashboard. + +## Why This Matters +YouTube Studio's built-in analytics are limited and don't let us slice data +the way we need. By owning the raw data, Travis can do proper analysis in +Python/R, and Jenny gets a clean dashboard showing what's actually working +in our content — especially where viewers drop off or rewatch. + +--- + +## Architecture Overview + +``` +YouTube Analytics API + | + Python Collector Script (PyCharm + UV) + | + SQLite Database (self-contained file) + | + ┌────┴────┐ + │ │ + n8n Streamlit +(schedule (dashboard ++ alerts) via Traefik) +``` + +- **Data Collection:** Python script using `google-api-python-client` + +`google-auth-oauthlib` +- **Storage:** SQLite database file (lightweight, portable, perfect for +read-heavy analytics) +- **Automation:** n8n triggers collection on schedule, sends Google Chat +alerts +- **Visualization:** Streamlit app served as Docker container behind Traefik + +--- + +## Phase 1: Google Cloud + API Setup +**Estimated Time:** 1-2 hours +**Goal:** Get API credentials and verify access to PBS YouTube data + +### Tasks +- [ ] Create Google Cloud project (or use existing PBS project) +- [ ] Enable YouTube Data API v3 +- [ ] Enable YouTube Analytics API v2 +- [ ] Configure OAuth consent screen (Internal if using Workspace, External +otherwise) +- [ ] Create OAuth 2.0 Desktop App credentials +- [ ] Download `client_secret.json` +- [ ] Test OAuth flow — authorize and confirm access to PBS channel data + +### Key Details +- **Required OAuth scope:** ` +https://www.googleapis.com/auth/yt-analytics.readonly` +- **Additional scope for video metadata:** ` +https://www.googleapis.com/auth/youtube.readonly` +- OAuth tokens will be stored securely and refreshed automatically +- First auth requires browser interaction; subsequent runs use refresh token + +### Deliverable +Working OAuth credentials that can query the PBS channel's analytics data + +--- + +## Phase 2: Python Data Collector +**Estimated Time:** 3-4 hours +**Goal:** Python script that pulls video stats and retention data into +SQLite +**Tools:** PyCharm Professional, UV package manager + +### Tasks +- [ ] Initialize project with UV (`uv init pbs-youtube-analytics`) +- [ ] Install dependencies: `google-api-python-client`, +`google-auth-oauthlib`, `google-auth-httplib2` +- [ ] Build OAuth2 auth module with token persistence (refresh token stored +in JSON) +- [ ] Build video list collector (pulls all PBS videos/shorts with metadata) +- [ ] Build retention data collector (audience retention curves per video) +- [ ] Build general metrics collector (views, watch time, likes, traffic +sources, etc.) +- [ ] Design and create SQLite schema +- [ ] Implement data ingestion with upsert logic (idempotent runs) +- [ ] Add CLI interface for manual runs and backfill +- [ ] Test with real PBS channel data + +### SQLite Schema (Initial Design) + +```sql +-- Video metadata from Data API +CREATE TABLE videos ( + video_id TEXT PRIMARY KEY, + title TEXT NOT NULL, + published_at TEXT NOT NULL, + duration_seconds INTEGER, + video_type TEXT CHECK(video_type IN ('video', 'short')), + thumbnail_url TEXT, + description TEXT, + created_at TEXT DEFAULT CURRENT_TIMESTAMP, + updated_at TEXT DEFAULT CURRENT_TIMESTAMP +); + +-- Daily aggregate metrics from Analytics API +CREATE TABLE video_daily_metrics ( + video_id TEXT NOT NULL, + date TEXT NOT NULL, + views INTEGER DEFAULT 0, + estimated_minutes_watched REAL DEFAULT 0, + average_view_duration_seconds REAL DEFAULT 0, + average_view_percentage REAL DEFAULT 0, + likes INTEGER DEFAULT 0, + dislikes INTEGER DEFAULT 0, + comments INTEGER DEFAULT 0, + shares INTEGER DEFAULT 0, + subscribers_gained INTEGER DEFAULT 0, + subscribers_lost INTEGER DEFAULT 0, + impressions INTEGER DEFAULT 0, + impressions_ctr REAL DEFAULT 0, + created_at TEXT DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (video_id, date), + FOREIGN KEY (video_id) REFERENCES videos(video_id) +); + +-- Audience retention curve (100 data points per video) +CREATE TABLE video_retention ( + video_id TEXT NOT NULL, + elapsed_ratio REAL NOT NULL, + audience_watch_ratio REAL NOT NULL, + relative_retention_performance REAL, + fetched_at TEXT DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (video_id, elapsed_ratio), + FOREIGN KEY (video_id) REFERENCES videos(video_id) +); + +-- Traffic source breakdown per video per day +CREATE TABLE video_traffic_sources ( + video_id TEXT NOT NULL, + date TEXT NOT NULL, + traffic_source TEXT NOT NULL, + views INTEGER DEFAULT 0, + estimated_minutes_watched REAL DEFAULT 0, + PRIMARY KEY (video_id, date, traffic_source), + FOREIGN KEY (video_id) REFERENCES videos(video_id) +); + +-- Channel-level daily summary +CREATE TABLE channel_daily_metrics ( + date TEXT PRIMARY KEY, + total_views INTEGER DEFAULT 0, + total_estimated_minutes_watched REAL DEFAULT 0, + subscribers_gained INTEGER DEFAULT 0, + subscribers_lost INTEGER DEFAULT 0, + net_subscribers INTEGER DEFAULT 0, + created_at TEXT DEFAULT CURRENT_TIMESTAMP +); +``` + +### API Details — Retention Data +- **Endpoint:** YouTube Analytics API v2 `reports.query` +- **Dimension:** `elapsedVideoTimeRatio` (100 data points, values 0.01 to +1.0) +- **Metrics available:** + - `audienceWatchRatio` — absolute retention (can exceed 1.0 for rewatched +segments) + - `relativeRetentionPerformance` — compared to similar-length YouTube +videos (0 to 1 scale) + - `startedWatching` — how often viewers started watching at this point + - `stoppedWatching` — how often viewers stopped watching at this point +- **Limitation:** Retention data is per-video only (one video per API call, +no further dimension splits) +- **Note:** For a 60-second Short, each data point ≈ 0.6 seconds. For a +10-minute video, each ≈ 6 seconds. + +### Deliverable +Python CLI tool that pulls all PBS video data + retention curves into a +local SQLite database + +--- + +## Phase 3: n8n Automation + Alerts +**Estimated Time:** 2-3 hours +**Goal:** Automate daily data collection and send performance alerts to +Google Chat + +### Tasks +- [ ] Deploy collector script to Linode server (alongside n8n) +- [ ] Create n8n workflow: daily scheduled trigger → Execute Command node → +runs Python collector +- [ ] Add error handling: notify Google Chat on collection failures +- [ ] Create weekly digest alert: top performing videos, notable retention +patterns +- [ ] Create threshold alerts: video crosses view milestones, unusual +engagement spikes +- [ ] Test scheduled execution end-to-end + +### Alert Ideas +- **Weekly Digest (for Jenny):** Top 5 videos this week by views, best +retention video, shorts vs long-form comparison +- **Spike Alert:** Video gets 2x+ its average daily views +- **Milestone Alert:** Video crosses 1K, 5K, 10K views +- **New Video Check-in:** 48-hour performance report for newly published +content + +### Deliverable +Automated daily collection with Google Chat alerts for notable events + +--- + +## Phase 4: Streamlit Dashboard +**Estimated Time:** 4-6 hours +**Goal:** Interactive web dashboard for Jenny and Travis to explore PBS +YouTube performance + +### Tasks +- [ ] Initialize Streamlit project with UV +- [ ] Build retention heatmap view (the star feature) +- [ ] Build video comparison view (side-by-side retention curves) +- [ ] Build channel overview page (trends over time) +- [ ] Build shorts vs long-form comparison view +- [ ] Build traffic source analysis view +- [ ] Dockerize Streamlit app +- [ ] Add to docker-compose with Traefik labels +- [ ] Deploy to staging first, then production +- [ ] Secure with Authelia (when SSO rollout happens) or basic auth +initially + +### Dashboard Pages (Initial Concept) +1. **Channel Overview** — subscriber trend, total views/watch time over +time, publishing cadence +2. **Video Deep Dive** — select a video, see retention curve, daily +metrics, traffic sources +3. **Retention Heatmap** — all videos on one view, color-coded by retention +quality at each time segment +4. **Shorts Lab** — Shorts-specific view comparing hook effectiveness +(first 3 seconds), rewatch rates +5. **What's Working** — auto-surfaced insights: best retention patterns, +top traffic sources, optimal video length + +### Deployment +- Streamlit container behind Traefik at `analytics.plantbasedsoutherner.com` +(or similar subdomain) +- Reads from same SQLite file populated by the collector +- Protected by basic auth initially, Authelia later + +### Deliverable +Live dashboard accessible to Jenny and Travis showing PBS YouTube +performance with retention analysis + +--- + +## Phase 5: Advanced Analysis & Iteration +**Estimated Time:** Ongoing +**Goal:** Leverage the data for deeper content strategy insights + +### Future Ideas +- [ ] Correlate retention patterns with recipe categories (link to +`pbs_recipes` table) +- [ ] A/B analysis: compare thumbnail styles, intro approaches, video +lengths +- [ ] Optimal posting time analysis using traffic source timing data +- [ ] Export data to R for statistical modeling +- [ ] Instagram vs YouTube cross-platform performance comparison +- [ ] Automated content recommendations based on what's performing + +--- + +## Prerequisites & Dependencies + +| Requirement | Status | Notes | +|---|---|---| +| Google Cloud project | Needed | May already exist for Google Workspace | +| YouTube Analytics API enabled | Needed | Free, quota-based | +| OAuth 2.0 credentials | Needed | Desktop app type | +| Python + UV | Ready | Travis's local dev setup | +| Linode server access | Ready | Same server running n8n | +| n8n operational | Ready | Already running PBS automation | +| Traefik reverse proxy | Ready | For Streamlit subdomain | +| SQLite | Ready | Ships with Python, no setup needed | + +--- + +## API Quotas & Limits +- YouTube Analytics API: 200 queries/day default (can request increase) +- YouTube Data API v3: 10,000 units/day (listing videos costs ~1-3 units +each) +- Retention data: one video per API call (plan batch collection accordingly) +- Data availability: typically 2-3 day delay from YouTube + +--- + +## Key Decisions + +| Decision | Choice | Rationale | +|---|---|---| +| Database | SQLite | Self-contained, portable, perfect for read-heavy +analytics workload. No server process needed. | +| Dashboard | Streamlit | Python-native, fast to build, interactive. Travis +can leverage data analyst skills directly. | +| API approach | YouTube Analytics API (targeted queries) | Real-time, +flexible dimensions/metrics. Better than Reporting API for our scale. | +| Hosting | Linode (same server) | Keeps everything centralized with +existing PBS infrastructure. | + +--- + +## Sequencing & Priority +1. **Phase 1** (API Setup) → unblocks everything +2. **Phase 2** (Python Collector) → gets data flowing, enables ad-hoc +analysis immediately +3. **Phase 3** (n8n Automation) → removes manual collection burden +4. **Phase 4** (Streamlit Dashboard) → gives Jenny self-service access to +insights +5. **Phase 5** (Advanced Analysis) → ongoing value extraction + +--- + +## Relationship to Other PBS Projects +- **PBS Content Hub (Phase 5):** Dashboard could eventually be a tab within +the Content Hub +- **Authelia SSO:** Will protect the Streamlit dashboard once rolled out +- **WordPress-to-MySQL sync:** Could correlate website recipe traffic with +YouTube performance +- **Instagram automation:** Cross-platform analysis potential (YouTube + +Instagram data in one place) + +--- + +*Next Step: Phase 1 — Set up Google Cloud project and enable YouTube APIs* \ No newline at end of file diff --git a/Sources/Dev/pbsii-cicd-pipeline.md b/Sources/Dev/pbsii-cicd-pipeline.md new file mode 100644 index 0000000..483f3a5 --- /dev/null +++ b/Sources/Dev/pbsii-cicd-pipeline.md @@ -0,0 +1,319 @@ +--- +created: 2026-04-18 +path: Sources/Dev +project: pbsii-cicd-pipeline +status: active +tags: +- pbs +- pbsii +- github-actions +- ansible +- cicd +- docker +- deployment +type: project-plan +updated: 2026-04-18 +--- + +# PBSII CI/CD Pipeline (Revised) + +## Goal + +Automate deployment of PBSII layers using GitHub Actions with a self-hosted +runner on `ustest1`. Each layer deploys independently based on which files +changed. Branch strategy controls target environment: `main` → production, +`staging` → staging. Start with the dashboard layer, then extend to +collector and parser. + +## Context + +- PBSII is a private monorepo on GitHub with subdirectories: `collector/`, +`parser/`, `dashboard/`, `n8n/` +- Ansible project (`wp-i`) handles all server config and Docker +deployments, lives on `ustest1` +- Each Docker container has its own folder and `docker-compose.yml` on the +server (e.g., `/opt/docker/dashboard/`) +- Ansible docker role currently deploys all containers in a loop — +migrating to per-container tags +- A standalone pbs-hub deploy role exists as a reference pattern +- Each PBSII layer deploys differently: dashboard is Docker, collector is +systemd, parser is Python/venv +- n8n workflows are version-controlled JSON — no CI/CD needed + +## Architecture + +``` +GitHub Push (to dashboard/ on main) + → GitHub signals self-hosted runner on ustest1 + → ustest1 already has wp-i, SSH keys, Ansible — everything it needs + → Runs Ansible playbook with dashboard tag + → Ansible SSHs to Linode production, deploys dashboard + → Google Chat notification on success/failure + +GitHub Push (to dashboard/ on staging) + → Same flow, but Ansible targets staging server +``` + +## Why Self-Hosted Runner + +- `ustest1` (Proxmox homelab) already has `wp-i`, SSH keys to Linode, and +Ansible installed +- No secrets to copy into GitHub cloud runners +- No cloning the `wp-i` repo onto temporary VMs +- No rebuilding Ansible environment on every run +- Solves the staging drift problem — staging deployments go through the +same Ansible path as production + +## Phase 1: Self-Hosted Runner Setup + +### 1a. Install GitHub Actions runner on ustest1 + +In the PBSII GitHub repo → Settings → Actions → Runners → New self-hosted +runner: + +- [ ] Follow GitHub's instructions to download and configure the runner +agent on `ustest1` +- [ ] Install as a systemd service so it starts on boot +- [ ] Verify runner shows as "Idle" in GitHub repo settings + +### 1b. Runner user permissions + +- [ ] The runner agent runs as a user on `ustest1` — that user needs SSH +access to Linode (production + staging) +- [ ] Ensure that user has access to `wp-i` on `ustest1` +- [ ] Ensure Ansible vault password is accessible (file or environment +variable) + +### 1c. Label the runner + +Add labels to the runner for targeting: + +- [ ] `self-hosted` (default) +- [ ] `ustest1` (custom, for clarity) + +## Phase 2: Ansible Dashboard Role + +### 2a. Create the dashboard Ansible role in wp-i + +Use the existing pbs-hub deploy role as a reference pattern. The role +should: + +- [ ] Copy dashboard source files to server at `/opt/docker/dashboard/` +- [ ] Copy `docker-compose.yml` and `Dockerfile` to the server +- [ ] Run `docker compose up -d --build` in the dashboard folder +- [ ] Healthcheck to verify dashboard is responding + +### 2b. Tag the role + +- [ ] Ensure the role can be called with `--tags dashboard` +- [ ] Verify it only touches the dashboard container, nothing else + +### 2c. Environment targeting + +The playbook needs to accept a variable (e.g., `target_env`) that +determines whether it deploys to production or staging: + +- [ ] `target_env: production` → production Linode IP, production compose +overrides +- [ ] `target_env: staging` → staging Linode IP, staging compose overrides + +This keeps the same role for both environments — only the target changes. + +## Phase 3: GitHub Actions Workflow (Dashboard) + +### 3a. Production workflow + +File: `.github/workflows/deploy-dashboard.yml` + +```yaml +name: Deploy Dashboard + +on: + push: + branches: [main] + paths: + - 'dashboard/**' + +jobs: + deploy: + runs-on: [self-hosted, ustest1] + steps: + - name: Checkout PBSII + uses: actions/checkout@v4 + + - name: Run Ansible playbook + run: | + cd /path/to/wp-i + ansible-playbook playbook.yml \ + --tags dashboard \ + -e target_env=production + + - name: Notify Google Chat (success) + if: success() + run: | + curl -X POST "$GOOGLE_CHAT_WEBHOOK" \ + -H 'Content-Type: application/json' \ + -d '{"text": "✅ Dashboard deployed to production (commit: ${{ +github.sha }})" }' + + - name: Notify Google Chat (failure) + if: failure() + run: | + curl -X POST "$GOOGLE_CHAT_WEBHOOK" \ + -H 'Content-Type: application/json' \ + -d '{"text": "❌ Dashboard deploy FAILED (commit: ${{ github.sha +}})" }' + env: + GOOGLE_CHAT_WEBHOOK: ${{ secrets.GOOGLE_CHAT_WEBHOOK }} +``` + +### 3b. Staging workflow + +File: `.github/workflows/deploy-dashboard-staging.yml` + +Same as above but: +- Trigger: `branches: [staging]` +- Ansible flag: `-e target_env=staging` +- Notification says "staging" instead of "production" + +Alternatively, this could be one workflow file with a branch conditional — +decide during implementation. + +### 3c. Test the workflow + +- [ ] Push a small change to `dashboard/` on `staging` branch first +- [ ] Watch the Actions tab in GitHub +- [ ] Verify dashboard redeploys on staging server +- [ ] Verify Google Chat notification arrives +- [ ] Verify pushing to other directories does NOT trigger this workflow +- [ ] Once staging is solid, test the `main` branch → production flow + +## Phase 4: Extend to Other Layers + +Once the dashboard pipeline is proven, repeat the pattern: + +### 4a. Collector pipeline + +File: `.github/workflows/deploy-collector.yml` + +Trigger: `paths: ['collector/**']` + +Ansible role needs to: +- [ ] Cross-compile Go binary in CI step (or build on server) +- [ ] Copy binary to server +- [ ] Restart the systemd service (`systemctl restart pbs-collector`) +- [ ] Verify collector responds on its HTTP API + +Note: Go cross-compiles easily — building on `ustest1` for Linux/amd64 +target and shipping the binary to Linode avoids needing Go installed on +production. + +### 4b. Parser pipeline + +File: `.github/workflows/deploy-parser.yml` + +Trigger: `paths: ['parser/**']` + +Ansible role needs to: +- [ ] Copy parser source to server +- [ ] Ensure Python venv exists with correct deps (uv sync) +- [ ] No restart needed — n8n triggers it on schedule + +## Phase 5: Google Chat Notifications + +### 5a. Setup + +- [ ] Add `GOOGLE_CHAT_WEBHOOK` to GitHub Secrets (can use same webhook as +existing n8n alerts, or create a dedicated one for deployments) +- [ ] Decide notification format — commit hash, branch, layer name, +success/failure + +### 5b. Notification content + +Success: `✅ [layer] deployed to [env] — commit abc1234` +Failure: `❌ [layer] deploy FAILED on [env] — commit abc1234 — check Actions +tab` + +## Repo Structure After Implementation + +``` +pbsii/ +├── .github/ +│ └── workflows/ +│ ├── deploy-dashboard.yml +│ ├── deploy-dashboard-staging.yml +│ ├── deploy-collector.yml +│ ├── deploy-collector-staging.yml +│ ├── deploy-parser.yml +│ └── deploy-parser-staging.yml +├── collector/ +│ ├── README.md +│ ├── go.mod +│ ├── main.go +│ ├── Makefile +│ └── pbs-collector.service +├── parser/ +│ ├── README.md +│ ├── pyproject.toml +│ └── parser.py +├── dashboard/ +│ ├── README.md +│ ├── pyproject.toml +│ ├── Dockerfile +│ ├── docker-compose.yml +│ └── app.py +├── n8n/ +│ └── workflows/ +│ └── *.json +├── sql/ +│ └── schema.sql +└── README.md +``` + +Ansible roles live in `wp-i`, not in this repo: + +``` +wp-i/ +└── roles/ + ├── dashboard/ + ├── collector/ + └── parser/ +``` + +## GitHub Secrets Required + +- [ ] `GOOGLE_CHAT_WEBHOOK` — webhook URL for deployment notifications +- [ ] `ANSIBLE_VAULT_PASSWORD` — if vault is needed for deploy (may not be +needed if runner user has access to vault password file on `ustest1`) + +Note: SSH keys and Ansible inventory are already on `ustest1` — no need to +store them in GitHub. + +## Open Questions + +- [ ] What is the path to `wp-i` on `ustest1`? Needed for the workflow `cd` +step. +- [ ] Which user does the runner agent run as on `ustest1`? That user needs +SSH + Ansible access. +- [ ] Does `ustest1` currently have SSH keys to both production and staging +Linode? If not, set up during Phase 1. +- [ ] One workflow file per layer per environment (6 files), or one per +layer with branch conditionals (3 files)? Start simple, refactor if it gets +noisy. +- [ ] Google Chat: dedicated deployment webhook or reuse existing? + +## Key Principles + +- Each layer deploys independently — a change to the parser never touches +the dashboard +- Ansible stays the deployment executor — GitHub Actions is just the trigger +- Self-hosted runner on `ustest1` — no secrets in the cloud, no rebuilding +environments +- Branch strategy solves staging drift — `staging` branch → staging server, +`main` → production +- Build on server for v1 — move to CI builds later if needed +- Google Chat notifications on every deploy — success and failure +- Start with dashboard, prove the pattern, then extend +- Test on staging first, always + +...sent from Jenny & Travis \ No newline at end of file diff --git a/Sources/Dev/pre-commit-framework-migration.md b/Sources/Dev/pre-commit-framework-migration.md new file mode 100644 index 0000000..9340e30 --- /dev/null +++ b/Sources/Dev/pre-commit-framework-migration.md @@ -0,0 +1,190 @@ +--- +created: 2026-05-03 +path: Sources/Dev +project: pre-commit-framework-migration +status: active +tags: +- homelab +- tooling +- git +- ansible +- security +type: project-plan +updated: 2026-05-03 +--- + +# pre-commit framework migration + +## Goal + +Replace the raw `.git/hooks/pre-commit` bash script (currently shared via +`core.hooksPath`) with the [pre-commit framework](https://pre-commit.com), +and adopt it as the standard for all new repos. Codify a curated set of +checks that catch the mistakes most likely to bite a solo homelab operator +working across Python, Go, Ansible, and YAML-heavy infra repos. + +## Why + +The current raw hook works but has limits: + +- not version-controlled (lives in `.git/hooks/`, not tracked) +- single-purpose (vault encryption check only) +- adding more checks means growing a bash script +- no portability across machines without re-running the `core.hooksPath` +setup + +The framework solves all four: config lives in `.pre-commit-config.yaml` at +the repo root, gets committed, and gives access to a large ecosystem of +pre-built hooks. + +## Scope + +### In scope + +- Install `pre-commit` framework on dev machines +- Migrate the existing ansible-vault check from raw bash to a framework hook +- Curate a default `.pre-commit-config.yaml` template covering Python, Go, +Ansible, YAML, and secret-detection +- Document the install/onboarding flow as the standard for new repos +- Decide how the framework coexists with (or replaces) the current +`core.hooksPath` setup +- Add the template to project scaffolding so new repos start with it +pre-wired + +### Out of scope + +- Migrating every existing repo today (do them as they're touched) +- Full CI integration (pre-commit can also run in GitHub Actions, but +that's a follow-up) +- Replacing `zero-check` or other post-generation validation skills + +## Decisions to confirm + +- **Coexistence with `core.hooksPath`** — once the framework is in a repo, +its `pre-commit install` will overwrite `.git/hooks/pre-commit` for that +repo. Need to decide: leave `core.hooksPath` as a fallback for repos +without `.pre-commit-config.yaml`, or unset it once all active repos are +migrated? +- **Vault check location** — keep the bash script in-repo at +`scripts/check-vault.sh` (referenced as a `local` hook), or rewrite as a +tiny standalone repo and pin like other framework hooks? +- **Template repo or scaffold-generated** — does the default +`.pre-commit-config.yaml` live in a `homelab-templates` repo for +`web_fetch` access, or get generated by the project scaffolding tool? + +## Curated check list + +Grouped by category. Each is a separate hook entry; opt in per repo by +what's actually relevant. + +### Universal (every repo) + +- `trailing-whitespace` — strip trailing spaces +- `end-of-file-fixer` — ensure files end with a newline +- `check-merge-conflict` — block commits with unresolved conflict markers +- `check-added-large-files` — block accidental large file commits (default +500KB) +- `mixed-line-ending` — enforce LF +- `detect-private-key` — block committing SSH/TLS private keys + +### Secrets & sensitive data + +- `detect-secrets` (Yelp) — broader secret scanning beyond just private +keys; catches AWS keys, API tokens, high-entropy strings +- `gitleaks` — alternative or complement; well-maintained, fast +- **Custom: ansible-vault encryption check** — port of the existing bash +hook + +### YAML / config + +- `check-yaml` — basic YAML syntax validation +- `yamllint` — style + structure (line length, indentation, truthy values) +- `check-json` — JSON syntax +- `check-toml` — TOML syntax + +### Python + +- `ruff` — lint + format in one tool (replaces flake8, isort, black for +most cases) +- `ruff-format` — formatter +- `mypy` — optional, type-checking (heavier; opt in per project) + +### Go + +- `go-fmt` — gofmt enforcement +- `go-vet` — basic static analysis +- `golangci-lint` — broader linting (opt in per project) + +### Ansible-specific + +- **Custom: ansible-vault encryption check** (existing logic, ported) +- `ansible-lint` — full Ansible playbook/role linting +- `yamllint` (already covered above, but Ansible repos lean on it heavily) + +### Shell scripts + +- `shellcheck` — catches common bash bugs and bad patterns +- `shfmt` — formatter for shell scripts + +### Markdown / docs + +- `markdownlint` — style/structure for `.md` files +- (Skip if it gets noisy on Obsidian-flavored markdown — the project notes +use front-matter and Obsidian syntax that vanilla markdownlint may complain +about.) + +## Migration path + +1. Pick one active repo as the pilot — likely the new project where this +conversation started. +2. `pip install pre-commit` (or `uv tool install pre-commit`) on the dev +machine. +3. Drop in a starter `.pre-commit-config.yaml` covering universal + secrets ++ ansible + relevant language checks. +4. Port the vault check as a `local` hook pointing at +`scripts/check-vault.sh`. +5. `pre-commit install` to wire it up. +6. `pre-commit run --all-files` to flush out anything the existing code +violates. +7. Iterate on the config until baseline is clean. +8. Once stable, copy `.pre-commit-config.yaml` to the template location +(TBD — see decisions). +9. Decide on `core.hooksPath` — leave or unset. +10. Add `pre-commit install` to the project scaffolding flow so new repos +get it automatically. + +## Tasks + +- [ ] Confirm decisions: coexistence with `core.hooksPath`, vault check +location, template hosting +- [ ] Install `pre-commit` framework on dev machines +- [ ] Pilot on one active repo +- [ ] Port vault encryption check as a `local` hook +- [ ] Build default `.pre-commit-config.yaml` template covering universal + +secrets + Python + Go + Ansible + YAML +- [ ] Run `pre-commit run --all-files` on pilot, fix or ignore findings +- [ ] Document the install/onboarding flow (README section or standalone +doc) +- [ ] Decide whether to keep or unset `core.hooksPath` +- [ ] Migrate active repos one by one as they're touched +- [ ] Add template to scaffolding tool or `homelab-templates` repo +- [ ] Evaluate whether to add CI run of `pre-commit run --all-files` on PRs +(follow-up) + +## Open questions + +- Does `pre-commit` play well with the `zero-check` skill, or is there +overlap to resolve? +- For Obsidian-flavored markdown, is `markdownlint` worth the noise or skip +entirely? +- Should `mypy` be in the default Python config, or opt-in per project? + +## References + +- pre-commit docs: https://pre-commit.com +- Hook list: https://pre-commit.com/hooks.html +- Existing raw hook: stored at `~/.config/git-hooks/pre-commit` (per +`core.hooksPath` setup) + + +...sent from Jenny & Travis \ No newline at end of file diff --git a/Sources/Dev/production-deploy-march-2026.md b/Sources/Dev/production-deploy-march-2026.md new file mode 100644 index 0000000..eb7bf14 --- /dev/null +++ b/Sources/Dev/production-deploy-march-2026.md @@ -0,0 +1,116 @@ +--- +created: 2026-03-29 +path: Sources/Dev +project: production-deploy-march-2026 +status: completed +tags: +- production +- docker +- traefik +- ansible +- cloudflare +- gitea +- n8n +type: session-notes +updated: 2026-03-29 +--- + +# Production Deploy - March 29, 2026 + +## Overview +Deployed infrastructure updates to production Linode via Ansible, including +new container configurations, database schema updates, and Gitea setup. +Used Cloudflare Worker for maintenance mode during deploy. + +## Accomplishments + +### Cloudflare Worker Maintenance Page +- Created `pbs-maintain` Worker with PBS-branded maintenance page (Sunnie +themed) +- Added IP bypass so Travis can access the site while maintenance page is +active for public +- Worker URL: https://pbs-maintain.tjherbranson.workers.dev +- Toggle on/off by adding/removing Workers Route for ` +plantbasedsoutherner.com/*` +- Worker uses `CF-Connecting-IP` header to check allowed IPs + +### n8n Document Pipeline Fix +- Identified race condition: two emails processed simultaneously caused +Gitea ref lock errors +- Root cause: Gitea Contents API creates a commit on every file +create/update — two simultaneous API calls create competing commits on +`main` +- Fix: Added Loop node before Gitea create/update nodes to serialize file +processing +- Key distinction: Loop node serializes items through the same path; Split +In Batches chunks data — these are different nodes with different behaviors + +### Production Deploy Process +- Cloned live Linode as backup before changes +- Ran Ansible playbook against the clone +- Deployed MySQL schema updates via phpMyAdmin (copy/paste with `IF NOT +EXISTS`) +- Updated DNS in Cloudflare to point to new clone server +- Clone became the new production server; original kept as rollback backup + +### Container Fixes During Deploy +- **pbs-api healthcheck**: Replaced curl-based healthcheck with Python +(`urllib.request`) since curl not installed in container +- **Missing README.md**: pbs-api build failed because `pyproject.toml` +referenced a README.md that didn't exist — created empty file +- **MySQL memory limits**: Added deploy block with 768M limit and tuning +flags to compose +- **WordPress memory limit**: Added 2000M deploy limit +- **Portainer stale container reference**: Restarted Portainer to clear +cached container IDs from pre-deploy + +### Gitea Production Setup +- Added DNS record for `gitea.plantbasedsoutherner.com` in Cloudflare +- Waited for DNS propagation before Traefik could issue Let's Encrypt cert +- Removed healthcheck from Gitea container (healthcheck returns 404 before +setup wizard completes, Traefik won't route to unhealthy containers) +- Completed setup wizard and created admin user +- Set `LANDING_PAGE = login` in app.ini (future task) + +### WordPress Staging Redirect Issue +- After Ansible deploy, site redirected to staging domain +- Root cause: One Traefik router label had staging URL, WordPress picked it +up and wrote it to `wp_options` table +- Fix: Updated `siteurl` and `home` in `wp_options` via phpMyAdmin, flushed +Redis, purged Cloudflare cache +- Lesson: WordPress can auto-update `wp_options` URLs based on incoming +request hostname + +## Key Learnings + +- **Cloudflare Worker IP bypass**: `return fetch(request)` re-fetches +through Cloudflare's network, so WordPress sees a Cloudflare edge IP +instead of your real IP — can trigger Wordfence lockouts +- **Cloudflare caches 301 redirects**: Always purge Cloudflare cache after +fixing redirect issues +- **Gitea API creates implicit commits**: Every file create/update via the +Contents API is a git commit — serialize multiple file operations to avoid +ref lock errors +- **Ansible staging drift**: Fixes applied directly on staging don't make +it back to Ansible automatically — fix on staging to unblock, but +immediately update Ansible too +- **DNS propagation for Let's Encrypt**: Traefik can't issue certs until +DNS propagates — recreate the container (`docker compose up -d +--force-recreate`) to trigger a retry without restarting all of Traefik +- **Wildcard DNS records**: `*.staging` in Cloudflare catches all +subdomains under staging automatically — convenient for staging, avoid on +production +- **`IF NOT EXISTS` MySQL warning**: The "less efficient" warning about +index generation during table creation is negligible for small schemas — +keep the safety of `IF NOT EXISTS` + +## Still To Do +- [ ] n8n and database configuration on production +- [ ] Set Gitea landing page to login in `app.ini` +- [ ] Configure Gitea email settings (deferred) +- [ ] Add Gitea healthcheck back with wget-based check after setup is stable +- [ ] Delete old Linode backup server after stability verification (~1 week) +- [ ] Continue work on per-container Ansible playbook +- [ ] Update Ansible with any fixes applied directly to production during +this session +...sent from Jenny & Travis \ No newline at end of file diff --git a/Sources/Dev/project-scaffolding-skill.md b/Sources/Dev/project-scaffolding-skill.md new file mode 100644 index 0000000..dbc0f13 --- /dev/null +++ b/Sources/Dev/project-scaffolding-skill.md @@ -0,0 +1,127 @@ +--- +created: 2026-04-22 +path: Sources/Dev +project: project-scaffolding-skill +status: active +tags: +- claude-code +- skills +- python +- go +- web +- homelab +type: project-plan +updated: 2026-04-22 +--- + +## Project Scaffolding Skill + +Fork and customize a Claude Code skill that scaffolds new projects with +opinionated defaults across Python, Go, and web stacks. Complements the +existing `zero-check` skill (validation) by handling the other end of the +lifecycle (initialization). + +### Goals + +- One skill that handles project creation for Python, Go, and web work +- Opinionated defaults baked in: uv + Ruff + Pyright for Python, Polars +over pandas for data work, standard Go layout +- ~12-15 project types covering real current and near-term needs — not 70 +- Deploys cleanly to Manjaro tower, ThinkPad, and Ubuntu VM +- Lives in a future `claude-skills` monorepo alongside `zero-check` + +### Approach + +Fork `hmohamed01/Claude-Code-Scaffolding-Skill` rather than build from +scratch. The scaffolding logic, wizard flow, and per-type templates are +already written. Trim to the ~13 needed types and customize defaults to +match stack preferences. + +Before committing to trim-vs-rebuild, read the fork's actual `SKILL.md` and +`scaffold.py` to assess code quality. If clean, trim. If messy, rebuild +using the draft already started (in `/home/claude/project-scaffolding/` +from the planning session) and keep the fork as structural reference. + +### Scope — project types to keep + +Python: FastAPI, Flask, CLI (Typer), Library (PyPI), Data analysis (Polars ++ Jupyter) +Go: Gin API, Chi API, CLI (Cobra), Module (library) +Web: React + Vite, Astro, Hono, T3 Stack + +Plus static HTML/CSS as a lightweight fallback — 13 total. + +### Opinions to apply + +- Python: uv + Ruff + Pyright (not ty, not mypy). Polars default for data, +pandas only when needed for sklearn compat. `src/` layout via `uv init +--package`. Python 3.13 (not 3.14). +- Ruff rules: `E4, E7, E9, F, B, I, UP, N, SIM, RET, PTH` — opinionated but +not ALL. +- Go: standard cmd/internal/pkg layout, Makefile included. +- Web: TypeScript by default, Tailwind for React/Astro. +- Every project: `src/` layout where applicable, git init + first commit, +`CLAUDE.md` referencing zero-check skill, MIT license default. + +### Tasks + +- [ ] Fork `hmohamed01/Claude-Code-Scaffolding-Skill` on GitHub to +`trucktrav/` +- [ ] Clone fork to Manjaro tower +- [ ] Read `project-scaffolding/SKILL.md` and `scripts/scaffold.py` to +assess structure +- [ ] Decide: trim existing vs. rebuild from scratch using drafted +`SKILL.md` in `/home/claude/project-scaffolding/` +- [ ] Delete unused project type references and scaffold.py logic +- [ ] Update `SKILL.md` decision table to list only kept types +- [ ] Swap Python defaults: mypy → Pyright, add Polars as data default, +customize Ruff rule set +- [ ] Update each kept reference file (or equivalent) with opinionated +templates +- [ ] Write `CLAUDE.md` template that references zero-check skill at +`~/zero-check-pipeline/validate/run-all.sh` +- [ ] Verify Go templates match existing Go project conventions +- [ ] Decide `claude-skills` monorepo now or later — if now, create repo +and move both skills in with symlink deploy pattern +- [ ] Deploy to `~/.claude/skills/project-scaffolding/` on Manjaro tower +- [ ] Test with a real new project (Python data analysis candidate: pick +something from the photo app or Immich tooling backlog) +- [ ] Test with the existing Go project pattern to verify Go template +matches +- [ ] Iterate on SKILL.md description for triggering accuracy +- [ ] Deploy to ThinkPad and `us-test-authy` once stable +- [ ] Update `claude-skills` README index with new skill entry + +### Open decisions + +- Monorepo now or wait until 3rd skill? +- Include a `web-static` (HTML/CSS) type, or skip? +- CLAUDE.md absolute path to zero-check — is `~/zero-check-pipeline/` +correct on all three target machines? +- Keep `scaffold.py` (programmatic scaffolding) or convert to +pure-instructional like `zero-check`? + +### References + +- Draft skill in planning session: +`/home/claude/project-scaffolding/SKILL.md` and `references/python-data.md` +- Upstream: `github.com/hmohamed01/Claude-Code-Scaffolding-Skill` +- Structural inspiration: `github.com/a5chin/python-uv` (Python template, +not a skill — for reading) +- Source article: ` +kdnuggets.com/python-project-setup-2026-uv-ruff-ty-polars` +- Existing skill: `zero-check` at `~/.claude/skills/zero-check/` on +`us-test-authy` + +### Next session + +On Manjaro tower with Claude Code: + +1. `gh repo fork hmohamed01/Claude-Code-Scaffolding-Skill --clone` (or fork +via web UI, then clone) +2. `cd Claude-Code-Scaffolding-Skill && cat project-scaffolding/SKILL.md` +3. Decide trim vs rebuild with Claude Code +4. Begin trimming / rebuilding + + +...sent from Jenny & Travis \ No newline at end of file diff --git a/Sources/Dev/recipe-cards-seed-packets.md b/Sources/Dev/recipe-cards-seed-packets.md new file mode 100644 index 0000000..b972d66 --- /dev/null +++ b/Sources/Dev/recipe-cards-seed-packets.md @@ -0,0 +1,129 @@ +--- +created: 2026-04-14 +path: Sources/Dev +project: recipe-cards-seed-packets +status: completed +tags: +- wordpress +- wpcode +- elementor +- css +- php +- branding +- sunnies +type: session-notes +updated: 2026-04-14 +--- + +# Session notes: recipe cards → seed packets + +## Outcome + +Replaced the deprecated `[su_posts]` shortcode (from the uninstalled +Shortcodes Ultimate plugin) with a custom `[pbs_recipes]` shortcode. +Evolved the output through three design iterations: plain list → clean card +grid → fully themed seed packet cards. Added AJAX load more (9 recipes per +batch) across all 7 category sections on the recipes page. Also +decommissioned the old (pre-migration) Linode server with a targeted tar +archive of the Docker project directory. + +## What shipped + +- Custom `[pbs_recipes]` shortcode registered via WPCode Lite PHP snippet +(Auto Insert → Run Everywhere) +- Pulls standard WP posts by category ID — matches the old `su_posts +tax_term="N"` behavior +- `post_status => publish` explicitly set so admins don't see drafts mixed +in +- Shared `pbs_render_recipe_cards()` function used by both the initial +shortcode render and the AJAX load-more handler +- AJAX endpoint via `admin-ajax.php` with nonce verification +- Event-delegated JS (single listener on document) handles load-more across +all 7 category sections at once +- Seed packet visual design: cream/brown double-border frame, Lora serif +body + Caveat script for packet title, "★ SOUTHERN HEIRLOOM ★" stamp, +"Recipe Packet №161" (phi-adjacent math joke), "Harvested [date]" language +- Hover overlay on image with "View Recipe →" CTA +- Full keyboard/focus support via anchor-wrapped cards + +## Key decisions + +| Decision | Rationale | +|---|---| +| Custom shortcode over reinstalling Shortcodes Ultimate | Only one page +used it; avoids adding a whole plugin for one feature | +| `cat` param (standard WP category) instead of WPRM taxonomy | Posts are +categorized in WP categories; WPRM cards are attached to posts, not +directly queried | +| CSS organized into GRID / CARD SHELL / FRAME / CONTENT / LOAD MORE +sections | Future seed-packet-to-new-design swaps only touch FRAME block; +content and grid stay stable | +| Auto Insert → Run Everywhere (not "On Demand") | `add_shortcode()` +registration is microseconds of work; "On Demand" would prevent +registration and break the shortcode | +| Skip WPRM meta (prep/cook time) for now | Not critical for v1; +placeholder commented in PHP for later | +| Drop the per-recipe packet number, hardcode №161 | Simpler than +stable-random or sequential numbering; phi joke works for all cards | +| Delete old Linode clone after targeted tar backup | Linode Images capped +at 6 GB, server disk was 19 GB. Docker project directory at `/opt/docker` +captured via `tar -czf`, scp'd locally. Site has been stable for 2+ weeks +post-migration. | + +## Learnings + +- **WPCode Lite "Auto Insert" vs "Shortcode" insertion modes** — for +snippets that *register* a custom shortcode (not snippets that *output* +content), use Auto Insert. The PHP needs to execute on page load so +`add_shortcode()` runs; it doesn't mean the work runs on every page — the +shortcode callback only fires when the shortcode appears in rendered +content. +- **Linode Images hard cap is 6 GB per image.** Servers with larger disks +cannot use the Images feature for archival regardless of quota. Options for +oversized servers: Linode Backup Service (not downloadable, tied to Linode +lifecycle) or manual tar + scp. +- **Targeted Docker project backup is sufficient for WordPress +migrations.** The entire compose directory (compose file, bind-mounted +volumes, configs) is ~2 GB vs. 19 GB full disk. OS, logs, apt packages all +rebuildable. +- **Elementor HTML widget's CSS injection.** The widget's built-in "Custom +CSS" field is Pro-only, but `` blocks inside the widget's HTML work +fine and keep CSS co-located with the markup it styles. Good trade-off vs. +Appearance → Customize → Additional CSS (easy to lose track of). +- **`tar: Removing leading '/' from member names`** is informational, not +an error — tar strips absolute paths for safer extraction. +- **WP_Query with `post_status` omitted defaults to 'publish' for +non-logged-in users, but includes drafts/private/future for admins.** +Explicitly setting `'post_status' => 'publish'` locks it down regardless of +viewer. + +## Open items + +- [ ] Live card polish: "Recipe Packet №161" and "Harvested [date]" +rendering as underlined links (theme global link styles bleeding into +anchor-wrapped card content); Caveat font not loading for packet title +- [ ] Consider adding WPRM meta (prep time / cook time) to cards when ready +— placeholder comment is in the PHP +- [ ] Seed packet design could extend to the Sunflower Garden member area +(LOE 1 Phase 3) — visual language already aligns +- [ ] Business Wiki email backup from old server (deferred — not flagged as +critical) + +## Tools / files touched + +- WordPress: WPCode Lite snippet (PHP), Elementor HTML widget on the +recipes page +- Shortcode name: `[pbs_recipes cat="N"]` with optional `per_page`, +`order`, `orderby` args +- Default `per_page`: 9 +- Fonts added: Lora (weights 500/600/700), Caveat (weights 600/700) via +Google Fonts +- Colors used: `#FBF6EC` (cream), `#F3E9D2` (packet tan), `#2E4D33` (deep +green), `#7A5A3F` (burnt brown) +- Removed: Shortcodes Ultimate (already uninstalled — confirmed only +`[su_posts]` was in use) +- Decommissioned: old Linode server (pre-migration clone), archived as tar +of `/opt/docker` + + +...sent from Jenny & Travis \ No newline at end of file diff --git a/Sources/Dev/second-brain.md b/Sources/Dev/second-brain.md new file mode 100644 index 0000000..13ec258 --- /dev/null +++ b/Sources/Dev/second-brain.md @@ -0,0 +1,215 @@ +--- +created: 2026-05-04 +path: Sources/Dev +project: second-brain +status: active +tags: +- homelab +- llm +- obsidian +- knowledge-management +- claude-code +type: project-plan +updated: 2026-05-04 +--- + +# Second Brain — Project Plan + +## Concept + +A personal knowledge system combining a video/article extraction pipeline +with an LLM-maintained wiki (Karpathy LLM Wiki pattern). Sources get +pulled, transcribed/extracted, reviewed, and compiled into an +Obsidian-resident wiki that grows and self-maintains over time. + +**Phase 1 success:** working knowledge extractor + working LLM wiki, +end-to-end on one vertical slice, generalizable to all four domains. + +**Phase 2+ (out of scope for now):** the "second brain" layer — static +user-context, active concerns, recent decisions, open questions, wiki-aware +extraction. Stretch goal, fun experiment, not a Phase 1 gate. + +## Architecture (3 layers) + +1. **Wiki** — long-term structured memory. Markdown files in Obsidian +vault, LLM-maintained. Karpathy three-layer pattern: raw sources +(immutable) → wiki (LLM-generated) → schema/config. +2. **Context assembler** — module that builds the prompt bundle for each +extraction. Phase 1 is thin (domain template + focus field + transcript) +but built as a real module from day one so Phase 2 layers slot in cleanly. +3. **Extractor** — stateless single-shot LLM call per source. Pure +function: bundle in, structured extraction out. + +## Pipeline + + +pull → transcribe → extract → review → publish (LLM-compiled to wiki) + + +Existing tool already has pull/transcribe/extract and a Flask web UI with +list/detail/accept. Extending it, not rewriting. + +## Decisions locked + +### Scope +- [x] Phase 1: extractor + wiki, all manual review +- [x] Phase 2 deferred: user-context, dynamic concerns, wiki-aware +extraction + +### Sources & domains +- [x] Sources: web pages, videos +- [x] Domains: `development`, `content`, `business`, `homelab` + +### Scheduler +- [x] Rate-limit smoother, not a batch processor +- [x] Spaces N items across an overnight window (e.g., 10pm–6am) +- [x] Token-aware budgeting preferred over pure time spacing (track tokens +per job) +- [x] Failures retry with backoff, dead-letter after N attempts +- [x] Idempotent — re-running a finished URL is a no-op + +### LLM access +- [x] Claude Code Python SDK (subscription-based, not API) +- [x] Single-shot calls — no agent loop, no tool use, no session continuity +- [x] `--bare` mode equivalent for reproducibility +- [x] Subscription auth on herbydev — needs verification before scheduler +goes live + +### Data model +- [x] Per-job `domain` field (one of four) +- [x] Per-job `focus` field (optional, free text — evaluative framing hint) +- [x] Status states: `pending → pulled → transcribed → analyzed → accepted +→ published` +- [x] Clean slate — no legacy data to migrate + +### Extraction output schema (contract between extractor and compiler) +yaml +source: + url, title, type, duration_or_length, published_at, ingested_at +domain: +focus: +summary: +key_points: [...] +entities: [...] # candidate wiki entities +claims: [...] # discrete claims tied to key_points +open_questions: [...] +contradictions: [...] + + +### Wiki publish step +- [x] Option 3: LLM-compiled (full Karpathy pattern, not templated, not +manual) +- [x] Compiler reads existing wiki, decides what to update/create, writes +changes +- [x] Auto-accept on compiler writes (no diff/dry-run gate in Phase 1) +- [x] Git-as-recovery: commit before each compile run, commit after + - Pre-write commit: `pre-compile: ` + - Post-write commit: `compile: ` with reasoning in body +(entities touched, created, flagged) + - `git log` becomes the audit trail; `git reset` is the rollback + +### Wiki page types +- [x] Default to LLM Wiki spec — include all types (entity, concept, +comparison, synthesis) +- [x] Skinny down later after running it + +### Source provenance +- [x] Default to LLM Wiki spec + +### Compiler decision authority +- [x] Default to LLM Wiki spec + +## Open items + +### Wiki folder structure +- [ ] Review LLM Wiki reference projects before deciding flat vs. nested +- [ ] Lean: may flatten (lots of folders not helpful) +- [ ] Decision needed before compiler is wired to vault + +### Reference implementations to review +- [ ] `Ar9av/obsidian-wiki` — skill-files for any agent, points at Obsidian +vault, provenance tagging +- [ ] `lucasastorian/llmwiki` — local app + MCP, indexes folder, Claude +writes pages +- [ ] `nvk/llm-wiki` — Claude Code plugin, parallel multi-agent research +- [ ] Karpathy original gist — abstract spec + +### Verification before scheduler runs +- [ ] Subscription auth in non-interactive cron context on herbydev + +## Build approach + +**Vertical slice first.** Pick one domain (likely `homelab`) + one source +type (likely YouTube). Build pull → transcribe → extract → review → publish +end-to-end. Generalizing to other domains is mostly config; adding article +ingestion is one new adapter. + +## Handoff to dev + +This plan covers the *what* and *why*. Dev session on herbydev handles the +*where* and *how-to-wire-it-up*: +- Repo layout under `~/dev/` +- Integration with existing extraction tool (lift working pieces, fresh +codebase) +- Obsidian vault path on herbydev +- Scheduler host/process structure (LXC, systemd timer, etc.) +- Existing extractor code review and selective port + +## Phase 1 explicitly out of scope +- Static user-context file +- Active concerns / recent decisions / open questions context layers +- Wiki-aware extraction (extractor doesn't read existing pages) +- Auto-accept on extractions (always lands in `analyzed` for review) +- LLM-maintained context +- Diff/dry-run mode on compiler writes + +## Notes & learning + +- LLM Wiki pattern (Karpathy, April 2026): three layers — immutable +sources, LLM-maintained wiki, schema/config file. Knowledge compiled once, +kept current — not re-derived per query like RAG. +- The "feeling of memory" in LLM systems is harness-driven, not +model-driven. Model is stateless; harness shapes the bundle. Phase 2 +"second brain" work is fundamentally context-assembler design, not +extractor design. +- SDK exercise: Claude Code Python SDK chosen partly for learning value. +Uses ~10% of its surface for Phase 1 (single-shot calls), but that's fine — +it's the right glue. +- Real intellectual work in Phase 2 lives in the context assembler, not the +extractor or the wiki. That's where the second-brain feeling actually comes +from. + +## Tasks + +### Pre-build +- [ ] Review LLM Wiki reference repos (30 min) +- [ ] Decide wiki folder structure (flat vs. nested) +- [ ] Verify subscription auth in cron on herbydev + +### Build (vertical slice) +- [ ] Repo layout + project skeleton (dev session) +- [ ] Port working pieces from existing extractor (dev session) +- [ ] Context assembler module (thin, but real) +- [ ] Extractor with Claude Code Python SDK +- [ ] Per-domain prompt template files +- [ ] Extraction output schema validation +- [ ] Queue/scheduler with token-aware spacing +- [ ] Web UI: domain filter, status filter, queue view, accept/publish +actions +- [ ] Wiki compiler module (reads vault, writes vault, two commits per run) +- [ ] End-to-end test on one homelab YouTube video + +### Generalize +- [ ] Add remaining three domains +- [ ] Add web article adapter +- [ ] Run for a week, observe, tune prompts + +### Phase 1 done when +- [ ] All four domains have working prompt templates +- [ ] Both source types ingest cleanly +- [ ] Scheduler runs unattended overnight without intervention +- [ ] Wiki has 20+ entity pages from real sources, with cross-references +- [ ] Git history shows clean compile commits with usable audit trail + + +...sent from Jenny & Travis \ No newline at end of file diff --git a/Sources/Dev/session-notes-convention.md b/Sources/Dev/session-notes-convention.md new file mode 100644 index 0000000..99557e5 --- /dev/null +++ b/Sources/Dev/session-notes-convention.md @@ -0,0 +1,150 @@ +--- +created: 2026-04-27 +path: Sources/Dev +project: session-notes-convention +status: active +tags: +- pbs +- documentation +- claude-code +- session-notes +- workflow +type: project-plan +updated: 2026-04-27 +--- + +# Session Notes Convention + +A standardized format for documenting coding session changes at the +individual-change level. Provides structured context for debugging, +deployment, agent handoffs, and independent code review. + +## Why + +Coding sessions produce changes that are hard to reconstruct after the +fact. Without a structured log, answering "what changed and why" requires +reading diffs, grepping commit messages, or asking the agent that did the +work. Session notes close that gap by capturing each logical change with +enough detail that a human or agent can understand it without looking at +the code. + +This convention also provides the context layer for future tooling — +specifically an independent code review skill that needs to know what +was intended in order to evaluate what was produced. + +## Consumers + +The same session note serves multiple consumers: + +- **Travis** — "what happened" for debugging and deployment +- **Loom dashboard** — session activity per project, loose thread scanning +- **Code review skill** — intent vs. implementation comparison +- **Next agent** — handoff context for what was done and what's next + +## Format + +Each session note file lives in the project directory (e.g., +`sessions/.md`). The file has a header with format +instructions, then dated session entries. Each entry is a list of changes. + +Each change has: + +- **Label** — short title (the `###` heading) +- **Narrative** — free text describing what was done and why, in plain + English +- **What** — one-line summary +- **Why** — the motivation +- **How** — implementation approach, key functions/patterns +- **Touches** — files modified + +### Example Entry + +markdown +## Session — 2026-04-27 + +### Trello REST Client +Wraps the Trello board API with httpx. Fetches lists and cards, groups +cards by label name (which is the project slug). Includes in-memory cache +with configurable TTL so the dashboard doesn't hit Trello on every page +load. + +- **What:** `TrelloClient` class with card grouping and TTL cache +- **Why:** Dashboard needs live Trello card state joined to vault data +- **How:** httpx.Client with 15s timeout. `_fetch_all()` populates cache + dict with lists and cards. `get_cards_by_project()` groups cards by + label name. `refresh()` invalidates cache. +- **Touches:** data/trello.py (new) + + +## Granularity Rule + +One entry per logical change — a new function, a new route, a new +template, a config addition, or a bug fix. If you'd explain it as a +separate thing in a code review, it's a separate entry. + +Don't roll multiple changes into one entry just because they were part +of the same feature. The test: could someone read this entry and +understand what was changed without looking at the diff? + +## CLAUDE.md Integration + +Each project's CLAUDE.md should include a Session Notes section at the +bottom pointing to the session notes file and restating the granularity +rule: + +markdown +## Session Notes + +After each coding session, append a session entry to +`sessions/.md` in this project folder. +See that file for the format convention and existing entries. + +Granularity: one entry per logical change. A new function, a new route, +a new template, a new config setting, or a bug fix each get their own +entry. Don't combine multiple changes into one entry. If it would be a +separate item in a code review, it's a separate entry here. + + +## Rollout + +### Phase 1 — Convention documented +- [x] Define session note format (label, narrative, what/why/how/touches) +- [x] Define granularity rule +- [x] Validate format against real project (Loom work-index-dashboard) +- [ ] Create session note template file + +### Phase 2 — Apply to existing projects +- [ ] Add session notes section to Loom CLAUDE.md (done by Lovebug) +- [ ] Retrofit session notes for any other active Claude Code projects +- [ ] Add convention to Claude Code skill or global CLAUDE.md so all + new projects pick it up automatically + +### Phase 3 — Tooling integration +- [ ] Loom dashboard reads session notes (already v2 of Loom) +- [ ] Code review skill uses session notes as reviewer context +- [ ] Evaluate whether session notes should auto-generate from git + diff + agent summary + +## Open Questions + +- Should the session notes file live at `sessions/.md` in the + project repo, or in the vault alongside the project plan? +- Should there be a global CLAUDE.md convention (e.g., in `~/.claude/`) + that applies the session notes rule to all projects automatically? +- Phase 3: should session notes be agent-written only, or should there + be a manual entry path too? + +## Success Criteria + +The convention is working when: +- Lovebug produces session notes at the right granularity without + being reminded +- Travis can open a session note and understand what changed without + reading the diff +- The code review skill can use session notes as context and produce + better reviews than a naked diff alone + + +...sent from Jenny & Travis + +...sent from Jenny & Travis \ No newline at end of file diff --git a/Sources/Dev/wiki-vault-migration.md b/Sources/Dev/wiki-vault-migration.md new file mode 100644 index 0000000..c74e344 --- /dev/null +++ b/Sources/Dev/wiki-vault-migration.md @@ -0,0 +1,263 @@ +--- +created: 2026-05-08 +path: Sources/Dev +project: wiki-vault-migration +status: active +tags: +- obsidian +- mcp +- automation +- wiki +type: project-plan +updated: 2026-05-08 +--- + +# Wiki Vault Migration — Unified Notes Repository + +## Goal + +Consolidate pbs-projects and homelab-projects notes into a single +`wiki-vault` repository with a clean two-layer structure (Sources + +Wiki) under four ownership domains. Update the MCP server schema to +match. Migrate existing files without disrupting the old repos. + +## Architecture + +### Two-layer structure + +``` +wiki-vault/ (/opt/projects/wiki-vault/) +├── CLAUDE.md # vault spec (read by compile skill) +├── index.md # front door — links into Sources + Wiki +├── log.md # compile audit trail +│ +├── Sources/ # layer 1: raw input (human + agent authored) +│ ├── index.md # sources catalog — all projects + sessions +│ ├── Dev/ +│ │ ├── index.md # Dev projects + sessions listing +│ │ └── *.md # project plans, session notes — flat +│ ├── Venture/ +│ │ ├── index.md +│ │ └── *.md +│ ├── Homelab/ +│ │ ├── index.md +│ │ └── *.md +│ └── Reference/ +│ ├── index.md +│ └── *.md +│ +└── Wiki/ # layer 2: compiled output (agent generated) + ├── index.md # wiki-wide catalog + ├── Dev/ + │ ├── index.md # Dev entities, topic landings, synthesis + │ └── *.md + ├── Venture/ + │ ├── index.md + │ └── *.md + ├── Homelab/ + │ ├── index.md + │ └── *.md + └── Reference/ + ├── index.md # cross-domain entities + └── *.md +``` + +### Layer responsibilities + +- **Sources** — project plans and session notes. Written by Travis, + Jenny, and LLM agents via the MCP server. Immutable once written + (edits create new versions, originals stay). The MCP `save_note` + tool writes here. +- **Wiki** — entity pages, topic landings, synthesis pages. Generated + and maintained by the wiki-maintenance compile skill. Never written + to directly by the MCP server or by agents outside the compile loop. +- **Spec layer** — `CLAUDE.md`, `index.md`, `log.md` at the root. + `CLAUDE.md` is the operating spec for the compile skill. Root + `index.md` is the navigation front door. + +### Navigation model + +All browsing happens through index pages, not the file tree. + +- **Root `index.md`** — links to Sources index, Wiki index, and + highlights (active projects, recent sessions, key entities) +- **`Sources/index.md`** — all projects grouped by status, recent + sessions by date, cross-domain view +- **`Sources/<Domain>/index.md`** — domain-specific project + session + listing +- **`Wiki/index.md`** — all entities, landings, synthesis pages +- **`Wiki/<Domain>/index.md`** — domain-specific compiled pages + +The compile loop maintains all index pages. Travis navigates via +wikilinks from the front door, never by drilling into folders. + +### Four ownership domains + +| Domain | Scope | +|--------|-------| +| **Dev** | Coding projects, dev environment, AI/ML, DeFi, tooling, language explorations | +| **Venture** | PBS business + content: recipes, video production, brand strategy, membership, marketing, revenue, partnerships — everything Travis and Jenny are building as a business | +| **Homelab** | Infrastructure, networking, hardware, sysadmin, storage, deployments, self-hosting | +| **Reference** | Cross-domain entities: people, systems, tools that span multiple domains (Lovebug, OB1, herbydev, Jenny, Sunnie, Tailscale, etc.) | + +Cross-domain content: prefer the domain where primary ownership lives. +If genuinely cross-cutting and entity-shaped, put in Reference. If +genuinely cross-cutting and project-shaped, pick one domain and tag +with the others. + +## Locked Decisions + +- [x] One repo (`wiki-vault`), four domains, two layers +- [x] Flat within each domain — type is in frontmatter, not folders +- [x] Domain name: Venture (not Content, PBS, Creative, or Forge) +- [x] Old repos (pbs-projects, homelab-projects) stay alive as backup +- [x] Migration copies files, does not move them +- [x] Sources indexes are compile-maintained, same as Wiki indexes +- [x] Navigation through index pages, not file tree browsing +- [x] MCP server updated: schema, folder paths, save_note targets +- [x] Wiki-maintenance skill updated: CLAUDE.md reflects new structure +- [x] Repo location: `/opt/projects/wiki-vault/` + +## MCP Schema Updates + +The `get_schema` tool needs these changes: + +### Domain list +Replace `Content` with `Venture`. Update descriptions. + +### Folder paths +``` +project-plan → Sources/<Domain>/ +session-notes → Sources/<Domain>/ +entity → Wiki/<Domain>/ (compile-only, not MCP-writable) +topic-landing → Wiki/<Domain>/ (compile-only) +synthesis → Wiki/<Domain>/ (compile-only) +``` + +### Section templates (new) +Add required H2 sections per note type so agents produce consistent +body structure: + +**project-plan required sections:** +- `## Goal` — one paragraph, what this project achieves +- `## Locked Decisions` — bulleted, dated as needed +- `## Open Items` — what's unresolved +- `## Phases` or `## Tasks` — work breakdown +- `## Notes` — learnings, context, references + +**session-notes required sections:** +- `## Outcome` — what came out of the session +- `## Topics Covered` — what was discussed +- `## Key Learnings` — what was learned +- `## Follow-ons` — checkbox list of next steps + +### Frontmatter path field +Update from `Tech/Projects` → `Sources/Dev`, `Sources/Venture`, etc. + +## Build Phases + +### Phase 1 — Create the repo and structure + +- [ ] Create `/opt/projects/wiki-vault/` with full folder tree +- [ ] Initialize git repo +- [ ] Write `CLAUDE.md` (updated spec reflecting two-layer structure) +- [ ] Write stub `index.md` at root, Sources, Wiki, and all 8 domain + folders +- [ ] Write initial `log.md` +- [ ] Create Gitea repo, configure remote + deploy key +- [ ] Push initial structure + +### Phase 2 — Migration script + +- [ ] Write Python script that reads both source repos +- [ ] For each markdown file with frontmatter: + - Determine domain (Dev, Venture, Homelab, Reference) from existing + path and tags + - Determine type (project-plan, session-notes) from frontmatter + - Update `path:` frontmatter to new location + - Copy to `Sources/<Domain>/<filename>` +- [ ] Generate a migration report: file count per domain, any files + that couldn't be auto-categorized +- [ ] Review report with Travis before committing +- [ ] Run the copy +- [ ] Commit with message: `migration: copy N files from pbs-projects + and homelab-projects` + +### Phase 3 — Update MCP server + +- [ ] Update `schema.py`: domains, folder paths, section templates +- [ ] Update `git_writer.py`: vault path points to wiki-vault +- [ ] Update `save_note` in `vault.py`: writes to `Sources/<Domain>/` +- [ ] Update `list_projects` and `get_project`: scan `Sources/` tree +- [ ] Update `check_vault_file`: scan new paths +- [ ] Update `.env`: `VAULT_HOST_PATH` points to wiki-vault +- [ ] Rebuild and deploy MCP server container +- [ ] Test: save a note via claude.ai, verify it lands in the right + Sources domain folder + +### Phase 4 — Update wiki-maintenance skill + +- [ ] Update `CLAUDE.md` in wiki-vault with new structure spec +- [ ] Update compile procedure: reads from `Sources/`, writes to `Wiki/` +- [ ] Update lint procedure: checks both layers +- [ ] Update migration runbook (if still relevant) or mark as complete +- [ ] Reinstall skill at `~/.claude/skills/wiki-maintenance/` + +### Phase 5 — Update wiki-context project + +- [ ] Update wiki-context `CLAUDE.md` to reference the two-layer model +- [ ] Align domain names (Content → Venture) +- [ ] Confirm compile skill still loads and reports correctly + +### Phase 6 — Initial compile run + +- [ ] Trigger compile against wiki-vault +- [ ] Generate initial Sources indexes (project + session listings) +- [ ] Generate initial Wiki indexes (empty until entities are promoted) +- [ ] Generate root index.md with navigation links +- [ ] Review output, iterate on index formatting +- [ ] Open wiki-vault in Obsidian, verify navigation flow works + +## File Categorization Rules (for migration script) + +| Current path pattern | Target domain | Rationale | +|---------------------|---------------|-----------| +| `Tech/Projects/*` | Dev | Technical project plans | +| `Tech/Sessions/*` | Dev | Technical session notes | +| `Tech/Reference/*` | Reference | Cross-domain entities | +| `PBS/Content/*` | Venture | PBS content planning | +| `PBS/Tech/Projects/*` | Dev (tag: pbs) | PBS tech work is Dev work | +| `PBS/Tech/Sessions/*` | Dev (tag: pbs) | PBS tech sessions | +| `PBS/Inbox/*` | Dev or Venture | Manual review needed | +| `Business/*` | Venture | Business planning | +| `PBS-Planning/*` | Venture | PBS planning | +| `settings/*` | Skip | Config files, not notes | + +Files without frontmatter: skip and list in the migration report. +Files with ambiguous domain: list for manual review. + +## Open Items + +- [ ] Decide whether `Sources/Reference/` is needed — Reference might + only contain Wiki-layer entity pages, not raw source notes. If so, + drop it from Sources. +- [ ] Confirm the n8n email pipeline can be updated to write to the + new paths (or if it stays writing to the old repos for now) +- [ ] Decide whether to update the Dispatch session plan templates to + reference wiki-vault instead of pbs-projects + +## Success Criteria + +- [ ] All project plans and session notes from both old repos exist in + wiki-vault under the correct domain +- [ ] MCP `save_note` writes to `Sources/<Domain>/` and commits +- [ ] MCP `get_schema` returns updated domains, paths, and section + templates +- [ ] MCP `list_projects` finds projects in the new structure +- [ ] Root `index.md` in Obsidian provides working navigation to all + content via wikilinks +- [ ] Sources and Wiki indexes are populated after first compile +- [ ] Old repos remain untouched and accessible as backup +- [ ] An agent posting a new project plan via MCP produces a file with + correct frontmatter, correct domain folder, and all required H2 + sections \ No newline at end of file diff --git a/Sources/Dev/wordpress-log-monitor.md b/Sources/Dev/wordpress-log-monitor.md new file mode 100644 index 0000000..53de59a --- /dev/null +++ b/Sources/Dev/wordpress-log-monitor.md @@ -0,0 +1,128 @@ +--- +created: 2026-04-07 +path: Sources/Dev +project: wordpress-log-monitor +status: active +tags: +- security +- docker +- python +- mysql +- streamlit +- n8n +- monitoring +type: Tech/Projects +updated: 2026-04-07 +--- + +# WordPress Log Monitor + +## Goal + +Build a security monitoring pipeline that parses WordPress Docker +logs, stores metrics in MySQL, and visualizes trends in Streamlit. +Replaces manual log grepping with an automated dashboard and Google +Chat alerts. + +## Problem Being Solved + +- 11,000+ brute force hits on wp-login.php went undetected for days +- No visibility into what traffic actually reaches the server vs what +Cloudflare blocks +- Memory crashes caused by Apache worker exhaustion were mysterious +without log data +- Manual grepping is not sustainable as site grows + +## Architecture + +Docker WordPress Logs → Python Parser → MySQL → n8n Orchestration → +Streamlit Dashboard + ↓ + Google Chat Alerts + +## Data To Capture + +### wp-login.php metrics +- Timestamp, IP address, HTTP method (GET vs POST), response code +- User agent string +- Hourly/daily hit counts +- POST attempts (actual login attempts) vs GET (page loads) + +### General security metrics +- 404 hit counts by IP (bot scanning indicator) +- Response code distribution +- Top IPs by request volume +- Slowest requests (response time > 5 seconds) + +### Apache health metrics +- Active worker count over time (correlate with memory spikes) +- Request rate per minute +- Memory usage correlation + +## Implementation Plan + +### Phase 1: Python Log Parser +- Use UV + venv, managed in PyCharm +- Parse WordPress Docker logs via docker logs command or log file mount +- Extract: timestamp, ip, method, uri, status, response_size, +response_time, user_agent +- Store raw events in MySQL pbs_security_events table +- Run on schedule via n8n (every 5-15 minutes) + +### Phase 2: MySQL Schema +- pbs_security_events: raw parsed log entries +- pbs_security_summary: hourly aggregates for dashboard performance +- pbs_blocked_ips: known bad actors for reference + +### Phase 3: n8n Orchestration +- Schedule Python collector every 15 minutes +- Alert workflow: if wp-login.php hits > threshold in last hour → +Google Chat alert +- Daily summary workflow → Google Chat digest with key metrics + +### Phase 4: Streamlit Dashboard +- wp-login.php hits over time (line chart) +- Top attacking IPs (table) +- Response code distribution (bar chart) +- Apache worker count vs memory usage (correlation chart) +- Cloudflare vs server hit comparison (manual input or CF API) + +## MySQL Schema (Draft) + +CREATE TABLE pbs_security_events ( + id BIGINT UNSIGNED AUTO_INCREMENT PRIMARY KEY, + logged_at DATETIME NOT NULL, + ip_address VARCHAR(45) NOT NULL, + method VARCHAR(10), + uri TEXT, + status_code SMALLINT, + response_size INT, + response_time_ms INT, + user_agent TEXT, + event_type VARCHAR(50), + created_at DATETIME DEFAULT CURRENT_TIMESTAMP, + INDEX idx_logged_at (logged_at), + INDEX idx_ip (ip_address), + INDEX idx_uri (uri(100)) +); + +## Alert Thresholds (Starting Points) + +- wp-login.php hits > 20 in any hour → Google Chat alert +- Any single IP > 50 requests in 10 minutes → Google Chat alert +- WordPress response time > 10 seconds → Google Chat alert +- Memory usage > 1.5GiB sustained 10 minutes → Google Chat alert + +## Future Expansion + +- Pull Cloudflare Security Events via API for comparison +- Correlate Apache worker count with memory stats +- Auto-block IPs in Wordfence via API when threshold exceeded +- Feed data into broader PBS Content Intelligence Platform + +## Notes + +- Use docker logs --since flag to avoid reprocessing old entries +- Store last processed timestamp in MySQL to handle restarts cleanly +- Keep collector lightweight — this runs every 15 minutes +- PyCharm Professional for development, UV for package management \ No newline at end of file diff --git a/Sources/Dev/work-index-dashboard.md b/Sources/Dev/work-index-dashboard.md new file mode 100644 index 0000000..4b30c76 --- /dev/null +++ b/Sources/Dev/work-index-dashboard.md @@ -0,0 +1,153 @@ +--- +created: 2026-04-27 +path: Sources/Dev +project: work-index-dashboard +status: active +tags: +- pbs +- homelab +- dashboard +- mcp +- trello +- obsidian +type: project-plan +updated: 2026-04-27 +--- + +# Work-Index Dashboard + +A queryable index over the vault, Trello, and (later) Claude Code session +logs. Static HTML dashboard for daily use, MCP read/write API for agents to +use the same data layer. + +## Why + +The vault is already unwieldy at ~20 files. Frontmatter is invisible in the +default file list, sorting and filtering aren't possible, and nothing +connects the vault to live project state in Trello or Claude Code sessions. +The result: "what did I commit to next" can't be answered without opening +multiple tools and reading individual files. + +This project closes that gap with a single dashboard surface, joined on +`project` slug across all three sources, plus an MCP server so agents can +read and write through the same layer. + +## Primary use case + +**"What did I commit to next."** Open the dashboard at the start of a work +session, see the current project list with active commitments, pick +something up. Commitment tracker, not activity feed. + +Secondary use cases (v2): "I'm lost, what was the last decision" and "what +loose threads have I left in session notes that never made it to Trello." + +## Architecture + +Three-source join on project slug: + +- **Vault** — frontmatter (`project`, `status`, `type`, `tags`, `created`, +`updated`, `path`) read directly from `.md` files +- **Trello** — single board, one label per project slug, cards as +commitments +- **Session notes** (v2) — markdown files under each project, written by +Claude Code sessions + +The data layer is plain Python. The dashboard renders it as HTML. The MCP +server exposes it as tools. Same functions, three surfaces. + +## Stack + +- Python managed via `uv` +- FastAPI (long-running process; MCP server eventually shares it) +- Jinja2 templates +- HTMX for interactions without a JS framework +- Tailwind for styling +- `py-trello` for Trello API (swap to `httpx` direct calls if it gets in +the way) +- `.env` for credentials +- In-memory cache for Trello responses (~60s TTL) so refreshes feel instant + +## Phasing + +### v1 — dashboard + +- Data layer: read vault frontmatter, read Trello cards by label +- Dashboard view: current project list with status, last vault update, open +Trello cards, links out to vault folder and Trello board +- On-demand refresh (no cron, no event-driven) +- Develop on bare metal on herbys-dev +- Deploy to Docker behind Traefik for the v1 cutover + +### v2 — session notes integration + +- Extend data layer to read session notes from each project +- Add "recent sessions" column to the project list +- Add "loose threads" view: `- [ ]` checkboxes in session notes that have +no matching Trello card +- Add search across session notes for the "I'm lost" case + +### v3 — MCP server + +- Wrap the same data layer as MCP tools +- Read tools: `get_projects`, `get_project_sessions`, `get_trello_cards`, +`search_sessions` +- Write tools: `log_session`, `update_project_status`, `create_vault_note`, +`update_card_status`, `create_card` +- stdio transport, local Claude Code only +- Closes the loop: sessions write structured logs and Trello updates +through MCP, dashboard reads the new state on next refresh + +## Out of scope (v1) + +- Graph visualization (revisit once queries are validated) +- Live updates (websockets, file watchers) +- Auth (homelab-internal, behind Traefik) +- Write-back from the dashboard UI + +## Tasks + +### v1 +- [ ] Scaffold FastAPI project with uv +- [ ] Implement vault frontmatter reader +- [ ] Implement Trello client wrapper using py-trello, label-based filtering +- [ ] Build join layer keyed on project slug +- [ ] Build project list view with Jinja + Tailwind +- [ ] Add HTMX-based refresh interaction +- [ ] Add in-memory cache with TTL for Trello calls +- [ ] Write Dockerfile +- [ ] Add Traefik labels and deploy to herbys-dev + +### v2 +- [ ] Extend data layer with session note reader +- [ ] Add recent-sessions column to project list +- [ ] Build loose-threads view (unfiled `- [ ]` items) +- [ ] Add session search view + +### v3 +- [ ] Scaffold MCP server alongside FastAPI app +- [ ] Implement read tools wrapping the data layer +- [ ] Implement write tools (vault, sessions, Trello) +- [ ] Configure Claude Code to use the MCP server +- [ ] Update CLAUDE.md conventions to call `log_session` at end of sessions + +## Open questions + +- Trello label naming — exact slug match, or prefixed (e.g. +`proj:work-index-dashboard`)? +- Where session notes live in each project repo — +`docs/sessions/YYYY-MM-DD.md` vs. another convention +- Dashboard hostname once Traefik routing is set up + +## Success criteria + +v1 ships when: +- Dashboard loads in under a second on herbys-dev +- It correctly shows every active project with its Trello commitments +- It's used at the start of a work session at least three times in the +first week + +If it isn't opened, it isn't earning its keep, and the design is wrong +before more layers get added on top. + + +...sent from Jenny & Travis \ No newline at end of file diff --git a/Sources/Dev/zero-check-pipeline.md b/Sources/Dev/zero-check-pipeline.md new file mode 100644 index 0000000..c6aa516 --- /dev/null +++ b/Sources/Dev/zero-check-pipeline.md @@ -0,0 +1,137 @@ +--- +created: 2026-04-18 +path: Sources/Dev +project: zero-check-pipeline +status: active +tags: +- homelab +- devops +- automation +- claude-code +type: project-plan +updated: 2026-04-18 +--- + +## Zero-Check Validation Pipeline + +A reusable, language-aware validation pipeline for post-LLM code +generation. Treats all generated code as untrusted and requires it to prove +safety before promotion. + +### Core Principles + +- LLM-agnostic — works on code from Claude, Gemini, or any generator +- Language-aware — detects Python and Go, runs appropriate tool chain +- Read-only checks — the gauntlet scans and reports, never modifies +- Scripts are locked — the LLM can fix code but cannot weaken the checks +- Phased commits — commit at phase boundaries for rollback granularity +- Git is the notebook, the gauntlet is the quality gate + +### Pipeline Flow + +**Phase 1 — Code Generation** +- LLM generates code on Ubuntu server (us-test-authy) +- Commit at phase boundaries as work progresses + +**Phase 2 — Gauntlet Loop (automated, max 3 iterations)** +- Secret detection (gitleaks) +- Linting (ruff for Python, golangci-lint for Go) +- SAST (semgrep with OWASP + language rulesets) +- Dependency audit (pip-audit for Python, govulncheck for Go) +- Unit tests (pytest for Python, go test for Go) +- Output is structured JSON so the LLM can parse failures and fix +- Each iteration gets a commit for traceability +- If not green after 3 attempts, escalate to human + +**Phase 3 — Container Verification (one-shot, not in the loop)** +- Build from Dockerfile in clean environment +- Run full test suite inside container +- Docker Compose for Tier 2 integration tests (databases, Redis, etc.) +- Failure here gets kicked back to LLM as a specific task + +**Phase 4 — Architect Sweep (human, ~5 minutes)** +- Review git diff from last clean baseline +- Check for outbound calls (requests, httpx, urllib) +- Check for auth bypasses (try/except: pass, commented-out decorators) +- Check for obfuscation (unexplained Base64/Hex strings) +- Approve or reject + +### Testing Tiers + +**Tier 1 — Self-contained (automated loop handles this)** +- Linting, SAST, secrets, dependency audit, unit tests with mocks +- No network, no services, no auth required + +**Tier 2 — Local services (container verification handles this)** +- App + database/Redis/queue via Docker Compose +- Isolated network, disposable, no real credentials + +**Tier 3 — External services (manual, deferred)** +- Real OAuth, external APIs, Cloudflare integration +- Mock boundaries in automated tests, manual smoke test during architect +sweep +- Dedicated test environment is a future evolution + +### Validation Artifacts + +**validate/results.json** — Machine-readable pipeline state. Timestamp, +pass/fail per check, iteration count, current status. Any new Claude +session reads this to resume context. + +**validate/SUMMARY.md** — Human-readable validation summary. Lives in repo, +visible in Gitea. Browsable from any device. + +Both committed and pushed with the project code. + +### Project Structure + +``` +validate/ + secrets.sh # runs gitleaks + lint.sh # runs ruff or golangci-lint + sast.sh # runs semgrep + deps.sh # runs pip-audit or govulncheck + tests.sh # runs pytest or go test + run-all.sh # calls each in order, stops on failure + results.json # machine-readable output (generated) + SUMMARY.md # human-readable output (generated) +``` + +Each script is 10-20 lines, single purpose, human-readable. + +### Orchestration + +- Claude Code is the meta-controller (the brain) +- Skills define the playbook (what to run, in what order, retry rules) +- Scripts are the hands (each runs one tool, returns structured output) +- No separate app needed — Claude Code + skills + scripts covers it + +### Build Order + +- [ ] Individual validation scripts (secrets, lint, sast, deps, tests) +- [ ] run-all.sh orchestrator with language detection +- [ ] Structured JSON output (results.json) +- [ ] Human-readable summary generation (SUMMARY.md) +- [ ] Claude Code skill for automated gauntlet loop with retry logic +- [ ] Dockerfile template for container verification +- [ ] Architect sweep checklist document +- [ ] Docker Compose template for Tier 2 integration tests + +### Evolution Path + +- **Current (v1):** Shell scripts + Claude Code skill, run locally +- **v2:** Taskfile (YAML-based task runner) if orchestration logic gets +complex +- **v3:** GitHub Actions as safety net on push +- **Future:** Dashboard app for cross-project validation history (dog-food +candidate for this pipeline) + +### Deferred + +- GitHub Actions integration +- Tier 3 external integration test environment +- Reporting/trend dashboards +- act (local GitHub Actions simulation) +- Dashboard app + +...sent from Jenny & Travis \ No newline at end of file diff --git a/Sources/Homelab/2026-05-06-homelab-mcp-server-stand-up.md b/Sources/Homelab/2026-05-06-homelab-mcp-server-stand-up.md new file mode 100644 index 0000000..82d3d88 --- /dev/null +++ b/Sources/Homelab/2026-05-06-homelab-mcp-server-stand-up.md @@ -0,0 +1,69 @@ +--- +created: '2026-05-06' +path: Sources/Homelab +project: homelab-mcp-server +status: active +tags: +- mcp +- authentik +- cloudflare +- obsidian +- n8n +type: session-notes +updated: '2026-05-06' +--- + +# Homelab MCP Server Stand-Up + +First successful end-to-end deployment of the homelab MCP server. This very note is the inaugural write through the new pipeline — vault saved via `save_vault_note` instead of email-to-n8n. + +## Outcome + +Working remote MCP connector reachable from Claude (web and mobile), authenticated via Authentik OAuth, fronted by Cloudflare Tunnel. Four Phase 1 tools live: `get_vault_schema`, `list_vault_projects`, `get_vault_project`, `save_vault_note`. + +## What got built + +Lovebug had the stack scaffolded before the session started — Python MCP server (FastMCP, Streamable HTTP), Authentik docker stack, cloudflared placeholder in compose, schema/git-writer modules. This session was the configuration and wiring pass. + +- Authentik admin account + OAuth2 provider + application created via the wizard +- Cloudflare tunnel `homelab-mcp` with two public hostnames: `mcp.herbylab.dev` → `homelab-mcp:8080`, `auth.herbylab.dev` → `authentik-server:9000` +- All containers on the auto-created default Docker network; cloudflared reaches services by container name, not localhost +- `.env` populated with public Authentik URLs for issuer/JWKS +- Connector registered on claude.ai with OAuth client ID/secret from Authentik + +## Issues hit and resolved + +- **Wrong tab in Cloudflare ZT dashboard.** Started in "Private Network" (which is the WARP-only feature) instead of "Public Hostname" / "Published applications." Cleared up by going into the tunnel detail page directly and using the right tab. +- **502 from initial curl.** Cloudflare tunnel URLs were set to `localhost:8080` and `localhost:9000`, but cloudflared runs as its own container with no host networking. Fix: change tunnel URLs to use container names (`homelab-mcp:8080`, `authentik-server:9000`) since all services share the default compose network. +- **Authentik external URL "problem" was a non-problem.** Authentik 2025.4 derives its issuer URL from the request Host header automatically. No brand setting change needed; the discovery URL through `auth.herbylab.dev` returned the correct public issuer immediately. +- **Missing OAuth Protected Resource Metadata.** Claude expects `/.well-known/oauth-protected-resource` per RFC 9728. Initial server only served `/.well-known/oauth-authorization-server`. Lovebug added the protected-resource document. +- **Invalid Host header rejection.** FastMCP's transport security check defaulted to localhost-only and was rejecting `mcp.herbylab.dev` with 421. Lovebug added `mcp.herbylab.dev` to the allowed hosts list. +- **`list_vault_projects` errors on directories ending in `.md`.** First test surfaced a bug — tool assumes every `*.md` match is a file and `os.open()`s it. Real vault has at least one folder named `pbs-membership-loe1-phase3.md`. Needs `os.path.isfile` filter. Open for Lovebug. + +## Architecture decisions worth recording + +- **Streamable HTTP transport, not SSE.** SSE deprecation is in motion; built on the durable transport. +- **Single MCP server, organized by domain modules internally.** Multiple servers were considered for trust scoping (vault read/write vs server-side execution). Decided trust separation lives between MCP and Dispatch instead — MCP for knowledge/state, Dispatch for execution. Code structured by domain so future split is mechanical, not a rewrite. +- **n8n stops being the front door but stays the integration hub.** Server writes vault directly + commits + pushes; n8n becomes downstream consumer for fan-out (tasks, Drive mirroring) when those tools come online. +- **Read-only-from-Travis enforcement by absence.** No edit/delete tools exist. Eliminates the concurrent-write problem class entirely. Vault repo on dev server, tower opens a local clone that pulls. +- **TLS terminates at Cloudflare's edge.** Internal cloudflared → service traffic stays HTTP since both are on the same host network. Standard tunnel pattern; no internal TLS plumbing needed. + +## Schema additions Lovebug shipped beyond the spec + +- `domains` taxonomy (Dev / Content / Homelab / Reference) layered above folder paths +- `knowledge_sources` populated for all three (vault, openbrain, llm_wiki) — placeholder design held up +- `frontmatter_templates` with required-field declarations and concrete examples per note type + +## What's next + +- Save the project plan as a separate `project-plan` note (use the draft from the design session) +- Fix `list_vault_projects` directory bug +- Retire the email-to-n8n trigger once a few real notes have flowed through +- Phase 2 catalog: search tools, generic task tools (n8n routes), cross-system glue, memory/OpenBrain integration when workflow patterns establish + +## Key learnings + +- Most "missing UI option" Authentik problems are version drift; check the discovery URL through the public domain *first* before hunting settings. +- For tunnels: `cloudflared` container's `localhost` is the container, not the host. Use container names when all services share a compose network. +- OAuth metadata for MCP requires *both* RFC 8414 (Authorization Server) *and* RFC 9728 (Protected Resource). Many implementations only ship the first. +- "Failed" tool calls are still a successful end-to-end test — every layer below the failure had to work to surface the bug. \ No newline at end of file diff --git a/Sources/Homelab/2026-05-08-homelab-vlan-renumber-switch-chip-to-bridge-alignment.md b/Sources/Homelab/2026-05-08-homelab-vlan-renumber-switch-chip-to-bridge-alignment.md new file mode 100644 index 0000000..96a4b7e --- /dev/null +++ b/Sources/Homelab/2026-05-08-homelab-vlan-renumber-switch-chip-to-bridge-alignment.md @@ -0,0 +1,106 @@ +--- +created: '2026-05-08' +path: Sources/Homelab +project: homelab-vlan-renumber +status: completed +tags: +- proxmox +- automation +type: session-notes +updated: '2026-05-08' +--- + +# Homelab VLAN Renumber — Switch Chip to Bridge Alignment + +## Outcome + +Renumbered OpenWrt's hardware switch chip VLANs to match the bridge VLAN filtering layer. Lab and private VLANs now use the same VID end-to-end (11 and 21) from the wire through the chip, sub-interface, and bridge. Unblocked Phase 1.3 — Knot LXC successfully reaches `10.0.11.1` over a tagged trunk from PVE. + +## The Problem + +OpenWrt was running two layers of VLAN configuration with different VID numbering: + +- **Switch chip (swconfig):** VID 1, 2, 3 (lab), 4 (private) +- **Bridge VLAN filtering:** VID 11, 21, 31, 61, 71 + +Bridge VLAN filtering rules quietly translated between them via `:u*` (untagged + PVID) on the sub-interface ports. NAS frames came up tagged VID 3 from the chip, hit `eth1.3` configured as `:u*` on bridge VID 11 — bridge applied PVID 11 to ingressing untagged frames. PVE same pattern: VID 4 → `eth1.4:u*` → bridge PVID 21. + +The translation worked, but it meant configuring PVE to send `tag=11` for "VLAN 11 lab" actually required sending `tag=3` (the chip's wire VID), which is the kind of hidden mismatch that loses an hour of debugging six months later. + +## The Architecture Change + +**Switch chip — added entries with bridge-matching VIDs:** + +``` +VLAN 11: ports='3 4t 6t' (NAS untagged on 3, PVE tagged on 4, CPU tagged on 6) +VLAN 21: ports='4 6t' (PVE untagged on 4, CPU tagged on 6) +``` + +**Linux — created explicit 8021q sub-interfaces:** + +``` +config device + option type '8021q' + option ifname 'eth1' + option vid '11' + option name 'eth1.11' + +config device + option type '8021q' + option ifname 'eth1' + option vid '21' + option name 'eth1.21' +``` + +These were **created explicitly** rather than relying on the legacy implicit `eth1.<N>` auto-creation. The previous `eth1.3` and `eth1.4` ports never had explicit device entries — Linux auto-spawned them when referenced in the bridge ports list. The explicit form is the modern OpenWrt-recommended pattern and makes the config self-documenting. + +**Bridge — `homebridge` ports updated:** + +``` +ports='eth1.3' → ports='eth1.21' 'eth1.11' +``` + +(`eth1.3` retained for ports 1/2 still on legacy VID 3 — see Open Items.) + +**Bridge VLAN filtering — sub-interfaces wired to matching VIDs:** + +``` +VID 11: 'eth1.11' tagged member, 'eth1.3:u*' (NAS legacy translation, retained) +VID 21: 'eth1.21:u*' (PVE untagged + PVID) +``` + +## How the Migration Stayed Live + +Parallel-build pattern. Built the new VID 21 plumbing alongside the existing VID 4 path before tearing anything down. PVE management kept flowing on the legacy path until the new path was verified, then `eth1.4` was removed in one cutover. Same approach for VID 11. PVE never dropped, NAS never dropped. + +## Hybrid Trunk on PVE Port + +Port 4 ended as a hybrid trunk: VID 21 untagged (PVE management default) + VID 11 tagged (LXC traffic). This required the LXC's `tag=11` parameter to match the wire VID for the first time — previously this would have needed `tag=3` due to the translation layer. + +## Key Learnings + +- **swconfig VIDs are real wire VIDs.** Earlier in the session I (Claude) misframed them as "internal slots" — they're actual 802.1Q tags on frames between the switch silicon and the CPU. The number you see in `swconfig` is the number on the wire. +- **Tagged port ≠ port adds tag.** Tagged means "expect frames to already carry an 802.1Q tag and preserve it." The sending device adds the tag. PVID on an untagged port is what tags ingressing untagged frames. +- **VLANs only do anything when something downstream enforces them.** A VID on a frame is just a number unless a switch or bridge filters on it. The original setup "worked" because the bridge wasn't VLAN-aware yet — turning on `bridge-vlan-aware yes` is what surfaced the latent chip/bridge mismatch. +- **Two VID namespaces stacked on top of each other is a footgun.** The translation pattern works but creates cognitive overhead every time you debug. Aligning the layers eliminates a class of "which number do I use here?" questions forever. +- **Explicit `8021q` device entries beat implicit sub-interface auto-spawn.** Self-documenting, idiomatic for current OpenWrt, easier to grep for in the config. The implicit form is legacy and hides the dependency on bridge `ports` list references. +- **`uci add_list` is non-destructive on bridge ports.** Adding a sub-interface to a bridge port list during a parallel build doesn't disrupt existing members. Both paths run side-by-side until the old one is removed. +- **Parallel-build > rename for live infrastructure.** Renaming `eth1.4` → `eth1.21` in place would have required atomically updating the device, bridge, and switch_vlan in one reload. Building `eth1.21` alongside, verifying, then removing `eth1.4` is recoverable at every step. + +## Open Items + +- [ ] Switch chip still has VID 3 with ports 1 and 2 as members (link down on 1, link up on 2). Need to identify what's on port 2 before retiring VID 3 entirely. +- [ ] Cosmetic: `eth1.21` listed as redundant tagged member on bridge VID 11 (harmless, future cleanup). +- [ ] Cosmetic: switch_vlan descriptions still say `lab-new` / `private-new`. Should rename to `lab` / `private` once VID 3 is fully retired. +- [ ] Document new wiring on Knot LXC — Phase 1.3 resume notes can now move from "blocked" to "infrastructure ready." + +## Final State + +| Layer | VID 11 (lab) | VID 21 (private) | +|---|---|---| +| Switch chip | `3 4t 6t` | `4 6t` | +| Sub-interface | `eth1.11` (explicit 8021q) | `eth1.21` (explicit 8021q) | +| Bridge port | `eth1.11` tagged, `eth1.3:u*` legacy | `eth1.21:u*` | +| End-to-end VID | 11 (NAS untagged on chip, all else 11) | 21 (PVE untagged on chip, all else 21) | + +NAS reachable, PVE management reachable, Knot LXC pinging gateway. Phase 1.3 unblocked. \ No newline at end of file diff --git a/Sources/Homelab/cli-standardization.md b/Sources/Homelab/cli-standardization.md new file mode 100644 index 0000000..3b607d1 --- /dev/null +++ b/Sources/Homelab/cli-standardization.md @@ -0,0 +1,75 @@ +--- +created: 2026-04-23 +path: Sources/Homelab +project: cli-standardization +status: active +tags: +- pbs +- tech-setup +- automation +type: project-plan +updated: 2026-04-23 +--- + +# CLI Standardization: Fish, Starship & Cross-Node Sync + +Standardize the terminal experience across the Manjaro dev rig, Proxmox +nodes, and Ubuntu NAS. This project moves away from manual "server pet" +configuration in favor of an automated, immutable CLI environment. + +## Objectives +- Deploy Fish 4.x+ and Starship to all primary nodes via Ansible. +- Establish a single source of truth for dotfiles using Chezmoi. +- Implement the `docker debug` protocol to avoid shell-bloat in containers. + +## Phase 1: Host Automation (Ansible) — COMPLETE +- [x] Create Ansible role `cli_modern` to install `fish`, `starship`, `fzf`, + `zoxide`, `bat`, `eza`, `fd`, `ripgrep`, `fastfetch`. +- [x] Handle Fish PPA for Ubuntu/Debian to get 4.x. +- [x] Add task to set Fish as the default shell for the primary user. +- [x] Deploy Fisher plugin manager with fzf.fish, z, sponge, autopair. +- [x] Write Fish config template (aliases, env vars, fastfetch on login). +- [x] Write Starship config template (git, Python/Go/Node, SSH hostname). +- [x] Support both Arch (Manjaro) and Debian (Ubuntu/Proxmox) via OS detection. +- [x] Pass ansible-lint at production profile. +- [ ] Fill in inventory with actual hosts. +- [ ] Deploy role to Proxmox cluster and Ubuntu NAS. + +## Phase 2: Dotfile Orchestration (Chezmoi) — COMPLETE (placeholders) +- [x] Initialize dotfiles Git repository with Chezmoi structure. +- [x] Create placeholder configs: Fish, Starship, Ghostty, Zellij. +- [x] Machine-type awareness (.chezmoiignore skips Ghostty/Zellij on servers). +- [x] Add Chezmoi install + apply tasks to cli_modern role (opt-in). +- [x] Configure `fish_plugins` file for Fisher automated plugin sync. +- [ ] Replace placeholder configs with real dotfiles from Manjaro rig. +- [ ] Push dotfiles repo to Gitea. +- [ ] Test `chezmoi apply` workflow on Manjaro rig. + +## Phase 3: Container Workflow +- [ ] Verify `docker debug` binary availability on nodes. +- [ ] Create an alias/function for `ddebug` that defaults to the Fish shell. +- [ ] Document the VS Code Dev Container configuration for consistent + project-level shells. + +## Repos + +| Repo | Location | Purpose | +|------|----------|---------| +| cli-standardization | `/opt/projects/pbs-workshop/cli-standardization/` | Ansible project with cli_modern role | +| dotfiles | `/opt/projects/pbs-workshop/dotfiles/` | Chezmoi-managed dotfiles (push to Gitea) | + +## Tools Installed + +| Tool | Purpose | Arch pkg | Ubuntu pkg | +|------|---------|----------|------------| +| Fish | Shell | fish | fish (PPA) | +| Starship | Prompt | binary | binary | +| fzf | Fuzzy finder | fzf | fzf | +| zoxide | Smart cd | zoxide | zoxide | +| bat | cat++ | bat | bat/batcat | +| eza | ls++ | eza | eza | +| fd | find++ | fd | fd-find | +| ripgrep | grep++ | ripgrep | ripgrep | +| fastfetch | System info | fastfetch | fastfetch | +| Fisher | Fish plugins | via script | via script | +| Chezmoi | Dotfiles | chezmoi | binary | \ No newline at end of file diff --git a/Sources/Homelab/dns-knot-pihole-end-to-end.md b/Sources/Homelab/dns-knot-pihole-end-to-end.md new file mode 100644 index 0000000..966beb8 --- /dev/null +++ b/Sources/Homelab/dns-knot-pihole-end-to-end.md @@ -0,0 +1,117 @@ +--- +created: 2026-05-08 +path: Sources/Homelab +project: dns-knot-pihole-end-to-end +status: completed +tags: +- homelab +- dns +- knot +- pihole +- openwrt +- traefik +type: session-notes +updated: 2026-05-08 +--- + +# DNS — Knot + Pi-hole End-to-End + +## Outcome + +Phase 1 of the Traefik project closed. Internal DNS architecture from +the homelab DNS decision doc is live: Knot DNS authoritative for +`herbylab.dev` on LXC 103, Pi-hole conditionally forwarding the zone, +OpenWrt passing answers through. All six initial records resolve +cleanly from a normal LAN client through the full chain. + +## Architecture (as built) + + +Client → OpenWrt (10.0.11.1) → Pi-hole (10.0.11.50) + ├─ herbylab.dev → Knot (10.0.11.10) + └─ everything else → Unbound → root + + +## Records served + +| Name | IP | VLAN | +|---|---|---| +| ns1 | 10.0.11.10 | Lab | +| immich | 10.0.11.62 | Lab | +| pihole | 10.0.11.50 | Lab | +| plex | 10.0.11.54 | Lab | +| pve | 10.0.21.188 | Private | +| traefik | 10.0.21.181 | Private | + +Migrated from `.lan` to `.dev` to enable LE wildcard via Cloudflare +DNS-01 in Phase 2. + +## Key Learnings + +- **`knotc reload` does not re-bind sockets.** It reloads the config +file but listening sockets only get re-evaluated on full daemon +restart. If `listen:` directives change (or were missing on first +start), `systemctl restart knot` is required. Symptom: zone loads, +`zone-status` reports correctly, but `dig` returns "connection +refused." +- **Knot config order matters for references.** `acl` blocks must be +defined before `zone` blocks that reference them — YAML validation is +single-pass, no forward references. Same pattern likely applies to +other ID-based references. +- **Fresh PVE LXCs inherit Tailscale resolvers from the host.** +`/etc/resolv.conf` gets populated with `100.100.100.100` by default. +If the LXC isn't on the tailnet, name resolution silently fails until +pointed at a reachable resolver. Permanent fix: configure DNS in the +LXC's PVE config or, for a DNS server LXC, point it at itself once +it's serving. +- **OpenWrt rebind protection silently filters RFC1918 answers.** When +dnsmasq has `rebind_protection='1'`, any DNS answer pointing to a +private IP gets stripped — query returns NOERROR with ANSWER:0 and EDE +15 (Blocked). The fix is whitelisting the domain via `rebind_domain`. +Easy to miss because the block happens *after* the upstream answers +correctly, so direct queries to Pi-hole or Knot work fine while +client-path queries fail. +- **EDE 15 (Blocked) doesn't always mean a blocklist match.** It's a +generic "this answer was filtered" code. Pi-hole's blocklists, +OpenWrt's rebind protection, and several other filters all use it. +Pi-hole's query log saying "Allowed" while `dig` reports EDE 15 is the +tell — block is happening downstream of Pi-hole. +- **Pi-hole v6 renamed conditional forwarding.** It's now under +reverse-server config with format +`,,,`. Same plumbing, different +UI surface. +- **Direct-vs-chain queries are the diagnostic shortcut.** When a DNS +chain fails, querying each link directly with `dig @` +immediately localizes the broken hop. Saved real time tonight +isolating the OpenWrt rebind issue. + +## Decisions Locked + +- Zone file lives at `/var/lib/knot/herbylab.dev.zone` on LXC 103, +owned by `knot:knot` +- Single NS record (`ns1`) for now; secondary will be added when NAS +instance comes up +- TTL strategy: 300 during buildout (current), raise to 3600 once stable +- SOA serial format: `YYYYMMDDNN` (current: `2026050801`) +- ACL allows transfer/notify from localhost only — will expand when +secondary is added +- Conditional forwarding scoped to `10.0.0.0/8` in Pi-hole — +VLAN/firewall layer enforces actual access boundaries + +## Open Items + +- [ ] LXC `/etc/resolv.conf` is a manual edit pointing at Pi-hole — +repoint at itself (10.0.11.10) with Pi-hole as fallback, make +persistent +- [ ] Knot zone file into git as source of truth +- [ ] Ansible role for Knot primary deployment +- [ ] Knot secondary on the NAS (after primary proven stable) +- [ ] Second Pi-hole instance for redundancy +- [ ] Pi-hole sync method decision (gravity-sync vs Teleporter) +- [ ] Move Traefik VM to Lab VLAN (currently on Private from original +bridge setup) +- [ ] Raise TTLs to 3600 once buildout settles + +## Next Session + +Phase 2 — wildcard cert via Traefik + Cloudflare DNS-01. DNS layer is done. \ No newline at end of file diff --git a/Sources/Homelab/editor-stack-upgrade.md b/Sources/Homelab/editor-stack-upgrade.md new file mode 100644 index 0000000..022b3e0 --- /dev/null +++ b/Sources/Homelab/editor-stack-upgrade.md @@ -0,0 +1,163 @@ +--- +created: 2026-04-08 +path: Sources/Homelab +project: editor-stack-upgrade +status: active +tags: +- editors +- neovim +- zed +- zellij +- lazyvim +- homelab +- workflow +type: project-plan +updated: 2026-04-08 +--- + +# Editor Stack Upgrade + +## Context + +VS Code Remote-SSH crashed production server after 6 instances were +accidentally left open (each spawns its own Node server process on the +remote). Replacing with a two-track editor workflow: a lightweight GUI for +heavier refactors, and a terminal-native stack for quick edits and street +cred. + +## Goals + +- [ ] Eliminate VS Code Remote-SSH resource bloat on production server +- [ ] Establish Zed as primary GUI editor with remote SSH workflow +- [ ] Build terminal-native editing muscle memory with Neovim + LazyVim +- [ ] Replace tmux workflow with Zellij (modern, discoverable keybinds) +- [ ] Keep production server stable during rollout (test everything on +tower first) + +## Track A: Zed (GUI path) + +### Phase A1 — Install on tower + +- [ ] Install Zed on Manjaro tower + ``` + curl -f https://zed.dev/install.sh | sh + ``` +- [ ] Verify binary at `~/.local/bin/zed` +- [ ] Launch and complete first-run setup +- [ ] Sign in (optional, for settings sync) + +### Phase A2 — Remote SSH setup + +- [ ] Confirm production server is in `~/.ssh/config` with alias +- [ ] Load SSH key into `ssh-agent` if passphrase-protected + ``` + eval $(ssh-agent) + ssh-add ~/.ssh/id_ed25519 + ``` +- [ ] In Zed: `ctrl+shift+p` → "remote: connect to host" → add prod server +- [ ] Verify remote agent installs to `~/.local/bin` on production server +- [ ] Open a test project over SSH and confirm LSP works + +### Phase A3 — Validate resource footprint + +- [ ] Check process count on prod server after Zed connect + ``` + ps aux | grep zed + ``` +- [ ] Compare RAM usage vs old VS Code Remote-SSH baseline +- [ ] Document findings in session notes + +## Track B: Terminal path (Zellij + Neovim + LazyVim) + +### Phase B1 — Zellij on tower + +- [ ] Install Zellij + ``` + cargo install --locked zellij + ``` + or via pacman/AUR if preferred +- [ ] Launch and work through built-in tutorial +- [ ] Learn core keybinds (shown at bottom of screen — that's the whole +point) +- [ ] Create first layout file for homelab workflow + ``` + ~/.config/zellij/layouts/homelab.kdl + ``` + +### Phase B2 — Neovim + LazyVim on tower + +- [ ] Install Neovim (need ≥0.9.0 for LazyVim) + ``` + sudo pacman -S neovim + ``` +- [ ] Install prerequisites: git, make, unzip, gcc, ripgrep, fd, a Nerd Font +- [ ] Back up any existing Neovim config + ``` + mv ~/.config/nvim ~/.config/nvim.bak + mv ~/.local/share/nvim ~/.local/share/nvim.bak + ``` +- [ ] Clone LazyVim starter + ``` + git clone https://github.com/LazyVim/starter ~/.config/nvim + rm -rf ~/.config/nvim/.git + ``` +- [ ] First launch: let Lazy.nvim install all plugins +- [ ] Run `:LazyHealth` and resolve any warnings +- [ ] Run `:Mason` and install LSPs for Python, Go, Bash, Lua + +### Phase B3 — Learn the basics (week 1) + +- [ ] Complete `vimtutor` (comes with Neovim, ~30 min) +- [ ] Daily driver for config edits only — don't force it on real work yet +- [ ] Learn: motions (h/j/k/l, w/b/e), modes (i/a/o, esc), save/quit (:w/:q) +- [ ] Learn LazyVim extras: `` leader menu, `ff` find files, +`sg` grep +- [ ] Learn file tree: `e` + +### Phase B4 — Zellij + Neovim integration + +- [ ] Create Zellij layout with Neovim + terminal panes +- [ ] Test detach/reattach workflow + ``` + zellij attach homelab + ``` +- [ ] Configure Neovim to respect Zellij pane navigation (if desired) + +### Phase B5 — Roll out to production server + +- [ ] Install Zellij on prod server +- [ ] Install Neovim + LazyVim on prod server +- [ ] Sync LazyVim config via git (store in `trucktrav/dotfiles` or similar) +- [ ] Create persistent Zellij session for prod work + ``` + zellij attach -c prod-work + ``` +- [ ] Document SSH + zellij attach workflow + +## Decision points + +- [ ] Dotfiles repo strategy — new repo or add to existing? +- [ ] LazyVim customization — keep starter minimal or add personal tweaks +early? +- [ ] Zed vs Neovim split — when to reach for which? (document after 2 +weeks of use) + +## Success criteria + +- [ ] Zero VS Code processes on production server +- [ ] Can SSH into prod, `zellij attach`, and resume work from any device +- [ ] Comfortable enough in Neovim to edit configs without reaching for Zed +- [ ] Production server RAM usage during editing sessions documented and +reduced + +## Notes & learnings + +_(populate as you go)_ + +## Next session + +Start with Phase A1 (Zed install on tower) — fastest win, immediately +usable, low risk. + + +...sent from Jenny & Travis \ No newline at end of file diff --git a/Sources/Homelab/env-file-hardening.md b/Sources/Homelab/env-file-hardening.md new file mode 100644 index 0000000..9316d22 --- /dev/null +++ b/Sources/Homelab/env-file-hardening.md @@ -0,0 +1,143 @@ +--- +created: 2026-04-19 +path: Sources/Homelab +project: env-file-hardening +status: active +tags: +- pbs +- security +- docker +- ansible +- secrets +type: project-plan +updated: 2026-04-19 +--- + +# Env File Hardening + +## Goal + +Remove the easy path to PBS secrets. Right now `.env` files in +`/opt/docker` are readable by anyone with filesystem access — including +`pbsdeploy`. If the deploy key is compromised, an attacker can `cat .env` +and grab every API key and password instantly. + +Goal: `pbsdeploy` can deploy containers without ever being able to read the +secrets they use. + +## Threat Model + +- **Primary risk:** `pbsdeploy` SSH key leaks → attacker SSHes in → reads +`.env` files → owns every service. +- **Mitigation goal:** eliminate the trivial read path. Raise the bar so an +attacker would need to escalate to root (no sudo on `pbsdeploy`), pivot +accounts, or break out of a container. +- **Accepted residual risk:** Determined attackers with deep access can +still cause damage. This is about removing low-hanging fruit, not achieving +zero risk. + +## Two-Phase Approach + +### Phase 1: Filesystem Permissions (80/20 win) + +Lock `.env` files down to `root:root 0600`. The Docker daemon runs as root, +so `docker compose` still reads them fine when processing compose files. +`pbsdeploy` runs `docker compose up`, which talks to the daemon via socket +— `pbsdeploy` itself never reads `.env` directly. + +**Net effect:** the trivial `cat .env` path is gone. No sidecar needed, no +app code changes, no runtime complexity. + +### Phase 2: Encryption at Rest on the Server (defense in depth, deferred) + +Currently secrets are encrypted in the `wp-i` repo via ansible-vault, but +land as plaintext on the server filesystem. Phase 2 keeps them encrypted on +disk and decrypts only at container start (via sidecar or similar). + +**Protects against:** backup leaks, Linode snapshot exfil, accidental +exposure, offline decrypt of historical filesystem copies. + +**Not urgent because:** secrets are already encrypted in the repo, Linode +account is behind 2FA, no compliance driver. This is "nice to have when +refactoring," per earlier conversation on the topic. + +## Design Decisions + +- **Phase 1 is just filesystem perms** — no sidecar, no Docker secrets, no +app changes. Docker daemon reads `.env` as root; `pbsdeploy` doesn't need +to. +- **Phase 2 uses sidecar pattern when we get there** — tiny init container +decrypts at runtime, injects env vars into main container, zero app code +changes. Compose secrets ruled out because it requires refactoring every +app to read from `/run/secrets/`. +- **External secret manager (Vault, etc.) ruled out** — overkill for +self-hosted prod scale. Revisit only if moving to Kubernetes. +- **Ansible stays the deploy tool** throughout — writes `.env` files as +root with tight perms. `pbsdeploy` has no sudoers entry, so it can't read +them back. +- **Staging first** for both phases. + +## How the Permissions Work (Phase 1) + +- Ansible SSHes in as `pbsdeploy` → uses `become: yes` to elevate for file +writes → files created as `root:root 0600`. +- `pbsdeploy` can't `cat` them (no read perms). Can't `sudo cat` them +either (no sudoers entry). +- Docker daemon runs as root → reads `.env` when processing compose files → +injects vars into containers. +- Main container sees env vars, doesn't know or care about file perms. + +## Next Steps (Phase 1) + +- [ ] Audit current state — which services have `.env` files in +`/opt/docker`, current ownership and perms +- [ ] Update Ansible role(s) to write `.env` as `root:root 0600` (not +`deployer` group) +- [ ] Pick a non-critical pilot service on staging +- [ ] Deploy change to pilot, verify container still works, verify +`pbsdeploy` can't read the file +- [ ] Roll out to all staging services +- [ ] Verify nothing breaks over a few deploys +- [ ] Roll out to production +- [ ] Remove `deployer` group read access to `.env` files (group can still +exist for other things if needed) + +## Next Steps (Phase 2 — later) + +- [ ] Decide on encryption approach: custom sidecar vs. SOPS + age vs. +something else +- [ ] Design sidecar container (Dockerfile, entrypoint, decryption strategy) +- [ ] Decide where the decryption key lives on the server and how it's +protected +- [ ] Pilot on one staging service +- [ ] Roll out across stack + +## Open Questions + +### Phase 1 + +- Does anything in `/opt/docker` currently rely on `deployer` group read +access to `.env` that we'd break by tightening perms? +- Are all services using `env_file:` in compose, or do some hardcode +secrets in the compose file itself (which would need separate handling)? + +### Phase 2 (deferred) + +- Sidecar implementation: custom shell-based init container vs. +off-the-shelf tool like SOPS? +- Where does the decryption key live on the server? (If it's in another +root-owned file, we're just moving the problem — though one layer deeper is +still an improvement.) + +## Depends On / Related + +- **SSH Login Alerting** — higher priority sibling project. Early warning +system for the threat this project mitigates. +- **PBSII CI/CD** — same `pbsdeploy` / `deployer` setup on staging; phase 1 +changes apply there too. +- **Previous conversation on secrets managers** — concluded that for +single-server self-hosted prod, encryption-at-rest on the server is "nice +to have," not urgent. Phase 2 is the realization of that future work. + + +...sent from Jenny & Travis \ No newline at end of file diff --git a/Sources/Homelab/homelab-dns-decision.md b/Sources/Homelab/homelab-dns-decision.md new file mode 100644 index 0000000..995b66f --- /dev/null +++ b/Sources/Homelab/homelab-dns-decision.md @@ -0,0 +1,116 @@ +--- +created: 2026-05-06 +path: Sources/Homelab +project: homelab-dns-decision +status: active +tags: +- homelab +- dns +- knot +- pihole +- unbound +- traefik +type: session-notes +updated: 2026-05-06 +--- + +# Homelab DNS — Architecture Decision + +## Outcome + +Locked the internal DNS architecture. **Knot DNS** as a dedicated +authoritative server for `herbylab.dev`, running active/passive across +Proxmox and the NAS. Pi-hole + Unbound stays as the recursive frontend and +ad-blocker. + +## Architecture + + +Client → Pi-hole (ad-block + cache) + ├─ herbylab.dev queries → Knot (authoritative) + └─ everything else → Unbound → root servers + + +Split-horizon: `*.herbylab.dev` resolves internally to the Traefik VM IP +via Knot. Cloudflare remains public authoritative for the same zone. Same +name, different answer depending on who's asking. + +## Decisions + +| Decision | Choice | Rationale | +|---|---|---| +| Authoritative DNS | Knot DNS | Purpose-built authoritative, native +AXFR/NOTIFY, ~40MB RAM, zone files in plain text, `knotc` CLI for runtime +changes | +| Recursive + ad-block | Pi-hole + Unbound (kept) | Already works, no +reason to replace the recursive layer just because zone management is +clunky | +| Redundancy model | Active/passive, separate hosts | Knot primary on PVE, +secondary on NAS — separate failure domains | +| Replication | Native AXFR + NOTIFY | Edit primary, secondary auto-syncs. +No Ansible run per record change | +| Source of truth | Zone file in git | Versioned, debuggable, deployed to +primary via Ansible | +| Failover at client | OpenWrt DHCP hands out both Pi-hole IPs | +Active/passive at the resolver level. Adds the second Pi-hole to the +existing pattern | +| DHCP | Stays on OpenWrt | Knot doesn't touch DHCP | +| Internal certs | Wildcard `*.herbylab.dev` via Traefik + Cloudflare +DNS-01 | Real LE cert, no browser warnings, independent of internal DNS +layer. Works because Cloudflare owns the public zone — LE never touches the +internal IP | +| Zone structure | Flat (one zone for all `*.herbylab.dev`) | Single source +of truth; revisit if it gets unwieldy | + +## Why Not the Other Contenders + +- **Technitium** — right answer when *replacing* Pi-hole + Unbound +entirely. Overkill once Pi-hole stays. +- **PowerDNS** — sexy on paper (REST API, web UI option) but adds SQL +backend, second daemon for the UI, heavier replication setup. Can write a +custom GUI on top of Knot later if needed. +- **CoreDNS** — fine, but not really designed for primary/secondary AXFR. +Better fit for K8s-style deployments. +- **NSD** — close runner-up. Knot edged it on the `knotc` runtime CLI and +slightly nicer config syntax. +- **BIND9** — overkill, config-heavy. Skipped. + +## Key Learnings + +- **Internal certs are a naming problem, not a DNS problem.** Using ` +herbylab.dev` for internal names (split-horizon) means a public CA will +sign a wildcard cert via DNS-01. A `.lan` or `.internal` zone forces a +private CA + root cert distribution — not worth it. +- **The job defines the tool.** Original DNS contender list (Technitium, +CoreDNS, PowerDNS, BIND9) assumed a full Pi-hole + Unbound replacement. +Once Pi-hole stays, the job collapses to "small authoritative server, one +zone, easy to replicate" — different list (Knot, NSD, CoreDNS, PowerDNS). +- **Cloudflare, Tailscale, Traefik, internal DNS each do one thing.** +Cloudflare = public auth + ACME validation. Tailscale MagicDNS = device +naming on the tailnet. Traefik = service routing once a request lands. +Internal DNS = telling LAN clients where to send the packet. They don't +overlap, they compose. + +## Open Items + +- [ ] Stand up second Pi-hole instance for redundancy (parallel to Knot +work, not blocking) +- [ ] Decide Pi-hole sync method (gravity-sync vs Teleporter) +- [ ] Pick deployment unit for Knot on each host (LXC on PVE likely; Docker +on NAS) +- [ ] Initial zone file content — inventory current `*.herbylab.dev` +records to migrate from Pi-hole local DNS +- [ ] TTL strategy — start at 300 during buildout, raise to 3600 once stable +- [ ] Ansible role for Knot primary deployment +- [ ] Confirm Pi-hole conditional forwarding config for `herbylab.dev` → +Knot + +## Next Session + +Return to the Traefik chat with this decision locked. DNS slot in the +broader stack is now filled. + + + + +...sent from Jenny & Travis \ No newline at end of file diff --git a/Sources/Homelab/homelab-mcp-server.md b/Sources/Homelab/homelab-mcp-server.md new file mode 100644 index 0000000..0a9c32b --- /dev/null +++ b/Sources/Homelab/homelab-mcp-server.md @@ -0,0 +1,185 @@ +--- +created: 2026-05-05 +path: Sources/Homelab +project: homelab-mcp-server +status: active +tags: +- homelab +- mcp +- obsidian +- n8n +- authentik +- cloudflare +type: project-plan +updated: 2026-05-05 +--- + +# Homelab MCP Server + +## Goal + +Replace the email → n8n → Obsidian pipeline for project and session notes +with a custom MCP server that LLMs can call directly. The MCP server +becomes the canonical surface for "things I want LLMs to know and record" +across the homelab. + +## Why + +- The email-as-transport pattern works but feels old-fashioned and forces +schema/template duplication across every LLM project. +- Schema changes (e.g., flattening folder structure) currently require +editing every project's instructions. With an MCP server, the contract +lives in one place — change the server, every LLM picks it up. +- Establishes the auth and deployment pattern for future homelab tooling +exposed to LLMs. +- Mobile-friendly: once added on claude.ai, the connector works from the +phone app. + +## Architecture + + +Claude (web/mobile/desktop) + ↓ HTTPS +Cloudflare Tunnel → mcp.herbylab.dev + ↓ +Authentik (OAuth) + ↓ validated token +MCP server (Python, us-test-authy) + ↓ + ┌────┴────┐ + ↓ ↓ +Vault git n8n webhook +(commit + (downstream fan-out: +push) tasks, Drive mirror, etc.) + + +**Trust separation:** MCP = read + scoped write. Dispatch and future +harnesses handle execution. The two surfaces never merge. + +## Locked Decisions + +- [x] Single MCP server, organized by domain modules internally +- [x] Streamable HTTP transport (not SSE — being deprecated) +- [x] Authentik for OAuth +- [x] Cloudflare tunnel → `mcp.herbylab.dev` → `us-test-authy` +- [x] Python implementation +- [x] Vault writes go direct (filesystem + git commit/push); n8n becomes +downstream consumer for fan-out +- [x] Read-only on Travis's side, enforced by not building edit/delete tools +- [x] Execution stays out of MCP entirely — Dispatch's job +- [x] Vault repo lives on `us-test-authy`; tower opens a local clone that +pulls +- [x] Gitea remote may move; not a blocker (one-line `git remote set-url`) + +## Code Structure + + +mcp_server/ + tools/ + vault.py # save_note, list_projects, get_project, get_schema + memory.py # placeholder for OpenBrain / LLM wiki + tasks.py # generic task tools, n8n forwards + core/ + auth.py # Authentik validation + git_writer.py # commit + push logic + schema.py # canonical schema dict + server.py # registers tools from each module + + +## Phase 1 Tool Surface + +- `save_note(title, content, project, type, tags, status?)` — write +markdown, assemble frontmatter, commit, push. Returns file path + git SHA. +- `list_projects()` — enumerate existing project slugs with metadata. +Prevents slug collisions. +- `get_project(slug)` — fetch a project's notes for context loading. +- `get_schema()` — return canonical valid values for `type`, `status`, +`tags`, folder paths. **Solves the folder-flattening problem.** Also +includes `knowledge_sources` field describing what each connected system is +for (vault, openbrain, llm_wiki). + +## Phase 2 Catalog (mapped, not built) + +**Vault depth:** +- `search_notes(query, filters)` — full-text + tag/type/date filters +- `list_recent_notes(days, filters)` +- `get_note(path)` +- `list_tags()` +- `summarize_project(slug)` + +**Tasks (generic, n8n routes):** +- `create_task(project, title, description?, frontmatter?, due?)` — n8n +decides destination based on frontmatter +- `list_tasks(project?, status?, filters?)` +- `modify_task(task_id, updates)` — generic update covers +complete/reopen/edit/reassign +- Task IDs opaque to MCP server; n8n owns the ID space + +**Cross-system glue:** +- `mirror_to_drive(note_path)` — Obsidian → Google Docs via n8n +- `record_session_artifact(session_id, type, location)` — track +session-generated files without putting them in the vault + +**Memory & knowledge (placeholder, design pending):** + +OpenBrain and LLM wiki integration deferred until workflow patterns +establish. Open questions: +- Unit of retrieval (note? fact? synthesized answer?) +- Authoritative source (vault → memory, memory → vault, or peers?) +- Read-only or read-write from LLM's side? +- API-shaped or markdown-on-disk? + +Likely tool shapes (names will change): +- `query_knowledge(question, sources?)` — federated semantic query +- `get_context(topic, depth?)` — background assembly before starting work +- `record_to_memory(content, system, metadata)` + +Architectural commitment: `tools/memory.py` exists from day one as empty +module. Boundary stays. + +## Implementation Order + +- [ ] Hello-world MCP server in Python (Streamable HTTP, single dummy tool) +- [ ] Cloudflare tunnel + DNS for `mcp.herbylab.dev` +- [ ] Verify end-to-end: connector added on claude.ai, tool callable from +chat +- [ ] Authentik OAuth integration +- [ ] Phase 1 tools: `save_note`, `list_projects`, `get_project`, +`get_schema` +- [ ] Git commit/push integration with deploy key +- [ ] Migrate from email flow; retire email trigger in n8n + +**Sequencing principle:** working transport before auth before real tools. +Don't debug "is it the auth, the network, or my code?" all at once. + +## Key Learnings From Design Session + +- **MCP is one of multiple LLM-facing surfaces.** MCP for knowledge/state, +Dispatch for execution. Don't conflate trust classes. +- **n8n stops being the front door but stays the integration hub.** Server +calls n8n for fan-out (tasks, Drive mirroring); n8n keeps Google/Trello +tight integration without owning the entry point. +- **Schema-as-API is the unlock.** `get_schema()` makes the MCP server the +canonical contract. Schema changes propagate without touching project +instructions. +- **Read-only-from-Travis eliminates the concurrent-write problem class.** +MCP server is sole writer; Travis is consumer. No merge conflicts, no +`.obsidian/workspace.json` thrash. +- **Single server is correct until proven otherwise.** Code structured by +domain so future split is mechanical, not a rewrite. +- **Streamable HTTP, not SSE.** SSE deprecation is in motion — build on the +durable transport from day one. +- **Anthropic connects from their cloud, not from the user's device.** +Tailscale alone won't work — server needs public HTTPS reachability. +Cloudflare tunnel handles this. + +## Open Items / Future Travis + +- Sharing model (collaborator access to specific tools) — defer until needed +- Decision on Gitea hosting location +- OpenBrain and LLM wiki workflow patterns before Phase 2 memory tools +- Whether `save_note` auto-fans-out tasks from `## Tasks` sections, or +stays explicit (current lean: explicit) + + +...sent from Jenny & Travis \ No newline at end of file diff --git a/Sources/Homelab/hunyuan3d-sunnie-pipeline.md b/Sources/Homelab/hunyuan3d-sunnie-pipeline.md new file mode 100644 index 0000000..a365ba9 --- /dev/null +++ b/Sources/Homelab/hunyuan3d-sunnie-pipeline.md @@ -0,0 +1,159 @@ +--- +created: 2026-04-27 +path: Sources/Homelab +project: hunyuan3d-sunnie-pipeline +status: active +tags: +- homelab +- docker +- ai-ml +- gpu +- blender +- pbs +- sunnie +type: project-plan +updated: 2026-04-27 +--- + +# Hunyuan3D-2 + Sunnie Animation Pipeline + +Self-hosted deployment of [Tencent Hunyuan3D-2]( +https://github.com/Tencent-Hunyuan/Hunyuan3D-2) on the Manjaro tower (RTX +3080), feeding into a Blender pipeline to produce a dancing 3D version of +Sunnie for PBS content. + +## Why Hunyuan3D-2 over InstantMesh + +For stylized characters, open-source models (Hunyuan, Trellis) preserve +cartoon aesthetics better than commercial tools (Tripo, Meshy) that +over-detail. Hunyuan3D-2 also outputs PBR textures (not just vertex +colors), which matters once Sunnie hits Blender for re-lighting. +InstantMesh stays in flight as a comparison data point. + +## Stack + +- **Host:** Manjaro tower (i7 / 128GB / RTX 3080, 10GB VRAM) +- **Runtime:** Docker + NVIDIA Container Toolkit +- **Model variant:** `Hunyuan3D-2mini` with `hunyuan3d-dit-v2-mini-turbo` +subfolder (10GB VRAM-friendly) +- **Required flags:** `--low_vram_mode --enable_flashvdm` +- **UI:** Gradio on `:8080` +- **Downstream:** Blender 4.x, Mixamo for auto-rig + dance animations + +## VRAM reality check + +The full `Hunyuan3D-2` model wants more than 10GB. The `mini-turbo` variant ++ low-VRAM flags is the target config for the 3080. If quality is +unacceptable, fallbacks: rent an A100/4090 hour on RunPod for final-quality +renders, or move workload to a future GPU upgrade. + +## Known issues + +- No official Tencent Docker image — community Dockerfiles only (referenced +in repo issues #125, #122) +- First-run model download is large (multiple GB across HuggingFace repos) +— persist to host volume +- Hunyuan3D-2.1 exists with better PBR but Docker support is rougher; +deferring until 2.0 baseline works + +## Phase 1 — Deploy Hunyuan3D-2 + +- [ ] Verify NVIDIA Container Toolkit: `docker run --rm --gpus all +nvidia/cuda:12.4.0-base-ubuntu22.04 nvidia-smi` +- [ ] Clone repo: `git clone +https://github.com/Tencent-Hunyuan/Hunyuan3D-2.git ~/projects/hunyuan3d-2` +- [ ] Write Dockerfile (community-style, base +`nvidia/cuda:12.4.0-devel-ubuntu22.04`) +- [ ] Build image: `docker compose build` +- [ ] Create model cache dir: `mkdir -p ~/hunyuan3d-models` +- [ ] First run: `docker compose up` — weights download to mounted volume +- [ ] Verify Gradio UI at `http://localhost:8080` +- [ ] Smoke test with sample image, save `.glb` output + +## Phase 2 — Sunnie mesh generation + +- [ ] Prepare Sunnie reference image (front-facing, transparent or white +background, clean lines) +- [ ] Generate mesh via Gradio UI +- [ ] Export as `.glb` or `.obj` with PBR textures +- [ ] Iterate prompt/source image until mesh is acceptable + +## Phase 3 — Blender cleanup + +- [ ] Import mesh into Blender 4.x +- [ ] Inspect topology — likely needs decimate or retopo for clean rig +deformation +- [ ] Verify UV maps and PBR material assignment +- [ ] Pose into T-pose if not already (Mixamo requirement) +- [ ] Export as `.fbx` for Mixamo upload + +## Phase 4 — Rig + animate via Mixamo + +- [ ] Upload `.fbx` to Mixamo, place rigging markers +- [ ] Verify auto-rig deformation +- [ ] Pick dance animation from Mixamo library +- [ ] Download rigged `.fbx` with skin + animation +- [ ] Re-import to Blender, verify playback + +## Phase 5 — Final render + +- [ ] Set up Blender scene (lighting, camera, background) +- [ ] Render dance sequence (Cycles or Eevee depending on quality target) +- [ ] Export video for PBS use + +## docker-compose.yml (starting point) + +yaml +services: + hunyuan3d: + build: + context: ./Hunyuan3D-2 + dockerfile: Dockerfile + image: hunyuan3d-2:local + container_name: hunyuan3d + ports: + - "8080:8080" + volumes: + - ~/hunyuan3d-models:/root/.cache/hy3dgen + - ./outputs:/workspace/outputs + command: > + python3 gradio_app.py + --model_path tencent/Hunyuan3D-2mini + --subfolder hunyuan3d-dit-v2-mini-turbo + --texgen_model_path tencent/Hunyuan3D-2 + --low_vram_mode + --enable_flashvdm + --host 0.0.0.0 + --port 8080 + deploy: + resources: + reservations: + devices: + - driver: nvidia + count: all + capabilities: [gpu] + restart: unless-stopped + + +## Open questions + +- [ ] If `mini-turbo` quality is insufficient for Sunnie, escalate to +Hunyuan3D-2.1 (more VRAM-hungry, rougher Docker) or RunPod cloud GPU? +- [ ] Reverse-proxy through Traefik on `*.lab.herbylab.dev`, or keep +local-only? +- [ ] Worth integrating with InstantMesh deploy for side-by-side mesh +comparison? +- [ ] Long-term: candidate tool for OpenClaw to call as a 3D-gen capability? + +## References + +- Hunyuan3D-2 repo: https://github.com/Tencent-Hunyuan/Hunyuan3D-2 +- Hunyuan3D-2.1 (newer, PBR-focused): +https://github.com/Tencent-Hunyuan/Hunyuan3D-2.1 +- Docker discussion: +https://github.com/Tencent-Hunyuan/Hunyuan3D-2/issues/125 +- Mixamo: https://www.mixamo.com +- Companion project: `instantmesh-docker` + + +...sent from Jenny & Travis \ No newline at end of file diff --git a/Sources/Homelab/index.md b/Sources/Homelab/index.md new file mode 100644 index 0000000..662afdc --- /dev/null +++ b/Sources/Homelab/index.md @@ -0,0 +1,23 @@ +--- +type: topic-landing +path: Sources/Homelab +tags: + - landing +updated: 2026-05-08 +--- + +# Homelab — Sources + +Project plans and session notes for the Homelab domain. + +## Active Projects + +*Updated by compile loop.* + +## Recent Sessions + +*Updated by compile loop.* + +## Completed Projects + +*Updated by compile loop.* diff --git a/Sources/Homelab/instantmesh-docker.md b/Sources/Homelab/instantmesh-docker.md new file mode 100644 index 0000000..92414b9 --- /dev/null +++ b/Sources/Homelab/instantmesh-docker.md @@ -0,0 +1,95 @@ +--- +created: 2026-04-27 +path: Sources/Homelab +project: instantmesh-docker +status: active +tags: +- homelab +- docker +- ai-ml +- gpu +type: project-plan +updated: 2026-04-27 +--- + +# InstantMesh Docker Deployment + +Self-hosted deployment of [TencentARC/InstantMesh]( +https://github.com/TencentARC/InstantMesh) on the Manjaro tower (RTX 3080) +for single-image to 3D mesh generation. Gradio UI exposed on port 43839. + +## Goals + +- Run InstantMesh locally via Docker Compose on the Manjaro tower +- Persist downloaded model weights outside the container (re-pulls are slow) +- Build from official repo source, patch the known `onnxruntime` issue from +upstream + +## Stack + +- **Host:** Manjaro tower (i7 / 128GB / RTX 3080, 10GB VRAM) +- **Runtime:** Docker + NVIDIA Container Toolkit +- **Base image:** Built from `TencentARC/InstantMesh/docker/Dockerfile` +- **UI:** Gradio on `:43839` +- **Model variant:** `instant-mesh-large` (default, fits in 10GB VRAM) + +## Known issue + +Upstream Dockerfile has a missing dep — container fails at runtime with +`ModuleNotFoundError: No module named 'onnxruntime'` (issue #175). Fix is +to add `onnxruntime` to the pip install step in the Dockerfile before +building, or install it post-build. + +## Setup steps + +- [ ] Verify NVIDIA Container Toolkit is functional: `docker run --rm +--gpus all nvidia/cuda:12.1.0-base-ubuntu22.04 nvidia-smi` +- [ ] Clone repo: `git clone https://github.com/TencentARC/InstantMesh.git +~/projects/instantmesh` +- [ ] Patch Dockerfile to add `onnxruntime` to pip install +- [ ] Build image: `docker compose build` +- [ ] Create model dir: `mkdir -p ~/instantmesh-models` +- [ ] First run: `docker compose up` — model weights download on first +launch (~10GB) +- [ ] Verify Gradio UI at `http://localhost:43839` +- [ ] Test inference with sample image + +## docker-compose.yml + +yaml +services: + instantmesh: + build: + context: ./InstantMesh + dockerfile: docker/Dockerfile + image: instantmesh:local + container_name: instantmesh + ports: + - "43839:43839" + volumes: + - ~/instantmesh-models:/workspace/instantmesh/models + deploy: + resources: + reservations: + devices: + - driver: nvidia + count: all + capabilities: [gpu] + restart: unless-stopped + + +## Open questions + +- [ ] Reverse proxy through Traefik on `*.lab.herbylab.dev`, or keep +local-only? +- [ ] Worth wiring into OpenClaw later as a callable 3D-gen tool? + +## References + +- Repo: https://github.com/TencentARC/InstantMesh +- Docker dir: https://github.com/TencentARC/InstantMesh/tree/main/docker +- onnxruntime issue: https://github.com/TencentARC/InstantMesh/issues/175 +- Community fix PR: https://github.com/TencentARC/InstantMesh/pull/181 + + +...sent from Jenny & Travis \ No newline at end of file diff --git a/Sources/Homelab/ob1-deployment.md b/Sources/Homelab/ob1-deployment.md new file mode 100644 index 0000000..21f59b1 --- /dev/null +++ b/Sources/Homelab/ob1-deployment.md @@ -0,0 +1,288 @@ +--- +created: 2026-05-04 +path: Sources/Homelab +project: ob1-deployment +status: active +tags: +- homelab +- ai +- memory +- ob1 +- supabase +- mcp +- addendum +type: project-plan +updated: 2026-05-04 +--- + +# OB1 Deployment — Addendum + +This addendum amends the original `ob1-deployment` project plan +based on architectural decisions made during the wiki-construct +design session on 2026-05-04. It does not replace the original +plan; it modifies specific phases and clarifies scope. Read this +alongside the original plan. + +## Context for the changes + +The wiki-construct session crystallized the framing for personal +AI memory across the homelab: + +- **Wiki** is Travis's interface to his own thinking. Human- + facing, organized for browsing in Obsidian. Agents read it + only when explicitly pointed at a file. +- **OB1** is Lovebug's working memory. Agent-facing, semantic + search via MCP, captured automatically from sessions and + conversations. + +The two systems are parallel and complementary, not unified. +This framing changes the OB1 plan less than expected — most of +the original plan still stands — but it sharpens scope around +what OB1 is and isn't responsible for. + +## Amendments to the original plan + +### Phase 4 — Authelia and Traefik (deferred) + +The original plan put Authelia + Traefik fronting in Phase 4 as +part of baseline deployment. **Defer this entire phase** until +there is a concrete user-facing reason to expose OB1 outside +Tailscale. + +Replacement scope for Phase 4: + +- Tailscale-only access for all OB1 surfaces (Studio UI, REST + API, MCP server) +- Document the URL map for the user (Tailscale hostnames / + ports for each surface) +- Confirm that the JWT-bearing MCP path stays Tailscale-only + forever — even when Authelia is eventually added in front of + browser surfaces, agent endpoints stay on Tailscale. This is + a design invariant, not a phase choice. + +Authelia goes back on the roadmap when Jenny's interface lands +or when there's another reason to put a browser surface on the +public web. Not before. + +This shortens the deployment by an estimated 30-40% of the +Phase 4 task list and removes the most complex coordination +with the existing Traefik / Authelia config. + +### Phase 3 — schema decisions (extended) + +The original Phase 3 deploys OB1's schema as-shipped. **Add one +forward-looking schema decision** before applying migrations: + +- **User scoping from day one.** Schema includes a `user_id` or + equivalent tenant column on every captured row, with RLS + policies enforced, even though the deployment is single-user + (Travis only) at first. + +Rationale: Jenny's eventual interface (PBS-public web surface +served from the PBS site) will require multi-user support. Row- +level security retrofitted later is painful. RLS designed in +from day one is cheap. + +This is a "design choice, not a feature" addition — no UI for +multi-user, no actual second user yet, just schema and policies +in place so the door isn't accidentally closed. + +### Phase 5 — client integration (refined) + +The original Phase 5 wires Claude Code, Claude Desktop, and the +n8n markdown-summary flow to OB1. All three remain in scope. +**Refinement:** the n8n flow modification should write to OB1 +*in parallel* with the existing Obsidian write, not in place of +it. The two systems serve different purposes: + +- The Obsidian write builds Travis's wiki (now structured per + the wiki-construct project) +- The OB1 write builds Lovebug's memory + +Same source, two destinations, neither replacing the other. + +The n8n node added in Phase 5 should: + +- POST the captured content to OB1's REST API +- Map relevant frontmatter fields to OB1 metadata (project, + domain, tags, dates) +- Tag OB1 captures with their wiki-vault path so cross- + reference is possible later if needed +- Run after the Obsidian write succeeds (Obsidian is the + primary destination; OB1 capture is additive) + +### Phase 6 — operational hygiene (no changes) + +The original Phase 6 stands. Backups, monitoring, Ansible +playbook, runbook, snapshot. Unchanged. + +### Phase 7 — Dispatch pre-compact hook (promoted from out-of-scope) + +The original plan listed the Dispatch pre-compact hook as out- +of-scope and "separate project". **Promote this to Phase 7 of +this project**, immediately following Phase 6, as the closing +phase. + +Reason: until the pre-compact hook is shipping captures into +OB1, OB1 is not actually serving its primary purpose as +Lovebug's working memory. Phases 1-6 deploy a database with +Claude Code MCP captures and email-summary captures, but no +Dispatch session captures. Dispatch sessions are where the bulk +of Lovebug's work happens. Without the hook, OB1 has the wrong +memory. + +Phase 7 scope: + +- Design what Dispatch sessions capture (full transcript, + decisions only, summaries only — needs design work) +- Design noise filtering (Dispatch sessions include a lot of + routine command output that shouldn't end up in semantic + memory) +- Implement the pre-compact hook integration +- Test capture end-to-end: run a Dispatch session, verify + meaningful content lands in OB1, verify retrieval surfaces + the right thing +- Document the hook for future modification + +Phase 7 is its own meaningful design exercise — what to capture +is not obvious. Treat it as a multi-session phase with its own +design step before implementation. + +## Scope clarifications (no plan change, just sharpening) + +### What OB1 is for + +- Lovebug's working memory across Dispatch sessions +- Lovebug's memory across Claude Code sessions +- Captures of Travis's email-summary intake (the same content + that builds the wiki, captured in parallel for semantic + search) +- Eventually: agent-execution session notes from coding + projects (currently captured alongside code in the project + repo; eventually flow into OB1 for cross-project recall) + +### What OB1 is not for + +- Travis's curated knowledge (that's the wiki) +- Long-form synthesis (that's the wiki) +- Human browsing (Studio UI is for inspection and + troubleshooting, not daily reference) +- Replacement for the wiki — the two systems coexist + +### How OB1 and the wiki interact + +In Phase 1 of both projects: they don't, beyond the parallel +n8n write described above. Travis uses the wiki to think. +Lovebug uses OB1 to remember its own work and conversations. + +In some Phase 2 future: a context assembler may query both +layers when bundling prompts — semantic search across OB1 plus +explicit wiki references when Travis points at specific pages. +This is deferred and should not influence Phase 1 architecture +decisions. + +The wiki should never be designed for OB1 to consume, and OB1 +should never be designed to maintain wiki content. Keep them +parallel. + +## Dependencies and ordering + +The wiki-construct project does not block OB1. They can run in +parallel. Travis's stated preference is wiki-first, but that's a +bandwidth decision, not a technical dependency. + +If wiki-construct ships first: + +- The n8n markdown-summary flow will already be writing to a + structured wiki when OB1 Phase 5 adds the parallel OB1 write +- Frontmatter conventions will already be stable, simplifying + the OB1 metadata mapping + +If OB1 ships first: + +- Wiki-construct's compile loop is unaffected (it doesn't read + from OB1) +- The Phase 5 n8n write will need to be revisited once the + wiki's frontmatter conventions land + +Either order works. + +## Open items + +- [ ] Decide cloud (OpenAI) vs. local (Ollama on RTX 3080) + embeddings during Phase 3 (recommendation in original plan + stands: local) +- [ ] Phase 7 design: what to capture from Dispatch sessions and + how to filter noise +- [ ] Confirm OB1's schema supports `user_id` / tenant scoping + cleanly during Phase 0 inventory; if not, this becomes a + schema extension rather than a configuration choice + +## Tasks (additions and changes only) + +### Phase 0 — Repository inspection (one addition) + +- [ ] Confirm OB1's schema supports user-scoping or determine + the minimal extension needed + +### Phase 3 — Schema and MCP server (one addition) + +- [ ] Apply user-scoping schema extension if needed +- [ ] Verify RLS policies enforce single-user access correctly + (Travis sees only Travis's rows, even though there's only + one user) + +### Phase 4 — Tailscale-only access (replaces original Phase 4) + +- [ ] Document Tailscale URL map for all OB1 surfaces +- [ ] Confirm MCP path is reachable over Tailscale from Travis's + dev rig and from herbydev +- [ ] Document the design invariant: MCP path stays Tailscale- + only permanently +- [ ] Defer Authelia + Traefik integration explicitly + +### Phase 5 — Client integration (one refinement) + +- [ ] Confirm n8n write to OB1 runs in parallel with Obsidian + write, not in place of it +- [ ] Tag OB1 captures from the email flow with their wiki-vault + path + +### Phase 7 — Dispatch pre-compact hook (new) + +- [ ] Design: what Dispatch sessions capture +- [ ] Design: noise filtering rules +- [ ] Implementation: pre-compact hook integration +- [ ] Test: end-to-end capture and retrieval from a Dispatch + session +- [ ] Document: hook configuration for future modification + +## Phase 1 done when (amended) + +The original plan's "project complete" trigger stands, with one +addition: **the Dispatch pre-compact hook is shipping captures +into OB1 and Lovebug is using them.** Without that, OB1 is +deployed but not yet doing its primary job. + +Consider OB1 truly operational only after Phase 7. Phases 1-6 +deploy the substrate; Phase 7 turns it on. + +## Notes + +- This addendum is not a re-plan. The original phases 1-3 and + 6 are unchanged. Phase 4 is shrunk. Phase 5 is refined. + Phase 7 is added. +- The wiki-construct project is the sibling project. Its plan + and skill bundle land separately. They do not depend on each + other. +- The framing decisions in this addendum (wiki for human, OB1 + for agent, parallel not unified) are load-bearing. If + anything in the deployment seems to push back on that + framing — e.g., "should OB1 also serve the wiki?" or "should + the wiki feed OB1 automatically?" — stop and confirm with + Travis before proceeding. The instinct to unify is wrong; + the instinct to keep parallel is correct. + + +-- +...sent from Jenny & Travis \ No newline at end of file diff --git a/Sources/Homelab/ob1-main-deployment.md b/Sources/Homelab/ob1-main-deployment.md new file mode 100644 index 0000000..8c47f4b --- /dev/null +++ b/Sources/Homelab/ob1-main-deployment.md @@ -0,0 +1,268 @@ +--- +created: 2026-05-04 +path: Sources/Homelab +project: ob1-main-deployment +status: active +tags: +- homelab +- ai +- memory +- ob1 +- supabase +- mcp +- docker +type: project-plan +updated: 2026-05-04 +--- + +# OB1 Deployment Project Plan + +Deploy OpenBrain (OB1) as the personal AI memory layer on the homelab. +OB1 is a self-hosted Postgres-plus-Supabase stack exposing MCP server +access for any compatible AI client. This becomes the persistent +memory layer underneath Claude Code Dispatch, Claude Desktop, and +eventually Pi or other harnesses. + +This plan is structured for hand-off to an agent after the OB1 repo is +cloned. Each phase has explicit deliverables, validation steps, and +stop conditions. The agent should pause after each phase for review +before proceeding. + +## Background context the agent needs + +- OB1 source: `github.com/NateBJones-Projects/OB1` + +- Architecture: Supabase (Postgres + GoTrue + PostgREST + Kong + +Studio + Edge Functions) with OB1's MCP server layered on top +- Target deployment node: Proxmox VM on the existing homelab (assume a +fresh Ubuntu 24.04 LTS VM unless directed otherwise) +- Network: Tailscale mesh for remote access; Traefik already running +on the homelab as the reverse proxy with HTTPS +- Auth model: Authelia in front of web-facing surfaces (Studio UI); +GoTrue stays for application identity and JWT issuance for AI clients +- This is a personal single-user deployment initially. RLS policies +should be in place but expect future expansion to a second user +(Jenny) for shared PBS context +- The user prefers Ansible for deployments and Docker Compose for +service orchestration. UV for any Python work. Never use +--break-system-packages + +## Phase 0 — Repository inspection and environment audit + +Goal: understand what the agent is actually deploying before touching +anything. + +Tasks: +- [ ] Read the OB1 README and any deployment or setup docs in the repo +- [ ] Inventory the docker-compose.yml — list every service, its +image, ports, volumes, dependencies +- [ ] Identify which Supabase services OB1 actually requires versus +which are bundled but unused (Realtime, Storage, ImgProxy, Logflare +are likely candidates for disabling) +- [ ] Check the OB1 MCP server implementation — language, runtime, +dependencies, how it connects to Supabase +- [ ] Note any environment variables, secrets, or external +dependencies the deployment requires +- [ ] Check schema or migration files — understand the data model +before touching it + +Deliverable: a written summary of what OB1 deploys, what's optional, +what's required, and what external resources (API keys, URLs, secrets) +the user needs to provide. + +Stop condition: present the summary and the proposed Phase 1 plan to +the user for review before proceeding. + +## Phase 1 — VM provisioning and base setup + +Goal: clean Ubuntu 24.04 VM ready to run OB1. + +Tasks: +- [ ] Confirm with user: VM hostname, target Proxmox node, resource +allocation (suggest 4 vCPU, 8GB RAM, 60GB disk as starting point — +adjust based on Phase 0 findings) +- [ ] Provision the VM via Proxmox (the user can do this manually; +agent should provide exact specs) +- [ ] Install Docker Engine and Docker Compose v2 (not the deprecated +docker-compose v1) +- [ ] Install Tailscale and join the mesh +- [ ] Confirm Tailscale connectivity from the user's dev rig +- [ ] Set up a non-root user for service operation (suggest +`ob1admin`) with Docker group membership +- [ ] Install basic operational tooling: htop, ncdu, git, curl, jq + +Deliverable: a working VM accessible over Tailscale, with Docker +installed and the OB1 repo cloned to `/opt/ob1/` (or wherever the user +prefers). + +Stop condition: confirm with user that the VM is reachable and base +setup is complete before proceeding to Phase 2. + +## Phase 2 — Trimmed Supabase deployment + +Goal: get the Supabase stack running with only the services OB1 actually +uses. + +Tasks: +- [ ] Based on Phase 0 inventory, modify docker-compose.yml to disable +services OB1 doesn't require. Likely candidates for removal: Realtime, +Storage, ImgProxy, Logflare, Vector +- [ ] Generate strong secrets for: Postgres password, JWT secret, anon +and service role keys, dashboard credentials. Use the Supabase secret +generation pattern; do not reuse defaults +- [ ] Configure Postgres with conservative resource settings: +max_connections=20, shared_buffers=256MB. These can be tuned upward +later if needed +- [ ] Configure Kong's routing to only expose the services that remain +- [ ] Bring up the stack with `docker compose up -d` and wait for all +services to report healthy +- [ ] Verify Postgres is reachable from inside the Docker network +- [ ] Verify GoTrue is issuing tokens by creating a test user via the auth +API +- [ ] Verify PostgREST is serving the auto-generated REST API +- [ ] Access Supabase Studio UI via Tailscale and confirm it loads + +Deliverable: a running, trimmed Supabase stack on the VM. All services +healthy. Studio UI reachable over Tailscale. + +Stop condition: confirm with user that Supabase is up and Studio is +accessible. Get approval before proceeding to Phase 3. + +## Phase 3 — OB1 schema and MCP server + +Goal: deploy OB1's database schema and MCP server on top of the +working Supabase stack. + +Tasks: +- [ ] Apply OB1's database migrations to Postgres — create the +thoughts table, any related tables, RLS policies, indexes including +the pgvector HNSW index +- [ ] Configure OB1's MCP server with the Supabase connection details, +JWT secret, and any API keys it needs (OpenAI for embeddings, or +Ollama URL for local embeddings — user preference) +- [ ] Decide with the user: cloud embeddings (OpenAI) for higher +quality, or local embeddings (Ollama on the dev rig with the 3080) for +zero cost and full data locality. Recommend local embeddings given +user's homelab philosophy and existing Ollama setup +- [ ] Run the MCP server (likely as a Docker container alongside +Supabase, or as a systemd service depending on what OB1 ships) +- [ ] Verify the MCP server starts cleanly and connects to Postgres +- [ ] Test capture: insert a test thought via the MCP server's capture +endpoint and verify it lands in the thoughts table with embedding and +metadata populated +- [ ] Test retrieval: query the MCP server's search endpoint and +verify it returns the test thought via semantic search + +Deliverable: OB1 MCP server running and operational. Test thought +captured and retrievable. + +Stop condition: confirm with user that capture and retrieval work +end-to-end before proceeding. + +## Phase 4 — Authelia integration and Traefik routing + +Goal: put OB1 behind the homelab's existing Authelia/Traefik perimeter. + +Tasks: +- [ ] Add Traefik labels (or update Traefik dynamic config) to route to: + - Supabase Studio UI (Authelia-protected) + - PostgREST API (Authelia-protected for direct access; bypassed for +MCP server which uses JWTs) + - GoTrue auth endpoints (no Authelia — these need to be reachable +for JWT operations) + - OB1 MCP server endpoint (no Authelia — clients use JWTs, not +browser sessions) +- [ ] Configure Authelia access control rules for the new routes +- [ ] Verify HTTPS works for all exposed endpoints via Traefik +- [ ] Test browser access to Studio: should hit Authelia login, then +proceed to Studio +- [ ] Test MCP server access: should bypass Authelia and authenticate +via JWT only +- [ ] Document the final URL map for the user + +Deliverable: OB1 properly fronted by Traefik with Authelia gating +browser surfaces. JWT-based clients reach the MCP server directly. + +Stop condition: confirm full access pattern works from both browser +(Authelia path) and AI client (JWT path). + +## Phase 5 — Client integration + +Goal: connect Claude Code, Claude Desktop, and the existing n8n flows to +OB1. + +Tasks: +- [ ] Generate a long-lived JWT or service-role API token for AI +client use. Store in the user's KeePassXC database +- [ ] Configure Claude Code's MCP integration to point at OB1's MCP +server. Test capture and search from a Claude Code session +- [ ] Configure Claude Desktop's MCP integration similarly. Test +capture and search from a Desktop conversation +- [ ] Add a node to the existing n8n markdown-summary intake flow that +POSTs captured content to OB1's REST API in parallel with the existing +Obsidian write. Map frontmatter fields to OB1 metadata +- [ ] Test the full capture loop: send a test summary email through +the existing flow and verify it lands in both Obsidian and OB1 +- [ ] Document the n8n flow modifications for the user + +Deliverable: OB1 receiving captures from Claude Code, Claude Desktop, +and the n8n email-intake pipeline. All three paths verified working. + +Stop condition: confirm all three capture paths work and the user can +perform an end-to-end test from any AI client. + +## Phase 6 — Operational hygiene + +Goal: make the deployment maintainable. + +Tasks: +- [ ] Set up a daily Postgres backup via pg_dump to the Ubuntu NAS +over Tailscale +- [ ] Add the OB1 VM to existing monitoring (Uptime Kuma if it's +monitoring infra services) +- [ ] Document the deploy in an Ansible playbook so future +re-provisioning is automated +- [ ] Write a brief runbook covering: how to update OB1, how to +restore from backup, how to rotate the JWT secret, how to add a new +client +- [ ] Snapshot the VM via Proxmox once everything is verified working + +Deliverable: backups running, monitoring in place, Ansible playbook +captures the deploy, runbook written, snapshot taken. + +Stop condition: project complete. Hand off to user for ongoing use. + +## Out of scope + +These are intentionally not part of this plan: + +- Building a custom technical-project extension for OB1 (separate +project, after baseline is operational) +- Adding Jenny as a second user with PBS-scoped access (separate +project, after Jenny's content workflow lands) +- Pre-compact hook for Dispatch transcript capture (separate project — +needs design work on what to capture and how to filter noise) +- Migrating existing Obsidian content into OB1 retroactively (separate +project — bulk import path) + +## Agent operating instructions + +- Work one phase at a time. Stop and report at the end of each phase. +Wait for user confirmation before proceeding to the next phase +- For any ambiguous decision, ask the user rather than guessing +- Never expose the homelab to the public internet during this +deployment. Tailscale mesh is the only access path +- All secrets generated during deployment go into the user's KeePassXC +database. Never write them to disk in plaintext outside the running +Docker environment +- Use the user's preferred tooling: Docker Compose for orchestration, +Ansible for repeatable deployment steps, UV for any Python work +- If a step fails, do not retry blindly. Report the failure with the +actual error output and propose next steps +- The user prefers concise updates — report what was done, what's +next, and any blockers. Skip the celebratory framing + + + +-- +...sent from Jenny & Travis \ No newline at end of file diff --git a/Sources/Homelab/pbs-infrastructure-intelligence.md b/Sources/Homelab/pbs-infrastructure-intelligence.md new file mode 100644 index 0000000..d5c4c7a --- /dev/null +++ b/Sources/Homelab/pbs-infrastructure-intelligence.md @@ -0,0 +1,481 @@ +--- +created: 2026-04-16 +path: Sources/Homelab +project: pbs-infrastructure-intelligence +status: active +tags: +- pbs +- monitoring +- docker +- go +- python +- mysql +- n8n +- streamlit +- security +type: project-plan +updated: 2026-04-16 +--- + +# PBS Infrastructure Intelligence (PBSII) + +## Goal + +Build a self-healing, intelligent monitoring system for the PBS +infrastructure. Four interconnected layers: a metrics collector, a +visualization dashboard, a threshold monitor, and an AI-powered healer. + +## Problem Being Solved + +- DoS attacks cause server thrashing with no early warning +- Container memory leaks go undetected until crashes occur +- Existing monitoring (Uptime Kuma) only detects hard-down states, not +degradation trends +- Manual log grepping is unsustainable as the site grows +- No visibility into what traffic reaches the server vs what Cloudflare +blocks +- Apache worker exhaustion causes memory crashes with no forensic data + +## Architecture Overview + +``` +Layer 1: Collector (Go, systemd) + → JSONL file (30s interval) + → MySQL sync (every few minutes) + → HTTP API (/metrics/current, /metrics/history, /health) + +Log Parser (Python, n8n-scheduled) + → docker logs --since + → MySQL pbs_security_events + pbs_security_summary + +Layer 2: Dashboard (Streamlit) + → Reads MySQL + → System metrics + security events + healer activity + +Layer 3: Monitor (n8n, every 5 min) + → Evaluates threshold rules against MySQL data + → Fires warnings (Google Chat) or invokes healer + +Layer 4: Healer (n8n + Claude API) + → Gathers diagnostic context from all layers + → Claude API reasons about the problem + → Executes allowed actions or escalates + → Logs all decisions to pbs_healer_actions +``` + +## Layer 1: Collector (Go Microservice) + +### Purpose + +Collect system and container metrics. Write to disk first, sync to MySQL +second. Serve current state via HTTP. No opinions, just data. + +### Technology + +- Language: Go (standard library first) +- Deployment: Single static binary, systemd service on host (not +containerized) +- Why systemd: Survives Docker daemon restarts, no Docker socket mount +needed for host metrics, resource-limited via unit file + +### Metrics Collected + +- Docker memory per container (replaces existing bash cron) +- Host total memory usage +- Host swap usage +- Host CPU usage +- Apache active worker count + +### Collection Architecture + +- Two goroutines in a single binary: + - **Collector goroutine:** Gathers metrics every 30 seconds, appends JSON +line to `/var/log/pbs-monitor/metrics.jsonl`. Cannot be slowed by MySQL. + - **Sync goroutine:** Reads new JSONL entries every few minutes, bulk +INSERTs to MySQL. If MySQL is unavailable, retries next cycle. File still +has the data. +- Log rotation via standard logrotate config + +### HTTP API Endpoints + +- `GET /metrics/current` — latest metrics snapshot +- `GET /metrics/history?range=1h` — recent history from MySQL +- `GET /health` — service health check +- Binds to localhost only; n8n reaches via Docker host bridge IP + +### Data Retention + +- Raw 30-second samples: 30 days in `pbs_system_metrics` +- Hourly rollup: kept indefinitely in `pbs_system_metrics_hourly` +- n8n handles both the hourly aggregation job and the 30-day cleanup job + +### Systemd Unit File + +```ini +[Unit] +Description=PBS Monitor - system metrics collector +After=network.target docker.service + +[Service] +Type=simple +ExecStart=/usr/local/bin/pbs-monitor +Restart=on-failure +RestartSec=5s +User=pbs-monitor +Group=pbs-monitor + +# Resource limits - blast radius containment +MemoryMax=256M +TasksMax=100 +CPUQuota=20% + +# Security hardening +NoNewPrivileges=true +ProtectSystem=strict +ProtectHome=true +ReadWritePaths=/var/log/pbs-monitor + +[Install] +WantedBy=multi-user.target +``` + +### Ansible Deployment + +- New Ansible role for systemd service deployment +- Tasks: copy binary, create config, create system user (in docker group), +install unit file, install logrotate config, enable and start service +- Follows existing staging-first deployment pattern + +## Log Parser (Python) + +### Purpose + +Parse WordPress Docker logs for security events and request patterns. Feeds +both the dashboard and Layer 3 alert rules. + +### Technology + +- Language: Python (UV + venv, PyCharm Professional) +- Orchestration: n8n schedules execution every 15 minutes +- Log source: `docker logs wordpress --since` command (no container changes +required) + +### Incremental Parsing + +- Query `MAX(logged_at)` from `pbs_security_events` to determine where to +start +- Use `--since` flag on `docker logs` to avoid reprocessing +- Self-healing on restart — just picks up from last stored timestamp + +### Fields Extracted + +- `logged_at` — timestamp +- `ip_address` — source IP +- `method` — GET/POST +- `uri` — request path +- `status_code` — HTTP response code +- `response_size` — bytes sent +- `user_agent` — client user agent +- `event_type` — derived category (e.g., `login_attempt`, `404`, `normal`) +- Note: `response_time_ms` deferred — requires custom Apache LogFormat, not +needed for v1 + +### MySQL Tables + +#### pbs_security_events (raw log entries) + +```sql +CREATE TABLE pbs_security_events ( + id BIGINT UNSIGNED AUTO_INCREMENT PRIMARY KEY, + logged_at DATETIME NOT NULL, + ip_address VARCHAR(45) NOT NULL, + method VARCHAR(10), + uri TEXT, + status_code SMALLINT, + response_size INT, + user_agent TEXT, + event_type VARCHAR(50), + created_at DATETIME DEFAULT CURRENT_TIMESTAMP, + INDEX idx_logged_at (logged_at), + INDEX idx_ip (ip_address), + INDEX idx_uri (uri(100)) +); +``` + +#### pbs_security_summary (hourly aggregates) + +```sql +CREATE TABLE pbs_security_summary ( + id BIGINT UNSIGNED AUTO_INCREMENT PRIMARY KEY, + hour_bucket DATETIME NOT NULL, + total_requests INT, + unique_ips INT, + login_attempts INT, + login_page_loads INT, + status_2xx_count INT, + status_3xx_count INT, + status_4xx_count INT, + status_404_count INT, + status_5xx_count INT, + top_ip VARCHAR(45), + top_ip_count INT, + avg_requests_per_minute DECIMAL(10,2), + peak_requests_per_minute INT, + created_at DATETIME DEFAULT CURRENT_TIMESTAMP, + UNIQUE INDEX idx_hour (hour_bucket) +); +``` + +- Hourly rollup performed by n8n workflow (separate from the parser) +- Raw events retention: grow unbounded initially, add cleanup policy when +volume is understood +- Summary table retention: indefinite + +## Layer 2: Dashboard (Streamlit) + +### Purpose + +Visualize system health and security data. Read-only, human-facing. + +### Technology + +- Streamlit, Python, reads from MySQL +- Charts via Plotly or Altair (time series) + +### Pages + +#### Page 1: System Metrics + +- Time series graphs: Docker memory per container, host memory, swap, CPU, +Apache workers +- Horizontal threshold lines showing Layer 3 warning and healer trigger +levels +- Time range selector: 24h (default), 7 days, 30 days + +#### Page 2: Security / Log Events + +- wp-login.php attempts over time +- Top attacking IPs (table) +- Status code distribution +- Request volume trends (avg and peak requests/min) + +#### Page 3: Healer Activity + +- Log of healer invocations: trigger, decision, action, outcome +- Timeline view of actions taken +- Deferred actions and their resolution (executed, auto-cancelled, +overridden) + +## Layer 3: Monitor (n8n) + +### Purpose + +Early warning system. Detects degradation trends before they become +outages. Evaluates rules, fires alerts or invokes the healer. + +### Mechanism + +- n8n workflow, scheduled every 5 minutes +- Queries MySQL for recent metrics +- Evaluates threshold rules +- Works alongside Uptime Kuma (not replacing it for now) + +### Starter Rules + +| Rule | Threshold | Action | +|---|---|---| +| Host RAM usage | >85% sustained 10 min | Google Chat warning | +| Host RAM usage | >95% sustained 5 min | Invoke healer | +| Swap usage | >50% of total swap | Google Chat warning | +| Container memory | >90% of cap sustained 5 min | Google Chat warning | +| Container memory | >98% of cap sustained 3 min | Invoke healer | +| Host CPU | >80% sustained 15 min | Google Chat warning (info only) | +| Apache workers | >80% of max sustained 10 min | Google Chat warning | +| wp-login.php POSTs | >30 in 5-min window | Google Chat warning | + +"Sustained" = all samples in the window exceed the threshold. Prevents +single-spike false positives. + +### Alert Tiers + +- **Warning** — Google Chat notification, no action taken +- **Invoke Healer** — hands off to Layer 4 with trigger context + +## Layer 4: Healer (n8n + Claude API) + +### Purpose + +Take corrective action when Layer 3 invokes it. Reason about the problem, +execute a safe response, report results. + +### Mechanism + +- n8n workflow triggered by Layer 3 or Uptime Kuma webhooks +- Portainer REST API for container operations +- Claude API for diagnostic reasoning + +### Available Actions + +| Action | Timing | Safety Rails | +|---|---|---| +| Container restart (non-protected) | Immediate | Max 1/hour, 3/24h per +container | +| WordPress restart | Deferred to quiet hours | Same caps + admin override +window | +| MySQL restart | Deferred to quiet hours | Same caps + admin override +window | +| Server reboot | Deferred to quiet hours | Max 1/24h + admin override +window | +| Alert only | Immediate | Always an option | +| Escalate | Immediate | When limits exceeded | + +**Protected containers** (deferred to quiet hours only): WordPress, MySQL + +### Quiet Hours + +- Window: 0200–0500 ET +- Deferred actions are standing orders, not scheduled events +- During quiet hours, n8n checks: is there a pending action? Is it still +pending (not overridden)? Do the original conditions still exist? +- If conditions resolved → auto-cancel with Google Chat notification +- If overridden → mark as overridden + +### Deferred Action Flow + +1. Healer decides a protected container or server needs restart +2. Sets flag in `pbs_healer_actions`: `deferred_action`, `deferred_status = +'pending'` +3. Google Chat notification with action ID and reasoning +4. During quiet hours, n8n evaluates: + - Pending action exists? → Check if overridden → Check if conditions +persist → Execute or auto-cancel +5. Status transitions: `pending` → `executed` | `auto_cancelled` | +`overridden` + +### Override Mechanism + +#### V1 + +- **pbs-hub web page:** Shows pending healer actions, click to +cancel/approve +- **Google Chat webhook:** Text command with action ID (e.g., "override +1234", "cancel 1234") +- Both hit the same pbs-hub API endpoint +- pbs-hub API also available for future integrations + +#### V2 (future) + +- Google Chat interactive cards with approve/cancel/defer buttons + +### Iteration Limits (Anti-Loop Protection) + +- Per-container cooldown: 1 hour between automatic restarts +- Per-container daily cap: 3 automatic restarts per 24 hours +- Global healer cap: 10 healer-initiated actions per 24 hours +- Cooldown on escalation: 6 hours of no auto-action for that container +after escalation +- If any limit hit → escalate instead of acting + +### Healer Decision Flow + +1. Receive trigger from Layer 3 or Uptime Kuma +2. Check iteration limits → if exceeded, escalate and stop +3. Gather context: + - Current metrics from pbs-monitor HTTP API + - Recent log events from pbs_security_events + - Container inspect + recent logs from Portainer API +4. Send structured context to Claude API +5. Claude returns: recommended action, reasoning, confidence +6. If action is allowed AND confidence is high → execute (or defer if +protected) +7. Else → escalate with Claude's reasoning +8. Log decision + action + outcome to `pbs_healer_actions` +9. Google Chat notification with: what happened, what healer did (or +didn't), Claude's reasoning + +### Claude API Prompt Shape + +- Structured input: trigger reason, current metrics snapshot, 30-min +history, recent logs, container inspect +- Structured output: diagnosis, recommended action (enum: +`restart_container`, `defer_restart`, `reboot_server`, `no_action`, +`escalate`), reasoning, confidence level + +### pbs_healer_actions Table + +```sql +CREATE TABLE pbs_healer_actions ( + id BIGINT UNSIGNED AUTO_INCREMENT PRIMARY KEY, + triggered_at DATETIME NOT NULL, + trigger_source VARCHAR(50), + trigger_rule VARCHAR(100), + container_name VARCHAR(100), + metrics_snapshot JSON, + claude_reasoning TEXT, + action_recommended VARCHAR(50), + action_taken VARCHAR(50), + action_result TEXT, + deferred_action VARCHAR(50), + deferred_status VARCHAR(20) DEFAULT NULL, + iteration_limit_hit BOOLEAN DEFAULT FALSE, + escalated BOOLEAN DEFAULT FALSE, + created_at DATETIME DEFAULT CURRENT_TIMESTAMP, + INDEX idx_triggered (triggered_at), + INDEX idx_container (container_name), + INDEX idx_deferred (deferred_status) +); +``` + +### Starting Conservatism + +- First period (target ~30 days): **dry-run mode** — healer makes +decisions, logs them, sends Google Chat with "I would have done X" but does +not execute +- After validation: flip to live mode, container restarts first +- Server reboot: enabled after container restart trust is established +- System reboot scheduling added only after override mechanism is validated + +## Existing Infrastructure Context + +- Docker/Traefik/Linode production stack +- Portainer running with REST API available +- Uptime Kuma operational for up/down checks and Google Chat alerts +- `pbs_automation` MySQL database exists with separate permissions +- n8n operational, Claude API integration is a known pattern +- Existing bash cron (60s, Docker memory only, flat file) — replaced by +Layer 1 +- Crowdsec security hardening is a separate project (not duplicated here) + +## Implementation Order + +- [ ] Phase 1: Layer 1 — Go collector (systemd, JSONL, MySQL sync, HTTP API) +- [ ] Phase 2: Log Parser — Python WordPress log parser + MySQL schema +- [ ] Phase 3: Layer 2 — Streamlit dashboard (system metrics + security + +thresholds) +- [ ] Phase 4: Layer 3 — n8n monitor workflow (threshold rules, Google Chat +alerts) +- [ ] Phase 5: Layer 4 — n8n healer workflow (dry-run mode, Claude API +integration) +- [ ] Phase 6: Layer 4 live — enable live actions, override page in pbs-hub +- [ ] Phase 7: Deferred actions — quiet hours logic, WordPress/MySQL/server +restart scheduling + +## Open Questions + +- [ ] Go project structure — standard layout or flat? Decide during Phase 1 +- [ ] Docker multi-stage build for Go binary — needed for CI, decide during +Phase 1 +- [ ] MySQL schema for `pbs_system_metrics` and `pbs_system_metrics_hourly` +— design during Phase 1 +- [ ] Apache max_workers value — needed to calculate percentage for Layer 3 +rules +- [ ] Portainer API auth — how is it currently secured? Needed for Layer 4 +- [ ] Claude API prompt engineering — detailed prompt design during Phase 5 +- [ ] pbs-hub override page routing — how does it fit into existing pbs-hub +Flask app? +- [ ] Google Chat webhook for override commands — new webhook or extend +existing? +- [ ] Streamlit deployment — containerized or host-level? Decide during +Phase 3 + +...sent from Jenny & Travis \ No newline at end of file diff --git a/Sources/Homelab/pbs-security-hardening.md b/Sources/Homelab/pbs-security-hardening.md new file mode 100644 index 0000000..7795379 --- /dev/null +++ b/Sources/Homelab/pbs-security-hardening.md @@ -0,0 +1,382 @@ +--- +created: 2026-04-02 +path: Sources/Homelab +project: pbs-security-hardening +status: active +tags: +- pbs +- docker +- traefik +- security +- crowdsec +- authelia +- cloudflare +- staging +- production +- automation +- n8n +type: project-plan +updated: 2026-04-02 +--- + +# PBS Security Hardening - Unified Project Plan + +## Project Goal + +Deploy a layered security stack for PBS infrastructure across staging and +production on Linode. Two components: + +1. **Crowdsec** — Perimeter security with intelligent alerting, replacing +manual Wordfence email monitoring +2. **Authelia** — SSO with 2FA for all admin/infrastructure services + +**Approach:** Deploy and validate on staging first, then roll to production. +--- +## Current Security Baseline + +### What's in place today +- Cloudflare: Edge WAF, DNS, CDN, Bot Fight Mode +- Wordfence: WordPress-level firewall, login lockout (5 attempts), malware +scanning +- 2FA: Enabled on WordPress admin accounts +- fail2ban: Server-level SSH protection +- UFW: Firewall with Docker-aware rules in after.rules +- Cloudflare proxy disabled for n8n and Gitea subdomains + +### Problems being solved +- ~20 lockout notification emails per day (noise) +- No automated intelligent blocking — manual review required +- Targeted login attempts using real admin usernames (concerning) +- No SSO — separate credentials for Portainer, Uptime Kuma, n8n, pbs-hub, +phpMyAdmin +- No confidence that attacks aren't slipping through +--- +## Architecture Overview + +### Traffic flow after implementation + +Internet → Cloudflare (edge WAF/CDN) + → Traefik + → Crowdsec Bouncer (block/allow decision) + → Authelia (SSO check for admin services) + → Service (Portainer, n8n, pbs-hub, etc.) + → WordPress (own auth, Wordfence for malware scanning only) + + +### Tiered alerting model (Crowdsec) +- **Tier 1 — Silent block:** Generic bot attempts (admin, test, wp-admin). +Automatic. No notification. +- **Tier 2 — Logged, weekly review:** Coordinated bot patterns, repeated +IPs. Crowdsec handles it. +- **Tier 3 — Immediate alert:** Real admin username targeted. Requires +WordPress auth log as additional Crowdsec data source. Notification via n8n +→ Google Chat. +- **Tier 4 — Critical alert:** Successful login from unknown IP, failed 2FA +on real account, file integrity changes. + +### Services protected by Authelia +- Portainer +- Uptime Kuma +- n8n +- pbs-hub +- phpMyAdmin +- Traefik dashboard + +### Services NOT behind Authelia +- WordPress (uses its own auth + Wordfence + 2FA) +--- +## Phase 1: Crowdsec on Staging + +### Estimated time: 3-4 hours + +### Step 1.1: Deploy Crowdsec container +- [ ] Add Crowdsec service to staging docker-compose.yml +- [ ] Configure to read Traefik access logs +- [ ] Install WordPress collection (crowdsecurity/wordpress) +- [ ] Install Traefik collection (crowdsecurity/traefik) +- [ ] Verify container starts and parses logs + +### Step 1.2: Install Traefik Bouncer +- [ ] Add Crowdsec Traefik bouncer plugin to Traefik config +- [ ] Configure bouncer API key +- [ ] Verify bouncer communicates with Crowdsec +- [ ] Test: manually ban an IP via `cscli`, confirm it gets blocked at +Traefik + +### Step 1.3: Add WordPress auth log source +- [ ] Add WPCode snippet or lightweight plugin to log failed logins with +usernames to a file +- [ ] Mount log file into Crowdsec container +- [ ] Configure Crowdsec to parse WordPress auth log as additional +datasource +- [ ] Verify Crowdsec sees username-level login data + +### Step 1.4: Configure community blocklists +- [ ] Register free Crowdsec console account +- [ ] Enroll staging instance +- [ ] Enable community blocklists +- [ ] Verify blocked IPs appear from community intelligence + +### Step 1.5: Validate on staging +- [ ] Simulate brute force from test IP — confirm auto-block +- [ ] Simulate real username attempt — confirm Tier 3 detection +- [ ] Verify WordPress still accessible for normal logins +- [ ] Verify admin tools still accessible +- [ ] Check `cscli metrics` for activity +- [ ] Check `cscli alerts list` for decisions +--- +## Phase 2: Crowdsec Alerting via n8n + +### Estimated time: 2-3 hours + +### Step 2.1: Crowdsec notification setup +- [ ] Configure Crowdsec HTTP notification plugin +- [ ] Point notifications at n8n webhook endpoint +- [ ] Define alert profiles matching tiered model: + - Tier 1: No notification (silent block) + - Tier 2: Logged only (weekly digest — future enhancement) + - Tier 3+: Immediate webhook to n8n + +### Step 2.2: n8n alerting workflow +- [ ] Create n8n workflow: Crowdsec webhook → parse alert → route by +severity +- [ ] Tier 3 alerts → Google Chat `webadmin` space via Notify Travis +sub-workflow +- [ ] Tier 4 alerts → Google Chat `webadmin` space with urgent flag +- [ ] Include: IP, reason, username (if available), timestamp, action taken +- [ ] Test end-to-end: trigger alert → verify Google Chat notification + +### Step 2.3: Validate alerting +- [ ] Confirm Tier 1 produces no notification +- [ ] Confirm Tier 3 sends Google Chat alert with username info +- [ ] Confirm Tier 4 sends urgent alert +- [ ] Verify no false positives from normal browsing +--- +## Phase 3: Authelia on Staging + +### Estimated time: 3-4 hours + +### Step 3.1: Directory and secrets setup +- [ ] Create directory structure on staging: `/opt/docker/authelia/config/` +- [ ] Generate secrets: JWT, session, storage encryption, OIDC HMAC +- [ ] Generate password hashes for Travis and Jenny (argon2id) +- [ ] Create users_database.yml with both accounts + +### Step 3.2: Authelia configuration +- [ ] Write configuration.yml: + - Session domain: staging domain + - Default policy: two_factor + - Storage: SQLite (local file) + - Notifier: filesystem (for staging — SMTP for production later) + - TOTP settings: SHA1, 6 digits, 30 second period +- [ ] Define access control rules: + - Portainer, Uptime Kuma, n8n, pbs-hub, phpMyAdmin, Traefik dashboard → +two_factor + - WordPress and public routes → bypass + +### Step 3.3: Docker and Traefik integration +- [ ] Create Authelia docker-compose.yml +- [ ] Add Authelia to Traefik network +- [ ] Configure Traefik forwardAuth middleware pointing to Authelia +- [ ] Add middleware labels to all protected services +- [ ] Leave WordPress compose labels unchanged + +### Step 3.4: Deploy and test +- [ ] Start Authelia container +- [ ] Navigate to a protected service — verify redirect to Authelia login +- [ ] Login with Travis account — verify redirect back to service +- [ ] Register TOTP device +- [ ] Verify 2FA prompt works +- [ ] Test SSO: login once, access other protected services without re-auth +- [ ] Verify WordPress login is unaffected +- [ ] Verify public site is unaffected +--- +## Phase 4: Integration Testing on Staging + +### Estimated time: 2 hours + +### Step 4.1: Crowdsec + Authelia together +- [ ] Verify Crowdsec bouncer doesn't interfere with Authelia redirects +- [ ] Verify blocked IPs get Crowdsec block page, not Authelia login +- [ ] Test: brute force Authelia login → Crowdsec detects and blocks +- [ ] Verify request flow: Cloudflare → Traefik → Crowdsec → Authelia → +Service + +### Step 4.2: Full stack validation +- [ ] All protected services accessible after Authelia login + 2FA +- [ ] WordPress admin login works independently +- [ ] Public site unaffected +- [ ] Crowdsec blocking bots silently +- [ ] Tier 3/4 alerts arriving in Google Chat +- [ ] No performance degradation +--- +## Phase 5: Roll to Production + +### Estimated time: 2-3 hours + +### Step 5.1: Pre-production prep +- [ ] Document all staging config that worked +- [ ] Generate fresh secrets for production (never reuse staging secrets) +- [ ] Update Authelia session domain to production domain +- [ ] Switch Authelia notifier from filesystem to SMTP (Google Workspace) +- [ ] Update Crowdsec webhook URL to production n8n + +### Step 5.2: Deploy Crowdsec to production +- [ ] Add Crowdsec to production docker-compose.yml +- [ ] Add Traefik bouncer plugin +- [ ] Add WordPress auth log source +- [ ] Enroll in Crowdsec console +- [ ] Verify blocking and alerting work + +### Step 5.3: Deploy Authelia to production +- [ ] Add Authelia to production docker-compose.yml +- [ ] Add forwardAuth middleware to all protected services +- [ ] Verify SSO and 2FA work +- [ ] Register TOTP devices for Travis and Jenny + +### Step 5.4: Post-deployment +- [ ] Monitor for 48 hours — watch for false positives +- [ ] Tune Crowdsec scenarios if needed +- [ ] Verify Wordfence lockout emails have dropped significantly +- [ ] Demote Wordfence to malware-scanning-only role +- [ ] Consider disabling Wordfence firewall features (now redundant) +--- +## Phase 6: Codify in Ansible + +### Estimated time: 3-4 hours + +### Step 6.1: Crowdsec Ansible role +- [ ] Create Crowdsec tasks with Ansible tags (`--tags crowdsec`) +- [ ] Template docker-compose.yml with variables +- [ ] Template Crowdsec acquis.yaml (log sources) +- [ ] Store bouncer API key in ansible-vault +- [ ] Store console enrollment key in ansible-vault + +### Step 6.2: Authelia Ansible role +- [ ] Create Authelia tasks with Ansible tags (`--tags authelia`) +- [ ] Template configuration.yml with variables +- [ ] Template users_database.yml +- [ ] Store all secrets in ansible-vault +- [ ] Add forwardAuth middleware labels to protected service templates + +### Step 6.3: Validate +- [ ] Destroy staging and rebuild from Ansible +- [ ] Verify Crowdsec + Authelia come up correctly +- [ ] Verify alerting works after rebuild +- [ ] Verify SSO works after rebuild +--- +## Wordfence Transition Plan + +After Crowdsec is stable on production: + +- [ ] Disable Wordfence firewall features (redundant with Crowdsec) +- [ ] Keep Wordfence for: scheduled malware scans (monthly), file integrity +checks +- [ ] Disable Wordfence login protection (Crowdsec handles this now) +- [ ] Disable lockout email notifications (the whole point!) +- [ ] Document remaining Wordfence role in tech wiki +- [ ] Long-term: when WordPress is replaced, Wordfence goes away entirely +--- +## Security Maintenance Cadence + +After full deployment: + +### Daily (glance, 1 min) +- Check Uptime Kuma — green means good +- Respond to any Tier 3/4 Google Chat alerts + +### Weekly (10 min) +- Review Crowdsec console dashboard for trends +- Check `cscli metrics` for volume patterns +- Update WordPress plugins on staging, then production + +### Monthly (30 min) +- Run Wordfence malware scan +- Review WordPress user accounts +- Check Docker image updates (Traefik, WordPress, Crowdsec, Authelia) +- Review Cloudflare analytics +- Spot-check Crowdsec community blocklist enrollment +--- +## Troubleshooting + +### Crowdsec not blocking +bash +# Check if bouncer is connected +cscli bouncers list + +# Check if decisions exist +cscli decisions list + +# Check log parsing +cscli metrics + +# Manually test a ban +cscli decisions add --ip 1.2.3.4 --reason "test ban" --duration 5m + + +### Authelia not redirecting +bash +# Check Authelia logs +docker logs authelia + +# Verify Traefik middleware +docker exec traefik wget -qO- http://localhost:8080/api/http/middlewares + +# Check forwardAuth is reaching Authelia +docker logs authelia | grep "authorization" + + +### False positives +bash +# Whitelist your own IP +cscli decisions delete --ip YOUR_IP +cscli parsers install crowdsecurity/whitelists +# Add your IP to /etc/crowdsec/parsers/s02-enrich/whitelists.yaml + + +### Locked out of Authelia +bash +# Reset user password +docker exec authelia authelia hash-password -- 'newpassword' +# Update users_database.yml with new hash +docker restart authelia + +--- +## Resources + +### Crowdsec +- Official docs: https://docs.crowdsec.net/ +- Traefik bouncer: +https://github.com/maxlerebourg/crowdsec-bouncer-traefik-plugin +- WordPress collection: +https://hub.crowdsec.net/author/crowdsecurity/collections/wordpress +- Console: https://app.crowdsec.net/ + +### Authelia +- Official docs: https://www.authelia.com/ +- Configuration reference: +https://www.authelia.com/configuration/prologue/introduction/ +- Traefik integration: https://www.authelia.com/integration/proxies/traefik/ +- File-based users: https://www.authelia.com/reference/guides/passwords/ + +### Traefik +- ForwardAuth middleware: +https://doc.traefik.io/traefik/middlewares/http/forwardauth/ +- Plugin catalog: https://plugins.traefik.io/ +--- +## Success Criteria + +- [ ] Wordfence lockout emails drop to near zero +- [ ] Crowdsec silently blocking bot traffic at Traefik layer +- [ ] Tier 3/4 alerts arriving in Google Chat when real threats detected +- [ ] All admin services behind Authelia SSO with 2FA +- [ ] WordPress auth unaffected +- [ ] Public site unaffected +- [ ] Everything codified in Ansible and reproducible +- [ ] Daily security overhead reduced from 15+ minutes to under 1 minute +--- +*Last Updated: April 2, 2026* +*Maintained by: Travis* +*Project: Plant Based Southerner Infrastructure* + +...sent from Jenny & Travis \ No newline at end of file diff --git a/Sources/Homelab/server-stability-and-security-hardening.md b/Sources/Homelab/server-stability-and-security-hardening.md new file mode 100644 index 0000000..2c72f06 --- /dev/null +++ b/Sources/Homelab/server-stability-and-security-hardening.md @@ -0,0 +1,248 @@ +--- +created: 2026-03-23 +path: Sources/Homelab +project: server-stability-and-security-hardening +status: active +tags: +- pbs +- docker +- production +- staging +- wordpress +- traefik +- cloudflare +- security +type: session-notes +updated: 2026-03-23 +--- + +# Server Stability, Security Hardening & Staging Fixes - March 23, 2026 + +## Session Summary + +Marathon session covering three major areas: (1) production server crash investigation and MySQL/WordPress memory capping, (2) staging Traefik upgrade and debugging, and (3) Cloudflare security and caching improvements. Two server crashes in 48 hours traced to MySQL OOM kills, with a third event tonight traced to WordPress memory bloat caused by bot traffic bursts. All three issues now mitigated with layered defenses. + +--- + +## Part 1: Production — MySQL OOM Investigation & Fix + +### Root Cause Confirmed +Both crashes (Saturday 3/22 ~6AM ET, Monday 3/23 ~6:20AM ET) were caused by MySQL being OOM-killed by the Linux kernel. Confirmed via `journalctl`: +- Saturday: `Out of memory: Killed process 4138817 (mysqld) total-vm:1841380kB` +- Monday: `Out of memory: Killed process 13015 (mysqld) total-vm:1828060kB` +- Both followed same pattern: MySQL OOM-killed → Docker restarts → system still starved → swapoff killed → cascading failure → manual Linode reboot + +### Server Timezone Note +Production server runs in **UTC**. Subtract 4 hours for Eastern time. Both crashes appeared as ~10AM UTC in logs but were ~6AM Eastern. + +### Journal Persistence Confirmed +- `/var/log/journal` exists and journals survive reboots +- `journalctl --list-boots` shows 5 boot sessions back to May 2025 +- For large time ranges, use `--since`/`--until` flags to avoid hanging + +### Investigation Results +- **WooCommerce Action Scheduler:** Cleared — all tasks showed completed status +- **Wordfence Scans:** Scan log showed ~1 minute scan on 3/19 at 10PM ET — doesn't align with crash window; scan schedule is automatic on free tier (no manual control) +- **htop threads:** Multiple MySQL rows in htop are threads, not processes — press `H` to toggle thread view + +### MySQL Memory Cap Applied +Added to `mysql` service in `/opt/docker/wordpress/compose.yml`: + +```yaml +mysql: + image: mysql:8.0 + container_name: wordpress_mysql + restart: unless-stopped + deploy: + resources: + limits: + memory: 768M + reservations: + memory: 256M + command: >- + --default-authentication-plugin=mysql_native_password + --innodb-buffer-pool-size=256M + --innodb-log-buffer-size=16M + --max-connections=50 + --key-buffer-size=16M + --tmp-table-size=32M + --max-heap-table-size=32M + --table-open-cache=256 + --performance-schema=OFF +``` + +**Key tuning notes:** +- `performance-schema=OFF` saves ~200-400MB alone +- `max-connections=50` reduced from default 151 +- `innodb-buffer-pool-size=256M` caps InnoDB's biggest memory consumer + +**Result:** MySQL dropped from 474MB (uncapped) to ~225MB (capped at 768MB, using 29% of cap) + +### Memory Monitoring Script Deployed +Created `/usr/local/bin/docker-mem-log.sh` — logs per-container memory every 5 minutes: + +```bash +#!/bin/bash +LOG_FILE="/var/log/pbs-monitoring/container-memory.log" +echo "$(date -u '+%Y-%m-%d %H:%M:%S UTC') | $(docker stats --no-stream --format '{{.Name}}:{{.MemUsage}}' | tr '\n' ' ')" >> "$LOG_FILE" +``` + +Cron: `/etc/cron.d/docker-mem-monitor` +``` +*/5 * * * * root /usr/local/bin/docker-mem-log.sh +``` + +Check with: `tail -20 /var/log/pbs-monitoring/container-memory.log` + +--- + +## Part 2: Production — WordPress Memory Spike & Bot Traffic Discovery + +### Memory Monitoring Pays Off +The monitoring script caught a WordPress memory spike in real time: + +| Time (UTC) | WordPress | MySQL | +|---|---|---| +| 02:15 | 1.12 GB | 245 MB | +| 02:20 | **2.34 GB** | 178 MB | +| 02:30 | **2.91 GB** | 141 MB | + +### Root Cause: Bot Traffic Burst +WordPress access logs at 02:16:59 UTC showed ~10+ simultaneous requests in 3 seconds: +- Multiple IPs hitting homepage simultaneously via Cloudflare +- Requests for random `.flac` and `.webm` files (classic bot probing) +- All using `http://` referrer (not `https://`) — not legitimate traffic +- Mix of spoofed user agents designed to look like different browsers +- Each uncached request spawned a PHP process, causing WordPress to spike to 2.9GB + +### WordPress Memory Cap Applied +Added to `wordpress` service in `/opt/docker/wordpress/compose.yml`: + +```yaml +deploy: + resources: + limits: + memory: 2000M +``` + +**Result:** WordPress now capped at ~2GB, currently running at ~866MB (43% of cap) + +### Cloudflare Traffic Analysis +24-hour stats showed 11.72k total requests with **10.4k uncached (89%)**. Two visible traffic spikes aligned with crash events. + +--- + +## Part 3: Cloudflare Security & Caching Hardening + +### Security Changes +1. **Bot Fight Mode** — Enabled (Security → Settings) +2. **WAF Rule: Block suspicious file probes** — Blocks requests ending in `.flac`, `.webm`, `.exe`, `.dll` +3. **Rate Limiting Rule: Homepage spam** — 30 requests per 10 seconds per IP, blocks for 10 seconds + +### Caching Changes +1. **Browser Cache TTL** — Increased from 4 hours to 1 day +2. **Always Online** — Enabled (serves cached pages when server is down) +3. **Cache Rule** — Applied Cloudflare "Cache Everything" template: + - Cache eligibility: Eligible for cache + - Edge TTL: Overrides origin cache-control headers + - Browser TTL: Set + - Serve stale while revalidating: Enabled + +**Important:** After publishing new content, purge cache via Caching → Configuration → Purge Cache + +--- + +## Part 4: Staging — Traefik Upgrade & Debugging + +### Docker API Version Mismatch +`apt-get upgrade` on staging updated Docker Engine to v29.2.1 (API v1.53, minimum client API v1.44). Traefik v3.5's built-in Docker client only spoke API v1.24 → Docker rejected all Traefik requests → entire site down. + +**Fix:** Updated Traefik from `v3.5` to `v3.6.11` +- v3.6.11 includes Docker API auto-negotiation fix +- Also patches 3 CVEs (CVE-2026-32595, CVE-2026-32305, CVE-2026-32695) + +**Production impact:** Must update Traefik on production **before** running `apt-get upgrade`, or the same break will occur. Update Traefik first, then Docker. + +### WordPress Unhealthy Container Issue +After Traefik upgrade, WordPress showed as "unhealthy" → Traefik v3.6 respects Docker health status and skips unhealthy containers → site returned 404. + +**Root cause:** MySQL `.env` password contained `$` character, which Docker compose interprets as variable substitution. Password was silently corrupted → WordPress couldn't connect to MySQL → healthcheck failed → Traefik wouldn't route. + +**Fix:** Escaped `$` characters in `.env` file. For future reference: `$` must be doubled (`$$`) in Docker `.env` files. + +**Lesson:** Traefik v3.6+ skips unhealthy containers entirely — they won't show up as routers in the dashboard. + +### PBS Manager Web App (Staging) +- Healthcheck using `curl` fails on `python:3.13-slim` (curl not installed) +- Fix: Use Python-based healthcheck instead: +```yaml +healthcheck: + test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:5000/api/health')"] + interval: 30s + timeout: 10s + retries: 3 + start_period: 30s +``` +- Code changes require `docker compose up -d --build` (not just `--force-recreate`) +- SQLAlchemy models must stay in sync with database schema changes + +--- + +## Layered Defense Summary + +| Layer | What It Does | Status | +|---|---|---| +| Cloudflare Bot Fight Mode | Auto-blocks known bots | ✅ Enabled | +| Cloudflare WAF rules | Blocks file probes (.flac, .webm, .exe, .dll) | ✅ Deployed | +| Cloudflare Rate Limiting | 30 req/10s per IP on homepage | ✅ Deployed | +| Cloudflare Caching | Cache everything, serve stale while revalidating | ✅ Deployed | +| Cloudflare Always Online | Serves cached site during outages | ✅ Enabled | +| WordPress memory cap | 2GB limit prevents runaway PHP | ✅ Applied | +| MySQL memory cap | 768MB limit with tuned buffers | ✅ Applied | +| Memory monitoring | Logs per-container stats every 5 min | ✅ Running | +| Journal persistence | OOM kill logs survive reboots | ✅ Confirmed | + +--- + +## Current Production Memory Snapshot (post-fixes) + +| Container | Memory | Limit | % of Limit | +|---|---|---|---| +| wordpress | 866 MB | 2,000 MB | 43% | +| n8n | 341 MB | System | 9% | +| wordpress_mysql | 190 MB | 768 MB | 25% | +| uptime-kuma | 124 MB | System | 3% | +| traefik | 56 MB | System | 1% | +| redis | 17 MB | 640 MB | 3% | +| wpcron | 16 MB | System | <1% | +| pbs-api | 14 MB | System | <1% | +| **Total** | **~1.62 GB** | | | + +--- + +## Still Open + +- [ ] Monitor overnight stability — check memory logs tomorrow AM +- [ ] Monitor Cloudflare cache hit rate over next 24 hours (should improve dramatically) +- [ ] Add log rotation for `/var/log/pbs-monitoring/container-memory.log` +- [ ] Update Traefik on production to v3.6.11 **before** running `apt-get upgrade` +- [ ] Disable `apt-daily.service` on production (automatic unattended updates) +- [ ] Investigate Cloudflare cache hit rate for wp-admin bypass if admin pages serve stale content +- [ ] Server sizing discussion still open — 4GB may be tight for Gitea + Authelia +- [ ] PBS Manager web app healthcheck and basicauth fixes on staging +- [ ] Consider Watchtower on staging only as a canary (discussed and decided against for production) + +--- + +## Key Learnings + +- **Docker `.env` files treat `$` as variable substitution** — double it (`$$`) or avoid `$` in passwords entirely +- **Traefik v3.6+ skips unhealthy containers** — if a container's healthcheck fails, Traefik won't route to it (no error, just missing from dashboard) +- **`docker compose up -d --force-recreate`** only recreates from existing image; use `--build` for code changes +- **Docker API versions ≠ Docker product versions** — API v1.24 vs v1.44 are protocol versions, not Docker Engine versions +- **`performance-schema=OFF`** in MySQL saves ~200-400MB with no downside for WordPress +- **89% uncached Cloudflare traffic** was caused by WordPress sending `no-cache` headers — override with Edge TTL rule +- **Bot traffic patterns:** simultaneous requests from multiple IPs, random file probes, `http://` referrers, mixed user agents +- **Memory monitoring script** proved its value immediately — caught WordPress spike in real time +- **Watchtower not recommended for production** — prefer deliberate manual updates tested on staging first +- **Always update Traefik before Docker Engine** — newer Docker can require minimum API versions that old Traefik can't speak \ No newline at end of file diff --git a/Sources/Homelab/server-stability-mysql-oom.md b/Sources/Homelab/server-stability-mysql-oom.md new file mode 100644 index 0000000..0925527 --- /dev/null +++ b/Sources/Homelab/server-stability-mysql-oom.md @@ -0,0 +1,146 @@ +--- +created: 2026-03-23 +path: Sources/Homelab +project: server-stability-mysql-oom +status: active +tags: +- pbs +- docker +- production +- wordpress +type: session-notes +updated: 2026-03-23 +--- + +# Server Stability - MySQL OOM Fix & Memory Monitoring + +## Summary + +Two server crashes in 48 hours (Saturday March 22 ~6AM ET, Monday March 23 ~6:20AM ET) traced to MySQL being OOM-killed by the Linux kernel. Root cause: MySQL had no memory limits and was consuming ~1.8GB before the OOM killer intervened, triggering a cascading failure that made the server completely unresponsive. + +## Investigation Findings + +### OOM Kill Evidence (from systemd journal) +- **Saturday crash:** `Out of memory: Killed process 4138817 (mysqld) total-vm:1841380kB` +- **Monday crash:** `Out of memory: Killed process 13015 (mysqld) total-vm:1828060kB` +- Both crashes followed the same pattern: MySQL OOM-killed → Docker restarts MySQL → system still memory-starved → swapoff killed → complete server lockup → manual Linode reboot required + +### Crash Timeline +- Both crashes occurred around 6:00-6:20 AM Eastern (10:00-10:20 UTC — server runs in UTC) +- WooCommerce installed Saturday — first crash Saturday night, second Monday morning +- WooCommerce Action Scheduler showed no failed/stuck tasks — likely not the direct trigger +- Wordfence scan logs showed a ~1 minute scan on March 19 at ~10PM ET — does not align with crash window +- Wordfence scan scheduling is automatic on free tier (no manual schedule control) + +### Ruled Out +- WooCommerce Action Scheduler runaway tasks (all showed completed status) +- Wordfence scan timing (didn't align with crash window) +- Multiple MySQL instances (htop showed threads, not separate processes — press `H` in htop to toggle thread view) + +### Not Yet Determined +- Exact trigger causing MySQL to balloon to 1.8GB overnight +- Whether WooCommerce's added baseline DB load is the tipping point +- `apt-daily.service` was running during Monday's crash — may be contributing to memory pressure + +## Changes Made + +### MySQL Memory Cap & Tuning (compose.yml) +Added to the `mysql` service in `/opt/docker/wordpress/compose.yml`: + +```yaml + mysql: + image: mysql:8.0 + container_name: wordpress_mysql + restart: unless-stopped + deploy: + resources: + limits: + memory: 768M + reservations: + memory: 256M + environment: + MYSQL_DATABASE: wordpress + MYSQL_USER: wordpress + MYSQL_PASSWORD: ${MYSQL_PASSWORD} + MYSQL_ROOT_PASSWORD: ${MYSQL_ROOT_PASSWORD} + volumes: + - mysql_data:/var/lib/mysql + networks: + - internal + command: >- + --default-authentication-plugin=mysql_native_password + --innodb-buffer-pool-size=256M + --innodb-log-buffer-size=16M + --max-connections=50 + --key-buffer-size=16M + --tmp-table-size=32M + --max-heap-table-size=32M + --table-open-cache=256 + --performance-schema=OFF +``` + +**What each setting does:** +- `limits: memory: 768M` — Docker kills MySQL if it exceeds 768MB (controlled restart vs kernel OOM) +- `reservations: memory: 256M` — Guarantees MySQL gets at least 256MB +- `innodb-buffer-pool-size=256M` — Caps InnoDB cache (MySQL's biggest memory consumer) +- `max-connections=50` — Reduced from default 151 (less memory per connection) +- `performance-schema=OFF` — Saves ~200-400MB (internal MySQL monitoring not needed) + +**Result:** +| Metric | Before | After | +|--------|--------|-------| +| MySQL memory usage | 474MB (uncapped, spiked to 1.8GB) | 225MB (capped at 768MB) | +| MySQL % of cap | N/A | 29% | +| Total stack memory | ~2.05GB | ~2.0GB | + +### Memory Monitoring Script +Created `/usr/local/bin/docker-mem-log.sh` — logs per-container memory usage every 5 minutes via cron: + +```bash +#!/bin/bash +LOG_FILE="/var/log/pbs-monitoring/container-memory.log" +echo "$(date -u '+%Y-%m-%d %H:%M:%S UTC') | $(docker stats --no-stream --format '{{.Name}}:{{.MemUsage}}' | tr '\n' ' ')" >> "$LOG_FILE" +``` + +Cron entry at `/etc/cron.d/docker-mem-monitor`: +``` +*/5 * * * * root /usr/local/bin/docker-mem-log.sh +``` + +**Check logs with:** `tail -20 /var/log/pbs-monitoring/container-memory.log` + +### Journal Persistence Confirmed +- `/var/log/journal` exists and is retaining logs across reboots +- `journalctl --list-boots` shows 5 boot sessions dating back to May 2025 +- OOM kill evidence was successfully retrieved from previous boots + +## Current Server Memory Snapshot (post-fix) +| Container | Memory | % of Limit | +|-----------|--------|------------| +| wordpress | 1.11 GB | 29% (of system) | +| wordpress_mysql | 225 MB | 29% (of 768MB cap) | +| n8n | 200 MB | 5% | +| uptime-kuma | 100 MB | 3% | +| traefik | 37 MB | 1% | +| pbs-api | 28 MB | 1% | +| redis | 13 MB | 2% (of 640MB cap) | +| wpcron | 8 MB | <1% | + +## Still Open + +- [ ] Monitor overnight stability — check memory logs tomorrow AM +- [ ] Add log rotation for `/var/log/pbs-monitoring/container-memory.log` +- [ ] Investigate `apt-daily.service` — consider disabling automatic apt updates +- [ ] Server sizing discussion: 4GB may be tight for adding Gitea + Authelia +- [ ] Determine if Wordfence free-tier scan is contributing to memory pressure +- [ ] Consider setting server timezone to Eastern for easier log reading +- [ ] Investigate root cause of MySQL memory bloat (WooCommerce correlation still strong) + +## Key Learnings + +- **htop shows threads, not processes** — press `H` to toggle thread visibility; one MySQL process can show as dozens of rows +- **systemd journal persists across reboots** if `/var/log/journal` exists and `Storage=auto` or `Storage=persistent` is set +- **`journalctl -b -1`** shows previous boot logs; use `--since`/`--until` for large time ranges to avoid hanging +- **`performance-schema=OFF`** in MySQL saves ~200-400MB with no downside for production WordPress +- **Docker `deploy.resources.limits.memory`** provides a controlled cap — Docker restarts the container instead of the kernel OOM-killing it and cascading +- **Server timezone is UTC** — subtract 4 hours for Eastern time when reading logs \ No newline at end of file diff --git a/Sources/Homelab/ssh-ca-with-step-ca-authentik.md b/Sources/Homelab/ssh-ca-with-step-ca-authentik.md new file mode 100644 index 0000000..af7c876 --- /dev/null +++ b/Sources/Homelab/ssh-ca-with-step-ca-authentik.md @@ -0,0 +1,114 @@ +--- +created: '2026-05-08' +path: Sources/Homelab +project: ssh-ca-step +status: active +tags: +- authentik +- ansible +- proxmox +- automation +type: project-plan +updated: '2026-05-08' +--- + +# SSH CA with step-ca + Authentik + +## Goal + +Replace long-lived SSH keypairs with short-lived certificates issued by a self-hosted CA, authenticated via Authentik OIDC. Cover human users, host identities (kill TOFU prompts), and service identities. + +## Architecture + +- **`step-ca`** in a Debian LXC on PVE (matches Knot pattern), VLAN 11 (Lab) +- **OIDC provisioner** pointed at Authentik for human cert requests +- **JWK provisioner** for service/automation cert requests (n8n, Ansible runners, etc.) +- **Host CA** signs server SSH host keys → clients trust the CA, no more `yes/no/fingerprint` prompts +- **User CA** signs client certs with TTL (8h human, shorter for service) +- Ansible role rolls out `TrustedUserCAKeys` and `HostCertificate` config to fleet +- Naming: `step-ca.herbylab.dev` (capability-based, fits the Traefik convention) + +## Why this shape + +- Authentik is already the SSO surface — same auth path as Traefik/MCP +- LXC keeps it lightweight and reuses Knot's deployment muscle memory +- One canary first → safer, lets us validate the Ansible role before fleet-wide +- Two CAs (user + host) is the recommended `step-ca` pattern; a single CA for both works but mixing principals gets messy + +## Phases + +### Phase 1 — CA foundation + +- [ ] **1.1** Create LXC on PVE: Debian 13, static IP on VLAN 11, hostname `step-ca` +- [ ] **1.2** Install `step-ca` + `step` CLI from Smallstep apt repo +- [ ] **1.3** Initialize CA: `step ca init` with two intermediate CAs (user, host) under one root, RSA or Ed25519 +- [ ] **1.4** Persist CA secrets: password file in `/etc/step-ca/`, root key offline-backup to KeePassXC +- [ ] **1.5** Systemd unit, enable + start, confirm `https://step-ca.herbylab.dev:9000/health` responds +- [ ] **1.6** DNS record in Knot once Knot is live; interim: `/etc/hosts` on canary +- [ ] **1.7** PVE snapshot: `phase1-ca-base` + +### Phase 2 — OIDC provisioner (Authentik) + +- [ ] **2.1** Create OAuth2/OIDC provider in Authentik for `step-ca`, scopes `openid email profile`, redirect `http://127.0.0.1/sso/oauth/callback` +- [ ] **2.2** Add provisioner to `step-ca`: `step ca provisioner add authentik --type=OIDC --client-id=... --client-secret=... --configuration-endpoint=https://auth.herbylab.dev/application/o/step-ca/.well-known/openid-configuration` +- [ ] **2.3** Configure principal mapping: Authentik `email` → Unix username (e.g., `tjcherb@plantbasedsoutherner.com` → `travadmin`); use `--admin` for admin emails +- [ ] **2.4** Test from tower: `step ssh login` → browser SSO → cert in agent +- [ ] **2.5** Verify cert contents: `step ssh inspect`, confirm TTL, principals, key-id + +### Phase 3 — Canary server (user certs only) + +- [ ] **3.1** Pick canary: `us-test-authy` (low blast radius, already main dev surface) +- [ ] **3.2** Fetch user CA pubkey, install to `/etc/ssh/ca-user.pub` +- [ ] **3.3** Add `TrustedUserCAKeys /etc/ssh/ca-user.pub` to `sshd_config`, reload +- [ ] **3.4** Test: SSH from tower with cert (key-based access still works as fallback) +- [ ] **3.5** Verify auth log shows cert-based login with correct principal +- [ ] **3.6** Run for 1 week, confirm no breakage + +### Phase 4 — Host certs on canary + +- [ ] **4.1** Issue host cert: `step ssh certificate --host us-test-authy us-test-authy.herbylab.dev /etc/ssh/ssh_host_ed25519_key.pub` +- [ ] **4.2** Configure `sshd_config`: `HostCertificate /etc/ssh/ssh_host_ed25519_key-cert.pub` +- [ ] **4.3** On clients: add host CA to `~/.ssh/known_hosts` as `@cert-authority *.herbylab.dev` +- [ ] **4.4** Confirm fingerprint prompt is gone on fresh client connection +- [ ] **4.5** Set up host cert renewal (systemd timer, weekly) + +### Phase 5 — Ansible role + fleet rollout + +- [ ] **5.1** Build `step-ca-client` Ansible role: installs `step` CLI, distributes user CA pubkey, configures `sshd_config`, manages host cert renewal timer +- [ ] **5.2** Inventory tagging: `step_ca_enrolled` group +- [ ] **5.3** Roll out to PVE host, NAS, Traefik VM, Knot LXC one-by-one +- [ ] **5.4** Update `~/.ssh/config` on tower + ThinkPad to use cert-based auth by default +- [ ] **5.5** Document the "new server" runbook: 1 Ansible play vs old key-shuffle dance + +### Phase 6 — Service identities + +- [ ] **6.1** Add JWK provisioner to `step-ca` for automation +- [ ] **6.2** Issue service certs to Ansible runner identity (replaces deploy keys for orchestration tasks where TTL works) +- [ ] **6.3** Decide what stays as static keys (true unattended workloads where cert renewal would be fragile) vs. what moves to certs +- [ ] **6.4** Document the tier model: human → OIDC certs, semi-attended → JWK certs, fully unattended service → static scoped keys + +### Phase 7 — Cleanup + +- [ ] **7.1** Audit `~/.ssh/` on tower + ThinkPad, archive obsolete keys to KeePassXC, delete from disk +- [ ] **7.2** Audit `authorized_keys` across fleet, remove keys made redundant by CA +- [ ] **7.3** Document key rotation + CA disaster recovery (root key restore from KeePassXC backup) +- [ ] **7.4** Add fish abbreviation `sshlogin` → `step ssh login` + +## Open questions + +- Cert TTL policy: 8h for human (re-auth daily) vs 24h (less friction)? Start at 8h, relax if painful. +- Do we want session recording? (Out of scope for step-ca; would need Teleport. Probably not.) +- Host cert renewal: systemd timer per host, or centralized via Ansible cron? Per-host is more resilient. + +## Dependencies / blockers + +- Authentik must be reachable at `auth.herbylab.dev` with TLS — already true +- Knot LXC blocker (separate project) doesn't block this; we can use `/etc/hosts` for `step-ca.herbylab.dev` until DNS is sorted +- Traefik project doesn't block this — `step-ca` exposes its own HTTPS on :9000, no ingress needed for the CA itself + +## Success criteria + +- Fresh laptop: install `step`, `step ssh login`, immediately SSH to any fleet server with no fingerprint prompts and no copied keys +- New server provisioning: 1 Ansible play installs CA trust + host cert, server is reachable +- Old laptop revoked: `step ca revoke` + 8h max blast radius +- `~/.ssh/` on personal machines contains only `config` and `known_hosts` \ No newline at end of file diff --git a/Sources/Homelab/ssh-login-alerting.md b/Sources/Homelab/ssh-login-alerting.md new file mode 100644 index 0000000..16784c1 --- /dev/null +++ b/Sources/Homelab/ssh-login-alerting.md @@ -0,0 +1,111 @@ +--- +created: 2026-04-19 +path: Sources/Homelab +project: ssh-login-alerting +status: active +tags: +- pbs +- security +- ssh +- journald +- python +- n8n +- monitoring +type: project-plan +updated: 2026-04-21 +--- + +# SSH Login Alerting + +## Goal + +Get notified in Google Chat the moment anyone SSHes into a PBS server. +Early warning system for compromised SSH keys or unauthorized access — +especially important now that `pbsdeploy` exists as a passphrase-less +key-based account. + +## Architecture + + +sshd → journald → Python sdjournal tailer → POST to n8n webhook → +n8n formats + sends to Google Chat + + +## Design Decisions + +- **journald over auditd** — v1 attempted `auditd → shell dispatcher → +regex → webhook` and was abandoned after silent event drops and two hours +of failed debugging. Root cause: auditd emits multiple records per logical +event with format variations that break line-by-line regex parsing. +Journald is one-line-per-event with structured metadata +(`_SYSTEMD_UNIT=ssh.service`), which eliminates the multi-record coalescing +problem entirely. +- **Python sdjournal tailer, not pam_exec** — pam_exec would give fully +structured data with zero regex via PAM env vars (`PAM_USER`, `PAM_RHOST`, +etc.), but touches the SSH auth path. Explicit tradeoff: accept one +remaining regex pattern in exchange for zero risk to login infrastructure. +If the tailer crashes, SSH keeps working. +- **Cursor persistence solves the silent-drop problem** — +`systemd.journal.Reader.seek_cursor()` with cursor saved to disk after each +successful webhook POST gives exactly-once delivery across restarts, +crashes, and n8n outages. This was the actual failure mode of v1. +- **Regex is contained and testable** — one pattern against sshd's +`MESSAGE` field (stable across OpenSSH versions for ~10 years), in one +Python function, with unit tests against fixture strings. When it breaks, +fix is one line in one place. +- **n8n handles notification routing** — unchanged from v1. Server fires +events; n8n formats and delivers. +- **Alert on every login for v1** — unchanged. Tune later if noise becomes +a problem. +- **Scope: all accounts, all PBS servers** — unchanged. Staging and +production, every user. + +## Why This Matters + +- `pbsdeploy` SSH key has no passphrase. If it leaks, the attacker's first +move is SSHing in. +- Linode account is already behind 2FA, so LISH console access isn't the +main threat vector. +- SSH is the primary attack surface — we need eyes on it. + +## Next Steps + +- [ ] Decide deployment model: host systemd service vs containerized +(leaning host systemd — simpler, more reliable for journal access) +- [ ] Write Python tailer using `systemd.journal.Reader` with +`add_match(_SYSTEMD_UNIT="ssh.service")`, cursor persistence, and HTTP POST +to n8n +- [ ] Unit test the MESSAGE regex against fixture strings (accepted +publickey, accepted password, failed attempts, invalid user) +- [ ] Deploy to staging, test cursor behavior across restarts +- [ ] Build n8n workflow: webhook trigger → format message → send to Google +Chat +- [ ] End-to-end test on staging (SSH in, kill tailer, SSH in again, +restart tailer, confirm replay) +- [ ] Add Uptime Kuma monitor for the tailer service +- [ ] Roll out to production via Ansible +- [ ] Decommission v1 auditd dispatcher + +## Open Questions + +- Host systemd service or containerized? Leaning host service — touches +host-level journal, simpler bind-mount story. Breaks Compose aesthetic but +fits the job. +- Where does cursor state live? (Path, ownership, permissions — probably +`/var/lib/ssh-login-alerter/cursor`.) +- Which Google Chat space — `webadmin` or a dedicated security space? +- Metadata beyond username + source IP + timestamp (e.g., geolocation on +source IP)? +- Separate "critical" alert path for root logins? +- How does Uptime Kuma know the tailer is alive? Heartbeat ping from the +tailer itself, or process check? + +## Priority + +Still higher priority than env file hardening. v1's silent drops mean we +currently have *no* working alert on the `pbsdeploy` account — the +force-multiplier argument is sharper now than it was at project inception. + +...sent from Jenny & Travis + +...sent from Jenny & Travis \ No newline at end of file diff --git a/Sources/Homelab/sshkm-local-ssh-key-manager.md b/Sources/Homelab/sshkm-local-ssh-key-manager.md new file mode 100644 index 0000000..b02046d --- /dev/null +++ b/Sources/Homelab/sshkm-local-ssh-key-manager.md @@ -0,0 +1,141 @@ +--- +created: '2026-05-08' +path: Sources/Homelab +project: sshkm +status: active +tags: +- python +- automation +- ansible +type: project-plan +updated: '2026-05-08' +--- + +# sshkm — local SSH key manager + +## Goal + +A small Python CLI that handles the day-to-day SSH key lifecycle (generate, register, deploy, list, remove) without the ceremony of a full CA. Solves the `~/.ssh/` graveyard problem today; designed to coexist with a future `step-ca` deployment rather than be replaced by it. + +## Design principles + +- **Capability/authorization split.** Generation is local-only file I/O — safe for agents to drive. Deployment requires interactive auth (target server password / agent unlock) — human-only by design. +- **Explicit state transitions.** A key is `floating`, `bound`, or `deployed`. Every transition is its own command. No silent rebinds. +- **Plain text everything.** Markdown ledger, plain `~/.ssh/config` blocks, no SQLite, no daemon. Inspect with `cat`, edit at your own risk. +- **Coexists with future CA.** This tool manages the long-lived keys that survive the cleanup; the CA project will replace human and host identities, leaving sshkm for the truly static service-key tier. + +## Architecture + +- **Typer + Rich** for the CLI surface +- **`ssh-keygen` / `ssh-copy-id`** as the heavy lifters (no paramiko, no shell-out reinvention) +- **`~/.ssh/managed/`** as sshkm's territory: keys, ledger, never touches anything outside +- **`~/.ssh/config`** gets `# managed-by-sshkm: <label>` blocks the tool can find and edit +- **Ledger** is markdown table at `~/.ssh/managed/ledger.md`, symlinkable into Obsidian + +## Key lifecycle + +``` +floating ──bind──▶ bound ──push──▶ deployed + ▲ │ + └────rebind──────┘ (--rebind flag, v1 errors out) +``` + +- **floating** — keypair exists, no target declared yet (LXC isn't built yet, etc.) +- **bound** — target host/user declared, ssh config block written, key not yet on target +- **deployed** — ssh-copy-id succeeded, connection test passed + +## Commands (v1) + +``` +sshkm generate -l <label> [-h <host> -u <user> -a <alias> -p <purpose>] + # No host/user → floating. With host/user → bound. Either way, key is created. + +sshkm push <label> [-h <host> -u <user> -a <alias>] + # Floating: requires host/user, transitions to deployed. + # Bound: requires nothing, transitions to deployed. + # Deployed: errors out (use --rebind in v2). + +sshkm list [--json] + # Shows label, status, alias, host, user, created, purpose. + +sshkm show <label> [--json] + # Full detail including key paths and fingerprint. + +sshkm remove <label> [--force] + # Local-only: removes config block, ledger row, key files. + # Warns that remote authorized_keys is untouched. + +sshkm import <key-path> -l <label> [-h <host> -u <user> --move] + # Adopts existing on-disk keys. Marks as deployed (assumes already in use). +``` + +## Agent boundary + +- **`generate`, `list`, `show`, `import`** — agent-friendly. `--json` + `--yes` flags everywhere. +- **`push`, `remove`** — human-only. Belt-and-suspenders enforcement: + - `sys.stdin.isatty()` check — refuses to run in non-interactive shells + - `SSHKM_AGENT=1` env-var bypass for legitimate scripted use, paired with logging + - Confirmation prompt by default, `--yes` only allowed when env-var set + +## Phases + +### Phase 1 — Core CLI + +- [ ] **1.1** Project skeleton: `pyproject.toml`, Typer + Rich, `uv tool install` target +- [ ] **1.2** `models.py` — `ManagedKey` dataclass with `Status` enum (`floating`, `bound`, `deployed`) +- [ ] **1.3** `paths.py`, `keygen.py`, `deploy.py` — filesystem and subprocess wrappers +- [ ] **1.4** `config.py` — render/append/remove blocks marked with `# managed-by-sshkm:` +- [ ] **1.5** `ledger.py` — append/read/update rows in markdown table + +### Phase 2 — Commands + +- [ ] **2.1** `generate` — handle both floating and bound modes +- [ ] **2.2** `push` — state-aware: validates current status, accepts host/user only when floating +- [ ] **2.3** `list` / `show` — Rich table for humans, `--json` for machines +- [ ] **2.4** `remove` — local cleanup with clear messaging about remote leftovers +- [ ] **2.5** `import` — adopt existing keys, status defaults to `deployed` + +### Phase 3 — Agent boundary + +- [ ] **3.1** TTY check + env-var override pattern for `push` and `remove` +- [ ] **3.2** `--json` output on read commands and on `generate` (so agents can chain) +- [ ] **3.3** `--yes` flag, gated on env-var +- [ ] **3.4** Document the boundary in README — explicit "what agents can/can't do" + +### Phase 4 — Real-world burn-in + +- [ ] **4.1** Install on tower via `uv tool install` +- [ ] **4.2** Use it for the next 3–5 real keys you'd otherwise create manually +- [ ] **4.3** Note friction points in a session note +- [ ] **4.4** Decide what (if anything) goes into v2 + +## v2 candidates (not in v1) + +- `rotate <label>` — generate fresh key, push, retire old one (needs remote cleanup) +- `--rebind` flag on push — explicitly allow same key on multiple hosts +- `--purge-remote` flag on remove — SSH in with the key and yank its own authorized_keys line +- Passphrase support — only matters once you're moving keys into KeePassXC's agent +- `audit` command — diff ledger against actual `~/.ssh/managed/` contents, find orphans + +## Non-goals + +- Replacing `ssh-agent`. KeePassXC handles that separately. +- Replacing the future CA. This tool manages static keys; the CA will issue ephemeral certs. +- Managing remote `authorized_keys` files as the source of truth. We touch the remote exactly once (push), never again. +- Cross-machine sync. The ledger is per-machine. If you want a unified view, that's a future MCP-or-script job, not this tool's concern. + +## Success criteria + +- `~/.ssh/` on the tower has only `config`, `known_hosts`, and `managed/` after the cleanup phase +- New key + new server flow takes one command (bound generate + push) instead of the manual dance +- Every key in `~/.ssh/managed/` has a corresponding ledger row — no orphans +- An agent driving Claude Code can prep a key for you without touching push + +## Interaction with the CA project + +When `step-ca` lands, this tool's role contracts: +- Human keys → migrate to OIDC-issued certs, sshkm becomes irrelevant for them +- Host keys → migrate to host certs, sshkm never managed these anyway +- Service keys with no good cert story → stay in sshkm + +So the CA project doesn't deprecate sshkm, it shrinks its surface to exactly the tier where short-lived certs don't fit. Good outcome either way. \ No newline at end of file diff --git a/Sources/Homelab/traefik-deployment.md b/Sources/Homelab/traefik-deployment.md new file mode 100644 index 0000000..b536c76 --- /dev/null +++ b/Sources/Homelab/traefik-deployment.md @@ -0,0 +1,189 @@ +--- +created: 2026-05-06 +path: Sources/Homelab +project: traefik-deployment +status: active +tags: +- homelab +- traefik +- dns +- knot +- tls +- cloudflare +- tailscale +- mcp +type: project-plan +updated: 2026-05-06 +--- + +# Traefik Deployment — Homelab Ingress + +## Goal + +Replace ad-hoc `host:port` access with consistent named endpoints for ~8 +services across three surfaces (public, private, east-west). Build the full +ingress stack — Traefik, Knot DNS, wildcard TLS — and migrate MCP behind it +as the first real service. + +## Locked Architecture + +### Topology + +- **Traefik VM** on PVE (own VM, runs Docker + tailscaled + cloudflared) +- **Knot primary** in LXC on PVE +- **Knot secondary** in Docker on NAS (Ubuntu bare metal) +- **Pi-hole + Unbound** stays as recursive frontend + ad-blocker +- Cross-VLAN firewall locks PVE↔NAS to explicit flows (Knot AXFR, +Traefik→NAS services) + +### DNS + +- Knot authoritative for `herbylab.dev` internal +- Active/passive replication via native AXFR + NOTIFY +- Pi-hole conditional forwards `herbylab.dev` queries to Knot +- Split-horizon: same name resolves to Traefik VM internally, Cloudflare +externally +- Zone file in git, Ansible deploys to primary + +### TLS + +- Wildcard `*.herbylab.dev` via Traefik + Cloudflare DNS-01 +- One cert covers public and private surfaces +- Tailscale certs dropped from the plan + +### Naming + +- Capability-based: `.herbylab.dev` regardless of host +- No `lab.` infix, no `dev.` prefix — domain is by function, not location + +### Routing surfaces + +- **Public:** Cloudflare → cloudflared (on Traefik VM) → Traefik → service +- **Private:** Tailscale → Traefik VM tailnet IP → service (DNS via Pi-hole +→ Knot) +- **East-west:** direct, never through Traefik + +### Visual + + + ┌─────────────────────┐ + │ Cloudflare Edge │ + │ *.herbylab.dev │ + └──────────┬──────────┘ + │ tunnel + tailnet ────────────────────┼──────────────── tailscale0 + │ + ┌──────────▼──────────┐ + │ Traefik VM (PVE) │ + │ - traefik │ + │ - cloudflared │ + │ - tailscaled │ + └──────────┬──────────┘ + │ + ┌────────────────┼────────────────┐ + │ │ │ + ┌───▼──┐ ┌───▼──┐ ┌───▼──┐ + │ MCP │ │whoami│ │ ... │ + └──────┘ └──────┘ └──────┘ + + +## Build Phases + +### Phase 1 — Foundations + +No services routed yet. Each step produces a verifiable artifact. + +- [ ] **1.1 Cloudflare API token** + - Scope: `herbylab.dev` zone, DNS edit permissions only + - Verify with `curl` against CF API + - Store in password manager +- [ ] **1.2 Traefik VM on PVE** + - Fresh Debian or Ubuntu, 1 vCPU, 1GB RAM, on PVE VLAN + - Install Docker, Tailscale; `tailscale up` + - Verify: pingable from laptop over tailnet + - Snapshot +- [ ] **1.3 Knot primary in LXC on PVE** + - Stand up LXC, install Knot + - Load minimal zone file with one test record + - Verify: `dig test.herbylab.dev @` returns expected answer +- [ ] **1.4 Knot secondary in Docker on NAS** + - Configure as AXFR slave + - Open cross-VLAN firewall rule for AXFR + - Verify: `dig test.herbylab.dev @` matches primary +- [ ] **1.5 Pi-hole conditional forwarding** + - Configure Pi-hole: `herbylab.dev` → Knot + - Verify from a normal client: `dig test.herbylab.dev` returns Knot's +answer +- [ ] **1.6 Wildcard cert via Traefik + Cloudflare DNS-01** + - Minimal Traefik config, ACME with DNS-01 challenge + - Request `*.herbylab.dev` + - Verify: cert in `acme.json`, expiry ~90 days out + - No services routed yet + +**Phase 1 checkpoint:** working DNS chain, working wildcard cert, Traefik +VM with no routes. Production unchanged. Safe abort point. + +### Phase 2 — Canary (`whoami`) + +- [ ] **2.1 Deploy `whoami` on Traefik VM** + - Docker compose, labels for both public and private surfaces +- [ ] **2.2 Add DNS records** + - Knot zone: `whoami.herbylab.dev` → Traefik VM tailnet IP + - Cloudflare: `whoami.herbylab.dev` → cloudflared tunnel +- [ ] **2.3 Four-position verification** + - Laptop on tailnet → `https://whoami.herbylab.dev` works, real cert + - Laptop off tailnet (LTE hotspot) → same URL works via cloudflared + - `dig` from inside LAN returns Traefik VM IP + - `dig` from outside returns Cloudflare IP +- [ ] **2.4 Decision: keep `whoami` as permanent diagnostic, or tear down** + +### Phase 3 — MCP Migration + +Parallel-path safety net. Existing `cloudflared → MCP` stays running until +the new path is verified. + +- [ ] **3.1 Add Traefik labels to MCP container** + - Add internal DNS record in Knot + - Existing public path untouched +- [ ] **3.2 Verify private surface first** + - From tailnet: `https://mcp.herbylab.dev` → Traefik → MCP works +- [ ] **3.3 Cut public traffic over** + - Flip Cloudflare DNS / tunnel config to land `mcp.herbylab.dev` on +Traefik + - Verify Claude can still call MCP + - Rollback path: flip DNS back +- [ ] **3.4 Decommission old direct cloudflared route** + - After a few days of stability + +## Open Items (Build-Time Decisions) + +- [ ] Inventory current `*.herbylab.dev` records to migrate from Pi-hole +local DNS into Knot zone file +- [ ] TTL strategy — start at 300 during buildout, raise to 3600 once stable +- [ ] Second Pi-hole instance for redundancy (parallel work, not blocking) +- [ ] Pi-hole sync method — gravity-sync vs Teleporter +- [ ] Ansible role structure for Knot primary deployment + +## Key Principles + +- **Capability-based naming.** Domain reflects function, not host. Service +can move hosts without renaming. +- **Surfaces compose, they don't overlap.** Cloudflare = public auth + ACME +validation. Tailscale = device naming on tailnet. Traefik = service routing +once a request lands. Internal DNS = telling LAN clients where to send the +packet. +- **East-west stays direct.** Service-to-service inside Docker/LAN never +traverses Traefik. +- **Build outside-in for foundations, inside-out for services.** Cert and +DNS first; canary before production. +- **Parallel paths for migration.** Never cut over working infrastructure +without a rollback flip. + +## Reference + +Prior session notes: `Tech/Sessions/homelab-dns-decision.md` (Knot +architecture lock-in, alternatives considered, why split-horizon with +public domain). + + +...sent from Jenny & Travis \ No newline at end of file diff --git a/Sources/Homelab/ufw-docker-outage-fix.md b/Sources/Homelab/ufw-docker-outage-fix.md new file mode 100644 index 0000000..3cea0e4 --- /dev/null +++ b/Sources/Homelab/ufw-docker-outage-fix.md @@ -0,0 +1,126 @@ +--- +path: Sources/Homelab +project: ufw-docker-outage-fix +status: completed +tags: +- pbs +- docker +- traefik +- production +- ufw +- security +- woocommerce +type: session-notes +--- + +# Server Outage & UFW Docker Rules Fix + +## Summary + +Production site became unresponsive after a server reboot. Root cause was +incomplete UFW firewall rules in `/etc/ufw/after.rules` on production — +Docker containers had no outbound internet access. WordPress plugins making +external HTTP calls (WooCommerce, Jetpack, Yoast, etc.) were timing out on +every page load, causing 60-second render times. + +## Timeline + +- Server became unresponsive overnight, required Linode dashboard reboot +- Site loaded but extremely slowly (15s+, then timeouts) +- WordPress container showed 60-second homepage render time +- Static files served in ~89ms — confirmed PHP processing was the bottleneck +- MySQL processlist was clean — not a database issue +- Discovered WordPress container could not reach the internet (`curl +google.com` failed, `ping 8.8.8.8` 100% packet loss) +- Compared `DOCKER-USER` iptables chain between production and staging +- Production was missing three critical rules that staging had +- Root cause: `after.rules` on production had an older version of the +Docker firewall rules that was never updated after Ansible playbook +improvements + +## Root Cause + +Production `/etc/ufw/after.rules` was missing: + +``` +-A DOCKER-USER -m conntrack --ctstate RELATED,ESTABLISHED -j RETURN +-A DOCKER-USER -p udp -m udp --dport 53 -j RETURN +-A DOCKER-USER -p tcp -m tcp --dport 53 -j RETURN +-A DOCKER-USER -i docker+ -o eth0 -j RETURN +``` + +Without these rules, containers could receive inbound traffic but could not +initiate outbound connections. The site worked before the reboot because +Docker's own iptables rules provided outbound access — but on reboot, UFW +reloaded from `after.rules` and overwrote them with the incomplete ruleset. + +## Fix Applied + +1. Backed up production `after.rules`: `sudo cp /etc/ufw/after.rules +/etc/ufw/after.rules.backup.2026-03-22` +2. Replaced production `after.rules` with staging's version (which matches +current Ansible playbook) +3. Ran `sudo ufw reload` +4. Verified: `docker exec traefik ping -c 2 8.8.8.8` — 0% packet loss +5. Homepage render time: 60 seconds → 276 milliseconds + +## Additional Cleanup + +- Cleaned 8,555 failed Action Scheduler tasks from +`wp_actionscheduler_actions` table (caused by +`image-optimization/cleanup/stuck-operation` hook accumulating since +December 2025) +- Cleaned 1,728 completed actions +- Flushed Redis cache + +## Key Learnings + +- **UFW + Docker is fragile on reboot:** Docker's runtime iptables rules +can mask incomplete UFW `after.rules` config. Everything works until a +reboot wipes Docker's rules and UFW reasserts its own. +- **Always re-run Ansible after playbook changes:** The playbook was +updated with correct Docker rules but never re-applied to production. +Staging got the fix, production didn't. +- **Container outbound networking failure presents as slow PHP:** Plugins +making external HTTP calls block the entire page render while waiting for +connection timeouts. Looks like a performance problem but is actually a +networking problem. +- **Cold cache + broken networking = compounding failure:** After reboot, +no Redis cache + no opcode cache + plugins timing out on external calls = +catastrophic page load times. +- **WooCommerce was a red herring:** It added overhead but wasn't the root +cause. The real issue predated the WooCommerce install. + +## Action Items + +- [ ] Investigate which plugin registers +`image-optimization/cleanup/stuck-operation` and fix or remove it +- [ ] Audit Ansible playbook vs production state — identify other drift +- [ ] Consider running Ansible against production with `--check --diff` to +see what would change before applying +- [ ] Add a monitoring check for container outbound connectivity (e.g., +Uptime Kuma ping to external host from inside a container) +- [ ] Document WooCommerce memory impact: WordPress container went from +~300-400MB to ~728MB + +## Diagnostic Commands Used + +```bash +# Check per-container resources +docker stats --no-stream --format "table +{{.Name}}\t{{.CPUPerc}}\t{{.MemUsage}}\t{{.MemPerc}}" + +# Test PHP render time +time docker exec wordpress curl -s -o /dev/null -w "%{http_code}" +http://localhost/ + +# Test container outbound access +docker exec wordpress php -r "var_dump(file_get_contents('http://google.com +'));" + +# Compare DOCKER-USER iptables rules +sudo iptables -L DOCKER-USER -n -v + +# Check UFW after.rules +sudo cat /etc/ufw/after.rules | grep -A 20 "DOCKER-USER" +``` \ No newline at end of file diff --git a/Sources/Homelab/wp-cron-supercronic-deploy.md b/Sources/Homelab/wp-cron-supercronic-deploy.md new file mode 100644 index 0000000..ed45e3c --- /dev/null +++ b/Sources/Homelab/wp-cron-supercronic-deploy.md @@ -0,0 +1,142 @@ +--- +created: 2026-04-15 +path: Sources/Homelab +project: wp-cron-supercronic-deploy +status: completed +tags: +- pbs +- wordpress +- traefik +- docker +- cron +type: session-notes +updated: 2026-04-15 +--- + +# WP-Cron → Supercronic Sidecar Deployment + +Session notes captured retroactively from March 7 work, before the n8n +email pipeline was in place. + +## Problem + +Uptime Kuma status checks were triggering WordPress's built-in WP-Cron on +every HTTP request, causing cron jobs to fire unpredictably on page load +instead of on a proper schedule. Added unnecessary load and made cron +behavior non-deterministic. + +## Solution + +Three-part fix: + +1. **Disable WP-Cron on page load** via `wp-config.php` +2. **Block external access to `wp-cron.php`** at the Traefik middleware +layer +3. **Deploy a Supercronic sidecar container** that calls wp-cron on a real +schedule via the internal Docker network + +## Implementation + +### 1. wp-config.php + +```php +define('ALTERNATE_WP_CRON', false); +define('DISABLE_WP_CRON', true); +``` + +`wp-config.php` is bind-mounted from the host so Ansible can manage it: + +```yaml +volumes: + - wordpress_data:/var/www/html + - ./wp-config.php:/var/www/html/wp-config.php +``` + +To pull a fresh copy out of a running container: +```bash +docker cp wordpress:/var/www/html/wp-config.php +/opt/docker/wordpress/wp-config.php +``` + +### 2. Traefik middleware — block external `wp-cron.php` + +```jinja +- traefik.http.routers.wordpress.middlewares=www-redirect,block-wpcron +- traefik.http.middlewares.block-wpcron.redirectregex.regex=^/wp-cron\.php.* +- traefik.http.middlewares.block-wpcron.redirectregex.replacement=/ +``` + +The router definition must include `block-wpcron` in the middlewares chain +or Traefik silently does not apply it. + +Verify: +```bash +curl -Lv https://plantbasedsoutherner.com/wp-cron.php 2>&1 | grep -E +"HTTP|Location" +# Expected: HTTP/2 302 + Location: https://plantbasedsoutherner.com +``` + +### 3. Supercronic sidecar + +Project structure: +``` +/opt/docker/wpcron/ + Dockerfile + crontab + docker-compose.yml +``` + +`crontab` (set to every minute — Wordfence and Jetpack schedule events at +this frequency): +``` +* * * * * curl -s http://wordpress/wp-cron.php?doing_wp_cron +``` + +`Dockerfile`: +```dockerfile +FROM alpine:latest + +ARG SUPERCRONIC_VERSION=v0.2.29 +ARG SUPERCRONIC_ARCH=amd64 + +RUN wget -q +https://github.com/aptible/supercronic/releases/download/${SUPERCRONIC_VERSION}/supercronic-linux-${SUPERCRONIC_ARCH} +\ + -O /usr/local/bin/supercronic && \ + chmod +x /usr/local/bin/supercronic && \ + apk add --no-cache curl + +COPY crontab /etc/crontab +CMD ["/usr/local/bin/supercronic", "/etc/crontab"] +``` + +The sidecar lives in its own compose project. To reach the WordPress +container, the wpcron compose file declares the WordPress network as +`external: true`. + +## Bonus Fix — Redis Auth on Production + +Discovered during the deploy that the production WordPress container was +missing the Redis password env var. Env vars had been moved to an `.env` +file on staging but the change never made it back to production. Fixed and +now part of the standard env handoff between environments. + +## Key Learnings + +- **Traefik `redirectregex` matches path only**, not the full URL — design +regex accordingly +- **`depends_on` only works within the same Docker Compose project** — for +cross-project dependencies, declare the network as `external: true` +- **`docker compose up -d --force-recreate `** is the right +command to pick up compose file changes (not `restart`, which reuses the +old config) +- **Cron interval matters:** Plugins like Wordfence and Jetpack schedule +per-minute events, so a 5-minute cron will leave them perpetually behind. +Every minute is the right default for this stack. + +## Status + +Deployed to staging then promoted to production. Running cleanly since. + + +...sent from Jenny & Travis \ No newline at end of file diff --git a/Sources/Reference/index.md b/Sources/Reference/index.md new file mode 100644 index 0000000..a0d4ed8 --- /dev/null +++ b/Sources/Reference/index.md @@ -0,0 +1,23 @@ +--- +type: topic-landing +path: Sources/Reference +tags: + - landing +updated: 2026-05-08 +--- + +# Reference — Sources + +Project plans and session notes for the Reference domain. + +## Active Projects + +*Updated by compile loop.* + +## Recent Sessions + +*Updated by compile loop.* + +## Completed Projects + +*Updated by compile loop.* diff --git a/Sources/Venture/editor-stack-upgrade.md b/Sources/Venture/editor-stack-upgrade.md new file mode 100644 index 0000000..814b508 --- /dev/null +++ b/Sources/Venture/editor-stack-upgrade.md @@ -0,0 +1,163 @@ +--- +created: 2026-04-08 +path: Sources/Venture +project: editor-stack-upgrade +status: active +tags: +- editors +- neovim +- zed +- zellij +- lazyvim +- homelab +- workflow +type: project-plan +updated: 2026-04-08 +--- + +# Editor Stack Upgrade + +## Context + +VS Code Remote-SSH crashed production server after 6 instances were +accidentally left open (each spawns its own Node server process on the +remote). Replacing with a two-track editor workflow: a lightweight GUI for +heavier refactors, and a terminal-native stack for quick edits and street +cred. + +## Goals + +- [ ] Eliminate VS Code Remote-SSH resource bloat on production server +- [ ] Establish Zed as primary GUI editor with remote SSH workflow +- [ ] Build terminal-native editing muscle memory with Neovim + LazyVim +- [ ] Replace tmux workflow with Zellij (modern, discoverable keybinds) +- [ ] Keep production server stable during rollout (test everything on +tower first) + +## Track A: Zed (GUI path) + +### Phase A1 — Install on tower + +- [ ] Install Zed on Manjaro tower + ``` + curl -f https://zed.dev/install.sh | sh + ``` +- [ ] Verify binary at `~/.local/bin/zed` +- [ ] Launch and complete first-run setup +- [ ] Sign in (optional, for settings sync) + +### Phase A2 — Remote SSH setup + +- [ ] Confirm production server is in `~/.ssh/config` with alias +- [ ] Load SSH key into `ssh-agent` if passphrase-protected + ``` + eval $(ssh-agent) + ssh-add ~/.ssh/id_ed25519 + ``` +- [ ] In Zed: `ctrl+shift+p` → "remote: connect to host" → add prod server +- [ ] Verify remote agent installs to `~/.local/bin` on production server +- [ ] Open a test project over SSH and confirm LSP works + +### Phase A3 — Validate resource footprint + +- [ ] Check process count on prod server after Zed connect + ``` + ps aux | grep zed + ``` +- [ ] Compare RAM usage vs old VS Code Remote-SSH baseline +- [ ] Document findings in session notes + +## Track B: Terminal path (Zellij + Neovim + LazyVim) + +### Phase B1 — Zellij on tower + +- [ ] Install Zellij + ``` + cargo install --locked zellij + ``` + or via pacman/AUR if preferred +- [ ] Launch and work through built-in tutorial +- [ ] Learn core keybinds (shown at bottom of screen — that's the whole +point) +- [ ] Create first layout file for homelab workflow + ``` + ~/.config/zellij/layouts/homelab.kdl + ``` + +### Phase B2 — Neovim + LazyVim on tower + +- [ ] Install Neovim (need ≥0.9.0 for LazyVim) + ``` + sudo pacman -S neovim + ``` +- [ ] Install prerequisites: git, make, unzip, gcc, ripgrep, fd, a Nerd Font +- [ ] Back up any existing Neovim config + ``` + mv ~/.config/nvim ~/.config/nvim.bak + mv ~/.local/share/nvim ~/.local/share/nvim.bak + ``` +- [ ] Clone LazyVim starter + ``` + git clone https://github.com/LazyVim/starter ~/.config/nvim + rm -rf ~/.config/nvim/.git + ``` +- [ ] First launch: let Lazy.nvim install all plugins +- [ ] Run `:LazyHealth` and resolve any warnings +- [ ] Run `:Mason` and install LSPs for Python, Go, Bash, Lua + +### Phase B3 — Learn the basics (week 1) + +- [ ] Complete `vimtutor` (comes with Neovim, ~30 min) +- [ ] Daily driver for config edits only — don't force it on real work yet +- [ ] Learn: motions (h/j/k/l, w/b/e), modes (i/a/o, esc), save/quit (:w/:q) +- [ ] Learn LazyVim extras: `` leader menu, `ff` find files, +`sg` grep +- [ ] Learn file tree: `e` + +### Phase B4 — Zellij + Neovim integration + +- [ ] Create Zellij layout with Neovim + terminal panes +- [ ] Test detach/reattach workflow + ``` + zellij attach homelab + ``` +- [ ] Configure Neovim to respect Zellij pane navigation (if desired) + +### Phase B5 — Roll out to production server + +- [ ] Install Zellij on prod server +- [ ] Install Neovim + LazyVim on prod server +- [ ] Sync LazyVim config via git (store in `trucktrav/dotfiles` or similar) +- [ ] Create persistent Zellij session for prod work + ``` + zellij attach -c prod-work + ``` +- [ ] Document SSH + zellij attach workflow + +## Decision points + +- [ ] Dotfiles repo strategy — new repo or add to existing? +- [ ] LazyVim customization — keep starter minimal or add personal tweaks +early? +- [ ] Zed vs Neovim split — when to reach for which? (document after 2 +weeks of use) + +## Success criteria + +- [ ] Zero VS Code processes on production server +- [ ] Can SSH into prod, `zellij attach`, and resume work from any device +- [ ] Comfortable enough in Neovim to edit configs without reaching for Zed +- [ ] Production server RAM usage during editing sessions documented and +reduced + +## Notes & learnings + +_(populate as you go)_ + +## Next session + +Start with Phase A1 (Zed install on tower) — fastest win, immediately +usable, low risk. + + +...sent from Jenny & Travis \ No newline at end of file diff --git a/Sources/Venture/index.md b/Sources/Venture/index.md new file mode 100644 index 0000000..f66620b --- /dev/null +++ b/Sources/Venture/index.md @@ -0,0 +1,23 @@ +--- +type: topic-landing +path: Sources/Venture +tags: + - landing +updated: 2026-05-08 +--- + +# Venture — Sources + +Project plans and session notes for the Venture domain. + +## Active Projects + +*Updated by compile loop.* + +## Recent Sessions + +*Updated by compile loop.* + +## Completed Projects + +*Updated by compile loop.* diff --git a/Sources/Venture/offsite-day1-business-dev.md b/Sources/Venture/offsite-day1-business-dev.md new file mode 100644 index 0000000..ad6dbba --- /dev/null +++ b/Sources/Venture/offsite-day1-business-dev.md @@ -0,0 +1,129 @@ +--- +created: 2026-04-20 +path: Sources/Venture +project: offsite-day1-business-dev +status: active +tags: +- pbs +- branding +- sunnies +- content +type: session-notes +updated: 2026-04-20 +--- + +# Off-Site Planning Session — Business Development Summary + +## Vision (Locked In) + +Plant Based Southerner becomes a household name. + +## Mission (Locked In) + +To provide guidance that empowers people to live a whole food plant-based +lifestyle. + +*Note: Mission is intentionally broad. Marketing and content target a +specific audience segment.* + +## Business Model Direction + +- Hybrid model: content drives multiple revenue streams (products, +community, education, etc.) +- Revenue must serve the education mission, not override it +- Monetization filter: only pursue streams that align with teaching people +to live well + +## Core Audience (Locked In) + +Busy women feeding families who want to eat a whole food plant-based diet +but struggle to do it consistently in their daily lives. + +### Key traits: +- Time-constrained +- Health-motivated +- Already believe in plant-based +- Managing family dynamics +- Struggling with consistency + +## Core Problem (Locked In) + +It's not a knowledge problem — it's an execution problem. They know how +they want to eat, but they can't make it work consistently within the +constraints of real life (time, energy, family, decisions). + +## Core Belief (Anchor Statement) + +"Most people don't fail because they don't know what to eat — they fail +because it doesn't work in real life." + +Everything PBS does should reinforce this. + +## Positioning (Locked In) + +Whole food plant-based eating made practical for busy women feeding their +families. We focus on helping them make it actually work in real life — not +just teaching what to eat. + +## Differentiation + +- Focus on execution, not information +- Integrate real life constraints (time, family, energy) +- Combine practical cooking, simple systems, and realistic expectations +- Teach how to make it work, not just what to do + +## Value Proposition (Locked In) + +PBS equips busy women with a repeatable system to consistently feed their +families a whole food plant-based diet — reducing daily decision-making and +making healthy eating sustainable in real life. + +In plain terms: less overwhelm, less decision fatigue, more consistency, +real-life sustainability. + +## Messaging Pillars (4 Pillars) + +1. **Progress Over Perfection** — You don't need to do this perfectly, just +consistently. +2. **Functional, Repeatable Cooking** — Food has to work in real life, not +just look good. +3. **Systems Over Willpower** — You need structure, not more discipline. +4. **Real Life Is the Standard** — If it doesn't work on your busiest day, +it doesn't work. + +## Content Tracks Within Core Audience + +- Culinary education (how to cook plant-based) +- Nutritional science (why plant-based matters for health) +- Quick and easy meals (just get it done) + +## Education Approach + +- Weave the "why" into the "how" rather than leading with the health case +- Show substitutions in context (no dairy, no oil, no meat as options +within familiar dishes) +- Avoid preachy or pushy tone + +## Key Discussion: Mission vs. Revenue Tension + +- Travis raised concern that revenue goals could drive decisions that +compromise education quality +- Root concern: neither are corporate people, and there's a fear that +making money dilutes authenticity +- Resolution: revenue streams should reinforce the mission — teach and sell +under one umbrella by being intentional about which streams to pursue +- Philosophical discussion set aside for now + +## Framework: 3-Day Off-Site Structure + +- Day 1: Problem and positioning (complete) +- Day 2: Business model and revenue streams +- Day 3: Go-to-market and metrics + +## Next Up + +- [ ] Define value proposition further through competitive analysis +- [ ] Map business model and revenue streams (Day 2) +- [ ] Go-to-market strategy and measurable goals (Day 3) + +...sent from Jenny & Travis \ No newline at end of file diff --git a/Sources/Venture/offsite-full-summary.md b/Sources/Venture/offsite-full-summary.md new file mode 100644 index 0000000..bd30597 --- /dev/null +++ b/Sources/Venture/offsite-full-summary.md @@ -0,0 +1,227 @@ +--- +created: 2026-04-20 +path: Sources/Venture +project: offsite-full-summary +status: active +tags: +- pbs +- branding +- sunnies +- content +- revenue +- business-model +- monetization +type: session-notes +updated: 2026-04-22 +--- + +# PBS Off-Site Planning — Full Summary + +## Day 1: Problem & Positioning + +### Vision (Locked In) + +Plant Based Southerner becomes a household name. + +### Mission (Locked In) + +To provide guidance that empowers people to live a whole food plant-based +lifestyle. + +Note: Mission is intentionally broad. Marketing and content target a +specific audience segment. + +### Core Problem (Locked In) + +It's not a knowledge problem — it's an execution problem. They know how +they want to eat, but they can't make it work consistently within the +constraints of real life (time, energy, family, decisions). + +### Core Belief (Anchor Statement) + +"Most people don't fail because they don't know what to eat — they fail +because it doesn't work in real life." + +Everything PBS does should reinforce this. + +### Core Audience (Locked In) + +Busy women feeding families who want to eat a whole food plant-based diet +but struggle to do it consistently in their daily lives. + +Key traits: +- Time-constrained +- Health-motivated +- Already believe in plant-based +- Managing family dynamics +- Struggling with consistency + +### Positioning (Locked In) + +Whole food plant-based eating made practical for busy women feeding their +families. We focus on helping them make it actually work in real life — not +just teaching what to eat. + +### Differentiation + +- Focus on execution, not information +- Integrate real life constraints (time, family, energy) +- Combine practical cooking, simple systems, and realistic expectations +- Teach how to make it work, not just what to do + +### Value Proposition (Locked In) + +PBS equips busy women with a repeatable system to consistently feed their +families a whole food plant-based diet — reducing daily decision-making and +making healthy eating sustainable in real life. + +In plain terms: less overwhelm, less decision fatigue, more consistency, +real-life sustainability. + +### Messaging Pillars (4 Pillars) + +1. Progress Over Perfection — You don't need to do this perfectly, just +consistently. +2. Functional, Repeatable Cooking — Food has to work in real life, not just +look good. +3. Systems Over Willpower — You need structure, not more discipline. +4. Real Life Is the Standard — If it doesn't work on your busiest day, it +doesn't work. + +### Education Approach + +- Weave the "why" into the "how" rather than leading with the health case +- Show substitutions in context (no dairy, no oil, no meat as options +within familiar dishes) +- Avoid preachy or pushy tone + +### Key Discussion: Mission vs. Revenue Tension + +- Travis raised concern that revenue goals could drive decisions that +compromise education quality +- Root concern: neither are corporate people, and there's a fear that +making money dilutes authenticity +- Resolution: revenue streams should reinforce the mission — teach and sell +under one umbrella by being intentional about which streams to pursue +--- +## Day 2: Business Model & Revenue Streams + +### Business Model (Locked In) + +Product-based business with content driving the marketing. + +- Primary products: Cookbooks, courses, membership (higher value) +- Secondary products: Merchandise, meal plans (lower value / nice-to-have) +- Passive/secondary income: Sponsorships, affiliate links, advertising +- Content is the marketing engine, not the core business + +### Current Assets + +- Instagram: ~13,000 followers, Reels 2x/week, Stories daily +- Pinterest: ~9,000 monthly views, drives significant website traffic +- YouTube: ~1 video/month +- Email list: ~80 subscribers (from Instagram via free guide) +- Free guide: 25 weeknight meals, 5-email funnel sequence, 25-45% open rate +- eCookbook: Plant Based Made Simple ($9.99 early / $12.99 public), 10-15 +early sales +- Website: plantbasedsoutherner.com (WordPress on Linode) +- TikTok and Pinterest: present but not heavily utilized + +### 12-Month Revenue Roadmap + +APRIL-MAY 2026 +- eCookbook hard launch (6-week push starting late April) +- Activate affiliate link promotion (organic integration) +- Begin sponsorship/partnership outreach + +JUNE-AUGUST 2026 +- Continue ebook sales and affiliate promotion +- Execute sponsorship deals +- Monitor sales data and audience feedback +- Begin planning second ebook + +SEPTEMBER-OCTOBER 2026 +- Launch second eCookbook +- Continue affiliate and sponsorship streams +- Prepare membership infrastructure (Travis coding) + +NOVEMBER-DECEMBER 2026 +- Launch free membership tier +- Launch tier one paid membership +- Holiday push on ebooks + +JANUARY-APRIL 2027 +- Optimize membership conversion and retention +- Continue all active streams +- Monitor YouTube growth toward monetization threshold +- Plan phase two (courses, expanded membership — 18-24 months out) + +### Revenue Streams (All 12 Months) + +Primary: +1. eCookbooks (2 launches — spring and fall) +2. Membership (free + tier one paid — winter launch) +3. Courses (18+ months out) + +Secondary: +4. Downloadable meal plans +5. Merchandise (towels, aprons, plushies, t-shirts) +6. Affiliate links (organic, ongoing) +7. Sponsorships and partnerships + +Future: +8. YouTube monetization (once threshold met) +9. Digital products (templates, guides) +10. Community add-ons (coaching, challenges, live demos) +--- +## Day 3: Go-to-Market & Metrics + +### Go-to-Market Strategy + +Channels: +- Instagram and Pinterest (double down — proven channels) +- YouTube (grow as third channel) + +Product Ecosystem (Discover → Try → Buy → Belong → Transform): +- Free content (Instagram, Pinterest, YouTube) attracts audience +- Free guide (25 weeknight meals) captures email +- eCookbook converts to first purchase +- Free membership builds community +- Paid membership deepens engagement +- Courses transform long-term (future) + +### Launch Plan: eCookbook (Now) + +- Soft launch: happening now +- Hard launch: 6-week push starting late April +- Channels: Instagram (special launch content + regular cadence) and email +(dedicated promo + regular newsletter) + +### Key Metrics + +Travis's north star: Audience growth (Instagram followers, email list, +YouTube subscribers) + +Jenny's north star: $1,000-$3,000/month in revenue within the next year + +6-Week Launch Targets: +- 1,500 new Instagram followers +- 100 ebook sales + +### Ownership + +- Travis: Tech and business +- Jenny: Products, marketing, and content +--- +## Next Steps + +- [ ] Execute 6-week eCookbook hard launch +- [ ] Activate affiliate link promotion +- [ ] Begin sponsorship/partnership outreach +- [ ] Define second ebook topic and timeline +- [ ] Design membership structure (free vs. paid tier features and pricing) +- [ ] Create sponsorship media kit +- [ ] Set up tracking for ebook sales and affiliate performance +- [ ] Define downloadable meal plan structure and pricing + +...sent from Jenny & Travis \ No newline at end of file diff --git a/Sources/index.md b/Sources/index.md new file mode 100644 index 0000000..7d6ee8e --- /dev/null +++ b/Sources/index.md @@ -0,0 +1,26 @@ +--- +type: topic-landing +path: Sources +tags: + - landing +updated: 2026-05-08 +--- + +# Sources + +All project plans and session notes across domains. + +## By Domain + +- [[Sources/Dev/index|Dev]] — coding projects, dev environment, AI/ML, tooling +- [[Sources/Venture/index|Venture]] — PBS business, content, brand, marketing +- [[Sources/Homelab/index|Homelab]] — infrastructure, networking, deployments +- [[Sources/Reference/index|Reference]] — cross-domain reference notes + +## Active Projects + +*Updated by compile loop. See domain indexes for full listings.* + +## Recent Sessions + +*Updated by compile loop.* diff --git a/Wiki/Dev/index.md b/Wiki/Dev/index.md new file mode 100644 index 0000000..9213659 --- /dev/null +++ b/Wiki/Dev/index.md @@ -0,0 +1,19 @@ +--- +type: topic-landing +path: Wiki/Dev +tags: + - landing +updated: 2026-05-08 +--- + +# Dev — Wiki + +Compiled entities, topic landings, and synthesis for the Dev domain. + +## Entities + +*Updated by compile loop.* + +## Synthesis Pages + +*Updated by compile loop.* diff --git a/Wiki/Homelab/index.md b/Wiki/Homelab/index.md new file mode 100644 index 0000000..3670539 --- /dev/null +++ b/Wiki/Homelab/index.md @@ -0,0 +1,19 @@ +--- +type: topic-landing +path: Wiki/Homelab +tags: + - landing +updated: 2026-05-08 +--- + +# Homelab — Wiki + +Compiled entities, topic landings, and synthesis for the Homelab domain. + +## Entities + +*Updated by compile loop.* + +## Synthesis Pages + +*Updated by compile loop.* diff --git a/Wiki/Reference/index.md b/Wiki/Reference/index.md new file mode 100644 index 0000000..1b347dc --- /dev/null +++ b/Wiki/Reference/index.md @@ -0,0 +1,19 @@ +--- +type: topic-landing +path: Wiki/Reference +tags: + - landing +updated: 2026-05-08 +--- + +# Reference — Wiki + +Compiled entities, topic landings, and synthesis for the Reference domain. + +## Entities + +*Updated by compile loop.* + +## Synthesis Pages + +*Updated by compile loop.* diff --git a/Wiki/Venture/index.md b/Wiki/Venture/index.md new file mode 100644 index 0000000..0ed95b8 --- /dev/null +++ b/Wiki/Venture/index.md @@ -0,0 +1,19 @@ +--- +type: topic-landing +path: Wiki/Venture +tags: + - landing +updated: 2026-05-08 +--- + +# Venture — Wiki + +Compiled entities, topic landings, and synthesis for the Venture domain. + +## Entities + +*Updated by compile loop.* + +## Synthesis Pages + +*Updated by compile loop.* diff --git a/Wiki/index.md b/Wiki/index.md new file mode 100644 index 0000000..4a9cc16 --- /dev/null +++ b/Wiki/index.md @@ -0,0 +1,27 @@ +--- +type: topic-landing +path: Wiki +tags: + - landing +updated: 2026-05-08 +--- + +# Wiki + +Compiled knowledge — entity pages, topic landings, and synthesis. +Generated and maintained by the wiki-maintenance compile skill. + +## By Domain + +- [[Wiki/Dev/index|Dev]] — dev entities, synthesis +- [[Wiki/Venture/index|Venture]] — PBS/business entities, synthesis +- [[Wiki/Homelab/index|Homelab]] — infrastructure entities, synthesis +- [[Wiki/Reference/index|Reference]] — cross-domain entities + +## Key Entities + +*Updated by compile loop.* + +## Synthesis Pages + +*Updated by compile loop.* diff --git a/index.md b/index.md new file mode 100644 index 0000000..ad4b0aa --- /dev/null +++ b/index.md @@ -0,0 +1,35 @@ +--- +type: topic-landing +tags: + - landing +updated: 2026-05-08 +--- + +# Wiki Vault + +Personal knowledge vault — project plans, session notes, and a compiled wiki. + +## Sources + +Raw project plans and session notes, organized by domain. + +- [[Sources/index|Browse all sources]] +- [[Sources/Dev/index|Dev]] — coding, AI/ML, tooling, DeFi +- [[Sources/Venture/index|Venture]] — PBS business, content, brand, marketing +- [[Sources/Homelab/index|Homelab]] — infrastructure, networking, deployments +- [[Sources/Reference/index|Reference]] — cross-domain notes + +## Wiki + +Compiled entity pages, topic landings, and synthesis — generated by the wiki-maintenance skill. + +- [[Wiki/index|Browse the wiki]] +- [[Wiki/Dev/index|Dev wiki]] +- [[Wiki/Venture/index|Venture wiki]] +- [[Wiki/Homelab/index|Homelab wiki]] +- [[Wiki/Reference/index|Reference wiki]] + +## Meta + +- [[CLAUDE|Vault spec]] +- [[log|Compile log]] diff --git a/log.md b/log.md new file mode 100644 index 0000000..f8301f2 --- /dev/null +++ b/log.md @@ -0,0 +1,10 @@ +# Compile Log + +Append-only event log. Each entry records a compile, lint, or migration run. + +## [2026-05-08 20:00] migration | initial repo creation + +- Repository created with two-layer structure (Sources + Wiki) +- Four domains: Dev, Venture, Homelab, Reference +- Migrated from pbs-projects and homelab-projects +- Notes: first entry, no prior compile history diff --git a/migrate.py b/migrate.py new file mode 100644 index 0000000..4611e49 --- /dev/null +++ b/migrate.py @@ -0,0 +1,268 @@ +#!/usr/bin/env python3 +"""Migration script — copies project plans and session notes from +pbs-projects and homelab-projects into wiki-vault's Sources/ layer. + +Copies only, never moves. Generates a report of what went where +and flags anything that couldn't be auto-categorized. + +Usage: + python3 migrate.py --dry-run # preview only + python3 migrate.py # copy files +""" + +import argparse +import shutil +import re +from pathlib import Path +from datetime import date + +try: + import frontmatter +except ImportError: + print("Install python-frontmatter: pip3 install python-frontmatter --break-system-packages") + raise SystemExit(1) + + +# --- Configuration --- + +PBS_PROJECTS = Path("/opt/projects/pbs-projects") +HOMELAB_PROJECTS = Path("/opt/projects/homelab-projects") +WIKI_VAULT = Path("/opt/projects/wiki-vault") +SOURCES = WIKI_VAULT / "Sources" + +# Path pattern → domain mapping +PATH_RULES = [ + # pbs-projects + (r"^Tech/Projects/", "Dev"), + (r"^Tech/Sessions/", "Dev"), + (r"^Tech/Reference/", "Reference"), + (r"^PBS/Content/", "Venture"), + (r"^PBS/Tech/Projects/", "Dev"), # PBS tech is Dev, tag with pbs + (r"^PBS/Tech/Sessions/", "Dev"), + (r"^PBS/Inbox/", None), # manual review + (r"^Business/", "Venture"), + (r"^PBS-Planning/", "Venture"), + # homelab-projects (only markdown notes, not code projects) + (r"^homelab/", "Homelab"), + (r"^Docker/", "Homelab"), + (r"^settings/", None), # skip config files +] + +# Slug-based overrides: files that land in Dev by path rules but belong in Homelab +# These are infrastructure/deployment/networking projects, not coding projects +HOMELAB_OVERRIDES = { + "traefik-deployment", + "ob1-deployment", + "ob1-main-deployment", + "homelab-mcp-server", + "ssh-login-alerting", + "ssh-ca-with-step-ca-authentik", + "sshkm-local-ssh-key-manager", + "env-file-hardening", + "pbs-security-hardening", + "pbs-infrastructure-intelligence", + "cli-standardization", + "instantmesh-docker", + "hunyuan3d-sunnie-pipeline", + # Sessions + "dns-knot-pihole-end-to-end", + "homelab-dns-decision", + "2026-05-08-homelab-vlan-renumber-switch-chip-to-bridge-alignment", + "server-stability-and-security-hardening", + "server-stability-mysql-oom", + "ufw-docker-outage-fix", + "wp-cron-supercronic-deploy", + "2026-05-06-homelab-mcp-server-stand-up", +} + +# Folders in homelab-projects that are code repos, not notes +SKIP_DIRS = { + "OB1", "homelab-mcp-server", "second-brain", + ".git", ".claude", ".venv", "node_modules", +} + + +def determine_domain(relative_path: str, meta: dict) -> str | None: + """Determine target domain from path and metadata.""" + # Check slug-based overrides first (infrastructure projects that path rules would put in Dev) + filename_stem = relative_path.rsplit("/", 1)[-1].replace(".md", "") + if filename_stem in HOMELAB_OVERRIDES: + return "Homelab" + + for pattern, domain in PATH_RULES: + if re.match(pattern, relative_path): + return domain + + # Fallback: check tags for hints + tags = meta.get("tags", []) + if isinstance(tags, list): + if "homelab" in tags: + return "Homelab" + if "pbs" in tags or "content" in tags: + return "Venture" + + return None + + +def is_note_file(path: Path) -> bool: + """Check if a file is a markdown note (not config, not code).""" + if not path.is_file(): + return False + if path.suffix != ".md": + return False + if path.name in ("README.md", "CLAUDE.md", "CONTRIBUTING.md", + "CODE_OF_CONDUCT.md", "SECURITY.md", "LICENSE.md", + "BUILD-LOG.md", "DEPLOYMENT-LOG.md", "SETUP.md"): + return False + return True + + +def scan_repo(repo_path: Path, repo_name: str) -> list[dict]: + """Scan a repo for markdown notes with frontmatter.""" + results = [] + + for md_file in sorted(repo_path.rglob("*.md")): + # Skip hidden dirs and code project dirs + rel = md_file.relative_to(repo_path) + parts = rel.parts + if any(p in SKIP_DIRS or p.startswith(".") for p in parts): + continue + + if not is_note_file(md_file): + continue + + try: + post = frontmatter.load(md_file) + meta = dict(post.metadata) + except Exception: + meta = {} + + note_type = meta.get("type", "unknown") + + # Only migrate project plans and session notes + if note_type not in ("project-plan", "session-notes", "session"): + # Check if it looks like a project plan even without correct type + content = md_file.read_text(encoding="utf-8", errors="replace") + if "## Goal" in content or "## Locked Decisions" in content or "## Phases" in content: + note_type = "project-plan" + elif "## Outcome" in content or "## Topics Covered" in content: + note_type = "session-notes" + else: + continue # skip non-note files + + rel_str = str(rel) + domain = determine_domain(rel_str, meta) + + results.append({ + "source_repo": repo_name, + "source_path": str(md_file), + "relative_path": rel_str, + "filename": md_file.name, + "type": note_type, + "domain": domain, + "meta": meta, + "project": meta.get("project", ""), + "status": meta.get("status", ""), + "tags": meta.get("tags", []), + }) + + return results + + +def update_frontmatter(content: str, new_path: str) -> str: + """Update the path field in frontmatter to reflect new location.""" + try: + post = frontmatter.loads(content) + post.metadata["path"] = new_path + return frontmatter.dumps(post) + except Exception: + return content + + +def copy_file(entry: dict, dry_run: bool = False) -> str: + """Copy a file to its target location. Returns status message.""" + domain = entry["domain"] + if not domain: + return f"SKIP (no domain): {entry['relative_path']}" + + target_dir = SOURCES / domain + target_path = target_dir / entry["filename"] + + if target_path.exists(): + return f"SKIP (exists): {entry['filename']} → Sources/{domain}/" + + if dry_run: + return f"WOULD COPY: {entry['relative_path']} → Sources/{domain}/{entry['filename']}" + + # Read, update frontmatter, write + source = Path(entry["source_path"]) + content = source.read_text(encoding="utf-8", errors="replace") + updated = update_frontmatter(content, f"Sources/{domain}") + target_path.write_text(updated, encoding="utf-8") + + return f"COPIED: {entry['relative_path']} → Sources/{domain}/{entry['filename']}" + + +def main(): + parser = argparse.ArgumentParser(description="Migrate notes to wiki-vault") + parser.add_argument("--dry-run", action="store_true", help="Preview only, don't copy") + args = parser.parse_args() + + print("Scanning repositories...\n") + + all_entries = [] + if PBS_PROJECTS.exists(): + all_entries.extend(scan_repo(PBS_PROJECTS, "pbs-projects")) + if HOMELAB_PROJECTS.exists(): + all_entries.extend(scan_repo(HOMELAB_PROJECTS, "homelab-projects")) + + print(f"Found {len(all_entries)} note files\n") + + # Categorize + by_domain = {"Dev": [], "Venture": [], "Homelab": [], "Reference": [], None: []} + for entry in all_entries: + by_domain.setdefault(entry["domain"], []).append(entry) + + # Report + print("=" * 60) + print("MIGRATION REPORT") + print("=" * 60) + + for domain in ["Dev", "Venture", "Homelab", "Reference"]: + entries = by_domain.get(domain, []) + print(f"\n{domain}: {len(entries)} files") + for e in entries: + print(f" {e['relative_path']} ({e['type']})") + + uncategorized = by_domain.get(None, []) + if uncategorized: + print(f"\nUNCATEGORIZED (manual review needed): {len(uncategorized)} files") + for e in uncategorized: + print(f" {e['source_repo']}: {e['relative_path']}") + + # Copy + print(f"\n{'=' * 60}") + print(f"{'DRY RUN' if args.dry_run else 'COPYING FILES'}") + print(f"{'=' * 60}\n") + + copied = 0 + skipped = 0 + for entry in all_entries: + result = copy_file(entry, dry_run=args.dry_run) + print(f" {result}") + if result.startswith("COPIED") or result.startswith("WOULD COPY"): + copied += 1 + else: + skipped += 1 + + print(f"\nDone: {copied} {'would be copied' if args.dry_run else 'copied'}, {skipped} skipped") + + if not args.dry_run and copied > 0: + print(f"\nNext steps:") + print(f" cd /opt/projects/wiki-vault") + print(f" git add -A") + print(f" git commit -m 'migration: copy {copied} files from pbs-projects and homelab-projects'") + + +if __name__ == "__main__": + main()