34a268d8dc
Two-layer structure: Sources (raw notes) + Wiki (compile output) Four domains: Dev (40), Venture (3), Homelab (23), Reference (0) Includes CLAUDE.md spec, index pages at all levels, compile log Co-Authored-By: Lovebug <lovebug@herbylab.dev>
269 lines
8.7 KiB
Python
269 lines
8.7 KiB
Python
#!/usr/bin/env python3
|
|
"""Migration script — copies project plans and session notes from
|
|
pbs-projects and homelab-projects into wiki-vault's Sources/ layer.
|
|
|
|
Copies only, never moves. Generates a report of what went where
|
|
and flags anything that couldn't be auto-categorized.
|
|
|
|
Usage:
|
|
python3 migrate.py --dry-run # preview only
|
|
python3 migrate.py # copy files
|
|
"""
|
|
|
|
import argparse
|
|
import shutil
|
|
import re
|
|
from pathlib import Path
|
|
from datetime import date
|
|
|
|
try:
|
|
import frontmatter
|
|
except ImportError:
|
|
print("Install python-frontmatter: pip3 install python-frontmatter --break-system-packages")
|
|
raise SystemExit(1)
|
|
|
|
|
|
# --- Configuration ---
|
|
|
|
PBS_PROJECTS = Path("/opt/projects/pbs-projects")
|
|
HOMELAB_PROJECTS = Path("/opt/projects/homelab-projects")
|
|
WIKI_VAULT = Path("/opt/projects/wiki-vault")
|
|
SOURCES = WIKI_VAULT / "Sources"
|
|
|
|
# Path pattern → domain mapping
|
|
PATH_RULES = [
|
|
# pbs-projects
|
|
(r"^Tech/Projects/", "Dev"),
|
|
(r"^Tech/Sessions/", "Dev"),
|
|
(r"^Tech/Reference/", "Reference"),
|
|
(r"^PBS/Content/", "Venture"),
|
|
(r"^PBS/Tech/Projects/", "Dev"), # PBS tech is Dev, tag with pbs
|
|
(r"^PBS/Tech/Sessions/", "Dev"),
|
|
(r"^PBS/Inbox/", None), # manual review
|
|
(r"^Business/", "Venture"),
|
|
(r"^PBS-Planning/", "Venture"),
|
|
# homelab-projects (only markdown notes, not code projects)
|
|
(r"^homelab/", "Homelab"),
|
|
(r"^Docker/", "Homelab"),
|
|
(r"^settings/", None), # skip config files
|
|
]
|
|
|
|
# Slug-based overrides: files that land in Dev by path rules but belong in Homelab
|
|
# These are infrastructure/deployment/networking projects, not coding projects
|
|
HOMELAB_OVERRIDES = {
|
|
"traefik-deployment",
|
|
"ob1-deployment",
|
|
"ob1-main-deployment",
|
|
"homelab-mcp-server",
|
|
"ssh-login-alerting",
|
|
"ssh-ca-with-step-ca-authentik",
|
|
"sshkm-local-ssh-key-manager",
|
|
"env-file-hardening",
|
|
"pbs-security-hardening",
|
|
"pbs-infrastructure-intelligence",
|
|
"cli-standardization",
|
|
"instantmesh-docker",
|
|
"hunyuan3d-sunnie-pipeline",
|
|
# Sessions
|
|
"dns-knot-pihole-end-to-end",
|
|
"homelab-dns-decision",
|
|
"2026-05-08-homelab-vlan-renumber-switch-chip-to-bridge-alignment",
|
|
"server-stability-and-security-hardening",
|
|
"server-stability-mysql-oom",
|
|
"ufw-docker-outage-fix",
|
|
"wp-cron-supercronic-deploy",
|
|
"2026-05-06-homelab-mcp-server-stand-up",
|
|
}
|
|
|
|
# Folders in homelab-projects that are code repos, not notes
|
|
SKIP_DIRS = {
|
|
"OB1", "homelab-mcp-server", "second-brain",
|
|
".git", ".claude", ".venv", "node_modules",
|
|
}
|
|
|
|
|
|
def determine_domain(relative_path: str, meta: dict) -> str | None:
|
|
"""Determine target domain from path and metadata."""
|
|
# Check slug-based overrides first (infrastructure projects that path rules would put in Dev)
|
|
filename_stem = relative_path.rsplit("/", 1)[-1].replace(".md", "")
|
|
if filename_stem in HOMELAB_OVERRIDES:
|
|
return "Homelab"
|
|
|
|
for pattern, domain in PATH_RULES:
|
|
if re.match(pattern, relative_path):
|
|
return domain
|
|
|
|
# Fallback: check tags for hints
|
|
tags = meta.get("tags", [])
|
|
if isinstance(tags, list):
|
|
if "homelab" in tags:
|
|
return "Homelab"
|
|
if "pbs" in tags or "content" in tags:
|
|
return "Venture"
|
|
|
|
return None
|
|
|
|
|
|
def is_note_file(path: Path) -> bool:
|
|
"""Check if a file is a markdown note (not config, not code)."""
|
|
if not path.is_file():
|
|
return False
|
|
if path.suffix != ".md":
|
|
return False
|
|
if path.name in ("README.md", "CLAUDE.md", "CONTRIBUTING.md",
|
|
"CODE_OF_CONDUCT.md", "SECURITY.md", "LICENSE.md",
|
|
"BUILD-LOG.md", "DEPLOYMENT-LOG.md", "SETUP.md"):
|
|
return False
|
|
return True
|
|
|
|
|
|
def scan_repo(repo_path: Path, repo_name: str) -> list[dict]:
|
|
"""Scan a repo for markdown notes with frontmatter."""
|
|
results = []
|
|
|
|
for md_file in sorted(repo_path.rglob("*.md")):
|
|
# Skip hidden dirs and code project dirs
|
|
rel = md_file.relative_to(repo_path)
|
|
parts = rel.parts
|
|
if any(p in SKIP_DIRS or p.startswith(".") for p in parts):
|
|
continue
|
|
|
|
if not is_note_file(md_file):
|
|
continue
|
|
|
|
try:
|
|
post = frontmatter.load(md_file)
|
|
meta = dict(post.metadata)
|
|
except Exception:
|
|
meta = {}
|
|
|
|
note_type = meta.get("type", "unknown")
|
|
|
|
# Only migrate project plans and session notes
|
|
if note_type not in ("project-plan", "session-notes", "session"):
|
|
# Check if it looks like a project plan even without correct type
|
|
content = md_file.read_text(encoding="utf-8", errors="replace")
|
|
if "## Goal" in content or "## Locked Decisions" in content or "## Phases" in content:
|
|
note_type = "project-plan"
|
|
elif "## Outcome" in content or "## Topics Covered" in content:
|
|
note_type = "session-notes"
|
|
else:
|
|
continue # skip non-note files
|
|
|
|
rel_str = str(rel)
|
|
domain = determine_domain(rel_str, meta)
|
|
|
|
results.append({
|
|
"source_repo": repo_name,
|
|
"source_path": str(md_file),
|
|
"relative_path": rel_str,
|
|
"filename": md_file.name,
|
|
"type": note_type,
|
|
"domain": domain,
|
|
"meta": meta,
|
|
"project": meta.get("project", ""),
|
|
"status": meta.get("status", ""),
|
|
"tags": meta.get("tags", []),
|
|
})
|
|
|
|
return results
|
|
|
|
|
|
def update_frontmatter(content: str, new_path: str) -> str:
|
|
"""Update the path field in frontmatter to reflect new location."""
|
|
try:
|
|
post = frontmatter.loads(content)
|
|
post.metadata["path"] = new_path
|
|
return frontmatter.dumps(post)
|
|
except Exception:
|
|
return content
|
|
|
|
|
|
def copy_file(entry: dict, dry_run: bool = False) -> str:
|
|
"""Copy a file to its target location. Returns status message."""
|
|
domain = entry["domain"]
|
|
if not domain:
|
|
return f"SKIP (no domain): {entry['relative_path']}"
|
|
|
|
target_dir = SOURCES / domain
|
|
target_path = target_dir / entry["filename"]
|
|
|
|
if target_path.exists():
|
|
return f"SKIP (exists): {entry['filename']} → Sources/{domain}/"
|
|
|
|
if dry_run:
|
|
return f"WOULD COPY: {entry['relative_path']} → Sources/{domain}/{entry['filename']}"
|
|
|
|
# Read, update frontmatter, write
|
|
source = Path(entry["source_path"])
|
|
content = source.read_text(encoding="utf-8", errors="replace")
|
|
updated = update_frontmatter(content, f"Sources/{domain}")
|
|
target_path.write_text(updated, encoding="utf-8")
|
|
|
|
return f"COPIED: {entry['relative_path']} → Sources/{domain}/{entry['filename']}"
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description="Migrate notes to wiki-vault")
|
|
parser.add_argument("--dry-run", action="store_true", help="Preview only, don't copy")
|
|
args = parser.parse_args()
|
|
|
|
print("Scanning repositories...\n")
|
|
|
|
all_entries = []
|
|
if PBS_PROJECTS.exists():
|
|
all_entries.extend(scan_repo(PBS_PROJECTS, "pbs-projects"))
|
|
if HOMELAB_PROJECTS.exists():
|
|
all_entries.extend(scan_repo(HOMELAB_PROJECTS, "homelab-projects"))
|
|
|
|
print(f"Found {len(all_entries)} note files\n")
|
|
|
|
# Categorize
|
|
by_domain = {"Dev": [], "Venture": [], "Homelab": [], "Reference": [], None: []}
|
|
for entry in all_entries:
|
|
by_domain.setdefault(entry["domain"], []).append(entry)
|
|
|
|
# Report
|
|
print("=" * 60)
|
|
print("MIGRATION REPORT")
|
|
print("=" * 60)
|
|
|
|
for domain in ["Dev", "Venture", "Homelab", "Reference"]:
|
|
entries = by_domain.get(domain, [])
|
|
print(f"\n{domain}: {len(entries)} files")
|
|
for e in entries:
|
|
print(f" {e['relative_path']} ({e['type']})")
|
|
|
|
uncategorized = by_domain.get(None, [])
|
|
if uncategorized:
|
|
print(f"\nUNCATEGORIZED (manual review needed): {len(uncategorized)} files")
|
|
for e in uncategorized:
|
|
print(f" {e['source_repo']}: {e['relative_path']}")
|
|
|
|
# Copy
|
|
print(f"\n{'=' * 60}")
|
|
print(f"{'DRY RUN' if args.dry_run else 'COPYING FILES'}")
|
|
print(f"{'=' * 60}\n")
|
|
|
|
copied = 0
|
|
skipped = 0
|
|
for entry in all_entries:
|
|
result = copy_file(entry, dry_run=args.dry_run)
|
|
print(f" {result}")
|
|
if result.startswith("COPIED") or result.startswith("WOULD COPY"):
|
|
copied += 1
|
|
else:
|
|
skipped += 1
|
|
|
|
print(f"\nDone: {copied} {'would be copied' if args.dry_run else 'copied'}, {skipped} skipped")
|
|
|
|
if not args.dry_run and copied > 0:
|
|
print(f"\nNext steps:")
|
|
print(f" cd /opt/projects/wiki-vault")
|
|
print(f" git add -A")
|
|
print(f" git commit -m 'migration: copy {copied} files from pbs-projects and homelab-projects'")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|