project init

This commit is contained in:
Travis Herbranson 2026-05-22 19:11:06 -04:00
commit fc85cd2abe
16 changed files with 2702 additions and 0 deletions

10
.gitignore vendored Normal file
View File

@ -0,0 +1,10 @@
# Python-generated files
__pycache__/
*.py[oc]
build/
dist/
wheels/
*.egg-info
# Virtual environments
.venv

1
.python-version Normal file
View File

@ -0,0 +1 @@
3.12

67
CLAUDE.md Normal file
View File

@ -0,0 +1,67 @@
# YouTube Analytics — CLAUDE.md
PBS YouTube channel analytics pipeline: collect data from YouTube APIs, store in SQLite, visualize with Streamlit.
## Project Structure
```
main.py — Click CLI entry point (init-db, seed, collect, dashboard)
src/
auth.py — OAuth2 authentication (YouTube Data API v3 + Analytics API v2)
collector.py — Data collection (videos, daily metrics, retention, traffic sources, channel stats)
database.py — SQLite schema (5 tables) and connection management (WAL mode, foreign keys)
dashboard.py — Streamlit dashboard (charts, tables, retention curves)
seed.py — Sample data generator (PBS plant-based cooking channel theme)
config/ — OAuth credentials (git-ignored, not committed)
data/ — SQLite database (git-ignored)
```
## Package Manager
Uses **uv** (not pip, not poetry). Python >=3.12.
```bash
uv sync # install deps from lockfile
uv add <pkg> # add a dependency
uv run python main.py <command> # run within managed venv
```
## CLI Commands
```bash
uv run python main.py init-db-cmd # create database tables
uv run python main.py seed # populate with sample data
uv run python main.py collect # pull real data (requires OAuth creds in config/)
uv run python main.py dashboard # launch Streamlit dashboard
```
## Database
SQLite at `data/pbs_youtube.db` with five tables:
- `videos` — video metadata (id, title, published_at, duration, type)
- `video_daily_metrics` — per-video daily stats (views, watch time, CTR, engagement)
- `video_retention` — 100-point audience retention curves per video
- `video_traffic_sources` — traffic source breakdown per video per day
- `channel_daily_metrics` — channel-level daily summary
All inserts use `ON CONFLICT DO UPDATE` for idempotent runs.
## Authentication
Real YouTube API access requires:
1. Create OAuth credentials at https://console.cloud.google.com/apis/credentials
2. Download client secret JSON → save as `config/client_secret.json`
3. First run of `collect` opens browser for OAuth consent, stores token at `config/token.json`
For development/testing, use `seed` command instead — no credentials needed.
## Key Dependencies
click, streamlit, plotly, pandas, google-api-python-client, google-auth-oauthlib
## Conventions
- All data collection functions live in `collector.py`, one function per data type
- Dashboard uses Plotly for charts via `streamlit` + `plotly`
- Duration parsing handles ISO 8601 (PT1H2M3S format)
- Shorts classified as videos ≤60 seconds

0
README.md Normal file
View File

12
config/client_secret.json Normal file
View File

@ -0,0 +1,12 @@
{
"installed":
{
"client_id":"918670827378-ngeo2grfmm82orjhru6bs8g5hrcht0cd.apps.googleusercontent.com",
"project_id":"pbs-site-486819",
"auth_uri":"https://accounts.google.com/o/oauth2/auth",
"token_uri":"https://oauth2.googleapis.com/token",
"auth_provider_x509_cert_url":"https://www.googleapis.com/oauth2/v1/certs",
"client_secret":"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",
"redirect_uris":["http://localhost"]
}
}

1
config/token.json Normal file
View File

@ -0,0 +1 @@
{"token": "ya29.a0AQvPyINTBUmYT0TS8M6C6QhWHVswoI0coM6ycnodz_mJpqITBBXXfh4sVILepbZ8fVz60Bfog95aqFl47T6NFkb5_GXGlSA_1iq-_Q0bQg_Qc2_L0J_8nZQ4ubpTr2FV5K2ipe-er3QU4999C4OXQSxdB9FEMCuNrT_r_Aa3ojqdb2OMZIKwPuw4Ctqp1cX_9RZ03kqCaCgYKAYgSARQSFQHGX2Mi62N5gg_yUtdFoY3t9Uh4lw0207", "refresh_token": "1//05Zaxlboo0cyOCgYIARAAGAUSNwF-L9IrAplENIYSCGlR4dcA07L0_f2344h793lvG2eRzFeFRgDKTYtMPso9D8DvYMbNDgNWGb4", "token_uri": "https://oauth2.googleapis.com/token", "client_id": "918670827378-ngeo2grfmm82orjhru6bs8g5hrcht0cd.apps.googleusercontent.com", "client_secret": "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", "scopes": ["https://www.googleapis.com/auth/yt-analytics.readonly", "https://www.googleapis.com/auth/youtube.readonly"], "universe_domain": "googleapis.com", "account": "", "expiry": "2026-05-04T02:25:42.357613Z"}

BIN
data/pbs_youtube.db Normal file

Binary file not shown.

159
main.py Normal file
View File

@ -0,0 +1,159 @@
"""CLI entry point for PBS YouTube Analytics."""
import click
from pathlib import Path
from src.database import init_db, get_connection, DEFAULT_DB_PATH
from src.seed import seed_database
@click.group()
def cli():
"""PBS YouTube Analytics CLI."""
pass
@cli.command()
def init_db_cmd():
"""Initialize the database schema."""
try:
init_db(DEFAULT_DB_PATH)
click.echo(click.style("✓ Database initialized successfully", fg="green"))
except Exception as e:
click.echo(click.style(f"✗ Error initializing database: {e}", fg="red"), err=True)
raise SystemExit(1)
@cli.command()
def seed():
"""Populate the database with sample data."""
try:
seed_database(DEFAULT_DB_PATH)
click.echo(click.style("✓ Database seeded successfully", fg="green"))
except Exception as e:
click.echo(click.style(f"✗ Error seeding database: {e}", fg="red"), err=True)
raise SystemExit(1)
@cli.command("clear-seed")
def clear_seed():
"""Remove sample seed data (videos with IDs matching short_* or long_*)."""
try:
conn = get_connection(DEFAULT_DB_PATH)
# Seed video IDs are short_NNN / long_NNN — real YouTube IDs are 11-char alphanumeric,
# so these patterns can never collide with collected data.
seed_pattern = "video_id LIKE 'short_%' OR video_id LIKE 'long_%'"
totals = {}
for table in ("video_traffic_sources", "video_retention", "video_daily_metrics"):
cur = conn.execute(f"DELETE FROM {table} WHERE {seed_pattern}")
totals[table] = cur.rowcount
cur = conn.execute(f"DELETE FROM videos WHERE {seed_pattern}")
totals["videos"] = cur.rowcount
conn.commit()
conn.close()
for table, count in totals.items():
click.echo(f" {table}: {count} rows deleted")
click.echo(click.style("✓ Seed data cleared", fg="green"))
click.echo(" Note: channel_daily_metrics rows have no seed identifier and are left untouched.")
except Exception as e:
click.echo(click.style(f"✗ Error clearing seed data: {e}", fg="red"), err=True)
raise SystemExit(1)
@cli.command()
@click.option("--start-date", default=None, help="Start date (YYYY-MM-DD), default: 30 days ago")
@click.option("--end-date", default=None, help="End date (YYYY-MM-DD), default: 2 days ago")
def collect(start_date, end_date):
"""Collect real data from YouTube API."""
import sqlite3
try:
from src.auth import get_authenticated_service
from src.collector import (
collect_channel_metrics,
collect_daily_metrics,
collect_retention,
collect_traffic_sources,
collect_video_list,
)
click.echo("Attempting to authenticate with YouTube API...")
youtube, youtube_analytics = get_authenticated_service()
click.echo(click.style("✓ Successfully authenticated with YouTube API", fg="green"))
# Ensure database exists
init_db(DEFAULT_DB_PATH)
conn = sqlite3.connect(DEFAULT_DB_PATH)
conn.execute("PRAGMA journal_mode=WAL")
conn.execute("PRAGMA foreign_keys=ON")
# Step 1: Collect video list
click.echo("\n📹 Collecting video list...")
video_ids = collect_video_list(youtube, conn)
# Step 2: Collect daily metrics per video
click.echo("📊 Collecting daily metrics...")
collect_daily_metrics(youtube_analytics, conn, video_ids, start_date, end_date)
# Step 3: Collect retention curves
click.echo("📈 Collecting retention data...")
collect_retention(youtube_analytics, conn, video_ids)
# Step 4: Collect traffic sources
click.echo("🔗 Collecting traffic sources...")
collect_traffic_sources(youtube_analytics, conn, video_ids, start_date, end_date)
# Step 5: Collect channel-level metrics
click.echo("📺 Collecting channel metrics...")
collect_channel_metrics(youtube_analytics, conn, start_date, end_date)
conn.close()
click.echo(click.style("\n✓ All data collected successfully!", fg="green"))
except FileNotFoundError as e:
click.echo(click.style(f"{e}", fg="red"), err=True)
click.echo("\nTo use the collect command:")
click.echo("1. Create credentials at: https://console.cloud.google.com/apis/credentials")
click.echo("2. Download OAuth credentials as client_secret.json")
click.echo("3. Save to: config/client_secret.json")
raise SystemExit(1)
except Exception as e:
click.echo(click.style(f"✗ Error during collection: {e}", fg="red"), err=True)
import traceback
traceback.print_exc()
raise SystemExit(1)
@cli.command()
def dashboard():
"""Launch the Streamlit dashboard."""
try:
import streamlit # noqa: F401 — just checking it's installed
except ImportError:
click.echo(
click.style("✗ Streamlit not installed. Install with: uv add streamlit", fg="red"),
err=True,
)
raise SystemExit(1)
import subprocess
import sys
dashboard_path = str(Path(__file__).parent / "src" / "dashboard.py")
try:
subprocess.run(
[sys.executable, "-m", "streamlit", "run", dashboard_path],
check=True,
)
except Exception as e:
click.echo(click.style(f"✗ Error launching dashboard: {e}", fg="red"), err=True)
raise SystemExit(1)
if __name__ == "__main__":
cli()

15
pyproject.toml Normal file
View File

@ -0,0 +1,15 @@
[project]
name = "youtube-analytics"
version = "0.1.0"
description = "Add your description here"
readme = "README.md"
requires-python = ">=3.12"
dependencies = [
"click>=8.3.2",
"google-api-python-client>=2.194.0",
"google-auth-httplib2>=0.3.1",
"google-auth-oauthlib>=1.3.1",
"pandas>=3.0.2",
"plotly>=6.7.0",
"streamlit>=1.56.0",
]

0
src/__init__.py Normal file
View File

56
src/auth.py Normal file
View File

@ -0,0 +1,56 @@
"""YouTube API authentication module with OAuth2 token persistence.
Replace PLACEHOLDER values with real credentials from Google Cloud Console.
See: https://console.cloud.google.com/apis/credentials
"""
from pathlib import Path
SCOPES = [
"https://www.googleapis.com/auth/yt-analytics.readonly",
"https://www.googleapis.com/auth/youtube.readonly",
]
CONFIG_DIR = Path(__file__).parent.parent / "config"
CLIENT_SECRET_PATH = CONFIG_DIR / "client_secret.json"
TOKEN_PATH = CONFIG_DIR / "token.json"
def get_authenticated_service():
"""Build and return authenticated YouTube API services.
Returns a tuple of (youtube_data_api, youtube_analytics_api).
On first run, opens a browser for OAuth consent.
Subsequent runs use the stored refresh token.
"""
from google.oauth2.credentials import Credentials
from google_auth_oauthlib.flow import InstalledAppFlow
from google.auth.transport.requests import Request
from googleapiclient.discovery import build
creds = None
if TOKEN_PATH.exists():
creds = Credentials.from_authorized_user_file(str(TOKEN_PATH), SCOPES)
if not creds or not creds.valid:
if creds and creds.expired and creds.refresh_token:
creds.refresh(Request())
else:
if not CLIENT_SECRET_PATH.exists():
raise FileNotFoundError(
f"Missing {CLIENT_SECRET_PATH}. Download OAuth credentials from "
"Google Cloud Console and save as config/client_secret.json"
)
flow = InstalledAppFlow.from_client_secrets_file(
str(CLIENT_SECRET_PATH), SCOPES
)
creds = flow.run_local_server(port=0)
TOKEN_PATH.write_text(creds.to_json())
youtube = build("youtube", "v3", credentials=creds)
youtube_analytics = build("youtubeAnalytics", "v2", credentials=creds)
return youtube, youtube_analytics

228
src/collector.py Normal file
View File

@ -0,0 +1,228 @@
"""YouTube data collector — pulls video metadata, metrics, and retention data."""
import sqlite3
from datetime import datetime, timedelta
def collect_video_list(youtube, conn: sqlite3.Connection) -> list[str]:
"""Fetch all videos from the channel and upsert into the videos table."""
video_ids = []
request = youtube.channels().list(part="contentDetails", mine=True)
response = request.execute()
uploads_playlist = response["items"][0]["contentDetails"]["relatedPlaylists"]["uploads"]
next_page = None
while True:
pl_request = youtube.playlistItems().list(
part="snippet,contentDetails",
playlistId=uploads_playlist,
maxResults=50,
pageToken=next_page,
)
pl_response = pl_request.execute()
ids = [item["contentDetails"]["videoId"] for item in pl_response["items"]]
video_ids.extend(ids)
# Batch video details (duration, type)
for i in range(0, len(ids), 50):
batch = ids[i : i + 50]
vid_request = youtube.videos().list(
part="snippet,contentDetails", id=",".join(batch)
)
vid_response = vid_request.execute()
for video in vid_response["items"]:
duration = _parse_duration(video["contentDetails"]["duration"])
video_type = "short" if duration <= 60 else "video"
conn.execute(
"""INSERT INTO videos (video_id, title, published_at, duration_seconds,
video_type, thumbnail_url, description, updated_at)
VALUES (?, ?, ?, ?, ?, ?, ?, CURRENT_TIMESTAMP)
ON CONFLICT(video_id) DO UPDATE SET
title=excluded.title, duration_seconds=excluded.duration_seconds,
video_type=excluded.video_type, thumbnail_url=excluded.thumbnail_url,
description=excluded.description, updated_at=CURRENT_TIMESTAMP""",
(
video["id"],
video["snippet"]["title"],
video["snippet"]["publishedAt"],
duration,
video_type,
video["snippet"]["thumbnails"].get("high", {}).get("url", ""),
video["snippet"].get("description", ""),
),
)
conn.commit()
next_page = pl_response.get("nextPageToken")
if not next_page:
break
print(f"Collected {len(video_ids)} videos")
return video_ids
def collect_daily_metrics(
youtube_analytics, conn: sqlite3.Connection, video_ids: list[str],
start_date: str | None = None, end_date: str | None = None,
) -> None:
"""Fetch daily metrics for each video from the Analytics API."""
if not end_date:
end_date = (datetime.now() - timedelta(days=2)).strftime("%Y-%m-%d")
if not start_date:
start_date = (datetime.now() - timedelta(days=30)).strftime("%Y-%m-%d")
for vid in video_ids:
response = youtube_analytics.reports().query(
ids="channel==MINE",
startDate=start_date,
endDate=end_date,
metrics="views,estimatedMinutesWatched,averageViewDuration,averageViewPercentage,likes,dislikes,comments,shares,subscribersGained,subscribersLost",
dimensions="day",
filters=f"video=={vid}",
sort="day",
).execute()
for row in response.get("rows", []):
conn.execute(
"""INSERT INTO video_daily_metrics
(video_id, date, views, estimated_minutes_watched,
average_view_duration_seconds, average_view_percentage,
likes, dislikes, comments, shares,
subscribers_gained, subscribers_lost)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
ON CONFLICT(video_id, date) DO UPDATE SET
views=excluded.views,
estimated_minutes_watched=excluded.estimated_minutes_watched,
average_view_duration_seconds=excluded.average_view_duration_seconds,
average_view_percentage=excluded.average_view_percentage,
likes=excluded.likes, dislikes=excluded.dislikes,
comments=excluded.comments, shares=excluded.shares,
subscribers_gained=excluded.subscribers_gained,
subscribers_lost=excluded.subscribers_lost""",
(vid, row[0], *row[1:]),
)
conn.commit()
print(f"Collected daily metrics for {len(video_ids)} videos")
def collect_retention(
youtube_analytics, conn: sqlite3.Connection, video_ids: list[str],
) -> None:
"""Fetch audience retention curves for each video."""
for vid in video_ids:
response = youtube_analytics.reports().query(
ids="channel==MINE",
startDate="2020-01-01",
endDate=datetime.now().strftime("%Y-%m-%d"),
metrics="audienceWatchRatio,relativeRetentionPerformance",
dimensions="elapsedVideoTimeRatio",
filters=f"video=={vid}",
sort="elapsedVideoTimeRatio",
).execute()
for row in response.get("rows", []):
conn.execute(
"""INSERT INTO video_retention
(video_id, elapsed_ratio, audience_watch_ratio,
relative_retention_performance, fetched_at)
VALUES (?, ?, ?, ?, CURRENT_TIMESTAMP)
ON CONFLICT(video_id, elapsed_ratio) DO UPDATE SET
audience_watch_ratio=excluded.audience_watch_ratio,
relative_retention_performance=excluded.relative_retention_performance,
fetched_at=CURRENT_TIMESTAMP""",
(vid, row[0], row[1], row[2]),
)
conn.commit()
print(f"Collected retention data for {len(video_ids)} videos")
def collect_traffic_sources(
youtube_analytics, conn: sqlite3.Connection, video_ids: list[str],
start_date: str | None = None, end_date: str | None = None,
) -> None:
"""Fetch traffic source breakdown for each video."""
if not end_date:
end_date = (datetime.now() - timedelta(days=2)).strftime("%Y-%m-%d")
if not start_date:
start_date = (datetime.now() - timedelta(days=30)).strftime("%Y-%m-%d")
for vid in video_ids:
response = youtube_analytics.reports().query(
ids="channel==MINE",
startDate=start_date,
endDate=end_date,
metrics="views,estimatedMinutesWatched",
dimensions="day,insightTrafficSourceType",
filters=f"video=={vid}",
sort="day",
).execute()
for row in response.get("rows", []):
conn.execute(
"""INSERT INTO video_traffic_sources
(video_id, date, traffic_source, views, estimated_minutes_watched)
VALUES (?, ?, ?, ?, ?)
ON CONFLICT(video_id, date, traffic_source) DO UPDATE SET
views=excluded.views,
estimated_minutes_watched=excluded.estimated_minutes_watched""",
(vid, row[0], row[1], row[2], row[3]),
)
conn.commit()
print(f"Collected traffic source data for {len(video_ids)} videos")
def collect_channel_metrics(
youtube_analytics, conn: sqlite3.Connection,
start_date: str | None = None, end_date: str | None = None,
) -> None:
"""Fetch channel-level daily summary metrics."""
if not end_date:
end_date = (datetime.now() - timedelta(days=2)).strftime("%Y-%m-%d")
if not start_date:
start_date = (datetime.now() - timedelta(days=30)).strftime("%Y-%m-%d")
response = youtube_analytics.reports().query(
ids="channel==MINE",
startDate=start_date,
endDate=end_date,
metrics="views,estimatedMinutesWatched,subscribersGained,subscribersLost",
dimensions="day",
sort="day",
).execute()
for row in response.get("rows", []):
conn.execute(
"""INSERT INTO channel_daily_metrics
(date, total_views, total_estimated_minutes_watched,
subscribers_gained, subscribers_lost, net_subscribers)
VALUES (?, ?, ?, ?, ?, ?)
ON CONFLICT(date) DO UPDATE SET
total_views=excluded.total_views,
total_estimated_minutes_watched=excluded.total_estimated_minutes_watched,
subscribers_gained=excluded.subscribers_gained,
subscribers_lost=excluded.subscribers_lost,
net_subscribers=excluded.net_subscribers""",
(row[0], row[1], row[2], row[3], row[4], row[3] - row[4]),
)
conn.commit()
print("Collected channel daily metrics")
def _parse_duration(iso_duration: str) -> int:
"""Parse ISO 8601 duration (PT1H2M3S) to total seconds."""
import re
match = re.match(r"PT(?:(\d+)H)?(?:(\d+)M)?(?:(\d+)S)?", iso_duration)
if not match:
return 0
hours = int(match.group(1) or 0)
minutes = int(match.group(2) or 0)
seconds = int(match.group(3) or 0)
return hours * 3600 + minutes * 60 + seconds

532
src/dashboard.py Normal file
View File

@ -0,0 +1,532 @@
"""Streamlit dashboard for PBS YouTube Analytics."""
import sqlite3
import streamlit as st
import pandas as pd
import plotly.graph_objects as go
import plotly.express as px
from datetime import datetime, timedelta
from pathlib import Path
from src.database import get_connection, DEFAULT_DB_PATH
def _render_video_tab(conn, video_type: str, start_date, end_date) -> None:
"""Render content for the Videos or Shorts tab."""
label = "Video" if video_type == "video" else "Short"
label_plural = "Videos" if video_type == "video" else "Shorts"
# Performance table
st.subheader(f"{label_plural} Performance")
try:
video_stats = pd.read_sql_query(
f"""SELECT
v.video_id,
v.title,
v.duration_seconds,
SUM(m.views) as total_views,
SUM(m.estimated_minutes_watched) as watch_time_minutes,
ROUND(AVG(m.average_view_percentage), 2) as avg_completion_pct,
SUM(m.likes) as likes,
SUM(m.comments) as comments
FROM videos v
LEFT JOIN video_daily_metrics m ON v.video_id = m.video_id
AND m.date BETWEEN ? AND ?
WHERE v.video_type = ?
GROUP BY v.video_id
ORDER BY total_views DESC""",
conn,
params=(start_date.isoformat(), end_date.isoformat(), video_type),
)
if not video_stats.empty:
video_stats["Duration"] = video_stats["duration_seconds"].apply(
lambda x: f"{int(x // 60)}:{int(x % 60):02d}" if x else "0:00"
)
video_stats["Watch Time (h)"] = (video_stats["watch_time_minutes"] / 60).round(1)
display_cols = ["title", "Duration", "total_views", "Watch Time (h)", "avg_completion_pct", "likes", "comments"]
st.dataframe(
video_stats[display_cols].rename(columns={
"title": "Title",
"total_views": "Views",
"avg_completion_pct": "Completion %",
}),
width='stretch',
hide_index=True,
)
else:
st.info(f"No {label_plural.lower()} data available.")
except Exception as e:
st.warning(f"Could not load {label_plural.lower()} statistics: {e}")
st.divider()
col1, col2 = st.columns(2)
with col1:
st.subheader(f"Top {label_plural} by Views")
try:
top_videos = pd.read_sql_query(
f"""SELECT
v.title,
SUM(m.views) as total_views
FROM videos v
LEFT JOIN video_daily_metrics m ON v.video_id = m.video_id
AND m.date BETWEEN ? AND ?
WHERE v.video_type = ?
GROUP BY v.video_id
ORDER BY total_views DESC
LIMIT 10""",
conn,
params=(start_date.isoformat(), end_date.isoformat(), video_type),
)
if not top_videos.empty:
fig = px.bar(
top_videos,
y="title",
x="total_views",
orientation="h",
labels={"title": "", "total_views": "Views"},
)
fig.update_yaxes(autorange="reversed")
fig.update_layout(showlegend=False, height=400)
st.plotly_chart(fig, width='stretch')
else:
st.info(f"No {label_plural.lower()} data available.")
except Exception as e:
st.warning(f"Could not load top {label_plural.lower()}: {e}")
with col2:
st.subheader("Traffic Source Breakdown")
try:
traffic = pd.read_sql_query(
f"""SELECT
t.traffic_source,
SUM(t.views) as views
FROM video_traffic_sources t
JOIN videos v ON t.video_id = v.video_id
WHERE t.date BETWEEN ? AND ?
AND v.video_type = ?
GROUP BY t.traffic_source
ORDER BY views DESC""",
conn,
params=(start_date.isoformat(), end_date.isoformat(), video_type),
)
if not traffic.empty:
fig = px.pie(
traffic,
names="traffic_source",
values="views",
labels={"traffic_source": "Source", "views": "Views"},
)
fig.update_layout(height=400)
st.plotly_chart(fig, width='stretch')
else:
st.info(f"No traffic data available for {label_plural.lower()}.")
except Exception as e:
st.warning(f"Could not load traffic sources: {e}")
st.divider()
# Per-video detail
st.subheader(f"{label} Detail")
try:
videos_of_type = pd.read_sql_query(
"SELECT video_id, title FROM videos WHERE video_type = ? ORDER BY title",
conn,
params=(video_type,),
)
if videos_of_type.empty:
st.info(f"No {label_plural.lower()} found.")
return
id_to_title = dict(zip(videos_of_type["video_id"], videos_of_type["title"]))
selected_videos = st.multiselect(
f"Select {label_plural.lower()} to explore",
options=videos_of_type["video_id"].tolist(),
format_func=lambda x: id_to_title[x],
key=f"multiselect_{video_type}",
)
if selected_videos:
metric_choice = st.selectbox(
"Metric",
options=["Views", "Watch Time"],
key=f"metric_select_{video_type}",
)
if metric_choice == "Views":
y_col = "views"
y_label = "Cumulative Views"
else:
y_col = "Watch Time (h)"
y_label = "Cumulative Watch Time (h)"
colors = px.colors.qualitative.Plotly
st.subheader("Daily Views & Watch Time")
try:
fig = go.Figure()
for i, vid_id in enumerate(selected_videos):
daily = pd.read_sql_query(
"""SELECT date, views, estimated_minutes_watched
FROM video_daily_metrics
WHERE video_id = ? AND date BETWEEN ? AND ?
ORDER BY date""",
conn,
params=(vid_id, start_date.isoformat(), end_date.isoformat()),
)
if not daily.empty:
daily["date"] = pd.to_datetime(daily["date"])
daily["Watch Time (h)"] = (daily["estimated_minutes_watched"] / 60).round(2)
daily["cumulative"] = daily[y_col].cumsum()
color = colors[i % len(colors)]
fig.add_trace(go.Scatter(
x=daily["date"],
y=daily["cumulative"],
name=id_to_title[vid_id],
mode="lines+markers",
line=dict(color=color),
))
fig.update_layout(
xaxis_title="Date",
yaxis_title=y_label,
hovermode="x unified",
)
st.plotly_chart(fig, width='stretch')
except Exception as e:
st.warning(f"Could not load daily metrics: {e}")
st.subheader("Retention Curves")
try:
fig = go.Figure()
for i, vid_id in enumerate(selected_videos):
retention = pd.read_sql_query(
"""SELECT elapsed_ratio * 100 as elapsed_pct, audience_watch_ratio
FROM video_retention
WHERE video_id = ?
ORDER BY elapsed_ratio""",
conn,
params=(vid_id,),
)
if not retention.empty:
color = colors[i % len(colors)]
fig.add_trace(go.Scatter(
x=retention["elapsed_pct"],
y=retention["audience_watch_ratio"],
mode="lines",
name=id_to_title[vid_id],
line=dict(color=color),
))
fig.update_layout(
xaxis_title="Video Progress (%)",
yaxis_title="Audience Watch Ratio (%)",
hovermode="x unified",
)
st.plotly_chart(fig, width='stretch')
except Exception as e:
st.warning(f"Could not load retention curves: {e}")
except Exception as e:
st.warning(f"Could not load {label.lower()} detail section: {e}")
def run_dashboard():
"""Run the Streamlit dashboard."""
st.set_page_config(page_title="PBS YouTube Analytics", layout="wide")
st.title("PBS YouTube Analytics Dashboard")
db_path = DEFAULT_DB_PATH
if not db_path.exists():
st.error(f"Database not found at {db_path}. Please run 'init-db' and 'seed' first.")
return
conn = get_connection(db_path)
# Sidebar — date range filter only
st.sidebar.header("Filters")
try:
min_date_result = conn.execute("SELECT MIN(date) FROM channel_daily_metrics").fetchone()
max_date_result = conn.execute("SELECT MAX(date) FROM channel_daily_metrics").fetchone()
if min_date_result[0] and max_date_result[0]:
min_date = datetime.fromisoformat(min_date_result[0]).date()
max_date = datetime.fromisoformat(max_date_result[0]).date()
else:
min_date = datetime.now().date() - timedelta(days=30)
max_date = datetime.now().date()
except Exception:
min_date = datetime.now().date() - timedelta(days=30)
max_date = datetime.now().date()
date_range = st.sidebar.date_input(
"Select date range",
value=(min_date, max_date),
min_value=min_date,
max_value=max_date,
)
if len(date_range) == 2:
start_date, end_date = date_range
else:
start_date, end_date = min_date, max_date
# Sidebar — Data Collection
st.sidebar.divider()
st.sidebar.header("Data Collection")
# Display any status message carried over from a previous collection run via session state
if "_collect_msg" in st.session_state:
_msg_type, _msg_text = st.session_state.pop("_collect_msg")
if _msg_type == "success":
st.sidebar.success(_msg_text)
else:
st.sidebar.error(_msg_text)
today = datetime.now().date()
collect_start = st.sidebar.date_input(
"Start date",
value=today - timedelta(days=30),
key="collect_start",
)
collect_end = st.sidebar.date_input(
"End date",
value=today - timedelta(days=2),
key="collect_end",
)
def _open_collection_conn():
sconn = sqlite3.connect(str(db_path))
sconn.execute("PRAGMA journal_mode=WAL")
sconn.execute("PRAGMA foreign_keys=ON")
return sconn
if st.sidebar.button("Collect All", type="primary", use_container_width=True):
with st.spinner("Collecting all data from YouTube API..."):
try:
from src.auth import get_authenticated_service
from src.collector import (
collect_channel_metrics,
collect_daily_metrics,
collect_retention,
collect_traffic_sources,
collect_video_list,
)
from src.database import init_db as _init_db
youtube, youtube_analytics = get_authenticated_service()
_init_db(db_path)
sconn = _open_collection_conn()
start_str = collect_start.isoformat()
end_str = collect_end.isoformat()
video_ids = collect_video_list(youtube, sconn)
collect_daily_metrics(youtube_analytics, sconn, video_ids, start_str, end_str)
collect_retention(youtube_analytics, sconn, video_ids)
collect_traffic_sources(youtube_analytics, sconn, video_ids, start_str, end_str)
collect_channel_metrics(youtube_analytics, sconn, start_str, end_str)
sconn.close()
st.session_state["_collect_msg"] = ("success", "All data collected successfully!")
st.rerun()
except FileNotFoundError as e:
st.sidebar.error(
f"Auth credentials not found. "
f"Save config/client_secret.json to enable collection.\n\n{e}"
)
except Exception as e:
st.sidebar.error(f"Collection error: {e}")
with st.sidebar.expander("Individual Steps"):
if st.button("Collect Video List", use_container_width=True, key="btn_videos"):
with st.spinner("Collecting video list..."):
try:
from src.auth import get_authenticated_service
from src.collector import collect_video_list
from src.database import init_db as _init_db
youtube, _ = get_authenticated_service()
_init_db(db_path)
sconn = _open_collection_conn()
collect_video_list(youtube, sconn)
sconn.close()
st.session_state["_collect_msg"] = ("success", "Video list collected!")
st.rerun()
except FileNotFoundError as e:
st.sidebar.error(f"Auth credentials not found: {e}")
except Exception as e:
st.sidebar.error(f"Error: {e}")
if st.button("Collect Daily Metrics", use_container_width=True, key="btn_daily"):
with st.spinner("Collecting daily metrics..."):
try:
from src.auth import get_authenticated_service
from src.collector import collect_daily_metrics
_, youtube_analytics = get_authenticated_service()
sconn = _open_collection_conn()
video_ids = [r[0] for r in sconn.execute("SELECT video_id FROM videos").fetchall()]
collect_daily_metrics(
youtube_analytics, sconn, video_ids,
collect_start.isoformat(), collect_end.isoformat(),
)
sconn.close()
st.session_state["_collect_msg"] = ("success", "Daily metrics collected!")
st.rerun()
except FileNotFoundError as e:
st.sidebar.error(f"Auth credentials not found: {e}")
except Exception as e:
st.sidebar.error(f"Error: {e}")
if st.button("Collect Retention", use_container_width=True, key="btn_retention"):
with st.spinner("Collecting retention data..."):
try:
from src.auth import get_authenticated_service
from src.collector import collect_retention
_, youtube_analytics = get_authenticated_service()
sconn = _open_collection_conn()
video_ids = [r[0] for r in sconn.execute("SELECT video_id FROM videos").fetchall()]
collect_retention(youtube_analytics, sconn, video_ids)
sconn.close()
st.session_state["_collect_msg"] = ("success", "Retention data collected!")
st.rerun()
except FileNotFoundError as e:
st.sidebar.error(f"Auth credentials not found: {e}")
except Exception as e:
st.sidebar.error(f"Error: {e}")
if st.button("Collect Traffic Sources", use_container_width=True, key="btn_traffic"):
with st.spinner("Collecting traffic sources..."):
try:
from src.auth import get_authenticated_service
from src.collector import collect_traffic_sources
_, youtube_analytics = get_authenticated_service()
sconn = _open_collection_conn()
video_ids = [r[0] for r in sconn.execute("SELECT video_id FROM videos").fetchall()]
collect_traffic_sources(
youtube_analytics, sconn, video_ids,
collect_start.isoformat(), collect_end.isoformat(),
)
sconn.close()
st.session_state["_collect_msg"] = ("success", "Traffic sources collected!")
st.rerun()
except FileNotFoundError as e:
st.sidebar.error(f"Auth credentials not found: {e}")
except Exception as e:
st.sidebar.error(f"Error: {e}")
if st.button("Collect Channel Metrics", use_container_width=True, key="btn_channel"):
with st.spinner("Collecting channel metrics..."):
try:
from src.auth import get_authenticated_service
from src.collector import collect_channel_metrics
_, youtube_analytics = get_authenticated_service()
sconn = _open_collection_conn()
collect_channel_metrics(
youtube_analytics, sconn,
collect_start.isoformat(), collect_end.isoformat(),
)
sconn.close()
st.session_state["_collect_msg"] = ("success", "Channel metrics collected!")
st.rerun()
except FileNotFoundError as e:
st.sidebar.error(f"Auth credentials not found: {e}")
except Exception as e:
st.sidebar.error(f"Error: {e}")
# Channel overview metrics (above tabs — channel-wide)
col1, col2, col3 = st.columns(3)
try:
metrics = conn.execute(
"""SELECT
SUM(total_views) as total_views,
SUM(total_estimated_minutes_watched) as total_watch_time,
SUM(subscribers_gained) as total_subs_gained
FROM channel_daily_metrics
WHERE date BETWEEN ? AND ?""",
(start_date.isoformat(), end_date.isoformat()),
).fetchone()
if metrics and metrics[0]:
with col1:
st.metric("Total Views", f"{int(metrics[0]):,}")
with col2:
st.metric("Watch Time (hours)", f"{int(metrics[1] / 60):,}")
with col3:
st.metric("Subscribers Gained", f"{int(metrics[2]):,}")
except Exception as e:
st.warning(f"Could not load channel metrics: {e}")
st.divider()
# Views over time (above tabs — channel-wide)
st.subheader("Views Over Time")
try:
daily_views = pd.read_sql_query(
"""SELECT m.date, v.video_type, SUM(m.views) as views
FROM video_daily_metrics m
JOIN videos v ON m.video_id = v.video_id
WHERE m.date BETWEEN ? AND ?
GROUP BY m.date, v.video_type
ORDER BY m.date""",
conn,
params=(start_date.isoformat(), end_date.isoformat()),
)
if not daily_views.empty:
daily_views["date"] = pd.to_datetime(daily_views["date"])
pivoted = daily_views.pivot_table(
index="date", columns="video_type", values="views", fill_value=0
).reset_index()
if "short" not in pivoted.columns:
pivoted["short"] = 0
if "video" not in pivoted.columns:
pivoted["video"] = 0
fig = go.Figure()
fig.add_trace(go.Scatter(
x=pivoted["date"],
y=pivoted["short"],
name="Shorts",
stackgroup="one",
mode="lines",
line=dict(color="#FF7043"),
fillcolor="rgba(255, 112, 67, 0.6)",
))
fig.add_trace(go.Scatter(
x=pivoted["date"],
y=pivoted["video"],
name="Videos",
stackgroup="one",
mode="lines",
line=dict(color="#1976D2"),
fillcolor="rgba(25, 118, 210, 0.6)",
))
fig.update_layout(
xaxis_title="Date",
yaxis_title="Views",
hovermode="x unified",
)
st.plotly_chart(fig, width='stretch')
else:
st.info("No data available for the selected date range.")
except Exception as e:
st.warning(f"Could not load views chart: {e}")
st.divider()
# Tabbed section — per-type content
videos_tab, shorts_tab = st.tabs(["Videos", "Shorts"])
with videos_tab:
_render_video_tab(conn, "video", start_date, end_date)
with shorts_tab:
_render_video_tab(conn, "short", start_date, end_date)
conn.close()
if __name__ == "__main__":
run_dashboard()

91
src/database.py Normal file
View File

@ -0,0 +1,91 @@
"""SQLite database setup and connection management for PBS YouTube Analytics."""
import sqlite3
from pathlib import Path
DEFAULT_DB_PATH = Path(__file__).parent.parent / "data" / "pbs_youtube.db"
SCHEMA = """
-- Video metadata from Data API
CREATE TABLE IF NOT EXISTS videos (
video_id TEXT PRIMARY KEY,
title TEXT NOT NULL,
published_at TEXT NOT NULL,
duration_seconds INTEGER,
video_type TEXT CHECK(video_type IN ('video', 'short')),
thumbnail_url TEXT,
description TEXT,
created_at TEXT DEFAULT CURRENT_TIMESTAMP,
updated_at TEXT DEFAULT CURRENT_TIMESTAMP
);
-- Daily aggregate metrics from Analytics API
CREATE TABLE IF NOT EXISTS video_daily_metrics (
video_id TEXT NOT NULL,
date TEXT NOT NULL,
views INTEGER DEFAULT 0,
estimated_minutes_watched REAL DEFAULT 0,
average_view_duration_seconds REAL DEFAULT 0,
average_view_percentage REAL DEFAULT 0,
likes INTEGER DEFAULT 0,
dislikes INTEGER DEFAULT 0,
comments INTEGER DEFAULT 0,
shares INTEGER DEFAULT 0,
subscribers_gained INTEGER DEFAULT 0,
subscribers_lost INTEGER DEFAULT 0,
impressions INTEGER DEFAULT 0,
impressions_ctr REAL DEFAULT 0,
created_at TEXT DEFAULT CURRENT_TIMESTAMP,
PRIMARY KEY (video_id, date),
FOREIGN KEY (video_id) REFERENCES videos(video_id)
);
-- Audience retention curve (100 data points per video)
CREATE TABLE IF NOT EXISTS video_retention (
video_id TEXT NOT NULL,
elapsed_ratio REAL NOT NULL,
audience_watch_ratio REAL NOT NULL,
relative_retention_performance REAL,
fetched_at TEXT DEFAULT CURRENT_TIMESTAMP,
PRIMARY KEY (video_id, elapsed_ratio),
FOREIGN KEY (video_id) REFERENCES videos(video_id)
);
-- Traffic source breakdown per video per day
CREATE TABLE IF NOT EXISTS video_traffic_sources (
video_id TEXT NOT NULL,
date TEXT NOT NULL,
traffic_source TEXT NOT NULL,
views INTEGER DEFAULT 0,
estimated_minutes_watched REAL DEFAULT 0,
PRIMARY KEY (video_id, date, traffic_source),
FOREIGN KEY (video_id) REFERENCES videos(video_id)
);
-- Channel-level daily summary
CREATE TABLE IF NOT EXISTS channel_daily_metrics (
date TEXT PRIMARY KEY,
total_views INTEGER DEFAULT 0,
total_estimated_minutes_watched REAL DEFAULT 0,
subscribers_gained INTEGER DEFAULT 0,
subscribers_lost INTEGER DEFAULT 0,
net_subscribers INTEGER DEFAULT 0,
created_at TEXT DEFAULT CURRENT_TIMESTAMP
);
"""
def get_connection(db_path: Path = DEFAULT_DB_PATH) -> sqlite3.Connection:
db_path.parent.mkdir(parents=True, exist_ok=True)
conn = sqlite3.connect(str(db_path))
conn.row_factory = sqlite3.Row
conn.execute("PRAGMA journal_mode=WAL")
conn.execute("PRAGMA foreign_keys=ON")
return conn
def init_db(db_path: Path = DEFAULT_DB_PATH) -> None:
conn = get_connection(db_path)
conn.executescript(SCHEMA)
conn.close()
print(f"Database initialized at {db_path}")

390
src/seed.py Normal file
View File

@ -0,0 +1,390 @@
"""Seed script to populate SQLite database with realistic sample data for a PBS plant-based cooking channel."""
import sqlite3
import random
from datetime import datetime, timedelta
from pathlib import Path
from src.database import get_connection, DEFAULT_DB_PATH
def seed_database(db_path: Path = DEFAULT_DB_PATH) -> None:
"""Populate the database with realistic sample data."""
conn = get_connection(db_path)
# Clear existing data
conn.execute("DELETE FROM video_traffic_sources")
conn.execute("DELETE FROM video_retention")
conn.execute("DELETE FROM video_daily_metrics")
conn.execute("DELETE FROM channel_daily_metrics")
conn.execute("DELETE FROM videos")
conn.commit()
# Seed videos
video_ids = _seed_videos(conn)
# Seed daily metrics for the last 30 days
_seed_daily_metrics(conn, video_ids)
# Seed retention curves
_seed_retention(conn, video_ids)
# Seed traffic sources
_seed_traffic_sources(conn, video_ids)
# Seed channel daily metrics
_seed_channel_metrics(conn)
conn.close()
print(f"Database seeded successfully at {db_path}")
def _seed_videos(conn: sqlite3.Connection) -> list[str]:
"""Create ~18 videos: mix of shorts and long-form, PBS plant-based cooking themed."""
videos = [
# Shorts (30-60 seconds)
{
"video_id": "short_001",
"title": "Quick Vegan Breakfast: 30-Second Oatmeal Hack",
"duration_seconds": 45,
"video_type": "short",
"published_at": (datetime.now() - timedelta(days=25)).isoformat() + "Z",
},
{
"video_id": "short_002",
"title": "Plant-Based Protein Smoothie Bowl",
"duration_seconds": 52,
"video_type": "short",
"published_at": (datetime.now() - timedelta(days=23)).isoformat() + "Z",
},
{
"video_id": "short_003",
"title": "Crispy Tofu in 60 Seconds",
"duration_seconds": 58,
"video_type": "short",
"published_at": (datetime.now() - timedelta(days=21)).isoformat() + "Z",
},
{
"video_id": "short_004",
"title": "No-Cook Hummus & Veggie Snack Hack",
"duration_seconds": 48,
"video_type": "short",
"published_at": (datetime.now() - timedelta(days=19)).isoformat() + "Z",
},
{
"video_id": "short_005",
"title": "5-Minute Mango Sorbet",
"duration_seconds": 55,
"video_type": "short",
"published_at": (datetime.now() - timedelta(days=17)).isoformat() + "Z",
},
{
"video_id": "short_006",
"title": "Instant Pasta Primavera Hack",
"duration_seconds": 50,
"video_type": "short",
"published_at": (datetime.now() - timedelta(days=15)).isoformat() + "Z",
},
# Long-form videos (5-20 minutes)
{
"video_id": "long_001",
"title": "Complete Plant-Based Dinner: Mediterranean Buddha Bowl",
"duration_seconds": 12 * 60 + 30,
"video_type": "video",
"published_at": (datetime.now() - timedelta(days=28)).isoformat() + "Z",
},
{
"video_id": "long_002",
"title": "Vegan Baking Masterclass: Chocolate Avocado Cake",
"duration_seconds": 18 * 60 + 45,
"video_type": "video",
"published_at": (datetime.now() - timedelta(days=26)).isoformat() + "Z",
},
{
"video_id": "long_003",
"title": "How to Make Cashew Cream from Scratch",
"duration_seconds": 8 * 60 + 15,
"video_type": "video",
"published_at": (datetime.now() - timedelta(days=24)).isoformat() + "Z",
},
{
"video_id": "long_004",
"title": "Budget Plant-Based Meal Prep for the Week",
"duration_seconds": 16 * 60 + 20,
"video_type": "video",
"published_at": (datetime.now() - timedelta(days=22)).isoformat() + "Z",
},
{
"video_id": "long_005",
"title": "Fermentation Basics: Making Kimchi & Sauerkraut",
"duration_seconds": 19 * 60 + 10,
"video_type": "video",
"published_at": (datetime.now() - timedelta(days=20)).isoformat() + "Z",
},
{
"video_id": "long_006",
"title": "Whole Plant-Based Thanksgiving Feast",
"duration_seconds": 22 * 60 + 30,
"video_type": "video",
"published_at": (datetime.now() - timedelta(days=18)).isoformat() + "Z",
},
{
"video_id": "long_007",
"title": "Nutritionist Deep Dive: Plant-Based Protein Myth Busting",
"duration_seconds": 14 * 60 + 50,
"video_type": "video",
"published_at": (datetime.now() - timedelta(days=16)).isoformat() + "Z",
},
{
"video_id": "long_008",
"title": "Zero-Waste Kitchen: Composting & Food Scraps",
"duration_seconds": 10 * 60 + 25,
"video_type": "video",
"published_at": (datetime.now() - timedelta(days=14)).isoformat() + "Z",
},
{
"video_id": "long_009",
"title": "Street Food Around the World: Vegan Edition",
"duration_seconds": 20 * 60 + 15,
"video_type": "video",
"published_at": (datetime.now() - timedelta(days=12)).isoformat() + "Z",
},
{
"video_id": "long_010",
"title": "How to Stock Your Plant-Based Pantry",
"duration_seconds": 11 * 60 + 40,
"video_type": "video",
"published_at": (datetime.now() - timedelta(days=10)).isoformat() + "Z",
},
{
"video_id": "long_011",
"title": "Interview: Local Organic Farmer About Seasonal Produce",
"duration_seconds": 17 * 60 + 35,
"video_type": "video",
"published_at": (datetime.now() - timedelta(days=8)).isoformat() + "Z",
},
{
"video_id": "long_012",
"title": "Plant-Based Athlete: How I Fuel My Training",
"duration_seconds": 15 * 60 + 20,
"video_type": "video",
"published_at": (datetime.now() - timedelta(days=6)).isoformat() + "Z",
},
]
thumbnails = [
"https://via.placeholder.com/320x180?text=Recipe",
"https://via.placeholder.com/320x180?text=Cooking",
"https://via.placeholder.com/320x180?text=Plant+Based",
]
descriptions = [
"Learn how to make delicious plant-based meals at home.",
"Quick and easy vegan cooking tips for busy weeknights.",
"Discover the joy of plant-based eating with our expert guides.",
"Sustainable cooking starts in your kitchen.",
"Healthy, delicious, and 100% plant-based.",
]
video_ids = []
for video in videos:
video_ids.append(video["video_id"])
conn.execute(
"""INSERT INTO videos
(video_id, title, published_at, duration_seconds, video_type, thumbnail_url, description)
VALUES (?, ?, ?, ?, ?, ?, ?)""",
(
video["video_id"],
video["title"],
video["published_at"],
video["duration_seconds"],
video["video_type"],
random.choice(thumbnails),
random.choice(descriptions),
),
)
conn.commit()
print(f"Seeded {len(video_ids)} videos")
return video_ids
def _seed_daily_metrics(conn: sqlite3.Connection, video_ids: list[str]) -> None:
"""Seed 30 days of daily metrics for each video."""
# Short videos get 50-500 views/day, long videos get 100-2000 views/day
start_date = datetime.now() - timedelta(days=30)
for video_id in video_ids:
is_short = video_id.startswith("short_")
for day_offset in range(30):
date = (start_date + timedelta(days=day_offset)).strftime("%Y-%m-%d")
if is_short:
views = random.randint(50, 500)
estimated_minutes = views * random.uniform(0.5, 2.0)
avg_duration = random.uniform(20, 45) # seconds
ctr = random.uniform(0.8, 3.5)
else:
views = random.randint(100, 2000)
estimated_minutes = views * random.uniform(3.0, 10.0)
avg_duration = random.uniform(200, 800) # seconds
ctr = random.uniform(1.2, 4.5)
avg_view_pct = (avg_duration / 720) * 100 if not is_short else (avg_duration / 60) * 100
avg_view_pct = min(avg_view_pct, 100)
likes = int(views * random.uniform(0.01, 0.05))
comments = int(views * random.uniform(0.002, 0.015))
shares = int(views * random.uniform(0.001, 0.008))
impressions = int(views / random.uniform(0.15, 0.35))
subs_gained = max(0, int(views * random.uniform(0.001, 0.012)) - 1)
conn.execute(
"""INSERT INTO video_daily_metrics
(video_id, date, views, estimated_minutes_watched,
average_view_duration_seconds, average_view_percentage,
likes, dislikes, comments, shares,
subscribers_gained, subscribers_lost, impressions, impressions_ctr)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""",
(
video_id, date, views, estimated_minutes,
avg_duration, avg_view_pct,
likes, 0, comments, shares,
subs_gained, 0, impressions, ctr,
),
)
conn.commit()
print(f"Seeded daily metrics for {len(video_ids)} videos over 30 days")
def _seed_retention(conn: sqlite3.Connection, video_ids: list[str]) -> None:
"""Seed audience retention curves (100 data points per video)."""
for video_id in video_ids:
is_short = video_id.startswith("short_")
# Retention typically starts high and drops off
# Shorts: steep drop, longs: gradual decline
for point in range(100):
elapsed_ratio = point / 100.0
if is_short:
# Shorts drop off faster
audience_ratio = 100 * (1 - elapsed_ratio ** 1.2)
else:
# Long-form videos drop off more gradually
audience_ratio = 100 * (1 - elapsed_ratio ** 0.8)
audience_ratio = max(1, audience_ratio) # Minimum 1%
relative_perf = random.uniform(-20, 30) # Performance vs avg
conn.execute(
"""INSERT INTO video_retention
(video_id, elapsed_ratio, audience_watch_ratio, relative_retention_performance)
VALUES (?, ?, ?, ?)""",
(video_id, elapsed_ratio, audience_ratio, relative_perf),
)
conn.commit()
print(f"Seeded retention curves for {len(video_ids)} videos")
def _seed_traffic_sources(conn: sqlite3.Connection, video_ids: list[str]) -> None:
"""Seed traffic source breakdown for each video-day combination."""
sources = [
"YouTube search",
"Suggested videos",
"Direct or unknown",
"External website",
"YouTube channel",
"Playlist",
]
start_date = datetime.now() - timedelta(days=30)
for video_id in video_ids:
for day_offset in range(30):
date = (start_date + timedelta(days=day_offset)).strftime("%Y-%m-%d")
# Get total views from video_daily_metrics
cursor = conn.execute(
"SELECT views FROM video_daily_metrics WHERE video_id = ? AND date = ?",
(video_id, date),
)
result = cursor.fetchone()
if not result:
continue
total_views = result[0]
# Distribute across sources
remaining_views = total_views
for source in sources[:-1]:
source_views = int(remaining_views * random.uniform(0.05, 0.35))
source_minutes = source_views * random.uniform(1.0, 6.0)
conn.execute(
"""INSERT INTO video_traffic_sources
(video_id, date, traffic_source, views, estimated_minutes_watched)
VALUES (?, ?, ?, ?, ?)""",
(video_id, date, source, source_views, source_minutes),
)
remaining_views -= source_views
# Last source gets remainder
if remaining_views > 0:
conn.execute(
"""INSERT INTO video_traffic_sources
(video_id, date, traffic_source, views, estimated_minutes_watched)
VALUES (?, ?, ?, ?, ?)""",
(video_id, date, sources[-1], remaining_views, remaining_views * random.uniform(1.0, 6.0)),
)
conn.commit()
print(f"Seeded traffic sources for {len(video_ids)} videos over 30 days")
def _seed_channel_metrics(conn: sqlite3.Connection) -> None:
"""Seed channel-level daily summary metrics for the last 30 days."""
start_date = datetime.now() - timedelta(days=30)
for day_offset in range(30):
date = (start_date + timedelta(days=day_offset)).strftime("%Y-%m-%d")
# Get daily totals from video metrics
cursor = conn.execute(
"""SELECT
SUM(views), SUM(estimated_minutes_watched)
FROM video_daily_metrics
WHERE date = ?""",
(date,),
)
result = cursor.fetchone()
if result and result[0]:
total_views = result[0]
total_minutes = result[1] or 0
else:
# Fallback if no video metrics yet
total_views = random.randint(2000, 8000)
total_minutes = total_views * random.uniform(3.0, 8.0)
subs_gained = max(0, int(total_views * random.uniform(0.001, 0.015)) - 1)
subs_lost = max(0, subs_gained - random.randint(0, int(subs_gained * 0.5)))
conn.execute(
"""INSERT INTO channel_daily_metrics
(date, total_views, total_estimated_minutes_watched,
subscribers_gained, subscribers_lost, net_subscribers)
VALUES (?, ?, ?, ?, ?, ?)""",
(date, total_views, total_minutes, subs_gained, subs_lost, subs_gained - subs_lost),
)
conn.commit()
print("Seeded channel daily metrics for 30 days")
if __name__ == "__main__":
seed_database()

1140
uv.lock generated Normal file

File diff suppressed because it is too large Load Diff