project init
This commit is contained in:
commit
fc85cd2abe
10
.gitignore
vendored
Normal file
10
.gitignore
vendored
Normal file
@ -0,0 +1,10 @@
|
||||
# Python-generated files
|
||||
__pycache__/
|
||||
*.py[oc]
|
||||
build/
|
||||
dist/
|
||||
wheels/
|
||||
*.egg-info
|
||||
|
||||
# Virtual environments
|
||||
.venv
|
||||
1
.python-version
Normal file
1
.python-version
Normal file
@ -0,0 +1 @@
|
||||
3.12
|
||||
67
CLAUDE.md
Normal file
67
CLAUDE.md
Normal file
@ -0,0 +1,67 @@
|
||||
# YouTube Analytics — CLAUDE.md
|
||||
|
||||
PBS YouTube channel analytics pipeline: collect data from YouTube APIs, store in SQLite, visualize with Streamlit.
|
||||
|
||||
## Project Structure
|
||||
|
||||
```
|
||||
main.py — Click CLI entry point (init-db, seed, collect, dashboard)
|
||||
src/
|
||||
auth.py — OAuth2 authentication (YouTube Data API v3 + Analytics API v2)
|
||||
collector.py — Data collection (videos, daily metrics, retention, traffic sources, channel stats)
|
||||
database.py — SQLite schema (5 tables) and connection management (WAL mode, foreign keys)
|
||||
dashboard.py — Streamlit dashboard (charts, tables, retention curves)
|
||||
seed.py — Sample data generator (PBS plant-based cooking channel theme)
|
||||
config/ — OAuth credentials (git-ignored, not committed)
|
||||
data/ — SQLite database (git-ignored)
|
||||
```
|
||||
|
||||
## Package Manager
|
||||
|
||||
Uses **uv** (not pip, not poetry). Python >=3.12.
|
||||
|
||||
```bash
|
||||
uv sync # install deps from lockfile
|
||||
uv add <pkg> # add a dependency
|
||||
uv run python main.py <command> # run within managed venv
|
||||
```
|
||||
|
||||
## CLI Commands
|
||||
|
||||
```bash
|
||||
uv run python main.py init-db-cmd # create database tables
|
||||
uv run python main.py seed # populate with sample data
|
||||
uv run python main.py collect # pull real data (requires OAuth creds in config/)
|
||||
uv run python main.py dashboard # launch Streamlit dashboard
|
||||
```
|
||||
|
||||
## Database
|
||||
|
||||
SQLite at `data/pbs_youtube.db` with five tables:
|
||||
- `videos` — video metadata (id, title, published_at, duration, type)
|
||||
- `video_daily_metrics` — per-video daily stats (views, watch time, CTR, engagement)
|
||||
- `video_retention` — 100-point audience retention curves per video
|
||||
- `video_traffic_sources` — traffic source breakdown per video per day
|
||||
- `channel_daily_metrics` — channel-level daily summary
|
||||
|
||||
All inserts use `ON CONFLICT DO UPDATE` for idempotent runs.
|
||||
|
||||
## Authentication
|
||||
|
||||
Real YouTube API access requires:
|
||||
1. Create OAuth credentials at https://console.cloud.google.com/apis/credentials
|
||||
2. Download client secret JSON → save as `config/client_secret.json`
|
||||
3. First run of `collect` opens browser for OAuth consent, stores token at `config/token.json`
|
||||
|
||||
For development/testing, use `seed` command instead — no credentials needed.
|
||||
|
||||
## Key Dependencies
|
||||
|
||||
click, streamlit, plotly, pandas, google-api-python-client, google-auth-oauthlib
|
||||
|
||||
## Conventions
|
||||
|
||||
- All data collection functions live in `collector.py`, one function per data type
|
||||
- Dashboard uses Plotly for charts via `streamlit` + `plotly`
|
||||
- Duration parsing handles ISO 8601 (PT1H2M3S format)
|
||||
- Shorts classified as videos ≤60 seconds
|
||||
12
config/client_secret.json
Normal file
12
config/client_secret.json
Normal file
@ -0,0 +1,12 @@
|
||||
{
|
||||
"installed":
|
||||
{
|
||||
"client_id":"918670827378-ngeo2grfmm82orjhru6bs8g5hrcht0cd.apps.googleusercontent.com",
|
||||
"project_id":"pbs-site-486819",
|
||||
"auth_uri":"https://accounts.google.com/o/oauth2/auth",
|
||||
"token_uri":"https://oauth2.googleapis.com/token",
|
||||
"auth_provider_x509_cert_url":"https://www.googleapis.com/oauth2/v1/certs",
|
||||
"client_secret":"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",
|
||||
"redirect_uris":["http://localhost"]
|
||||
}
|
||||
}
|
||||
1
config/token.json
Normal file
1
config/token.json
Normal file
@ -0,0 +1 @@
|
||||
{"token": "ya29.a0AQvPyINTBUmYT0TS8M6C6QhWHVswoI0coM6ycnodz_mJpqITBBXXfh4sVILepbZ8fVz60Bfog95aqFl47T6NFkb5_GXGlSA_1iq-_Q0bQg_Qc2_L0J_8nZQ4ubpTr2FV5K2ipe-er3QU4999C4OXQSxdB9FEMCuNrT_r_Aa3ojqdb2OMZIKwPuw4Ctqp1cX_9RZ03kqCaCgYKAYgSARQSFQHGX2Mi62N5gg_yUtdFoY3t9Uh4lw0207", "refresh_token": "1//05Zaxlboo0cyOCgYIARAAGAUSNwF-L9IrAplENIYSCGlR4dcA07L0_f2344h793lvG2eRzFeFRgDKTYtMPso9D8DvYMbNDgNWGb4", "token_uri": "https://oauth2.googleapis.com/token", "client_id": "918670827378-ngeo2grfmm82orjhru6bs8g5hrcht0cd.apps.googleusercontent.com", "client_secret": "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", "scopes": ["https://www.googleapis.com/auth/yt-analytics.readonly", "https://www.googleapis.com/auth/youtube.readonly"], "universe_domain": "googleapis.com", "account": "", "expiry": "2026-05-04T02:25:42.357613Z"}
|
||||
BIN
data/pbs_youtube.db
Normal file
BIN
data/pbs_youtube.db
Normal file
Binary file not shown.
159
main.py
Normal file
159
main.py
Normal file
@ -0,0 +1,159 @@
|
||||
"""CLI entry point for PBS YouTube Analytics."""
|
||||
|
||||
import click
|
||||
from pathlib import Path
|
||||
|
||||
from src.database import init_db, get_connection, DEFAULT_DB_PATH
|
||||
from src.seed import seed_database
|
||||
|
||||
|
||||
@click.group()
|
||||
def cli():
|
||||
"""PBS YouTube Analytics CLI."""
|
||||
pass
|
||||
|
||||
|
||||
@cli.command()
|
||||
def init_db_cmd():
|
||||
"""Initialize the database schema."""
|
||||
try:
|
||||
init_db(DEFAULT_DB_PATH)
|
||||
click.echo(click.style("✓ Database initialized successfully", fg="green"))
|
||||
except Exception as e:
|
||||
click.echo(click.style(f"✗ Error initializing database: {e}", fg="red"), err=True)
|
||||
raise SystemExit(1)
|
||||
|
||||
|
||||
@cli.command()
|
||||
def seed():
|
||||
"""Populate the database with sample data."""
|
||||
try:
|
||||
seed_database(DEFAULT_DB_PATH)
|
||||
click.echo(click.style("✓ Database seeded successfully", fg="green"))
|
||||
except Exception as e:
|
||||
click.echo(click.style(f"✗ Error seeding database: {e}", fg="red"), err=True)
|
||||
raise SystemExit(1)
|
||||
|
||||
|
||||
@cli.command("clear-seed")
|
||||
def clear_seed():
|
||||
"""Remove sample seed data (videos with IDs matching short_* or long_*)."""
|
||||
try:
|
||||
conn = get_connection(DEFAULT_DB_PATH)
|
||||
|
||||
# Seed video IDs are short_NNN / long_NNN — real YouTube IDs are 11-char alphanumeric,
|
||||
# so these patterns can never collide with collected data.
|
||||
seed_pattern = "video_id LIKE 'short_%' OR video_id LIKE 'long_%'"
|
||||
|
||||
totals = {}
|
||||
for table in ("video_traffic_sources", "video_retention", "video_daily_metrics"):
|
||||
cur = conn.execute(f"DELETE FROM {table} WHERE {seed_pattern}")
|
||||
totals[table] = cur.rowcount
|
||||
|
||||
cur = conn.execute(f"DELETE FROM videos WHERE {seed_pattern}")
|
||||
totals["videos"] = cur.rowcount
|
||||
|
||||
conn.commit()
|
||||
conn.close()
|
||||
|
||||
for table, count in totals.items():
|
||||
click.echo(f" {table}: {count} rows deleted")
|
||||
click.echo(click.style("✓ Seed data cleared", fg="green"))
|
||||
click.echo(" Note: channel_daily_metrics rows have no seed identifier and are left untouched.")
|
||||
except Exception as e:
|
||||
click.echo(click.style(f"✗ Error clearing seed data: {e}", fg="red"), err=True)
|
||||
raise SystemExit(1)
|
||||
|
||||
|
||||
@cli.command()
|
||||
@click.option("--start-date", default=None, help="Start date (YYYY-MM-DD), default: 30 days ago")
|
||||
@click.option("--end-date", default=None, help="End date (YYYY-MM-DD), default: 2 days ago")
|
||||
def collect(start_date, end_date):
|
||||
"""Collect real data from YouTube API."""
|
||||
import sqlite3
|
||||
|
||||
try:
|
||||
from src.auth import get_authenticated_service
|
||||
from src.collector import (
|
||||
collect_channel_metrics,
|
||||
collect_daily_metrics,
|
||||
collect_retention,
|
||||
collect_traffic_sources,
|
||||
collect_video_list,
|
||||
)
|
||||
|
||||
click.echo("Attempting to authenticate with YouTube API...")
|
||||
youtube, youtube_analytics = get_authenticated_service()
|
||||
click.echo(click.style("✓ Successfully authenticated with YouTube API", fg="green"))
|
||||
|
||||
# Ensure database exists
|
||||
init_db(DEFAULT_DB_PATH)
|
||||
conn = sqlite3.connect(DEFAULT_DB_PATH)
|
||||
conn.execute("PRAGMA journal_mode=WAL")
|
||||
conn.execute("PRAGMA foreign_keys=ON")
|
||||
|
||||
# Step 1: Collect video list
|
||||
click.echo("\n📹 Collecting video list...")
|
||||
video_ids = collect_video_list(youtube, conn)
|
||||
|
||||
# Step 2: Collect daily metrics per video
|
||||
click.echo("📊 Collecting daily metrics...")
|
||||
collect_daily_metrics(youtube_analytics, conn, video_ids, start_date, end_date)
|
||||
|
||||
# Step 3: Collect retention curves
|
||||
click.echo("📈 Collecting retention data...")
|
||||
collect_retention(youtube_analytics, conn, video_ids)
|
||||
|
||||
# Step 4: Collect traffic sources
|
||||
click.echo("🔗 Collecting traffic sources...")
|
||||
collect_traffic_sources(youtube_analytics, conn, video_ids, start_date, end_date)
|
||||
|
||||
# Step 5: Collect channel-level metrics
|
||||
click.echo("📺 Collecting channel metrics...")
|
||||
collect_channel_metrics(youtube_analytics, conn, start_date, end_date)
|
||||
|
||||
conn.close()
|
||||
click.echo(click.style("\n✓ All data collected successfully!", fg="green"))
|
||||
|
||||
except FileNotFoundError as e:
|
||||
click.echo(click.style(f"✗ {e}", fg="red"), err=True)
|
||||
click.echo("\nTo use the collect command:")
|
||||
click.echo("1. Create credentials at: https://console.cloud.google.com/apis/credentials")
|
||||
click.echo("2. Download OAuth credentials as client_secret.json")
|
||||
click.echo("3. Save to: config/client_secret.json")
|
||||
raise SystemExit(1)
|
||||
except Exception as e:
|
||||
click.echo(click.style(f"✗ Error during collection: {e}", fg="red"), err=True)
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
raise SystemExit(1)
|
||||
|
||||
|
||||
@cli.command()
|
||||
def dashboard():
|
||||
"""Launch the Streamlit dashboard."""
|
||||
try:
|
||||
import streamlit # noqa: F401 — just checking it's installed
|
||||
except ImportError:
|
||||
click.echo(
|
||||
click.style("✗ Streamlit not installed. Install with: uv add streamlit", fg="red"),
|
||||
err=True,
|
||||
)
|
||||
raise SystemExit(1)
|
||||
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
dashboard_path = str(Path(__file__).parent / "src" / "dashboard.py")
|
||||
try:
|
||||
subprocess.run(
|
||||
[sys.executable, "-m", "streamlit", "run", dashboard_path],
|
||||
check=True,
|
||||
)
|
||||
except Exception as e:
|
||||
click.echo(click.style(f"✗ Error launching dashboard: {e}", fg="red"), err=True)
|
||||
raise SystemExit(1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
cli()
|
||||
15
pyproject.toml
Normal file
15
pyproject.toml
Normal file
@ -0,0 +1,15 @@
|
||||
[project]
|
||||
name = "youtube-analytics"
|
||||
version = "0.1.0"
|
||||
description = "Add your description here"
|
||||
readme = "README.md"
|
||||
requires-python = ">=3.12"
|
||||
dependencies = [
|
||||
"click>=8.3.2",
|
||||
"google-api-python-client>=2.194.0",
|
||||
"google-auth-httplib2>=0.3.1",
|
||||
"google-auth-oauthlib>=1.3.1",
|
||||
"pandas>=3.0.2",
|
||||
"plotly>=6.7.0",
|
||||
"streamlit>=1.56.0",
|
||||
]
|
||||
0
src/__init__.py
Normal file
0
src/__init__.py
Normal file
56
src/auth.py
Normal file
56
src/auth.py
Normal file
@ -0,0 +1,56 @@
|
||||
"""YouTube API authentication module with OAuth2 token persistence.
|
||||
|
||||
Replace PLACEHOLDER values with real credentials from Google Cloud Console.
|
||||
See: https://console.cloud.google.com/apis/credentials
|
||||
"""
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
SCOPES = [
|
||||
"https://www.googleapis.com/auth/yt-analytics.readonly",
|
||||
"https://www.googleapis.com/auth/youtube.readonly",
|
||||
]
|
||||
|
||||
CONFIG_DIR = Path(__file__).parent.parent / "config"
|
||||
CLIENT_SECRET_PATH = CONFIG_DIR / "client_secret.json"
|
||||
TOKEN_PATH = CONFIG_DIR / "token.json"
|
||||
|
||||
|
||||
def get_authenticated_service():
|
||||
"""Build and return authenticated YouTube API services.
|
||||
|
||||
Returns a tuple of (youtube_data_api, youtube_analytics_api).
|
||||
|
||||
On first run, opens a browser for OAuth consent.
|
||||
Subsequent runs use the stored refresh token.
|
||||
"""
|
||||
from google.oauth2.credentials import Credentials
|
||||
from google_auth_oauthlib.flow import InstalledAppFlow
|
||||
from google.auth.transport.requests import Request
|
||||
from googleapiclient.discovery import build
|
||||
|
||||
creds = None
|
||||
|
||||
if TOKEN_PATH.exists():
|
||||
creds = Credentials.from_authorized_user_file(str(TOKEN_PATH), SCOPES)
|
||||
|
||||
if not creds or not creds.valid:
|
||||
if creds and creds.expired and creds.refresh_token:
|
||||
creds.refresh(Request())
|
||||
else:
|
||||
if not CLIENT_SECRET_PATH.exists():
|
||||
raise FileNotFoundError(
|
||||
f"Missing {CLIENT_SECRET_PATH}. Download OAuth credentials from "
|
||||
"Google Cloud Console and save as config/client_secret.json"
|
||||
)
|
||||
flow = InstalledAppFlow.from_client_secrets_file(
|
||||
str(CLIENT_SECRET_PATH), SCOPES
|
||||
)
|
||||
creds = flow.run_local_server(port=0)
|
||||
|
||||
TOKEN_PATH.write_text(creds.to_json())
|
||||
|
||||
youtube = build("youtube", "v3", credentials=creds)
|
||||
youtube_analytics = build("youtubeAnalytics", "v2", credentials=creds)
|
||||
|
||||
return youtube, youtube_analytics
|
||||
228
src/collector.py
Normal file
228
src/collector.py
Normal file
@ -0,0 +1,228 @@
|
||||
"""YouTube data collector — pulls video metadata, metrics, and retention data."""
|
||||
|
||||
import sqlite3
|
||||
from datetime import datetime, timedelta
|
||||
|
||||
|
||||
def collect_video_list(youtube, conn: sqlite3.Connection) -> list[str]:
|
||||
"""Fetch all videos from the channel and upsert into the videos table."""
|
||||
video_ids = []
|
||||
request = youtube.channels().list(part="contentDetails", mine=True)
|
||||
response = request.execute()
|
||||
|
||||
uploads_playlist = response["items"][0]["contentDetails"]["relatedPlaylists"]["uploads"]
|
||||
|
||||
next_page = None
|
||||
while True:
|
||||
pl_request = youtube.playlistItems().list(
|
||||
part="snippet,contentDetails",
|
||||
playlistId=uploads_playlist,
|
||||
maxResults=50,
|
||||
pageToken=next_page,
|
||||
)
|
||||
pl_response = pl_request.execute()
|
||||
|
||||
ids = [item["contentDetails"]["videoId"] for item in pl_response["items"]]
|
||||
video_ids.extend(ids)
|
||||
|
||||
# Batch video details (duration, type)
|
||||
for i in range(0, len(ids), 50):
|
||||
batch = ids[i : i + 50]
|
||||
vid_request = youtube.videos().list(
|
||||
part="snippet,contentDetails", id=",".join(batch)
|
||||
)
|
||||
vid_response = vid_request.execute()
|
||||
|
||||
for video in vid_response["items"]:
|
||||
duration = _parse_duration(video["contentDetails"]["duration"])
|
||||
video_type = "short" if duration <= 60 else "video"
|
||||
|
||||
conn.execute(
|
||||
"""INSERT INTO videos (video_id, title, published_at, duration_seconds,
|
||||
video_type, thumbnail_url, description, updated_at)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, CURRENT_TIMESTAMP)
|
||||
ON CONFLICT(video_id) DO UPDATE SET
|
||||
title=excluded.title, duration_seconds=excluded.duration_seconds,
|
||||
video_type=excluded.video_type, thumbnail_url=excluded.thumbnail_url,
|
||||
description=excluded.description, updated_at=CURRENT_TIMESTAMP""",
|
||||
(
|
||||
video["id"],
|
||||
video["snippet"]["title"],
|
||||
video["snippet"]["publishedAt"],
|
||||
duration,
|
||||
video_type,
|
||||
video["snippet"]["thumbnails"].get("high", {}).get("url", ""),
|
||||
video["snippet"].get("description", ""),
|
||||
),
|
||||
)
|
||||
|
||||
conn.commit()
|
||||
next_page = pl_response.get("nextPageToken")
|
||||
if not next_page:
|
||||
break
|
||||
|
||||
print(f"Collected {len(video_ids)} videos")
|
||||
return video_ids
|
||||
|
||||
|
||||
def collect_daily_metrics(
|
||||
youtube_analytics, conn: sqlite3.Connection, video_ids: list[str],
|
||||
start_date: str | None = None, end_date: str | None = None,
|
||||
) -> None:
|
||||
"""Fetch daily metrics for each video from the Analytics API."""
|
||||
if not end_date:
|
||||
end_date = (datetime.now() - timedelta(days=2)).strftime("%Y-%m-%d")
|
||||
if not start_date:
|
||||
start_date = (datetime.now() - timedelta(days=30)).strftime("%Y-%m-%d")
|
||||
|
||||
for vid in video_ids:
|
||||
response = youtube_analytics.reports().query(
|
||||
ids="channel==MINE",
|
||||
startDate=start_date,
|
||||
endDate=end_date,
|
||||
metrics="views,estimatedMinutesWatched,averageViewDuration,averageViewPercentage,likes,dislikes,comments,shares,subscribersGained,subscribersLost",
|
||||
dimensions="day",
|
||||
filters=f"video=={vid}",
|
||||
sort="day",
|
||||
).execute()
|
||||
|
||||
for row in response.get("rows", []):
|
||||
conn.execute(
|
||||
"""INSERT INTO video_daily_metrics
|
||||
(video_id, date, views, estimated_minutes_watched,
|
||||
average_view_duration_seconds, average_view_percentage,
|
||||
likes, dislikes, comments, shares,
|
||||
subscribers_gained, subscribers_lost)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
ON CONFLICT(video_id, date) DO UPDATE SET
|
||||
views=excluded.views,
|
||||
estimated_minutes_watched=excluded.estimated_minutes_watched,
|
||||
average_view_duration_seconds=excluded.average_view_duration_seconds,
|
||||
average_view_percentage=excluded.average_view_percentage,
|
||||
likes=excluded.likes, dislikes=excluded.dislikes,
|
||||
comments=excluded.comments, shares=excluded.shares,
|
||||
subscribers_gained=excluded.subscribers_gained,
|
||||
subscribers_lost=excluded.subscribers_lost""",
|
||||
(vid, row[0], *row[1:]),
|
||||
)
|
||||
|
||||
conn.commit()
|
||||
print(f"Collected daily metrics for {len(video_ids)} videos")
|
||||
|
||||
|
||||
def collect_retention(
|
||||
youtube_analytics, conn: sqlite3.Connection, video_ids: list[str],
|
||||
) -> None:
|
||||
"""Fetch audience retention curves for each video."""
|
||||
for vid in video_ids:
|
||||
response = youtube_analytics.reports().query(
|
||||
ids="channel==MINE",
|
||||
startDate="2020-01-01",
|
||||
endDate=datetime.now().strftime("%Y-%m-%d"),
|
||||
metrics="audienceWatchRatio,relativeRetentionPerformance",
|
||||
dimensions="elapsedVideoTimeRatio",
|
||||
filters=f"video=={vid}",
|
||||
sort="elapsedVideoTimeRatio",
|
||||
).execute()
|
||||
|
||||
for row in response.get("rows", []):
|
||||
conn.execute(
|
||||
"""INSERT INTO video_retention
|
||||
(video_id, elapsed_ratio, audience_watch_ratio,
|
||||
relative_retention_performance, fetched_at)
|
||||
VALUES (?, ?, ?, ?, CURRENT_TIMESTAMP)
|
||||
ON CONFLICT(video_id, elapsed_ratio) DO UPDATE SET
|
||||
audience_watch_ratio=excluded.audience_watch_ratio,
|
||||
relative_retention_performance=excluded.relative_retention_performance,
|
||||
fetched_at=CURRENT_TIMESTAMP""",
|
||||
(vid, row[0], row[1], row[2]),
|
||||
)
|
||||
|
||||
conn.commit()
|
||||
print(f"Collected retention data for {len(video_ids)} videos")
|
||||
|
||||
|
||||
def collect_traffic_sources(
|
||||
youtube_analytics, conn: sqlite3.Connection, video_ids: list[str],
|
||||
start_date: str | None = None, end_date: str | None = None,
|
||||
) -> None:
|
||||
"""Fetch traffic source breakdown for each video."""
|
||||
if not end_date:
|
||||
end_date = (datetime.now() - timedelta(days=2)).strftime("%Y-%m-%d")
|
||||
if not start_date:
|
||||
start_date = (datetime.now() - timedelta(days=30)).strftime("%Y-%m-%d")
|
||||
|
||||
for vid in video_ids:
|
||||
response = youtube_analytics.reports().query(
|
||||
ids="channel==MINE",
|
||||
startDate=start_date,
|
||||
endDate=end_date,
|
||||
metrics="views,estimatedMinutesWatched",
|
||||
dimensions="day,insightTrafficSourceType",
|
||||
filters=f"video=={vid}",
|
||||
sort="day",
|
||||
).execute()
|
||||
|
||||
for row in response.get("rows", []):
|
||||
conn.execute(
|
||||
"""INSERT INTO video_traffic_sources
|
||||
(video_id, date, traffic_source, views, estimated_minutes_watched)
|
||||
VALUES (?, ?, ?, ?, ?)
|
||||
ON CONFLICT(video_id, date, traffic_source) DO UPDATE SET
|
||||
views=excluded.views,
|
||||
estimated_minutes_watched=excluded.estimated_minutes_watched""",
|
||||
(vid, row[0], row[1], row[2], row[3]),
|
||||
)
|
||||
|
||||
conn.commit()
|
||||
print(f"Collected traffic source data for {len(video_ids)} videos")
|
||||
|
||||
|
||||
def collect_channel_metrics(
|
||||
youtube_analytics, conn: sqlite3.Connection,
|
||||
start_date: str | None = None, end_date: str | None = None,
|
||||
) -> None:
|
||||
"""Fetch channel-level daily summary metrics."""
|
||||
if not end_date:
|
||||
end_date = (datetime.now() - timedelta(days=2)).strftime("%Y-%m-%d")
|
||||
if not start_date:
|
||||
start_date = (datetime.now() - timedelta(days=30)).strftime("%Y-%m-%d")
|
||||
|
||||
response = youtube_analytics.reports().query(
|
||||
ids="channel==MINE",
|
||||
startDate=start_date,
|
||||
endDate=end_date,
|
||||
metrics="views,estimatedMinutesWatched,subscribersGained,subscribersLost",
|
||||
dimensions="day",
|
||||
sort="day",
|
||||
).execute()
|
||||
|
||||
for row in response.get("rows", []):
|
||||
conn.execute(
|
||||
"""INSERT INTO channel_daily_metrics
|
||||
(date, total_views, total_estimated_minutes_watched,
|
||||
subscribers_gained, subscribers_lost, net_subscribers)
|
||||
VALUES (?, ?, ?, ?, ?, ?)
|
||||
ON CONFLICT(date) DO UPDATE SET
|
||||
total_views=excluded.total_views,
|
||||
total_estimated_minutes_watched=excluded.total_estimated_minutes_watched,
|
||||
subscribers_gained=excluded.subscribers_gained,
|
||||
subscribers_lost=excluded.subscribers_lost,
|
||||
net_subscribers=excluded.net_subscribers""",
|
||||
(row[0], row[1], row[2], row[3], row[4], row[3] - row[4]),
|
||||
)
|
||||
|
||||
conn.commit()
|
||||
print("Collected channel daily metrics")
|
||||
|
||||
|
||||
def _parse_duration(iso_duration: str) -> int:
|
||||
"""Parse ISO 8601 duration (PT1H2M3S) to total seconds."""
|
||||
import re
|
||||
match = re.match(r"PT(?:(\d+)H)?(?:(\d+)M)?(?:(\d+)S)?", iso_duration)
|
||||
if not match:
|
||||
return 0
|
||||
hours = int(match.group(1) or 0)
|
||||
minutes = int(match.group(2) or 0)
|
||||
seconds = int(match.group(3) or 0)
|
||||
return hours * 3600 + minutes * 60 + seconds
|
||||
532
src/dashboard.py
Normal file
532
src/dashboard.py
Normal file
@ -0,0 +1,532 @@
|
||||
"""Streamlit dashboard for PBS YouTube Analytics."""
|
||||
|
||||
import sqlite3
|
||||
import streamlit as st
|
||||
import pandas as pd
|
||||
import plotly.graph_objects as go
|
||||
import plotly.express as px
|
||||
from datetime import datetime, timedelta
|
||||
from pathlib import Path
|
||||
|
||||
from src.database import get_connection, DEFAULT_DB_PATH
|
||||
|
||||
|
||||
def _render_video_tab(conn, video_type: str, start_date, end_date) -> None:
|
||||
"""Render content for the Videos or Shorts tab."""
|
||||
label = "Video" if video_type == "video" else "Short"
|
||||
label_plural = "Videos" if video_type == "video" else "Shorts"
|
||||
|
||||
# Performance table
|
||||
st.subheader(f"{label_plural} Performance")
|
||||
try:
|
||||
video_stats = pd.read_sql_query(
|
||||
f"""SELECT
|
||||
v.video_id,
|
||||
v.title,
|
||||
v.duration_seconds,
|
||||
SUM(m.views) as total_views,
|
||||
SUM(m.estimated_minutes_watched) as watch_time_minutes,
|
||||
ROUND(AVG(m.average_view_percentage), 2) as avg_completion_pct,
|
||||
SUM(m.likes) as likes,
|
||||
SUM(m.comments) as comments
|
||||
FROM videos v
|
||||
LEFT JOIN video_daily_metrics m ON v.video_id = m.video_id
|
||||
AND m.date BETWEEN ? AND ?
|
||||
WHERE v.video_type = ?
|
||||
GROUP BY v.video_id
|
||||
ORDER BY total_views DESC""",
|
||||
conn,
|
||||
params=(start_date.isoformat(), end_date.isoformat(), video_type),
|
||||
)
|
||||
|
||||
if not video_stats.empty:
|
||||
video_stats["Duration"] = video_stats["duration_seconds"].apply(
|
||||
lambda x: f"{int(x // 60)}:{int(x % 60):02d}" if x else "0:00"
|
||||
)
|
||||
video_stats["Watch Time (h)"] = (video_stats["watch_time_minutes"] / 60).round(1)
|
||||
display_cols = ["title", "Duration", "total_views", "Watch Time (h)", "avg_completion_pct", "likes", "comments"]
|
||||
st.dataframe(
|
||||
video_stats[display_cols].rename(columns={
|
||||
"title": "Title",
|
||||
"total_views": "Views",
|
||||
"avg_completion_pct": "Completion %",
|
||||
}),
|
||||
width='stretch',
|
||||
hide_index=True,
|
||||
)
|
||||
else:
|
||||
st.info(f"No {label_plural.lower()} data available.")
|
||||
except Exception as e:
|
||||
st.warning(f"Could not load {label_plural.lower()} statistics: {e}")
|
||||
|
||||
st.divider()
|
||||
|
||||
col1, col2 = st.columns(2)
|
||||
|
||||
with col1:
|
||||
st.subheader(f"Top {label_plural} by Views")
|
||||
try:
|
||||
top_videos = pd.read_sql_query(
|
||||
f"""SELECT
|
||||
v.title,
|
||||
SUM(m.views) as total_views
|
||||
FROM videos v
|
||||
LEFT JOIN video_daily_metrics m ON v.video_id = m.video_id
|
||||
AND m.date BETWEEN ? AND ?
|
||||
WHERE v.video_type = ?
|
||||
GROUP BY v.video_id
|
||||
ORDER BY total_views DESC
|
||||
LIMIT 10""",
|
||||
conn,
|
||||
params=(start_date.isoformat(), end_date.isoformat(), video_type),
|
||||
)
|
||||
if not top_videos.empty:
|
||||
fig = px.bar(
|
||||
top_videos,
|
||||
y="title",
|
||||
x="total_views",
|
||||
orientation="h",
|
||||
labels={"title": "", "total_views": "Views"},
|
||||
)
|
||||
fig.update_yaxes(autorange="reversed")
|
||||
fig.update_layout(showlegend=False, height=400)
|
||||
st.plotly_chart(fig, width='stretch')
|
||||
else:
|
||||
st.info(f"No {label_plural.lower()} data available.")
|
||||
except Exception as e:
|
||||
st.warning(f"Could not load top {label_plural.lower()}: {e}")
|
||||
|
||||
with col2:
|
||||
st.subheader("Traffic Source Breakdown")
|
||||
try:
|
||||
traffic = pd.read_sql_query(
|
||||
f"""SELECT
|
||||
t.traffic_source,
|
||||
SUM(t.views) as views
|
||||
FROM video_traffic_sources t
|
||||
JOIN videos v ON t.video_id = v.video_id
|
||||
WHERE t.date BETWEEN ? AND ?
|
||||
AND v.video_type = ?
|
||||
GROUP BY t.traffic_source
|
||||
ORDER BY views DESC""",
|
||||
conn,
|
||||
params=(start_date.isoformat(), end_date.isoformat(), video_type),
|
||||
)
|
||||
if not traffic.empty:
|
||||
fig = px.pie(
|
||||
traffic,
|
||||
names="traffic_source",
|
||||
values="views",
|
||||
labels={"traffic_source": "Source", "views": "Views"},
|
||||
)
|
||||
fig.update_layout(height=400)
|
||||
st.plotly_chart(fig, width='stretch')
|
||||
else:
|
||||
st.info(f"No traffic data available for {label_plural.lower()}.")
|
||||
except Exception as e:
|
||||
st.warning(f"Could not load traffic sources: {e}")
|
||||
|
||||
st.divider()
|
||||
|
||||
# Per-video detail
|
||||
st.subheader(f"{label} Detail")
|
||||
try:
|
||||
videos_of_type = pd.read_sql_query(
|
||||
"SELECT video_id, title FROM videos WHERE video_type = ? ORDER BY title",
|
||||
conn,
|
||||
params=(video_type,),
|
||||
)
|
||||
|
||||
if videos_of_type.empty:
|
||||
st.info(f"No {label_plural.lower()} found.")
|
||||
return
|
||||
|
||||
id_to_title = dict(zip(videos_of_type["video_id"], videos_of_type["title"]))
|
||||
|
||||
selected_videos = st.multiselect(
|
||||
f"Select {label_plural.lower()} to explore",
|
||||
options=videos_of_type["video_id"].tolist(),
|
||||
format_func=lambda x: id_to_title[x],
|
||||
key=f"multiselect_{video_type}",
|
||||
)
|
||||
|
||||
if selected_videos:
|
||||
metric_choice = st.selectbox(
|
||||
"Metric",
|
||||
options=["Views", "Watch Time"],
|
||||
key=f"metric_select_{video_type}",
|
||||
)
|
||||
if metric_choice == "Views":
|
||||
y_col = "views"
|
||||
y_label = "Cumulative Views"
|
||||
else:
|
||||
y_col = "Watch Time (h)"
|
||||
y_label = "Cumulative Watch Time (h)"
|
||||
|
||||
colors = px.colors.qualitative.Plotly
|
||||
|
||||
st.subheader("Daily Views & Watch Time")
|
||||
try:
|
||||
fig = go.Figure()
|
||||
for i, vid_id in enumerate(selected_videos):
|
||||
daily = pd.read_sql_query(
|
||||
"""SELECT date, views, estimated_minutes_watched
|
||||
FROM video_daily_metrics
|
||||
WHERE video_id = ? AND date BETWEEN ? AND ?
|
||||
ORDER BY date""",
|
||||
conn,
|
||||
params=(vid_id, start_date.isoformat(), end_date.isoformat()),
|
||||
)
|
||||
if not daily.empty:
|
||||
daily["date"] = pd.to_datetime(daily["date"])
|
||||
daily["Watch Time (h)"] = (daily["estimated_minutes_watched"] / 60).round(2)
|
||||
daily["cumulative"] = daily[y_col].cumsum()
|
||||
color = colors[i % len(colors)]
|
||||
fig.add_trace(go.Scatter(
|
||||
x=daily["date"],
|
||||
y=daily["cumulative"],
|
||||
name=id_to_title[vid_id],
|
||||
mode="lines+markers",
|
||||
line=dict(color=color),
|
||||
))
|
||||
fig.update_layout(
|
||||
xaxis_title="Date",
|
||||
yaxis_title=y_label,
|
||||
hovermode="x unified",
|
||||
)
|
||||
st.plotly_chart(fig, width='stretch')
|
||||
except Exception as e:
|
||||
st.warning(f"Could not load daily metrics: {e}")
|
||||
|
||||
st.subheader("Retention Curves")
|
||||
try:
|
||||
fig = go.Figure()
|
||||
for i, vid_id in enumerate(selected_videos):
|
||||
retention = pd.read_sql_query(
|
||||
"""SELECT elapsed_ratio * 100 as elapsed_pct, audience_watch_ratio
|
||||
FROM video_retention
|
||||
WHERE video_id = ?
|
||||
ORDER BY elapsed_ratio""",
|
||||
conn,
|
||||
params=(vid_id,),
|
||||
)
|
||||
if not retention.empty:
|
||||
color = colors[i % len(colors)]
|
||||
fig.add_trace(go.Scatter(
|
||||
x=retention["elapsed_pct"],
|
||||
y=retention["audience_watch_ratio"],
|
||||
mode="lines",
|
||||
name=id_to_title[vid_id],
|
||||
line=dict(color=color),
|
||||
))
|
||||
fig.update_layout(
|
||||
xaxis_title="Video Progress (%)",
|
||||
yaxis_title="Audience Watch Ratio (%)",
|
||||
hovermode="x unified",
|
||||
)
|
||||
st.plotly_chart(fig, width='stretch')
|
||||
except Exception as e:
|
||||
st.warning(f"Could not load retention curves: {e}")
|
||||
|
||||
except Exception as e:
|
||||
st.warning(f"Could not load {label.lower()} detail section: {e}")
|
||||
|
||||
|
||||
def run_dashboard():
|
||||
"""Run the Streamlit dashboard."""
|
||||
st.set_page_config(page_title="PBS YouTube Analytics", layout="wide")
|
||||
st.title("PBS YouTube Analytics Dashboard")
|
||||
|
||||
db_path = DEFAULT_DB_PATH
|
||||
if not db_path.exists():
|
||||
st.error(f"Database not found at {db_path}. Please run 'init-db' and 'seed' first.")
|
||||
return
|
||||
|
||||
conn = get_connection(db_path)
|
||||
|
||||
# Sidebar — date range filter only
|
||||
st.sidebar.header("Filters")
|
||||
|
||||
try:
|
||||
min_date_result = conn.execute("SELECT MIN(date) FROM channel_daily_metrics").fetchone()
|
||||
max_date_result = conn.execute("SELECT MAX(date) FROM channel_daily_metrics").fetchone()
|
||||
|
||||
if min_date_result[0] and max_date_result[0]:
|
||||
min_date = datetime.fromisoformat(min_date_result[0]).date()
|
||||
max_date = datetime.fromisoformat(max_date_result[0]).date()
|
||||
else:
|
||||
min_date = datetime.now().date() - timedelta(days=30)
|
||||
max_date = datetime.now().date()
|
||||
except Exception:
|
||||
min_date = datetime.now().date() - timedelta(days=30)
|
||||
max_date = datetime.now().date()
|
||||
|
||||
date_range = st.sidebar.date_input(
|
||||
"Select date range",
|
||||
value=(min_date, max_date),
|
||||
min_value=min_date,
|
||||
max_value=max_date,
|
||||
)
|
||||
|
||||
if len(date_range) == 2:
|
||||
start_date, end_date = date_range
|
||||
else:
|
||||
start_date, end_date = min_date, max_date
|
||||
|
||||
# Sidebar — Data Collection
|
||||
st.sidebar.divider()
|
||||
st.sidebar.header("Data Collection")
|
||||
|
||||
# Display any status message carried over from a previous collection run via session state
|
||||
if "_collect_msg" in st.session_state:
|
||||
_msg_type, _msg_text = st.session_state.pop("_collect_msg")
|
||||
if _msg_type == "success":
|
||||
st.sidebar.success(_msg_text)
|
||||
else:
|
||||
st.sidebar.error(_msg_text)
|
||||
|
||||
today = datetime.now().date()
|
||||
collect_start = st.sidebar.date_input(
|
||||
"Start date",
|
||||
value=today - timedelta(days=30),
|
||||
key="collect_start",
|
||||
)
|
||||
collect_end = st.sidebar.date_input(
|
||||
"End date",
|
||||
value=today - timedelta(days=2),
|
||||
key="collect_end",
|
||||
)
|
||||
|
||||
def _open_collection_conn():
|
||||
sconn = sqlite3.connect(str(db_path))
|
||||
sconn.execute("PRAGMA journal_mode=WAL")
|
||||
sconn.execute("PRAGMA foreign_keys=ON")
|
||||
return sconn
|
||||
|
||||
if st.sidebar.button("Collect All", type="primary", use_container_width=True):
|
||||
with st.spinner("Collecting all data from YouTube API..."):
|
||||
try:
|
||||
from src.auth import get_authenticated_service
|
||||
from src.collector import (
|
||||
collect_channel_metrics,
|
||||
collect_daily_metrics,
|
||||
collect_retention,
|
||||
collect_traffic_sources,
|
||||
collect_video_list,
|
||||
)
|
||||
from src.database import init_db as _init_db
|
||||
|
||||
youtube, youtube_analytics = get_authenticated_service()
|
||||
_init_db(db_path)
|
||||
sconn = _open_collection_conn()
|
||||
start_str = collect_start.isoformat()
|
||||
end_str = collect_end.isoformat()
|
||||
video_ids = collect_video_list(youtube, sconn)
|
||||
collect_daily_metrics(youtube_analytics, sconn, video_ids, start_str, end_str)
|
||||
collect_retention(youtube_analytics, sconn, video_ids)
|
||||
collect_traffic_sources(youtube_analytics, sconn, video_ids, start_str, end_str)
|
||||
collect_channel_metrics(youtube_analytics, sconn, start_str, end_str)
|
||||
sconn.close()
|
||||
st.session_state["_collect_msg"] = ("success", "All data collected successfully!")
|
||||
st.rerun()
|
||||
except FileNotFoundError as e:
|
||||
st.sidebar.error(
|
||||
f"Auth credentials not found. "
|
||||
f"Save config/client_secret.json to enable collection.\n\n{e}"
|
||||
)
|
||||
except Exception as e:
|
||||
st.sidebar.error(f"Collection error: {e}")
|
||||
|
||||
with st.sidebar.expander("Individual Steps"):
|
||||
if st.button("Collect Video List", use_container_width=True, key="btn_videos"):
|
||||
with st.spinner("Collecting video list..."):
|
||||
try:
|
||||
from src.auth import get_authenticated_service
|
||||
from src.collector import collect_video_list
|
||||
from src.database import init_db as _init_db
|
||||
|
||||
youtube, _ = get_authenticated_service()
|
||||
_init_db(db_path)
|
||||
sconn = _open_collection_conn()
|
||||
collect_video_list(youtube, sconn)
|
||||
sconn.close()
|
||||
st.session_state["_collect_msg"] = ("success", "Video list collected!")
|
||||
st.rerun()
|
||||
except FileNotFoundError as e:
|
||||
st.sidebar.error(f"Auth credentials not found: {e}")
|
||||
except Exception as e:
|
||||
st.sidebar.error(f"Error: {e}")
|
||||
|
||||
if st.button("Collect Daily Metrics", use_container_width=True, key="btn_daily"):
|
||||
with st.spinner("Collecting daily metrics..."):
|
||||
try:
|
||||
from src.auth import get_authenticated_service
|
||||
from src.collector import collect_daily_metrics
|
||||
|
||||
_, youtube_analytics = get_authenticated_service()
|
||||
sconn = _open_collection_conn()
|
||||
video_ids = [r[0] for r in sconn.execute("SELECT video_id FROM videos").fetchall()]
|
||||
collect_daily_metrics(
|
||||
youtube_analytics, sconn, video_ids,
|
||||
collect_start.isoformat(), collect_end.isoformat(),
|
||||
)
|
||||
sconn.close()
|
||||
st.session_state["_collect_msg"] = ("success", "Daily metrics collected!")
|
||||
st.rerun()
|
||||
except FileNotFoundError as e:
|
||||
st.sidebar.error(f"Auth credentials not found: {e}")
|
||||
except Exception as e:
|
||||
st.sidebar.error(f"Error: {e}")
|
||||
|
||||
if st.button("Collect Retention", use_container_width=True, key="btn_retention"):
|
||||
with st.spinner("Collecting retention data..."):
|
||||
try:
|
||||
from src.auth import get_authenticated_service
|
||||
from src.collector import collect_retention
|
||||
|
||||
_, youtube_analytics = get_authenticated_service()
|
||||
sconn = _open_collection_conn()
|
||||
video_ids = [r[0] for r in sconn.execute("SELECT video_id FROM videos").fetchall()]
|
||||
collect_retention(youtube_analytics, sconn, video_ids)
|
||||
sconn.close()
|
||||
st.session_state["_collect_msg"] = ("success", "Retention data collected!")
|
||||
st.rerun()
|
||||
except FileNotFoundError as e:
|
||||
st.sidebar.error(f"Auth credentials not found: {e}")
|
||||
except Exception as e:
|
||||
st.sidebar.error(f"Error: {e}")
|
||||
|
||||
if st.button("Collect Traffic Sources", use_container_width=True, key="btn_traffic"):
|
||||
with st.spinner("Collecting traffic sources..."):
|
||||
try:
|
||||
from src.auth import get_authenticated_service
|
||||
from src.collector import collect_traffic_sources
|
||||
|
||||
_, youtube_analytics = get_authenticated_service()
|
||||
sconn = _open_collection_conn()
|
||||
video_ids = [r[0] for r in sconn.execute("SELECT video_id FROM videos").fetchall()]
|
||||
collect_traffic_sources(
|
||||
youtube_analytics, sconn, video_ids,
|
||||
collect_start.isoformat(), collect_end.isoformat(),
|
||||
)
|
||||
sconn.close()
|
||||
st.session_state["_collect_msg"] = ("success", "Traffic sources collected!")
|
||||
st.rerun()
|
||||
except FileNotFoundError as e:
|
||||
st.sidebar.error(f"Auth credentials not found: {e}")
|
||||
except Exception as e:
|
||||
st.sidebar.error(f"Error: {e}")
|
||||
|
||||
if st.button("Collect Channel Metrics", use_container_width=True, key="btn_channel"):
|
||||
with st.spinner("Collecting channel metrics..."):
|
||||
try:
|
||||
from src.auth import get_authenticated_service
|
||||
from src.collector import collect_channel_metrics
|
||||
|
||||
_, youtube_analytics = get_authenticated_service()
|
||||
sconn = _open_collection_conn()
|
||||
collect_channel_metrics(
|
||||
youtube_analytics, sconn,
|
||||
collect_start.isoformat(), collect_end.isoformat(),
|
||||
)
|
||||
sconn.close()
|
||||
st.session_state["_collect_msg"] = ("success", "Channel metrics collected!")
|
||||
st.rerun()
|
||||
except FileNotFoundError as e:
|
||||
st.sidebar.error(f"Auth credentials not found: {e}")
|
||||
except Exception as e:
|
||||
st.sidebar.error(f"Error: {e}")
|
||||
|
||||
# Channel overview metrics (above tabs — channel-wide)
|
||||
col1, col2, col3 = st.columns(3)
|
||||
try:
|
||||
metrics = conn.execute(
|
||||
"""SELECT
|
||||
SUM(total_views) as total_views,
|
||||
SUM(total_estimated_minutes_watched) as total_watch_time,
|
||||
SUM(subscribers_gained) as total_subs_gained
|
||||
FROM channel_daily_metrics
|
||||
WHERE date BETWEEN ? AND ?""",
|
||||
(start_date.isoformat(), end_date.isoformat()),
|
||||
).fetchone()
|
||||
|
||||
if metrics and metrics[0]:
|
||||
with col1:
|
||||
st.metric("Total Views", f"{int(metrics[0]):,}")
|
||||
with col2:
|
||||
st.metric("Watch Time (hours)", f"{int(metrics[1] / 60):,}")
|
||||
with col3:
|
||||
st.metric("Subscribers Gained", f"{int(metrics[2]):,}")
|
||||
except Exception as e:
|
||||
st.warning(f"Could not load channel metrics: {e}")
|
||||
|
||||
st.divider()
|
||||
|
||||
# Views over time (above tabs — channel-wide)
|
||||
st.subheader("Views Over Time")
|
||||
try:
|
||||
daily_views = pd.read_sql_query(
|
||||
"""SELECT m.date, v.video_type, SUM(m.views) as views
|
||||
FROM video_daily_metrics m
|
||||
JOIN videos v ON m.video_id = v.video_id
|
||||
WHERE m.date BETWEEN ? AND ?
|
||||
GROUP BY m.date, v.video_type
|
||||
ORDER BY m.date""",
|
||||
conn,
|
||||
params=(start_date.isoformat(), end_date.isoformat()),
|
||||
)
|
||||
if not daily_views.empty:
|
||||
daily_views["date"] = pd.to_datetime(daily_views["date"])
|
||||
pivoted = daily_views.pivot_table(
|
||||
index="date", columns="video_type", values="views", fill_value=0
|
||||
).reset_index()
|
||||
if "short" not in pivoted.columns:
|
||||
pivoted["short"] = 0
|
||||
if "video" not in pivoted.columns:
|
||||
pivoted["video"] = 0
|
||||
fig = go.Figure()
|
||||
fig.add_trace(go.Scatter(
|
||||
x=pivoted["date"],
|
||||
y=pivoted["short"],
|
||||
name="Shorts",
|
||||
stackgroup="one",
|
||||
mode="lines",
|
||||
line=dict(color="#FF7043"),
|
||||
fillcolor="rgba(255, 112, 67, 0.6)",
|
||||
))
|
||||
fig.add_trace(go.Scatter(
|
||||
x=pivoted["date"],
|
||||
y=pivoted["video"],
|
||||
name="Videos",
|
||||
stackgroup="one",
|
||||
mode="lines",
|
||||
line=dict(color="#1976D2"),
|
||||
fillcolor="rgba(25, 118, 210, 0.6)",
|
||||
))
|
||||
fig.update_layout(
|
||||
xaxis_title="Date",
|
||||
yaxis_title="Views",
|
||||
hovermode="x unified",
|
||||
)
|
||||
st.plotly_chart(fig, width='stretch')
|
||||
else:
|
||||
st.info("No data available for the selected date range.")
|
||||
except Exception as e:
|
||||
st.warning(f"Could not load views chart: {e}")
|
||||
|
||||
st.divider()
|
||||
|
||||
# Tabbed section — per-type content
|
||||
videos_tab, shorts_tab = st.tabs(["Videos", "Shorts"])
|
||||
|
||||
with videos_tab:
|
||||
_render_video_tab(conn, "video", start_date, end_date)
|
||||
|
||||
with shorts_tab:
|
||||
_render_video_tab(conn, "short", start_date, end_date)
|
||||
|
||||
conn.close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
run_dashboard()
|
||||
91
src/database.py
Normal file
91
src/database.py
Normal file
@ -0,0 +1,91 @@
|
||||
"""SQLite database setup and connection management for PBS YouTube Analytics."""
|
||||
|
||||
import sqlite3
|
||||
from pathlib import Path
|
||||
|
||||
DEFAULT_DB_PATH = Path(__file__).parent.parent / "data" / "pbs_youtube.db"
|
||||
|
||||
SCHEMA = """
|
||||
-- Video metadata from Data API
|
||||
CREATE TABLE IF NOT EXISTS videos (
|
||||
video_id TEXT PRIMARY KEY,
|
||||
title TEXT NOT NULL,
|
||||
published_at TEXT NOT NULL,
|
||||
duration_seconds INTEGER,
|
||||
video_type TEXT CHECK(video_type IN ('video', 'short')),
|
||||
thumbnail_url TEXT,
|
||||
description TEXT,
|
||||
created_at TEXT DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TEXT DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
|
||||
-- Daily aggregate metrics from Analytics API
|
||||
CREATE TABLE IF NOT EXISTS video_daily_metrics (
|
||||
video_id TEXT NOT NULL,
|
||||
date TEXT NOT NULL,
|
||||
views INTEGER DEFAULT 0,
|
||||
estimated_minutes_watched REAL DEFAULT 0,
|
||||
average_view_duration_seconds REAL DEFAULT 0,
|
||||
average_view_percentage REAL DEFAULT 0,
|
||||
likes INTEGER DEFAULT 0,
|
||||
dislikes INTEGER DEFAULT 0,
|
||||
comments INTEGER DEFAULT 0,
|
||||
shares INTEGER DEFAULT 0,
|
||||
subscribers_gained INTEGER DEFAULT 0,
|
||||
subscribers_lost INTEGER DEFAULT 0,
|
||||
impressions INTEGER DEFAULT 0,
|
||||
impressions_ctr REAL DEFAULT 0,
|
||||
created_at TEXT DEFAULT CURRENT_TIMESTAMP,
|
||||
PRIMARY KEY (video_id, date),
|
||||
FOREIGN KEY (video_id) REFERENCES videos(video_id)
|
||||
);
|
||||
|
||||
-- Audience retention curve (100 data points per video)
|
||||
CREATE TABLE IF NOT EXISTS video_retention (
|
||||
video_id TEXT NOT NULL,
|
||||
elapsed_ratio REAL NOT NULL,
|
||||
audience_watch_ratio REAL NOT NULL,
|
||||
relative_retention_performance REAL,
|
||||
fetched_at TEXT DEFAULT CURRENT_TIMESTAMP,
|
||||
PRIMARY KEY (video_id, elapsed_ratio),
|
||||
FOREIGN KEY (video_id) REFERENCES videos(video_id)
|
||||
);
|
||||
|
||||
-- Traffic source breakdown per video per day
|
||||
CREATE TABLE IF NOT EXISTS video_traffic_sources (
|
||||
video_id TEXT NOT NULL,
|
||||
date TEXT NOT NULL,
|
||||
traffic_source TEXT NOT NULL,
|
||||
views INTEGER DEFAULT 0,
|
||||
estimated_minutes_watched REAL DEFAULT 0,
|
||||
PRIMARY KEY (video_id, date, traffic_source),
|
||||
FOREIGN KEY (video_id) REFERENCES videos(video_id)
|
||||
);
|
||||
|
||||
-- Channel-level daily summary
|
||||
CREATE TABLE IF NOT EXISTS channel_daily_metrics (
|
||||
date TEXT PRIMARY KEY,
|
||||
total_views INTEGER DEFAULT 0,
|
||||
total_estimated_minutes_watched REAL DEFAULT 0,
|
||||
subscribers_gained INTEGER DEFAULT 0,
|
||||
subscribers_lost INTEGER DEFAULT 0,
|
||||
net_subscribers INTEGER DEFAULT 0,
|
||||
created_at TEXT DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
"""
|
||||
|
||||
|
||||
def get_connection(db_path: Path = DEFAULT_DB_PATH) -> sqlite3.Connection:
|
||||
db_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
conn = sqlite3.connect(str(db_path))
|
||||
conn.row_factory = sqlite3.Row
|
||||
conn.execute("PRAGMA journal_mode=WAL")
|
||||
conn.execute("PRAGMA foreign_keys=ON")
|
||||
return conn
|
||||
|
||||
|
||||
def init_db(db_path: Path = DEFAULT_DB_PATH) -> None:
|
||||
conn = get_connection(db_path)
|
||||
conn.executescript(SCHEMA)
|
||||
conn.close()
|
||||
print(f"Database initialized at {db_path}")
|
||||
390
src/seed.py
Normal file
390
src/seed.py
Normal file
@ -0,0 +1,390 @@
|
||||
"""Seed script to populate SQLite database with realistic sample data for a PBS plant-based cooking channel."""
|
||||
|
||||
import sqlite3
|
||||
import random
|
||||
from datetime import datetime, timedelta
|
||||
from pathlib import Path
|
||||
|
||||
from src.database import get_connection, DEFAULT_DB_PATH
|
||||
|
||||
|
||||
def seed_database(db_path: Path = DEFAULT_DB_PATH) -> None:
|
||||
"""Populate the database with realistic sample data."""
|
||||
conn = get_connection(db_path)
|
||||
|
||||
# Clear existing data
|
||||
conn.execute("DELETE FROM video_traffic_sources")
|
||||
conn.execute("DELETE FROM video_retention")
|
||||
conn.execute("DELETE FROM video_daily_metrics")
|
||||
conn.execute("DELETE FROM channel_daily_metrics")
|
||||
conn.execute("DELETE FROM videos")
|
||||
conn.commit()
|
||||
|
||||
# Seed videos
|
||||
video_ids = _seed_videos(conn)
|
||||
|
||||
# Seed daily metrics for the last 30 days
|
||||
_seed_daily_metrics(conn, video_ids)
|
||||
|
||||
# Seed retention curves
|
||||
_seed_retention(conn, video_ids)
|
||||
|
||||
# Seed traffic sources
|
||||
_seed_traffic_sources(conn, video_ids)
|
||||
|
||||
# Seed channel daily metrics
|
||||
_seed_channel_metrics(conn)
|
||||
|
||||
conn.close()
|
||||
print(f"Database seeded successfully at {db_path}")
|
||||
|
||||
|
||||
def _seed_videos(conn: sqlite3.Connection) -> list[str]:
|
||||
"""Create ~18 videos: mix of shorts and long-form, PBS plant-based cooking themed."""
|
||||
videos = [
|
||||
# Shorts (30-60 seconds)
|
||||
{
|
||||
"video_id": "short_001",
|
||||
"title": "Quick Vegan Breakfast: 30-Second Oatmeal Hack",
|
||||
"duration_seconds": 45,
|
||||
"video_type": "short",
|
||||
"published_at": (datetime.now() - timedelta(days=25)).isoformat() + "Z",
|
||||
},
|
||||
{
|
||||
"video_id": "short_002",
|
||||
"title": "Plant-Based Protein Smoothie Bowl",
|
||||
"duration_seconds": 52,
|
||||
"video_type": "short",
|
||||
"published_at": (datetime.now() - timedelta(days=23)).isoformat() + "Z",
|
||||
},
|
||||
{
|
||||
"video_id": "short_003",
|
||||
"title": "Crispy Tofu in 60 Seconds",
|
||||
"duration_seconds": 58,
|
||||
"video_type": "short",
|
||||
"published_at": (datetime.now() - timedelta(days=21)).isoformat() + "Z",
|
||||
},
|
||||
{
|
||||
"video_id": "short_004",
|
||||
"title": "No-Cook Hummus & Veggie Snack Hack",
|
||||
"duration_seconds": 48,
|
||||
"video_type": "short",
|
||||
"published_at": (datetime.now() - timedelta(days=19)).isoformat() + "Z",
|
||||
},
|
||||
{
|
||||
"video_id": "short_005",
|
||||
"title": "5-Minute Mango Sorbet",
|
||||
"duration_seconds": 55,
|
||||
"video_type": "short",
|
||||
"published_at": (datetime.now() - timedelta(days=17)).isoformat() + "Z",
|
||||
},
|
||||
{
|
||||
"video_id": "short_006",
|
||||
"title": "Instant Pasta Primavera Hack",
|
||||
"duration_seconds": 50,
|
||||
"video_type": "short",
|
||||
"published_at": (datetime.now() - timedelta(days=15)).isoformat() + "Z",
|
||||
},
|
||||
# Long-form videos (5-20 minutes)
|
||||
{
|
||||
"video_id": "long_001",
|
||||
"title": "Complete Plant-Based Dinner: Mediterranean Buddha Bowl",
|
||||
"duration_seconds": 12 * 60 + 30,
|
||||
"video_type": "video",
|
||||
"published_at": (datetime.now() - timedelta(days=28)).isoformat() + "Z",
|
||||
},
|
||||
{
|
||||
"video_id": "long_002",
|
||||
"title": "Vegan Baking Masterclass: Chocolate Avocado Cake",
|
||||
"duration_seconds": 18 * 60 + 45,
|
||||
"video_type": "video",
|
||||
"published_at": (datetime.now() - timedelta(days=26)).isoformat() + "Z",
|
||||
},
|
||||
{
|
||||
"video_id": "long_003",
|
||||
"title": "How to Make Cashew Cream from Scratch",
|
||||
"duration_seconds": 8 * 60 + 15,
|
||||
"video_type": "video",
|
||||
"published_at": (datetime.now() - timedelta(days=24)).isoformat() + "Z",
|
||||
},
|
||||
{
|
||||
"video_id": "long_004",
|
||||
"title": "Budget Plant-Based Meal Prep for the Week",
|
||||
"duration_seconds": 16 * 60 + 20,
|
||||
"video_type": "video",
|
||||
"published_at": (datetime.now() - timedelta(days=22)).isoformat() + "Z",
|
||||
},
|
||||
{
|
||||
"video_id": "long_005",
|
||||
"title": "Fermentation Basics: Making Kimchi & Sauerkraut",
|
||||
"duration_seconds": 19 * 60 + 10,
|
||||
"video_type": "video",
|
||||
"published_at": (datetime.now() - timedelta(days=20)).isoformat() + "Z",
|
||||
},
|
||||
{
|
||||
"video_id": "long_006",
|
||||
"title": "Whole Plant-Based Thanksgiving Feast",
|
||||
"duration_seconds": 22 * 60 + 30,
|
||||
"video_type": "video",
|
||||
"published_at": (datetime.now() - timedelta(days=18)).isoformat() + "Z",
|
||||
},
|
||||
{
|
||||
"video_id": "long_007",
|
||||
"title": "Nutritionist Deep Dive: Plant-Based Protein Myth Busting",
|
||||
"duration_seconds": 14 * 60 + 50,
|
||||
"video_type": "video",
|
||||
"published_at": (datetime.now() - timedelta(days=16)).isoformat() + "Z",
|
||||
},
|
||||
{
|
||||
"video_id": "long_008",
|
||||
"title": "Zero-Waste Kitchen: Composting & Food Scraps",
|
||||
"duration_seconds": 10 * 60 + 25,
|
||||
"video_type": "video",
|
||||
"published_at": (datetime.now() - timedelta(days=14)).isoformat() + "Z",
|
||||
},
|
||||
{
|
||||
"video_id": "long_009",
|
||||
"title": "Street Food Around the World: Vegan Edition",
|
||||
"duration_seconds": 20 * 60 + 15,
|
||||
"video_type": "video",
|
||||
"published_at": (datetime.now() - timedelta(days=12)).isoformat() + "Z",
|
||||
},
|
||||
{
|
||||
"video_id": "long_010",
|
||||
"title": "How to Stock Your Plant-Based Pantry",
|
||||
"duration_seconds": 11 * 60 + 40,
|
||||
"video_type": "video",
|
||||
"published_at": (datetime.now() - timedelta(days=10)).isoformat() + "Z",
|
||||
},
|
||||
{
|
||||
"video_id": "long_011",
|
||||
"title": "Interview: Local Organic Farmer About Seasonal Produce",
|
||||
"duration_seconds": 17 * 60 + 35,
|
||||
"video_type": "video",
|
||||
"published_at": (datetime.now() - timedelta(days=8)).isoformat() + "Z",
|
||||
},
|
||||
{
|
||||
"video_id": "long_012",
|
||||
"title": "Plant-Based Athlete: How I Fuel My Training",
|
||||
"duration_seconds": 15 * 60 + 20,
|
||||
"video_type": "video",
|
||||
"published_at": (datetime.now() - timedelta(days=6)).isoformat() + "Z",
|
||||
},
|
||||
]
|
||||
|
||||
thumbnails = [
|
||||
"https://via.placeholder.com/320x180?text=Recipe",
|
||||
"https://via.placeholder.com/320x180?text=Cooking",
|
||||
"https://via.placeholder.com/320x180?text=Plant+Based",
|
||||
]
|
||||
|
||||
descriptions = [
|
||||
"Learn how to make delicious plant-based meals at home.",
|
||||
"Quick and easy vegan cooking tips for busy weeknights.",
|
||||
"Discover the joy of plant-based eating with our expert guides.",
|
||||
"Sustainable cooking starts in your kitchen.",
|
||||
"Healthy, delicious, and 100% plant-based.",
|
||||
]
|
||||
|
||||
video_ids = []
|
||||
for video in videos:
|
||||
video_ids.append(video["video_id"])
|
||||
conn.execute(
|
||||
"""INSERT INTO videos
|
||||
(video_id, title, published_at, duration_seconds, video_type, thumbnail_url, description)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?)""",
|
||||
(
|
||||
video["video_id"],
|
||||
video["title"],
|
||||
video["published_at"],
|
||||
video["duration_seconds"],
|
||||
video["video_type"],
|
||||
random.choice(thumbnails),
|
||||
random.choice(descriptions),
|
||||
),
|
||||
)
|
||||
|
||||
conn.commit()
|
||||
print(f"Seeded {len(video_ids)} videos")
|
||||
return video_ids
|
||||
|
||||
|
||||
def _seed_daily_metrics(conn: sqlite3.Connection, video_ids: list[str]) -> None:
|
||||
"""Seed 30 days of daily metrics for each video."""
|
||||
# Short videos get 50-500 views/day, long videos get 100-2000 views/day
|
||||
start_date = datetime.now() - timedelta(days=30)
|
||||
|
||||
for video_id in video_ids:
|
||||
is_short = video_id.startswith("short_")
|
||||
|
||||
for day_offset in range(30):
|
||||
date = (start_date + timedelta(days=day_offset)).strftime("%Y-%m-%d")
|
||||
|
||||
if is_short:
|
||||
views = random.randint(50, 500)
|
||||
estimated_minutes = views * random.uniform(0.5, 2.0)
|
||||
avg_duration = random.uniform(20, 45) # seconds
|
||||
ctr = random.uniform(0.8, 3.5)
|
||||
else:
|
||||
views = random.randint(100, 2000)
|
||||
estimated_minutes = views * random.uniform(3.0, 10.0)
|
||||
avg_duration = random.uniform(200, 800) # seconds
|
||||
ctr = random.uniform(1.2, 4.5)
|
||||
|
||||
avg_view_pct = (avg_duration / 720) * 100 if not is_short else (avg_duration / 60) * 100
|
||||
avg_view_pct = min(avg_view_pct, 100)
|
||||
|
||||
likes = int(views * random.uniform(0.01, 0.05))
|
||||
comments = int(views * random.uniform(0.002, 0.015))
|
||||
shares = int(views * random.uniform(0.001, 0.008))
|
||||
impressions = int(views / random.uniform(0.15, 0.35))
|
||||
subs_gained = max(0, int(views * random.uniform(0.001, 0.012)) - 1)
|
||||
|
||||
conn.execute(
|
||||
"""INSERT INTO video_daily_metrics
|
||||
(video_id, date, views, estimated_minutes_watched,
|
||||
average_view_duration_seconds, average_view_percentage,
|
||||
likes, dislikes, comments, shares,
|
||||
subscribers_gained, subscribers_lost, impressions, impressions_ctr)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""",
|
||||
(
|
||||
video_id, date, views, estimated_minutes,
|
||||
avg_duration, avg_view_pct,
|
||||
likes, 0, comments, shares,
|
||||
subs_gained, 0, impressions, ctr,
|
||||
),
|
||||
)
|
||||
|
||||
conn.commit()
|
||||
print(f"Seeded daily metrics for {len(video_ids)} videos over 30 days")
|
||||
|
||||
|
||||
def _seed_retention(conn: sqlite3.Connection, video_ids: list[str]) -> None:
|
||||
"""Seed audience retention curves (100 data points per video)."""
|
||||
for video_id in video_ids:
|
||||
is_short = video_id.startswith("short_")
|
||||
|
||||
# Retention typically starts high and drops off
|
||||
# Shorts: steep drop, longs: gradual decline
|
||||
for point in range(100):
|
||||
elapsed_ratio = point / 100.0
|
||||
|
||||
if is_short:
|
||||
# Shorts drop off faster
|
||||
audience_ratio = 100 * (1 - elapsed_ratio ** 1.2)
|
||||
else:
|
||||
# Long-form videos drop off more gradually
|
||||
audience_ratio = 100 * (1 - elapsed_ratio ** 0.8)
|
||||
|
||||
audience_ratio = max(1, audience_ratio) # Minimum 1%
|
||||
|
||||
relative_perf = random.uniform(-20, 30) # Performance vs avg
|
||||
|
||||
conn.execute(
|
||||
"""INSERT INTO video_retention
|
||||
(video_id, elapsed_ratio, audience_watch_ratio, relative_retention_performance)
|
||||
VALUES (?, ?, ?, ?)""",
|
||||
(video_id, elapsed_ratio, audience_ratio, relative_perf),
|
||||
)
|
||||
|
||||
conn.commit()
|
||||
print(f"Seeded retention curves for {len(video_ids)} videos")
|
||||
|
||||
|
||||
def _seed_traffic_sources(conn: sqlite3.Connection, video_ids: list[str]) -> None:
|
||||
"""Seed traffic source breakdown for each video-day combination."""
|
||||
sources = [
|
||||
"YouTube search",
|
||||
"Suggested videos",
|
||||
"Direct or unknown",
|
||||
"External website",
|
||||
"YouTube channel",
|
||||
"Playlist",
|
||||
]
|
||||
|
||||
start_date = datetime.now() - timedelta(days=30)
|
||||
|
||||
for video_id in video_ids:
|
||||
for day_offset in range(30):
|
||||
date = (start_date + timedelta(days=day_offset)).strftime("%Y-%m-%d")
|
||||
|
||||
# Get total views from video_daily_metrics
|
||||
cursor = conn.execute(
|
||||
"SELECT views FROM video_daily_metrics WHERE video_id = ? AND date = ?",
|
||||
(video_id, date),
|
||||
)
|
||||
result = cursor.fetchone()
|
||||
if not result:
|
||||
continue
|
||||
|
||||
total_views = result[0]
|
||||
|
||||
# Distribute across sources
|
||||
remaining_views = total_views
|
||||
for source in sources[:-1]:
|
||||
source_views = int(remaining_views * random.uniform(0.05, 0.35))
|
||||
source_minutes = source_views * random.uniform(1.0, 6.0)
|
||||
|
||||
conn.execute(
|
||||
"""INSERT INTO video_traffic_sources
|
||||
(video_id, date, traffic_source, views, estimated_minutes_watched)
|
||||
VALUES (?, ?, ?, ?, ?)""",
|
||||
(video_id, date, source, source_views, source_minutes),
|
||||
)
|
||||
remaining_views -= source_views
|
||||
|
||||
# Last source gets remainder
|
||||
if remaining_views > 0:
|
||||
conn.execute(
|
||||
"""INSERT INTO video_traffic_sources
|
||||
(video_id, date, traffic_source, views, estimated_minutes_watched)
|
||||
VALUES (?, ?, ?, ?, ?)""",
|
||||
(video_id, date, sources[-1], remaining_views, remaining_views * random.uniform(1.0, 6.0)),
|
||||
)
|
||||
|
||||
conn.commit()
|
||||
print(f"Seeded traffic sources for {len(video_ids)} videos over 30 days")
|
||||
|
||||
|
||||
def _seed_channel_metrics(conn: sqlite3.Connection) -> None:
|
||||
"""Seed channel-level daily summary metrics for the last 30 days."""
|
||||
start_date = datetime.now() - timedelta(days=30)
|
||||
|
||||
for day_offset in range(30):
|
||||
date = (start_date + timedelta(days=day_offset)).strftime("%Y-%m-%d")
|
||||
|
||||
# Get daily totals from video metrics
|
||||
cursor = conn.execute(
|
||||
"""SELECT
|
||||
SUM(views), SUM(estimated_minutes_watched)
|
||||
FROM video_daily_metrics
|
||||
WHERE date = ?""",
|
||||
(date,),
|
||||
)
|
||||
result = cursor.fetchone()
|
||||
|
||||
if result and result[0]:
|
||||
total_views = result[0]
|
||||
total_minutes = result[1] or 0
|
||||
else:
|
||||
# Fallback if no video metrics yet
|
||||
total_views = random.randint(2000, 8000)
|
||||
total_minutes = total_views * random.uniform(3.0, 8.0)
|
||||
|
||||
subs_gained = max(0, int(total_views * random.uniform(0.001, 0.015)) - 1)
|
||||
subs_lost = max(0, subs_gained - random.randint(0, int(subs_gained * 0.5)))
|
||||
|
||||
conn.execute(
|
||||
"""INSERT INTO channel_daily_metrics
|
||||
(date, total_views, total_estimated_minutes_watched,
|
||||
subscribers_gained, subscribers_lost, net_subscribers)
|
||||
VALUES (?, ?, ?, ?, ?, ?)""",
|
||||
(date, total_views, total_minutes, subs_gained, subs_lost, subs_gained - subs_lost),
|
||||
)
|
||||
|
||||
conn.commit()
|
||||
print("Seeded channel daily metrics for 30 days")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
seed_database()
|
||||
Loading…
Reference in New Issue
Block a user