#!/usr/bin/env python3
"""Prepare and render Soren's podcast-discovery queue with bounded model context.

The deterministic stages handle subscription exclusions, prior-candidate
exclusions, known-source RSS/YouTube collection, lexical pre-ranking,
authoritative enclosure checks, reaction archiving, payload encoding, and
dashboard rendering. The model sees only a compact shortlist and supplies the
semantic judgment: track, fit, novelty, and expertise.
"""

from __future__ import annotations

import argparse
import base64
from concurrent.futures import ThreadPoolExecutor, as_completed
from datetime import datetime, timedelta, timezone
import email.utils
import html
import json
import os
from pathlib import Path
import re
import subprocess
import tempfile
from typing import Dict, Iterable, List, Optional, Sequence, Set, Tuple
from urllib.request import Request, urlopen
from xml.etree import ElementTree as ET


PODCAST_ROOT = Path("/Users/soren/Podcasts")
VAULT_ROOT = Path("/Users/soren/Library/Mobile Documents/iCloud~md~obsidian/Documents/Personal Brain")
DEFAULT_OPML = VAULT_ROOT / "Library/Imports/Pocket Casts - 2026-08-16.opml"
DEFAULT_DASHBOARD = VAULT_ROOT / "_Dashboards/Podcast Discovery.md"
DEFAULT_REACTIONS = VAULT_ROOT / "Library/Podcast Discovery Reactions.md"
DEFAULT_PREFERENCES = VAULT_ROOT / "Library/Podcast Preferences.md"
DEFAULT_STATE = PODCAST_ROOT / "discoveries/state.json"
DEFAULT_SOURCES = PODCAST_ROOT / "discovery_sources.json"
DEFAULT_BRIEF = Path("/tmp/podcast-discovery-brief.json")
DISCOVERIES_SCRIPT = PODCAST_ROOT / "discoveries.py"
PODSYNC_FEEDS = (PODCAST_ROOT / "work_playlist.xml", PODCAST_ROOT / "my_playlist.xml")
SYNC_STATUS = PODCAST_ROOT / "sync-status.json"

ITUNES = "http://www.itunes.com/dtds/podcast-1.0.dtd"
ATOM = "http://www.w3.org/2005/Atom"
MEDIA = "http://search.yahoo.com/mrss/"
YT = "http://www.youtube.com/xml/schemas/2015"

TRACK_TERMS = {
    "AI institutions": (
        "artificial intelligence", " ai ", "algorithm", "evaluation", "benchmark",
        "measurement", "standards", "supervision", "regulator", "infrastructure",
        "automation", "language model", "epistem", "verification", "audit",
    ),
    "Oriental Orthodox formation": (
        "coptic", "syriac", "ethiopian", "oriental orthodox", "miaphys",
        "alexandria", "severus", "cyril", "patrist", "christology", "liturgy",
        "monastic", "late antique", "chalcedon", "church history",
    ),
    "Governance and state capacity": (
        "congress", "administrative", "state capacity", "bureaucr", "public management",
        "government", "implementation", "legitimacy", "institution", "constitutional",
        "civil service", "regulatory capacity", "public administration",
    ),
    "Philosophy and studies of science": (
        "philosophy of science", "science studies", "pragmat", "peirce", "latour",
        "complex systems", "expertise", "scientific", "model", "evidence",
        "inference", "fallibil", "semiotic", "metrology",
    ),
    "Anthropology of knowledge": (
        "anthropology", "ethnograph", "knowledge-making", "epistemic culture",
        "ritual", "formation", "authority", "memory", "communal", "practice",
    ),
}

NEGATIVE_TERMS = (
    "weekly news", "news roundup", "top stories", "beginner's guide", "beginner guide",
    "how to use chatgpt", "grow your business", "transformation consultant",
    "sponsored by", "founder story", "sales leader",
)

REACTION_NAMES = (
    "Good", "Not for me", "More like this", "Too basic", "Too chatty",
    "Too long", "Already knew this",
)

ROTATING_SEARCH_TRACKS = (
    "AI as institutional infrastructure, evaluation, measurement, and epistemology",
    "Coptic, Syriac, Ethiopian, and Oriental Orthodox theology and formation",
    "governance, administration, Congress, state capacity, and legitimacy",
    "Peirce, pragmatism, philosophy and social studies of science, models, and evidence",
    "cross-track anthropology of knowledge, institutions, religion, science, and technology",
)


def atomic_text(path: Path, value: str) -> None:
    path.parent.mkdir(parents=True, exist_ok=True)
    fd, tmp_name = tempfile.mkstemp(prefix=".%s." % path.name, dir=str(path.parent))
    try:
        with os.fdopen(fd, "w", encoding="utf-8") as handle:
            handle.write(value)
            handle.flush()
            os.fsync(handle.fileno())
        os.replace(tmp_name, str(path))
    except Exception:
        try:
            os.unlink(tmp_name)
        except FileNotFoundError:
            pass
        raise


def load_json(path: Path, default):
    if not path.exists():
        return default
    return json.loads(path.read_text(encoding="utf-8"))


def clean(value: str, limit: int = 0) -> str:
    value = html.unescape(re.sub(r"<[^>]+>", " ", value or ""))
    value = re.sub(r"\s+", " ", value).strip()
    if limit and len(value) > limit:
        return value[: limit - 1].rstrip() + "…"
    return value


def norm(value: str) -> str:
    value = clean(value).lower().replace("&", " and ")
    return re.sub(r"[^a-z0-9]+", "", value)


def parse_date(value: str) -> Optional[datetime]:
    if not value:
        return None
    try:
        parsed = email.utils.parsedate_to_datetime(value)
        if parsed.tzinfo is None:
            parsed = parsed.replace(tzinfo=timezone.utc)
        return parsed.astimezone(timezone.utc)
    except (TypeError, ValueError):
        try:
            return datetime.fromisoformat(value.replace("Z", "+00:00")).astimezone(timezone.utc)
        except ValueError:
            return None


def rfc2822(value: Optional[datetime]) -> str:
    return email.utils.format_datetime(value or datetime.now(timezone.utc))


def child_text(node: Optional[ET.Element], path: str) -> str:
    if node is None:
        return ""
    found = node.find(path)
    return (found.text or "").strip() if found is not None else ""


def opml_exclusions(path: Path) -> Tuple[Set[str], Set[str]]:
    root = ET.parse(str(path)).getroot()
    feeds, shows = set(), set()
    for outline in root.findall(".//outline[@xmlUrl]"):
        feed = (outline.get("xmlUrl") or "").strip()
        show = (outline.get("text") or outline.get("title") or "").strip()
        if feed:
            feeds.add(feed.rstrip("/"))
        if show:
            shows.add(norm(show))
    return feeds, shows


def ledger_exclusions(path: Path) -> Tuple[Set[str], Set[str]]:
    if not path.exists():
        return set(), set()
    text = path.read_text(encoding="utf-8")
    ids = set(re.findall(r"\*\*Candidate ID:\*\* `([^`]+)`", text))
    titles = {norm(title) for title in re.findall(r"^## (.+)$", text, flags=re.MULTILINE)}
    return ids, titles


def state_exclusions(path: Path) -> Tuple[Set[str], Set[str], Set[str], List[dict]]:
    state = load_json(path, {"items": []})
    items = state.get("items", [])
    ids = {item.get("guid", "") for item in items if item.get("guid")}
    titles = {norm(item.get("title", "")) for item in items if item.get("title")}
    enclosures = {item.get("enclosure_url", "") for item in items if item.get("enclosure_url")}
    return ids, titles, enclosures, items


def podsync_exclusions(paths: Iterable[Path]) -> Tuple[Set[str], Set[str], Set[str]]:
    ids, titles, links = set(), set(), set()
    for path in paths:
        if not path.exists():
            continue
        try:
            root = ET.parse(str(path)).getroot()
        except ET.ParseError:
            continue
        for item in root.findall(".//item"):
            guid = child_text(item, "guid")
            title = child_text(item, "title")
            link = child_text(item, "link")
            enclosure = item.find("enclosure")
            if guid:
                ids.add(guid)
            if title:
                titles.add(norm(title))
            if link:
                links.add(link)
            if enclosure is not None and enclosure.get("url"):
                links.add(enclosure.get("url", ""))
    return ids, titles, links


def decode_payload(value: str) -> dict:
    padding = "=" * (-len(value) % 4)
    return json.loads(base64.urlsafe_b64decode(value + padding).decode("utf-8"))


def encode_payload(value: dict) -> str:
    raw = json.dumps(value, ensure_ascii=False, separators=(",", ":")).encode("utf-8")
    return base64.urlsafe_b64encode(raw).decode("ascii").rstrip("=")


def dashboard_candidates(path: Path) -> List[dict]:
    if not path.exists():
        return []
    text = path.read_text(encoding="utf-8")
    updated = re.search(r"^updated:\s*(.+)$", text, flags=re.MULTILINE)
    first_shown = (updated.group(1)[:10] if updated else datetime.now().date().isoformat())
    pattern = re.compile(
        r"^### (?P<title>.+?)\n\n"
        r"<!-- recommendation-candidate:(?P<payload>[A-Za-z0-9_-]+) -->\n"
        r"(?P<body>.*?)(?=^### |^## |\Z)",
        flags=re.MULTILINE | re.DOTALL,
    )
    results = []
    for match in pattern.finditer(text):
        data = decode_payload(match.group("payload"))
        body = match.group("body")
        new_heading = text.rfind("## New discoveries", 0, match.start())
        optional_heading = text.rfind("## Optional discoveries", 0, match.start())
        candidate_id = (
            "youtube:%s" % re.search(r"(?:v=|youtu\.be/)([A-Za-z0-9_-]+)", data.get("url", "")).group(1)
            if data.get("kind") == "youtube" and re.search(r"(?:v=|youtu\.be/)([A-Za-z0-9_-]+)", data.get("url", ""))
            else "rss:%s" % data.get("guid", "")
        )
        details = dict(re.findall(r"^  - \*\*(.+?):\*\*\s*(.*)$", body, flags=re.MULTILINE))
        reactions = [name for name in REACTION_NAMES if re.search(r"^- \[[xX]\] %s$" % re.escape(name), body, flags=re.MULTILINE)]
        results.append({
            "candidate_id": candidate_id,
            "title": match.group("title").strip(),
            "show": details.get("Show/channel", data.get("show", "")),
            "link": details.get("Direct link", data.get("link", data.get("url", ""))),
            "first_shown": first_shown,
            "added": bool(re.search(r"^- \[[xX]\] Add to discoveries feed", body, flags=re.MULTILINE)),
            "reactions": reactions,
            "section": "optional" if optional_heading > new_heading else "high",
            "raw": match.group(0).rstrip(),
        })
    return results


def source_registry(path: Path, state_items: Sequence[dict], subscribed_feeds: Set[str]) -> List[dict]:
    registry = load_json(path, {"sources": []}).get("sources", [])
    by_url = {source.get("url", "").rstrip("/"): source for source in registry if source.get("url")}
    for item in state_items:
        url = (item.get("source_feed") or "").strip()
        if not url:
            continue
        if "youtube.com/channel/" in url:
            channel_id = url.rstrip("/").split("/")[-1]
            url = "https://www.youtube.com/feeds/videos.xml?channel_id=%s" % channel_id
            kind = "youtube"
        elif "/feeds/videos.xml" in url:
            kind = "youtube"
        elif url.startswith("http"):
            kind = "rss"
        else:
            continue
        key = url.rstrip("/")
        if key in subscribed_feeds or key in by_url:
            continue
        by_url[key] = {
            "name": item.get("show", "Previously discovered source"),
            "kind": kind,
            "url": url,
            "priority": 1,
            "enabled": True,
        }
    return [source for source in by_url.values() if source.get("enabled", True)]


def score_candidate(title: str, description: str, source_priority: int) -> Tuple[int, Dict[str, int], str]:
    title_haystack = " %s " % clean(title).lower()
    desc_haystack = " %s " % clean(description).lower()
    scores = {}
    for track, terms in TRACK_TERMS.items():
        title_hits = sum(1 for term in terms if term in title_haystack)
        desc_hits = sum(1 for term in terms if term in desc_haystack)
        scores[track] = title_hits * 3 + desc_hits
    active = [track for track, value in scores.items() if value]
    total = sum(scores.values()) + max(0, len(active) - 1) * 2 + int(source_priority)
    combined = title_haystack + desc_haystack
    total -= sum(3 for term in NEGATIVE_TERMS if term in combined)
    primary = max(scores, key=scores.get) if active else ""
    return total, scores, primary


def fetch_source(source: dict, cutoff: datetime, now: datetime) -> Tuple[List[dict], Optional[str]]:
    url = source["url"]
    try:
        request = Request(url, headers={"User-Agent": "SorenPodcastDiscovery/2.0"})
        with urlopen(request, timeout=20) as response:
            root = ET.fromstring(response.read(8_000_000))
        if source.get("kind") == "youtube" or root.tag.endswith("feed"):
            return parse_youtube_feed(root, source, cutoff, now), None
        return parse_rss_feed(root, source, cutoff, now), None
    except Exception as exc:
        return [], "%s: %s: %s" % (source.get("name", url), type(exc).__name__, str(exc)[:140])


def normalize_duration(value: str) -> str:
    value = clean(value)
    if not re.fullmatch(r"\d+", value):
        return value
    seconds = int(value)
    hours, remainder = divmod(seconds, 3600)
    minutes, seconds = divmod(remainder, 60)
    if hours:
        return "%d:%02d:%02d" % (hours, minutes, seconds)
    return "%d:%02d" % (minutes, seconds)


def parse_rss_feed(root: ET.Element, source: dict, cutoff: datetime, now: datetime) -> List[dict]:
    channel = root.find("channel") or root.find(".//channel")
    show = child_text(channel, "title") or source.get("name", "")
    show_image_node = channel.find("{%s}image" % ITUNES) if channel is not None else None
    show_image = show_image_node.get("href", "") if show_image_node is not None else child_text(channel, "image/url")
    results = []
    for item in root.findall(".//item"):
        title = clean(child_text(item, "title"))
        description = clean(child_text(item, "description") or child_text(item, "{%s}summary" % ITUNES), 600)
        published = parse_date(child_text(item, "pubDate") or child_text(item, "{%s}date" % ATOM))
        if not published or published < cutoff or published > now + timedelta(days=1):
            continue
        enclosure = item.find("enclosure")
        if enclosure is None or not enclosure.get("url"):
            continue
        link = child_text(item, "link") or enclosure.get("url", "")
        guid = child_text(item, "guid") or link or enclosure.get("url", "")
        image_node = item.find("{%s}image" % ITUNES)
        score, track_scores, primary = score_candidate(title, description, source.get("priority", 1))
        results.append({
            "candidate_id": "rss:%s" % guid,
            "kind": "rss",
            "guid": guid,
            "title": title,
            "show": show,
            "link": link,
            "description": description,
            "pub_date": rfc2822(published),
            "published_iso": published.isoformat(),
            "enclosure_url": enclosure.get("url", ""),
            "enclosure_type": enclosure.get("type", "audio/mpeg"),
            "length": int(enclosure.get("length") or 0),
            "duration": normalize_duration(child_text(item, "{%s}duration" % ITUNES)),
            "image": image_node.get("href", "") if image_node is not None else show_image,
            "source_feed": source["url"],
            "source_priority": source.get("priority", 1),
            "keyword_score": score,
            "track_scores": track_scores,
            "primary_track": primary,
        })
    return results


def parse_youtube_feed(root: ET.Element, source: dict, cutoff: datetime, now: datetime) -> List[dict]:
    show = child_text(root, "{%s}title" % ATOM) or source.get("name", "")
    results = []
    for entry in root.findall("{%s}entry" % ATOM):
        video_id = child_text(entry, "{%s}videoId" % YT)
        title = clean(child_text(entry, "{%s}title" % ATOM))
        published = parse_date(child_text(entry, "{%s}published" % ATOM))
        if not video_id or not published or published < cutoff or published > now + timedelta(days=1):
            continue
        group = entry.find("{%s}group" % MEDIA)
        description = clean(child_text(group, "{%s}description" % MEDIA), 600)
        score, track_scores, primary = score_candidate(title, description, source.get("priority", 1))
        results.append({
            "candidate_id": "youtube:%s" % video_id,
            "kind": "youtube",
            "url": "https://www.youtube.com/watch?v=%s" % video_id,
            "title": title,
            "show": show,
            "link": "https://www.youtube.com/watch?v=%s" % video_id,
            "description": description,
            "pub_date": rfc2822(published),
            "published_iso": published.isoformat(),
            "duration": "",
            "image": "https://i.ytimg.com/vi/%s/maxresdefault.jpg" % video_id,
            "source_feed": source["url"],
            "source_priority": source.get("priority", 1),
            "keyword_score": score,
            "track_scores": track_scores,
            "primary_track": primary,
        })
    return results


def enrich_youtube(candidates: List[dict], limit: int = 5) -> None:
    yt_dlp = Path("/opt/homebrew/bin/yt-dlp")
    if not yt_dlp.exists():
        return
    targets = [item for item in candidates if item.get("kind") == "youtube"][:limit]

    def load(item: dict):
        result = subprocess.run(
            [str(yt_dlp), "--dump-single-json", "--skip-download", "--no-playlist", item["url"]],
            capture_output=True, text=True, timeout=35,
        )
        if result.returncode:
            return None
        return item, json.loads(result.stdout)

    with ThreadPoolExecutor(max_workers=min(3, len(targets) or 1)) as pool:
        futures = [pool.submit(load, item) for item in targets]
        for future in as_completed(futures):
            loaded = future.result()
            if not loaded:
                continue
            item, metadata = loaded
            item["title"] = metadata.get("title") or item["title"]
            item["show"] = metadata.get("channel") or metadata.get("uploader") or item["show"]
            item["description"] = clean(metadata.get("description") or item["description"], 600)
            item["duration"] = metadata.get("duration_string") or str(metadata.get("duration") or "")
            item["image"] = metadata.get("thumbnail") or item["image"]
            item["source_feed"] = metadata.get("channel_url") or item["source_feed"]


def candidate_source_key(item: dict) -> str:
    return norm(item.get("source_feed", "") or item.get("show", "") or "unknown-source")


def balanced_shortlist(candidates: Sequence[dict], limit: int, per_source_limit: int = 3) -> List[dict]:
    ranked = sorted(
        candidates,
        key=lambda item: (item.get("keyword_score", 0), item.get("published_iso", ""), item.get("source_priority", 0)),
        reverse=True,
    )
    buckets = {track: [] for track in TRACK_TERMS}
    other = []
    for item in ranked:
        track = item.get("primary_track")
        (buckets[track] if track in buckets else other).append(item)
    chosen, seen, source_counts = [], set(), {}

    def eligible(item: dict) -> bool:
        source = candidate_source_key(item)
        return item["candidate_id"] not in seen and source_counts.get(source, 0) < per_source_limit

    def choose(item: dict) -> None:
        source = candidate_source_key(item)
        chosen.append(item)
        seen.add(item["candidate_id"])
        source_counts[source] = source_counts.get(source, 0) + 1

    while len(chosen) < limit:
        added = False
        for track in TRACK_TERMS:
            while buckets[track] and not eligible(buckets[track][0]):
                buckets[track].pop(0)
            if buckets[track] and len(chosen) < limit:
                item = buckets[track].pop(0)
                choose(item)
                added = True
        if not added:
            break
    for item in other + ranked:
        if len(chosen) >= limit:
            break
        if eligible(item):
            choose(item)
    return chosen


def prepare(args) -> int:
    now = datetime.now(timezone.utc)
    cutoff = now - timedelta(days=args.max_age_days)
    subscribed_feeds, subscribed_shows = opml_exclusions(args.opml)
    ledger_ids, ledger_titles = ledger_exclusions(args.reactions)
    state_ids, state_titles, state_enclosures, state_items = state_exclusions(args.state)
    pod_ids, pod_titles, pod_links = podsync_exclusions(PODSYNC_FEEDS)
    prior_queue = dashboard_candidates(args.dashboard)
    prior_ids = {item["candidate_id"] for item in prior_queue}

    sources = source_registry(args.sources, state_items, subscribed_feeds)
    found, errors = [], []
    with ThreadPoolExecutor(max_workers=min(10, len(sources) or 1)) as pool:
        futures = {pool.submit(fetch_source, source, cutoff, now): source for source in sources}
        for future in as_completed(futures):
            items, error = future.result()
            found.extend(items)
            if error:
                errors.append(error)

    excluded_ids = ledger_ids | state_ids | pod_ids | prior_ids
    excluded_titles = ledger_titles | state_titles | pod_titles | {norm(item["title"]) for item in prior_queue}
    filtered = []
    for item in found:
        if item["candidate_id"] in excluded_ids:
            continue
        if norm(item.get("title", "")) in excluded_titles:
            continue
        if item.get("enclosure_url") in state_enclosures or item.get("enclosure_url") in pod_links:
            continue
        if item.get("source_feed", "").rstrip("/") in subscribed_feeds:
            continue
        if norm(item.get("show", "")) in subscribed_shows:
            continue
        if item.get("keyword_score", 0) < 2:
            continue
        filtered.append(item)

    shortlist = balanced_shortlist(filtered, args.max_candidates)
    enrich_youtube(shortlist)
    sync_status = load_json(SYNC_STATUS, {})
    brief = {
        "schema_version": 1,
        "generated_at": datetime.now().astimezone().isoformat(timespec="seconds"),
        "search_track": ROTATING_SEARCH_TRACKS[datetime.now().weekday() % len(ROTATING_SEARCH_TRACKS)],
        "limits": {
            "max_model_candidates": args.max_candidates,
            "max_candidates_per_source": 3,
            "web_search_queries_if_needed": 2,
            "web_results_to_open_if_needed": 3,
            "high_confidence_max": 3,
            "optional_max": 5,
        },
        "selection_schema": {
            "summary": "One sentence explaining today's selection quality.",
            "discoveries": [{
                "candidate_id": "Use an ID from candidates, or assign web:rss:<stable-guid> / web:youtube:<video-id>.",
                "confidence": "high or optional",
                "track": "Specific intellectual track",
                "why_it_fits": "One concrete sentence",
                "what_is_novel": "One concrete sentence",
                "guest_expertise": "Role and relevant expertise",
                "metadata": "Omit for brief candidates. For a web candidate include all authoritative RSS payload fields, or kind/youtube URL plus title, show, link, pub_date, duration, description, and source_feed.",
            }],
        },
        "exclusion_counts": {
            "subscriptions": len(subscribed_feeds),
            "ledger": len(ledger_ids),
            "discoveries_feed": len(state_ids),
            "podsync_items": len(pod_ids),
        },
        "current_queue": prior_queue,
        "known_sources_scanned": len(sources),
        "source_errors": errors[:8],
        "sync_status": sync_status,
        "candidates": shortlist,
    }
    atomic_text(args.output, json.dumps(brief, ensure_ascii=False, indent=2) + "\n")
    print(json.dumps({
        "brief": str(args.output),
        "bytes": args.output.stat().st_size,
        "candidates": len(shortlist),
        "known_sources": len(sources),
        "errors": len(errors),
        "search_track": brief["search_track"],
    }, ensure_ascii=False))
    return 0


def union_reactions(old: str, new: Sequence[str]) -> str:
    values = [] if old.strip().lower().startswith("none") else [part.strip() for part in old.split(";") if part.strip()]
    for value in new:
        if value not in values:
            values.append(value)
    return "; ".join(values) if values else "None yet"


def archive_dashboard(dashboard: Path, reactions: Path, timestamp: str, candidates: Optional[Sequence[dict]] = None) -> int:
    candidates = list(candidates) if candidates is not None else dashboard_candidates(dashboard)
    candidates = [candidate for candidate in candidates if candidate["reactions"]]
    if not candidates:
        return 0
    ledger = reactions.read_text(encoding="utf-8") if reactions.exists() else "# Podcast Discovery Reactions\n"
    parts = re.split(r"(?=^## )", ledger, flags=re.MULTILINE)
    archived = 0
    for candidate in candidates:
        existing_index = next((i for i, part in enumerate(parts) if candidate["candidate_id"] in part), None)
        if existing_index is not None:
            part = parts[existing_index]
            reaction_match = re.search(r"^- \*\*Reactions:\*\* (.*)$", part, flags=re.MULTILINE)
            merged = union_reactions(reaction_match.group(1), candidate["reactions"]) if reaction_match else union_reactions("", candidate["reactions"])
            if reaction_match:
                part = part[:reaction_match.start(1)] + merged + part[reaction_match.end(1):]
            elif candidate["reactions"]:
                part = part.rstrip() + "\n- **Reactions:** %s\n" % merged
            if candidate["added"]:
                part = re.sub(r"^- \*\*Added to discoveries:\*\* No$", "- **Added to discoveries:** Yes", part, flags=re.MULTILINE)
            parts[existing_index] = part
            continue
        block = (
            "\n## {title}\n\n"
            "- **Candidate ID:** `{candidate_id}`\n"
            "- **Show/channel:** {show}\n"
            "- **First shown:** {first_shown}\n"
            "- **Added to discoveries:** {added}\n"
            "- **Reactions:** {reactions}\n"
            "- **Direct link:** {link}\n"
        ).format(
            title=candidate["title"], candidate_id=candidate["candidate_id"],
            show=candidate["show"] or "Unknown", first_shown=candidate["first_shown"],
            added="Yes" if candidate["added"] else "No",
            reactions="; ".join(candidate["reactions"]) if candidate["reactions"] else "None yet",
            link=candidate["link"],
        )
        parts.append(block)
        archived += 1
    ledger = "".join(parts)
    if re.search(r"^updated:", ledger, flags=re.MULTILINE):
        ledger = re.sub(r"^updated:.*$", "updated: %s" % timestamp, ledger, count=1, flags=re.MULTILINE)
    atomic_text(reactions, ledger.rstrip() + "\n")
    return archived


def verify_url(url: str, expected_prefix: str) -> Tuple[str, int]:
    if not url:
        raise ValueError("missing URL")

    def acceptable(mime: str) -> bool:
        return not expected_prefix or mime.startswith(expected_prefix) or mime == "application/octet-stream"

    def metadata(response) -> Tuple[str, int]:
        mime = response.headers.get_content_type()
        length = int(response.headers.get("Content-Length") or 0)
        content_range = response.headers.get("Content-Range", "")
        total_match = re.search(r"/(\d+)$", content_range)
        if total_match:
            length = int(total_match.group(1))
        return mime, length

    last_mime = "unknown"
    last_error = None
    probes = (
        Request(url, method="HEAD", headers={"User-Agent": "SorenPodcastDiscovery/2.0"}),
        Request(url, headers={"User-Agent": "SorenPodcastDiscovery/2.0", "Range": "bytes=0-0"}),
    )
    for request in probes:
        try:
            response = urlopen(request, timeout=25)
            with response:
                mime, length = metadata(response)
            last_mime = mime
            if acceptable(mime):
                return mime, length
        except Exception as exc:
            last_error = exc

    if last_mime != "unknown":
        raise ValueError("unexpected MIME %s for %s" % (last_mime, url))
    raise ValueError("could not verify %s: %s" % (url, last_error))


def selection_items(brief: dict, selection: dict) -> List[dict]:
    lookup = {item["candidate_id"]: item for item in brief.get("candidates", [])}
    results = []
    for choice in selection.get("discoveries", []):
        candidate_id = choice.get("candidate_id", "")
        metadata = dict(lookup.get(candidate_id, {}))
        metadata.update(choice.get("metadata") or {})
        if not metadata:
            raise ValueError("selection has no metadata for %s" % candidate_id)
        metadata["candidate_id"] = candidate_id or metadata.get("candidate_id")
        metadata["confidence"] = choice.get("confidence", "optional")
        common_required = ("title", "show", "pub_date", "duration")
        missing_common = [field for field in common_required if not metadata.get(field)]
        if missing_common:
            raise ValueError("%s missing display fields: %s" % (candidate_id, ", ".join(missing_common)))
        for field in ("track", "why_it_fits", "what_is_novel", "guest_expertise"):
            metadata[field] = clean(choice.get(field, ""))
            if not metadata[field]:
                raise ValueError("%s is missing %s" % (metadata.get("title", candidate_id), field))
        if metadata.get("kind") == "rss":
            required = ("guid", "title", "show", "link", "pub_date", "enclosure_url", "duration", "image", "source_feed")
            missing = [field for field in required if not metadata.get(field)]
            if missing:
                raise ValueError("%s missing RSS fields: %s" % (metadata.get("title"), ", ".join(missing)))
            mime, length = verify_url(metadata["enclosure_url"], "audio/")
            metadata["enclosure_type"] = mime
            if not metadata.get("length"):
                metadata["length"] = length
            verify_url(metadata["image"], "image/")
        elif metadata.get("kind") == "youtube":
            if not metadata.get("url"):
                metadata["url"] = metadata.get("link", "")
            if not metadata.get("url"):
                raise ValueError("YouTube selection missing URL")
            if not metadata.get("link"):
                metadata["link"] = metadata["url"]
        else:
            raise ValueError("unsupported candidate kind: %s" % metadata.get("kind"))
        results.append(metadata)
    return results


def candidate_payload(item: dict) -> dict:
    if item["kind"] == "youtube":
        return {"kind": "youtube", "url": item["url"]}
    return {
        "kind": "rss", "guid": item["guid"], "title": item["title"], "show": item["show"],
        "link": item["link"], "description": clean(item.get("description", ""), 500),
        "pub_date": item["pub_date"], "enclosure_url": item["enclosure_url"],
        "enclosure_type": item.get("enclosure_type", "audio/mpeg"),
        "length": int(item.get("length") or 0), "duration": str(item.get("duration", "")),
        "image": item.get("image", ""), "source_feed": item.get("source_feed", ""),
    }


def display_date(value: str) -> str:
    parsed = parse_date(value)
    return parsed.strftime("%B %-d, %Y") if parsed else value


def render_candidate(item: dict) -> str:
    checked = "x" if item["confidence"] == "high" else " "
    payload = encode_payload(candidate_payload(item))
    return "\n".join([
        "### %s" % item["title"],
        "",
        "<!-- recommendation-candidate:%s -->" % payload,
        "- [%s] Add to discoveries feed%s" % (checked, " (high-confidence automatic addition)" if checked == "x" else " (optional)"),
        "- [ ] Good",
        "- [ ] Not for me",
        "- [ ] More like this",
        "- [ ] Too basic",
        "- [ ] Too chatty",
        "- [ ] Too long",
        "- [ ] Already knew this",
        "  - **Show/channel:** %s" % item.get("show", ""),
        "  - **Track:** %s" % item["track"],
        "  - **Publication date:** %s" % display_date(item.get("pub_date", "")),
        "  - **Duration:** %s" % item.get("duration", ""),
        "  - **Why it fits:** %s" % item["why_it_fits"],
        "  - **What is novel:** %s" % item["what_is_novel"],
        "  - **Guest expertise:** %s" % item["guest_expertise"],
        "  - **Direct link:** %s" % (item.get("link") or item.get("url", "")),
    ])


def render_dashboard(items: Sequence[dict], summary: str, timestamp: str, retained: Sequence[dict] = ()) -> str:
    high = sorted([item for item in items if item["confidence"] == "high"], key=lambda item: item.get("published_iso", ""), reverse=True)
    optional = sorted([item for item in items if item["confidence"] != "high"], key=lambda item: item.get("published_iso", ""), reverse=True)
    retained_high = [candidate["raw"] for candidate in retained if candidate["section"] == "high"]
    retained_optional = [candidate["raw"] for candidate in retained if candidate["section"] == "optional"]
    high_blocks = [render_candidate(item) for item in high] + retained_high
    optional_blocks = [render_candidate(item) for item in optional] + retained_optional
    high_text = "\n\n".join(high_blocks) if high_blocks else "No high-confidence candidate cleared today’s novelty and value-per-minute threshold."
    optional_text = "\n\n".join(optional_blocks) if optional_blocks else "No lower-confidence candidate cleared today’s novelty and value-per-minute threshold."
    return """---
type: podcast-discovery
updated: {timestamp}
tags: [dashboard, podcasts, discovery]
---

# Podcast Discovery

This dashboard is a working queue containing discoveries from outside your Pocket Casts subscriptions. A candidate remains here until you score it with an evaluative reaction; scored candidates move to [[Library/Podcast Discovery Reactions|Podcast Discovery Reactions]].

- **Discoveries feed:** https://mini.taila1ac2d.ts.net/discoveries.xml
- **Favorite People feed:** https://mini.taila1ac2d.ts.net/recommendations.xml

## New discoveries

{summary}

{high_text}

## Optional discoveries

{optional_text}

## Feedback

Use the checkboxes on each candidate; no written feedback is required. Add status alone is not a score. Once you check Good, Not for me, More like this, Too basic, Too chatty, Too long, or Already knew this, the candidate moves to the reaction ledger on the next run.

## How selection works

Subscriptions, prior discoveries, and the reaction ledger are filtered programmatically before model review. The model judges only a compact external shortlist and may conduct one bounded rotating-track search when known sources are thin. David Bentley Hart and Fr. Anthony Messeh are handled separately in **Favorite People**.
""".format(timestamp=timestamp, summary=summary, high_text=high_text, optional_text=optional_text)


def update_registry(path: Path, items: Sequence[dict], subscribed_feeds: Set[str]) -> int:
    data = load_json(path, {"version": 1, "sources": []})
    sources = data.setdefault("sources", [])
    urls = {source.get("url", "").rstrip("/") for source in sources}
    added = 0
    for item in items:
        url = (item.get("source_feed") or "").strip()
        kind = item.get("kind", "rss")
        if kind == "youtube" and "youtube.com/channel/" in url:
            url = "https://www.youtube.com/feeds/videos.xml?channel_id=%s" % url.rstrip("/").split("/")[-1]
        if not url or not url.startswith("http") or url.rstrip("/") in urls or url.rstrip("/") in subscribed_feeds:
            continue
        sources.append({
            "name": item.get("show", "Selected source"), "kind": kind,
            "url": url, "priority": 1, "enabled": True,
        })
        urls.add(url.rstrip("/"))
        added += 1
    if added:
        data["updated"] = datetime.now().astimezone().isoformat(timespec="seconds")
        atomic_text(path, json.dumps(data, ensure_ascii=False, indent=2) + "\n")
    return added


def render(args) -> int:
    brief = load_json(args.brief, {})
    selection = load_json(args.selection, {})
    items = selection_items(brief, selection)
    if len([item for item in items if item["confidence"] == "high"]) > 3:
        raise ValueError("selection exceeds three high-confidence discoveries")
    if len([item for item in items if item["confidence"] != "high"]) > 5:
        raise ValueError("selection exceeds five optional discoveries")
    prior_candidates = dashboard_candidates(args.dashboard)
    harvested_output = ""
    if args.publish and args.dashboard.exists():
        harvested = subprocess.run(
            [str(DISCOVERIES_SCRIPT), "sync-dashboard", "--read-only-dashboard", str(args.dashboard)],
            capture_output=True, text=True,
        )
        harvested_output = clean(harvested.stdout or harvested.stderr, 500)
        if harvested.returncode:
            raise RuntimeError("existing dashboard harvest failed: %s" % harvested_output)
    timestamp = datetime.now().astimezone().isoformat(timespec="seconds")
    archived = archive_dashboard(args.dashboard, args.reactions, timestamp, prior_candidates)
    new_ids = {item["candidate_id"] for item in items}
    retained = [candidate for candidate in prior_candidates if not candidate["reactions"] and candidate["candidate_id"] not in new_ids]
    summary = clean(selection.get("summary", "")) or "No candidate cleared today’s novelty and value-per-minute threshold."
    atomic_text(args.dashboard, render_dashboard(items, summary, timestamp, retained))
    subscribed_feeds, _ = opml_exclusions(args.opml)
    new_sources = update_registry(args.sources, items, subscribed_feeds)
    publish_output = ""
    if args.publish:
        result = subprocess.run(
            [str(DISCOVERIES_SCRIPT), "sync-dashboard", str(args.dashboard)],
            capture_output=True, text=True,
        )
        publish_output = clean(result.stdout or result.stderr, 500)
        if result.returncode:
            raise RuntimeError("feed publication failed: %s" % publish_output)
    print(json.dumps({
        "dashboard": str(args.dashboard), "selected": len(items), "archived": archived,
        "retained": len(retained),
        "new_sources": new_sources, "published": bool(args.publish), "publish_output": publish_output,
        "harvested_output": harvested_output,
    }, ensure_ascii=False))
    return 0


def build_parser() -> argparse.ArgumentParser:
    parser = argparse.ArgumentParser(description=__doc__)
    sub = parser.add_subparsers(dest="command", required=True)
    prep = sub.add_parser("prepare", help="Build a compact, pre-filtered candidate brief")
    prep.add_argument("--output", type=Path, default=DEFAULT_BRIEF)
    prep.add_argument("--opml", type=Path, default=DEFAULT_OPML)
    prep.add_argument("--dashboard", type=Path, default=DEFAULT_DASHBOARD)
    prep.add_argument("--reactions", type=Path, default=DEFAULT_REACTIONS)
    prep.add_argument("--state", type=Path, default=DEFAULT_STATE)
    prep.add_argument("--sources", type=Path, default=DEFAULT_SOURCES)
    prep.add_argument("--max-candidates", type=int, default=24)
    prep.add_argument("--max-age-days", type=int, default=240)
    prep.set_defaults(func=prepare)

    renderer = sub.add_parser("render", help="Archive the old queue and render a model selection")
    renderer.add_argument("selection", type=Path)
    renderer.add_argument("--brief", type=Path, default=DEFAULT_BRIEF)
    renderer.add_argument("--opml", type=Path, default=DEFAULT_OPML)
    renderer.add_argument("--dashboard", type=Path, default=DEFAULT_DASHBOARD)
    renderer.add_argument("--reactions", type=Path, default=DEFAULT_REACTIONS)
    renderer.add_argument("--sources", type=Path, default=DEFAULT_SOURCES)
    renderer.add_argument("--publish", action="store_true")
    renderer.set_defaults(func=render)
    return parser


def main() -> int:
    args = build_parser().parse_args()
    return args.func(args)


if __name__ == "__main__":
    raise SystemExit(main())