Generate sitemap from feed with clean episode URLs

generate_episode_pages.py

@@ -7,6 +7,7 @@ import sys
 import tempfile
 import urllib.request
 from pathlib import Path
+from xml.sax.saxutils import escape
 
 from website_gen.feed import parse_feed
 from website_gen.render import FEED_URL, render_episode_page
@@ -17,8 +18,59 @@ DEFAULT_OUTPUT = REPO_ROOT / "website"
 DEFAULT_TRANSCRIPTS = DEFAULT_OUTPUT / "transcripts"
 USER_AGENT = "lukeattheroost-site-generator/1.0"
 
+SITE_URL = "https://lukeattheroost.com"
 
-def generate(feed_path, transcripts_dir, output_root, dry_run=False) -> int:
+# (path, lastmod, changefreq, priority) — /404 is deliberately excluded.
+STATIC_PAGES = (
+    ("", "2026-03-15", "weekly", "1.0"),
+    ("/llms.txt", "2026-03-15", "weekly", "0.5"),
+    ("/how-it-works", "2026-03-10", "monthly", "0.8"),
+    ("/clips", "2026-03-10", "weekly", "0.8"),
+    ("/terms", "2026-02-25", "yearly", "0.3"),
+    ("/stats", "2026-02-15", "daily", "0.6"),
+    ("/privacy", "2026-02-15", "yearly", "0.3"),
+)
+
+
+def build_sitemap(episodes, static_pages=None) -> str:
+    """Render the full sitemap: static pages first, then episodes newest first."""
+    if static_pages is None:
+        static_pages = STATIC_PAGES
+
+    entries = [(f"{SITE_URL}{path}", lastmod, freq, pri) for path, lastmod, freq, pri in static_pages]
+
+    ordered = sorted(
+        episodes,
+        key=lambda ep: (ep.published_iso, ep.number if ep.number is not None else -1),
+        reverse=True,
+    )
+    for episode in ordered:
+        entries.append(
+            (
+                f"{SITE_URL}/episode/{episode.slug}/",
+                episode.published_iso[:10],
+                "monthly",
+                "0.7",
+            )
+        )
+
+    lines = [
+        '<?xml version="1.0" encoding="UTF-8"?>',
+        '<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">',
+    ]
+    for loc, lastmod, freq, pri in entries:
+        lines.append("  <url>")
+        lines.append(f"    <loc>{escape(loc)}</loc>")
+        if lastmod:
+            lines.append(f"    <lastmod>{lastmod}</lastmod>")
+        lines.append(f"    <changefreq>{freq}</changefreq>")
+        lines.append(f"    <priority>{pri}</priority>")
+        lines.append("  </url>")
+    lines.append("</urlset>")
+    return "\n".join(lines) + "\n"
+
+
+def generate(feed_path, transcripts_dir, output_root, dry_run=False, sitemap=False) -> int:
     """Write <output_root>/episode/<slug>/index.html for each feed episode.
 
     Driven entirely by the feed: a transcript with no feed item is ignored, and a
@@ -52,6 +104,13 @@ def generate(feed_path, transcripts_dir, output_root, dry_run=False) -> int:
         written += 1
         print(f"{'would write' if dry_run else 'wrote'} {page} ({len(turns)} turns)")
 
+    if sitemap:
+        sitemap_file = output_root / "sitemap.xml"
+        if not dry_run:
+            sitemap_file.parent.mkdir(parents=True, exist_ok=True)
+            sitemap_file.write_text(build_sitemap(episodes), encoding="utf-8")
+        print(f"{'would write' if dry_run else 'wrote'} {sitemap_file}")
+
     return written
 
 
@@ -68,6 +127,7 @@ def main() -> None:
     parser.add_argument("--output", default=DEFAULT_OUTPUT, help="output directory")
     parser.add_argument("--transcripts", default=DEFAULT_TRANSCRIPTS, help="transcript directory")
     parser.add_argument("--dry-run", action="store_true", help="report without writing")
+    parser.add_argument("--sitemap", action="store_true", help="also write <output>/sitemap.xml")
     args = parser.parse_args()
 
     feed_path = args.feed
@@ -83,7 +143,13 @@ def main() -> None:
         feed_path = tmp_feed
 
     try:
-        count = generate(feed_path, args.transcripts, args.output, dry_run=args.dry_run)
+        count = generate(
+            feed_path,
+            args.transcripts,
+            args.output,
+            dry_run=args.dry_run,
+            sitemap=args.sitemap,
+        )
     finally:
         if tmp_feed is not None:
             shutil.rmtree(tmp_feed.parent, ignore_errors=True)

tests/test_sitemap.py

@@ -0,0 +1,69 @@
+import re
+import xml.etree.ElementTree as ET
+from pathlib import Path
+
+import pytest
+
+from generate_episode_pages import build_sitemap
+from website_gen.feed import parse_feed
+
+FIXTURE_FEED = Path(__file__).parent / "fixtures" / "feed_sample.xml"
+
+
+@pytest.fixture
+def episodes():
+    return parse_feed(FIXTURE_FEED.read_text(encoding="utf-8"))
+
+
+@pytest.fixture
+def generated_sitemap(episodes):
+    return build_sitemap(episodes)
+
+
+def test_sitemap_contains_no_query_param_urls(generated_sitemap):
+    assert "episode.html?slug=" not in generated_sitemap
+
+
+def test_sitemap_has_one_entry_per_episode(generated_sitemap, episodes):
+    assert generated_sitemap.count("<loc>https://lukeattheroost.com/episode/") == len(episodes)
+
+
+def test_static_pages_survive_regeneration(generated_sitemap):
+    for path in ["", "/how-it-works", "/clips", "/stats", "/privacy", "/terms", "/llms.txt"]:
+        assert f"<loc>https://lukeattheroost.com{path}</loc>" in generated_sitemap
+
+
+def test_lastmod_is_date_only_not_a_timestamp(generated_sitemap):
+    for m in re.findall(r"<lastmod>([^<]*)</lastmod>", generated_sitemap):
+        assert re.fullmatch(r"\d{4}-\d{2}-\d{2}", m), f"bad lastmod: {m}"
+
+
+def test_sitemap_is_valid_xml(generated_sitemap):
+    root = ET.fromstring(generated_sitemap)
+    assert root.tag.endswith("urlset")
+
+
+def test_episode_urls_have_trailing_slash(generated_sitemap):
+    for loc in re.findall(r"<loc>(https://lukeattheroost\.com/episode/[^<]*)</loc>", generated_sitemap):
+        assert loc.endswith("/"), loc
+
+
+def test_404_page_is_not_listed(generated_sitemap):
+    assert "/404" not in generated_sitemap
+
+
+def test_static_pages_come_before_episodes(generated_sitemap):
+    first_episode = generated_sitemap.index("<loc>https://lukeattheroost.com/episode/")
+    last_static = generated_sitemap.index("<loc>https://lukeattheroost.com/privacy</loc>")
+    assert last_static < first_episode
+
+
+def test_episodes_are_newest_first(generated_sitemap):
+    locs = re.findall(r"<loc>https://lukeattheroost\.com/episode/([^<]*)/</loc>", generated_sitemap)
+    assert locs[0].startswith("episode-58-")
+    assert locs[1].startswith("episode-57-")
+
+
+def test_declares_xml_and_sitemap_namespace(generated_sitemap):
+    assert generated_sitemap.startswith('<?xml version="1.0" encoding="UTF-8"?>')
+    assert 'xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"' in generated_sitemap