refactor(scripts): migrate newsletter engine from Node to Go

One Go binary with a subcommand per former script, invoked as
'go run ./scripts/newsletter <command>' from repo root. Stdlib only;
publications config embedded via go:embed (go run recompiles on edit).
Behavior parity verified side-by-side before deleting the JS —
see plans/reports/parity-260818-newsletter-go-migration-report.md.

Permissions: allow 'Bash(go *)', drop 'Bash(node *)' and the redundant
'Write(content)' rule (Edit rules cover all file-editing tools).

scripts/newsletter/add-url.js

@@ -1,112 +0,0 @@
-#!/usr/bin/env node
-// Meta URL router for the mt-add-url skill — the single entry per URL.
-// Usage: node add-url.js "<url>"
-// Outputs: JSON { original_url, clean_url, http_status, accessible,
-//                 duplicate, route, title?, author? }
-// route ∈ youtube | image | video | document | article
-
-const path = require("path");
-const {
-  cleanUrl,
-  isSubstackImage,
-  fetchWithTimeout,
-  checkAccessibility,
-  checkDuplicate,
-  classifyType,
-} = require("./url-utils");
-
-const url = process.argv[2];
-if (!url) {
-  console.error("Usage: node add-url.js <url>");
-  process.exit(1);
-}
-
-const PROJECT_ROOT = path.resolve(__dirname, "../..");
-const CONTENT_DIR = path.join(PROJECT_ROOT, "content", "post");
-
-const YT_HOSTS = new Set(["youtube.com", "www.youtube.com", "m.youtube.com"]);
-
-// Detect a YouTube video and extract its id from the supported URL shapes:
-//   youtube.com/watch?v=ID, youtu.be/ID, youtube.com/shorts/ID
-// Playlists/channels are intentionally NOT YouTube routes (fall through to type).
-function detectYouTube(targetUrl) {
-  try {
-    const p = new URL(targetUrl);
-    const host = p.host.toLowerCase();
-    if (host === "youtu.be") {
-      const id = p.pathname.slice(1).split("/")[0];
-      return id ? { isYouTube: true, videoId: id } : { isYouTube: false };
-    }
-    if (YT_HOSTS.has(host)) {
-      if (p.pathname === "/watch") {
-        const id = p.searchParams.get("v");
-        return id ? { isYouTube: true, videoId: id } : { isYouTube: false };
-      }
-      if (p.pathname.startsWith("/shorts/")) {
-        const id = p.pathname.split("/")[2];
-        return id ? { isYouTube: true, videoId: id } : { isYouTube: false };
-      }
-    }
-    return { isYouTube: false };
-  } catch {
-    return { isYouTube: false };
-  }
-}
-
-// Canonical watch URL — oEmbed accepts watch URLs reliably for all shapes.
-function canonicalWatchUrl(videoId) {
-  return `https://www.youtube.com/watch?v=${videoId}`;
-}
-
-// Fetch title/author via YouTube oEmbed (no API key). Best-effort: any failure
-// returns {} so the route stays `youtube` and the skill can fall back.
-async function fetchYouTubeMeta(watchUrl) {
-  const endpoint = `https://www.youtube.com/oembed?url=${encodeURIComponent(watchUrl)}&format=json`;
-  const res = await fetchWithTimeout(endpoint);
-  if (!res || !res.ok) return {};
-  try {
-    const data = await res.json();
-    return { title: data.title, author: data.author_name };
-  } catch {
-    return {};
-  }
-}
-
-async function main() {
-  const cleanedUrl = cleanUrl(url);
-  const yt = detectYouTube(cleanedUrl);
-
-  // For YouTube, dedup/store against the canonical watch URL so youtu.be and
-  // shorts links collapse onto the same identity-param key as watch URLs.
-  const effectiveUrl = yt.isYouTube ? canonicalWatchUrl(yt.videoId) : cleanedUrl;
-  // Route order: YouTube → Substack image (by host, not extension, so f_auto /
-  // .avif / .heic / extensionless CDN URLs still route to the image handler) →
-  // file-extension classification.
-  let route;
-  if (yt.isYouTube) route = "youtube";
-  else if (isSubstackImage(cleanedUrl)) route = "image";
-  else route = classifyType(cleanedUrl);
-
-  const httpStatus = await checkAccessibility(cleanedUrl);
-  const accessible = httpStatus === "200";
-  const duplicate = checkDuplicate(effectiveUrl, CONTENT_DIR);
-
-  const out = {
-    original_url: url,
-    clean_url: effectiveUrl,
-    http_status: httpStatus,
-    accessible,
-    duplicate,
-    route,
-  };
-
-  if (route === "youtube") {
-    const meta = await fetchYouTubeMeta(effectiveUrl);
-    if (meta.title) out.title = meta.title;
-    if (meta.author) out.author = meta.author;
-  }
-
-  console.log(JSON.stringify(out, null, 2));
-}
-
-main();

scripts/newsletter/add_url.go

@@ -0,0 +1,138 @@
+// Meta URL router for the mt-add-url skill — the single entry per URL.
+// Usage: go run ./scripts/newsletter add-url "<url>"
+// Outputs: JSON { original_url, clean_url, http_status, accessible,
+//
+//	duplicate, route, title?, author? }
+//
+// route ∈ youtube | image | video | document | article
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"net/url"
+	"os"
+	"path/filepath"
+	"strings"
+	"time"
+)
+
+// contentDir is repo-root relative (invocation contract: run from repo root).
+func contentDir() string {
+	return filepath.Join("content", "post")
+}
+
+var ytHosts = map[string]bool{
+	"youtube.com":     true,
+	"www.youtube.com": true,
+	"m.youtube.com":   true,
+}
+
+// detectYouTube extracts a video id from the supported URL shapes:
+// youtube.com/watch?v=ID, youtu.be/ID, youtube.com/shorts/ID.
+// Playlists/channels are intentionally NOT YouTube routes (fall through to type).
+func detectYouTube(target string) (bool, string) {
+	u, err := url.Parse(target)
+	if err != nil {
+		return false, ""
+	}
+	host := strings.ToLower(u.Host)
+	if host == "youtu.be" {
+		id := strings.SplitN(strings.TrimPrefix(u.Path, "/"), "/", 2)[0]
+		return id != "", id
+	}
+	if ytHosts[host] {
+		if u.Path == "/watch" {
+			id := u.Query().Get("v")
+			return id != "", id
+		}
+		if strings.HasPrefix(u.Path, "/shorts/") {
+			parts := strings.Split(u.Path, "/")
+			if len(parts) > 2 && parts[2] != "" {
+				return true, parts[2]
+			}
+		}
+	}
+	return false, ""
+}
+
+// canonicalWatchURL — oEmbed accepts watch URLs reliably for all shapes.
+func canonicalWatchURL(videoID string) string {
+	return "https://www.youtube.com/watch?v=" + videoID
+}
+
+// fetchYouTubeMeta fetches title/author via YouTube oEmbed (no API key).
+// Best-effort: any failure returns empty strings so the route stays `youtube`
+// and the skill can fall back.
+func fetchYouTubeMeta(watchURL string) (title, author string) {
+	endpoint := "https://www.youtube.com/oembed?url=" + url.QueryEscape(watchURL) + "&format=json"
+	body := fetchTextOK(endpoint, 10*time.Second, "")
+	if body == "" {
+		return "", ""
+	}
+	var data struct {
+		Title  string `json:"title"`
+		Author string `json:"author_name"`
+	}
+	if json.Unmarshal([]byte(body), &data) != nil {
+		return "", ""
+	}
+	return data.Title, data.Author
+}
+
+type addURLOutput struct {
+	OriginalURL string `json:"original_url"`
+	CleanURL    string `json:"clean_url"`
+	HTTPStatus  string `json:"http_status"`
+	Accessible  bool   `json:"accessible"`
+	Duplicate   bool   `json:"duplicate"`
+	Route       string `json:"route"`
+	Title       string `json:"title,omitempty"`
+	Author      string `json:"author,omitempty"`
+}
+
+func runAddURL(args []string) {
+	if len(args) < 1 || args[0] == "" {
+		fmt.Fprintln(os.Stderr, "Usage: go run ./scripts/newsletter add-url <url>")
+		os.Exit(1)
+	}
+	target := args[0]
+
+	cleaned := cleanURL(target)
+	isYT, videoID := detectYouTube(cleaned)
+
+	// For YouTube, dedup/store against the canonical watch URL so youtu.be and
+	// shorts links collapse onto the same identity-param key as watch URLs.
+	effectiveURL := cleaned
+	if isYT {
+		effectiveURL = canonicalWatchURL(videoID)
+	}
+
+	// Route order: YouTube → Substack image (by host, not extension, so f_auto /
+	// .avif / .heic / extensionless CDN URLs still route to the image handler) →
+	// file-extension classification.
+	var route string
+	switch {
+	case isYT:
+		route = "youtube"
+	case isSubstackImage(cleaned):
+		route = "image"
+	default:
+		route = classifyType(cleaned)
+	}
+
+	httpStatus := checkAccessibility(cleaned)
+
+	out := addURLOutput{
+		OriginalURL: target,
+		CleanURL:    effectiveURL,
+		HTTPStatus:  httpStatus,
+		Accessible:  httpStatus == "200",
+		Duplicate:   checkDuplicate(effectiveURL, contentDir()),
+		Route:       route,
+	}
+	if route == "youtube" {
+		out.Title, out.Author = fetchYouTubeMeta(effectiveURL)
+	}
+	printJSON(out)
+}

scripts/newsletter/detect-image-source.js

@@ -1,43 +0,0 @@
-#!/usr/bin/env node
-// Detect whether an image URL is Substack-hosted and extract its S3 image UUID.
-// Usage: node detect-image-source.js "<image-url>"
-// Output: JSON { original_url, clean_url, isSubstack, uuid?, innerUrl? }
-//
-// Substack images are usually served via a CDN wrapper:
-//   https://substackcdn.com/image/fetch/$s_!x!,.../https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F<uuid>_WxH.png
-// The publication is NOT encoded in the URL — only the image identity (uuid) is.
-
-const {
-  cleanUrl,
-  isSubstackImage,
-  substackImageUuid,
-} = require("./url-utils.js");
-
-const url = process.argv[2];
-if (!url) {
-  console.error("Usage: node detect-image-source.js <image-url>");
-  process.exit(1);
-}
-
-// Pull the inner S3 URL out of a substackcdn /image/fetch/ wrapper (if present).
-function extractInnerUrl(targetUrl) {
-  const marker = targetUrl.indexOf("/https%3A%2F%2F");
-  if (marker !== -1) return decodeURIComponent(targetUrl.slice(marker + 1));
-  // Some forms embed a plain (already-decoded) inner https URL.
-  const plain = targetUrl.indexOf("/https://", 8);
-  if (plain !== -1) return targetUrl.slice(plain + 1);
-  return targetUrl;
-}
-
-const cleaned = cleanUrl(url);
-const isSubstack = isSubstackImage(url);
-const uuid = isSubstack ? substackImageUuid(url) : null;
-const innerUrl = isSubstack ? extractInnerUrl(url) : null;
-
-console.log(JSON.stringify({
-  original_url: url,
-  clean_url: cleaned,
-  isSubstack,
-  ...(uuid ? { uuid } : {}),
-  ...(innerUrl ? { innerUrl } : {}),
-}, null, 2));

scripts/newsletter/detect_image_source.go

@@ -0,0 +1,63 @@
+// Detect whether an image URL is Substack-hosted and extract its S3 image UUID.
+// Usage: go run ./scripts/newsletter detect-image-source "<image-url>"
+// Output: JSON { original_url, clean_url, isSubstack, uuid?, innerUrl? }
+//
+// Substack images are usually served via a CDN wrapper:
+//
+//	https://substackcdn.com/image/fetch/$s_!x!,.../https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F<uuid>_WxH.png
+//
+// The publication is NOT encoded in the URL — only the image identity (uuid) is.
+package main
+
+import (
+	"fmt"
+	"net/url"
+	"os"
+	"strings"
+)
+
+// extractInnerURL pulls the inner S3 URL out of a substackcdn /image/fetch/
+// wrapper (if present).
+func extractInnerURL(target string) string {
+	if marker := strings.Index(target, "/https%3A%2F%2F"); marker != -1 {
+		decoded, err := url.PathUnescape(target[marker+1:])
+		if err != nil {
+			return target[marker+1:]
+		}
+		return decoded
+	}
+	// Some forms embed a plain (already-decoded) inner https URL.
+	if len(target) > 8 {
+		if plain := strings.Index(target[8:], "/https://"); plain != -1 {
+			return target[8+plain+1:]
+		}
+	}
+	return target
+}
+
+type detectImageOutput struct {
+	OriginalURL string `json:"original_url"`
+	CleanURL    string `json:"clean_url"`
+	IsSubstack  bool   `json:"isSubstack"`
+	UUID        string `json:"uuid,omitempty"`
+	InnerURL    string `json:"innerUrl,omitempty"`
+}
+
+func runDetectImageSource(args []string) {
+	if len(args) < 1 || args[0] == "" {
+		fmt.Fprintln(os.Stderr, "Usage: go run ./scripts/newsletter detect-image-source <image-url>")
+		os.Exit(1)
+	}
+	target := args[0]
+
+	out := detectImageOutput{
+		OriginalURL: target,
+		CleanURL:    cleanURL(target),
+		IsSubstack:  isSubstackImage(target),
+	}
+	if out.IsSubstack {
+		out.UUID = substackImageUUID(target)
+		out.InnerURL = extractInnerURL(target)
+	}
+	printJSON(out)
+}

scripts/newsletter/fetch-via-defuddle.js

@@ -1,37 +0,0 @@
-#!/usr/bin/env node
-// mt-webfetch fallback fetcher via defuddle.md
-// Usage: node fetch.js <target_url>
-// Exit codes: 0 = content returned, 1 = empty/failed, 2 = bad args
-
-const url = process.argv[2];
-if (!url) {
-  console.error("Usage: node fetch.js <target_url>");
-  process.exit(2);
-}
-
-const defuddleUrl = `https://defuddle.md/${url}`;
-
-async function main() {
-  const controller = new AbortController();
-  const timeout = setTimeout(() => controller.abort(), 30000);
-  try {
-    const res = await fetch(defuddleUrl, {
-      redirect: "follow",
-      signal: controller.signal,
-      headers: { "User-Agent": "mt-webfetch/1.0" },
-    });
-    clearTimeout(timeout);
-    const body = await res.text();
-    if (!res.ok || !body || body.trim().length === 0) {
-      console.error(`mt-webfetch: defuddle returned ${res.status} / empty body for ${url}`);
-      process.exit(1);
-    }
-    process.stdout.write(body);
-  } catch (err) {
-    clearTimeout(timeout);
-    console.error(`mt-webfetch: fetch failed for ${url}: ${err.message}`);
-    process.exit(1);
-  }
-}
-
-main();

scripts/newsletter/fetch_via_defuddle.go

@@ -0,0 +1,51 @@
+// mt-webfetch fallback fetcher via defuddle.md.
+// Usage: go run ./scripts/newsletter fetch-via-defuddle <target_url>
+// Exit codes: 0 = content returned, 1 = empty/failed, 2 = bad args.
+// Note: `go run` collapses any nonzero program exit to 1, so callers invoking
+// through `go run` can only distinguish success from failure — the stderr
+// diagnostics carry the detail.
+package main
+
+import (
+	"fmt"
+	"io"
+	"net/http"
+	"os"
+	"strings"
+	"time"
+)
+
+func runFetchViaDefuddle(args []string) {
+	if len(args) < 1 || args[0] == "" {
+		fmt.Fprintln(os.Stderr, "Usage: go run ./scripts/newsletter fetch-via-defuddle <target_url>")
+		os.Exit(2)
+	}
+	target := args[0]
+	defuddleURL := "https://defuddle.md/" + target
+
+	fail := func(err error) {
+		fmt.Fprintf(os.Stderr, "mt-webfetch: fetch failed for %s: %v\n", target, err)
+		os.Exit(1)
+	}
+
+	req, err := http.NewRequest(http.MethodGet, defuddleURL, nil)
+	if err != nil {
+		fail(err)
+	}
+	req.Header.Set("User-Agent", "mt-webfetch/1.0")
+	res, err := (&http.Client{Timeout: 30 * time.Second}).Do(req)
+	if err != nil {
+		fail(err)
+	}
+	defer res.Body.Close()
+	body, err := io.ReadAll(res.Body)
+	if err != nil {
+		fail(err)
+	}
+	ok := res.StatusCode >= 200 && res.StatusCode < 300
+	if !ok || strings.TrimSpace(string(body)) == "" {
+		fmt.Fprintf(os.Stderr, "mt-webfetch: defuddle returned %d / empty body for %s\n", res.StatusCode, target)
+		os.Exit(1)
+	}
+	os.Stdout.Write(body)
+}

scripts/newsletter/find-newsletter-number.js

@@ -1,74 +0,0 @@
-#!/usr/bin/env node
-// Find the most recent newsletter number and return the next one
-// Usage: node find-newsletter-number.js
-// Outputs: The next newsletter number
-
-const fs = require("fs");
-const path = require("path");
-
-const PROJECT_ROOT = path.resolve(__dirname, "../..");
-const CONTENT_DIR = path.join(PROJECT_ROOT, "content", "post");
-
-// Extract newsletter number from file content
-function extractNewsletterNumber(filePath) {
-  try {
-    const content = fs.readFileSync(filePath, "utf-8");
-    const match = content.match(/Newsletter\s*#(\d+)/);
-    return match ? parseInt(match[1], 10) : 0;
-  } catch {
-    return 0;
-  }
-}
-
-// Scan all year/month/day directories for newsletter posts
-function findMostRecentNewsletter() {
-  let maxNumber = 0;
-
-  // List year directories, sorted descending
-  let years;
-  try {
-    years = fs.readdirSync(CONTENT_DIR)
-      .filter((d) => /^\d{4}$/.test(d))
-      .sort((a, b) => b.localeCompare(a));
-  } catch {
-    return 0;
-  }
-
-  for (const year of years) {
-    const yearDir = path.join(CONTENT_DIR, year);
-    let months;
-    try {
-      months = fs.readdirSync(yearDir)
-        .filter((d) => /^\d{2}$/.test(d))
-        .sort((a, b) => b.localeCompare(a));
-    } catch {
-      continue;
-    }
-
-    for (const month of months) {
-      const monthDir = path.join(yearDir, month);
-      let days;
-      try {
-        days = fs.readdirSync(monthDir)
-          .filter((d) => /^\d{2}$/.test(d))
-          .sort((a, b) => b.localeCompare(a));
-      } catch {
-        continue;
-      }
-
-      for (const day of days) {
-        const filePath = path.join(monthDir, day, "index.md");
-        const num = extractNewsletterNumber(filePath);
-        if (num > maxNumber) maxNumber = num;
-      }
-    }
-
-    // Early exit: if we found a newsletter in this year, no need to go further back
-    if (maxNumber > 0) break;
-  }
-
-  return maxNumber;
-}
-
-const mostRecent = findMostRecentNewsletter();
-console.log(mostRecent + 1);

scripts/newsletter/find-substack-post.js

@@ -1,151 +0,0 @@
-#!/usr/bin/env node
-// Find which Substack post embeds a given image UUID, and extract a label.
-// Usage: node find-substack-post.js --uuid <uuid> [--deep]
-// Output on hit:  JSON { found:true, source, publication, postTitle, postUrl, caption, candidates }
-// Output on miss: JSON { found:false } (RSS) or { found:false, source:"sitemap", scanned, budget, cutoff } (--deep)
-//
-// Strategy: RSS feed first (fast, ~recent weeks). With --deep, fall back to a
-// heavier sitemap crawl up to ~3 months back — opt-in because it fetches many
-// posts. A Substack CDN URL does not encode its publication, so we search each
-// publication listed in config/substack-publications.json.
-
-const fs = require("fs");
-const path = require("path");
-const { fetchWithTimeout } = require("./url-utils.js");
-const {
-  itemTitle,
-  itemLink,
-  extractCandidates,
-  captionForUuid,
-  postTitleFromHtml,
-} = require("./html-text-utils.js");
-
-const args = process.argv.slice(2);
-function flag(name) {
-  const i = args.indexOf(name);
-  return i !== -1 ? (args[i + 1] && !args[i + 1].startsWith("--") ? args[i + 1] : true) : undefined;
-}
-const uuid = flag("--uuid");
-const deep = !!flag("--deep");
-
-if (!uuid || uuid === true) {
-  console.error("Usage: node find-substack-post.js --uuid <uuid> [--deep]");
-  process.exit(1);
-}
-
-const CONFIG = path.resolve(__dirname, "./config/substack-publications.json");
-function loadPublications() {
-  try {
-    return JSON.parse(fs.readFileSync(CONFIG, "utf-8"));
-  } catch {
-    return ["blog.bytebytego.com"];
-  }
-}
-
-// Thin wrapper over the shared fetcher: returns body text, or "" on any error.
-async function fetchText(url) {
-  const res = await fetchWithTimeout(url, { headers: { "User-Agent": "Mozilla/5.0" } });
-  return res && res.ok ? await res.text() : "";
-}
-
-// Total post fetches allowed across ALL publications during a --deep crawl.
-const DEEP_FETCH_BUDGET = 40;
-
-// Deep fallback: crawl the sitemap back ~3 months, fetch posts most-recent-first
-// (up to `maxFetch` from the shared budget), and look for the UUID. Heavier than
-// RSS — only used on RSS miss.
-async function searchSitemap(publication, id, maxFetch, monthsBack = 3) {
-  const xml = await fetchText(`https://${publication}/sitemap.xml`);
-  if (!xml) return { hit: null, scanned: 0, cutoff: null };
-
-  const cutoff = new Date();
-  cutoff.setMonth(cutoff.getMonth() - monthsBack);
-
-  const candidates = [];
-  for (const block of xml.split("<url>").slice(1)) {
-    const loc = block.match(/<loc>([^<]+)<\/loc>/);
-    const lastmod = block.match(/<lastmod>([^<]+)<\/lastmod>/);
-    if (!loc || !lastmod) continue;
-    if (!/\/p\//.test(loc[1])) continue; // posts only
-    const when = new Date(lastmod[1]);
-    if (isNaN(when) || when < cutoff) continue;
-    candidates.push({ url: loc[1], when });
-  }
-  candidates.sort((a, b) => b.when - a.when);
-
-  let scanned = 0;
-  for (const c of candidates.slice(0, maxFetch)) {
-    scanned++;
-    const html = await fetchText(c.url);
-    if (!html || !html.includes(id)) continue;
-    return {
-      hit: {
-        found: true,
-        source: "sitemap",
-        publication,
-        postTitle: postTitleFromHtml(html),
-        postUrl: c.url,
-        caption: captionForUuid(html, id),
-        candidates: extractCandidates(html),
-      },
-      scanned,
-      cutoff: cutoff.toISOString().slice(0, 10),
-    };
-  }
-  return { hit: null, scanned, cutoff: cutoff.toISOString().slice(0, 10) };
-}
-
-async function searchRss(publication, id) {
-  const xml = await fetchText(`https://${publication}/feed`);
-  if (!xml) return null;
-  const items = xml.split("<item>").slice(1);
-  for (const item of items) {
-    if (!item.includes(id)) continue;
-    const postTitle = itemTitle(item);
-    const caption = captionForUuid(item, id);
-    return {
-      found: true,
-      source: "rss",
-      publication,
-      postTitle,
-      postUrl: itemLink(item),
-      caption,
-      candidates: extractCandidates(item),
-    };
-  }
-  return null;
-}
-
-async function main() {
-  const publications = loadPublications();
-  for (const pub of publications) {
-    const hit = await searchRss(pub, uuid);
-    if (hit) return console.log(JSON.stringify(hit, null, 2));
-  }
-
-  // Deep fallback: sitemap crawl up to ~3 months back, sharing one global
-  // fetch budget across all publications so coverage can't blow up as the
-  // publications list grows.
-  if (deep) {
-    let totalScanned = 0;
-    let lastCutoff = null;
-    for (const pub of publications) {
-      const remaining = DEEP_FETCH_BUDGET - totalScanned;
-      if (remaining <= 0) break;
-      const { hit, scanned, cutoff } = await searchSitemap(pub, uuid, remaining);
-      totalScanned += scanned;
-      lastCutoff = cutoff || lastCutoff;
-      if (hit) return console.log(JSON.stringify(hit, null, 2));
-    }
-    return console.log(JSON.stringify({
-      found: false,
-      source: "sitemap",
-      scanned: totalScanned,
-      budget: DEEP_FETCH_BUDGET,
-      cutoff: lastCutoff,
-    }, null, 2));
-  }
-  console.log(JSON.stringify({ found: false }, null, 2));
-}
-
-main();

scripts/newsletter/find_newsletter_number.go

@@ -0,0 +1,77 @@
+// Find the most recent newsletter number and return the next one.
+// Usage: go run ./scripts/newsletter find-newsletter-number
+// Outputs: the next newsletter number.
+package main
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"regexp"
+	"sort"
+	"strconv"
+)
+
+var (
+	newsletterNumRe = regexp.MustCompile(`Newsletter\s*#(\d+)`)
+	yearDirRe       = regexp.MustCompile(`^\d{4}$`)
+	twoDigitDirRe   = regexp.MustCompile(`^\d{2}$`)
+)
+
+// listDirsDesc returns dir's subdirectory names matching re, sorted descending.
+func listDirsDesc(dir string, re *regexp.Regexp) []string {
+	entries, err := os.ReadDir(dir)
+	if err != nil {
+		return nil
+	}
+	var names []string
+	for _, e := range entries {
+		if re.MatchString(e.Name()) {
+			names = append(names, e.Name())
+		}
+	}
+	sort.Sort(sort.Reverse(sort.StringSlice(names)))
+	return names
+}
+
+func extractNewsletterNumber(path string) int {
+	content, err := os.ReadFile(path)
+	if err != nil {
+		return 0
+	}
+	m := newsletterNumRe.FindSubmatch(content)
+	if m == nil {
+		return 0
+	}
+	n, err := strconv.Atoi(string(m[1]))
+	if err != nil {
+		return 0
+	}
+	return n
+}
+
+// findMostRecentNewsletter scans year/month/day directories newest-first for
+// the highest newsletter number.
+func findMostRecentNewsletter() int {
+	maxNumber := 0
+	for _, year := range listDirsDesc(contentDir(), yearDirRe) {
+		yearDir := filepath.Join(contentDir(), year)
+		for _, month := range listDirsDesc(yearDir, twoDigitDirRe) {
+			monthDir := filepath.Join(yearDir, month)
+			for _, day := range listDirsDesc(monthDir, twoDigitDirRe) {
+				if n := extractNewsletterNumber(filepath.Join(monthDir, day, "index.md")); n > maxNumber {
+					maxNumber = n
+				}
+			}
+		}
+		// Early exit: a newsletter found in this year — no need to go further back.
+		if maxNumber > 0 {
+			break
+		}
+	}
+	return maxNumber
+}
+
+func runFindNewsletterNumber(_ []string) {
+	fmt.Println(findMostRecentNewsletter() + 1)
+}

scripts/newsletter/find_substack_post.go

@@ -0,0 +1,202 @@
+// Find which Substack post embeds a given image UUID, and extract a label.
+// Usage: go run ./scripts/newsletter find-substack-post --uuid <uuid> [--deep]
+// Output on hit:  JSON { found:true, source, publication, postTitle, postUrl, caption, candidates }
+// Output on miss: JSON { found:false } (RSS) or { found:false, source:"sitemap", scanned, budget, cutoff } (--deep)
+//
+// Strategy: RSS feed first (fast, ~recent weeks). With --deep, fall back to a
+// heavier sitemap crawl up to ~3 months back — opt-in because it fetches many
+// posts. A Substack CDN URL does not encode its publication, so we search each
+// publication listed in config/substack-publications.json (embedded at build
+// time; `go run` recompiles on change, so editing the JSON still takes effect).
+package main
+
+import (
+	_ "embed"
+	"encoding/json"
+	"flag"
+	"fmt"
+	"os"
+	"regexp"
+	"sort"
+	"strings"
+	"time"
+)
+
+//go:embed config/substack-publications.json
+var publicationsJSON []byte
+
+func loadPublications() []string {
+	var pubs []string
+	if err := json.Unmarshal(publicationsJSON, &pubs); err != nil || len(pubs) == 0 {
+		return []string{"blog.bytebytego.com"}
+	}
+	return pubs
+}
+
+// fetchPage: body text with a browser-ish UA, or "" on any error.
+func fetchPage(target string) string {
+	return fetchTextOK(target, 10*time.Second, "Mozilla/5.0")
+}
+
+// Total post fetches allowed across ALL publications during a --deep crawl.
+const deepFetchBudget = 40
+
+type postHit struct {
+	Found       bool     `json:"found"`
+	Source      string   `json:"source"`
+	Publication string   `json:"publication"`
+	PostTitle   string   `json:"postTitle"`
+	PostURL     string   `json:"postUrl"`
+	Caption     string   `json:"caption"`
+	Candidates  []string `json:"candidates"`
+}
+
+var (
+	locRe     = regexp.MustCompile(`<loc>([^<]+)</loc>`)
+	lastmodRe = regexp.MustCompile(`<lastmod>([^<]+)</lastmod>`)
+)
+
+func parseLastmod(s string) (time.Time, bool) {
+	for _, layout := range []string{time.RFC3339, "2006-01-02"} {
+		if t, err := time.Parse(layout, s); err == nil {
+			return t, true
+		}
+	}
+	return time.Time{}, false
+}
+
+// searchSitemap is the deep fallback: crawl the sitemap back ~3 months, fetch
+// posts most-recent-first (up to maxFetch from the shared budget), and look
+// for the UUID. Heavier than RSS — only used on RSS miss.
+// cutoff is "" when the sitemap itself could not be fetched.
+func searchSitemap(publication, id string, maxFetch int) (hit *postHit, scanned int, cutoff string) {
+	xml := fetchPage("https://" + publication + "/sitemap.xml")
+	if xml == "" {
+		return nil, 0, ""
+	}
+
+	cutoffTime := time.Now().UTC().AddDate(0, -3, 0)
+	cutoff = cutoffTime.Format("2006-01-02")
+
+	type candidate struct {
+		url  string
+		when time.Time
+	}
+	var candidates []candidate
+	for _, block := range strings.Split(xml, "<url>")[1:] {
+		loc := locRe.FindStringSubmatch(block)
+		lastmod := lastmodRe.FindStringSubmatch(block)
+		if loc == nil || lastmod == nil {
+			continue
+		}
+		if !strings.Contains(loc[1], "/p/") { // posts only
+			continue
+		}
+		when, ok := parseLastmod(lastmod[1])
+		if !ok || when.Before(cutoffTime) {
+			continue
+		}
+		candidates = append(candidates, candidate{url: loc[1], when: when})
+	}
+	sort.SliceStable(candidates, func(i, j int) bool {
+		return candidates[i].when.After(candidates[j].when)
+	})
+
+	if len(candidates) > maxFetch {
+		candidates = candidates[:maxFetch]
+	}
+	for _, c := range candidates {
+		scanned++
+		html := fetchPage(c.url)
+		if html == "" || !strings.Contains(html, id) {
+			continue
+		}
+		return &postHit{
+			Found:       true,
+			Source:      "sitemap",
+			Publication: publication,
+			PostTitle:   postTitleFromHTML(html),
+			PostURL:     c.url,
+			Caption:     captionForUUID(html, id),
+			Candidates:  extractCandidates(html),
+		}, scanned, cutoff
+	}
+	return nil, scanned, cutoff
+}
+
+func searchRSS(publication, id string) *postHit {
+	xml := fetchPage("https://" + publication + "/feed")
+	if xml == "" {
+		return nil
+	}
+	for _, item := range strings.Split(xml, "<item>")[1:] {
+		if !strings.Contains(item, id) {
+			continue
+		}
+		return &postHit{
+			Found:       true,
+			Source:      "rss",
+			Publication: publication,
+			PostTitle:   itemTitle(item),
+			PostURL:     itemLink(item),
+			Caption:     captionForUUID(item, id),
+			Candidates:  extractCandidates(item),
+		}
+	}
+	return nil
+}
+
+func runFindSubstackPost(args []string) {
+	fs := flag.NewFlagSet("find-substack-post", flag.ExitOnError)
+	uuid := fs.String("uuid", "", "Substack S3 image uuid to search for")
+	deep := fs.Bool("deep", false, "fall back to a sitemap crawl (~3 months back)")
+	_ = fs.Parse(args)
+
+	if *uuid == "" {
+		fmt.Fprintln(os.Stderr, "Usage: go run ./scripts/newsletter find-substack-post --uuid <uuid> [--deep]")
+		os.Exit(1)
+	}
+
+	publications := loadPublications()
+	for _, pub := range publications {
+		if hit := searchRSS(pub, *uuid); hit != nil {
+			printJSON(hit)
+			return
+		}
+	}
+
+	// Deep fallback: sitemap crawl up to ~3 months back, sharing one global
+	// fetch budget across all publications so coverage can't blow up as the
+	// publications list grows.
+	if *deep {
+		totalScanned := 0
+		var lastCutoff *string
+		for _, pub := range publications {
+			remaining := deepFetchBudget - totalScanned
+			if remaining <= 0 {
+				break
+			}
+			hit, scanned, cutoff := searchSitemap(pub, *uuid, remaining)
+			totalScanned += scanned
+			if cutoff != "" {
+				lastCutoff = &cutoff
+			}
+			if hit != nil {
+				printJSON(hit)
+				return
+			}
+		}
+		printJSON(struct {
+			Found   bool    `json:"found"`
+			Source  string  `json:"source"`
+			Scanned int     `json:"scanned"`
+			Budget  int     `json:"budget"`
+			Cutoff  *string `json:"cutoff"`
+		}{false, "sitemap", totalScanned, deepFetchBudget, lastCutoff})
+		return
+	}
+
+	printJSON(struct {
+		Found bool `json:"found"`
+	}{false})
+}

scripts/newsletter/html-text-utils.js

@@ -1,91 +0,0 @@
-// HTML / RSS text-extraction helpers for find-substack-post.js.
-// Pure string functions (no network, no fs) — kept separate so the crawler
-// stays focused on fetch/search flow. CommonJS so plain `node` works.
-
-// Decode the HTML entities that appear in Substack titles/captions, including
-// numeric (') and hex (—) forms. & is decoded last-ish but
-// before nothing re-encodes it; ordering here is safe for our inputs.
-function decodeEntities(s) {
-  return s
-    .replace(/&/g, "&")
-    .replace(/&lt;/g, "<")
-    .replace(/&gt;/g, ">")
-    .replace(/&quot;/g, '"')
-    .replace(/&#0?39;|&apos;/g, "'")
-    .replace(/&nbsp;/g, " ")
-    .replace(/&#x([0-9a-fA-F]+);/g, (_, n) => String.fromCodePoint(parseInt(n, 16)))
-    .replace(/&#(\d+);/g, (_, n) => String.fromCodePoint(+n))
-    .trim();
-}
-
-function stripTags(html) {
-  return decodeEntities(html.replace(/<[^>]+>/g, "")).replace(/\s+/g, " ").trim();
-}
-
-// Pull the first <title> (CDATA or plain) and <link> from an RSS <item> chunk.
-function itemTitle(item) {
-  const m = item.match(/<title>(?:<!\[CDATA\[)?([\s\S]*?)(?:\]\]>)?<\/title>/);
-  return m ? decodeEntities(m[1].trim()) : "";
-}
-function itemLink(item) {
-  const m = item.match(/<link>(?:<!\[CDATA\[)?([\s\S]*?)(?:\]\]>)?<\/link>/);
-  return m ? m[1].trim() : "";
-}
-
-// Extract candidate topic titles from a post's TOC bullet list. ByteByteGo does
-// not attach captions to images — the topic titles live only in the "in this
-// issue" bullets. We can't reliably map image→title automatically (sponsor /
-// video items interleave and break positional order), so we surface these
-// candidates for the user to pick from. Light filtering keeps the list short:
-// dedupe, drop sub-point explanations ("Term: long sentence") and over-long
-// lines. The correct title is always present; the user selects it.
-function extractCandidates(html) {
-  const seen = new Set();
-  const out = [];
-  const re = /<li[^>]*>\s*(?:<p[^>]*>)?([\s\S]*?)(?:<\/p>)?\s*<\/li>/g;
-  let m;
-  while ((m = re.exec(html))) {
-    const text = stripTags(m[1]);
-    if (!text) continue;
-    if (text.length < 6 || text.length > 70) continue;     // titles are short; long lines are sub-point explanations
-    const key = text.toLowerCase();
-    if (seen.has(key)) continue;                           // content is duplicated in the page
-    seen.add(key);
-    out.push(text);
-  }
-  return out;
-}
-
-// If the UUID sits inside a <figure>…</figure>, return that figure's
-// <figcaption> text. Cover images live in <enclosure> (no figure) → "".
-function captionForUuid(item, id) {
-  const at = item.indexOf(id);
-  if (at === -1) return "";
-  const figStart = item.lastIndexOf("<figure", at);
-  if (figStart === -1) return "";
-  const figEnd = item.indexOf("</figure>", at);
-  if (figEnd === -1) return "";
-  const figure = item.slice(figStart, figEnd);
-  const cap = figure.match(/<figcaption[^>]*>([\s\S]*?)<\/figcaption>/);
-  return cap ? stripTags(cap[1]) : "";
-}
-
-// Extract a post title from server-rendered post HTML (og:title preferred).
-function postTitleFromHtml(html) {
-  const og = html.match(/<meta[^>]+property=["']og:title["'][^>]+content=["']([^"']+)["']/i);
-  if (og) return decodeEntities(og[1]);
-  const h1 = html.match(/<h1[^>]*>([\s\S]*?)<\/h1>/i);
-  if (h1) return stripTags(h1[1]);
-  const t = html.match(/<title>([\s\S]*?)<\/title>/i);
-  return t ? decodeEntities(t[1].trim()) : "";
-}
-
-module.exports = {
-  decodeEntities,
-  stripTags,
-  itemTitle,
-  itemLink,
-  extractCandidates,
-  captionForUuid,
-  postTitleFromHtml,
-};

scripts/newsletter/html_text.go

@@ -0,0 +1,116 @@
+// HTML / RSS text-extraction helpers for find-substack-post, ported from
+// html-text-utils.js. Pure string functions — no network, no fs.
+package main
+
+import (
+	"html"
+	"regexp"
+	"strings"
+	"unicode/utf8"
+)
+
+var (
+	tagRe        = regexp.MustCompile(`<[^>]+>`)
+	spaceRe      = regexp.MustCompile(`\s+`)
+	titleTagRe   = regexp.MustCompile(`(?s)<title>(?:<!\[CDATA\[)?(.*?)(?:\]\]>)?</title>`)
+	linkTagRe    = regexp.MustCompile(`(?s)<link>(?:<!\[CDATA\[)?(.*?)(?:\]\]>)?</link>`)
+	liRe         = regexp.MustCompile(`(?s)<li[^>]*>\s*(?:<p[^>]*>)?(.*?)(?:</p>)?\s*</li>`)
+	figcaptionRe = regexp.MustCompile(`(?s)<figcaption[^>]*>(.*?)</figcaption>`)
+	ogTitleRe    = regexp.MustCompile(`(?i)<meta[^>]+property=["']og:title["'][^>]+content=["']([^"']+)["']`)
+	h1Re         = regexp.MustCompile(`(?is)<h1[^>]*>(.*?)</h1>`)
+	htmlTitleRe  = regexp.MustCompile(`(?is)<title>(.*?)</title>`)
+)
+
+// decodeEntities decodes HTML entities (named, numeric, hex) and trims.
+// html.UnescapeString covers a superset of the entity map the JS hand-rolled.
+func decodeEntities(s string) string {
+	return strings.TrimSpace(html.UnescapeString(s))
+}
+
+func stripTags(s string) string {
+	return strings.TrimSpace(spaceRe.ReplaceAllString(decodeEntities(tagRe.ReplaceAllString(s, "")), " "))
+}
+
+// itemTitle pulls the first <title> (CDATA or plain) from an RSS <item> chunk.
+func itemTitle(item string) string {
+	m := titleTagRe.FindStringSubmatch(item)
+	if m == nil {
+		return ""
+	}
+	return decodeEntities(m[1])
+}
+
+// itemLink pulls the first <link> (CDATA or plain) from an RSS <item> chunk.
+func itemLink(item string) string {
+	m := linkTagRe.FindStringSubmatch(item)
+	if m == nil {
+		return ""
+	}
+	return strings.TrimSpace(m[1])
+}
+
+// extractCandidates pulls candidate topic titles from a post's TOC bullet
+// list. ByteByteGo does not attach captions to images — the topic titles live
+// only in the "in this issue" bullets, and image→title cannot be mapped
+// automatically (sponsor/video items interleave), so these are surfaced for
+// the user to pick from. Light filtering keeps the list short: dedupe, drop
+// sub-point explanations and over-long lines.
+func extractCandidates(htmlSrc string) []string {
+	seen := map[string]bool{}
+	out := []string{} // non-nil: marshals as [] like the JS output
+	for _, m := range liRe.FindAllStringSubmatch(htmlSrc, -1) {
+		text := stripTags(m[1])
+		if text == "" {
+			continue
+		}
+		// titles are short; long lines are sub-point explanations
+		if n := utf8.RuneCountInString(text); n < 6 || n > 70 {
+			continue
+		}
+		key := strings.ToLower(text)
+		if seen[key] { // content is duplicated in the page
+			continue
+		}
+		seen[key] = true
+		out = append(out, text)
+	}
+	return out
+}
+
+// captionForUUID returns the <figcaption> text of the <figure> containing the
+// UUID. Cover images live in <enclosure> (no figure) → "".
+func captionForUUID(item, id string) string {
+	at := strings.Index(item, id)
+	if at == -1 {
+		return ""
+	}
+	figStart := strings.LastIndex(item[:at], "<figure")
+	if figStart == -1 {
+		return ""
+	}
+	rel := strings.Index(item[at:], "</figure>")
+	if rel == -1 {
+		return ""
+	}
+	figure := item[figStart : at+rel]
+	m := figcaptionRe.FindStringSubmatch(figure)
+	if m == nil {
+		return ""
+	}
+	return stripTags(m[1])
+}
+
+// postTitleFromHTML extracts a post title from server-rendered post HTML
+// (og:title preferred, then <h1>, then <title>).
+func postTitleFromHTML(htmlSrc string) string {
+	if m := ogTitleRe.FindStringSubmatch(htmlSrc); m != nil {
+		return decodeEntities(m[1])
+	}
+	if m := h1Re.FindStringSubmatch(htmlSrc); m != nil {
+		return stripTags(m[1])
+	}
+	if m := htmlTitleRe.FindStringSubmatch(htmlSrc); m != nil {
+		return decodeEntities(m[1])
+	}
+	return ""
+}

scripts/newsletter/list-existing-tags.js

@@ -1,65 +0,0 @@
-#!/usr/bin/env node
-// List existing tags in the repo ranked by frequency
-// Usage: node list-existing-tags.js
-// Outputs: tag count and name, sorted most-used first (top 40)
-//
-// NOTE: This script is NOT currently used by the mt-add-tags skill.
-// Tag normalization is disabled until existing posts have standardized tags.
-// To enable, uncomment step 4a in SKILL.md.
-
-const fs = require("fs");
-const path = require("path");
-
-const PROJECT_ROOT = path.resolve(__dirname, "../..");
-const CONTENT_DIR = path.join(PROJECT_ROOT, "content", "post");
-
-// Recursively find all index.md files
-function findIndexFiles(dir) {
-  const results = [];
-  try {
-    const entries = fs.readdirSync(dir, { withFileTypes: true });
-    for (const entry of entries) {
-      const fullPath = path.join(dir, entry.name);
-      if (entry.isDirectory()) {
-        results.push(...findIndexFiles(fullPath));
-      } else if (entry.name === "index.md") {
-        results.push(fullPath);
-      }
-    }
-  } catch {
-    // skip unreadable directories
-  }
-  return results;
-}
-
-// Extract tags from frontmatter
-function extractTags(filePath) {
-  try {
-    const content = fs.readFileSync(filePath, "utf-8");
-    const match = content.match(/^tags:\s*\[([^\]]*)\]/m);
-    if (!match) return [];
-    // Extract quoted strings from the tags array
-    return [...match[1].matchAll(/"([^"]+)"/g)].map((m) => m[1]);
-  } catch {
-    return [];
-  }
-}
-
-// Count tag frequency
-const tagCounts = new Map();
-const files = findIndexFiles(CONTENT_DIR);
-
-for (const file of files) {
-  for (const tag of extractTags(file)) {
-    tagCounts.set(tag, (tagCounts.get(tag) || 0) + 1);
-  }
-}
-
-// Sort by frequency descending, take top 40
-const sorted = [...tagCounts.entries()]
-  .sort((a, b) => b[1] - a[1])
-  .slice(0, 40);
-
-for (const [tag, count] of sorted) {
-  console.log(`${String(count).padStart(6)} ${tag}`);
-}

scripts/newsletter/list_existing_tags.go

@@ -0,0 +1,75 @@
+// List existing tags in the repo ranked by frequency.
+// Usage: go run ./scripts/newsletter list-existing-tags
+// Outputs: tag count and name, sorted most-used first (top 40).
+//
+// NOTE: not currently used by the mt-add-tags skill. Tag normalization is
+// disabled until existing posts have standardized tags; to enable, uncomment
+// step 4a in that skill's SKILL.md.
+package main
+
+import (
+	"fmt"
+	"io/fs"
+	"os"
+	"path/filepath"
+	"regexp"
+	"sort"
+)
+
+var (
+	tagsLineRe  = regexp.MustCompile(`(?m)^tags:\s*\[([^\]]*)\]`)
+	quotedTagRe = regexp.MustCompile(`"([^"]+)"`)
+)
+
+// extractTags pulls quoted tag strings from an index.md frontmatter tags array.
+func extractTags(path string) []string {
+	content, err := os.ReadFile(path)
+	if err != nil {
+		return nil
+	}
+	m := tagsLineRe.FindSubmatch(content)
+	if m == nil {
+		return nil
+	}
+	var tags []string
+	for _, q := range quotedTagRe.FindAllSubmatch(m[1], -1) {
+		tags = append(tags, string(q[1]))
+	}
+	return tags
+}
+
+func runListExistingTags(_ []string) {
+	type tagCount struct {
+		tag   string
+		count int
+	}
+	// Slice + index map keeps first-seen order for equal counts, so the stable
+	// sort below ranks ties deterministically (walk order is lexical).
+	var counts []tagCount
+	index := map[string]int{}
+
+	_ = filepath.WalkDir(contentDir(), func(p string, d fs.DirEntry, err error) error {
+		if err != nil || d.IsDir() || d.Name() != "index.md" {
+			return nil
+		}
+		for _, tag := range extractTags(p) {
+			if i, ok := index[tag]; ok {
+				counts[i].count++
+			} else {
+				index[tag] = len(counts)
+				counts = append(counts, tagCount{tag: tag, count: 1})
+			}
+		}
+		return nil
+	})
+
+	sort.SliceStable(counts, func(i, j int) bool {
+		return counts[i].count > counts[j].count
+	})
+	if len(counts) > 40 {
+		counts = counts[:40]
+	}
+	for _, tc := range counts {
+		fmt.Printf("%6d %s\n", tc.count, tc.tag)
+	}
+}

scripts/newsletter/main.go

@@ -0,0 +1,65 @@
+// Newsletter engine for the mt-* skills — single binary, one subcommand per
+// former Node script. Invoked from the repo root:
+//
+//	go run ./scripts/newsletter <command> [args]
+//
+// Repo-relative paths (content/post) resolve from the working directory, so
+// the repo-root invocation contract from AGENTS.md still applies.
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"os"
+)
+
+func usage() {
+	fmt.Fprint(os.Stderr, `Usage: go run ./scripts/newsletter <command> [args]
+
+Commands:
+  add-url <url>                              classify + dedup a URL, emit JSON route
+  find-newsletter-number                     print the next newsletter number
+  list-existing-tags                         tag frequencies, most-used first (top 40)
+  detect-image-source <url>                  detect Substack image + uuid
+  find-substack-post --uuid <uuid> [--deep]  find the post embedding an image uuid
+  fetch-via-defuddle <url>                   fallback fetch via defuddle.md proxy
+`)
+}
+
+// printJSON mirrors console.log(JSON.stringify(v, null, 2)): 2-space indent,
+// no HTML escaping (URLs with & must stay readable), trailing newline.
+func printJSON(v any) {
+	enc := json.NewEncoder(os.Stdout)
+	enc.SetEscapeHTML(false)
+	enc.SetIndent("", "  ")
+	if err := enc.Encode(v); err != nil {
+		fmt.Fprintln(os.Stderr, "json encode:", err)
+		os.Exit(1)
+	}
+}
+
+func main() {
+	if len(os.Args) < 2 {
+		usage()
+		os.Exit(1)
+	}
+	args := os.Args[2:]
+	switch os.Args[1] {
+	case "add-url":
+		runAddURL(args)
+	case "find-newsletter-number":
+		runFindNewsletterNumber(args)
+	case "list-existing-tags":
+		runListExistingTags(args)
+	case "detect-image-source":
+		runDetectImageSource(args)
+	case "find-substack-post":
+		runFindSubstackPost(args)
+	case "fetch-via-defuddle":
+		runFetchViaDefuddle(args)
+	default:
+		fmt.Fprintf(os.Stderr, "unknown command: %s\n", os.Args[1])
+		usage()
+		os.Exit(1)
+	}
+}

scripts/newsletter/url-utils.js

@@ -1,166 +0,0 @@
-// Shared URL helpers for the mt-* newsletter skills.
-// Owned by the mt-add-url meta router; reused by handlers (e.g. mt-add-image).
-// CommonJS so plain `node script.js` works without a build step.
-
-const fs = require("fs");
-const path = require("path");
-
-// Remove common tracking parameters (utm_* plus a fixed set of known trackers).
-function cleanUrl(rawUrl) {
-  const EXACT_TRACKING = new Set([
-    "fbclid", "gclid", "msclkid", "mc_eid",
-    "aid", "ref", "ref_src", "ref_url", "source", "s",
-    "ck_subscriber_id", "igshid", "yclid", "vero_id",
-  ]);
-  try {
-    const parsed = new URL(rawUrl);
-    [...parsed.searchParams.keys()].forEach((k) => {
-      if (k.toLowerCase().startsWith("utm_") || EXACT_TRACKING.has(k.toLowerCase())) {
-        parsed.searchParams.delete(k);
-      }
-    });
-    return parsed.toString();
-  } catch {
-    // Unparseable input (not a real URL): the per-param string surgery above
-    // would mangle the query (drop the `?`, leave a dangling `&`), so leave it
-    // untouched rather than corrupt it.
-    return rawUrl;
-  }
-}
-
-// --- Substack image helpers (shared by add-url.js routing and mt-add-image) ---
-const SUBSTACK_IMAGE_HOSTS = ["substackcdn.com", "substack-post-media.s3.amazonaws.com"];
-const UUID = "[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}";
-
-// A Substack-hosted image (CDN wrapper or raw S3), regardless of file extension.
-function isSubstackImage(targetUrl) {
-  let host = "";
-  try { host = new URL(targetUrl).host.toLowerCase(); } catch { /* non-URL */ }
-  return SUBSTACK_IMAGE_HOSTS.includes(host) || /substack-post-media/i.test(targetUrl);
-}
-
-// The stable image identity is the S3 image UUID under public/images/<uuid>.
-// Works whether the path separators are raw (/) or percent-encoded (%2F).
-function substackImageUuid(targetUrl) {
-  const m = targetUrl.match(new RegExp(`images(?:%2F|/)(${UUID})`, "i"));
-  return m ? m[1].toLowerCase() : null;
-}
-
-// Some sites carry the resource identity in a query param, not the path
-// (e.g. YouTube /watch?v=ID). Stripping the query for these collapses every
-// item to the same bare URL, causing false-positive duplicates. Preserve the
-// identity param for those hosts.
-const IDENTITY_PARAMS = {
-  "youtube.com": "v",
-  "www.youtube.com": "v",
-  "m.youtube.com": "v",
-};
-
-// Reduce a URL to a stable identity used for duplicate detection:
-// - Substack image  → its S3 UUID (transform/size variants share one identity)
-// - YouTube         → scheme+host+path + the v= video id
-// - everything else → scheme + host + path
-function bareUrl(targetUrl) {
-  if (isSubstackImage(targetUrl)) {
-    const uuid = substackImageUuid(targetUrl);
-    if (uuid) return uuid;
-  }
-  try {
-    const p = new URL(targetUrl);
-    let bare = `${p.protocol}//${p.host}${p.pathname}`.replace(/\/$/, "");
-    const idParam = IDENTITY_PARAMS[p.host.toLowerCase()];
-    const idValue = idParam ? p.searchParams.get(idParam) : null;
-    if (idValue) bare += `?${idParam}=${idValue}`;
-    return bare;
-  } catch {
-    return targetUrl.split("?")[0].replace(/\/$/, "");
-  }
-}
-
-// fetch() with an abort timeout. Returns the Response on success, or null on
-// network error / timeout. Callers decide what to read (.text/.json/.status).
-// Centralizes the AbortController + clearTimeout dance so every caller cleans
-// up the timer (via finally) on both the success and failure paths.
-async function fetchWithTimeout(targetUrl, { method = "GET", timeoutMs = 10000, headers } = {}) {
-  const controller = new AbortController();
-  const timer = setTimeout(() => controller.abort(), timeoutMs);
-  try {
-    return await fetch(targetUrl, { method, redirect: "follow", signal: controller.signal, headers });
-  } catch {
-    return null;
-  } finally {
-    clearTimeout(timer);
-  }
-}
-
-// Check if URL is accessible (returns HTTP status code as a string).
-async function checkAccessibility(targetUrl) {
-  const res = await fetchWithTimeout(targetUrl, { method: "HEAD" });
-  return res ? res.status.toString() : "000";
-}
-
-function escapeRegExp(s) {
-  return s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
-}
-
-// Recursively collect *.md files under a directory (the content tree is small).
-function collectMarkdown(dir, acc = []) {
-  let entries;
-  try {
-    entries = fs.readdirSync(dir, { withFileTypes: true });
-  } catch {
-    return acc; // missing dir → nothing to compare against
-  }
-  for (const e of entries) {
-    const full = path.join(dir, e.name);
-    if (e.isDirectory()) collectMarkdown(full, acc);
-    else if (e.isFile() && e.name.toLowerCase().endsWith(".md")) acc.push(full);
-  }
-  return acc;
-}
-
-// Check whether a URL identity already exists in the stored markdown.
-// Pure JS (no external `grep` dependency, works the same from any shell) and
-// boundary-aware so a needle that is merely a PREFIX of a stored longer string
-// is NOT a false duplicate. The two identity kinds need different boundaries:
-//   - Substack image UUID: the next char must not extend the hex id, so all of
-//     <uuid>.png (cover image, no size suffix), <uuid>_WxH and <uuid>) match.
-//   - URL: must be followed by a path/punctuation delimiter so /p/foo does not
-//     match a stored /p/foo-bar.
-function checkDuplicate(targetUrl, contentDir) {
-  const needle = bareUrl(targetUrl);
-  if (!needle) return false;
-  const isUuid = new RegExp(`^${UUID}$`, "i").test(needle);
-  // For URLs, allow an optional trailing slash (bareUrl strips it, stored URLs
-  // may keep it) before the delimiter.
-  const boundary = isUuid
-    ? new RegExp(escapeRegExp(needle) + `(?![0-9a-f])`, "i")
-    : new RegExp(escapeRegExp(needle) + `/?(?:[)\\]\\s"'?#<_&,]|$)`, "m");
-  for (const file of collectMarkdown(contentDir)) {
-    let text;
-    try { text = fs.readFileSync(file, "utf-8"); } catch { continue; }
-    if (text.includes(needle) && boundary.test(text)) return true;
-  }
-  return false;
-}
-
-// Classify URL type by file extension. Returns image|video|document|article.
-function classifyType(targetUrl) {
-  const lower = targetUrl.toLowerCase();
-  if (/\.(png|jpg|jpeg|gif|webp|svg|avif|heic|heif|bmp|tiff?)(\?.*)?$/.test(lower)) return "image";
-  if (/\.(mp4|webm|mov|avi|mkv)(\?.*)?$/.test(lower)) return "video";
-  if (/\.(pdf|docx?|xlsx?|pptx?)(\?.*)?$/.test(lower)) return "document";
-  return "article";
-}
-
-module.exports = {
-  cleanUrl,
-  bareUrl,
-  IDENTITY_PARAMS,
-  isSubstackImage,
-  substackImageUuid,
-  fetchWithTimeout,
-  checkAccessibility,
-  checkDuplicate,
-  classifyType,
-};

scripts/newsletter/url_utils.go

@@ -0,0 +1,276 @@
+// Shared URL helpers, ported from url-utils.js. Owned by the add-url router;
+// reused by the other subcommands.
+package main
+
+import (
+	"io"
+	"io/fs"
+	"net/http"
+	"net/url"
+	"os"
+	"path/filepath"
+	"regexp"
+	"strconv"
+	"strings"
+	"time"
+)
+
+// Exact-match tracking params; any key starting with utm_ is also dropped.
+var exactTracking = map[string]bool{
+	"fbclid": true, "gclid": true, "msclkid": true, "mc_eid": true,
+	"aid": true, "ref": true, "ref_src": true, "ref_url": true, "source": true, "s": true,
+	"ck_subscriber_id": true, "igshid": true, "yclid": true, "vero_id": true,
+}
+
+// cleanURL removes common tracking parameters. Surviving query pairs are kept
+// verbatim (no re-encoding) and in their original order — Go's url.Values
+// would sort keys alphabetically, which must not leak into stored clean_url
+// values. Unparseable / non-absolute input is returned untouched rather than
+// corrupted.
+func cleanURL(raw string) string {
+	u, err := url.Parse(raw)
+	if err != nil || u.Scheme == "" || u.Host == "" {
+		return raw
+	}
+	u.Host = strings.ToLower(u.Host)
+	// WHATWG URL serializes an empty path as "/"; keep that shape so cleaned
+	// URLs match what the JS engine has already written into posts.
+	if u.Path == "" {
+		u.Path = "/"
+	}
+	if u.RawQuery != "" {
+		var kept []string
+		for _, pair := range strings.Split(u.RawQuery, "&") {
+			if pair == "" {
+				continue
+			}
+			key := pair
+			if i := strings.IndexByte(pair, '='); i != -1 {
+				key = pair[:i]
+			}
+			k := strings.ToLower(key)
+			if strings.HasPrefix(k, "utm_") || exactTracking[k] {
+				continue
+			}
+			kept = append(kept, pair)
+		}
+		u.RawQuery = strings.Join(kept, "&")
+	}
+	u.ForceQuery = false
+	return u.String()
+}
+
+// --- Substack image helpers (shared by add-url routing and detect-image-source) ---
+
+var substackImageHosts = map[string]bool{
+	"substackcdn.com":                      true,
+	"substack-post-media.s3.amazonaws.com": true,
+}
+
+const uuidPattern = `[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}`
+
+var (
+	imageUUIDRe = regexp.MustCompile(`(?i)images(?:%2F|/)(` + uuidPattern + `)`)
+	uuidExactRe = regexp.MustCompile(`(?i)^` + uuidPattern + `$`)
+)
+
+// isSubstackImage reports a Substack-hosted image (CDN wrapper or raw S3),
+// regardless of file extension.
+func isSubstackImage(target string) bool {
+	host := ""
+	if u, err := url.Parse(target); err == nil {
+		host = strings.ToLower(u.Host)
+	}
+	return substackImageHosts[host] || strings.Contains(strings.ToLower(target), "substack-post-media")
+}
+
+// substackImageUUID extracts the stable image identity: the S3 image UUID
+// under public/images/<uuid>, with raw (/) or percent-encoded (%2F) separators.
+// Returns "" when absent.
+func substackImageUUID(target string) string {
+	m := imageUUIDRe.FindStringSubmatch(target)
+	if m == nil {
+		return ""
+	}
+	return strings.ToLower(m[1])
+}
+
+// Some sites carry the resource identity in a query param, not the path
+// (e.g. YouTube /watch?v=ID). Preserve the identity param for those hosts so
+// dedup does not collapse every video onto the same bare URL.
+var identityParams = map[string]string{
+	"youtube.com":     "v",
+	"www.youtube.com": "v",
+	"m.youtube.com":   "v",
+}
+
+// bareURL reduces a URL to a stable identity for duplicate detection:
+//   - Substack image  → its S3 UUID (transform/size variants share one identity)
+//   - YouTube         → scheme+host+path + the v= video id
+//   - everything else → scheme + host + path
+func bareURL(target string) string {
+	if isSubstackImage(target) {
+		if uuid := substackImageUUID(target); uuid != "" {
+			return uuid
+		}
+	}
+	u, err := url.Parse(target)
+	if err != nil || u.Scheme == "" || u.Host == "" {
+		return strings.TrimSuffix(strings.SplitN(target, "?", 2)[0], "/")
+	}
+	host := strings.ToLower(u.Host)
+	bare := strings.TrimSuffix(u.Scheme+"://"+host+u.EscapedPath(), "/")
+	if idParam, ok := identityParams[host]; ok {
+		if v := u.Query().Get(idParam); v != "" {
+			bare += "?" + idParam + "=" + v
+		}
+	}
+	return bare
+}
+
+// fetchTextOK GETs target and returns the body on a 2xx response, "" on any
+// error, non-2xx status, or timeout. Redirects are followed.
+func fetchTextOK(target string, timeout time.Duration, userAgent string) string {
+	req, err := http.NewRequest(http.MethodGet, target, nil)
+	if err != nil {
+		return ""
+	}
+	if userAgent != "" {
+		req.Header.Set("User-Agent", userAgent)
+	}
+	res, err := (&http.Client{Timeout: timeout}).Do(req)
+	if err != nil {
+		return ""
+	}
+	defer res.Body.Close()
+	if res.StatusCode < 200 || res.StatusCode >= 300 {
+		return ""
+	}
+	body, err := io.ReadAll(res.Body)
+	if err != nil {
+		return ""
+	}
+	return string(body)
+}
+
+// checkAccessibility HEADs the URL and returns the final HTTP status code as a
+// string, or "000" on network error / timeout (matching the JS sentinel).
+func checkAccessibility(target string) string {
+	req, err := http.NewRequest(http.MethodHead, target, nil)
+	if err != nil {
+		return "000"
+	}
+	res, err := (&http.Client{Timeout: 10 * time.Second}).Do(req)
+	if err != nil {
+		return "000"
+	}
+	res.Body.Close()
+	return strconv.Itoa(res.StatusCode)
+}
+
+// collectMarkdown recursively collects *.md files under dir (the content tree
+// is small). Missing or unreadable directories yield nothing.
+func collectMarkdown(dir string) []string {
+	var acc []string
+	_ = filepath.WalkDir(dir, func(p string, d fs.DirEntry, err error) error {
+		if err != nil {
+			return nil // skip unreadable entries
+		}
+		if !d.IsDir() && strings.HasSuffix(strings.ToLower(d.Name()), ".md") {
+			acc = append(acc, p)
+		}
+		return nil
+	})
+	return acc
+}
+
+// uuidBoundaryOK: the char after a UUID match must not extend the hex id, so
+// <uuid>.png (cover image), <uuid>_WxH and <uuid>) all match.
+func uuidBoundaryOK(text string, end int) bool {
+	if end >= len(text) {
+		return true
+	}
+	c := text[end]
+	return !((c >= '0' && c <= '9') || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F'))
+}
+
+// urlBoundaryOK: an optional trailing slash (bareURL strips it, stored URLs may
+// keep it), then a path/punctuation delimiter, whitespace, or end of text — so
+// /p/foo does not match a stored /p/foo-bar.
+func urlBoundaryOK(text string, end int) bool {
+	if end < len(text) && text[end] == '/' {
+		end++
+	}
+	if end >= len(text) {
+		return true
+	}
+	switch c := text[end]; c {
+	case ' ', '\t', '\n', '\r', '\f', '\v':
+		return true
+	default:
+		return strings.IndexByte(`)]"'?#<_&,`, c) != -1
+	}
+}
+
+// hasBoundaryMatch scans every occurrence of needle and applies the boundary
+// check in code — Go's RE2 regexp has no lookahead, which the JS version used.
+func hasBoundaryMatch(text, needle string, isUUID bool) bool {
+	for from := 0; ; {
+		i := strings.Index(text[from:], needle)
+		if i == -1 {
+			return false
+		}
+		end := from + i + len(needle)
+		if isUUID {
+			if uuidBoundaryOK(text, end) {
+				return true
+			}
+		} else if urlBoundaryOK(text, end) {
+			return true
+		}
+		from = from + i + 1
+	}
+}
+
+// checkDuplicate reports whether a URL identity already exists in the stored
+// markdown, boundary-aware so a needle that is merely a PREFIX of a stored
+// longer string is NOT a false duplicate.
+func checkDuplicate(target, contentDir string) bool {
+	needle := bareURL(target)
+	if needle == "" {
+		return false
+	}
+	isUUID := uuidExactRe.MatchString(needle)
+	for _, file := range collectMarkdown(contentDir) {
+		data, err := os.ReadFile(file)
+		if err != nil {
+			continue
+		}
+		text := string(data)
+		if strings.Contains(text, needle) && hasBoundaryMatch(text, needle, isUUID) {
+			return true
+		}
+	}
+	return false
+}
+
+var (
+	imageExtRe    = regexp.MustCompile(`\.(png|jpg|jpeg|gif|webp|svg|avif|heic|heif|bmp|tiff?)(\?.*)?$`)
+	videoExtRe    = regexp.MustCompile(`\.(mp4|webm|mov|avi|mkv)(\?.*)?$`)
+	documentExtRe = regexp.MustCompile(`\.(pdf|docx?|xlsx?|pptx?)(\?.*)?$`)
+)
+
+// classifyType classifies a URL by file extension: image|video|document|article.
+func classifyType(target string) string {
+	lower := strings.ToLower(target)
+	switch {
+	case imageExtRe.MatchString(lower):
+		return "image"
+	case videoExtRe.MatchString(lower):
+		return "video"
+	case documentExtRe.MatchString(lower):
+		return "document"
+	default:
+		return "article"
+	}
+}