From e21c12eefe349b24c9f81b7b5d6d03857cb42ab5 Mon Sep 17 00:00:00 2001 From: tiennm99 Date: Fri, 18 Sep 2026 16:24:42 +0700 Subject: [PATCH] chore(newsletter): delete the Go engine Parity is proven and every call site now invokes the JavaScript engine, so the Go implementation has no callers. It remains recoverable from history. The newsletter engine was Go's only consumer here: the Hugo theme is vendored under themes/ and the site configuration declares no module block, so nothing else needs a toolchain. Dropping go.mod also disposes of a latent defect, since it declared a newer Go version than either build environment pinned and only worked because the toolchain was fetched on demand. The Go patterns in .gitignore go with it. --- .gitignore | 24 -- go.mod | 3 - scripts/newsletter/add_url.go | 138 --------- scripts/newsletter/detect_image_source.go | 63 ----- scripts/newsletter/fetch_via_defuddle.go | 51 ---- scripts/newsletter/find_newsletter_number.go | 77 ------ scripts/newsletter/find_substack_post.go | 202 -------------- scripts/newsletter/html_text.go | 116 -------- scripts/newsletter/list_existing_tags.go | 75 ----- scripts/newsletter/main.go | 68 ----- scripts/newsletter/post_stats.go | 101 ------- scripts/newsletter/url_utils.go | 277 ------------------- 12 files changed, 1195 deletions(-) delete mode 100644 go.mod delete mode 100644 scripts/newsletter/add_url.go delete mode 100644 scripts/newsletter/detect_image_source.go delete mode 100644 scripts/newsletter/fetch_via_defuddle.go delete mode 100644 scripts/newsletter/find_newsletter_number.go delete mode 100644 scripts/newsletter/find_substack_post.go delete mode 100644 scripts/newsletter/html_text.go delete mode 100644 scripts/newsletter/list_existing_tags.go delete mode 100644 scripts/newsletter/main.go delete mode 100644 scripts/newsletter/post_stats.go delete mode 100644 scripts/newsletter/url_utils.go diff --git a/.gitignore b/.gitignore index 08d705e..6cd7e1b 100644 --- a/.gitignore +++ b/.gitignore @@ -5,30 +5,6 @@ settings.local.json -# Compiled Object files, Static and Dynamic libs (Shared Objects) -*.o -*.a -*.so - -# Folders -_obj -_test - -# Architecture specific extensions/prefixes -*.[568vq] -[568vq].out - -*.cgo1.go -*.cgo2.c -_cgo_defun.c -_cgo_gotypes.go -_cgo_export.* - -_testmain.go - -*.exe -*.test - public themes resources diff --git a/go.mod b/go.mod deleted file mode 100644 index 08d10c8..0000000 --- a/go.mod +++ /dev/null @@ -1,3 +0,0 @@ -module github.com/tiennm99/blog - -go 1.26 diff --git a/scripts/newsletter/add_url.go b/scripts/newsletter/add_url.go deleted file mode 100644 index 70893af..0000000 --- a/scripts/newsletter/add_url.go +++ /dev/null @@ -1,138 +0,0 @@ -// Meta URL router for the mt-add-url skill — the single entry per URL. -// Usage: go run ./scripts/newsletter add-url "" -// Outputs: JSON { original_url, clean_url, http_status, accessible, -// -// duplicate, route, title?, author? } -// -// route ∈ youtube | image | video | document | article -package main - -import ( - "encoding/json" - "fmt" - "net/url" - "os" - "path/filepath" - "strings" - "time" -) - -// contentDir is repo-root relative (invocation contract: run from repo root). -func contentDir() string { - return filepath.Join("content", "post") -} - -var ytHosts = map[string]bool{ - "youtube.com": true, - "www.youtube.com": true, - "m.youtube.com": true, -} - -// detectYouTube extracts a video id from the supported URL shapes: -// youtube.com/watch?v=ID, youtu.be/ID, youtube.com/shorts/ID. -// Playlists/channels are intentionally NOT YouTube routes (fall through to type). -func detectYouTube(target string) (bool, string) { - u, err := url.Parse(target) - if err != nil { - return false, "" - } - host := strings.ToLower(u.Host) - if host == "youtu.be" { - id := strings.SplitN(strings.TrimPrefix(u.Path, "/"), "/", 2)[0] - return id != "", id - } - if ytHosts[host] { - if u.Path == "/watch" { - id := u.Query().Get("v") - return id != "", id - } - if strings.HasPrefix(u.Path, "/shorts/") { - parts := strings.Split(u.Path, "/") - if len(parts) > 2 && parts[2] != "" { - return true, parts[2] - } - } - } - return false, "" -} - -// canonicalWatchURL — oEmbed accepts watch URLs reliably for all shapes. -func canonicalWatchURL(videoID string) string { - return "https://www.youtube.com/watch?v=" + videoID -} - -// fetchYouTubeMeta fetches title/author via YouTube oEmbed (no API key). -// Best-effort: any failure returns empty strings so the route stays `youtube` -// and the skill can fall back. -func fetchYouTubeMeta(watchURL string) (title, author string) { - endpoint := "https://www.youtube.com/oembed?url=" + url.QueryEscape(watchURL) + "&format=json" - body := fetchTextOK(endpoint, 10*time.Second, "") - if body == "" { - return "", "" - } - var data struct { - Title string `json:"title"` - Author string `json:"author_name"` - } - if json.Unmarshal([]byte(body), &data) != nil { - return "", "" - } - return data.Title, data.Author -} - -type addURLOutput struct { - OriginalURL string `json:"original_url"` - CleanURL string `json:"clean_url"` - HTTPStatus string `json:"http_status"` - Accessible bool `json:"accessible"` - Duplicate bool `json:"duplicate"` - Route string `json:"route"` - Title string `json:"title,omitempty"` - Author string `json:"author,omitempty"` -} - -func runAddURL(args []string) { - if len(args) < 1 || args[0] == "" { - fmt.Fprintln(os.Stderr, "Usage: go run ./scripts/newsletter add-url ") - os.Exit(1) - } - target := args[0] - - cleaned := cleanURL(target) - isYT, videoID := detectYouTube(cleaned) - - // For YouTube, dedup/store against the canonical watch URL so youtu.be and - // shorts links collapse onto the same identity-param key as watch URLs. - effectiveURL := cleaned - if isYT { - effectiveURL = canonicalWatchURL(videoID) - } - - // Route order: YouTube → Substack image (by host, not extension, so f_auto / - // .avif / .heic / extensionless CDN URLs still route to the image handler) → - // file-extension classification. - var route string - switch { - case isYT: - route = "youtube" - case isSubstackImage(cleaned): - route = "image" - default: - route = classifyType(cleaned) - } - - httpStatus := checkAccessibility(cleaned) - - out := addURLOutput{ - OriginalURL: target, - CleanURL: effectiveURL, - HTTPStatus: httpStatus, - Accessible: httpStatus == "200", - Duplicate: checkDuplicate(effectiveURL, contentDir()), - Route: route, - } - if route == "youtube" { - out.Title, out.Author = fetchYouTubeMeta(effectiveURL) - } - printJSON(out) -} diff --git a/scripts/newsletter/detect_image_source.go b/scripts/newsletter/detect_image_source.go deleted file mode 100644 index 845b74a..0000000 --- a/scripts/newsletter/detect_image_source.go +++ /dev/null @@ -1,63 +0,0 @@ -// Detect whether an image URL is Substack-hosted and extract its S3 image UUID. -// Usage: go run ./scripts/newsletter detect-image-source "" -// Output: JSON { original_url, clean_url, isSubstack, uuid?, innerUrl? } -// -// Substack images are usually served via a CDN wrapper: -// -// https://substackcdn.com/image/fetch/$s_!x!,.../https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F_WxH.png -// -// The publication is NOT encoded in the URL — only the image identity (uuid) is. -package main - -import ( - "fmt" - "net/url" - "os" - "strings" -) - -// extractInnerURL pulls the inner S3 URL out of a substackcdn /image/fetch/ -// wrapper (if present). -func extractInnerURL(target string) string { - if marker := strings.Index(target, "/https%3A%2F%2F"); marker != -1 { - decoded, err := url.PathUnescape(target[marker+1:]) - if err != nil { - return target[marker+1:] - } - return decoded - } - // Some forms embed a plain (already-decoded) inner https URL. - if len(target) > 8 { - if plain := strings.Index(target[8:], "/https://"); plain != -1 { - return target[8+plain+1:] - } - } - return target -} - -type detectImageOutput struct { - OriginalURL string `json:"original_url"` - CleanURL string `json:"clean_url"` - IsSubstack bool `json:"isSubstack"` - UUID string `json:"uuid,omitempty"` - InnerURL string `json:"innerUrl,omitempty"` -} - -func runDetectImageSource(args []string) { - if len(args) < 1 || args[0] == "" { - fmt.Fprintln(os.Stderr, "Usage: go run ./scripts/newsletter detect-image-source ") - os.Exit(1) - } - target := args[0] - - out := detectImageOutput{ - OriginalURL: target, - CleanURL: cleanURL(target), - IsSubstack: isSubstackImage(target), - } - if out.IsSubstack { - out.UUID = substackImageUUID(target) - out.InnerURL = extractInnerURL(target) - } - printJSON(out) -} diff --git a/scripts/newsletter/fetch_via_defuddle.go b/scripts/newsletter/fetch_via_defuddle.go deleted file mode 100644 index da4d3e1..0000000 --- a/scripts/newsletter/fetch_via_defuddle.go +++ /dev/null @@ -1,51 +0,0 @@ -// mt-webfetch fallback fetcher via defuddle.md. -// Usage: go run ./scripts/newsletter fetch-via-defuddle -// Exit codes: 0 = content returned, 1 = empty/failed, 2 = bad args. -// Note: `go run` collapses any nonzero program exit to 1, so callers invoking -// through `go run` can only distinguish success from failure — the stderr -// diagnostics carry the detail. -package main - -import ( - "fmt" - "io" - "net/http" - "os" - "strings" - "time" -) - -func runFetchViaDefuddle(args []string) { - if len(args) < 1 || args[0] == "" { - fmt.Fprintln(os.Stderr, "Usage: go run ./scripts/newsletter fetch-via-defuddle ") - os.Exit(2) - } - target := args[0] - defuddleURL := "https://defuddle.md/" + target - - fail := func(err error) { - fmt.Fprintf(os.Stderr, "mt-webfetch: fetch failed for %s: %v\n", target, err) - os.Exit(1) - } - - req, err := http.NewRequest(http.MethodGet, defuddleURL, nil) - if err != nil { - fail(err) - } - req.Header.Set("User-Agent", "mt-webfetch/1.0") - res, err := (&http.Client{Timeout: 30 * time.Second}).Do(req) - if err != nil { - fail(err) - } - defer res.Body.Close() - body, err := io.ReadAll(res.Body) - if err != nil { - fail(err) - } - ok := res.StatusCode >= 200 && res.StatusCode < 300 - if !ok || strings.TrimSpace(string(body)) == "" { - fmt.Fprintf(os.Stderr, "mt-webfetch: defuddle returned %d / empty body for %s\n", res.StatusCode, target) - os.Exit(1) - } - os.Stdout.Write(body) -} diff --git a/scripts/newsletter/find_newsletter_number.go b/scripts/newsletter/find_newsletter_number.go deleted file mode 100644 index b30987f..0000000 --- a/scripts/newsletter/find_newsletter_number.go +++ /dev/null @@ -1,77 +0,0 @@ -// Find the most recent newsletter number and return the next one. -// Usage: go run ./scripts/newsletter find-newsletter-number -// Outputs: the next newsletter number. -package main - -import ( - "fmt" - "os" - "path/filepath" - "regexp" - "sort" - "strconv" -) - -var ( - newsletterNumRe = regexp.MustCompile(`Newsletter\s*#(\d+)`) - yearDirRe = regexp.MustCompile(`^\d{4}$`) - twoDigitDirRe = regexp.MustCompile(`^\d{2}$`) -) - -// listDirsDesc returns dir's subdirectory names matching re, sorted descending. -func listDirsDesc(dir string, re *regexp.Regexp) []string { - entries, err := os.ReadDir(dir) - if err != nil { - return nil - } - var names []string - for _, e := range entries { - if re.MatchString(e.Name()) { - names = append(names, e.Name()) - } - } - sort.Sort(sort.Reverse(sort.StringSlice(names))) - return names -} - -func extractNewsletterNumber(path string) int { - content, err := os.ReadFile(path) - if err != nil { - return 0 - } - m := newsletterNumRe.FindSubmatch(content) - if m == nil { - return 0 - } - n, err := strconv.Atoi(string(m[1])) - if err != nil { - return 0 - } - return n -} - -// findMostRecentNewsletter scans year/month/day directories newest-first for -// the highest newsletter number. -func findMostRecentNewsletter() int { - maxNumber := 0 - for _, year := range listDirsDesc(contentDir(), yearDirRe) { - yearDir := filepath.Join(contentDir(), year) - for _, month := range listDirsDesc(yearDir, twoDigitDirRe) { - monthDir := filepath.Join(yearDir, month) - for _, day := range listDirsDesc(monthDir, twoDigitDirRe) { - if n := extractNewsletterNumber(filepath.Join(monthDir, day, "index.md")); n > maxNumber { - maxNumber = n - } - } - } - // Early exit: a newsletter found in this year — no need to go further back. - if maxNumber > 0 { - break - } - } - return maxNumber -} - -func runFindNewsletterNumber(_ []string) { - fmt.Println(findMostRecentNewsletter() + 1) -} diff --git a/scripts/newsletter/find_substack_post.go b/scripts/newsletter/find_substack_post.go deleted file mode 100644 index f5d755f..0000000 --- a/scripts/newsletter/find_substack_post.go +++ /dev/null @@ -1,202 +0,0 @@ -// Find which Substack post embeds a given image UUID, and extract a label. -// Usage: go run ./scripts/newsletter find-substack-post --uuid [--deep] -// Output on hit: JSON { found:true, source, publication, postTitle, postUrl, caption, candidates } -// Output on miss: JSON { found:false } (RSS) or { found:false, source:"sitemap", scanned, budget, cutoff } (--deep) -// -// Strategy: RSS feed first (fast, ~recent weeks). With --deep, fall back to a -// heavier sitemap crawl up to ~3 months back — opt-in because it fetches many -// posts. A Substack CDN URL does not encode its publication, so we search each -// publication listed in config/substack-publications.json (embedded at build -// time; `go run` recompiles on change, so editing the JSON still takes effect). -package main - -import ( - _ "embed" - "encoding/json" - "flag" - "fmt" - "os" - "regexp" - "sort" - "strings" - "time" -) - -//go:embed config/substack-publications.json -var publicationsJSON []byte - -func loadPublications() []string { - var pubs []string - if err := json.Unmarshal(publicationsJSON, &pubs); err != nil || len(pubs) == 0 { - return []string{"blog.bytebytego.com"} - } - return pubs -} - -// fetchPage: body text with a browser-ish UA, or "" on any error. -func fetchPage(target string) string { - return fetchTextOK(target, 10*time.Second, "Mozilla/5.0") -} - -// Total post fetches allowed across ALL publications during a --deep crawl. -const deepFetchBudget = 40 - -type postHit struct { - Found bool `json:"found"` - Source string `json:"source"` - Publication string `json:"publication"` - PostTitle string `json:"postTitle"` - PostURL string `json:"postUrl"` - Caption string `json:"caption"` - Candidates []string `json:"candidates"` -} - -var ( - locRe = regexp.MustCompile(`([^<]+)`) - lastmodRe = regexp.MustCompile(`([^<]+)`) -) - -func parseLastmod(s string) (time.Time, bool) { - for _, layout := range []string{time.RFC3339, "2006-01-02"} { - if t, err := time.Parse(layout, s); err == nil { - return t, true - } - } - return time.Time{}, false -} - -// searchSitemap is the deep fallback: crawl the sitemap back ~3 months, fetch -// posts most-recent-first (up to maxFetch from the shared budget), and look -// for the UUID. Heavier than RSS — only used on RSS miss. -// cutoff is "" when the sitemap itself could not be fetched. -func searchSitemap(publication, id string, maxFetch int) (hit *postHit, scanned int, cutoff string) { - xml := fetchPage("https://" + publication + "/sitemap.xml") - if xml == "" { - return nil, 0, "" - } - - cutoffTime := time.Now().UTC().AddDate(0, -3, 0) - cutoff = cutoffTime.Format("2006-01-02") - - type candidate struct { - url string - when time.Time - } - var candidates []candidate - for _, block := range strings.Split(xml, "")[1:] { - loc := locRe.FindStringSubmatch(block) - lastmod := lastmodRe.FindStringSubmatch(block) - if loc == nil || lastmod == nil { - continue - } - if !strings.Contains(loc[1], "/p/") { // posts only - continue - } - when, ok := parseLastmod(lastmod[1]) - if !ok || when.Before(cutoffTime) { - continue - } - candidates = append(candidates, candidate{url: loc[1], when: when}) - } - sort.SliceStable(candidates, func(i, j int) bool { - return candidates[i].when.After(candidates[j].when) - }) - - if len(candidates) > maxFetch { - candidates = candidates[:maxFetch] - } - for _, c := range candidates { - scanned++ - html := fetchPage(c.url) - if html == "" || !strings.Contains(html, id) { - continue - } - return &postHit{ - Found: true, - Source: "sitemap", - Publication: publication, - PostTitle: postTitleFromHTML(html), - PostURL: c.url, - Caption: captionForUUID(html, id), - Candidates: extractCandidates(html), - }, scanned, cutoff - } - return nil, scanned, cutoff -} - -func searchRSS(publication, id string) *postHit { - xml := fetchPage("https://" + publication + "/feed") - if xml == "" { - return nil - } - for _, item := range strings.Split(xml, "")[1:] { - if !strings.Contains(item, id) { - continue - } - return &postHit{ - Found: true, - Source: "rss", - Publication: publication, - PostTitle: itemTitle(item), - PostURL: itemLink(item), - Caption: captionForUUID(item, id), - Candidates: extractCandidates(item), - } - } - return nil -} - -func runFindSubstackPost(args []string) { - fs := flag.NewFlagSet("find-substack-post", flag.ExitOnError) - uuid := fs.String("uuid", "", "Substack S3 image uuid to search for") - deep := fs.Bool("deep", false, "fall back to a sitemap crawl (~3 months back)") - _ = fs.Parse(args) - - if *uuid == "" { - fmt.Fprintln(os.Stderr, "Usage: go run ./scripts/newsletter find-substack-post --uuid [--deep]") - os.Exit(1) - } - - publications := loadPublications() - for _, pub := range publications { - if hit := searchRSS(pub, *uuid); hit != nil { - printJSON(hit) - return - } - } - - // Deep fallback: sitemap crawl up to ~3 months back, sharing one global - // fetch budget across all publications so coverage can't blow up as the - // publications list grows. - if *deep { - totalScanned := 0 - var lastCutoff *string - for _, pub := range publications { - remaining := deepFetchBudget - totalScanned - if remaining <= 0 { - break - } - hit, scanned, cutoff := searchSitemap(pub, *uuid, remaining) - totalScanned += scanned - if cutoff != "" { - lastCutoff = &cutoff - } - if hit != nil { - printJSON(hit) - return - } - } - printJSON(struct { - Found bool `json:"found"` - Source string `json:"source"` - Scanned int `json:"scanned"` - Budget int `json:"budget"` - Cutoff *string `json:"cutoff"` - }{false, "sitemap", totalScanned, deepFetchBudget, lastCutoff}) - return - } - - printJSON(struct { - Found bool `json:"found"` - }{false}) -} diff --git a/scripts/newsletter/html_text.go b/scripts/newsletter/html_text.go deleted file mode 100644 index df669a8..0000000 --- a/scripts/newsletter/html_text.go +++ /dev/null @@ -1,116 +0,0 @@ -// HTML / RSS text-extraction helpers for find-substack-post, ported from -// html-text-utils.js. Pure string functions — no network, no fs. -package main - -import ( - "html" - "regexp" - "strings" - "unicode/utf8" -) - -var ( - tagRe = regexp.MustCompile(`<[^>]+>`) - spaceRe = regexp.MustCompile(`\s+`) - titleTagRe = regexp.MustCompile(`(?s)(?:<!\[CDATA\[)?(.*?)(?:\]\]>)?`) - linkTagRe = regexp.MustCompile(`(?s)(?:)?`) - liRe = regexp.MustCompile(`(?s)]*>\s*(?:]*>)?(.*?)(?:

)?\s*`) - figcaptionRe = regexp.MustCompile(`(?s)]*>(.*?)`) - ogTitleRe = regexp.MustCompile(`(?i)]+property=["']og:title["'][^>]+content=["']([^"']+)["']`) - h1Re = regexp.MustCompile(`(?is)]*>(.*?)`) - htmlTitleRe = regexp.MustCompile(`(?is)(.*?)`) -) - -// decodeEntities decodes HTML entities (named, numeric, hex) and trims. -// html.UnescapeString covers a superset of the entity map the JS hand-rolled. -func decodeEntities(s string) string { - return strings.TrimSpace(html.UnescapeString(s)) -} - -func stripTags(s string) string { - return strings.TrimSpace(spaceRe.ReplaceAllString(decodeEntities(tagRe.ReplaceAllString(s, "")), " ")) -} - -// itemTitle pulls the first (CDATA or plain) from an RSS <item> chunk. -func itemTitle(item string) string { - m := titleTagRe.FindStringSubmatch(item) - if m == nil { - return "" - } - return decodeEntities(m[1]) -} - -// itemLink pulls the first <link> (CDATA or plain) from an RSS <item> chunk. -func itemLink(item string) string { - m := linkTagRe.FindStringSubmatch(item) - if m == nil { - return "" - } - return strings.TrimSpace(m[1]) -} - -// extractCandidates pulls candidate topic titles from a post's TOC bullet -// list. ByteByteGo does not attach captions to images — the topic titles live -// only in the "in this issue" bullets, and image→title cannot be mapped -// automatically (sponsor/video items interleave), so these are surfaced for -// the user to pick from. Light filtering keeps the list short: dedupe, drop -// sub-point explanations and over-long lines. -func extractCandidates(htmlSrc string) []string { - seen := map[string]bool{} - out := []string{} // non-nil: marshals as [] like the JS output - for _, m := range liRe.FindAllStringSubmatch(htmlSrc, -1) { - text := stripTags(m[1]) - if text == "" { - continue - } - // titles are short; long lines are sub-point explanations - if n := utf8.RuneCountInString(text); n < 6 || n > 70 { - continue - } - key := strings.ToLower(text) - if seen[key] { // content is duplicated in the page - continue - } - seen[key] = true - out = append(out, text) - } - return out -} - -// captionForUUID returns the <figcaption> text of the <figure> containing the -// UUID. Cover images live in <enclosure> (no figure) → "". -func captionForUUID(item, id string) string { - at := strings.Index(item, id) - if at == -1 { - return "" - } - figStart := strings.LastIndex(item[:at], "<figure") - if figStart == -1 { - return "" - } - rel := strings.Index(item[at:], "</figure>") - if rel == -1 { - return "" - } - figure := item[figStart : at+rel] - m := figcaptionRe.FindStringSubmatch(figure) - if m == nil { - return "" - } - return stripTags(m[1]) -} - -// postTitleFromHTML extracts a post title from server-rendered post HTML -// (og:title preferred, then <h1>, then <title>). -func postTitleFromHTML(htmlSrc string) string { - if m := ogTitleRe.FindStringSubmatch(htmlSrc); m != nil { - return decodeEntities(m[1]) - } - if m := h1Re.FindStringSubmatch(htmlSrc); m != nil { - return stripTags(m[1]) - } - if m := htmlTitleRe.FindStringSubmatch(htmlSrc); m != nil { - return decodeEntities(m[1]) - } - return "" -} diff --git a/scripts/newsletter/list_existing_tags.go b/scripts/newsletter/list_existing_tags.go deleted file mode 100644 index 0560a72..0000000 --- a/scripts/newsletter/list_existing_tags.go +++ /dev/null @@ -1,75 +0,0 @@ -// List existing tags in the repo ranked by frequency. -// Usage: go run ./scripts/newsletter list-existing-tags -// Outputs: tag count and name, sorted most-used first (top 40). -// -// NOTE: not currently used by the mt-add-tags skill. Tag normalization is -// disabled until existing posts have standardized tags; to enable, uncomment -// step 4a in that skill's SKILL.md. -package main - -import ( - "fmt" - "io/fs" - "os" - "path/filepath" - "regexp" - "sort" -) - -var ( - tagsLineRe = regexp.MustCompile(`(?m)^tags:\s*\[([^\]]*)\]`) - quotedTagRe = regexp.MustCompile(`"([^"]+)"`) -) - -// extractTags pulls quoted tag strings from an index.md frontmatter tags array. -func extractTags(path string) []string { - content, err := os.ReadFile(path) - if err != nil { - return nil - } - m := tagsLineRe.FindSubmatch(content) - if m == nil { - return nil - } - var tags []string - for _, q := range quotedTagRe.FindAllSubmatch(m[1], -1) { - tags = append(tags, string(q[1])) - } - return tags -} - -func runListExistingTags(_ []string) { - type tagCount struct { - tag string - count int - } - // Slice + index map keeps first-seen order for equal counts, so the stable - // sort below ranks ties deterministically (walk order is lexical). - var counts []tagCount - index := map[string]int{} - - _ = filepath.WalkDir(contentDir(), func(p string, d fs.DirEntry, err error) error { - if err != nil || d.IsDir() || d.Name() != "index.md" { - return nil - } - for _, tag := range extractTags(p) { - if i, ok := index[tag]; ok { - counts[i].count++ - } else { - index[tag] = len(counts) - counts = append(counts, tagCount{tag: tag, count: 1}) - } - } - return nil - }) - - sort.SliceStable(counts, func(i, j int) bool { - return counts[i].count > counts[j].count - }) - if len(counts) > 40 { - counts = counts[:40] - } - for _, tc := range counts { - fmt.Printf("%6d %s\n", tc.count, tc.tag) - } -} diff --git a/scripts/newsletter/main.go b/scripts/newsletter/main.go deleted file mode 100644 index b49941d..0000000 --- a/scripts/newsletter/main.go +++ /dev/null @@ -1,68 +0,0 @@ -// Newsletter engine for the mt-* skills — single binary, one subcommand per -// former Node script. Invoked from the repo root: -// -// go run ./scripts/newsletter <command> [args] -// -// Repo-relative paths (content/post) resolve from the working directory, so -// the repo-root invocation contract from AGENTS.md still applies. -package main - -import ( - "encoding/json" - "fmt" - "os" -) - -func usage() { - fmt.Fprint(os.Stderr, `Usage: go run ./scripts/newsletter <command> [args] - -Commands: - add-url <url> classify + dedup a URL, emit JSON route - find-newsletter-number print the next newsletter number - list-existing-tags tag frequencies, most-used first (top 40) - detect-image-source <url> detect Substack image + uuid - find-substack-post --uuid <uuid> [--deep] find the post embedding an image uuid - fetch-via-defuddle <url> fallback fetch via defuddle.md proxy - post-stats <path/to/index.md> count the post's articles/images/videos/documents -`) -} - -// printJSON mirrors console.log(JSON.stringify(v, null, 2)): 2-space indent, -// no HTML escaping (URLs with & must stay readable), trailing newline. -func printJSON(v any) { - enc := json.NewEncoder(os.Stdout) - enc.SetEscapeHTML(false) - enc.SetIndent("", " ") - if err := enc.Encode(v); err != nil { - fmt.Fprintln(os.Stderr, "json encode:", err) - os.Exit(1) - } -} - -func main() { - if len(os.Args) < 2 { - usage() - os.Exit(1) - } - args := os.Args[2:] - switch os.Args[1] { - case "add-url": - runAddURL(args) - case "find-newsletter-number": - runFindNewsletterNumber(args) - case "list-existing-tags": - runListExistingTags(args) - case "detect-image-source": - runDetectImageSource(args) - case "find-substack-post": - runFindSubstackPost(args) - case "fetch-via-defuddle": - runFetchViaDefuddle(args) - case "post-stats": - runPostStats(args) - default: - fmt.Fprintf(os.Stderr, "unknown command: %s\n", os.Args[1]) - usage() - os.Exit(1) - } -} diff --git a/scripts/newsletter/post_stats.go b/scripts/newsletter/post_stats.go deleted file mode 100644 index ab42d3b..0000000 --- a/scripts/newsletter/post_stats.go +++ /dev/null @@ -1,101 +0,0 @@ -// Count the entries already present in a newsletter post, so a handler can -// report a running tally after each insertion. -// Usage: go run ./scripts/newsletter post-stats <path/to/index.md> -// Outputs: JSON { post, newsletter, articles, images, videos, documents, total } -package main - -import ( - "fmt" - "os" - "regexp" - "strconv" - "strings" -) - -// Entry shapes, per the Bonus format in the shared post mechanics: -// -// articles "## [Title](url)" (level-2 heading, main content) -// images "![label](url)" (under **Images:**) -// videos "[Title](url)" (under **Videos:**) -// documents "[PDF: title](url)" (under **Documents:**) -var ( - articleHeadingRe = regexp.MustCompile(`^##\s+\[`) - bonusHeadingRe = regexp.MustCompile(`^###\s+Bonus\b`) - subsectionRe = regexp.MustCompile(`^\*\*(Images|Videos|Documents):\*\*`) - imageEntryRe = regexp.MustCompile(`^!\[`) - linkEntryRe = regexp.MustCompile(`^\[`) -) - -type postStats struct { - Post string `json:"post"` - Newsletter int `json:"newsletter"` - Articles int `json:"articles"` - Images int `json:"images"` - Videos int `json:"videos"` - Documents int `json:"documents"` - Total int `json:"total"` -} - -// countPostEntries walks the post once. Article headings are counted anywhere -// outside Bonus; asset entries are attributed to whichever subsection is open. -func countPostEntries(content string) postStats { - var s postStats - inBonus := false - subsection := "" - - for _, raw := range strings.Split(content, "\n") { - line := strings.TrimSpace(raw) - switch { - case bonusHeadingRe.MatchString(line): - inBonus = true - subsection = "" - continue - case subsectionRe.MatchString(line): - subsection = subsectionRe.FindStringSubmatch(line)[1] - continue - case articleHeadingRe.MatchString(line): - s.Articles++ - continue - } - if !inBonus { - continue - } - switch subsection { - case "Images": - if imageEntryRe.MatchString(line) { - s.Images++ - } - case "Videos": - // A direct video file entry looks the same as a YouTube entry; - // both belong to the Videos tally. - if linkEntryRe.MatchString(line) { - s.Videos++ - } - case "Documents": - if linkEntryRe.MatchString(line) { - s.Documents++ - } - } - } - s.Total = s.Articles + s.Images + s.Videos + s.Documents - return s -} - -func runPostStats(args []string) { - if len(args) < 1 { - fmt.Fprintln(os.Stderr, "usage: post-stats <path/to/index.md>") - os.Exit(1) - } - path := args[0] - content, err := os.ReadFile(path) - if err != nil { - fmt.Fprintln(os.Stderr, "read post:", err) - os.Exit(1) - } - stats := countPostEntries(string(content)) - stats.Post = path - if m := newsletterNumRe.FindSubmatch(content); m != nil { - stats.Newsletter, _ = strconv.Atoi(string(m[1])) - } - printJSON(stats) -} diff --git a/scripts/newsletter/url_utils.go b/scripts/newsletter/url_utils.go deleted file mode 100644 index 305a2f3..0000000 --- a/scripts/newsletter/url_utils.go +++ /dev/null @@ -1,277 +0,0 @@ -// Shared URL helpers, ported from url-utils.js. Owned by the add-url router; -// reused by the other subcommands. -package main - -import ( - "io" - "io/fs" - "net/http" - "net/url" - "os" - "path/filepath" - "regexp" - "strconv" - "strings" - "time" -) - -// Exact-match tracking params; any key starting with utm_ is also dropped. -var exactTracking = map[string]bool{ - "fbclid": true, "gclid": true, "msclkid": true, "mc_eid": true, - "aid": true, "ref": true, "ref_src": true, "ref_url": true, "source": true, "s": true, - "ck_subscriber_id": true, "igshid": true, "yclid": true, "vero_id": true, -} - -// cleanURL removes common tracking parameters. Surviving query pairs are kept -// verbatim (no re-encoding) and in their original order — Go's url.Values -// would sort keys alphabetically, which must not leak into stored clean_url -// values. Unparseable / non-absolute input is returned untouched rather than -// corrupted. -func cleanURL(raw string) string { - u, err := url.Parse(raw) - if err != nil || u.Scheme == "" || u.Host == "" { - return raw - } - u.Host = strings.ToLower(u.Host) - // WHATWG URL serializes an empty path as "/"; keep that shape so cleaned - // URLs match what the JS engine has already written into posts. - if u.Path == "" { - u.Path = "/" - } - if u.RawQuery != "" { - var kept []string - for _, pair := range strings.Split(u.RawQuery, "&") { - if pair == "" { - continue - } - key := pair - if i := strings.IndexByte(pair, '='); i != -1 { - key = pair[:i] - } - k := strings.ToLower(key) - if strings.HasPrefix(k, "utm_") || exactTracking[k] { - continue - } - kept = append(kept, pair) - } - u.RawQuery = strings.Join(kept, "&") - } - u.ForceQuery = false - return u.String() -} - -// --- Substack image helpers (shared by add-url routing and detect-image-source) --- - -var substackImageHosts = map[string]bool{ - "substackcdn.com": true, - "substack-post-media.s3.amazonaws.com": true, -} - -const uuidPattern = `[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}` - -var ( - imageUUIDRe = regexp.MustCompile(`(?i)images(?:%2F|/)(` + uuidPattern + `)`) - uuidExactRe = regexp.MustCompile(`(?i)^` + uuidPattern + `$`) -) - -// isSubstackImage reports a Substack-hosted image (CDN wrapper or raw S3), -// regardless of file extension. -func isSubstackImage(target string) bool { - host := "" - if u, err := url.Parse(target); err == nil { - host = strings.ToLower(u.Host) - } - return substackImageHosts[host] || strings.Contains(strings.ToLower(target), "substack-post-media") -} - -// substackImageUUID extracts the stable image identity: the S3 image UUID -// under public/images/<uuid>, with raw (/) or percent-encoded (%2F) separators. -// Returns "" when absent. -func substackImageUUID(target string) string { - m := imageUUIDRe.FindStringSubmatch(target) - if m == nil { - return "" - } - return strings.ToLower(m[1]) -} - -// Some sites carry the resource identity in a query param, not the path -// (e.g. YouTube /watch?v=ID). Preserve the identity param for those hosts so -// dedup does not collapse every video onto the same bare URL. -var identityParams = map[string]string{ - "youtube.com": "v", - "www.youtube.com": "v", - "m.youtube.com": "v", -} - -// bareURL reduces a URL to a stable identity for duplicate detection: -// - Substack image → its S3 UUID (transform/size variants share one identity) -// - YouTube → scheme+host+path + the v= video id -// - everything else → scheme + host + path -func bareURL(target string) string { - if isSubstackImage(target) { - if uuid := substackImageUUID(target); uuid != "" { - return uuid - } - } - u, err := url.Parse(target) - if err != nil || u.Scheme == "" || u.Host == "" { - return strings.TrimSuffix(strings.SplitN(target, "?", 2)[0], "/") - } - host := strings.ToLower(u.Host) - bare := strings.TrimSuffix(u.Scheme+"://"+host+u.EscapedPath(), "/") - if idParam, ok := identityParams[host]; ok { - if v := u.Query().Get(idParam); v != "" { - bare += "?" + idParam + "=" + v - } - } - return bare -} - -// fetchTextOK GETs target and returns the body on a 2xx response, "" on any -// error, non-2xx status, or timeout. Redirects are followed. -func fetchTextOK(target string, timeout time.Duration, userAgent string) string { - req, err := http.NewRequest(http.MethodGet, target, nil) - if err != nil { - return "" - } - if userAgent != "" { - req.Header.Set("User-Agent", userAgent) - } - res, err := (&http.Client{Timeout: timeout}).Do(req) - if err != nil { - return "" - } - defer res.Body.Close() - if res.StatusCode < 200 || res.StatusCode >= 300 { - return "" - } - body, err := io.ReadAll(res.Body) - if err != nil { - return "" - } - return string(body) -} - -// checkAccessibility HEADs the URL and returns the final HTTP status code as a -// string, or "000" on network error / timeout (matching the JS sentinel). -func checkAccessibility(target string) string { - req, err := http.NewRequest(http.MethodHead, target, nil) - if err != nil { - return "000" - } - res, err := (&http.Client{Timeout: 10 * time.Second}).Do(req) - if err != nil { - return "000" - } - res.Body.Close() - return strconv.Itoa(res.StatusCode) -} - -// collectMarkdown recursively collects *.md files under dir (the content tree -// is small). Missing or unreadable directories yield nothing. -func collectMarkdown(dir string) []string { - var acc []string - _ = filepath.WalkDir(dir, func(p string, d fs.DirEntry, err error) error { - if err != nil { - return nil // skip unreadable entries - } - if !d.IsDir() && strings.HasSuffix(strings.ToLower(d.Name()), ".md") { - acc = append(acc, p) - } - return nil - }) - return acc -} - -// uuidBoundaryOK: the char after a UUID match must not extend the hex id, so -// <uuid>.png (cover image), <uuid>_WxH and <uuid>) all match. -func uuidBoundaryOK(text string, end int) bool { - if end >= len(text) { - return true - } - c := text[end] - return !((c >= '0' && c <= '9') || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F')) -} - -// urlBoundaryOK: an optional trailing slash (bareURL strips it, stored URLs may -// keep it), then a path/punctuation delimiter, whitespace, or end of text — so -// /p/foo does not match a stored /p/foo-bar. The '>' delimiter covers URLs -// stored in markdown autolink form <https://…>, which older posts use. -func urlBoundaryOK(text string, end int) bool { - if end < len(text) && text[end] == '/' { - end++ - } - if end >= len(text) { - return true - } - switch c := text[end]; c { - case ' ', '\t', '\n', '\r', '\f', '\v': - return true - default: - return strings.IndexByte(`)]"'?#<>_&,`, c) != -1 - } -} - -// hasBoundaryMatch scans every occurrence of needle and applies the boundary -// check in code — Go's RE2 regexp has no lookahead, which the JS version used. -func hasBoundaryMatch(text, needle string, isUUID bool) bool { - for from := 0; ; { - i := strings.Index(text[from:], needle) - if i == -1 { - return false - } - end := from + i + len(needle) - if isUUID { - if uuidBoundaryOK(text, end) { - return true - } - } else if urlBoundaryOK(text, end) { - return true - } - from = from + i + 1 - } -} - -// checkDuplicate reports whether a URL identity already exists in the stored -// markdown, boundary-aware so a needle that is merely a PREFIX of a stored -// longer string is NOT a false duplicate. -func checkDuplicate(target, contentDir string) bool { - needle := bareURL(target) - if needle == "" { - return false - } - isUUID := uuidExactRe.MatchString(needle) - for _, file := range collectMarkdown(contentDir) { - data, err := os.ReadFile(file) - if err != nil { - continue - } - text := string(data) - if strings.Contains(text, needle) && hasBoundaryMatch(text, needle, isUUID) { - return true - } - } - return false -} - -var ( - imageExtRe = regexp.MustCompile(`\.(png|jpg|jpeg|gif|webp|svg|avif|heic|heif|bmp|tiff?)(\?.*)?$`) - videoExtRe = regexp.MustCompile(`\.(mp4|webm|mov|avi|mkv)(\?.*)?$`) - documentExtRe = regexp.MustCompile(`\.(pdf|docx?|xlsx?|pptx?)(\?.*)?$`) -) - -// classifyType classifies a URL by file extension: image|video|document|article. -func classifyType(target string) string { - lower := strings.ToLower(target) - switch { - case imageExtRe.MatchString(lower): - return "image" - case videoExtRe.MatchString(lower): - return "video" - case documentExtRe.MatchString(lower): - return "document" - default: - return "article" - } -}