diff --git a/.gitignore b/.gitignore index 08d705e..6cd7e1b 100644 --- a/.gitignore +++ b/.gitignore @@ -5,30 +5,6 @@ settings.local.json -# Compiled Object files, Static and Dynamic libs (Shared Objects) -*.o -*.a -*.so - -# Folders -_obj -_test - -# Architecture specific extensions/prefixes -*.[568vq] -[568vq].out - -*.cgo1.go -*.cgo2.c -_cgo_defun.c -_cgo_gotypes.go -_cgo_export.* - -_testmain.go - -*.exe -*.test - public themes resources diff --git a/go.mod b/go.mod deleted file mode 100644 index 08d10c8..0000000 --- a/go.mod +++ /dev/null @@ -1,3 +0,0 @@ -module github.com/tiennm99/blog - -go 1.26 diff --git a/scripts/newsletter/add_url.go b/scripts/newsletter/add_url.go deleted file mode 100644 index 70893af..0000000 --- a/scripts/newsletter/add_url.go +++ /dev/null @@ -1,138 +0,0 @@ -// Meta URL router for the mt-add-url skill — the single entry per URL. -// Usage: go run ./scripts/newsletter add-url "" -// Outputs: JSON { original_url, clean_url, http_status, accessible, -// -// duplicate, route, title?, author? } -// -// route ∈ youtube | image | video | document | article -package main - -import ( - "encoding/json" - "fmt" - "net/url" - "os" - "path/filepath" - "strings" - "time" -) - -// contentDir is repo-root relative (invocation contract: run from repo root). -func contentDir() string { - return filepath.Join("content", "post") -} - -var ytHosts = map[string]bool{ - "youtube.com": true, - "www.youtube.com": true, - "m.youtube.com": true, -} - -// detectYouTube extracts a video id from the supported URL shapes: -// youtube.com/watch?v=ID, youtu.be/ID, youtube.com/shorts/ID. -// Playlists/channels are intentionally NOT YouTube routes (fall through to type). -func detectYouTube(target string) (bool, string) { - u, err := url.Parse(target) - if err != nil { - return false, "" - } - host := strings.ToLower(u.Host) - if host == "youtu.be" { - id := strings.SplitN(strings.TrimPrefix(u.Path, "/"), "/", 2)[0] - return id != "", id - } - if ytHosts[host] { - if u.Path == "/watch" { - id := u.Query().Get("v") - return id != "", id - } - if strings.HasPrefix(u.Path, "/shorts/") { - parts := strings.Split(u.Path, "/") - if len(parts) > 2 && parts[2] != "" { - return true, parts[2] - } - } - } - return false, "" -} - -// canonicalWatchURL — oEmbed accepts watch URLs reliably for all shapes. -func canonicalWatchURL(videoID string) string { - return "https://www.youtube.com/watch?v=" + videoID -} - -// fetchYouTubeMeta fetches title/author via YouTube oEmbed (no API key). -// Best-effort: any failure returns empty strings so the route stays `youtube` -// and the skill can fall back. -func fetchYouTubeMeta(watchURL string) (title, author string) { - endpoint := "https://www.youtube.com/oembed?url=" + url.QueryEscape(watchURL) + "&format=json" - body := fetchTextOK(endpoint, 10*time.Second, "") - if body == "" { - return "", "" - } - var data struct { - Title string `json:"title"` - Author string `json:"author_name"` - } - if json.Unmarshal([]byte(body), &data) != nil { - return "", "" - } - return data.Title, data.Author -} - -type addURLOutput struct { - OriginalURL string `json:"original_url"` - CleanURL string `json:"clean_url"` - HTTPStatus string `json:"http_status"` - Accessible bool `json:"accessible"` - Duplicate bool `json:"duplicate"` - Route string `json:"route"` - Title string `json:"title,omitempty"` - Author string `json:"author,omitempty"` -} - -func runAddURL(args []string) { - if len(args) < 1 || args[0] == "" { - fmt.Fprintln(os.Stderr, "Usage: go run ./scripts/newsletter add-url ") - os.Exit(1) - } - target := args[0] - - cleaned := cleanURL(target) - isYT, videoID := detectYouTube(cleaned) - - // For YouTube, dedup/store against the canonical watch URL so youtu.be and - // shorts links collapse onto the same identity-param key as watch URLs. - effectiveURL := cleaned - if isYT { - effectiveURL = canonicalWatchURL(videoID) - } - - // Route order: YouTube → Substack image (by host, not extension, so f_auto / - // .avif / .heic / extensionless CDN URLs still route to the image handler) → - // file-extension classification. - var route string - switch { - case isYT: - route = "youtube" - case isSubstackImage(cleaned): - route = "image" - default: - route = classifyType(cleaned) - } - - httpStatus := checkAccessibility(cleaned) - - out := addURLOutput{ - OriginalURL: target, - CleanURL: effectiveURL, - HTTPStatus: httpStatus, - Accessible: httpStatus == "200", - Duplicate: checkDuplicate(effectiveURL, contentDir()), - Route: route, - } - if route == "youtube" { - out.Title, out.Author = fetchYouTubeMeta(effectiveURL) - } - printJSON(out) -} diff --git a/scripts/newsletter/detect_image_source.go b/scripts/newsletter/detect_image_source.go deleted file mode 100644 index 845b74a..0000000 --- a/scripts/newsletter/detect_image_source.go +++ /dev/null @@ -1,63 +0,0 @@ -// Detect whether an image URL is Substack-hosted and extract its S3 image UUID. -// Usage: go run ./scripts/newsletter detect-image-source "" -// Output: JSON { original_url, clean_url, isSubstack, uuid?, innerUrl? } -// -// Substack images are usually served via a CDN wrapper: -// -// https://substackcdn.com/image/fetch/$s_!x!,.../https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F_WxH.png -// -// The publication is NOT encoded in the URL — only the image identity (uuid) is. -package main - -import ( - "fmt" - "net/url" - "os" - "strings" -) - -// extractInnerURL pulls the inner S3 URL out of a substackcdn /image/fetch/ -// wrapper (if present). -func extractInnerURL(target string) string { - if marker := strings.Index(target, "/https%3A%2F%2F"); marker != -1 { - decoded, err := url.PathUnescape(target[marker+1:]) - if err != nil { - return target[marker+1:] - } - return decoded - } - // Some forms embed a plain (already-decoded) inner https URL. - if len(target) > 8 { - if plain := strings.Index(target[8:], "/https://"); plain != -1 { - return target[8+plain+1:] - } - } - return target -} - -type detectImageOutput struct { - OriginalURL string `json:"original_url"` - CleanURL string `json:"clean_url"` - IsSubstack bool `json:"isSubstack"` - UUID string `json:"uuid,omitempty"` - InnerURL string `json:"innerUrl,omitempty"` -} - -func runDetectImageSource(args []string) { - if len(args) < 1 || args[0] == "" { - fmt.Fprintln(os.Stderr, "Usage: go run ./scripts/newsletter detect-image-source ") - os.Exit(1) - } - target := args[0] - - out := detectImageOutput{ - OriginalURL: target, - CleanURL: cleanURL(target), - IsSubstack: isSubstackImage(target), - } - if out.IsSubstack { - out.UUID = substackImageUUID(target) - out.InnerURL = extractInnerURL(target) - } - printJSON(out) -} diff --git a/scripts/newsletter/fetch_via_defuddle.go b/scripts/newsletter/fetch_via_defuddle.go deleted file mode 100644 index da4d3e1..0000000 --- a/scripts/newsletter/fetch_via_defuddle.go +++ /dev/null @@ -1,51 +0,0 @@ -// mt-webfetch fallback fetcher via defuddle.md. -// Usage: go run ./scripts/newsletter fetch-via-defuddle -// Exit codes: 0 = content returned, 1 = empty/failed, 2 = bad args. -// Note: `go run` collapses any nonzero program exit to 1, so callers invoking -// through `go run` can only distinguish success from failure — the stderr -// diagnostics carry the detail. -package main - -import ( - "fmt" - "io" - "net/http" - "os" - "strings" - "time" -) - -func runFetchViaDefuddle(args []string) { - if len(args) < 1 || args[0] == "" { - fmt.Fprintln(os.Stderr, "Usage: go run ./scripts/newsletter fetch-via-defuddle ") - os.Exit(2) - } - target := args[0] - defuddleURL := "https://defuddle.md/" + target - - fail := func(err error) { - fmt.Fprintf(os.Stderr, "mt-webfetch: fetch failed for %s: %v\n", target, err) - os.Exit(1) - } - - req, err := http.NewRequest(http.MethodGet, defuddleURL, nil) - if err != nil { - fail(err) - } - req.Header.Set("User-Agent", "mt-webfetch/1.0") - res, err := (&http.Client{Timeout: 30 * time.Second}).Do(req) - if err != nil { - fail(err) - } - defer res.Body.Close() - body, err := io.ReadAll(res.Body) - if err != nil { - fail(err) - } - ok := res.StatusCode >= 200 && res.StatusCode < 300 - if !ok || strings.TrimSpace(string(body)) == "" { - fmt.Fprintf(os.Stderr, "mt-webfetch: defuddle returned %d / empty body for %s\n", res.StatusCode, target) - os.Exit(1) - } - os.Stdout.Write(body) -} diff --git a/scripts/newsletter/find_newsletter_number.go b/scripts/newsletter/find_newsletter_number.go deleted file mode 100644 index b30987f..0000000 --- a/scripts/newsletter/find_newsletter_number.go +++ /dev/null @@ -1,77 +0,0 @@ -// Find the most recent newsletter number and return the next one. -// Usage: go run ./scripts/newsletter find-newsletter-number -// Outputs: the next newsletter number. -package main - -import ( - "fmt" - "os" - "path/filepath" - "regexp" - "sort" - "strconv" -) - -var ( - newsletterNumRe = regexp.MustCompile(`Newsletter\s*#(\d+)`) - yearDirRe = regexp.MustCompile(`^\d{4}$`) - twoDigitDirRe = regexp.MustCompile(`^\d{2}$`) -) - -// listDirsDesc returns dir's subdirectory names matching re, sorted descending. -func listDirsDesc(dir string, re *regexp.Regexp) []string { - entries, err := os.ReadDir(dir) - if err != nil { - return nil - } - var names []string - for _, e := range entries { - if re.MatchString(e.Name()) { - names = append(names, e.Name()) - } - } - sort.Sort(sort.Reverse(sort.StringSlice(names))) - return names -} - -func extractNewsletterNumber(path string) int { - content, err := os.ReadFile(path) - if err != nil { - return 0 - } - m := newsletterNumRe.FindSubmatch(content) - if m == nil { - return 0 - } - n, err := strconv.Atoi(string(m[1])) - if err != nil { - return 0 - } - return n -} - -// findMostRecentNewsletter scans year/month/day directories newest-first for -// the highest newsletter number. -func findMostRecentNewsletter() int { - maxNumber := 0 - for _, year := range listDirsDesc(contentDir(), yearDirRe) { - yearDir := filepath.Join(contentDir(), year) - for _, month := range listDirsDesc(yearDir, twoDigitDirRe) { - monthDir := filepath.Join(yearDir, month) - for _, day := range listDirsDesc(monthDir, twoDigitDirRe) { - if n := extractNewsletterNumber(filepath.Join(monthDir, day, "index.md")); n > maxNumber { - maxNumber = n - } - } - } - // Early exit: a newsletter found in this year — no need to go further back. - if maxNumber > 0 { - break - } - } - return maxNumber -} - -func runFindNewsletterNumber(_ []string) { - fmt.Println(findMostRecentNewsletter() + 1) -} diff --git a/scripts/newsletter/find_substack_post.go b/scripts/newsletter/find_substack_post.go deleted file mode 100644 index f5d755f..0000000 --- a/scripts/newsletter/find_substack_post.go +++ /dev/null @@ -1,202 +0,0 @@ -// Find which Substack post embeds a given image UUID, and extract a label. -// Usage: go run ./scripts/newsletter find-substack-post --uuid [--deep] -// Output on hit: JSON { found:true, source, publication, postTitle, postUrl, caption, candidates } -// Output on miss: JSON { found:false } (RSS) or { found:false, source:"sitemap", scanned, budget, cutoff } (--deep) -// -// Strategy: RSS feed first (fast, ~recent weeks). With --deep, fall back to a -// heavier sitemap crawl up to ~3 months back — opt-in because it fetches many -// posts. A Substack CDN URL does not encode its publication, so we search each -// publication listed in config/substack-publications.json (embedded at build -// time; `go run` recompiles on change, so editing the JSON still takes effect). -package main - -import ( - _ "embed" - "encoding/json" - "flag" - "fmt" - "os" - "regexp" - "sort" - "strings" - "time" -) - -//go:embed config/substack-publications.json -var publicationsJSON []byte - -func loadPublications() []string { - var pubs []string - if err := json.Unmarshal(publicationsJSON, &pubs); err != nil || len(pubs) == 0 { - return []string{"blog.bytebytego.com"} - } - return pubs -} - -// fetchPage: body text with a browser-ish UA, or "" on any error. -func fetchPage(target string) string { - return fetchTextOK(target, 10*time.Second, "Mozilla/5.0") -} - -// Total post fetches allowed across ALL publications during a --deep crawl. -const deepFetchBudget = 40 - -type postHit struct { - Found bool `json:"found"` - Source string `json:"source"` - Publication string `json:"publication"` - PostTitle string `json:"postTitle"` - PostURL string `json:"postUrl"` - Caption string `json:"caption"` - Candidates []string `json:"candidates"` -} - -var ( - locRe = regexp.MustCompile(`([^<]+)`) - lastmodRe = regexp.MustCompile(`([^<]+)`) -) - -func parseLastmod(s string) (time.Time, bool) { - for _, layout := range []string{time.RFC3339, "2006-01-02"} { - if t, err := time.Parse(layout, s); err == nil { - return t, true - } - } - return time.Time{}, false -} - -// searchSitemap is the deep fallback: crawl the sitemap back ~3 months, fetch -// posts most-recent-first (up to maxFetch from the shared budget), and look -// for the UUID. Heavier than RSS — only used on RSS miss. -// cutoff is "" when the sitemap itself could not be fetched. -func searchSitemap(publication, id string, maxFetch int) (hit *postHit, scanned int, cutoff string) { - xml := fetchPage("https://" + publication + "/sitemap.xml") - if xml == "" { - return nil, 0, "" - } - - cutoffTime := time.Now().UTC().AddDate(0, -3, 0) - cutoff = cutoffTime.Format("2006-01-02") - - type candidate struct { - url string - when time.Time - } - var candidates []candidate - for _, block := range strings.Split(xml, "")[1:] { - loc := locRe.FindStringSubmatch(block) - lastmod := lastmodRe.FindStringSubmatch(block) - if loc == nil || lastmod == nil { - continue - } - if !strings.Contains(loc[1], "/p/") { // posts only - continue - } - when, ok := parseLastmod(lastmod[1]) - if !ok || when.Before(cutoffTime) { - continue - } - candidates = append(candidates, candidate{url: loc[1], when: when}) - } - sort.SliceStable(candidates, func(i, j int) bool { - return candidates[i].when.After(candidates[j].when) - }) - - if len(candidates) > maxFetch { - candidates = candidates[:maxFetch] - } - for _, c := range candidates { - scanned++ - html := fetchPage(c.url) - if html == "" || !strings.Contains(html, id) { - continue - } - return &postHit{ - Found: true, - Source: "sitemap", - Publication: publication, - PostTitle: postTitleFromHTML(html), - PostURL: c.url, - Caption: captionForUUID(html, id), - Candidates: extractCandidates(html), - }, scanned, cutoff - } - return nil, scanned, cutoff -} - -func searchRSS(publication, id string) *postHit { - xml := fetchPage("https://" + publication + "/feed") - if xml == "" { - return nil - } - for _, item := range strings.Split(xml, "")[1:] { - if !strings.Contains(item, id) { - continue - } - return &postHit{ - Found: true, - Source: "rss", - Publication: publication, - PostTitle: itemTitle(item), - PostURL: itemLink(item), - Caption: captionForUUID(item, id), - Candidates: extractCandidates(item), - } - } - return nil -} - -func runFindSubstackPost(args []string) { - fs := flag.NewFlagSet("find-substack-post", flag.ExitOnError) - uuid := fs.String("uuid", "", "Substack S3 image uuid to search for") - deep := fs.Bool("deep", false, "fall back to a sitemap crawl (~3 months back)") - _ = fs.Parse(args) - - if *uuid == "" { - fmt.Fprintln(os.Stderr, "Usage: go run ./scripts/newsletter find-substack-post --uuid [--deep]") - os.Exit(1) - } - - publications := loadPublications() - for _, pub := range publications { - if hit := searchRSS(pub, *uuid); hit != nil { - printJSON(hit) - return - } - } - - // Deep fallback: sitemap crawl up to ~3 months back, sharing one global - // fetch budget across all publications so coverage can't blow up as the - // publications list grows. - if *deep { - totalScanned := 0 - var lastCutoff *string - for _, pub := range publications { - remaining := deepFetchBudget - totalScanned - if remaining <= 0 { - break - } - hit, scanned, cutoff := searchSitemap(pub, *uuid, remaining) - totalScanned += scanned - if cutoff != "" { - lastCutoff = &cutoff - } - if hit != nil { - printJSON(hit) - return - } - } - printJSON(struct { - Found bool `json:"found"` - Source string `json:"source"` - Scanned int `json:"scanned"` - Budget int `json:"budget"` - Cutoff *string `json:"cutoff"` - }{false, "sitemap", totalScanned, deepFetchBudget, lastCutoff}) - return - } - - printJSON(struct { - Found bool `json:"found"` - }{false}) -} diff --git a/scripts/newsletter/html_text.go b/scripts/newsletter/html_text.go deleted file mode 100644 index df669a8..0000000 --- a/scripts/newsletter/html_text.go +++ /dev/null @@ -1,116 +0,0 @@ -// HTML / RSS text-extraction helpers for find-substack-post, ported from -// html-text-utils.js. Pure string functions — no network, no fs. -package main - -import ( - "html" - "regexp" - "strings" - "unicode/utf8" -) - -var ( - tagRe = regexp.MustCompile(`<[^>]+>`) - spaceRe = regexp.MustCompile(`\s+`) - titleTagRe = regexp.MustCompile(`(?s)(?:<!\[CDATA\[)?(.*?)(?:\]\]>)?`) - linkTagRe = regexp.MustCompile(`(?s)(?:)?`) - liRe = regexp.MustCompile(`(?s)]*>\s*(?:]*>)?(.*?)(?:

)?\s*`) - figcaptionRe = regexp.MustCompile(`(?s)]*>(.*?)`) - ogTitleRe = regexp.MustCompile(`(?i)]+property=["']og:title["'][^>]+content=["']([^"']+)["']`) - h1Re = regexp.MustCompile(`(?is)]*>(.*?)`) - htmlTitleRe = regexp.MustCompile(`(?is)(.*?)`) -) - -// decodeEntities decodes HTML entities (named, numeric, hex) and trims. -// html.UnescapeString covers a superset of the entity map the JS hand-rolled. -func decodeEntities(s string) string { - return strings.TrimSpace(html.UnescapeString(s)) -} - -func stripTags(s string) string { - return strings.TrimSpace(spaceRe.ReplaceAllString(decodeEntities(tagRe.ReplaceAllString(s, "")), " ")) -} - -// itemTitle pulls the first (CDATA or plain) from an RSS <item> chunk. -func itemTitle(item string) string { - m := titleTagRe.FindStringSubmatch(item) - if m == nil { - return "" - } - return decodeEntities(m[1]) -} - -// itemLink pulls the first <link> (CDATA or plain) from an RSS <item> chunk. -func itemLink(item string) string { - m := linkTagRe.FindStringSubmatch(item) - if m == nil { - return "" - } - return strings.TrimSpace(m[1]) -} - -// extractCandidates pulls candidate topic titles from a post's TOC bullet -// list. ByteByteGo does not attach captions to images — the topic titles live -// only in the "in this issue" bullets, and image→title cannot be mapped -// automatically (sponsor/video items interleave), so these are surfaced for -// the user to pick from. Light filtering keeps the list short: dedupe, drop -// sub-point explanations and over-long lines. -func extractCandidates(htmlSrc string) []string { - seen := map[string]bool{} - out := []string{} // non-nil: marshals as [] like the JS output - for _, m := range liRe.FindAllStringSubmatch(htmlSrc, -1) { - text := stripTags(m[1]) - if text == "" { - continue - } - // titles are short; long lines are sub-point explanations - if n := utf8.RuneCountInString(text); n < 6 || n > 70 { - continue - } - key := strings.ToLower(text) - if seen[key] { // content is duplicated in the page - continue - } - seen[key] = true - out = append(out, text) - } - return out -} - -// captionForUUID returns the <figcaption> text of the <figure> containing the -// UUID. Cover images live in <enclosure> (no figure) → "". -func captionForUUID(item, id string) string { - at := strings.Index(item, id) - if at == -1 { - return "" - } - figStart := strings.LastIndex(item[:at], "<figure") - if figStart == -1 { - return "" - } - rel := strings.Index(item[at:], "</figure>") - if rel == -1 { - return "" - } - figure := item[figStart : at+rel] - m := figcaptionRe.FindStringSubmatch(figure) - if m == nil { - return "" - } - return stripTags(m[1]) -} - -// postTitleFromHTML extracts a post title from server-rendered post HTML -// (og:title preferred, then <h1>, then <title>). -func postTitleFromHTML(htmlSrc string) string { - if m := ogTitleRe.FindStringSubmatch(htmlSrc); m != nil { - return decodeEntities(m[1]) - } - if m := h1Re.FindStringSubmatch(htmlSrc); m != nil { - return stripTags(m[1]) - } - if m := htmlTitleRe.FindStringSubmatch(htmlSrc); m != nil { - return decodeEntities(m[1]) - } - return "" -} diff --git a/scripts/newsletter/list_existing_tags.go b/scripts/newsletter/list_existing_tags.go deleted file mode 100644 index 0560a72..0000000 --- a/scripts/newsletter/list_existing_tags.go +++ /dev/null @@ -1,75 +0,0 @@ -// List existing tags in the repo ranked by frequency. -// Usage: go run ./scripts/newsletter list-existing-tags -// Outputs: tag count and name, sorted most-used first (top 40). -// -// NOTE: not currently used by the mt-add-tags skill. Tag normalization is -// disabled until existing posts have standardized tags; to enable, uncomment -// step 4a in that skill's SKILL.md. -package main - -import ( - "fmt" - "io/fs" - "os" - "path/filepath" - "regexp" - "sort" -) - -var ( - tagsLineRe = regexp.MustCompile(`(?m)^tags:\s*\[([^\]]*)\]`) - quotedTagRe = regexp.MustCompile(`"([^"]+)"`) -) - -// extractTags pulls quoted tag strings from an index.md frontmatter tags array. -func extractTags(path string) []string { - content, err := os.ReadFile(path) - if err != nil { - return nil - } - m := tagsLineRe.FindSubmatch(content) - if m == nil { - return nil - } - var tags []string - for _, q := range quotedTagRe.FindAllSubmatch(m[1], -1) { - tags = append(tags, string(q[1])) - } - return tags -} - -func runListExistingTags(_ []string) { - type tagCount struct { - tag string - count int - } - // Slice + index map keeps first-seen order for equal counts, so the stable - // sort below ranks ties deterministically (walk order is lexical). - var counts []tagCount - index := map[string]int{} - - _ = filepath.WalkDir(contentDir(), func(p string, d fs.DirEntry, err error) error { - if err != nil || d.IsDir() || d.Name() != "index.md" { - return nil - } - for _, tag := range extractTags(p) { - if i, ok := index[tag]; ok { - counts[i].count++ - } else { - index[tag] = len(counts) - counts = append(counts, tagCount{tag: tag, count: 1}) - } - } - return nil - }) - - sort.SliceStable(counts, func(i, j int) bool { - return counts[i].count > counts[j].count - }) - if len(counts) > 40 { - counts = counts[:40] - } - for _, tc := range counts { - fmt.Printf("%6d %s\n", tc.count, tc.tag) - } -} diff --git a/scripts/newsletter/main.go b/scripts/newsletter/main.go deleted file mode 100644 index b49941d..0000000 --- a/scripts/newsletter/main.go +++ /dev/null @@ -1,68 +0,0 @@ -// Newsletter engine for the mt-* skills — single binary, one subcommand per -// former Node script. Invoked from the repo root: -// -// go run ./scripts/newsletter <command> [args] -// -// Repo-relative paths (content/post) resolve from the working directory, so -// the repo-root invocation contract from AGENTS.md still applies. -package main - -import ( - "encoding/json" - "fmt" - "os" -) - -func usage() { - fmt.Fprint(os.Stderr, `Usage: go run ./scripts/newsletter <command> [args] - -Commands: - add-url <url> classify + dedup a URL, emit JSON route - find-newsletter-number print the next newsletter number - list-existing-tags tag frequencies, most-used first (top 40) - detect-image-source <url> detect Substack image + uuid - find-substack-post --uuid <uuid> [--deep] find the post embedding an image uuid - fetch-via-defuddle <url> fallback fetch via defuddle.md proxy - post-stats <path/to/index.md> count the post's articles/images/videos/documents -`) -} - -// printJSON mirrors console.log(JSON.stringify(v, null, 2)): 2-space indent, -// no HTML escaping (URLs with & must stay readable), trailing newline. -func printJSON(v any) { - enc := json.NewEncoder(os.Stdout) - enc.SetEscapeHTML(false) - enc.SetIndent("", " ") - if err := enc.Encode(v); err != nil { - fmt.Fprintln(os.Stderr, "json encode:", err) - os.Exit(1) - } -} - -func main() { - if len(os.Args) < 2 { - usage() - os.Exit(1) - } - args := os.Args[2:] - switch os.Args[1] { - case "add-url": - runAddURL(args) - case "find-newsletter-number": - runFindNewsletterNumber(args) - case "list-existing-tags": - runListExistingTags(args) - case "detect-image-source": - runDetectImageSource(args) - case "find-substack-post": - runFindSubstackPost(args) - case "fetch-via-defuddle": - runFetchViaDefuddle(args) - case "post-stats": - runPostStats(args) - default: - fmt.Fprintf(os.Stderr, "unknown command: %s\n", os.Args[1]) - usage() - os.Exit(1) - } -} diff --git a/scripts/newsletter/post_stats.go b/scripts/newsletter/post_stats.go deleted file mode 100644 index ab42d3b..0000000 --- a/scripts/newsletter/post_stats.go +++ /dev/null @@ -1,101 +0,0 @@ -// Count the entries already present in a newsletter post, so a handler can -// report a running tally after each insertion. -// Usage: go run ./scripts/newsletter post-stats <path/to/index.md> -// Outputs: JSON { post, newsletter, articles, images, videos, documents, total } -package main - -import ( - "fmt" - "os" - "regexp" - "strconv" - "strings" -) - -// Entry shapes, per the Bonus format in the shared post mechanics: -// -// articles "## [Title](url)" (level-2 heading, main content) -// images "![label](url)" (under **Images:**) -// videos "[Title](url)" (under **Videos:**) -// documents "[PDF: title](url)" (under **Documents:**) -var ( - articleHeadingRe = regexp.MustCompile(`^##\s+\[`) - bonusHeadingRe = regexp.MustCompile(`^###\s+Bonus\b`) - subsectionRe = regexp.MustCompile(`^\*\*(Images|Videos|Documents):\*\*`) - imageEntryRe = regexp.MustCompile(`^!\[`) - linkEntryRe = regexp.MustCompile(`^\[`) -) - -type postStats struct { - Post string `json:"post"` - Newsletter int `json:"newsletter"` - Articles int `json:"articles"` - Images int `json:"images"` - Videos int `json:"videos"` - Documents int `json:"documents"` - Total int `json:"total"` -} - -// countPostEntries walks the post once. Article headings are counted anywhere -// outside Bonus; asset entries are attributed to whichever subsection is open. -func countPostEntries(content string) postStats { - var s postStats - inBonus := false - subsection := "" - - for _, raw := range strings.Split(content, "\n") { - line := strings.TrimSpace(raw) - switch { - case bonusHeadingRe.MatchString(line): - inBonus = true - subsection = "" - continue - case subsectionRe.MatchString(line): - subsection = subsectionRe.FindStringSubmatch(line)[1] - continue - case articleHeadingRe.MatchString(line): - s.Articles++ - continue - } - if !inBonus { - continue - } - switch subsection { - case "Images": - if imageEntryRe.MatchString(line) { - s.Images++ - } - case "Videos": - // A direct video file entry looks the same as a YouTube entry; - // both belong to the Videos tally. - if linkEntryRe.MatchString(line) { - s.Videos++ - } - case "Documents": - if linkEntryRe.MatchString(line) { - s.Documents++ - } - } - } - s.Total = s.Articles + s.Images + s.Videos + s.Documents - return s -} - -func runPostStats(args []string) { - if len(args) < 1 { - fmt.Fprintln(os.Stderr, "usage: post-stats <path/to/index.md>") - os.Exit(1) - } - path := args[0] - content, err := os.ReadFile(path) - if err != nil { - fmt.Fprintln(os.Stderr, "read post:", err) - os.Exit(1) - } - stats := countPostEntries(string(content)) - stats.Post = path - if m := newsletterNumRe.FindSubmatch(content); m != nil { - stats.Newsletter, _ = strconv.Atoi(string(m[1])) - } - printJSON(stats) -} diff --git a/scripts/newsletter/url_utils.go b/scripts/newsletter/url_utils.go deleted file mode 100644 index 305a2f3..0000000 --- a/scripts/newsletter/url_utils.go +++ /dev/null @@ -1,277 +0,0 @@ -// Shared URL helpers, ported from url-utils.js. Owned by the add-url router; -// reused by the other subcommands. -package main - -import ( - "io" - "io/fs" - "net/http" - "net/url" - "os" - "path/filepath" - "regexp" - "strconv" - "strings" - "time" -) - -// Exact-match tracking params; any key starting with utm_ is also dropped. -var exactTracking = map[string]bool{ - "fbclid": true, "gclid": true, "msclkid": true, "mc_eid": true, - "aid": true, "ref": true, "ref_src": true, "ref_url": true, "source": true, "s": true, - "ck_subscriber_id": true, "igshid": true, "yclid": true, "vero_id": true, -} - -// cleanURL removes common tracking parameters. Surviving query pairs are kept -// verbatim (no re-encoding) and in their original order — Go's url.Values -// would sort keys alphabetically, which must not leak into stored clean_url -// values. Unparseable / non-absolute input is returned untouched rather than -// corrupted. -func cleanURL(raw string) string { - u, err := url.Parse(raw) - if err != nil || u.Scheme == "" || u.Host == "" { - return raw - } - u.Host = strings.ToLower(u.Host) - // WHATWG URL serializes an empty path as "/"; keep that shape so cleaned - // URLs match what the JS engine has already written into posts. - if u.Path == "" { - u.Path = "/" - } - if u.RawQuery != "" { - var kept []string - for _, pair := range strings.Split(u.RawQuery, "&") { - if pair == "" { - continue - } - key := pair - if i := strings.IndexByte(pair, '='); i != -1 { - key = pair[:i] - } - k := strings.ToLower(key) - if strings.HasPrefix(k, "utm_") || exactTracking[k] { - continue - } - kept = append(kept, pair) - } - u.RawQuery = strings.Join(kept, "&") - } - u.ForceQuery = false - return u.String() -} - -// --- Substack image helpers (shared by add-url routing and detect-image-source) --- - -var substackImageHosts = map[string]bool{ - "substackcdn.com": true, - "substack-post-media.s3.amazonaws.com": true, -} - -const uuidPattern = `[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}` - -var ( - imageUUIDRe = regexp.MustCompile(`(?i)images(?:%2F|/)(` + uuidPattern + `)`) - uuidExactRe = regexp.MustCompile(`(?i)^` + uuidPattern + `$`) -) - -// isSubstackImage reports a Substack-hosted image (CDN wrapper or raw S3), -// regardless of file extension. -func isSubstackImage(target string) bool { - host := "" - if u, err := url.Parse(target); err == nil { - host = strings.ToLower(u.Host) - } - return substackImageHosts[host] || strings.Contains(strings.ToLower(target), "substack-post-media") -} - -// substackImageUUID extracts the stable image identity: the S3 image UUID -// under public/images/<uuid>, with raw (/) or percent-encoded (%2F) separators. -// Returns "" when absent. -func substackImageUUID(target string) string { - m := imageUUIDRe.FindStringSubmatch(target) - if m == nil { - return "" - } - return strings.ToLower(m[1]) -} - -// Some sites carry the resource identity in a query param, not the path -// (e.g. YouTube /watch?v=ID). Preserve the identity param for those hosts so -// dedup does not collapse every video onto the same bare URL. -var identityParams = map[string]string{ - "youtube.com": "v", - "www.youtube.com": "v", - "m.youtube.com": "v", -} - -// bareURL reduces a URL to a stable identity for duplicate detection: -// - Substack image → its S3 UUID (transform/size variants share one identity) -// - YouTube → scheme+host+path + the v= video id -// - everything else → scheme + host + path -func bareURL(target string) string { - if isSubstackImage(target) { - if uuid := substackImageUUID(target); uuid != "" { - return uuid - } - } - u, err := url.Parse(target) - if err != nil || u.Scheme == "" || u.Host == "" { - return strings.TrimSuffix(strings.SplitN(target, "?", 2)[0], "/") - } - host := strings.ToLower(u.Host) - bare := strings.TrimSuffix(u.Scheme+"://"+host+u.EscapedPath(), "/") - if idParam, ok := identityParams[host]; ok { - if v := u.Query().Get(idParam); v != "" { - bare += "?" + idParam + "=" + v - } - } - return bare -} - -// fetchTextOK GETs target and returns the body on a 2xx response, "" on any -// error, non-2xx status, or timeout. Redirects are followed. -func fetchTextOK(target string, timeout time.Duration, userAgent string) string { - req, err := http.NewRequest(http.MethodGet, target, nil) - if err != nil { - return "" - } - if userAgent != "" { - req.Header.Set("User-Agent", userAgent) - } - res, err := (&http.Client{Timeout: timeout}).Do(req) - if err != nil { - return "" - } - defer res.Body.Close() - if res.StatusCode < 200 || res.StatusCode >= 300 { - return "" - } - body, err := io.ReadAll(res.Body) - if err != nil { - return "" - } - return string(body) -} - -// checkAccessibility HEADs the URL and returns the final HTTP status code as a -// string, or "000" on network error / timeout (matching the JS sentinel). -func checkAccessibility(target string) string { - req, err := http.NewRequest(http.MethodHead, target, nil) - if err != nil { - return "000" - } - res, err := (&http.Client{Timeout: 10 * time.Second}).Do(req) - if err != nil { - return "000" - } - res.Body.Close() - return strconv.Itoa(res.StatusCode) -} - -// collectMarkdown recursively collects *.md files under dir (the content tree -// is small). Missing or unreadable directories yield nothing. -func collectMarkdown(dir string) []string { - var acc []string - _ = filepath.WalkDir(dir, func(p string, d fs.DirEntry, err error) error { - if err != nil { - return nil // skip unreadable entries - } - if !d.IsDir() && strings.HasSuffix(strings.ToLower(d.Name()), ".md") { - acc = append(acc, p) - } - return nil - }) - return acc -} - -// uuidBoundaryOK: the char after a UUID match must not extend the hex id, so -// <uuid>.png (cover image), <uuid>_WxH and <uuid>) all match. -func uuidBoundaryOK(text string, end int) bool { - if end >= len(text) { - return true - } - c := text[end] - return !((c >= '0' && c <= '9') || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F')) -} - -// urlBoundaryOK: an optional trailing slash (bareURL strips it, stored URLs may -// keep it), then a path/punctuation delimiter, whitespace, or end of text — so -// /p/foo does not match a stored /p/foo-bar. The '>' delimiter covers URLs -// stored in markdown autolink form <https://…>, which older posts use. -func urlBoundaryOK(text string, end int) bool { - if end < len(text) && text[end] == '/' { - end++ - } - if end >= len(text) { - return true - } - switch c := text[end]; c { - case ' ', '\t', '\n', '\r', '\f', '\v': - return true - default: - return strings.IndexByte(`)]"'?#<>_&,`, c) != -1 - } -} - -// hasBoundaryMatch scans every occurrence of needle and applies the boundary -// check in code — Go's RE2 regexp has no lookahead, which the JS version used. -func hasBoundaryMatch(text, needle string, isUUID bool) bool { - for from := 0; ; { - i := strings.Index(text[from:], needle) - if i == -1 { - return false - } - end := from + i + len(needle) - if isUUID { - if uuidBoundaryOK(text, end) { - return true - } - } else if urlBoundaryOK(text, end) { - return true - } - from = from + i + 1 - } -} - -// checkDuplicate reports whether a URL identity already exists in the stored -// markdown, boundary-aware so a needle that is merely a PREFIX of a stored -// longer string is NOT a false duplicate. -func checkDuplicate(target, contentDir string) bool { - needle := bareURL(target) - if needle == "" { - return false - } - isUUID := uuidExactRe.MatchString(needle) - for _, file := range collectMarkdown(contentDir) { - data, err := os.ReadFile(file) - if err != nil { - continue - } - text := string(data) - if strings.Contains(text, needle) && hasBoundaryMatch(text, needle, isUUID) { - return true - } - } - return false -} - -var ( - imageExtRe = regexp.MustCompile(`\.(png|jpg|jpeg|gif|webp|svg|avif|heic|heif|bmp|tiff?)(\?.*)?$`) - videoExtRe = regexp.MustCompile(`\.(mp4|webm|mov|avi|mkv)(\?.*)?$`) - documentExtRe = regexp.MustCompile(`\.(pdf|docx?|xlsx?|pptx?)(\?.*)?$`) -) - -// classifyType classifies a URL by file extension: image|video|document|article. -func classifyType(target string) string { - lower := strings.ToLower(target) - switch { - case imageExtRe.MatchString(lower): - return "image" - case videoExtRe.MatchString(lower): - return "video" - case documentExtRe.MatchString(lower): - return "document" - default: - return "article" - } -}