mirror of
https://github.com/tiennm99/MTTools.git
synced 2026-10-12 16:15:27 +00:00
refactor: make crawler importable and add export pipeline
Move monkeyd and pdfout out of internal/ so other modules can import them, and add export.Export, which holds the crawl-to-PDF sequence the CLI used to inline. The CLI now parses flags and delegates, so an embedding program gets the same defaults and validation.
This commit is contained in:
1 parent
2855ce5e53
commit
3633e96cf5
15 files changed
+490
-128
No files matched your search
@@ -104,10 +104,34 @@ chapter ordering, and the numbering gap. No network access required.
|
||||
|
||||
```
|
||||
cmd/monkeyd-crawler/ CLI
|
||||
internal/monkeyd/ fetching, HTML/CSS parsing, crawl orchestration
|
||||
internal/pdfout/ PDF rendering and font discovery
|
||||
export/ URL -> PDF in one call; shared by the CLI and importers
|
||||
monkeyd/ fetching, HTML/CSS parsing, crawl orchestration
|
||||
pdfout/ PDF rendering and font discovery
|
||||
```
|
||||
|
||||
## Use as a library
|
||||
|
||||
The packages are importable, so another Go program can produce the same PDF
|
||||
without shelling out to the binary. `export.Export` is the whole pipeline —
|
||||
chapter list, fetch, font discovery, render:
|
||||
|
||||
```go
|
||||
result, err := export.Export(ctx, export.Request{
|
||||
NovelURL: "https://monkeydd.com/tro-lai-nam-thang-cu.html",
|
||||
OutDir: tmpDir,
|
||||
})
|
||||
```
|
||||
|
||||
Only `NovelURL` is required; each zero-valued field falls back to the same
|
||||
default as the matching CLI flag. Because zero means "unset", ask for *no*
|
||||
cache or *no* request delay with the `NoCache` and `NoDelay` fields rather than
|
||||
by zeroing `CacheDir` or `Delay`. Pass a `Log` function to receive the progress
|
||||
messages the CLI prints to stderr.
|
||||
|
||||
Callers running in a container should note that `pdfout.FindFont` searches
|
||||
system font paths: a minimal image with no fonts installed needs either a font
|
||||
present or an explicit `FontFile`.
|
||||
|
||||
## Scope
|
||||
|
||||
Downloaded text stays on your machine; only fetch content you are allowed to read offline, and
|
||||
|
||||
@@ -6,34 +6,16 @@ import (
|
||||
"context"
|
||||
"flag"
|
||||
"fmt"
|
||||
"net/url"
|
||||
"os"
|
||||
"os/signal"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"strings"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/tiennm99/monkeyd-crawler/internal/monkeyd"
|
||||
"github.com/tiennm99/monkeyd-crawler/internal/pdfout"
|
||||
"github.com/tiennm99/monkeyd-crawler/export"
|
||||
"github.com/tiennm99/monkeyd-crawler/pdfout"
|
||||
)
|
||||
|
||||
type config struct {
|
||||
novelURL string
|
||||
out string
|
||||
page string
|
||||
fontFile string
|
||||
fontSize float64
|
||||
lineSpacing float64
|
||||
margin float64
|
||||
workers int
|
||||
delay time.Duration
|
||||
retries int
|
||||
limit int
|
||||
cacheDir string
|
||||
}
|
||||
|
||||
func main() {
|
||||
if err := run(); err != nil {
|
||||
fmt.Fprintln(os.Stderr, "error:", err)
|
||||
@@ -42,7 +24,7 @@ func main() {
|
||||
}
|
||||
|
||||
func run() error {
|
||||
cfg, err := parseFlags()
|
||||
req, err := parseFlags()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -51,78 +33,41 @@ func run() error {
|
||||
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
|
||||
defer stop()
|
||||
|
||||
crawler := &monkeyd.Crawler{
|
||||
Client: monkeyd.NewClient(cfg.delay, cfg.retries),
|
||||
CacheDir: cfg.cacheDir,
|
||||
Workers: cfg.workers,
|
||||
Log: func(format string, args ...any) {
|
||||
fmt.Fprintf(os.Stderr, format+"\n", args...)
|
||||
},
|
||||
req.Log = func(format string, args ...any) {
|
||||
fmt.Fprintf(os.Stderr, format+"\n", args...)
|
||||
}
|
||||
|
||||
novel, err := crawler.Novel(ctx, cfg.novelURL)
|
||||
result, err := export.Export(ctx, *req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if cfg.limit > 0 && cfg.limit < len(novel.Chapters) {
|
||||
fmt.Fprintf(os.Stderr, "limiting to first %d of %d chapters\n", cfg.limit, len(novel.Chapters))
|
||||
novel.Chapters = novel.Chapters[:cfg.limit]
|
||||
}
|
||||
|
||||
chapters, err := crawler.Chapters(ctx, novel)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
fontFile := cfg.fontFile
|
||||
if fontFile == "" {
|
||||
if fontFile, err = pdfout.FindFont(); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
outPath := cfg.out
|
||||
if outPath == "" {
|
||||
outPath = safeFileName(novel.Title, novel.Slug) + ".pdf"
|
||||
}
|
||||
|
||||
opts := pdfout.Options{
|
||||
Page: pdfout.Presets[cfg.page],
|
||||
Margin: cfg.margin,
|
||||
FontFile: fontFile,
|
||||
FontSize: cfg.fontSize,
|
||||
LineSpacing: cfg.lineSpacing,
|
||||
Title: novel.Title,
|
||||
SourceURL: novel.URL,
|
||||
}
|
||||
if err := pdfout.Write(outPath, opts, toPDFChapters(chapters)); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
fmt.Fprintf(os.Stderr, "\n%s\n", monkeyd.Describe(novel, chapters))
|
||||
fmt.Fprintf(os.Stderr, "\n%s\n", result.Summary())
|
||||
fmt.Fprintf(os.Stderr, "font: %s at %.0fpt on %s page (%.0f x %.0f mm)\n",
|
||||
filepath.Base(fontFile), cfg.fontSize, opts.Page.Name, opts.Page.W, opts.Page.H)
|
||||
fmt.Println(outPath)
|
||||
filepath.Base(result.FontFile), req.FontSize, result.Page.Name, result.Page.W, result.Page.H)
|
||||
fmt.Println(result.Path)
|
||||
return nil
|
||||
}
|
||||
|
||||
func parseFlags() (*config, error) {
|
||||
cfg := &config{}
|
||||
// parseFlags builds the export request from the command line. Validation of the
|
||||
// resulting values lives in export.Export, so the CLI and an embedding program
|
||||
// reject the same inputs.
|
||||
func parseFlags() (*export.Request, error) {
|
||||
req := &export.Request{}
|
||||
|
||||
flag.StringVar(&cfg.novelURL, "url", "", "novel page URL, e.g. https://monkeydd.com/tro-lai-nam-thang-cu.html")
|
||||
flag.StringVar(&cfg.out, "out", "", "output PDF path (default: novel title)")
|
||||
flag.StringVar(&cfg.page, "page", "phone",
|
||||
flag.StringVar(&req.NovelURL, "url", "", "novel page URL, e.g. https://monkeydd.com/tro-lai-nam-thang-cu.html")
|
||||
flag.StringVar(&req.OutPath, "out", "", "output PDF path (default: novel title)")
|
||||
flag.StringVar(&req.Page, "page", export.DefaultPage,
|
||||
"page size: "+strings.Join(pdfout.PresetNames(), ", "))
|
||||
flag.StringVar(&cfg.fontFile, "font", "", "path to a .ttf font (default: a Vietnamese-capable system font)")
|
||||
flag.Float64Var(&cfg.fontSize, "font-size", 12, "body font size in points")
|
||||
flag.Float64Var(&cfg.lineSpacing, "line-spacing", 1.55, "line height as a multiple of font size")
|
||||
flag.Float64Var(&cfg.margin, "margin", 6, "page margin in millimetres")
|
||||
flag.IntVar(&cfg.workers, "workers", 4, "concurrent chapter fetches")
|
||||
flag.DurationVar(&cfg.delay, "delay", 400*time.Millisecond, "minimum delay between requests")
|
||||
flag.IntVar(&cfg.retries, "retries", 3, "retries per request")
|
||||
flag.IntVar(&cfg.limit, "limit", 0, "only fetch the first N chapters (0 = all)")
|
||||
flag.StringVar(&cfg.cacheDir, "cache", ".cache",
|
||||
flag.StringVar(&req.FontFile, "font", "", "path to a .ttf font (default: a Vietnamese-capable system font)")
|
||||
flag.Float64Var(&req.FontSize, "font-size", export.DefaultFontSize, "body font size in points")
|
||||
flag.Float64Var(&req.LineSpacing, "line-spacing", export.DefaultLineSpacing, "line height as a multiple of font size")
|
||||
flag.Float64Var(&req.Margin, "margin", export.DefaultMargin, "page margin in millimetres")
|
||||
flag.IntVar(&req.Workers, "workers", export.DefaultWorkers, "concurrent chapter fetches")
|
||||
flag.DurationVar(&req.Delay, "delay", export.DefaultDelay, "minimum delay between requests")
|
||||
flag.IntVar(&req.Retries, "retries", export.DefaultRetries, "retries per request")
|
||||
flag.IntVar(&req.Limit, "limit", 0, "only fetch the first N chapters (0 = all)")
|
||||
flag.StringVar(&req.CacheDir, "cache", export.DefaultCacheDir,
|
||||
"directory for cached pages, so re-exports need no requests (empty to disable)")
|
||||
|
||||
flag.Usage = func() {
|
||||
@@ -133,52 +78,14 @@ func parseFlags() (*config, error) {
|
||||
}
|
||||
flag.Parse()
|
||||
|
||||
if cfg.novelURL == "" {
|
||||
if req.NovelURL == "" {
|
||||
flag.Usage()
|
||||
return nil, fmt.Errorf("-url is required")
|
||||
}
|
||||
parsed, err := url.Parse(cfg.novelURL)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("invalid -url: %w", err)
|
||||
}
|
||||
if parsed.Scheme != "http" && parsed.Scheme != "https" {
|
||||
return nil, fmt.Errorf("invalid -url: want an http(s) URL, got %q", cfg.novelURL)
|
||||
}
|
||||
if _, ok := pdfout.Presets[cfg.page]; !ok {
|
||||
return nil, fmt.Errorf("unknown -page %q: want one of %s",
|
||||
cfg.page, strings.Join(pdfout.PresetNames(), ", "))
|
||||
}
|
||||
if cfg.fontSize <= 0 {
|
||||
return nil, fmt.Errorf("-font-size must be positive")
|
||||
}
|
||||
if cfg.lineSpacing <= 0 {
|
||||
return nil, fmt.Errorf("-line-spacing must be positive")
|
||||
}
|
||||
if cfg.margin < 0 {
|
||||
return nil, fmt.Errorf("-margin cannot be negative")
|
||||
}
|
||||
if cfg.workers < 1 {
|
||||
return nil, fmt.Errorf("-workers must be at least 1")
|
||||
}
|
||||
return cfg, nil
|
||||
}
|
||||
|
||||
func toPDFChapters(chapters []*monkeyd.Chapter) []pdfout.Chapter {
|
||||
out := make([]pdfout.Chapter, 0, len(chapters))
|
||||
for _, ch := range chapters {
|
||||
out = append(out, pdfout.Chapter{Heading: ch.Heading(), Paragraphs: ch.Paragraphs})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
var unsafeNameChars = regexp.MustCompile(`[^\p{L}\p{N}]+`)
|
||||
|
||||
// safeFileName builds a file name from the novel title, falling back to the
|
||||
// slug when the title has no usable characters.
|
||||
func safeFileName(title, fallback string) string {
|
||||
name := strings.Trim(unsafeNameChars.ReplaceAllString(title, "-"), "-")
|
||||
if name == "" {
|
||||
return fallback
|
||||
}
|
||||
return name
|
||||
// The flag defaults above are already applied, so a zero value here can
|
||||
// only come from the user asking for none. Say so explicitly: left as the
|
||||
// zero value, Export would read it as "unset" and restore the default.
|
||||
req.NoCache = req.CacheDir == ""
|
||||
req.NoDelay = req.Delay == 0
|
||||
return req, nil
|
||||
}
|
||||
@@ -0,0 +1,244 @@
|
||||
// Package export turns a novel URL into a PDF file. It holds the sequence the
|
||||
// CLI and any embedding program both need — resolve the chapter list, fetch the
|
||||
// chapters, pick a font, render the PDF — so neither has to reassemble it.
|
||||
package export
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/url"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/tiennm99/monkeyd-crawler/monkeyd"
|
||||
"github.com/tiennm99/monkeyd-crawler/pdfout"
|
||||
)
|
||||
|
||||
// Defaults for every tunable field of Request. Exported so a caller's own
|
||||
// flags or config can advertise the same values instead of restating them.
|
||||
const (
|
||||
DefaultPage = "phone"
|
||||
DefaultFontSize = 12.0
|
||||
DefaultLineSpacing = 1.55
|
||||
DefaultMargin = 6.0
|
||||
DefaultWorkers = 4
|
||||
DefaultDelay = 400 * time.Millisecond
|
||||
DefaultRetries = 3
|
||||
DefaultCacheDir = ".cache"
|
||||
)
|
||||
|
||||
// Request describes one export. Only NovelURL is required; every zero-valued
|
||||
// tunable falls back to its Default above, so a caller that only has a URL can
|
||||
// leave the rest alone.
|
||||
type Request struct {
|
||||
NovelURL string
|
||||
|
||||
// OutPath is the exact PDF path to write. When empty the file is named
|
||||
// after the novel title and placed in OutDir.
|
||||
OutPath string
|
||||
OutDir string
|
||||
|
||||
Page string // preset key: phone, a5, a4
|
||||
FontFile string // path to a .ttf; empty means discover a system font
|
||||
FontSize float64 // points
|
||||
LineSpacing float64 // multiple of font size
|
||||
Margin float64 // millimetres
|
||||
|
||||
Workers int
|
||||
Retries int
|
||||
|
||||
// Delay is the minimum spacing between requests. Set NoDelay to remove
|
||||
// the spacing rather than setting this to 0, which is read as "unset"
|
||||
// and gets the default back.
|
||||
Delay time.Duration
|
||||
NoDelay bool
|
||||
|
||||
// Limit caps the export to the first N chapters. 0 means every chapter.
|
||||
Limit int
|
||||
|
||||
// CacheDir stores raw pages so a re-export costs no requests. Set
|
||||
// NoCache to opt out rather than clearing this field, which would be
|
||||
// read as "unset" and get the default back.
|
||||
CacheDir string
|
||||
NoCache bool
|
||||
|
||||
// Log receives progress messages. Optional.
|
||||
Log func(format string, args ...any)
|
||||
}
|
||||
|
||||
// Result reports what was produced.
|
||||
type Result struct {
|
||||
Path string
|
||||
Title string
|
||||
SourceURL string
|
||||
Chapters int
|
||||
Words int
|
||||
FontFile string
|
||||
Page pdfout.PageSize
|
||||
}
|
||||
|
||||
// Summary renders a one-line description of the exported book.
|
||||
func (r *Result) Summary() string {
|
||||
return fmt.Sprintf("%s — %d chapters, %d words", r.Title, r.Chapters, r.Words)
|
||||
}
|
||||
|
||||
// Export fetches the novel at req.NovelURL and writes it as a PDF, returning
|
||||
// where it landed. The context bounds the whole crawl; cancelling it abandons
|
||||
// the run without leaving a partial PDF behind.
|
||||
func Export(ctx context.Context, req Request) (*Result, error) {
|
||||
req.applyDefaults()
|
||||
if err := req.validate(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
crawler := &monkeyd.Crawler{
|
||||
Client: monkeyd.NewClient(req.Delay, req.Retries),
|
||||
CacheDir: req.CacheDir,
|
||||
Workers: req.Workers,
|
||||
Log: req.Log,
|
||||
}
|
||||
|
||||
novel, err := crawler.Novel(ctx, req.NovelURL)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if req.Limit > 0 && req.Limit < len(novel.Chapters) {
|
||||
req.logf("limiting to first %d of %d chapters", req.Limit, len(novel.Chapters))
|
||||
novel.Chapters = novel.Chapters[:req.Limit]
|
||||
}
|
||||
|
||||
chapters, err := crawler.Chapters(ctx, novel)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
fontFile := req.FontFile
|
||||
if fontFile == "" {
|
||||
if fontFile, err = pdfout.FindFont(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
outPath := req.OutPath
|
||||
if outPath == "" {
|
||||
outPath = filepath.Join(req.OutDir, SafeFileName(novel.Title, novel.Slug)+".pdf")
|
||||
}
|
||||
|
||||
page := pdfout.Presets[req.Page]
|
||||
opts := pdfout.Options{
|
||||
Page: page,
|
||||
Margin: req.Margin,
|
||||
FontFile: fontFile,
|
||||
FontSize: req.FontSize,
|
||||
LineSpacing: req.LineSpacing,
|
||||
Title: novel.Title,
|
||||
SourceURL: novel.URL,
|
||||
}
|
||||
if err := pdfout.Write(outPath, opts, toPDFChapters(chapters)); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &Result{
|
||||
Path: outPath,
|
||||
Title: novel.Title,
|
||||
SourceURL: novel.URL,
|
||||
Chapters: len(chapters),
|
||||
Words: monkeyd.TotalWords(chapters),
|
||||
FontFile: fontFile,
|
||||
Page: page,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (r *Request) applyDefaults() {
|
||||
if r.Page == "" {
|
||||
r.Page = DefaultPage
|
||||
}
|
||||
if r.FontSize == 0 {
|
||||
r.FontSize = DefaultFontSize
|
||||
}
|
||||
if r.LineSpacing == 0 {
|
||||
r.LineSpacing = DefaultLineSpacing
|
||||
}
|
||||
if r.Margin == 0 {
|
||||
r.Margin = DefaultMargin
|
||||
}
|
||||
if r.Workers == 0 {
|
||||
r.Workers = DefaultWorkers
|
||||
}
|
||||
switch {
|
||||
case r.NoDelay:
|
||||
r.Delay = 0
|
||||
case r.Delay == 0:
|
||||
r.Delay = DefaultDelay
|
||||
}
|
||||
if r.Retries == 0 {
|
||||
r.Retries = DefaultRetries
|
||||
}
|
||||
switch {
|
||||
case r.NoCache:
|
||||
r.CacheDir = ""
|
||||
case r.CacheDir == "":
|
||||
r.CacheDir = DefaultCacheDir
|
||||
}
|
||||
}
|
||||
|
||||
// validate rejects a Request before any request is made, so a typo costs no
|
||||
// fetches. Call applyDefaults first: it checks the effective values.
|
||||
func (r *Request) validate() error {
|
||||
if r.NovelURL == "" {
|
||||
return fmt.Errorf("novel url is required")
|
||||
}
|
||||
parsed, err := url.Parse(r.NovelURL)
|
||||
if err != nil {
|
||||
return fmt.Errorf("invalid novel url: %w", err)
|
||||
}
|
||||
if parsed.Scheme != "http" && parsed.Scheme != "https" {
|
||||
return fmt.Errorf("invalid novel url: want an http(s) URL, got %q", r.NovelURL)
|
||||
}
|
||||
if _, ok := pdfout.Presets[r.Page]; !ok {
|
||||
return fmt.Errorf("unknown page %q: want one of %s",
|
||||
r.Page, strings.Join(pdfout.PresetNames(), ", "))
|
||||
}
|
||||
if r.FontSize <= 0 {
|
||||
return fmt.Errorf("font size must be positive")
|
||||
}
|
||||
if r.LineSpacing <= 0 {
|
||||
return fmt.Errorf("line spacing must be positive")
|
||||
}
|
||||
if r.Margin < 0 {
|
||||
return fmt.Errorf("margin cannot be negative")
|
||||
}
|
||||
if r.Workers < 1 {
|
||||
return fmt.Errorf("workers must be at least 1")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *Request) logf(format string, args ...any) {
|
||||
if r.Log != nil {
|
||||
r.Log(format, args...)
|
||||
}
|
||||
}
|
||||
|
||||
func toPDFChapters(chapters []*monkeyd.Chapter) []pdfout.Chapter {
|
||||
out := make([]pdfout.Chapter, 0, len(chapters))
|
||||
for _, ch := range chapters {
|
||||
out = append(out, pdfout.Chapter{Heading: ch.Heading(), Paragraphs: ch.Paragraphs})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
var unsafeNameChars = regexp.MustCompile(`[^\p{L}\p{N}]+`)
|
||||
|
||||
// SafeFileName builds a file name from the novel title, falling back to the
|
||||
// slug when the title has no usable characters. The result has no extension.
|
||||
func SafeFileName(title, fallback string) string {
|
||||
name := strings.Trim(unsafeNameChars.ReplaceAllString(title, "-"), "-")
|
||||
if name == "" {
|
||||
return fallback
|
||||
}
|
||||
return name
|
||||
}
|
||||
@@ -0,0 +1,187 @@
|
||||
package export
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestApplyDefaultsFillsUnsetFields(t *testing.T) {
|
||||
req := Request{NovelURL: "https://monkeydd.com/example.html"}
|
||||
req.applyDefaults()
|
||||
|
||||
if req.Page != DefaultPage {
|
||||
t.Errorf("Page = %q, want %q", req.Page, DefaultPage)
|
||||
}
|
||||
if req.FontSize != DefaultFontSize {
|
||||
t.Errorf("FontSize = %v, want %v", req.FontSize, DefaultFontSize)
|
||||
}
|
||||
if req.LineSpacing != DefaultLineSpacing {
|
||||
t.Errorf("LineSpacing = %v, want %v", req.LineSpacing, DefaultLineSpacing)
|
||||
}
|
||||
if req.Margin != DefaultMargin {
|
||||
t.Errorf("Margin = %v, want %v", req.Margin, DefaultMargin)
|
||||
}
|
||||
if req.Workers != DefaultWorkers {
|
||||
t.Errorf("Workers = %v, want %v", req.Workers, DefaultWorkers)
|
||||
}
|
||||
if req.Delay != DefaultDelay {
|
||||
t.Errorf("Delay = %v, want %v", req.Delay, DefaultDelay)
|
||||
}
|
||||
if req.Retries != DefaultRetries {
|
||||
t.Errorf("Retries = %v, want %v", req.Retries, DefaultRetries)
|
||||
}
|
||||
if req.CacheDir != DefaultCacheDir {
|
||||
t.Errorf("CacheDir = %q, want %q", req.CacheDir, DefaultCacheDir)
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplyDefaultsKeepsExplicitValues(t *testing.T) {
|
||||
req := Request{
|
||||
NovelURL: "https://monkeydd.com/example.html",
|
||||
Page: "a5",
|
||||
FontSize: 14,
|
||||
LineSpacing: 1.4,
|
||||
Margin: 10,
|
||||
Workers: 2,
|
||||
Delay: time.Second,
|
||||
Retries: 1,
|
||||
CacheDir: "/tmp/pages",
|
||||
}
|
||||
req.applyDefaults()
|
||||
|
||||
if req.Page != "a5" || req.FontSize != 14 || req.LineSpacing != 1.4 {
|
||||
t.Errorf("layout fields overwritten: %+v", req)
|
||||
}
|
||||
if req.Margin != 10 || req.Workers != 2 || req.Delay != time.Second || req.Retries != 1 {
|
||||
t.Errorf("fetch fields overwritten: %+v", req)
|
||||
}
|
||||
if req.CacheDir != "/tmp/pages" {
|
||||
t.Errorf("CacheDir = %q, want /tmp/pages", req.CacheDir)
|
||||
}
|
||||
}
|
||||
|
||||
// A zero Margin is indistinguishable from "unset", so it becomes the default.
|
||||
// NoCache and NoDelay exist precisely because "none" must survive that rule.
|
||||
func TestApplyDefaultsHonoursOptOuts(t *testing.T) {
|
||||
req := Request{NovelURL: "https://monkeydd.com/example.html", NoCache: true, NoDelay: true}
|
||||
req.applyDefaults()
|
||||
|
||||
if req.CacheDir != "" {
|
||||
t.Errorf("NoCache left CacheDir = %q, want empty", req.CacheDir)
|
||||
}
|
||||
if req.Delay != 0 {
|
||||
t.Errorf("NoDelay left Delay = %v, want 0", req.Delay)
|
||||
}
|
||||
}
|
||||
|
||||
// Opt-outs win over an explicitly set value so a caller cannot end up with both.
|
||||
func TestApplyDefaultsOptOutsOverrideExplicitValues(t *testing.T) {
|
||||
req := Request{
|
||||
NovelURL: "https://monkeydd.com/example.html",
|
||||
CacheDir: "/tmp/pages",
|
||||
NoCache: true,
|
||||
Delay: time.Second,
|
||||
NoDelay: true,
|
||||
}
|
||||
req.applyDefaults()
|
||||
|
||||
if req.CacheDir != "" {
|
||||
t.Errorf("CacheDir = %q, want empty", req.CacheDir)
|
||||
}
|
||||
if req.Delay != 0 {
|
||||
t.Errorf("Delay = %v, want 0", req.Delay)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidate(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
req Request
|
||||
wantErr bool
|
||||
}{
|
||||
{
|
||||
name: "defaults are valid",
|
||||
req: Request{NovelURL: "https://monkeydd.com/example.html"},
|
||||
},
|
||||
{
|
||||
name: "missing url",
|
||||
req: Request{},
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "non-http scheme",
|
||||
req: Request{NovelURL: "ftp://monkeydd.com/example.html"},
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "scheme-less url",
|
||||
req: Request{NovelURL: "monkeydd.com/example.html"},
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "unknown page preset",
|
||||
req: Request{NovelURL: "https://monkeydd.com/example.html", Page: "letter"},
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "negative font size",
|
||||
req: Request{NovelURL: "https://monkeydd.com/example.html", FontSize: -1},
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "negative line spacing",
|
||||
req: Request{NovelURL: "https://monkeydd.com/example.html", LineSpacing: -1},
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "negative margin",
|
||||
req: Request{NovelURL: "https://monkeydd.com/example.html", Margin: -1},
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "zero workers is filled by defaults, negative is not",
|
||||
req: Request{NovelURL: "https://monkeydd.com/example.html", Workers: -1},
|
||||
wantErr: true,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
req := tt.req
|
||||
req.applyDefaults()
|
||||
err := req.validate()
|
||||
if tt.wantErr && err == nil {
|
||||
t.Error("validate() = nil, want error")
|
||||
}
|
||||
if !tt.wantErr && err != nil {
|
||||
t.Errorf("validate() = %v, want nil", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestSafeFileName(t *testing.T) {
|
||||
tests := []struct {
|
||||
title string
|
||||
fallback string
|
||||
want string
|
||||
}{
|
||||
{"Trở Lại Năm Tháng Cũ", "slug", "Trở-Lại-Năm-Tháng-Cũ"},
|
||||
{"Chapter: One / Two", "slug", "Chapter-One-Two"},
|
||||
{" --- ", "slug", "slug"},
|
||||
{"", "slug", "slug"},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
if got := SafeFileName(tt.title, tt.fallback); got != tt.want {
|
||||
t.Errorf("SafeFileName(%q, %q) = %q, want %q", tt.title, tt.fallback, got, tt.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestResultSummary(t *testing.T) {
|
||||
r := &Result{Title: "Example", Chapters: 12, Words: 3400}
|
||||
want := "Example — 12 chapters, 3400 words"
|
||||
if got := r.Summary(); got != want {
|
||||
t.Errorf("Summary() = %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
Reference in new issue
Block a user