mirror of
https://github.com/tiennm99/MTTools.git
synced 2026-10-16 08:15:25 +00:00
tiennm99
fb0964bb6e
Port both scripts into one Go program with a command per behavior: gallery walks the numbered ghibli.jp film galleries, scrape downloads every image referenced by a page. Downloads now run through a bounded worker pool, and non-200 responses are no longer written to disk as image files. Drop the orphaned package-lock.json, which pinned the deprecated request dependency and was the source of the repository's Dependabot alerts.
35 lines
985 B
Go
35 lines
985 B
Go
package main
|
|
|
|
import (
|
|
"fmt"
|
|
"net/url"
|
|
"path/filepath"
|
|
)
|
|
|
|
// galleryJobs builds the download list for the numbered ghibli.jp galleries:
|
|
// for each film, images are named <film>001.jpg through <film><count>.jpg and
|
|
// are stored in a directory named after the film.
|
|
func galleryJobs(base string, films []string, count int, out string) ([]job, error) {
|
|
baseURL, err := url.Parse(base)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("parsing base URL %q: %w", base, err)
|
|
}
|
|
if baseURL.Scheme == "" || baseURL.Host == "" {
|
|
return nil, fmt.Errorf("base URL %q must be absolute", base)
|
|
}
|
|
|
|
jobs := make([]job, 0, len(films)*count)
|
|
for _, film := range films {
|
|
dir := filepath.Join(out, film)
|
|
for i := 1; i <= count; i++ {
|
|
name := fmt.Sprintf("%s%03d.jpg", film, i)
|
|
imageURL, err := url.JoinPath(base, name)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("building URL for %s: %w", name, err)
|
|
}
|
|
jobs = append(jobs, job{url: imageURL, dir: dir})
|
|
}
|
|
}
|
|
return jobs, nil
|
|
}
|