diff --git a/.github/workflows/update-themes.yml b/.github/workflows/update-themes.yml index 46542db..96ff79d 100644 --- a/.github/workflows/update-themes.yml +++ b/.github/workflows/update-themes.yml @@ -10,6 +10,7 @@ jobs: runs-on: ubuntu-latest env: HUGO_CACHEDIR: /tmp/hugo_cache + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} steps: - uses: actions/checkout@v3 - uses: actions/setup-go@v3 @@ -19,11 +20,11 @@ jobs: shell: bash working-directory: ./cmd/hugothemesitebuilder run: | - go run main.go build -configOnly - cd build git config --global user.email "bep@users.noreply.github.com" git config --global user.name "bep" go install github.com/gohugoio/hugo@${{ env.HUGO_VERSION }} + go run main.go build -skipSiteBuild -cleanCache + cd build hugo mod get hugo mod tidy git add . diff --git a/cmd/hugothemesitebuilder/build/cache/0000.githubrepos.json b/cmd/hugothemesitebuilder/build/cache/githubrepos.json similarity index 99% rename from cmd/hugothemesitebuilder/build/cache/0000.githubrepos.json rename to cmd/hugothemesitebuilder/build/cache/githubrepos.json index f32f569..3aa30d6 100644 --- a/cmd/hugothemesitebuilder/build/cache/0000.githubrepos.json +++ b/cmd/hugothemesitebuilder/build/cache/githubrepos.json @@ -596,11 +596,11 @@ "github.com/Vimux/mainroad": { "id": 76561407, "created_at": "2016-12-15T13:21:09Z", - "updated_at": "2023-10-23T11:01:20Z", + "updated_at": "2023-10-29T09:21:04Z", "name": "Mainroad", "description": "Responsive, simple, clean and content-focused Hugo theme based on the MH Magazine lite WordPress theme", "html_url": "https://github.com/Vimux/Mainroad", - "stargazers_count": 808 + "stargazers_count": 809 }, "github.com/WingLim/hugo-tania": { "id": 316264307, @@ -659,11 +659,11 @@ "github.com/adityatelange/hugo-PaperMod": { "id": 281297416, "created_at": "2020-07-21T04:45:10Z", - "updated_at": "2023-10-29T06:32:51Z", + "updated_at": "2023-10-29T10:48:51Z", "name": "hugo-PaperMod", "description": " A fast, clean, responsive Hugo theme.", "html_url": "https://github.com/adityatelange/hugo-PaperMod", - "stargazers_count": 7339 + "stargazers_count": 7340 }, "github.com/adityatelange/hugo-index": { "id": 625463549, @@ -740,11 +740,11 @@ "github.com/alex-shpak/hugo-book": { "id": 147530276, "created_at": "2018-09-05T14:21:31Z", - "updated_at": "2023-10-28T22:44:41Z", + "updated_at": "2023-10-29T11:49:35Z", "name": "hugo-book", "description": "Hugo documentation theme as simple as plain book", "html_url": "https://github.com/alex-shpak/hugo-book", - "stargazers_count": 2521 + "stargazers_count": 2522 }, "github.com/alexandrevicenzi/soho": { "id": 239386232, @@ -812,11 +812,11 @@ "github.com/apvarun/blist-hugo-theme": { "id": 386273576, "created_at": "2021-07-15T11:59:59Z", - "updated_at": "2023-10-21T20:18:58Z", + "updated_at": "2023-10-29T10:20:33Z", "name": "blist-hugo-theme", "description": "Blist is a clean and fast blog theme for your Hugo site.", "html_url": "https://github.com/apvarun/blist-hugo-theme", - "stargazers_count": 294 + "stargazers_count": 295 }, "github.com/apvarun/digital-garden-hugo-theme": { "id": 439232849, @@ -857,11 +857,11 @@ "github.com/athul/archie": { "id": 253427252, "created_at": "2020-04-06T07:35:38Z", - "updated_at": "2023-10-28T14:10:15Z", + "updated_at": "2023-10-29T10:38:14Z", "name": "archie", "description": "A minimal Hugo Theme", "html_url": "https://github.com/athul/archie", - "stargazers_count": 763 + "stargazers_count": 764 }, "github.com/austingebauer/devise": { "id": 193039721, @@ -1766,11 +1766,11 @@ "github.com/google/docsy": { "id": 153180924, "created_at": "2018-10-15T20:59:47Z", - "updated_at": "2023-10-26T02:49:40Z", + "updated_at": "2023-10-29T10:25:42Z", "name": "docsy", "description": "A set of Hugo doc templates for launching open source content.", "html_url": "https://github.com/google/docsy", - "stargazers_count": 2301 + "stargazers_count": 2302 }, "github.com/guangmean/Niello": { "id": 163953752, @@ -2063,11 +2063,11 @@ "github.com/imfing/hextra": { "id": 668453226, "created_at": "2023-07-19T21:03:57Z", - "updated_at": "2023-10-28T16:18:53Z", + "updated_at": "2023-10-29T11:39:34Z", "name": "hextra", "description": "🔯 Modern, batteries-included Hugo theme for creating beautiful doc, blog and static websites", "html_url": "https://github.com/imfing/hextra", - "stargazers_count": 101 + "stargazers_count": 102 }, "github.com/ineesalmeida/almeida-cv": { "id": 287938234, @@ -3008,11 +3008,11 @@ "github.com/onweru/newsroom": { "id": 202606752, "created_at": "2019-08-15T20:25:37Z", - "updated_at": "2023-10-26T03:07:00Z", + "updated_at": "2023-10-29T09:29:50Z", "name": "newsroom", "description": "A simple, minimalistic Hugo theme. View Demo here", "html_url": "https://github.com/onweru/newsroom", - "stargazers_count": 252 + "stargazers_count": 253 }, "github.com/opera7133/Blonde": { "id": 287219987, @@ -4025,20 +4025,20 @@ "github.com/wowchemy/starter-hugo-academic": { "id": 111604813, "created_at": "2017-11-21T21:47:01Z", - "updated_at": "2023-10-28T15:38:18Z", + "updated_at": "2023-10-29T11:02:11Z", "name": "starter-hugo-academic", "description": "🎓 无需编写任何代码即可轻松创建漂亮的学术网站 Easily create a beautiful academic résumé or educational website using Hugo and GitHub. No code.", "html_url": "https://github.com/wowchemy/starter-hugo-academic", - "stargazers_count": 3184 + "stargazers_count": 3185 }, "github.com/wowchemy/wowchemy-hugo-themes/modules/wowchemy/v5": { "id": 57165232, "created_at": "2016-04-26T22:09:15Z", - "updated_at": "2023-10-28T17:25:27Z", + "updated_at": "2023-10-29T11:01:39Z", "name": "wowchemy-hugo-themes", "description": "😍 EASILY BUILD THE WEBSITE YOU WANT - NO CODE, JUST MARKDOWN BLOCKS! 使用块轻松创建任何类型的网站 - 无需代码。 一个应用程序，没有依赖项，没有 JS", "html_url": "https://github.com/wowchemy/wowchemy-hugo-themes", - "stargazers_count": 7435 + "stargazers_count": 7437 }, "github.com/wtoll/venture": { "id": 374216323, diff --git a/pkg/buildcmd/build.go b/pkg/buildcmd/build.go index c3274c4..cea8a41 100644 --- a/pkg/buildcmd/build.go +++ b/pkg/buildcmd/build.go @@ -25,15 +25,18 @@ import ( "github.com/gohugoio/hugoThemesSiteBuilder/pkg/rootcmd" ) -// Config for the get subcommand. +// Config for the build subcommand. type Config struct { // Do not delete the old /content folder // This is useful when doing theme edits. Hugo gets confused // when the entire content vanishes. noClean bool - // Only rebuild the config.json file. - configOnly bool + // Skip the final site build. + skipSiteBuild bool + + // Set to true to remove the GitHub cache before building. + cleanCache bool rootConfig *rootcmd.Config } @@ -46,8 +49,8 @@ func New(rootConfig *rootcmd.Config) *ffcli.Command { fs := flag.NewFlagSet(rootcmd.CommandName+" build", flag.ExitOnError) fs.BoolVar(&cfg.noClean, "noClean", false, "do not clean out /content before building") - fs.BoolVar(&cfg.configOnly, "configOnly", false, "only build the config.json file") - + fs.BoolVar(&cfg.skipSiteBuild, "skipSiteBuild", false, "skip the final site build") + fs.BoolVar(&cfg.cleanCache, "cleanCache", false, "clean the GitHub cache before building") rootConfig.RegisterFlags(fs) return &ffcli.Command{ @@ -79,7 +82,15 @@ func (c *Config) Exec(ctx context.Context, args []string) error { return err } - if c.configOnly { + var err error + bc.mmap, err = bc.GetHugoModulesMap(configAll) + if err != nil { + return err + } + + bc.initGhRepos(c.cleanCache) + + if c.skipSiteBuild { return nil } @@ -88,12 +99,6 @@ func (c *Config) Exec(ctx context.Context, args []string) error { } client.CheckErr(os.MkdirAll(contentDir, 0o777)) - var err error - bc.mmap, err = bc.GetHugoModulesMap(configAll) - if err != nil { - return err - } - if err := bc.writeThemesContent(); err != nil { return err } @@ -128,9 +133,9 @@ func (c *buildClient) err(err error) { c.buildErrs = append(c.buildErrs, err) } -func (c *buildClient) getGitHubRepo(path string) client.GitHubRepo { +func (c *buildClient) initGhRepos(cleanCache bool) { c.ghReposInit.Do(func() { - ghRepos, err := c.GetGitHubRepos(c.mmap) + ghRepos, err := c.GetGitHubRepos(c.mmap, cleanCache) client.CheckErr(err) maxStars := 0 for _, ghRepo := range ghRepos { @@ -141,7 +146,10 @@ func (c *buildClient) getGitHubRepo(path string) client.GitHubRepo { c.maxStars = maxStars c.ghRepos = ghRepos }) +} +func (c *buildClient) getGitHubRepo(path string) client.GitHubRepo { + c.initGhRepos(false) return c.ghRepos[path] } diff --git a/pkg/client/client.go b/pkg/client/client.go index b6b0b09..700fdbf 100644 --- a/pkg/client/client.go +++ b/pkg/client/client.go @@ -13,8 +13,6 @@ import ( "os/exec" "path/filepath" "runtime" - "sort" - "strconv" "strings" "sync" "time" @@ -198,96 +196,53 @@ func (c *Client) TimeTrack(start time.Time, name string) { fmt.Fprintf(c.logWriter, "%s in %v ms\n", name, int(1000*elapsed.Seconds())) } -const cacheFileSuffix = "githubrepos.json" +const gitHubReposCacheFilename = "githubrepos.json" // GetGitHubRepos will first look in the chache folder for GitHub repo // information for mods. If not found, it will ask GitHub and then store // it in the cache. // // If you start with an empty cache, you will need to set a GITHUB_TOKEN environment variable. -func (c *Client) GetGitHubRepos(mods ModulesMap) (map[string]GitHubRepo, error) { +func (c *Client) GetGitHubRepos(mods ModulesMap, cleanCache bool) (map[string]GitHubRepo, error) { c.Logf("Get GitHub repos") defer c.TimeTrack(time.Now(), "Got GitHub repos") - cacheFiles := c.getGithubReposCacheFilesSorted() - m := make(map[string]GitHubRepo) - for _, cacheFile := range cacheFiles { - cacheFilename := filepath.Join(c.outDir, cacheDir, cacheFile) - b, err := os.ReadFile(cacheFilename) + cacheFilename := filepath.Join(c.outDir, cacheDir, gitHubReposCacheFilename) + if cleanCache { + os.Remove(cacheFilename) + } + b, err := os.ReadFile(cacheFilename) + if err != nil { + if !os.IsNotExist(err) { + return nil, err + } + + // Fetch the github repos and store in cache. + m, err := c.fetchGitHubRepos(mods) if err != nil { return nil, err } - m2 := make(map[string]GitHubRepo) - if err = json.Unmarshal(b, &m2); err != nil { - return nil, err - } - - for k, v := range m2 { - m[k] = v - } - } - - missing := ModulesMap{} - for k, v := range mods { - if _, found := m[k]; !found { - missing[k] = v - } - } - - if len(missing) > 0 { - cacheNum := 0 - if len(cacheFiles) > 0 { - last := cacheFiles[len(cacheFiles)-1] - cacheNum, _ = strconv.Atoi(last[:strings.Index(last, ".")]) - cacheNum++ - } - nextCacheFilename := filepath.Join(c.outDir, cacheDir, fmt.Sprintf("%0*d.%s", 4, cacheNum, cacheFileSuffix)) - m2, err := c.fetchGitHubRepos(missing) - if err != nil { - return nil, err - } - - if len(m2) > 0 { - b, err := json.MarshalIndent(m2, "", " ") + if len(m) > 0 { + b, err := json.MarshalIndent(m, "", " ") if err != nil { return nil, err } - - for k, v := range m2 { - m[k] = v - } - - CheckErr(os.MkdirAll(filepath.Dir(nextCacheFilename), 0o777)) - CheckErr(os.WriteFile(nextCacheFilename, b, 0o666)) + CheckErr(os.MkdirAll(filepath.Dir(cacheFilename), 0o777)) + CheckErr(os.WriteFile(cacheFilename, b, 0o666)) } + + return m, nil + } + + m := make(map[string]GitHubRepo) + + if err = json.Unmarshal(b, &m); err != nil { + return nil, err } return m, nil } -func (c *Client) getGithubReposCacheFilesSorted() []string { - cacheDir := filepath.Join(c.outDir, cacheDir) - CheckErr(os.MkdirAll(cacheDir, 0o777)) - fis, err := os.ReadDir(cacheDir) - CheckErr(err) - - var entries []string - - for _, fi := range fis { - if fi.IsDir() { - continue - } - - if strings.HasSuffix(fi.Name(), cacheFileSuffix) { - entries = append(entries, fi.Name()) - } - } - - sort.Strings(entries) - - return entries -} - func (c *Client) fetchGitHubRepo(m Module) (GitHubRepo, error) { var repo GitHubRepo