From e221d1927a9c04f799291ca55975a84de74244bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Erik=20Pedersen?= Date: Thu, 10 Sep 2026 13:20:35 +0200 Subject: [PATCH] Skip building codeberg.org themes for now They have been very unreliable lately, presumably due people moving repos over from GitHub for similar reasons. Oh, the irony. Note that we could build using a Go proxy, which would have fixed the problem and made the builds faster, but people submitting the themes have not opted having their theme mirrored in a Go proxy server. --- README.md | 6 +++ netlify.toml | 3 ++ pkg/buildcmd/build.go | 33 +++++++++++- pkg/buildcmd/build_test.go | 9 ++++ pkg/client/client.go | 104 ++++++++++++++++++++++++++++++++++--- pkg/client/client_test.go | 62 ++++++++++++++++++++++ 6 files changed, 207 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index fb5f8bf..0a04cd4 100644 --- a/README.md +++ b/README.md @@ -180,6 +180,12 @@ The Netlify deploy preview can fail for a variety of reasons. The following step +# Maintainer notes + +## Git host outages + +Themes are fetched directly from their Git hosts (no Go module proxy), so an unstable host can fail the build. If a host is unstable for a longer period, its themes can be left out of the build by setting the `HUGO_THEMES_SKIP_HOSTS` environment variable (or the `-skipHosts` flag) to a comma separated list of hosts, e.g. `codeberg.org`. This is meant for the Netlify build (see `netlify.toml`); the themes will be missing from the site until the variable is removed again. Do not set it in the scheduled GitHub workflow, as that would remove the themes from the committed `go.mod`. + # Outdated themes Themes are removed if they haven't been updated in the last 18 months. Even if your theme works now, we strongly recommend you periodically check its compatibility with the latest version of Hugo. diff --git a/netlify.toml b/netlify.toml index b4b2f37..2e63114 100644 --- a/netlify.toml +++ b/netlify.toml @@ -12,3 +12,6 @@ [build.environment] HUGO_VERSION = "0.166.0" GO_VERSION = "1.26.0" + # Uncomment to leave the themes of one or more unstable Git hosts out + # of the build (they will be missing from the site until removed again). + HUGO_THEMES_SKIP_HOSTS = "codeberg.org" diff --git a/pkg/buildcmd/build.go b/pkg/buildcmd/build.go index cde7130..9de942b 100644 --- a/pkg/buildcmd/build.go +++ b/pkg/buildcmd/build.go @@ -38,9 +38,31 @@ type Config struct { // Set to true to remove the GitHub cache before building. cleanCache bool + // Comma separated list of Git hosts (e.g. "codeberg.org") to leave out + // of this build. Useful when a host is unstable. + skipHosts string + rootConfig *rootcmd.Config } +// skipHostsEnv is the environment variable that provides the default for +// the -skipHosts flag, so it can be set in e.g. the Netlify build settings. +const skipHostsEnv = "HUGO_THEMES_SKIP_HOSTS" + +func (c *Config) skipHostsList() []string { + return splitHosts(c.skipHosts) +} + +func splitHosts(s string) []string { + var hosts []string + for _, h := range strings.Split(s, ",") { + if h = strings.TrimSpace(h); h != "" { + hosts = append(hosts, h) + } + } + return hosts +} + // New returns a usable ffcli.Command for the get subcommand. func New(rootConfig *rootcmd.Config) *ffcli.Command { cfg := Config{ @@ -51,6 +73,7 @@ func New(rootConfig *rootcmd.Config) *ffcli.Command { fs.BoolVar(&cfg.noClean, "noClean", false, "do not clean out /content before building") fs.BoolVar(&cfg.skipSiteBuild, "skipSiteBuild", false, "skip the final site build") fs.BoolVar(&cfg.cleanCache, "cleanCache", false, "clean the GitHub cache before building") + fs.StringVar(&cfg.skipHosts, "skipHosts", os.Getenv(skipHostsEnv), "comma separated list of Git hosts (e.g. codeberg.org) to skip in this build (default from "+skipHostsEnv+")") rootConfig.RegisterFlags(fs) return &ffcli.Command{ @@ -119,10 +142,16 @@ func (c *Config) Exec(ctx context.Context, args []string) error { } } - if err := bc.CreateThemesConfig(); err != nil { + skipHosts := c.skipHostsList() + + if err := bc.CreateThemesConfig(skipHosts); err != nil { return err } + if err := bc.RemoveHostsFromGoMod(skipHosts); err != nil { + return fmt.Errorf("failed to remove skipped hosts from go.mod: %w", err) + } + var err error bc.mmap, err = bc.GetHugoModulesMap(configAll) if err != nil { @@ -153,7 +182,7 @@ func (c *Config) Exec(ctx context.Context, args []string) error { if err := bc.RemoveModulePathFromThemesTxt(failedModulePath); err != nil { return fmt.Errorf("failed to remove module %q from themes.txt: %w", failedModulePath, err) } - if err := bc.CreateThemesConfig(); err != nil { + if err := bc.CreateThemesConfig(skipHosts); err != nil { return err } } diff --git a/pkg/buildcmd/build_test.go b/pkg/buildcmd/build_test.go index 52c6439..4dae0f0 100644 --- a/pkg/buildcmd/build_test.go +++ b/pkg/buildcmd/build_test.go @@ -29,3 +29,12 @@ block content c.Assert(fixReadmeContent(s), qt.Equals, "\n{{%/* details \"summary title\" */%}}\n\nblock content\n\n{{%/* /details */%}}\n") } + +func TestSplitHosts(t *testing.T) { + c := qt.New(t) + + c.Assert(splitHosts(""), qt.IsNil) + c.Assert(splitHosts(" , "), qt.IsNil) + c.Assert(splitHosts("codeberg.org"), qt.DeepEquals, []string{"codeberg.org"}) + c.Assert(splitHosts(" codeberg.org, gitlab.com ,"), qt.DeepEquals, []string{"codeberg.org", "gitlab.com"}) +} diff --git a/pkg/client/client.go b/pkg/client/client.go index 8a8c339..3d05b44 100644 --- a/pkg/client/client.go +++ b/pkg/client/client.go @@ -195,7 +195,9 @@ func (c *Client) RemoveModulePathFromThemesTxt(module string) error { // CreateThemesConfig reads themes.txt and creates a config.json // suitable for Hugo. Note that we're only using that config to // get the full module listing. -func (c *Client) CreateThemesConfig() error { +// +// Themes hosted on any of skipHosts (e.g. "codeberg.org") are left out. +func (c *Client) CreateThemesConfig(skipHosts []string) error { // This looks a little funky, but we want the themes.txt to be // easily visible for users to add to in the root of the project. f, err := os.Open(c.themesTxtFilename()) @@ -207,23 +209,33 @@ func (c *Client) CreateThemesConfig() error { config := make(map[string]interface{}) var imports []map[string]interface{} + skipped := 0 scanner := bufio.NewScanner(f) for scanner.Scan() { line := strings.TrimSpace(scanner.Text()) - if !strings.HasPrefix(line, "#") { - imports = append(imports, map[string]interface{}{ - "path": line, - "ignoreImports": true, - "ignoreConfig": true, - "noMounts": true, - }) + if line == "" || strings.HasPrefix(line, "#") { + continue } + if hostMatches(line, skipHosts) { + skipped++ + continue + } + imports = append(imports, map[string]interface{}{ + "path": line, + "ignoreImports": true, + "ignoreConfig": true, + "noMounts": true, + }) } if err := scanner.Err(); err != nil { return err } + if skipped > 0 { + c.Logf("Skipping %d themes hosted on %s", skipped, strings.Join(skipHosts, ", ")) + } + config["module"] = map[string]interface{}{ "hugoVersion": map[string]interface{}{ "min": "0.115.0", @@ -239,6 +251,82 @@ func (c *Client) CreateThemesConfig() error { return os.WriteFile(filepath.Join(c.outDir, "config.json"), b, 0o666) } +// RemoveHostsFromGoMod drops the require directives for modules hosted on +// any of skipHosts from go.mod. This is needed in addition to leaving them +// out of the imports: Hugo runs "go mod download" for every module in go.mod, +// not only for the ones imported, so a host that is down would still fail +// the build. +// +// Note that this modifies go.mod in place, so only use it in throwaway +// builds (e.g. Netlify), never in the workflow that commits go.mod. +func (c *Client) RemoveHostsFromGoMod(skipHosts []string) error { + if len(skipHosts) == 0 { + return nil + } + + filename := filepath.Join(c.outDir, "go.mod") + b, err := os.ReadFile(filename) + if err != nil { + if os.IsNotExist(err) { + return nil + } + return err + } + + var ( + lines []string + removed int + inRequire bool + ) + + for _, line := range strings.Split(string(b), "\n") { + trimmed := strings.TrimSpace(line) + var modulePath string + + switch { + case trimmed == "require (": + inRequire = true + case inRequire && trimmed == ")": + inRequire = false + case inRequire: + if fields := strings.Fields(trimmed); len(fields) > 0 && !strings.HasPrefix(fields[0], "//") { + modulePath = fields[0] + } + case strings.HasPrefix(trimmed, "require "): + if fields := strings.Fields(trimmed); len(fields) > 1 { + modulePath = fields[1] + } + } + + if modulePath != "" && hostMatches(modulePath, skipHosts) { + removed++ + continue + } + + lines = append(lines, line) + } + + if removed == 0 { + return nil + } + + c.Logf("Removing %d modules hosted on %s from go.mod", removed, strings.Join(skipHosts, ", ")) + + return os.WriteFile(filename, []byte(strings.Join(lines, "\n")), 0o666) +} + +// hostMatches reports whether the host part of modulePath (e.g. "codeberg.org" +// in "codeberg.org/user/theme") is one of hosts. +func hostMatches(modulePath string, hosts []string) bool { + host, _, _ := strings.Cut(modulePath, "/") + for _, h := range hosts { + if strings.EqualFold(host, h) { + return true + } + } + return false +} + func (c *Client) JoinOutPath(elem ...string) string { return filepath.Join(append([]string{c.outDir}, elem...)...) } diff --git a/pkg/client/client_test.go b/pkg/client/client_test.go index 5e3a118..3a4f688 100644 --- a/pkg/client/client_test.go +++ b/pkg/client/client_test.go @@ -1,6 +1,9 @@ package client import ( + "io" + "os" + "path/filepath" "testing" qt "github.com/frankban/quicktest" @@ -26,3 +29,62 @@ func TestPathWithoutVersion(t *testing.T) { c.Assert(PathWithoutVersion("github.com/gohugoio/hugo/v2"), qt.Equals, "github.com/gohugoio/hugo") c.Assert(PathWithoutVersion("github.com/gohugoio/hugo"), qt.Equals, "github.com/gohugoio/hugo") } + +func TestHostMatches(t *testing.T) { + c := qt.New(t) + + hosts := []string{"codeberg.org", "gitlab.com"} + c.Assert(hostMatches("codeberg.org/user/theme", hosts), qt.IsTrue) + c.Assert(hostMatches("Codeberg.org/user/theme", hosts), qt.IsTrue) + c.Assert(hostMatches("gitlab.com/user/theme/v2", hosts), qt.IsTrue) + c.Assert(hostMatches("github.com/user/theme", hosts), qt.IsFalse) + c.Assert(hostMatches("codeberg.org", hosts), qt.IsTrue) + c.Assert(hostMatches("codeberg.org/user/theme", nil), qt.IsFalse) +} + +func TestRemoveHostsFromGoMod(t *testing.T) { + c := qt.New(t) + + dir := t.TempDir() + client := &Client{logWriter: io.Discard, outDir: dir} + + // No go.mod is fine. + c.Assert(client.RemoveHostsFromGoMod([]string{"codeberg.org"}), qt.IsNil) + + goMod := `module example.com/build + +go 1.27.0 + +require codeberg.org/single/theme v0.1.0 + +require ( + codeberg.org/user/theme v0.0.0-20260321101414-d37fb5f2a994 // indirect + // codeberg.org/user/commented v0.1.0 + github.com/user/theme v1.1.36 // indirect + gitlab.com/user/theme v0.2.0 // indirect +) +` + filename := filepath.Join(dir, "go.mod") + c.Assert(os.WriteFile(filename, []byte(goMod), 0o666), qt.IsNil) + + // Nothing to skip leaves the file untouched. + c.Assert(client.RemoveHostsFromGoMod(nil), qt.IsNil) + b, err := os.ReadFile(filename) + c.Assert(err, qt.IsNil) + c.Assert(string(b), qt.Equals, goMod) + + c.Assert(client.RemoveHostsFromGoMod([]string{"codeberg.org"}), qt.IsNil) + b, err = os.ReadFile(filename) + c.Assert(err, qt.IsNil) + c.Assert(string(b), qt.Equals, `module example.com/build + +go 1.27.0 + + +require ( + // codeberg.org/user/commented v0.1.0 + github.com/user/theme v1.1.36 // indirect + gitlab.com/user/theme v0.2.0 // indirect +) +`) +}