Skip building codeberg.org themes for now

They have been very unreliable lately, presumably due people moving repos over from GitHub for similar reasons. Oh, the irony.

Note that we could build using a Go proxy, which would have fixed the problem and made the builds faster, but people submitting the themes have not opted having their theme mirrored in a Go proxy server.

README.md

@@ -180,6 +180,12 @@ The Netlify deploy preview can fail for a variety of reasons. The following step
    
    </details>
 
+# Maintainer notes
+
+## Git host outages
+
+Themes are fetched directly from their Git hosts (no Go module proxy), so an unstable host can fail the build. If a host is unstable for a longer period, its themes can be left out of the build by setting the `HUGO_THEMES_SKIP_HOSTS` environment variable (or the `-skipHosts` flag) to a comma separated list of hosts, e.g. `codeberg.org`. This is meant for the Netlify build (see `netlify.toml`); the themes will be missing from the site until the variable is removed again. Do not set it in the scheduled GitHub workflow, as that would remove the themes from the committed `go.mod`.
+
 # Outdated themes
 
 Themes are removed if they haven't been updated in the last 18 months. Even if your theme works now, we strongly recommend you periodically check its compatibility with the latest version of Hugo.

netlify.toml

@@ -12,3 +12,6 @@
   [build.environment]
     HUGO_VERSION = "0.166.0"
     GO_VERSION   = "1.26.0"
+    # Uncomment to leave the themes of one or more unstable Git hosts out
+    # of the build (they will be missing from the site until removed again).
+    HUGO_THEMES_SKIP_HOSTS = "codeberg.org"

pkg/buildcmd/build.go

@@ -38,9 +38,31 @@ type Config struct {
 	// Set to true to remove the GitHub cache before building.
 	cleanCache bool
 
+	// Comma separated list of Git hosts (e.g. "codeberg.org") to leave out
+	// of this build. Useful when a host is unstable.
+	skipHosts string
+
 	rootConfig *rootcmd.Config
 }
 
+// skipHostsEnv is the environment variable that provides the default for
+// the -skipHosts flag, so it can be set in e.g. the Netlify build settings.
+const skipHostsEnv = "HUGO_THEMES_SKIP_HOSTS"
+
+func (c *Config) skipHostsList() []string {
+	return splitHosts(c.skipHosts)
+}
+
+func splitHosts(s string) []string {
+	var hosts []string
+	for _, h := range strings.Split(s, ",") {
+		if h = strings.TrimSpace(h); h != "" {
+			hosts = append(hosts, h)
+		}
+	}
+	return hosts
+}
+
 // New returns a usable ffcli.Command for the get subcommand.
 func New(rootConfig *rootcmd.Config) *ffcli.Command {
 	cfg := Config{
@@ -51,6 +73,7 @@ func New(rootConfig *rootcmd.Config) *ffcli.Command {
 	fs.BoolVar(&cfg.noClean, "noClean", false, "do not clean out /content before building")
 	fs.BoolVar(&cfg.skipSiteBuild, "skipSiteBuild", false, "skip the final site build")
 	fs.BoolVar(&cfg.cleanCache, "cleanCache", false, "clean the GitHub cache before building")
+	fs.StringVar(&cfg.skipHosts, "skipHosts", os.Getenv(skipHostsEnv), "comma separated list of Git hosts (e.g. codeberg.org) to skip in this build (default from "+skipHostsEnv+")")
 	rootConfig.RegisterFlags(fs)
 
 	return &ffcli.Command{
@@ -119,10 +142,16 @@ func (c *Config) Exec(ctx context.Context, args []string) error {
 		}
 	}
 
-	if err := bc.CreateThemesConfig(); err != nil {
+	skipHosts := c.skipHostsList()
+
+	if err := bc.CreateThemesConfig(skipHosts); err != nil {
 		return err
 	}
 
+	if err := bc.RemoveHostsFromGoMod(skipHosts); err != nil {
+		return fmt.Errorf("failed to remove skipped hosts from go.mod: %w", err)
+	}
+
 	var err error
 	bc.mmap, err = bc.GetHugoModulesMap(configAll)
 	if err != nil {
@@ -153,7 +182,7 @@ func (c *Config) Exec(ctx context.Context, args []string) error {
 			if err := bc.RemoveModulePathFromThemesTxt(failedModulePath); err != nil {
 				return fmt.Errorf("failed to remove module %q from themes.txt: %w", failedModulePath, err)
 			}
-			if err := bc.CreateThemesConfig(); err != nil {
+			if err := bc.CreateThemesConfig(skipHosts); err != nil {
 				return err
 			}
 		}

pkg/buildcmd/build_test.go

@@ -29,3 +29,12 @@ block content
 
 	c.Assert(fixReadmeContent(s), qt.Equals, "\n{{%/* details \"summary title\" */%}}\n\nblock content\n\n{{%/* /details */%}}\n")
 }
+
+func TestSplitHosts(t *testing.T) {
+	c := qt.New(t)
+
+	c.Assert(splitHosts(""), qt.IsNil)
+	c.Assert(splitHosts(" , "), qt.IsNil)
+	c.Assert(splitHosts("codeberg.org"), qt.DeepEquals, []string{"codeberg.org"})
+	c.Assert(splitHosts(" codeberg.org, gitlab.com ,"), qt.DeepEquals, []string{"codeberg.org", "gitlab.com"})
+}

pkg/client/client.go

@@ -195,7 +195,9 @@ func (c *Client) RemoveModulePathFromThemesTxt(module string) error {
 // CreateThemesConfig reads themes.txt and creates a config.json
 // suitable for Hugo. Note that we're only using that config to
 // get the full module listing.
-func (c *Client) CreateThemesConfig() error {
+//
+// Themes hosted on any of skipHosts (e.g. "codeberg.org") are left out.
+func (c *Client) CreateThemesConfig(skipHosts []string) error {
 	// This looks a little funky, but we want the themes.txt to be
 	// easily visible for users to add to in the root of the project.
 	f, err := os.Open(c.themesTxtFilename())
@@ -207,23 +209,33 @@ func (c *Client) CreateThemesConfig() error {
 	config := make(map[string]interface{})
 	var imports []map[string]interface{}
 
+	skipped := 0
 	scanner := bufio.NewScanner(f)
 	for scanner.Scan() {
 		line := strings.TrimSpace(scanner.Text())
-		if !strings.HasPrefix(line, "#") {
-			imports = append(imports, map[string]interface{}{
-				"path":          line,
-				"ignoreImports": true,
-				"ignoreConfig":  true,
-				"noMounts":      true,
-			})
+		if line == "" || strings.HasPrefix(line, "#") {
+			continue
 		}
+		if hostMatches(line, skipHosts) {
+			skipped++
+			continue
+		}
+		imports = append(imports, map[string]interface{}{
+			"path":          line,
+			"ignoreImports": true,
+			"ignoreConfig":  true,
+			"noMounts":      true,
+		})
 	}
 
 	if err := scanner.Err(); err != nil {
 		return err
 	}
 
+	if skipped > 0 {
+		c.Logf("Skipping %d themes hosted on %s", skipped, strings.Join(skipHosts, ", "))
+	}
+
 	config["module"] = map[string]interface{}{
 		"hugoVersion": map[string]interface{}{
 			"min": "0.115.0",
@@ -239,6 +251,82 @@ func (c *Client) CreateThemesConfig() error {
 	return os.WriteFile(filepath.Join(c.outDir, "config.json"), b, 0o666)
 }
 
+// RemoveHostsFromGoMod drops the require directives for modules hosted on
+// any of skipHosts from go.mod. This is needed in addition to leaving them
+// out of the imports: Hugo runs "go mod download" for every module in go.mod,
+// not only for the ones imported, so a host that is down would still fail
+// the build.
+//
+// Note that this modifies go.mod in place, so only use it in throwaway
+// builds (e.g. Netlify), never in the workflow that commits go.mod.
+func (c *Client) RemoveHostsFromGoMod(skipHosts []string) error {
+	if len(skipHosts) == 0 {
+		return nil
+	}
+
+	filename := filepath.Join(c.outDir, "go.mod")
+	b, err := os.ReadFile(filename)
+	if err != nil {
+		if os.IsNotExist(err) {
+			return nil
+		}
+		return err
+	}
+
+	var (
+		lines     []string
+		removed   int
+		inRequire bool
+	)
+
+	for _, line := range strings.Split(string(b), "\n") {
+		trimmed := strings.TrimSpace(line)
+		var modulePath string
+
+		switch {
+		case trimmed == "require (":
+			inRequire = true
+		case inRequire && trimmed == ")":
+			inRequire = false
+		case inRequire:
+			if fields := strings.Fields(trimmed); len(fields) > 0 && !strings.HasPrefix(fields[0], "//") {
+				modulePath = fields[0]
+			}
+		case strings.HasPrefix(trimmed, "require "):
+			if fields := strings.Fields(trimmed); len(fields) > 1 {
+				modulePath = fields[1]
+			}
+		}
+
+		if modulePath != "" && hostMatches(modulePath, skipHosts) {
+			removed++
+			continue
+		}
+
+		lines = append(lines, line)
+	}
+
+	if removed == 0 {
+		return nil
+	}
+
+	c.Logf("Removing %d modules hosted on %s from go.mod", removed, strings.Join(skipHosts, ", "))
+
+	return os.WriteFile(filename, []byte(strings.Join(lines, "\n")), 0o666)
+}
+
+// hostMatches reports whether the host part of modulePath (e.g. "codeberg.org"
+// in "codeberg.org/user/theme") is one of hosts.
+func hostMatches(modulePath string, hosts []string) bool {
+	host, _, _ := strings.Cut(modulePath, "/")
+	for _, h := range hosts {
+		if strings.EqualFold(host, h) {
+			return true
+		}
+	}
+	return false
+}
+
 func (c *Client) JoinOutPath(elem ...string) string {
 	return filepath.Join(append([]string{c.outDir}, elem...)...)
 }

pkg/client/client_test.go

@@ -1,6 +1,9 @@
 package client
 
 import (
+	"io"
+	"os"
+	"path/filepath"
 	"testing"
 
 	qt "github.com/frankban/quicktest"
@@ -26,3 +29,62 @@ func TestPathWithoutVersion(t *testing.T) {
 	c.Assert(PathWithoutVersion("github.com/gohugoio/hugo/v2"), qt.Equals, "github.com/gohugoio/hugo")
 	c.Assert(PathWithoutVersion("github.com/gohugoio/hugo"), qt.Equals, "github.com/gohugoio/hugo")
 }
+
+func TestHostMatches(t *testing.T) {
+	c := qt.New(t)
+
+	hosts := []string{"codeberg.org", "gitlab.com"}
+	c.Assert(hostMatches("codeberg.org/user/theme", hosts), qt.IsTrue)
+	c.Assert(hostMatches("Codeberg.org/user/theme", hosts), qt.IsTrue)
+	c.Assert(hostMatches("gitlab.com/user/theme/v2", hosts), qt.IsTrue)
+	c.Assert(hostMatches("github.com/user/theme", hosts), qt.IsFalse)
+	c.Assert(hostMatches("codeberg.org", hosts), qt.IsTrue)
+	c.Assert(hostMatches("codeberg.org/user/theme", nil), qt.IsFalse)
+}
+
+func TestRemoveHostsFromGoMod(t *testing.T) {
+	c := qt.New(t)
+
+	dir := t.TempDir()
+	client := &Client{logWriter: io.Discard, outDir: dir}
+
+	// No go.mod is fine.
+	c.Assert(client.RemoveHostsFromGoMod([]string{"codeberg.org"}), qt.IsNil)
+
+	goMod := `module example.com/build
+
+go 1.27.0
+
+require codeberg.org/single/theme v0.1.0
+
+require (
+	codeberg.org/user/theme v0.0.0-20260321101414-d37fb5f2a994 // indirect
+	// codeberg.org/user/commented v0.1.0
+	github.com/user/theme v1.1.36 // indirect
+	gitlab.com/user/theme v0.2.0 // indirect
+)
+`
+	filename := filepath.Join(dir, "go.mod")
+	c.Assert(os.WriteFile(filename, []byte(goMod), 0o666), qt.IsNil)
+
+	// Nothing to skip leaves the file untouched.
+	c.Assert(client.RemoveHostsFromGoMod(nil), qt.IsNil)
+	b, err := os.ReadFile(filename)
+	c.Assert(err, qt.IsNil)
+	c.Assert(string(b), qt.Equals, goMod)
+
+	c.Assert(client.RemoveHostsFromGoMod([]string{"codeberg.org"}), qt.IsNil)
+	b, err = os.ReadFile(filename)
+	c.Assert(err, qt.IsNil)
+	c.Assert(string(b), qt.Equals, `module example.com/build
+
+go 1.27.0
+
+
+require (
+	// codeberg.org/user/commented v0.1.0
+	github.com/user/theme v1.1.36 // indirect
+	gitlab.com/user/theme v0.2.0 // indirect
+)
+`)
+}