Refactor

cmd/hugothemesitebuilder/build/cache/0002.githubrepos.json

@@ -0,0 +1 @@
+{"github.com/2-REC/hugo-myportfolio-theme":{"id":148112554,"name":"hugo-myportfolio-theme","description":"Adaptation of the HUGO Creative Theme to support several portfolios.","html_url":"https://github.com/2-REC/hugo-myportfolio-theme","stargazers_count":8},"github.com/AlexFinn/simple-a":{"id":20995016,"name":"simple-a","description":"Minimalistic Hugo theme","html_url":"https://github.com/alxschwarz/simple-a","stargazers_count":53},"github.com/AmazingRise/hugo-theme-diary":{"id":220989613,"name":"hugo-theme-diary","description":"Moments piled up. A Hugo theme ported from SumiMakito/hexo-theme-Journal.","html_url":"https://github.com/AmazingRise/hugo-theme-diary","stargazers_count":195},"github.com/AngeloStavrow/indigo":{"id":144703412,"name":"indigo","description":"An IndieWeb-friendly custom theme for Hugo","html_url":"https://github.com/AngeloStavrow/indigo","stargazers_count":52},"github.com/Chen-Zhe/photo-grid":{"id":240859368,"name":"photo-grid","description":"A Hugo theme for manage photography display in a dynamic grid layout","html_url":"https://github.com/Chen-Zhe/photo-grid","stargazers_count":23},"github.com/EmielH/hallo-hugo":{"id":155071757,"name":"hallo-hugo","description":"Hallo is a single-page Hugo theme to introduce yourself.","html_url":"https://github.com/EmielH/hallo-hugo","stargazers_count":84},"github.com/EmielH/stip-hugo":{"id":178725944,"name":"stip-hugo","description":"Stip is a single-page Hugo theme to introduce yourself, based on Material Design's tap target design pattern.","html_url":"https://github.com/EmielH/stip-hugo","stargazers_count":18},"github.com/EmielH/tale-hugo":{"id":142682503,"name":"tale-hugo","description":"A port of the Tale theme for Hugo. Tale is a minimal theme curated for storytellers.","html_url":"https://github.com/EmielH/tale-hugo","stargazers_count":199},"github.com/ExchangeRate-API/strange-case":{"id":66580052,"name":"strange-case","description":"Strange Case is a Hugo theme for people who like the Hyde theme ported from Jekyll but prefer using Bootstrap.","html_url":"https://github.com/ExchangeRate-API/strange-case","stargazers_count":26},"github.com/Fastbyte01/KeepIt":{"id":168930851,"name":"KeepIt","description":"The most powerful minimal Hugo theme.","html_url":"https://github.com/Fastbyte01/KeepIt","stargazers_count":160},"github.com/GDGToulouse/devfest-theme-hugo":{"id":165132952,"name":"devfest-theme-hugo","description":"A theme for a conference website. Created for DevFest Toulouse 2019","html_url":"https://github.com/GDGToulouse/devfest-theme-hugo","stargazers_count":65},"github.com/IvanChou/hugo-theme-vec":{"id":67712359,"name":"hugo-theme-vec","description":"Vec is a minimal, clean and beautiful theme for Hugo.","html_url":"https://github.com/yiichou/hugo-theme-vec","stargazers_count":37},"github.com/JugglerX/hugo-hero-theme":{"id":160591048,"name":"hugo-hero-theme","description":"A multi-page Hugo theme with fullscreen hero images and fullwidth sections.","html_url":"https://github.com/zerostaticthemes/hugo-hero-theme","stargazers_count":134},"github.com/JugglerX/hugo-serif-theme":{"id":159459907,"name":"hugo-serif-theme","description":"Serif is a modern business theme for Hugo.","html_url":"https://github.com/zerostaticthemes/hugo-serif-theme","stargazers_count":176},"github.com/JugglerX/hugo-whisper-theme":{"id":171089926,"name":"hugo-whisper-theme","description":"Whisper is a minimal documentation theme for Hugo.","html_url":"https://github.com/zerostaticthemes/hugo-whisper-theme","stargazers_count":166},"github.com/Lednerb/bilberry-hugo-theme":{"id":108488686,"name":"bilberry-hugo-theme","description":"Premium theme for the hugo site builder. DEMO:","html_url":"https://github.com/Lednerb/bilberry-hugo-theme","stargazers_count":218},"github.com/LordMathis/hugo-theme-nix":{"id":69647603,"name":"hugo-theme-nix","description":"Nix is a simple, minimal theme for Hugo","html_url":"https://github.com/LordMathis/hugo-theme-nix","stargazers_count":108},"github.com/MarcusVirg/forty":{"id":111167907,"name":"forty","description":"Forty theme - Hugo theme ported from HTML5UP origrinal theme called Forty.","html_url":"https://github.com/MarcusVirg/forty","stargazers_count":107},"github.com/MeiK2333/github-style":{"id":216473473,"name":"github-style","description":"","html_url":"https://github.com/MeiK2333/github-style","stargazers_count":193},"github.com/Mitrichius/hugo-theme-anubis":{"id":233350753,"name":"hugo-theme-anubis","description":"Anubis is a simple minimalist theme for Hugo blog engine","html_url":"https://github.com/Mitrichius/hugo-theme-anubis","stargazers_count":170},"github.com/MunifTanjim/minimo":{"id":91743121,"name":"minimo","description":"Minimo - Minimalist theme for Hugo","html_url":"https://github.com/MunifTanjim/minimo","stargazers_count":463},"github.com/NormandErwan/Blogpaper":{"id":219223151,"name":"Blogpaper","description":"A graphical newspaper like blog theme for Hugo.","html_url":"https://github.com/NormandErwan/BlogpLine truncated

cmd/hugothemesitebuilder/build/go.mod

@@ -1,4 +1,4 @@
-module github.com/gohugoio/hugoThemeSiteBuilder/cmd/hugothemesitebuilder/build
+module github.com/gohugoio/hugoThemesSiteBuilder/cmd/hugothemesitebuilder/build
 
 go 1.16
 

pkg/buildcmd/build.go

@@ -3,8 +3,19 @@ package buildcmd
 import (
 	"context"
 	"flag"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"os"
+	"path"
+	"path/filepath"
+	"regexp"
+	"strings"
+	"time"
 
+	"github.com/gohugoio/hugoThemesSiteBuilder/pkg/client"
 	"github.com/peterbourgon/ff/v3/ffcli"
+	"gopkg.in/yaml.v3"
 
 	"github.com/gohugoio/hugoThemesSiteBuilder/pkg/rootcmd"
 )
@@ -36,15 +47,12 @@ func New(rootConfig *rootcmd.Config) *ffcli.Command {
 // Exec function for this command.
 func (c *Config) Exec(ctx context.Context, args []string) error {
 	const configAll = "config.json"
-	client := c.rootConfig.Client
+	client := &buildClient{Client: c.rootConfig.Client}
 
 	if err := client.CreateThemesConfig(); err != nil {
 		return err
 	}
 
-	if true {
-		return nil
-	}
 	if !client.OutFileExists("go.mod") {
 		// Initialize the Hugo Module
 		if err := client.InitModule(configAll); err != nil {
@@ -57,10 +65,148 @@ func (c *Config) Exec(ctx context.Context, args []string) error {
 		return err
 	}
 
-	if err := client.WriteThemesContent(mmap); err != nil {
+	if err := client.writeThemesContent(mmap); err != nil {
 		return err
 	}
 
 	return nil
 
 }
+
+type buildClient struct {
+	*client.Client
+
+	mmap client.ModulesMap
+}
+
+func (c *buildClient) writeThemesContent(mm client.ModulesMap) error {
+	githubrepos, err := c.GetGitHubRepos(mm)
+	if err != nil {
+		return err
+	}
+	maxStars := 0
+	for _, ghRepo := range githubrepos {
+		if ghRepo.Stars > maxStars {
+			maxStars = ghRepo.Stars
+		}
+	}
+
+	contentDir := c.JoinOutPath("site", "content")
+	client.CheckErr(os.RemoveAll(contentDir))
+	client.CheckErr(os.MkdirAll(contentDir, 0777))
+
+	for k, m := range mm {
+
+		themeName := strings.ToLower(path.Base(k))
+
+		themeDir := filepath.Join(contentDir, "themes", themeName)
+		client.CheckErr(os.MkdirAll(themeDir, 0777))
+
+		copyIfExists := func(sourcePath, targetPath string) {
+			fs, err := os.Open(filepath.Join(m.Dir, sourcePath))
+			if err != nil {
+				return
+			}
+			defer fs.Close()
+			targetFilename := filepath.Join(themeDir, targetPath)
+			client.CheckErr(os.MkdirAll(filepath.Dir(targetFilename), 0777))
+			ft, err := os.Create(targetFilename)
+			client.CheckErr(err)
+			defer ft.Close()
+
+			_, err = io.Copy(ft, fs)
+			client.CheckErr(err)
+		}
+
+		fixReadMeContent := func(s string) string {
+			// Tell Hugo not to process shortcode samples
+			s = regexp.MustCompile(`(?s){\{%([^\/].*?)%\}\}`).ReplaceAllString(s, `{{%/*$1*/%}}`)
+			s = regexp.MustCompile(`(?s){\{<([^\/].*?)>\}\}`).ReplaceAllString(s, `{{</*$1*/>}}`)
+
+			return s
+		}
+
+		getReadMeContent := func() string {
+			files, err := os.ReadDir(m.Dir)
+			client.CheckErr(err)
+			for _, fi := range files {
+				if fi.IsDir() {
+					continue
+				}
+				if strings.EqualFold(fi.Name(), "readme.md") {
+					b, err := ioutil.ReadFile(filepath.Join(m.Dir, fi.Name()))
+					client.CheckErr(err)
+					return fixReadMeContent(string(b))
+				}
+			}
+			return ""
+		}
+
+		title := strings.Title(themeName)
+		readMeContent := getReadMeContent()
+		ghRepo := githubrepos[m.Path]
+
+		// 30 days.
+		d30 := 30 * 24 * time.Hour
+		const boost = 50
+
+		// Higher is better.
+		weight := maxStars + 500
+		weight -= ghRepo.Stars
+		// Boost themes updated recently.
+		if !m.Time.IsZero() {
+			// Add some weight to recently updated themes.
+			age := time.Since(m.Time)
+			if age < (3 * d30) {
+				weight -= (boost * 2)
+			} else if age < (6 * d30) {
+				weight -= boost
+			}
+		}
+
+		// Boost themes with a Hugo version indicator set that covers.
+		// the current Hugo version.
+		if m.HugoVersion.IsValid() {
+			weight -= boost
+		}
+
+		// TODO(bep) we don't build any demo site anymore, but
+		// we could and should probably build a simple site and
+		// count warnings and error and use that to
+		// either pull it down the list with weight or skip it.
+
+		//c.Logf("Processing theme %q with weight %d", themeName, weight)
+
+		// TODO1 tags, normalized.
+
+		frontmatter := map[string]interface{}{
+			"title":       title,
+			"slug":        themeName,
+			"aliases":     []string{"/" + themeName},
+			"weight":      weight,
+			"lastMod":     m.Time,
+			"hugoVersion": m.HugoVersion,
+			"meta":        m.Meta,
+			"githubInfo":  ghRepo,
+		}
+
+		b, err := yaml.Marshal(frontmatter)
+		client.CheckErr(err)
+
+		content := fmt.Sprintf(`---
+%s
+---
+%s
+`, string(b), readMeContent)
+
+		if err := ioutil.WriteFile(filepath.Join(themeDir, "index.md"), []byte(content), 0666); err != nil {
+			return err
+		}
+
+		copyIfExists("images/tn.png", "tn-featured.png")
+		copyIfExists("images/screenshot.png", "screenshot.png")
+
+	}
+
+	return nil
+}

pkg/client/client.go

@@ -10,15 +10,13 @@ import (
 	"net/http"
 	"os"
 	"os/exec"
-	"path"
 	"path/filepath"
-	"regexp"
 	"runtime"
+	"sort"
+	"strconv"
 	"strings"
 	"time"
 
-	"gopkg.in/yaml.v3"
-
 	"github.com/gohugoio/hugo/modules"
 )
 
@@ -122,168 +120,101 @@ func (c *Client) CreateThemesConfig() error {
 
 }
 
+func (c *Client) JoinOutPath(elem ...string) string {
+	return filepath.Join(append([]string{c.outDir}, elem...)...)
+}
+
 func (c *Client) TimeTrack(start time.Time, name string) {
 	elapsed := time.Since(start)
 	fmt.Fprintf(c.logWriter, "%s in %v ms\n", name, int(1000*elapsed.Seconds()))
 }
 
-func (c *Client) WriteThemesContent(mm ModulesMap) error {
-	githubrepos, err := c.GetGitHubRepos(mm)
-	if err != nil {
-		return err
-	}
-	maxStars := 0
-	for _, ghRepo := range githubrepos {
-		if ghRepo.Stars > maxStars {
-			maxStars = ghRepo.Stars
+const cacheFileSuffix = "githubrepos.json"
+
+// GetGitHubRepos will first look in the chache folder for GitHub repo
+// information for mods. If not found, it will ask GitHub and then store
+// it in the cache.
+//
+// If you start with an empty cache, you will need to set a GITHUB_TOKEN environment variable.
+func (c *Client) GetGitHubRepos(mods ModulesMap) (map[string]GitHubRepo, error) {
+	cacheFiles := c.getGithubReposCacheFilesSorted()
+	m := make(map[string]GitHubRepo)
+	for _, cacheFile := range cacheFiles {
+		cacheFilename := filepath.Join(c.outDir, cacheDir, cacheFile)
+		b, err := ioutil.ReadFile(cacheFilename)
+		if err != nil {
+			return nil, err
+		}
+
+		m2 := make(map[string]GitHubRepo)
+		if err = json.Unmarshal(b, &m2); err != nil {
+			return nil, err
+		}
+
+		for k, v := range m2 {
+			m[k] = v
+
 		}
 	}
 
-	contentDir := filepath.Join(c.outDir, "site", "content")
-	checkErr(os.RemoveAll(contentDir))
-	checkErr(os.MkdirAll(contentDir, 0777))
-
-	for k, m := range mm {
-
-		themeName := strings.ToLower(path.Base(k))
-
-		themeDir := filepath.Join(contentDir, "themes", themeName)
-		checkErr(os.MkdirAll(themeDir, 0777))
-
-		copyIfExists := func(sourcePath, targetPath string) {
-			fs, err := os.Open(filepath.Join(m.Dir, sourcePath))
-			if err != nil {
-				return
-			}
-			defer fs.Close()
-			targetFilename := filepath.Join(themeDir, targetPath)
-			checkErr(os.MkdirAll(filepath.Dir(targetFilename), 0777))
-			ft, err := os.Create(targetFilename)
-			checkErr(err)
-			defer ft.Close()
-
-			_, err = io.Copy(ft, fs)
-			checkErr(err)
+	missing := ModulesMap{}
+	for k, v := range mods {
+		if _, found := m[k]; !found {
+			missing[k] = v
 		}
-
-		fixReadMeContent := func(s string) string {
-			// Tell Hugo not to process shortcode samples
-			s = regexp.MustCompile(`(?s){\{%([^\/].*?)%\}\}`).ReplaceAllString(s, `{{%/*$1*/%}}`)
-			s = regexp.MustCompile(`(?s){\{<([^\/].*?)>\}\}`).ReplaceAllString(s, `{{</*$1*/>}}`)
-
-			return s
-		}
-
-		getReadMeContent := func() string {
-			files, err := os.ReadDir(m.Dir)
-			checkErr(err)
-			for _, fi := range files {
-				if fi.IsDir() {
-					continue
-				}
-				if strings.EqualFold(fi.Name(), "readme.md") {
-					b, err := ioutil.ReadFile(filepath.Join(m.Dir, fi.Name()))
-					checkErr(err)
-					return fixReadMeContent(string(b))
-				}
-			}
-			return ""
-		}
-
-		title := strings.Title(themeName)
-		readMeContent := getReadMeContent()
-		ghRepo := githubrepos[m.Path]
-
-		// 30 days.
-		d30 := 30 * 24 * time.Hour
-		const boost = 50
-
-		// Higher is better.
-		weight := maxStars + 500
-		weight -= ghRepo.Stars
-		// Boost themes updated recently.
-		if !m.Time.IsZero() {
-			// Add some weight to recently updated themes.
-			age := time.Since(m.Time)
-			if age < (3 * d30) {
-				weight -= (boost * 2)
-			} else if age < (6 * d30) {
-				weight -= boost
-			}
-		}
-
-		// Boost themes with a Hugo version indicator set that covers.
-		// the current Hugo version.
-		if m.HugoVersion.IsValid() {
-			weight -= boost
-		}
-
-		// TODO(bep) we don't build any demo site anymore, but
-		// we could and should probably build a simple site and
-		// count warnings and error and use that to
-		// either pull it down the list with weight or skip it.
-
-		c.Logf("Processing theme %q with weight %d", themeName, weight)
-
-		// TODO1 tags, normalized.
-
-		frontmatter := map[string]interface{}{
-			"title":       title,
-			"slug":        themeName,
-			"aliases":     []string{"/" + themeName},
-			"weight":      weight,
-			"lastMod":     m.Time,
-			"hugoVersion": m.HugoVersion,
-			"meta":        m.Meta,
-			"githubInfo":  ghRepo,
-		}
-
-		b, err := yaml.Marshal(frontmatter)
-		checkErr(err)
-
-		content := fmt.Sprintf(`---
-%s
----
-%s
-`, string(b), readMeContent)
-
-		if err := ioutil.WriteFile(filepath.Join(themeDir, "index.md"), []byte(content), 0666); err != nil {
-			return err
-		}
-
-		copyIfExists("images/tn.png", "tn-featured.png")
-		copyIfExists("images/screenshot.png", "screenshot.png")
-
-		return nil
-
 	}
 
-	return nil
+	if len(missing) > 0 {
+		cacheNum := 0
+		if len(cacheFiles) > 0 {
+			last := cacheFiles[len(cacheFiles)-1]
+			cacheNum, _ = strconv.Atoi(last[:strings.Index(last, ".")])
+			cacheNum++
+		}
+		nextCacheFilename := filepath.Join(c.outDir, cacheDir, fmt.Sprintf("%0*d.%s", 4, cacheNum, cacheFileSuffix))
+		m2, err := c.fetchGitHubRepos(mods)
+		if err != nil {
+			return nil, err
+		}
+
+		b, err := json.Marshal(m2)
+		if err != nil {
+			return nil, err
+		}
+
+		for k, v := range m2 {
+			m[k] = v
+
+		}
+
+		CheckErr(os.MkdirAll(filepath.Dir(nextCacheFilename), 0777))
+		CheckErr(ioutil.WriteFile(nextCacheFilename, b, 0666))
+	}
+
+	return m, nil
+
 }
 
-func (c *Client) GetGitHubRepos(mods ModulesMap) (map[string]GitHubRepo, error) {
-	const cacheFile = "githubrepos.json"
-	cacheFilename := filepath.Join(c.outDir, cacheDir, cacheFile)
-	b, err := ioutil.ReadFile(cacheFilename)
-	if err == nil {
-		m := make(map[string]GitHubRepo)
-		err := json.Unmarshal(b, &m)
-		return m, err
+func (c *Client) getGithubReposCacheFilesSorted() []string {
+
+	fis, err := os.ReadDir(filepath.Join(c.outDir, cacheDir))
+	CheckErr(err)
+
+	var entries []string
+
+	for _, fi := range fis {
+		if fi.IsDir() {
+			continue
+		}
+
+		if strings.HasSuffix(fi.Name(), cacheFileSuffix) {
+			entries = append(entries, fi.Name())
+		}
 	}
 
-	m, err := c.fetchGitHubRepos(mods)
-	if err != nil {
-		return nil, err
-	}
+	sort.Strings(entries)
 
-	b, err = json.Marshal(m)
-	if err != nil {
-		return nil, err
-	}
-
-	checkErr(os.MkdirAll(filepath.Dir(cacheFilename), 0777))
-
-	return m, ioutil.WriteFile(cacheFilename, b, 0666)
+	return entries
 
 }
 
@@ -305,7 +236,7 @@ func (c *Client) fetchGitHubRepo(m Module) (GitHubRepo, error) {
 
 	err = doGitHubRequest(req, &repo)
 	if err != nil {
-		return repo, fmt.Errorf("failed to get GitHub repo for %q: %s", apiURL, err)
+		return repo, fmt.Errorf("failed to get GitHub repo for %q: %s. Set GITHUB_TOKEN if you get rate limiting errors.", apiURL, err)
 	}
 	return repo, nil
 }
@@ -384,7 +315,7 @@ func addGitHubToken(req *http.Request) {
 	}
 }
 
-func checkErr(err error) {
+func CheckErr(err error) {
 	if err != nil {
 		panic(err)
 	}