feat(go): rewrite in Go

crawl-prime/README.md

@@ -1,6 +1,14 @@
 # crawl-prime
 
-Crawl all prime available on http://compoasso.free.fr/primelistweb/page/prime/liste_online_en.php
+Crawl all prime numbers available on http://compoasso.free.fr/primelistweb/page/prime/liste_online_en.php
+
+*In 2025, I rewrite this project using Go. The Python version of this project can be found at [feature/python](https://github.com/tiennm99/crawl-prime/tree/feature/python) branch.*
+
+## Requirements
+
+- Go 1.18 or higher
+- Internet connection to access the prime list website
+- Write permissions in the current directory (for `primes.txt`)
 
 ## Installation
 
@@ -8,20 +16,40 @@ Crawl all prime available on http://compoasso.free.fr/primelistweb/page/prime/li
    ```bash
    git clone https://github.com/tiennm99/crawl-prime.git
    cd crawl-prime
-
-2. Create a virtual environment:
-   ```bash
-   python -m venv venv
-   source venv/bin/activate   # On macOS/Linux
-   venv\Scripts\activate      # On Windows
    ```
 
-3. Install the dependencies:
+2. Install Go (if not already installed):
    ```bash
-    pip install -r requirements.txt
+   # Download and install Go from https://golang.org/dl/
-    ```
+   # Or use your package manager:
+   # Ubuntu/Debian: sudo apt install golang
+   # macOS: brew install go
+   # Windows: Download from official website
+   ```
+
+3. Install dependencies:
+   ```bash
+   go mod tidy
+   ```
 
 4. Run the script:
    ```bash
-    python main.py
+   go run main.go
-    ```
+   ```
+
+## Usage
+
+The program will automatically:
+- Start crawling from prime number 1
+- Process pages in batches of 600 primes each
+- Write all found primes to `primes.txt`
+- Display progress information during execution
+- Stop when it reaches 1,000,000 primes or encounters an error
+
+## Output
+
+- `primes.txt` - Contains all crawled prime numbers, one per line
+
+## License
+
+This project is for educational purposes. Please respect the terms of service of the prime list website.

crawl-prime/go.mod

@@ -0,0 +1,12 @@
+module github.com/tiennm99/crawl-prime
+
+go 1.24.0
+
+toolchain go1.24.10
+
+require github.com/anaskhan96/soup v1.2.5
+
+require (
+	golang.org/x/net v0.47.0 // indirect
+	golang.org/x/text v0.31.0 // indirect
+)

crawl-prime/go.sum

@@ -0,0 +1,20 @@
+github.com/anaskhan96/soup v1.2.5 h1:V/FHiusdTrPrdF4iA1YkVxsOpdNcgvqT1hG+YtcZ5hM=
+github.com/anaskhan96/soup v1.2.5/go.mod h1:6YnEp9A2yywlYdM4EgDz9NEHclocMepEtku7wg6Cq3s=
+github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0=
+github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.47.0 h1:Mx+4dIFzqraBXUugkia1OOvlD6LemFo1ALMHjrXDOhY=
+golang.org/x/net v0.47.0/go.mod h1:/jNxtkgq5yWUGYkaZGqo27cfGZ1c5Nen03aYrrKpVRU=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.31.0 h1:aC8ghyu4JhP8VojJ2lEHBnochRno1sgL6nEi9WGFGMM=
+golang.org/x/text v0.31.0/go.mod h1:tKRAlv61yKIjGGHX/4tP1LTbc13YSec1pxVEWXzfoeM=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

crawl-prime/main.go

@@ -0,0 +1,176 @@
+package main
+
+import (
+	"fmt"
+	"io"
+	"log"
+	"net/http"
+	"net/url"
+	"os"
+	"strconv"
+	"strings"
+
+	"github.com/anaskhan96/soup"
+)
+
+const (
+	targetURL    = "http://compoasso.free.fr/primelistweb/page/prime/liste_online_en.php"
+	maxPrimes    = 1000000
+	outputFile   = "primes.txt"
+	perPageParam = "600"
+)
+
+func main() {
+	current := 0
+	totalWritten := 0
+
+	for current < maxPrimes {
+		fmt.Printf("Processing primes starting from: %d\n", current)
+
+		primes, nextCurrent, err := fetchPrimesPage(current)
+		if err != nil {
+			log.Printf("Error fetching page starting at %d: %v", current, err)
+			continue
+		}
+
+		if len(primes) == 0 {
+			log.Printf("No primes found on page starting at %d", current)
+			break
+		}
+
+		// Write primes to file
+		err = writePrimesToFile(primes)
+		if err != nil {
+			log.Printf("Error writing primes to file: %v", err)
+			continue
+		}
+
+		totalWritten += len(primes)
+		fmt.Printf("Wrote %d primes (total: %d)\n", len(primes), totalWritten)
+
+		if nextCurrent <= current {
+			log.Printf("Next current (%d) is not greater than current (%d), stopping", nextCurrent, current)
+			break
+		}
+
+		current = nextCurrent
+	}
+
+	fmt.Printf("Completed! Total primes written: %d\n", totalWritten)
+}
+
+func fetchPrimesPage(start int) ([]string, int, error) {
+	// Prepare form data
+	formData := url.Values{
+		"primePageInput": {perPageParam},
+		"numberInput":    {strconv.Itoa(start)},
+	}
+
+	// Create request
+	req, err := http.NewRequest("POST", targetURL, strings.NewReader(formData.Encode()))
+	if err != nil {
+		return nil, 0, fmt.Errorf("failed to create request: %w", err)
+	}
+
+	req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
+	req.Header.Set("User-Agent", "Prime-Crawler/1.0")
+
+	// Send request
+	client := &http.Client{}
+	resp, err := client.Do(req)
+	if err != nil {
+		return nil, 0, fmt.Errorf("failed to send request: %w", err)
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		return nil, 0, fmt.Errorf("unexpected status code: %d", resp.StatusCode)
+	}
+
+	// Parse HTML
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return nil, 0, fmt.Errorf("failed to read response body: %w", err)
+	}
+
+	return parsePrimesFromHTML(body, start)
+}
+
+func parsePrimesFromHTML(htmlContent []byte, start int) ([]string, int, error) {
+	// Parse HTML using soup (similar to BeautifulSoup)
+	htmlStr := string(htmlContent)
+	doc := soup.HTMLParse(htmlStr)
+
+	// Find the first table (matching Python soup.find('table'))
+	table := doc.Find("table")
+	if table.Error != nil {
+		return nil, 0, fmt.Errorf("failed to find table: %w", table.Error)
+	}
+
+	// Find form within the table (matching Python table.find('form'))
+	form := table.Find("form")
+	if form.Error != nil {
+		return nil, 0, fmt.Errorf("failed to find form: %w", form.Error)
+	}
+
+	// Find inner table within the form (matching Python form.find('table'))
+	table1 := form.Find("table")
+	if table1.Error != nil {
+		return nil, 0, fmt.Errorf("failed to find inner table: %w", table1.Error)
+	}
+
+	// Find all td elements (matching Python table1.find_all('td'))
+	tdElements := table1.FindAll("td")
+	if len(tdElements) == 0 {
+		return nil, 0, fmt.Errorf("no td elements found")
+	}
+
+	// Extract primes from all td elements except last 10 (matching Python td_elements[0:-10])
+	var primes []string
+	for i, td := range tdElements {
+		if i < len(tdElements)-10 {
+			text := strings.TrimSpace(td.Text())
+			if text != "" {
+				primes = append(primes, text)
+			}
+		}
+	}
+
+	// Get next current from the 11th element from the end (matching Python td_elements[-11])
+	var nextCurrent int
+	if len(tdElements) >= 11 {
+		nextTd := tdElements[len(tdElements)-11]
+		text := strings.TrimSpace(nextTd.Text())
+		if text != "" {
+			var err error
+			nextCurrent, err = strconv.Atoi(text)
+			if err != nil {
+				return nil, 0, fmt.Errorf("failed to parse next current number: %w", err)
+			}
+			nextCurrent += 1
+		}
+	}
+
+	if len(primes) == 0 {
+		return nil, 0, fmt.Errorf("no primes found in HTML response")
+	}
+
+	return primes, nextCurrent, nil
+}
+
+func writePrimesToFile(primes []string) error {
+	file, err := os.OpenFile(outputFile, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
+	if err != nil {
+		return fmt.Errorf("failed to open output file: %w", err)
+	}
+	defer file.Close()
+
+	for _, prime := range primes {
+		_, err := file.WriteString(prime + "\n")
+		if err != nil {
+			return fmt.Errorf("failed to write prime to file: %w", err)
+		}
+	}
+
+	return nil
+}

crawl-prime/main.py

@@ -1,33 +0,0 @@
-import requests
-from bs4 import BeautifulSoup
-
-url = "http://compoasso.free.fr/primelistweb/page/prime/liste_online_en.php"
-max = 1_000_000
-current = 0
-
-while current < max:
-  data = {
-    "primePageInput": "600",
-    "numberInput": current,
-  }
-
-  response = requests.post(url, data=data)
-
-  if response.status_code == 200:
-    soup = BeautifulSoup(response.text, 'html.parser')
-    table = soup.find('table')
-    if table:
-      form = table.find('form')
-      if form:
-        table1 = form.find('table')
-        if table1:
-          td_elements = table1.find_all('td')
-          output_file = "primes.txt"
-
-          with open(output_file, "a", encoding="utf-8") as file:
-            for td in td_elements[0:-10]:
-              file.write(td.text + "\n")
-
-          current = int(td_elements[-11].text) + 1
-  else:
-    print("Error - current: " + current)