mttools/crawl-prime/main.go

package main

import (
	"fmt"
	"io"
	"log"
	"net/http"
	"net/url"
	"os"
	"strconv"
	"strings"

	"github.com/anaskhan96/soup"
)

const (
	targetURL    = "http://compoasso.free.fr/primelistweb/page/prime/liste_online_en.php"
	maxPrimes    = 1000000
	outputFile   = "primes.txt"
	perPageParam = "600"
)

func main() {
	current := 0
	totalWritten := 0

	for current < maxPrimes {
		fmt.Printf("Processing primes starting from: %d\n", current)

		primes, nextCurrent, err := fetchPrimesPage(current)
		if err != nil {
			log.Printf("Error fetching page starting at %d: %v", current, err)
			continue
		}

		if len(primes) == 0 {
			log.Printf("No primes found on page starting at %d", current)
			break
		}

		// Write primes to file
		err = writePrimesToFile(primes)
		if err != nil {
			log.Printf("Error writing primes to file: %v", err)
			continue
		}

		totalWritten += len(primes)
		fmt.Printf("Wrote %d primes (total: %d)\n", len(primes), totalWritten)

		if nextCurrent <= current {
			log.Printf("Next current (%d) is not greater than current (%d), stopping", nextCurrent, current)
			break
		}

		current = nextCurrent
	}

	fmt.Printf("Completed! Total primes written: %d\n", totalWritten)
}

func fetchPrimesPage(start int) ([]string, int, error) {
	// Prepare form data
	formData := url.Values{
		"primePageInput": {perPageParam},
		"numberInput":    {strconv.Itoa(start)},
	}

	// Create request
	req, err := http.NewRequest("POST", targetURL, strings.NewReader(formData.Encode()))
	if err != nil {
		return nil, 0, fmt.Errorf("failed to create request: %w", err)
	}

	req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
	req.Header.Set("User-Agent", "Prime-Crawler/1.0")

	// Send request
	client := &http.Client{}
	resp, err := client.Do(req)
	if err != nil {
		return nil, 0, fmt.Errorf("failed to send request: %w", err)
	}
	defer resp.Body.Close()

	if resp.StatusCode != http.StatusOK {
		return nil, 0, fmt.Errorf("unexpected status code: %d", resp.StatusCode)
	}

	// Parse HTML
	body, err := io.ReadAll(resp.Body)
	if err != nil {
		return nil, 0, fmt.Errorf("failed to read response body: %w", err)
	}

	return parsePrimesFromHTML(body, start)
}

func parsePrimesFromHTML(htmlContent []byte, start int) ([]string, int, error) {
	// Parse HTML using soup (similar to BeautifulSoup)
	htmlStr := string(htmlContent)
	doc := soup.HTMLParse(htmlStr)

	// Find the first table (matching Python soup.find('table'))
	table := doc.Find("table")
	if table.Error != nil {
		return nil, 0, fmt.Errorf("failed to find table: %w", table.Error)
	}

	// Find form within the table (matching Python table.find('form'))
	form := table.Find("form")
	if form.Error != nil {
		return nil, 0, fmt.Errorf("failed to find form: %w", form.Error)
	}

	// Find inner table within the form (matching Python form.find('table'))
	table1 := form.Find("table")
	if table1.Error != nil {
		return nil, 0, fmt.Errorf("failed to find inner table: %w", table1.Error)
	}

	// Find all td elements (matching Python table1.find_all('td'))
	tdElements := table1.FindAll("td")
	if len(tdElements) == 0 {
		return nil, 0, fmt.Errorf("no td elements found")
	}

	// Extract primes from all td elements except last 10 (matching Python td_elements[0:-10])
	var primes []string
	for i, td := range tdElements {
		if i < len(tdElements)-10 {
			text := strings.TrimSpace(td.Text())
			if text != "" {
				primes = append(primes, text)
			}
		}
	}

	// Get next current from the 11th element from the end (matching Python td_elements[-11])
	var nextCurrent int
	if len(tdElements) >= 11 {
		nextTd := tdElements[len(tdElements)-11]
		text := strings.TrimSpace(nextTd.Text())
		if text != "" {
			var err error
			nextCurrent, err = strconv.Atoi(text)
			if err != nil {
				return nil, 0, fmt.Errorf("failed to parse next current number: %w", err)
			}
			nextCurrent += 1
		}
	}

	if len(primes) == 0 {
		return nil, 0, fmt.Errorf("no primes found in HTML response")
	}

	return primes, nextCurrent, nil
}

func writePrimesToFile(primes []string) error {
	file, err := os.OpenFile(outputFile, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
	if err != nil {
		return fmt.Errorf("failed to open output file: %w", err)
	}
	defer file.Close()

	for _, prime := range primes {
		_, err := file.WriteString(prime + "\n")
		if err != nil {
			return fmt.Errorf("failed to write prime to file: %w", err)
		}
	}

	return nil
}