diff --git a/crawl-prime/README.md b/crawl-prime/README.md index c1d11f1..2bded01 100644 --- a/crawl-prime/README.md +++ b/crawl-prime/README.md @@ -1,6 +1,14 @@ # crawl-prime -Crawl all prime available on http://compoasso.free.fr/primelistweb/page/prime/liste_online_en.php +Crawl all prime numbers available on http://compoasso.free.fr/primelistweb/page/prime/liste_online_en.php + +*In 2025, I rewrite this project using Go. The Python version of this project can be found at [feature/python](https://github.com/tiennm99/crawl-prime/tree/feature/python) branch.* + +## Requirements + +- Go 1.18 or higher +- Internet connection to access the prime list website +- Write permissions in the current directory (for `primes.txt`) ## Installation @@ -8,20 +16,40 @@ Crawl all prime available on http://compoasso.free.fr/primelistweb/page/prime/li ```bash git clone https://github.com/tiennm99/crawl-prime.git cd crawl-prime - -2. Create a virtual environment: - ```bash - python -m venv venv - source venv/bin/activate # On macOS/Linux - venv\Scripts\activate # On Windows ``` -3. Install the dependencies: +2. Install Go (if not already installed): ```bash - pip install -r requirements.txt - ``` + # Download and install Go from https://golang.org/dl/ + # Or use your package manager: + # Ubuntu/Debian: sudo apt install golang + # macOS: brew install go + # Windows: Download from official website + ``` + +3. Install dependencies: + ```bash + go mod tidy + ``` 4. Run the script: ```bash - python main.py - ``` + go run main.go + ``` + +## Usage + +The program will automatically: +- Start crawling from prime number 1 +- Process pages in batches of 600 primes each +- Write all found primes to `primes.txt` +- Display progress information during execution +- Stop when it reaches 1,000,000 primes or encounters an error + +## Output + +- `primes.txt` - Contains all crawled prime numbers, one per line + +## License + +This project is for educational purposes. Please respect the terms of service of the prime list website. diff --git a/crawl-prime/go.mod b/crawl-prime/go.mod new file mode 100644 index 0000000..cfa9699 --- /dev/null +++ b/crawl-prime/go.mod @@ -0,0 +1,12 @@ +module github.com/tiennm99/crawl-prime + +go 1.24.0 + +toolchain go1.24.10 + +require github.com/anaskhan96/soup v1.2.5 + +require ( + golang.org/x/net v0.47.0 // indirect + golang.org/x/text v0.31.0 // indirect +) diff --git a/crawl-prime/go.sum b/crawl-prime/go.sum new file mode 100644 index 0000000..a749019 --- /dev/null +++ b/crawl-prime/go.sum @@ -0,0 +1,20 @@ +github.com/anaskhan96/soup v1.2.5 h1:V/FHiusdTrPrdF4iA1YkVxsOpdNcgvqT1hG+YtcZ5hM= +github.com/anaskhan96/soup v1.2.5/go.mod h1:6YnEp9A2yywlYdM4EgDz9NEHclocMepEtku7wg6Cq3s= +github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.47.0 h1:Mx+4dIFzqraBXUugkia1OOvlD6LemFo1ALMHjrXDOhY= +golang.org/x/net v0.47.0/go.mod h1:/jNxtkgq5yWUGYkaZGqo27cfGZ1c5Nen03aYrrKpVRU= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.31.0 h1:aC8ghyu4JhP8VojJ2lEHBnochRno1sgL6nEi9WGFGMM= +golang.org/x/text v0.31.0/go.mod h1:tKRAlv61yKIjGGHX/4tP1LTbc13YSec1pxVEWXzfoeM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/crawl-prime/main.go b/crawl-prime/main.go new file mode 100644 index 0000000..0b95d7f --- /dev/null +++ b/crawl-prime/main.go @@ -0,0 +1,176 @@ +package main + +import ( + "fmt" + "io" + "log" + "net/http" + "net/url" + "os" + "strconv" + "strings" + + "github.com/anaskhan96/soup" +) + +const ( + targetURL = "http://compoasso.free.fr/primelistweb/page/prime/liste_online_en.php" + maxPrimes = 1000000 + outputFile = "primes.txt" + perPageParam = "600" +) + +func main() { + current := 0 + totalWritten := 0 + + for current < maxPrimes { + fmt.Printf("Processing primes starting from: %d\n", current) + + primes, nextCurrent, err := fetchPrimesPage(current) + if err != nil { + log.Printf("Error fetching page starting at %d: %v", current, err) + continue + } + + if len(primes) == 0 { + log.Printf("No primes found on page starting at %d", current) + break + } + + // Write primes to file + err = writePrimesToFile(primes) + if err != nil { + log.Printf("Error writing primes to file: %v", err) + continue + } + + totalWritten += len(primes) + fmt.Printf("Wrote %d primes (total: %d)\n", len(primes), totalWritten) + + if nextCurrent <= current { + log.Printf("Next current (%d) is not greater than current (%d), stopping", nextCurrent, current) + break + } + + current = nextCurrent + } + + fmt.Printf("Completed! Total primes written: %d\n", totalWritten) +} + +func fetchPrimesPage(start int) ([]string, int, error) { + // Prepare form data + formData := url.Values{ + "primePageInput": {perPageParam}, + "numberInput": {strconv.Itoa(start)}, + } + + // Create request + req, err := http.NewRequest("POST", targetURL, strings.NewReader(formData.Encode())) + if err != nil { + return nil, 0, fmt.Errorf("failed to create request: %w", err) + } + + req.Header.Set("Content-Type", "application/x-www-form-urlencoded") + req.Header.Set("User-Agent", "Prime-Crawler/1.0") + + // Send request + client := &http.Client{} + resp, err := client.Do(req) + if err != nil { + return nil, 0, fmt.Errorf("failed to send request: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return nil, 0, fmt.Errorf("unexpected status code: %d", resp.StatusCode) + } + + // Parse HTML + body, err := io.ReadAll(resp.Body) + if err != nil { + return nil, 0, fmt.Errorf("failed to read response body: %w", err) + } + + return parsePrimesFromHTML(body, start) +} + +func parsePrimesFromHTML(htmlContent []byte, start int) ([]string, int, error) { + // Parse HTML using soup (similar to BeautifulSoup) + htmlStr := string(htmlContent) + doc := soup.HTMLParse(htmlStr) + + // Find the first table (matching Python soup.find('table')) + table := doc.Find("table") + if table.Error != nil { + return nil, 0, fmt.Errorf("failed to find table: %w", table.Error) + } + + // Find form within the table (matching Python table.find('form')) + form := table.Find("form") + if form.Error != nil { + return nil, 0, fmt.Errorf("failed to find form: %w", form.Error) + } + + // Find inner table within the form (matching Python form.find('table')) + table1 := form.Find("table") + if table1.Error != nil { + return nil, 0, fmt.Errorf("failed to find inner table: %w", table1.Error) + } + + // Find all td elements (matching Python table1.find_all('td')) + tdElements := table1.FindAll("td") + if len(tdElements) == 0 { + return nil, 0, fmt.Errorf("no td elements found") + } + + // Extract primes from all td elements except last 10 (matching Python td_elements[0:-10]) + var primes []string + for i, td := range tdElements { + if i < len(tdElements)-10 { + text := strings.TrimSpace(td.Text()) + if text != "" { + primes = append(primes, text) + } + } + } + + // Get next current from the 11th element from the end (matching Python td_elements[-11]) + var nextCurrent int + if len(tdElements) >= 11 { + nextTd := tdElements[len(tdElements)-11] + text := strings.TrimSpace(nextTd.Text()) + if text != "" { + var err error + nextCurrent, err = strconv.Atoi(text) + if err != nil { + return nil, 0, fmt.Errorf("failed to parse next current number: %w", err) + } + nextCurrent += 1 + } + } + + if len(primes) == 0 { + return nil, 0, fmt.Errorf("no primes found in HTML response") + } + + return primes, nextCurrent, nil +} + +func writePrimesToFile(primes []string) error { + file, err := os.OpenFile(outputFile, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644) + if err != nil { + return fmt.Errorf("failed to open output file: %w", err) + } + defer file.Close() + + for _, prime := range primes { + _, err := file.WriteString(prime + "\n") + if err != nil { + return fmt.Errorf("failed to write prime to file: %w", err) + } + } + + return nil +} diff --git a/crawl-prime/main.py b/crawl-prime/main.py deleted file mode 100644 index e35ce79..0000000 --- a/crawl-prime/main.py +++ /dev/null @@ -1,33 +0,0 @@ -import requests -from bs4 import BeautifulSoup - -url = "http://compoasso.free.fr/primelistweb/page/prime/liste_online_en.php" -max = 1_000_000 -current = 0 - -while current < max: - data = { - "primePageInput": "600", - "numberInput": current, - } - - response = requests.post(url, data=data) - - if response.status_code == 200: - soup = BeautifulSoup(response.text, 'html.parser') - table = soup.find('table') - if table: - form = table.find('form') - if form: - table1 = form.find('table') - if table1: - td_elements = table1.find_all('td') - output_file = "primes.txt" - - with open(output_file, "a", encoding="utf-8") as file: - for td in td_elements[0:-10]: - file.write(td.text + "\n") - - current = int(td_elements[-11].text) + 1 - else: - print("Error - current: " + current) diff --git a/crawl-prime/requirements.txt b/crawl-prime/requirements.txt deleted file mode 100644 index 0844986..0000000 Binary files a/crawl-prime/requirements.txt and /dev/null differ