feat: Get Proxy Cloudflare Checker Working

Not finished yet.
Encountering issues with proxies rotation.

Author: kangoo13

.gitignore

@@ -13,3 +13,9 @@
 
 # Dependency directories (remove the comment below to include it)
 # vendor/
+
+proxies.txt
+cloudflare-proxy-ban-checker
+bad.txt
+good.txt
+.idea

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2020 Aurélien SCHILTZ
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

@@ -1,2 +1,29 @@
 # cloudflare-proxy-ban-checker
-Checks a set of proxies against a protected cloudflare website to check if the proxies are banned or not.
+
+Checks a set of proxies against a protected CloudFlare website to check if the proxies are banned or not.
+
+# Installation
+
+Very simple :
+
+```
+git clone git@github.com:kangoo13/cloudflare-proxy-ban-checker.git
+cd cloudflare-proxy-ban-checker
+go build
+./cloudflare-proxy-ban-checker -h
+Usage: cloudflare-proxy-ban-checker [--goodproxiespath GOODPROXIESPATH] [--badproxiespath BADPROXIESPATH] [--timeoutproxy TIMEOUTPROXY] WEBSITE PROXYLIST
+
+Positional arguments:
+  WEBSITE                website to test proxy against cloudflare
+  PROXYLIST              path to the proxyList
+
+Options:
+  --goodproxiespath GOODPROXIESPATH
+                         path to the good proxies identified [default: good.txt]
+  --badproxiespath BADPROXIESPATH
+                         path to the bad proxies identified [default: bad.txt]
+  --timeoutproxy TIMEOUTPROXY
+                         timeout proxy duration [default: 5]
+  --help, -h             display this help and exit
+```
+

cfprxchecker/checker.go

@@ -0,0 +1,110 @@
+package cfprxchecker
+
+import (
+	"github.com/gocolly/colly"
+	"github.com/gocolly/colly/extensions"
+	"log"
+	"net/http"
+	"os"
+	"strings"
+	"sync"
+	"time"
+)
+
+type Args struct {
+	WebsiteToCrawl        string
+	ProxyList             []string
+	GoodProxiesOutputFile *os.File
+	BadProxiesOutputFile  *os.File
+	TimeoutProxy          time.Duration
+	muBad                 sync.Mutex
+	muGood                sync.Mutex
+}
+
+func (a *Args) writeGoodProxy(proxy string) {
+	if proxy != "" {
+
+		var b strings.Builder
+		a.muGood.Lock()
+		defer a.muGood.Unlock()
+
+		b.WriteString(proxy)
+		b.WriteString("\n")
+		_, err := a.GoodProxiesOutputFile.WriteString(b.String())
+		if err != nil {
+			log.Fatalf("[WriteGoodProxy] error while appending to file %s", err)
+		}
+	}
+}
+
+func (a *Args) writeBadProxy(proxy string) {
+	if proxy != "" {
+
+		var b strings.Builder
+
+		a.muBad.Lock()
+		defer a.muBad.Unlock()
+
+		b.WriteString(proxy)
+		b.WriteString("\n")
+		_, err := a.BadProxiesOutputFile.WriteString(b.String())
+		if err != nil {
+			log.Fatalf("[WriteBadProxy] error while appending to file %s", err)
+		}
+	}
+}
+
+func CheckProxiesAgainstCloudFlare(args *Args) {
+	// Rotate the proxies
+	rp, err := RoundRobinProxySwitcher(args.ProxyList...)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	// Instantiate default collector
+	c := colly.NewCollector(
+		colly.Async(true),
+	)
+
+	c.IgnoreRobotsTxt = true
+	c.AllowURLRevisit = true
+	c.CacheDir = ""
+
+	c.WithTransport(&http.Transport{
+		Proxy:               rp,
+		DisableKeepAlives:   true,
+		MaxIdleConns:        100,
+		MaxIdleConnsPerHost: 100,
+	})
+
+	// Limit the maximum parallelism to 24
+	err = c.Limit(&colly.LimitRule{DomainGlob: "*", Parallelism: 24})
+
+	if err != nil {
+		log.Printf("error while doing c.Limit %s", err)
+	}
+
+	extensions.RandomUserAgent(c)
+
+	if args.BadProxiesOutputFile != nil {
+		c.OnError(func(response *colly.Response, err error) {
+			log.Printf("[DEBUG] Bad proxy found error status %d [%s] [%s]", response.StatusCode, response.Request.ProxyURL, err)
+			args.writeBadProxy(response.Request.ProxyURL)
+		})
+	}
+
+	c.OnResponse(func(response *colly.Response) {
+		log.Printf("[DEBUG] Good proxy found [%s]", response.Request.ProxyURL)
+		args.writeGoodProxy(response.Request.ProxyURL)
+	})
+
+	for _, proxy := range args.ProxyList {
+		log.Printf("[DEBUG] Doing %s", proxy)
+		if err = c.Visit(args.WebsiteToCrawl); err != nil {
+			log.Printf("Error happening doing Visit %s", err)
+		}
+	}
+
+	// Wait until threads are finished
+	c.Wait()
+}

cfprxchecker/custom_round_robin_proxy.go

@@ -0,0 +1,59 @@
+package cfprxchecker
+
+import (
+	"context"
+	"github.com/gocolly/colly"
+	"log"
+	"net/http"
+	"net/url"
+	"sync/atomic"
+)
+
+type roundRobinSwitcher struct {
+	proxyURLs    []*url.URL
+	proxyURLsMap map[string]*url.URL
+	index        uint32
+}
+
+func (r *roundRobinSwitcher) GetProxy(pr *http.Request) (*url.URL, error) {
+	var u *url.URL
+	// Use Same Proxy when Redirect
+	if pr.Response != nil && pr.Response.StatusCode == 301 {
+		u = r.proxyURLsMap[pr.Response.Request.Context().Value(colly.ProxyURLKey).(string)]
+		log.Printf("titi %s", u.String())
+	} else if pr.Response != nil {
+		log.Printf("tototototo")
+	} else {
+		u = r.proxyURLs[r.index%uint32(len(r.proxyURLs))]
+		atomic.AddUint32(&r.index, 1)
+		log.Printf("grosminet %s", u.String())
+	}
+
+	ctx := context.WithValue(pr.Context(), colly.ProxyURLKey, u.String())
+	*pr = *pr.WithContext(ctx)
+
+	return u, nil
+}
+
+// RoundRobinProxySwitcher creates a proxy switcher function which rotates
+// ProxyURLs on every request.
+// The proxy type is determined by the URL scheme. "http", "https"
+// and "socks5" are supported. If the scheme is empty,
+// "http" is assumed.
+func RoundRobinProxySwitcher(ProxyURLs ...string) (colly.ProxyFunc, error) {
+	urls := make([]*url.URL, len(ProxyURLs))
+	urlsMap := make(map[string]*url.URL, len(ProxyURLs))
+	for i, u := range ProxyURLs {
+		parsedU, err := url.Parse(u)
+		if err != nil {
+			return nil, err
+		}
+		urls[i] = parsedU
+		urlsMap[parsedU.String()] = parsedU
+	}
+	return (&roundRobinSwitcher{
+		proxyURLs:    urls,
+		proxyURLsMap: urlsMap,
+		index:        0,
+	}).GetProxy, nil
+}

go.mod

@@ -0,0 +1,21 @@
+module github.com/kangoo13/cloudflare-proxy-ban-checker
+
+go 1.13
+
+require (
+	github.com/PuerkitoBio/goquery v1.5.1 // indirect
+	github.com/alexflint/go-arg v1.3.0
+	github.com/antchfx/htmlquery v1.2.2 // indirect
+	github.com/antchfx/xmlquery v1.2.3 // indirect
+	github.com/antchfx/xpath v1.1.4 // indirect
+	github.com/gobwas/glob v0.2.3 // indirect
+	github.com/gocolly/colly v1.2.0
+	github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect
+	github.com/golang/protobuf v1.3.4 // indirect
+	github.com/jawher/mow.cli v1.1.0 // indirect
+	github.com/kennygrant/sanitize v1.2.4 // indirect
+	github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca // indirect
+	github.com/temoto/robotstxt v1.1.1 // indirect
+	golang.org/x/net v0.0.0-20200301022130-244492dfa37a // indirect
+	google.golang.org/appengine v1.6.5 // indirect
+)

go.sum

@@ -0,0 +1,50 @@
+github.com/PuerkitoBio/goquery v1.5.1 h1:PSPBGne8NIUWw+/7vFBV+kG2J/5MOjbzc7154OaKCSE=
+github.com/PuerkitoBio/goquery v1.5.1/go.mod h1:GsLWisAFVj4WgDibEWF4pvYnkVQBpKBKeU+7zCJoLcc=
+github.com/alexflint/go-arg v1.3.0 h1:UfldqSdFWeLtoOuVRosqofU4nmhI1pYEbT4ZFS34Bdo=
+github.com/alexflint/go-arg v1.3.0/go.mod h1:9iRbDxne7LcR/GSvEr7ma++GLpdIU1zrghf2y2768kM=
+github.com/alexflint/go-scalar v1.0.0 h1:NGupf1XV/Xb04wXskDFzS0KWOLH632W/EO4fAFi+A70=
+github.com/alexflint/go-scalar v1.0.0/go.mod h1:GpHzbCOZXEKMEcygYQ5n/aa4Aq84zbxjy3MxYW0gjYw=
+github.com/andybalholm/cascadia v1.1.0 h1:BuuO6sSfQNFRu1LppgbD25Hr2vLYW25JvxHs5zzsLTo=
+github.com/andybalholm/cascadia v1.1.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y=
+github.com/antchfx/htmlquery v1.2.2 h1:exe4hUStBqXdRZ+9nB7EYA+W2zfIHIq3rRFpChh+VSk=
+github.com/antchfx/htmlquery v1.2.2/go.mod h1:MS9yksVSQXls00iXkiMqXr0J+umL/AmxXKuP28SUJM8=
+github.com/antchfx/xmlquery v1.2.3 h1:++irmxT+Pkn55FGtSTkUTHarZ6E0b1yyR+UiPZRA+eY=
+github.com/antchfx/xmlquery v1.2.3/go.mod h1:/+CnyD/DzHRnv2eRxrVbieRU/FIF6N0C+7oTtyUtCKk=
+github.com/antchfx/xpath v1.1.4 h1:naPIpjBGeT3eX0Vw7E8iyHsY8FGt6EbGdkcd8EZCo+g=
+github.com/antchfx/xpath v1.1.4/go.mod h1:Yee4kTMuNiPYJ7nSNorELQMr1J33uOpXDMByNYhvtNk=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y=
+github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8=
+github.com/gocolly/colly v1.2.0 h1:qRz9YAn8FIH0qzgNUw+HT9UN7wm1oF9OBAilwEWpyrI=
+github.com/gocolly/colly v1.2.0/go.mod h1:Hof5T3ZswNVsOHYmba1u03W65HDWgpV5HifSuueE0EA=
+github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e h1:1r7pUrabqp18hOBcwBwiTsbnFeTZHV9eER/QT5JVZxY=
+github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.4 h1:87PNWwrRvUSnqS4dlcBU/ftvOIBep4sYuBLlh6rX2wk=
+github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
+github.com/jawher/mow.cli v1.1.0/go.mod h1:aNaQlc7ozF3vw6IJ2dHjp2ZFiA4ozMIYY6PyuRJwlUg=
+github.com/kennygrant/sanitize v1.2.4 h1:gN25/otpP5vAsO2djbMhF/LQX6R7+O1TB4yv8NzpJ3o=
+github.com/kennygrant/sanitize v1.2.4/go.mod h1:LGsjYYtgxbetdg5owWB2mpgUL6e2nfw2eObZ0u0qvak=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca h1:NugYot0LIVPxTvN8n+Kvkn6TrbMyxQiuvKdEwFdR9vI=
+github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca/go.mod h1:uugorj2VCxiV1x+LzaIdVa9b4S4qGAcH6cbhh4qVxOU=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE=
+github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/temoto/robotstxt v1.1.1 h1:Gh8RCs8ouX3hRSxxK7B1mO5RFByQ4CmJZDwgom++JaA=
+github.com/temoto/robotstxt v1.1.1/go.mod h1:+1AmkuG3IYkh1kv0d2qEB9Le88ehNO0zwOr3ujewlOo=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
+golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200301022130-244492dfa37a h1:GuSPYbZzB5/dcLNCwLQLsg3obCJtX9IJhpXkvY7kzk0=
+golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
+golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+google.golang.org/appengine v1.6.5 h1:tycE03LOZYQNhDpS27tcQdAzLCVMaj7QT2SXxebnpCM=
+google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=

main.go

@@ -0,0 +1,75 @@
+package main
+
+import (
+	"bufio"
+	"github.com/alexflint/go-arg"
+	"github.com/kangoo13/cloudflare-proxy-ban-checker/cfprxchecker"
+	"log"
+	"os"
+	"strings"
+	"time"
+)
+
+func main() {
+	var args cfprxchecker.Args
+	var inputArgs struct {
+		Website         string `arg:"positional,required" help:"website to test proxy against cloudflare"`
+		ProxyList       string `arg:"positional,required" help:"path to the proxyList"`
+		GoodProxiesPath string `default:"good.txt" help:"path to the good proxies identified"`
+		BadProxiesPath  string `default:"bad.txt" help:"path to the bad proxies identified"`
+		TimeoutProxy    int64  `default:"5" help:"timeout proxy duration"`
+	}
+
+	arg.MustParse(&inputArgs)
+
+	if inputArgs.GoodProxiesPath != "" {
+		args.GoodProxiesOutputFile = OpenFile(inputArgs.GoodProxiesPath)
+		defer args.GoodProxiesOutputFile.Close()
+	}
+
+	if inputArgs.BadProxiesPath != "" {
+		args.BadProxiesOutputFile = OpenFile(inputArgs.BadProxiesPath)
+		defer args.BadProxiesOutputFile.Close()
+	}
+
+	args.WebsiteToCrawl = inputArgs.Website
+	args.ProxyList = FileToStringSlice(inputArgs.ProxyList)
+	args.TimeoutProxy = time.Duration(inputArgs.TimeoutProxy) * time.Second
+
+	cfprxchecker.CheckProxiesAgainstCloudFlare(&args)
+}
+
+func OpenFile(filePath string) *os.File {
+	f, err := os.OpenFile(filePath, os.O_APPEND|os.O_WRONLY|os.O_CREATE, os.ModeAppend|0660)
+	if err != nil {
+		log.Fatalf("[OpenFile] error while opening file %s", err)
+	}
+
+	return f
+}
+
+func FileToStringSlice(filePath string) []string {
+	var (
+		fileTextLines []string
+	)
+	readFile, err := os.Open(filePath)
+
+	if err != nil {
+		log.Fatalf("failed to open file: %s", err)
+	}
+
+	defer readFile.Close()
+
+	fileScanner := bufio.NewScanner(readFile)
+	fileScanner.Split(bufio.ScanLines)
+
+	for fileScanner.Scan() {
+		newLine := fileScanner.Text()
+		if strings.Trim(newLine, "\n\t\r") != "" {
+			fileTextLines = append(fileTextLines, fileScanner.Text())
+		}
+	}
+
+	return fileTextLines
+
+}