Initial commmit.

.gitignore

@@ -0,0 +1,5 @@
+*.txt
+*.bak
+*.swp
+*.save
+*.list

Makefile

@@ -0,0 +1,188 @@
+# The binary to build (just the basename).
+BIN := endless
+
+# Where to push the docker image.
+REGISTRY ?= TODO
+
+# This version-strategy uses git tags to set the version string
+# VERSION := $(shell git describe --tags --always --dirty)
+#
+# This version-strategy uses a manual value to set the version string
+VERSION := 0.0.1
+
+###
+### These variables should not need tweaking.
+###
+
+SRC_DIRS := cmd pkg # directories which hold app source (not vendored)
+
+ALL_PLATFORMS := linux/amd64 linux/arm linux/arm64 linux/ppc64le linux/s390x
+
+# Used internally.  Users should pass GOOS and/or GOARCH.
+OS := $(if $(GOOS),$(GOOS),$(shell go env GOOS))
+ARCH := $(if $(GOARCH),$(GOARCH),$(shell go env GOARCH))
+
+BASEIMAGE ?= gcr.io/distroless/static
+
+IMAGE := $(REGISTRY)/$(BIN)
+TAG := $(VERSION)__$(OS)_$(ARCH)
+
+BUILD_IMAGE ?= golang:latest
+
+# If you want to build all binaries, see the 'all-build' rule.
+# If you want to build all containers, see the 'all-container' rule.
+# If you want to build AND push all containers, see the 'all-push' rule.
+all: build
+
+# For the following OS/ARCH expansions, we transform OS/ARCH into OS_ARCH
+# because make pattern rules don't match with embedded '/' characters.
+
+build-%:
+	@$(MAKE) build                        \
+	    --no-print-directory              \
+	    GOOS=$(firstword $(subst _, ,$*)) \
+	    GOARCH=$(lastword $(subst _, ,$*))
+
+container-%:
+	@$(MAKE) container                    \
+	    --no-print-directory              \
+	    GOOS=$(firstword $(subst _, ,$*)) \
+	    GOARCH=$(lastword $(subst _, ,$*))
+
+push-%:
+	@$(MAKE) push                         \
+	    --no-print-directory              \
+	    GOOS=$(firstword $(subst _, ,$*)) \
+	    GOARCH=$(lastword $(subst _, ,$*))
+
+all-build: $(addprefix build-, $(subst /,_, $(ALL_PLATFORMS)))
+
+all-container: $(addprefix container-, $(subst /,_, $(ALL_PLATFORMS)))
+
+all-push: $(addprefix push-, $(subst /,_, $(ALL_PLATFORMS)))
+
+build: bin/$(OS)_$(ARCH)/$(BIN)
+
+# Directories that we need created to build/test.
+BUILD_DIRS := bin/$(OS)_$(ARCH)     \
+              .go/bin/$(OS)_$(ARCH) \
+              .go/cache
+
+# The following structure defeats Go's (intentional) behavior to always touch
+# result files, even if they have not changed.  This will still run `go` but
+# will not trigger further work if nothing has actually changed.
+OUTBIN = bin/$(OS)_$(ARCH)/$(BIN)
+$(OUTBIN): .go/$(OUTBIN).stamp
+	@true
+
+# This will build the binary under ./.go and update the real binary iff needed.
+.PHONY: .go/$(OUTBIN).stamp
+.go/$(OUTBIN).stamp: $(BUILD_DIRS)
+	@echo "making $(OUTBIN)"
+	@docker run                                                 \
+	    -i                                                      \
+	    --rm                                                    \
+	    -u $$(id -u):$$(id -g)                                  \
+	    -v $$(pwd):/src                                         \
+	    -w /src                                                 \
+	    -v $$(pwd)/.go/bin/$(OS)_$(ARCH):/go/bin                \
+	    -v $$(pwd)/.go/bin/$(OS)_$(ARCH):/go/bin/$(OS)_$(ARCH)  \
+	    -v $$(pwd)/.go/cache:/.cache                            \
+	    --env HTTP_PROXY=$(HTTP_PROXY)                          \
+	    --env HTTPS_PROXY=$(HTTPS_PROXY)                        \
+	    $(BUILD_IMAGE)                                          \
+	    /bin/sh -c "                                            \
+	        ARCH=$(ARCH)                                        \
+	        OS=$(OS)                                            \
+	        VERSION=$(VERSION)                                  \
+	        ./build/build.sh                                    \
+	    "
+	@if ! cmp -s .go/$(OUTBIN) $(OUTBIN); then \
+	    mv .go/$(OUTBIN) $(OUTBIN);            \
+	    date >$@;                              \
+	fi
+
+# Example: make shell CMD="-c 'date > datefile'"
+shell: $(BUILD_DIRS)
+	@echo "launching a shell in the containerized build environment"
+	@docker run                                                 \
+	    -ti                                                     \
+	    --rm                                                    \
+	    -u $$(id -u):$$(id -g)                                  \
+	    -v $$(pwd):/src                                         \
+	    -w /src                                                 \
+	    -v $$(pwd)/.go/bin/$(OS)_$(ARCH):/go/bin                \
+	    -v $$(pwd)/.go/bin/$(OS)_$(ARCH):/go/bin/$(OS)_$(ARCH)  \
+	    -v $$(pwd)/.go/cache:/.cache                            \
+	    --env HTTP_PROXY=$(HTTP_PROXY)                          \
+	    --env HTTPS_PROXY=$(HTTPS_PROXY)                        \
+	    $(BUILD_IMAGE)                                          \
+	    /bin/sh $(CMD)
+
+# Used to track state in hidden files.
+DOTFILE_IMAGE = $(subst /,_,$(IMAGE))-$(TAG)
+
+container: .container-$(DOTFILE_IMAGE) say_container_name
+.container-$(DOTFILE_IMAGE): bin/$(OS)_$(ARCH)/$(BIN) Dockerfile.in
+	@sed                                 \
+	    -e 's|{ARG_BIN}|$(BIN)|g'        \
+	    -e 's|{ARG_ARCH}|$(ARCH)|g'      \
+	    -e 's|{ARG_OS}|$(OS)|g'          \
+	    -e 's|{ARG_FROM}|$(BASEIMAGE)|g' \
+	    Dockerfile.in > .dockerfile-$(OS)_$(ARCH)
+	@docker build -t $(IMAGE):$(TAG) -f .dockerfile-$(OS)_$(ARCH) .
+	@docker images -q $(IMAGE):$(TAG) > $@
+
+say_container_name:
+	@echo "container: $(IMAGE):$(TAG)"
+
+push: .push-$(DOTFILE_IMAGE) say_push_name
+.push-$(DOTFILE_IMAGE): .container-$(DOTFILE_IMAGE)
+	@docker push $(IMAGE):$(TAG)
+
+say_push_name:
+	@echo "pushed: $(IMAGE):$(TAG)"
+
+manifest-list: all-push
+	platforms=$$(echo $(ALL_PLATFORMS) | sed 's/ /,/g');  \
+	manifest-tool                                         \
+	    --username=oauth2accesstoken                      \
+	    --password=$$(gcloud auth print-access-token)     \
+	    push from-args                                    \
+	    --platforms "$$platforms"                         \
+	    --template $(REGISTRY)/$(BIN):$(VERSION)__OS_ARCH \
+	    --target $(REGISTRY)/$(BIN):$(VERSION)
+
+version:
+	@echo $(VERSION)
+
+test: $(BUILD_DIRS)
+	@docker run                                                 \
+	    -i                                                      \
+	    --rm                                                    \
+	    -u $$(id -u):$$(id -g)                                  \
+	    -v $$(pwd):/src                                         \
+	    -w /src                                                 \
+	    -v $$(pwd)/.go/bin/$(OS)_$(ARCH):/go/bin                \
+	    -v $$(pwd)/.go/bin/$(OS)_$(ARCH):/go/bin/$(OS)_$(ARCH)  \
+	    -v $$(pwd)/.go/cache:/.cache                            \
+	    --env HTTP_PROXY=$(HTTP_PROXY)                          \
+	    --env HTTPS_PROXY=$(HTTPS_PROXY)                        \
+	    $(BUILD_IMAGE)                                          \
+	    /bin/sh -c "                                            \
+	        ARCH=$(ARCH)                                        \
+	        OS=$(OS)                                            \
+	        VERSION=$(VERSION)                                  \
+	        ./build/test.sh $(SRC_DIRS)                         \
+	    "
+
+$(BUILD_DIRS):
+	@mkdir -p $@
+
+clean: container-clean bin-clean
+
+container-clean:
+	rm -rf .container-* .dockerfile-* .push-*
+
+bin-clean:
+	rm -rf .go bin

README.md

@@ -0,0 +1,24 @@
+# Groceries
+
+Groceries will collect a configurable number of open source proxies from numerous provided sources, check the proxies, sort by numerous categories including geolocation, privacy and of course proxy type.
+
+## Installation
+```
+git clone https://git.tcp.direct/moony/groceries
+cd groceries
+make
+```
+
+## Requirements
+```
+go 1.19+
+```
+
+## Usage
+__TODO__: _Usage example here._
+
+### External Integrations
+
+[Prox5](https://git.tcp.direct/kayos/prox5)
+
+Groceries feeds it's proxy pool into [Prox5](https://git.tcp.direct/kayos/prox5) to create a very nice SOCKS5 proxy upstream by design. While easy to modify for other uses, [Prox5](https://git.tcp.direct/kayos/prox5) was intended to be integrated during development.

build/build.sh

@@ -0,0 +1,29 @@
+#!/bin/bash
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+if [ -z "${OS:-}" ]; then
+  echo "OS must be set"
+  exit 1
+fi
+if [ -z "${ARCH:-}" ]; then
+  echo "ARCH must be set"
+  exit 1
+fi
+if [ -z "${VERSION:-}" ]; then
+  echo "VERSION mus be set"
+  exit 1
+fi
+
+export CGO_ENABLED=0
+export GOARCH="${ARCH}"
+export GOOS="${OS}"
+export GO111MODULE=on
+export GOFLAGS="-mod=vendor"
+
+go install                                                    \
+  -installsuffix "static"                                     \
+  -ldflags "-x $(go list -m)/pkg/version.VERSION=${VERSION}"  \
+  ./...

build/test.sh

@@ -0,0 +1,39 @@
+#!/bin/bash
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+export CGO_ENABLED=0
+export GO111MODULE=on
+export GOFLAGS="-mod=vendor"
+
+TARGETS=$(for d in "$@"; do echo ./$d/...; done)
+
+echo "Running tests: "
+go test -iinstallsuffix "static" ${TARGETS}
+echo
+
+echo -n "Checking gofmt: "
+ERRS=$(find "$@" -type f -name \*.go | xargs gofmt -l 2>&1 || true)
+if [ -n "${ERRS}" ]; then
+  echo "FAIL - the files need to be gofmt'ed:"
+  for e in ${ERRS}; do
+    echo "  $e"
+  done
+  echo
+  exit 1
+fi
+echo "PASS"
+echo
+
+echo -n "Checking go vet: "
+ERRS=$(go vet "${TARGETS}" 2>&1 || true)
+if [ -n "${ERRS}" ]; then
+  echo "FAIL"
+  echo "${ERRS}"
+  echo
+  exit 1
+fi
+echo "PASS"
+echo

cmd/groceries/main.go

@@ -0,0 +1,48 @@
+package endless
+
+import (
+	"components/modules"
+	"components/utils"
+	"fmt"
+	"time"
+
+	"github.com/zenthangplus/goccm"
+)
+
+func main() {
+	utils.PrintLogo()
+	utils.LoadConfig()
+
+	if utils.Config.Options.Scrape {
+		modules.Scrape()
+	}
+
+	proxies, err := utils.ReadLines("proxies.txt")
+	if utils.HandleError(err) {
+		return
+	}
+
+	proxies = utils.RemoveDuplicateStr(proxies)
+	utils.Log(fmt.Sprintf("[*] Loaded [%d] proxies", len(proxies)))
+
+	StartTime := time.Now()
+	c := goccm.New(utils.Config.Options.Threads)
+
+	for _, proxy := range proxies {
+		c.Wait()
+
+		go func(proxy string) {
+			modules.CheckProxy(proxy)
+			c.Done()
+		}(proxy)
+	}
+
+	c.WaitAllDone()
+	utils.Log(
+		fmt.Sprintf(
+			"[*] Checked [%d] proxies in [%fs]",
+			len(proxies),
+			time.Since(StartTime).Seconds(),
+		),
+	)
+}

components/modules/checker.go

@@ -0,0 +1,81 @@
+package modules
+
+import (
+	"components/utils"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"net/url"
+	"strings"
+	"time"
+
+	"h12.io/socks"
+)
+
+func GetHttpTransport(Proxy string) *http.Transport {
+	ProxyUrl, err := url.Parse(Proxy)
+	if utils.HandleError(err) {
+		return &http.Transport{}
+	}
+
+	return &http.Transport{
+		Proxy: http.ProxyURL(ProxyUrl),
+	}
+}
+
+func GetSocksTranport(Proxy string) *http.Transport {
+	return &http.Transport{
+		Dial: socks.Dial(fmt.Sprintf("%s?timeout=%ds", Proxy, utils.Config.Filter.Timeout)),
+	}
+}
+
+func GetTransport(Proxy string) *http.Transport {
+	if strings.Contains(Proxy, "http://") {
+		return GetHttpTransport(Proxy)
+	} else {
+		return GetSocksTranport(Proxy)
+	}
+}
+
+func ProxyReq(req string, proxy string) (res *http.Response, err error) {
+	ReqUrl, err := url.Parse(req)
+	if utils.HandleError(err) {
+		return nil, err
+	}
+
+	client := &http.Client{
+		Timeout:   time.Duration(time.Duration(utils.Config.Filter.Timeout) * time.Second),
+		Transport: GetTransport(proxy),
+	}
+
+	res, err = client.Get(ReqUrl.String())
+	return res, err
+}
+
+func CheckProxy(Proxy string) {
+	response, err := ProxyReq("https://api.ipify.org", Proxy)
+	if err != nil {
+		if utils.Config.Options.ShowDeadProxies {
+			utils.Log(fmt.Sprintf("[x] [DEAD] [%s]", Proxy))
+		}
+		return
+	}
+
+	defer response.Body.Close()
+
+	content, err := ioutil.ReadAll(response.Body)
+	if utils.HandleError(err) {
+		return
+	}
+
+	is_elite := string(content) != utils.ActualIp
+
+	utils.Log(fmt.Sprintf("[!] [ALIVE] [ELITE: %v] [%s]", is_elite, Proxy))
+	utils.Valid++
+
+	if !is_elite && !utils.Config.Options.SaveTransparent {
+		return
+	}
+
+	utils.AppendFile("checked.txt", Proxy)
+}

components/modules/scraper.go

@@ -0,0 +1,72 @@
+package modules
+
+import (
+	"components/utils"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"strings"
+	"time"
+
+	"github.com/zenthangplus/goccm"
+)
+
+func ScrapeUrl(Url string, ProxyType string) {
+	utils.Log(fmt.Sprintf("[+] Scraping [%s] proxies from [%s]", ProxyType, Url))
+
+	resp, err := http.Get(Url)
+	if utils.HandleError(err) {
+		return
+	}
+
+	defer resp.Body.Close()
+
+	if resp.StatusCode == 403 || resp.StatusCode == 404 {
+		return
+	}
+
+	content, err := ioutil.ReadAll(resp.Body)
+	if utils.HandleError(err) {
+		return
+	}
+
+	lines := strings.Split(string(content), "\n")
+
+	for _, proxy := range lines {
+		if proxy == "" {
+			continue
+		}
+
+		utils.AppendFile("proxies.txt", fmt.Sprintf("%s://%s", ProxyType, proxy))
+	}
+}
+
+func Scrape() {
+	url_list, err := utils.ReadLines("url,csv")
+	if utils.HandleError(err) {
+		return
+	}
+
+	StartTime := time.Now()
+	c := goccm.New(utils.Config.Options.ScrapeThreads)
+
+	for _, url := range url_list {
+		c.Wait()
+
+		s := strings.Split(url, ",")
+
+		go func(u string, t string) {
+			ScrapeUrl(u, t)
+			c.Done()
+		}(s[1], s[0])
+	}
+
+	c.WaitAllDone()
+	utils.Log(
+		fmt.Sprintf(
+			"[*] Scraped [%d] urils in [%fs]",
+			len(url_list),
+			time.Since(StartTime).Seconds(),
+		),
+	)
+}

components/utils/config.go

@@ -0,0 +1,55 @@
+package utils
+
+import (
+	"io"
+	"net/http"
+
+	"github.com/BurntSushi/toml"
+)
+
+var (
+	Config   = ConfigStruct{}
+	ActualIp string
+	Valid    = 0
+)
+
+type ConfigStruct struct {
+	Filter struct {
+		Timeout int `toml:"timeout"`
+	} `toml:"filter"`
+
+	Dev struct {
+		Debug bool `toml:"debug"`
+	} `toml:"dev"`
+
+	Options struct {
+		Scrape          bool `toml:"scrape"`
+		Threads         int  `toml:"threads"`
+		ScrapeThreads   int  `toml:"scrape_threads"`
+		SaveTransparent bool `toml:"save_transparent"`
+		ShowDeadProxies bool `toml:"show_dead_proxies"`
+	} `toml:"options"`
+}
+
+func GetActualIp() string {
+	res, err := http.Get("https://api.ipify.org")
+	if HandleError(err) {
+		return ""
+	}
+
+	defer res.Body.Close()
+
+	content, err := io.ReadAll(res.Body)
+	if HandleError(err) {
+		return ""
+	}
+
+	return string(content)
+}
+
+func LoadConfig() {
+	if _, err := toml.DecodeFile("script/config.toml", &Config); err != nil {
+		panic(err)
+	}
+	ActualIp = GetActualIp()
+}

components/utils/console.go

@@ -0,0 +1,47 @@
+package utils
+
+import (
+	"fmt"
+	"strings"
+	"time"
+
+	"github.com/gookit/color"
+	"github.com/inancgumus/screen"
+)
+
+func PrintLogo() {
+	screen.Clear()
+	screen.MoveTopLeft()
+
+	fmt.Println(`
+	__  ______                      
+	\ \/ /  _ \ _ __ _____  ___   _ 
+	 \  /| |_) | '__/ _ \ \/ / | | |
+	 /  \|  __/| | | (_) >  <| |_| |
+	/_/\_\_|   |_|  \___/_/\_\\__, |
+                                   |___/ 
+    
+      Coded by Moony - 9d5 until infinity...
+   `)
+}
+
+func Log(Content string) {
+	date := strings.ReplaceAll(time.Now().Format("15:04:05"), ":", "<fg=353a3b>:</>")
+	content := fmt.Sprintf("[%s] [%d] %s.", date, Valid, Content)
+
+	content = strings.ReplaceAll(content, "DEAD", "<fg=f5291b>DEAD</>")
+	content = strings.ReplaceAll(content, "ALIVE", "<fg=61eb42>ALIVE</>")
+
+	color.Println(content)
+}
+
+func HandleError(Err error) bool {
+	if Err != nil {
+		if Config.Dev.Debug {
+			fmt.Println(Err)
+		}
+		return true
+	}
+
+	return false
+}

components/utils/io.go

@@ -0,0 +1,64 @@
+package utils
+
+import (
+	"bufio"
+	"fmt"
+	"os"
+	"strings"
+)
+
+func ReadLines(path string) ([]string, error) {
+	f, err := os.Open(fmt.Sprintf("data/%s", path))
+	if err != nil {
+		return nil, err
+	}
+	defer f.Close()
+
+	r := bufio.NewReader(f)
+	bytes := []byte{}
+	lines := []string{}
+	for {
+		line, isPrefix, err := r.ReadLine()
+		if err != nil {
+			break
+		}
+		bytes = append(bytes, line...)
+		if !isPrefix {
+			str := strings.TrimSpace(string(bytes))
+			if len(str) > 0 {
+				lines = append(lines, str)
+				bytes = []byte{}
+			}
+		}
+	}
+	if len(bytes) > 0 {
+		lines = append(lines, string(bytes))
+	}
+	return lines, nil
+}
+
+func AppendFile(FileName string, Content string) {
+	File, err := os.OpenFile(fmt.Sprintf("data/%s", FileName), os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
+	if HandleError(err) {
+		return
+	}
+
+	_, err = File.WriteString(Content + "\n")
+	if HandleError(err) {
+		return
+	}
+
+	File.Close()
+}
+
+func RemoveDuplicateStr(strSlice []string) []string {
+	allKeys := make(map[string]bool)
+	list := []string{}
+	for _, item := range strSlice {
+		if _, value := allKeys[item]; !value {
+			allKeys[item] = true
+			list = append(list, item)
+		}
+	}
+	return list
+}

data/url.csv

@@ -0,0 +1,105 @@
+http,http://ab57.ru/downloads/proxyold.txt
+http,http://alexa.lr2b.com/proxylist.txt
+http,http://proxysearcher.sourceforge.net/Proxy%20List.php?type=http
+http,http://rootjazz.com/proxies/proxies.txt
+http,http://spys.me/proxy.txt
+http,http://worm.rip/http.txt
+http,http://www.proxyserverlist24.top/feeds/posts/default
+http,https://api.openproxylist.xyz/http.txt
+http,https://api.proxyscrape.com/?request=displayproxies&proxytype=http
+http,https://api.proxyscrape.com/v2/?request=getproxies&protocol=http
+http,https://api.proxyscrape.com/v2/?request=getproxies&protocol=https
+http,https://multiproxy.org/txt_all/proxy.txt
+http,https://proxy-spider.com/api/proxies.example.txt
+http,https://proxyspace.pro/http.txt
+http,https://proxyspace.pro/https.txt
+http,https://raw.githubusercontent.com/B4RC0DE-TM/proxy-list/main/HTTP.txt
+http,https://raw.githubusercontent.com/clarketm/proxy-list/master/proxy-list-raw.txt
+http,https://raw.githubusercontent.com/hookzof/socks5_list/master/proxy.txt
+http,https://raw.githubusercontent.com/HyperBeats/proxy-list/main/http.txt
+http,https://raw.githubusercontent.com/HyperBeats/proxy-list/main/https.txt
+http,https://raw.githubusercontent.com/jetkai/proxy-list/main/online-proxies/txt/proxies-http.txt
+http,https://raw.githubusercontent.com/mmpx12/proxy-list/master/http.txt
+http,https://raw.githubusercontent.com/monosans/proxy-list/main/proxies/http.txt
+http,https://raw.githubusercontent.com/monosans/proxy-list/main/proxies_anonymous/http.txt
+http,https://raw.githubusercontent.com/opsxcq/proxy-list/master/list.txt
+http,https://raw.githubusercontent.com/proxy4parsing/proxy-list/main/http.txt
+http,https://raw.githubusercontent.com/roosterkid/openproxylist/main/HTTPS_RAW.txt
+http,https://raw.githubusercontent.com/RX4096/proxy-list/main/online/all.txt
+http,https://raw.githubusercontent.com/saschazesiger/Free-Proxies/master/proxies/http.txt
+http,https://raw.githubusercontent.com/ShiftyTR/Proxy-List/master/http.txt
+http,https://raw.githubusercontent.com/ShiftyTR/Proxy-List/master/https.txt
+http,https://raw.githubusercontent.com/shiftytr/proxy-list/master/proxy.txt
+http,https://raw.githubusercontent.com/sunny9577/proxy-scraper/master/proxies.txt
+http,https://raw.githubusercontent.com/TheSpeedX/PROXY-List/master/http.txt
+http,https://raw.githubusercontent.com/TheSpeedX/SOCKS-List/master/http.txt
+http,https://raw.githubusercontent.com/UserR3X/proxy-list/main/online/http.txt
+http,https://raw.githubusercontent.com/UserR3X/proxy-list/main/online/https.txt
+http,https://sheesh.rip/http.txt
+http,https://www.freeproxychecker.com/result/http_proxies.txt
+http,https://www.my-proxy.com/free-anonymous-proxy.html
+http,https://www.my-proxy.com/free-proxy-list-10.html
+http,https://www.my-proxy.com/free-proxy-list-2.html
+http,https://www.my-proxy.com/free-proxy-list-3.html
+http,https://www.my-proxy.com/free-proxy-list-4.html
+http,https://www.my-proxy.com/free-proxy-list-5.html
+http,https://www.my-proxy.com/free-proxy-list-6.html
+http,https://www.my-proxy.com/free-proxy-list-7.html
+http,https://www.my-proxy.com/free-proxy-list-8.html
+http,https://www.my-proxy.com/free-proxy-list-9.html
+http,https://www.my-proxy.com/free-proxy-list.html
+http,https://www.my-proxy.com/free-transparent-proxy.html
+http,https://www.proxy-list.download/api/v1/get?type=http
+http,https://www.proxyscan.io/download?type=http
+http,https://www.proxyscan.io/download?type=https
+http,https://proxy11.com/api/proxy.txt?key=NDAzNg.YYHPVA.QB8moHDjsHJ_R_q8lkgkUV3wt2c
+socks4,https://proxylist.geonode.com/api/proxy-list?limit=50&page=1&sort_by=lastChecked&sort_type=desc&protocols=socks4
+socks4,http://worm.rip/socks4.txt
+socks4,http://www.socks24.org/feeds/posts/default
+socks4,https://api.openproxylist.xyz/socks4.txt
+socks4,https://api.proxyscrape.com/?request=displayproxies&proxytype=socks4
+socks4,https://api.proxyscrape.com/?request=displayproxies&proxytype=socks4&country=all
+socks4,https://api.proxyscrape.com/v2/?request=displayproxies&protocol=socks4
+socks4,https://api.proxyscrape.com/v2/?request=getproxies&protocol=socks4
+socks4,https://proxyspace.pro/socks4.txt
+socks4,https://raw.githubusercontent.com/B4RC0DE-TM/proxy-list/main/SOCKS4.txt
+socks4,https://raw.githubusercontent.com/HyperBeats/proxy-list/main/socks4.txt
+socks4,https://raw.githubusercontent.com/jetkai/proxy-list/main/online-proxies/txt/proxies-socks4.txt
+socks4,https://raw.githubusercontent.com/mmpx12/proxy-list/master/socks4.txt
+socks4,https://raw.githubusercontent.com/monosans/proxy-list/main/proxies/socks4.txt
+socks4,https://raw.githubusercontent.com/monosans/proxy-list/main/proxies_anonymous/socks4.txt
+socks4,https://raw.githubusercontent.com/roosterkid/openproxylist/main/SOCKS4_RAW.txt
+socks4,https://raw.githubusercontent.com/saschazesiger/Free-Proxies/master/proxies/socks4.txt
+socks4,https://raw.githubusercontent.com/ShiftyTR/Proxy-List/master/socks4.txt
+socks4,https://raw.githubusercontent.com/TheSpeedX/PROXY-List/blob/master/socks4.txt
+socks4,https://raw.githubusercontent.com/TheSpeedX/PROXY-List/master/socks4.txt
+socks4,https://raw.githubusercontent.com/TheSpeedX/SOCKS-List/master/socks4.txt
+socks4,https://www.freeproxychecker.com/result/socks4_proxies.txt
+socks4,https://www.my-proxy.com/free-socks-4-proxy.html
+socks4,https://www.proxy-list.download/api/v1/get?type=socks4
+socks4,https://www.proxyscan.io/download?type=socks4
+socks5,https://proxylist.geonode.com/api/proxy-list?limit=50&page=1&sort_by=lastChecked&sort_type=desc&protocols=socks5
+socks5,http://worm.rip/socks5.txt
+socks5,http://www.socks24.org/feeds/posts/default
+socks5,https://api.openproxylist.xyz/socks5.txt
+socks5,https://api.proxyscrape.com/?request=displayproxies&proxytype=socks5
+socks5,https://api.proxyscrape.com/v2/?request=displayproxies&protocol=socks5
+socks5,https://api.proxyscrape.com/v2/?request=getproxies&protocol=socks5
+socks5,https://api.proxyscrape.com/v2/?request=getproxies&protocol=socks5&timeout=10000&country=all&simplified=true
+socks5,https://proxyspace.pro/socks5.txt
+socks5,https://raw.githubusercontent.com/B4RC0DE-TM/proxy-list/main/SOCKS5.txt
+socks5,https://raw.githubusercontent.com/hookzof/socks5_list/master/proxy.txt
+socks5,https://raw.githubusercontent.com/HyperBeats/proxy-list/main/socks5.txt
+socks5,https://raw.githubusercontent.com/jetkai/proxy-list/main/online-proxies/txt/proxies-socks5.txt
+socks5,https://raw.githubusercontent.com/manuGMG/proxy-365/main/SOCKS5.txt
+socks5,https://raw.githubusercontent.com/mmpx12/proxy-list/master/socks5.txt
+socks5,https://raw.githubusercontent.com/monosans/proxy-list/main/proxies/socks5.txt
+socks5,https://raw.githubusercontent.com/monosans/proxy-list/main/proxies_anonymous/socks5.txt
+socks5,https://raw.githubusercontent.com/roosterkid/openproxylist/main/SOCKS5_RAW.txt
+socks5,https://raw.githubusercontent.com/saschazesiger/Free-Proxies/master/proxies/socks5.txt
+socks5,https://raw.githubusercontent.com/ShiftyTR/Proxy-List/master/socks5.txt
+socks5,https://raw.githubusercontent.com/TheSpeedX/PROXY-List/master/socks5.txt
+socks5,https://www.freeproxychecker.com/result/socks5_proxies.txt
+socks5,https://www.my-proxy.com/free-socks-5-proxy.html
+socks5,https://www.proxy-list.download/api/v1/get?type=socks5
+socks5,https://www.proxyscan.io/download?type=socks5

go.mod

@@ -0,0 +1,18 @@
+module endless
+
+go 1.20
+
+require (
+	github.com/BurntSushi/toml v1.2.1
+	github.com/gookit/color v1.5.2
+	github.com/inancgumus/screen v0.0.0-20190314163918-06e984b86ed3
+	github.com/zenthangplus/goccm v1.1.3
+	h12.io/socks v1.0.3
+)
+
+require (
+	github.com/xo/terminfo v0.0.0-20210125001918-ca9a967f8778 // indirect
+	golang.org/x/crypto v0.7.0 // indirect
+	golang.org/x/sys v0.6.0 // indirect
+	golang.org/x/term v0.6.0 // indirect
+)

go.sum

@@ -0,0 +1,37 @@
+github.com/BurntSushi/toml v1.2.1 h1:9F2/+DoOYIOksmaJFPw1tGFy1eDnIJXg+UHjuD8lTak=
+github.com/BurntSushi/toml v1.2.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/gookit/color v1.5.2 h1:uLnfXcaFjlrDnQDT+NCBcfhrXqYTx/rcCa6xn01Y8yI=
+github.com/gookit/color v1.5.2/go.mod h1:w8h4bGiHeeBpvQVePTutdbERIUf3oJE5lZ8HM0UgXyg=
+github.com/h12w/go-socks5 v0.0.0-20200522160539-76189e178364 h1:5XxdakFhqd9dnXoAZy1Mb2R/DZ6D1e+0bGC/JhucGYI=
+github.com/h12w/go-socks5 v0.0.0-20200522160539-76189e178364/go.mod h1:eDJQioIyy4Yn3MVivT7rv/39gAJTrA7lgmYr8EW950c=
+github.com/inancgumus/screen v0.0.0-20190314163918-06e984b86ed3 h1:fO9A67/izFYFYky7l1pDP5Dr0BTCRkaQJUG6Jm5ehsk=
+github.com/inancgumus/screen v0.0.0-20190314163918-06e984b86ed3/go.mod h1:Ey4uAp+LvIl+s5jRbOHLcZpUDnkjLBROl15fZLwPlTM=
+github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2 h1:JhzVVoYvbOACxoUmOs6V/G4D5nPVUW73rKvXxP4XUJc=
+github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2/go.mod h1:iIss55rKnNBTvrwdmkUpLnDpZoAHvWaiq5+iMmen4AE=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
+github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk=
+github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
+github.com/xo/terminfo v0.0.0-20210125001918-ca9a967f8778 h1:QldyIu/L63oPpyvQmHgvgickp1Yw510KJOqX7H24mg8=
+github.com/xo/terminfo v0.0.0-20210125001918-ca9a967f8778/go.mod h1:2MuV+tbUrU1zIOPMxZ5EncGwgmMJsa+9ucAQZXxsObs=
+github.com/zenthangplus/goccm v1.1.3 h1:66XVj24yexO2fCkBum8b+y6tOJ7Giq05LIn2vn3whGE=
+github.com/zenthangplus/goccm v1.1.3/go.mod h1:DUzu/BC4TkgUfXP8J1P6Md73Djt+0l0CHq001Pt4weA=
+golang.org/x/crypto v0.7.0 h1:AvwMYaRytfdeVt3u6mLaxYtErKYjxA2OXjJ1HHq6t3A=
+golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU=
+golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.6.0 h1:MVltZSvRTcU2ljQOhs94SXPftV6DCNnZViHeQps87pQ=
+golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/term v0.6.0 h1:clScbb1cHjoCkyRbWwBEUZ5H/tIFu5TAXIqaZD0Gcjw=
+golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+h12.io/socks v1.0.3 h1:Ka3qaQewws4j4/eDQnOdpr4wXsC//dXtWvftlIcCQUo=
+h12.io/socks v1.0.3/go.mod h1:AIhxy1jOId/XCz9BO+EIgNL2rQiPTBNnOfnVnQ+3Eck=

pkg/version/version.go

@@ -0,0 +1,3 @@
+package version
+
+var VERSION = "0.0.1"

scripts/config.toml

@@ -0,0 +1,12 @@
+[filter]
+timeout = 10                     # proxies timeout in seconds.
+
+[options]
+scrape = true                    # scrape proxies from url.csv.
+threads = 800                    # maximum checking threads.
+scrape_threads = 50              # maximum threads while scraping proxies.
+save_transparent = false         # transparent proxies don't hide your real IP.
+show_dead_proxies = false        # print dead proxies in console, disable it can give better performances
+
+[dev]
+debug = false                     # show errors

scripts/parse.bat

@@ -0,0 +1,5 @@
+@echo off
+cd ..
+title XProxy parser - git.tcp.direct/moony
+python parser.py
+pause

scripts/parser.py

@@ -0,0 +1,16 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+# This script is helpful to parse checked proxies.
+with open("../script/socks4.txt", "w+") as socks4_fd:
+    with open("../script/socks5.txt", "w+") as socks5_fd:
+        with open("../script/http.txt", "w+") as http_fd:
+            for proxy in list(set(open("../data/checked.txt", "r+").read().splitlines())):
+                if "socks4" in proxy:
+                    socks4_fd.write(f'{proxy.split("://")[1]}\n')
+
+                if "socks5" in proxy:
+                    socks5_fd.write(f'{proxy.split("://")[1]}\n')
+
+                if "http" in proxy:
+                    http_fd.write(f'{proxy.split("://")[1]}\n')

scripts/start.bat

@@ -0,0 +1,5 @@
+@echo off
+cd ..
+title Groceries - git.tcp.direct/moony
+go run .
+pause