o

.gitignore

@@ -1,2 +1,7 @@
 **/__pycache__
-**/.venv
+**/.venv
+
+**/dead_proxies.txt
+**/proxies_to_check.txt
+**/proxy_log.log
+**/valid_proxies.txt

proxy_lists.txt

@@ -0,0 +1,13 @@
+https://raw.githubusercontent.com/TheSpeedX/SOCKS-List/master/socks5.txt
+https://raw.githubusercontent.com/TheSpeedX/SOCKS-List/master/socks4.txt
+https://raw.githubusercontent.com/stamparm/aux/master/fetch-some-list.txt
+https://api.proxyscrape.com/v2/?request=getproxies&protocol=socks5&timeout=10000&country=all&simplified=true
+https://api.proxyscrape.com/v2/?request=getproxies&protocol=socks4&timeout=10000&country=all&simplified=true
+https://spys.me/proxy.txt
+https://spys.me/socks.txt
+https://raw.githubusercontent.com/clarketm/proxy-list/master/proxy-list-raw.txt
+https://raw.githubusercontent.com/sunny9577/proxy-scraper/master/proxies.txt
+https://raw.githubusercontent.com/jetkai/proxy-list/main/online-proxies/txt/proxies-socks4%2B5.txt
+https://raw.githubusercontent.com/roosterkid/openproxylist/main/SOCKS5_RAW.txt
+https://raw.githubusercontent.com/roosterkid/openproxylist/main/SOCKS4_RAW.txt
+https://raw.githubusercontent.com/User-R3X/proxy-list/main/online/socks4%2B5.txt

pull_lists.py

@@ -0,0 +1,43 @@
+from requests import request
+from pathlib import Path
+from re import compile
+from tqdm import tqdm
+
+proxy_re = compile(
+    r"(?=^((?:(?:(?P<protocol>socks[4-5]):\/\/)?(?:(?P<username>[a-z0-9A-Z]+):(?P<password>[a-z0-9A-Z]+)@)?)?(?P<ipaddr>\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}):(?P<port>[^\n\s]+)))\1"
+)
+
+PULLED = Path("./pulled_proxies.txt")
+LISTS = Path("./proxy_lists.txt")
+PROXIES = set()
+
+
+with LISTS.open("r") as f:
+    for line in tqdm(f):
+        line = line.rstrip("\n")
+        with request("GET", line) as resp:
+            try:
+                data = resp.json()
+                if isinstance(data, list):
+                    for proxy in data:
+                        PROXIES.add(f"{proxy['ip']}:{proxy['port']}")
+                    continue
+
+            except Exception as e:
+                ...
+
+            text = resp.text
+
+            for proxy in text.splitlines():
+                match = proxy_re.search(proxy.rstrip('\n'))
+                if match:
+                    PROXIES.add(match.group(0))
+                continue
+
+with PULLED.open("a+") as f:
+    for p in tqdm(PROXIES):
+        f.write(f"{p}""\n")
+
+# https://raw.githubusercontent.com/User-R3X/proxy-list/main/archive/all.txt
+
+# https://raw.githubusercontent.com/jetkai/proxy-list/main/archive/txt/working-proxies-history.txt

run.py

@@ -2,6 +2,7 @@ from asyncio import Queue, get_running_loop, new_event_loop, gather, Event
 from ipaddress import AddressValueError, IPv4Address
 from logging import basicConfig, log
 from re import compile
+from tqdm import tqdm
 
 from python_socks.async_.asyncio import ProxyType
 from python_socks.async_.asyncio.v2 import Proxy
@@ -21,12 +22,8 @@ except ImportError:
     print("Unable to find ssl, we will not be able to make any connections using 443 unless ssl is installed")
     CHECK_PORT = 80
 
-CHECK_ADDR: str | IPv4Address = IPv4Address("198.204.232.125")
-CHECK_HOST_NAME: str = "tcp.ac"
-CHECK_TIMEOUT: float = 10.0
-WORKERS = 1000
 
-CHECK_PROXIES = "./proxies_to_check.txt"
+CHECK_PROXIES = "./pulled_proxies.txt"
 VALID_PROXIES = "./valid_proxies.txt"
 DEAD_PROXIES = "./dead_proxies.txt"
 CHECKING_LOG = "./proxy_log.log"
@@ -34,6 +31,11 @@ CHECKING_LOG = "./proxy_log.log"
 LOG_FMT = "%(asctime)s:%(levelname)s:%(levelno)s:%(lineno)d - %(message)s"
 basicConfig(filename=CHECKING_LOG, filemode="w+", level=1, format=LOG_FMT, datefmt='[%D %H:%M:%S]', )
 
+CHECK_ADDR: str | IPv4Address = IPv4Address("198.204.232.125")
+CHECK_HOST_NAME: str = "tcp.ac"
+CHECK_TIMEOUT: float = 10.0
+WORKERS = 1000
+
 loop = new_event_loop()
 
 TASKS = []
@@ -44,7 +46,7 @@ UNRESPONSIVE = set()
 CLOSING = Event()
 
 proxy_re = compile(
-    r"(?=^(?:(?P<protocol>socks[4-5]):\/\/)?(?:(?P<username>[^:]+):(?P<password>[^@]+)@)?(?P<ipaddr>[^:]+):(?P<port>[^$]+)$)"
+    r"(?=^(?:(?P<protocol>socks[4-5]):\/\/)?(?:(?P<username>[^:]+):(?P<password>[^@]+)@)?(?P<ipaddr>[^:]+):(?P<port>[^\n]+)\n)"
 )
 
 with open(CHECK_PROXIES, "r") as f:
@@ -55,7 +57,7 @@ with open(CHECK_PROXIES, "r") as f:
 
         # TODO: Pretty sure could identify by port used but just check all anyways for now
 
-        PROXIES.add((ProxyType.HTTP, match.group("ipaddr"), match.group("port"), match.groupdict().get("username", None), match.groupdict().get("password", None),))
+        # PROXIES.add((ProxyType.HTTP, match.group("ipaddr"), match.group("port"), match.groupdict().get("username", None), match.groupdict().get("password", None),))
         PROXIES.add((ProxyType.SOCKS4, match.group("ipaddr"), match.group("port"), match.groupdict().get("username", None), match.groupdict().get("password", None),))
         PROXIES.add((ProxyType.SOCKS5, match.group("ipaddr"), match.group("port"), match.groupdict().get("username", None), match.groupdict().get("password", None),))
 
@@ -72,10 +74,18 @@ Cache-Control: max-age=0""".encode("utf8")
 
 
 def save_proxies():
-    with open(VALID_PROXIES, "w+") as f:
+    with open(VALID_PROXIES, "a+") as f:
+        for line in f:
+            LIVE_PROXIES.add(line)
+
+        f.seek(0)
         f.writelines(LIVE_PROXIES)
 
-    with open(DEAD_PROXIES, "w+") as f:
+    with open(DEAD_PROXIES, "a+") as f:
+        for line in f:
+            UNRESPONSIVE.add(line)
+
+        f.seek(0)
         f.writelines(UNRESPONSIVE)
 
 
@@ -134,10 +144,10 @@ async def proxy_check():
             while reading and _stream:
                 try:
                     if not page_requested:
-                        await _stream.write_all(REQUEST_PAGE)
+                        await _stream.write_all(REQUEST_PAGE)  # type: ignore
                         page_requested = True
 
-                    resp = await _stream.reader.readline()
+                    resp = await _stream.reader.readline()  # type: ignore
 
                 except Exception as e:
                     log(10, f"Exception when interacting with stream | {type(e)} - {e}")
@@ -153,7 +163,7 @@ async def begin():
     for _ in range(WORKERS):
         TASKS.append(_loop.create_task(proxy_check()))
 
-    for proxy in PROXIES:
+    for proxy in tqdm(PROXIES):
         await PROXY_QUEUE.put(proxy)
 
     CLOSING.set()