2021-09-13 08:30:49 +00:00
|
|
|
package pxndscvm
|
|
|
|
|
|
|
|
import (
|
2021-09-13 19:08:41 +00:00
|
|
|
"fmt"
|
2021-09-13 08:30:49 +00:00
|
|
|
"sync"
|
|
|
|
"time"
|
|
|
|
|
2021-09-13 19:08:41 +00:00
|
|
|
"github.com/alitto/pond"
|
2021-09-13 08:30:49 +00:00
|
|
|
rl "github.com/yunginnanet/Rate5"
|
|
|
|
)
|
|
|
|
|
2021-09-13 19:08:41 +00:00
|
|
|
// SwampStatus represents the current state of our Swamp.
|
|
|
|
type SwampStatus int
|
|
|
|
|
|
|
|
const (
|
|
|
|
// Running means the proxy pool is currently taking in proxys and validating them, and is available to dispense proxies.
|
|
|
|
Running SwampStatus = iota
|
|
|
|
// Paused means the proxy pool has been with Swamp.Pause() and may be resumed with Swamp.Resume()
|
|
|
|
Paused
|
|
|
|
)
|
|
|
|
|
2021-09-13 08:30:49 +00:00
|
|
|
// Swamp represents a proxy pool
|
|
|
|
type Swamp struct {
|
|
|
|
// Socks5 is a constant stream of verified Socks5 proxies
|
2021-09-13 11:26:18 +00:00
|
|
|
Socks5 chan Proxy
|
2021-09-13 08:30:49 +00:00
|
|
|
// Socks4 is a constant stream of verified Socks4 proxies
|
2021-09-13 11:26:18 +00:00
|
|
|
Socks4 chan Proxy
|
2021-09-13 08:30:49 +00:00
|
|
|
// Socks4a is a constant stream of verified Socks5 proxies
|
2021-09-13 11:26:18 +00:00
|
|
|
Socks4a chan Proxy
|
2021-09-13 08:30:49 +00:00
|
|
|
|
2021-09-13 09:37:59 +00:00
|
|
|
// Stats holds the Statistics for our swamp
|
|
|
|
Stats *Statistics
|
|
|
|
|
2021-09-13 19:08:41 +00:00
|
|
|
Status SwampStatus
|
|
|
|
|
2021-09-13 08:30:49 +00:00
|
|
|
// Pending is a constant stream of proxy strings to be verified
|
|
|
|
Pending chan string
|
|
|
|
|
2021-09-13 19:08:41 +00:00
|
|
|
quit chan bool
|
2021-09-13 08:30:49 +00:00
|
|
|
scvm []string
|
2021-09-13 19:08:41 +00:00
|
|
|
pool *pond.WorkerPool
|
2021-09-13 08:30:49 +00:00
|
|
|
swampopt *SwampOptions
|
2021-09-13 10:08:04 +00:00
|
|
|
started bool
|
2021-09-13 08:30:49 +00:00
|
|
|
mu *sync.RWMutex
|
|
|
|
}
|
|
|
|
|
|
|
|
var (
|
2021-09-13 16:01:19 +00:00
|
|
|
defaultStaleTime = 1 * time.Hour
|
2021-09-13 10:08:04 +00:00
|
|
|
defWorkers = 100
|
2021-09-13 17:55:51 +00:00
|
|
|
// Note: I've chosen to use https here exclusively assuring all validated proxies are SSL capable.
|
2021-09-13 17:40:23 +00:00
|
|
|
defaultChecks = []string{"https://wtfismyip.com/text", "https://myexternalip.com/raw", "https://ipinfo.io/ip", "https://api.ipify.org", "https://icanhazip.com/", "https://ifconfig.me/ip", "https://www.trackip.net/ip", "https://checkip.amazonaws.com/"}
|
2021-09-13 08:30:49 +00:00
|
|
|
)
|
|
|
|
|
|
|
|
func defOpt() *SwampOptions {
|
2021-09-13 10:08:04 +00:00
|
|
|
return &SwampOptions{
|
2021-09-14 01:50:39 +00:00
|
|
|
UserAgents: defaultUserAgents,
|
2021-09-13 17:40:23 +00:00
|
|
|
CheckEndpoints: defaultChecks,
|
|
|
|
Stale: defaultStaleTime,
|
|
|
|
MaxWorkers: defWorkers,
|
|
|
|
ValidationTimeout: 5,
|
|
|
|
Debug: false,
|
2021-09-13 10:08:04 +00:00
|
|
|
}
|
2021-09-13 08:30:49 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// SwampOptions holds our configuration for Swamp instances
|
|
|
|
type SwampOptions struct {
|
|
|
|
// UserAgents contains a list of UserAgents to be randomly drawn from for proxied requests, this should be supplied via SetUserAgents
|
|
|
|
UserAgents []string
|
|
|
|
// Stale is the amount of time since verification that qualifies a proxy going stale.
|
2021-09-13 09:37:59 +00:00
|
|
|
// if a stale proxy is drawn during the use of our getter functions, it will be skipped.
|
2021-09-13 08:30:49 +00:00
|
|
|
Stale time.Duration
|
2021-09-13 09:37:59 +00:00
|
|
|
// Debug when enabled will print results as they come in
|
|
|
|
Debug bool
|
2021-09-13 15:36:30 +00:00
|
|
|
// CheckEndpoints includes web services that respond with (just) the WAN IP of the connection for validation purposes
|
|
|
|
CheckEndpoints []string
|
2021-09-13 10:08:04 +00:00
|
|
|
// MaxWorkers determines the maximum amount of workers used for checking proxies
|
|
|
|
MaxWorkers int
|
2021-09-13 17:40:23 +00:00
|
|
|
// ValidationTimeout defines the timeout (in seconds) for proxy validation operations.
|
|
|
|
// This will apply for both the initial quick check (dial), and the second check (HTTP GET).
|
|
|
|
ValidationTimeout int
|
2021-09-13 08:30:49 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
var (
|
2021-09-13 16:32:07 +00:00
|
|
|
useProx *rl.Limiter
|
|
|
|
badProx *rl.Limiter
|
2021-09-13 08:30:49 +00:00
|
|
|
)
|
|
|
|
|
|
|
|
// Proxy represents an individual proxy
|
|
|
|
type Proxy struct {
|
2021-09-13 09:37:59 +00:00
|
|
|
// Endpoint is the address:port of the proxy that we connect to
|
2021-09-13 08:30:49 +00:00
|
|
|
Endpoint string
|
|
|
|
// ProxiedIP is the address that we end up having when making proxied requests through this proxy
|
|
|
|
ProxiedIP string
|
|
|
|
// Proto is the version/Protocol (currently SOCKS* only) of the proxy
|
|
|
|
Proto string
|
|
|
|
// Verified is the time this proxy was last verified working
|
|
|
|
Verified time.Time
|
|
|
|
}
|
|
|
|
|
|
|
|
// UniqueKey is an implementation of the Identity interface from Rate5
|
2021-09-13 11:26:18 +00:00
|
|
|
func (p Proxy) UniqueKey() string {
|
2021-09-13 08:30:49 +00:00
|
|
|
return p.Endpoint
|
|
|
|
}
|
|
|
|
|
|
|
|
func init() {
|
2021-09-14 04:34:09 +00:00
|
|
|
// see: https://pkg.go.dev/github.com/yunginnanet/Rate5
|
2021-09-13 08:30:49 +00:00
|
|
|
useProx = rl.NewLimiter(60, 2)
|
2021-09-14 04:34:09 +00:00
|
|
|
badProx = rl.NewStrictLimiter(60, 5)
|
2021-09-13 08:30:49 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// NewDefaultSwamp returns a Swamp with basic options.
|
|
|
|
func NewDefaultSwamp() *Swamp {
|
2021-09-13 10:08:04 +00:00
|
|
|
s := &Swamp{
|
2021-09-13 14:23:44 +00:00
|
|
|
Socks5: make(chan Proxy, 1000),
|
|
|
|
Socks4: make(chan Proxy, 1000),
|
|
|
|
Socks4a: make(chan Proxy, 1000),
|
|
|
|
Pending: make(chan string, 500),
|
2021-09-13 08:30:49 +00:00
|
|
|
|
2021-09-13 09:37:59 +00:00
|
|
|
Stats: &Statistics{
|
2021-09-13 15:17:20 +00:00
|
|
|
Valid4: 0,
|
|
|
|
Valid4a: 0,
|
|
|
|
Valid5: 0,
|
2021-09-13 11:26:18 +00:00
|
|
|
Dispensed: 0,
|
2021-09-14 02:47:16 +00:00
|
|
|
birthday: time.Now(),
|
2021-09-13 15:17:20 +00:00
|
|
|
mu: &sync.Mutex{},
|
2021-09-13 09:37:59 +00:00
|
|
|
},
|
|
|
|
|
2021-09-13 19:08:41 +00:00
|
|
|
quit: make(chan bool),
|
2021-09-13 08:30:49 +00:00
|
|
|
swampopt: defOpt(),
|
|
|
|
mu: &sync.RWMutex{},
|
|
|
|
}
|
2021-09-13 10:08:04 +00:00
|
|
|
|
2021-09-13 19:08:41 +00:00
|
|
|
s.pool = pond.New(s.swampopt.MaxWorkers, 10000, pond.PanicHandler(func(p interface{}) {
|
|
|
|
fmt.Println("WORKER PANIC! ", p)
|
|
|
|
}))
|
|
|
|
|
2021-09-13 11:26:18 +00:00
|
|
|
return s
|
2021-09-13 08:30:49 +00:00
|
|
|
}
|
2021-09-13 19:48:42 +00:00
|
|
|
|
2021-09-14 01:50:39 +00:00
|
|
|
// defaultUserAgents is a small list of user agents to use during validation.
|
|
|
|
var defaultUserAgents = []string{
|
2021-09-13 19:48:42 +00:00
|
|
|
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:60.0) Gecko/20100101 Firefox/60.0",
|
|
|
|
"Mozilla/5.0 (Windows NT 6.2; WOW64; rv:34.0) Gecko/20100101 Firefox/34.0",
|
|
|
|
"Mozilla/5.0 (Windows NT 6.2; Win64; x64; rv:24.0) Gecko/20140419 Firefox/24.0 PaleMoon/24.5.0",
|
|
|
|
"Mozilla/5.0 (X11; Ubuntu; Linux i686; rv:44.0) Gecko/20100101 Firefox/44.0",
|
|
|
|
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.9; rv:49.0) Gecko/20100101 Firefox/49.0",
|
|
|
|
"Mozilla/5.0 (X11; Ubuntu; Linux i686; rv:55.0) Gecko/20100101 Firefox/55.0",
|
|
|
|
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.11; rv:47.0) Gecko/20100101 Firefox/--.0",
|
|
|
|
"Mozilla/5.0 (Windows NT 6.0; rv:19.0) Gecko/20100101 Firefox/19.0",
|
|
|
|
"Mozilla/5.0 (X11; Ubuntu; Linux i686; rv:45.0) Gecko/20100101 Firefox/45.0",
|
|
|
|
"Mozilla/5.0 (Windows NT 6.0; WOW64; rv:45.0) Gecko/20100101 Firefox/45.0",
|
|
|
|
"Mozilla/5.0 (FreeBSD; Viera; rv:34.0) Gecko/20100101 Firefox/34.0",
|
|
|
|
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.7; rv:20.0) Gecko/20100101 Firefox/20.0",
|
|
|
|
"Mozilla/5.0 (Android 6.0; Mobile; rv:60.0) Gecko/20100101 Firefox/60.0",
|
|
|
|
"Mozilla/5.0 (Windows NT 5.1; rv:37.0) Gecko/20100101 Firefox/37.0",
|
|
|
|
"Mozilla/5.0 (Windows NT 6.1; WOW64; rv:35.0) Gecko/20100101 Firefox/35.0 evaliant",
|
|
|
|
"Mozilla/5.0 (Windows NT 6.1; WOW64; rv:28.0) Gecko/20100101 Firefox/28.0",
|
|
|
|
"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:58.0) Gecko/20100101 Firefox/58.0",
|
|
|
|
"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:60.0) Gecko/20100101 Firefox/60.0",
|
|
|
|
"Mozilla/5.0 (Windows NT 10.0; WOW64; rv:45.0) Gecko/20100101 Firefox/45.0",
|
|
|
|
"Mozilla/5.0 (Windows NT 6.2; WOW64; rv:41.0) Gecko/20100101 Firefox/41.0",
|
|
|
|
}
|