303 lines
9.4 KiB
Go
303 lines
9.4 KiB
Go
package prox5
|
|
|
|
import (
|
|
"container/list"
|
|
"context"
|
|
"sync"
|
|
"sync/atomic"
|
|
"time"
|
|
|
|
"git.tcp.direct/kayos/common/entropy"
|
|
cmap "github.com/orcaman/concurrent-map/v2"
|
|
"github.com/panjf2000/ants/v2"
|
|
rl "github.com/yunginnanet/Rate5"
|
|
|
|
"git.tcp.direct/kayos/prox5/internal/scaler"
|
|
"git.tcp.direct/kayos/prox5/logger"
|
|
)
|
|
|
|
type proxyList struct {
|
|
*list.List
|
|
*sync.RWMutex
|
|
}
|
|
|
|
func (pl proxyList) add(p *Proxy) {
|
|
pl.Lock()
|
|
defer pl.Unlock()
|
|
pl.PushBack(p)
|
|
}
|
|
|
|
func (pl proxyList) pop() *Proxy {
|
|
pl.Lock()
|
|
if pl.Len() < 1 {
|
|
pl.Unlock()
|
|
return nil
|
|
}
|
|
p := pl.Remove(pl.Front()).(*Proxy)
|
|
pl.Unlock()
|
|
return p
|
|
}
|
|
|
|
// ProxyChannels will likely be unexported in the future.
|
|
type ProxyChannels struct {
|
|
// SOCKS5 is a constant stream of verified SOCKS5 proxies
|
|
SOCKS5 proxyList
|
|
// SOCKS4 is a constant stream of verified SOCKS4 proxies
|
|
SOCKS4 proxyList
|
|
// SOCKS4a is a constant stream of verified SOCKS5 proxies
|
|
SOCKS4a proxyList
|
|
// HTTP is a constant stream of verified SOCKS5 proxies
|
|
HTTP proxyList
|
|
}
|
|
|
|
// Slice returns a slice of all proxyLists in ProxyChannels, note that HTTP is not included.
|
|
func (pc ProxyChannels) Slice() []*proxyList {
|
|
lists := []*proxyList{&pc.SOCKS5, &pc.SOCKS4, &pc.SOCKS4a}
|
|
entropy.GetOptimizedRand().Shuffle(3, func(i, j int) {
|
|
lists[i], lists[j] = lists[j], lists[i]
|
|
})
|
|
return lists
|
|
}
|
|
|
|
// ProxyEngine represents a proxy pool
|
|
type ProxyEngine struct {
|
|
Valids ProxyChannels
|
|
DebugLogger logger.Logger
|
|
|
|
// stats holds the Statistics for ProxyEngine
|
|
stats *Statistics
|
|
|
|
Status uint32
|
|
|
|
// Pending is a constant stream of proxy strings to be verified
|
|
Pending proxyList
|
|
|
|
// see: https://pkg.go.dev/github.com/yunginnanet/Rate5
|
|
useProx *rl.Limiter
|
|
badProx *rl.Limiter
|
|
|
|
dispenseMiddleware func(*Proxy) (*Proxy, bool)
|
|
|
|
conCtx context.Context
|
|
killConns context.CancelFunc
|
|
ctx context.Context
|
|
quit context.CancelFunc
|
|
|
|
httpOptsDirty *atomic.Bool
|
|
httpClients *sync.Pool
|
|
|
|
proxyMap proxyMap
|
|
|
|
// reaper sync.Pool
|
|
|
|
recycleMu *sync.Mutex
|
|
mu *sync.RWMutex
|
|
pool *ants.Pool
|
|
|
|
scaler *scaler.AutoScaler
|
|
scaleTimer *time.Ticker
|
|
|
|
recycleTimer *time.Ticker
|
|
|
|
opt *config
|
|
runningdaemons int32
|
|
conductor chan bool
|
|
}
|
|
|
|
var (
|
|
defaultStaleTime = 30 * time.Minute
|
|
defaultWorkerCount = 20
|
|
defaultBailout = 20
|
|
defaultRemoveAfter = 25
|
|
// Note: I've chosen to use https here exclusively assuring all validated proxies are SSL capable.
|
|
defaultChecks = []string{
|
|
"https://wtfismyip.com/text",
|
|
"https://myexternalip.com/raw",
|
|
"https://ipinfo.io/ip",
|
|
"https://api.ipify.org/",
|
|
"https://icanhazip.com/",
|
|
"https://ifconfig.me/ip",
|
|
"https://www.trackip.net/ip",
|
|
"https://checkip.amazonaws.com/",
|
|
}
|
|
)
|
|
|
|
// Returns a pointer to our default options (modified and accessed later through concurrent safe getters and setters)
|
|
func defOpt() *config {
|
|
sm := &config{
|
|
useProxConfig: defaultUseProxyRatelimiter,
|
|
badProxConfig: defaultBadProxyRateLimiter,
|
|
|
|
checkEndpoints: defaultChecks,
|
|
userAgents: defaultUserAgents,
|
|
RWMutex: &sync.RWMutex{},
|
|
removeafter: defaultRemoveAfter,
|
|
recycle: true,
|
|
debug: true,
|
|
dialerBailout: defaultBailout,
|
|
stale: defaultStaleTime,
|
|
maxWorkers: defaultWorkerCount,
|
|
redact: false,
|
|
tlsVerify: false,
|
|
shuffle: true,
|
|
}
|
|
sm.validationTimeout = time.Duration(18) * time.Second
|
|
sm.serverTimeout = time.Duration(180) * time.Second
|
|
return sm
|
|
}
|
|
|
|
// config holds our configuration for ProxyEngine instances.
|
|
// This is implemented as a pointer, and should be interacted with via the setter and getter functions.
|
|
type config struct {
|
|
// stale is the amount of time since verification that qualifies a proxy going stale.
|
|
// if a stale proxy is drawn during the use of our getter functions, it will be skipped.
|
|
stale time.Duration
|
|
// userAgents contains a list of userAgents to be randomly drawn from for proxied requests, this should be supplied via SetUserAgents
|
|
userAgents []string
|
|
// debug when enabled will print results as they come in
|
|
debug bool
|
|
// checkEndpoints includes web services that respond with (just) the WAN IP of the connection for validation purposes
|
|
checkEndpoints []string
|
|
// maxWorkers determines the maximum amount of workers used for checking proxies
|
|
maxWorkers int
|
|
// validationTimeout defines the timeout for proxy validation operations.
|
|
// This will apply for both the initial quick check (dial), and the second check (HTTP GET).
|
|
validationTimeout time.Duration
|
|
// serverTimeout defines the timeout for outgoing connections made with the mysteryDialer.
|
|
serverTimeout time.Duration
|
|
// dialerBailout defines the amount of times a dial atttempt can fail before giving up and returning an error.
|
|
dialerBailout int
|
|
// redact when enabled will redact the target string from the debug output
|
|
redact bool
|
|
// recycle determines whether or not we recycle proxies pack into the pending channel after we dispense them
|
|
recycle bool
|
|
// remove proxy from recycling after being marked bad this many times
|
|
removeafter int
|
|
// shuffle determines whether or not we shuffle proxies when we recycle them.
|
|
shuffle bool
|
|
// tlsVerify determines whether or not we verify the TLS certificate of the endpoints the http client connects to.
|
|
tlsVerify bool
|
|
|
|
// TODO: make getters and setters for these
|
|
useProxConfig rl.Policy
|
|
badProxConfig rl.Policy
|
|
|
|
*sync.RWMutex
|
|
}
|
|
|
|
// NewDefaultSwamp returns a new ProxyEngine instance.
|
|
//
|
|
// Deprecated: use NewProxyEngine instead.
|
|
func NewDefaultSwamp() *Swamp {
|
|
return &Swamp{NewProxyEngine()}
|
|
}
|
|
|
|
// Swamp is a deprecated alias for ProxyEngine
|
|
//
|
|
// Deprecated: use ProxyEngine instead.
|
|
type Swamp struct {
|
|
*ProxyEngine
|
|
}
|
|
|
|
// NewProxyEngine returns a ProxyEngine with default options.
|
|
// After calling this you may use the various "setters" to change the options before calling ProxyEngine.Start().
|
|
func NewProxyEngine() *ProxyEngine {
|
|
pe := &ProxyEngine{
|
|
stats: &Statistics{birthday: time.Now()},
|
|
DebugLogger: &basicPrinter{},
|
|
|
|
opt: defOpt(),
|
|
|
|
conductor: make(chan bool),
|
|
mu: &sync.RWMutex{},
|
|
recycleMu: &sync.Mutex{},
|
|
httpOptsDirty: &atomic.Bool{},
|
|
Status: uint32(stateNew),
|
|
}
|
|
|
|
pe.httpOptsDirty.Store(false)
|
|
pe.httpClients = &sync.Pool{New: func() interface{} { return pe.newHTTPClient() }}
|
|
|
|
stats := []int64{pe.stats.Valid4, pe.stats.Valid4a, pe.stats.Valid5, pe.stats.ValidHTTP, pe.stats.Dispensed}
|
|
for i := range stats {
|
|
atomic.StoreInt64(&stats[i], 0)
|
|
}
|
|
|
|
lists := []*proxyList{&pe.Valids.SOCKS5, &pe.Valids.SOCKS4, &pe.Valids.SOCKS4a, &pe.Valids.HTTP, &pe.Pending}
|
|
for _, c := range lists {
|
|
*c = proxyList{
|
|
List: &list.List{},
|
|
RWMutex: &sync.RWMutex{},
|
|
}
|
|
}
|
|
|
|
pe.dispenseMiddleware = func(p *Proxy) (*Proxy, bool) {
|
|
return p, true
|
|
}
|
|
pe.ctx, pe.quit = context.WithCancel(context.Background())
|
|
pe.conCtx, pe.killConns = context.WithCancel(context.Background())
|
|
pe.proxyMap = newProxyMap(pe)
|
|
|
|
atomic.StoreUint32(&pe.Status, uint32(stateNew))
|
|
atomic.StoreInt32(&pe.runningdaemons, 0)
|
|
|
|
pe.useProx = rl.NewCustomLimiter(pe.opt.useProxConfig)
|
|
pe.badProx = rl.NewCustomLimiter(pe.opt.badProxConfig)
|
|
|
|
var err error
|
|
pe.pool, err = ants.NewPool(pe.opt.maxWorkers, ants.WithOptions(ants.Options{
|
|
ExpiryDuration: 2 * time.Minute,
|
|
PanicHandler: pe.pondPanic,
|
|
}))
|
|
|
|
pe.scaler = scaler.NewAutoScaler(pe.opt.maxWorkers, pe.opt.maxWorkers+100, 50)
|
|
pe.scaleTimer = time.NewTicker(1 * time.Second)
|
|
pe.recycleTimer = time.NewTicker(500 * time.Millisecond)
|
|
|
|
if err != nil {
|
|
buf := strs.Get()
|
|
buf.MustWriteString("CRITICAL: ")
|
|
buf.MustWriteString(err.Error())
|
|
pe.dbgPrint(buf)
|
|
panic(err)
|
|
}
|
|
|
|
return pe
|
|
}
|
|
|
|
func newProxyMap(pe *ProxyEngine) proxyMap {
|
|
return proxyMap{
|
|
plot: cmap.New[*Proxy](),
|
|
parent: pe,
|
|
}
|
|
}
|
|
|
|
func (p5 *ProxyEngine) pondPanic(p interface{}) {
|
|
panic(p)
|
|
// pe.dbgPrint("Worker panic: " + fmt.Sprintf("%v", p))
|
|
}
|
|
|
|
// defaultUserAgents is a small list of user agents to use during validation.
|
|
var defaultUserAgents = []string{
|
|
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:60.0) Gecko/20100101 Firefox/60.0",
|
|
"Mozilla/5.0 (Windows NT 6.2; WOW64; rv:34.0) Gecko/20100101 Firefox/34.0",
|
|
"Mozilla/5.0 (Windows NT 6.2; Win64; x64; rv:24.0) Gecko/20140419 Firefox/24.0 PaleMoon/24.5.0",
|
|
"Mozilla/5.0 (X11; Ubuntu; Linux i686; rv:44.0) Gecko/20100101 Firefox/44.0",
|
|
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.9; rv:49.0) Gecko/20100101 Firefox/49.0",
|
|
"Mozilla/5.0 (X11; Ubuntu; Linux i686; rv:55.0) Gecko/20100101 Firefox/55.0",
|
|
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.11; rv:47.0) Gecko/20100101 Firefox/--.0",
|
|
"Mozilla/5.0 (Windows NT 6.0; rv:19.0) Gecko/20100101 Firefox/19.0",
|
|
"Mozilla/5.0 (X11; Ubuntu; Linux i686; rv:45.0) Gecko/20100101 Firefox/45.0",
|
|
"Mozilla/5.0 (Windows NT 6.0; WOW64; rv:45.0) Gecko/20100101 Firefox/45.0",
|
|
"Mozilla/5.0 (FreeBSD; Viera; rv:34.0) Gecko/20100101 Firefox/34.0",
|
|
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.7; rv:20.0) Gecko/20100101 Firefox/20.0",
|
|
"Mozilla/5.0 (Android 6.0; Mobile; rv:60.0) Gecko/20100101 Firefox/60.0",
|
|
"Mozilla/5.0 (Windows NT 5.1; rv:37.0) Gecko/20100101 Firefox/37.0",
|
|
"Mozilla/5.0 (Windows NT 6.1; WOW64; rv:35.0) Gecko/20100101 Firefox/35.0 evaliant",
|
|
"Mozilla/5.0 (Windows NT 6.1; WOW64; rv:28.0) Gecko/20100101 Firefox/28.0",
|
|
"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:58.0) Gecko/20100101 Firefox/58.0",
|
|
"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:60.0) Gecko/20100101 Firefox/60.0",
|
|
"Mozilla/5.0 (Windows NT 10.0; WOW64; rv:45.0) Gecko/20100101 Firefox/45.0",
|
|
"Mozilla/5.0 (Windows NT 6.2; WOW64; rv:41.0) Gecko/20100101 Firefox/41.0",
|
|
}
|