prox5/defs.go

303 lines
9.4 KiB
Go
Raw Normal View History

2022-05-23 01:05:50 +00:00
package prox5
2021-09-13 08:30:49 +00:00
import (
2023-01-31 09:21:29 +00:00
"container/list"
2022-05-30 10:42:18 +00:00
"context"
2021-09-13 08:30:49 +00:00
"sync"
2021-09-24 19:07:56 +00:00
"sync/atomic"
2021-09-13 08:30:49 +00:00
"time"
2023-01-31 09:21:29 +00:00
"git.tcp.direct/kayos/common/entropy"
2022-12-27 12:47:37 +00:00
cmap "github.com/orcaman/concurrent-map/v2"
"github.com/panjf2000/ants/v2"
2021-09-13 08:30:49 +00:00
rl "github.com/yunginnanet/Rate5"
2022-10-16 10:53:04 +00:00
"git.tcp.direct/kayos/prox5/internal/scaler"
"git.tcp.direct/kayos/prox5/logger"
2021-09-13 08:30:49 +00:00
)
2023-01-31 09:21:29 +00:00
type proxyList struct {
*list.List
*sync.RWMutex
}
func (pl proxyList) add(p *Proxy) {
pl.Lock()
defer pl.Unlock()
pl.PushBack(p)
}
func (pl proxyList) pop() *Proxy {
pl.Lock()
if pl.Len() < 1 {
pl.Unlock()
return nil
}
p := pl.Remove(pl.Front()).(*Proxy)
pl.Unlock()
return p
}
2022-12-27 15:25:10 +00:00
// ProxyChannels will likely be unexported in the future.
2022-06-26 02:51:42 +00:00
type ProxyChannels struct {
// SOCKS5 is a constant stream of verified SOCKS5 proxies
2023-01-31 09:21:29 +00:00
SOCKS5 proxyList
2022-06-26 02:51:42 +00:00
// SOCKS4 is a constant stream of verified SOCKS4 proxies
2023-01-31 09:21:29 +00:00
SOCKS4 proxyList
2022-06-26 02:51:42 +00:00
// SOCKS4a is a constant stream of verified SOCKS5 proxies
2023-01-31 09:21:29 +00:00
SOCKS4a proxyList
2022-06-26 02:51:42 +00:00
// HTTP is a constant stream of verified SOCKS5 proxies
2023-01-31 09:21:29 +00:00
HTTP proxyList
}
// Slice returns a slice of all proxyLists in ProxyChannels, note that HTTP is not included.
func (pc ProxyChannels) Slice() []*proxyList {
lists := []*proxyList{&pc.SOCKS5, &pc.SOCKS4, &pc.SOCKS4a}
2023-01-31 09:26:11 +00:00
entropy.GetOptimizedRand().Shuffle(3, func(i, j int) {
2023-01-31 09:21:29 +00:00
lists[i], lists[j] = lists[j], lists[i]
})
return lists
2022-06-26 02:51:42 +00:00
}
2022-10-16 10:53:04 +00:00
// ProxyEngine represents a proxy pool
type ProxyEngine struct {
Valids ProxyChannels
DebugLogger logger.Logger
2021-09-13 08:30:49 +00:00
2022-10-16 10:53:04 +00:00
// stats holds the Statistics for ProxyEngine
2022-10-16 07:38:49 +00:00
stats *Statistics
2021-09-30 19:19:09 +00:00
Status uint32
2023-02-09 14:19:34 +00:00
2021-09-13 08:30:49 +00:00
// Pending is a constant stream of proxy strings to be verified
2023-01-31 09:21:29 +00:00
Pending proxyList
2021-09-13 08:30:49 +00:00
// see: https://pkg.go.dev/github.com/yunginnanet/Rate5
useProx *rl.Limiter
badProx *rl.Limiter
dispenseMiddleware func(*Proxy) (*Proxy, bool)
2021-09-30 19:19:09 +00:00
2022-10-16 10:53:04 +00:00
conCtx context.Context
killConns context.CancelFunc
ctx context.Context
quit context.CancelFunc
2021-09-20 01:23:18 +00:00
2023-01-31 09:21:29 +00:00
httpOptsDirty *atomic.Bool
httpClients *sync.Pool
2022-10-16 10:53:04 +00:00
proxyMap proxyMap
2021-09-20 01:23:18 +00:00
2022-05-30 10:42:18 +00:00
// reaper sync.Pool
2021-09-24 19:07:56 +00:00
2023-01-31 09:21:29 +00:00
recycleMu *sync.Mutex
mu *sync.RWMutex
pool *ants.Pool
2022-12-27 15:25:10 +00:00
scaler *scaler.AutoScaler
scaleTimer *time.Ticker
2023-01-31 09:21:29 +00:00
recycleTimer *time.Ticker
2023-01-01 00:40:32 +00:00
2022-10-16 10:53:04 +00:00
opt *config
2022-05-30 10:42:18 +00:00
runningdaemons int32
conductor chan bool
2021-09-13 08:30:49 +00:00
}
var (
2022-08-28 16:37:40 +00:00
defaultStaleTime = 30 * time.Minute
defaultWorkerCount = 20
defaultBailout = 20
defaultRemoveAfter = 25
// Note: I've chosen to use https here exclusively assuring all validated proxies are SSL capable.
defaultChecks = []string{
"https://wtfismyip.com/text",
"https://myexternalip.com/raw",
"https://ipinfo.io/ip",
"https://api.ipify.org/",
"https://icanhazip.com/",
"https://ifconfig.me/ip",
"https://www.trackip.net/ip",
"https://checkip.amazonaws.com/",
}
2021-09-13 08:30:49 +00:00
)
// Returns a pointer to our default options (modified and accessed later through concurrent safe getters and setters)
2022-06-26 02:51:42 +00:00
func defOpt() *config {
sm := &config{
useProxConfig: defaultUseProxyRatelimiter,
badProxConfig: defaultBadProxyRateLimiter,
checkEndpoints: defaultChecks,
2021-09-28 06:25:00 +00:00
userAgents: defaultUserAgents,
RWMutex: &sync.RWMutex{},
removeafter: defaultRemoveAfter,
recycle: true,
debug: true,
dialerBailout: defaultBailout,
stale: defaultStaleTime,
maxWorkers: defaultWorkerCount,
2022-10-19 12:36:31 +00:00
redact: false,
2023-01-31 09:21:29 +00:00
tlsVerify: false,
shuffle: true,
}
2022-08-28 16:37:40 +00:00
sm.validationTimeout = time.Duration(18) * time.Second
sm.serverTimeout = time.Duration(180) * time.Second
2021-09-24 19:07:56 +00:00
return sm
}
2022-10-16 10:53:04 +00:00
// config holds our configuration for ProxyEngine instances.
// This is implemented as a pointer, and should be interacted with via the setter and getter functions.
2022-06-26 02:51:42 +00:00
type config struct {
// stale is the amount of time since verification that qualifies a proxy going stale.
// if a stale proxy is drawn during the use of our getter functions, it will be skipped.
stale time.Duration
// userAgents contains a list of userAgents to be randomly drawn from for proxied requests, this should be supplied via SetUserAgents
userAgents []string
// debug when enabled will print results as they come in
debug bool
// checkEndpoints includes web services that respond with (just) the WAN IP of the connection for validation purposes
checkEndpoints []string
// maxWorkers determines the maximum amount of workers used for checking proxies
maxWorkers int
2021-09-24 19:07:56 +00:00
// validationTimeout defines the timeout for proxy validation operations.
// This will apply for both the initial quick check (dial), and the second check (HTTP GET).
validationTimeout time.Duration
2022-12-27 12:55:03 +00:00
// serverTimeout defines the timeout for outgoing connections made with the mysteryDialer.
serverTimeout time.Duration
// dialerBailout defines the amount of times a dial atttempt can fail before giving up and returning an error.
dialerBailout int
// redact when enabled will redact the target string from the debug output
redact bool
// recycle determines whether or not we recycle proxies pack into the pending channel after we dispense them
recycle bool
2021-09-20 08:49:06 +00:00
// remove proxy from recycling after being marked bad this many times
removeafter int
2023-01-31 09:21:29 +00:00
// shuffle determines whether or not we shuffle proxies when we recycle them.
shuffle bool
// tlsVerify determines whether or not we verify the TLS certificate of the endpoints the http client connects to.
tlsVerify bool
// TODO: make getters and setters for these
useProxConfig rl.Policy
badProxConfig rl.Policy
2021-09-13 08:30:49 +00:00
*sync.RWMutex
2021-09-13 08:30:49 +00:00
}
2022-10-16 10:53:04 +00:00
// NewDefaultSwamp returns a new ProxyEngine instance.
2022-09-23 00:04:14 +00:00
//
// Deprecated: use NewProxyEngine instead.
func NewDefaultSwamp() *Swamp {
2022-10-16 10:53:04 +00:00
return &Swamp{NewProxyEngine()}
}
// Swamp is a deprecated alias for ProxyEngine
//
// Deprecated: use ProxyEngine instead.
type Swamp struct {
*ProxyEngine
2022-09-23 00:04:14 +00:00
}
2022-10-16 10:53:04 +00:00
// NewProxyEngine returns a ProxyEngine with default options.
// After calling this you may use the various "setters" to change the options before calling ProxyEngine.Start().
func NewProxyEngine() *ProxyEngine {
pe := &ProxyEngine{
2022-10-16 09:43:12 +00:00
stats: &Statistics{birthday: time.Now()},
2022-08-31 18:38:44 +00:00
DebugLogger: &basicPrinter{},
2022-10-16 10:53:04 +00:00
opt: defOpt(),
2023-01-31 09:21:29 +00:00
conductor: make(chan bool),
mu: &sync.RWMutex{},
recycleMu: &sync.Mutex{},
httpOptsDirty: &atomic.Bool{},
Status: uint32(stateNew),
2021-09-13 08:30:49 +00:00
}
2023-01-31 09:21:29 +00:00
pe.httpOptsDirty.Store(false)
pe.httpClients = &sync.Pool{New: func() interface{} { return pe.newHTTPClient() }}
2022-06-26 02:51:42 +00:00
stats := []int64{pe.stats.Valid4, pe.stats.Valid4a, pe.stats.Valid5, pe.stats.ValidHTTP, pe.stats.Dispensed}
2022-07-25 07:14:26 +00:00
for i := range stats {
atomic.StoreInt64(&stats[i], 0)
}
2023-01-31 09:21:29 +00:00
lists := []*proxyList{&pe.Valids.SOCKS5, &pe.Valids.SOCKS4, &pe.Valids.SOCKS4a, &pe.Valids.HTTP, &pe.Pending}
for _, c := range lists {
*c = proxyList{
List: &list.List{},
RWMutex: &sync.RWMutex{},
}
}
2021-09-20 01:23:18 +00:00
2022-06-26 02:51:42 +00:00
pe.dispenseMiddleware = func(p *Proxy) (*Proxy, bool) {
return p, true
}
2022-06-26 02:51:42 +00:00
pe.ctx, pe.quit = context.WithCancel(context.Background())
2022-10-16 10:53:04 +00:00
pe.conCtx, pe.killConns = context.WithCancel(context.Background())
pe.proxyMap = newProxyMap(pe)
2021-09-30 19:19:09 +00:00
2023-01-31 09:21:29 +00:00
atomic.StoreUint32(&pe.Status, uint32(stateNew))
2022-06-26 02:51:42 +00:00
atomic.StoreInt32(&pe.runningdaemons, 0)
2022-10-16 10:53:04 +00:00
pe.useProx = rl.NewCustomLimiter(pe.opt.useProxConfig)
pe.badProx = rl.NewCustomLimiter(pe.opt.badProxConfig)
var err error
2022-10-16 10:53:04 +00:00
pe.pool, err = ants.NewPool(pe.opt.maxWorkers, ants.WithOptions(ants.Options{
ExpiryDuration: 2 * time.Minute,
2022-06-26 02:51:42 +00:00
PanicHandler: pe.pondPanic,
}))
2022-12-27 15:25:10 +00:00
pe.scaler = scaler.NewAutoScaler(pe.opt.maxWorkers, pe.opt.maxWorkers+100, 50)
2023-02-02 05:37:59 +00:00
pe.scaleTimer = time.NewTicker(1 * time.Second)
pe.recycleTimer = time.NewTicker(500 * time.Millisecond)
2022-10-16 12:56:40 +00:00
if err != nil {
buf := strs.Get()
buf.MustWriteString("CRITICAL: ")
buf.MustWriteString(err.Error())
2022-07-25 06:23:12 +00:00
pe.dbgPrint(buf)
panic(err)
}
2022-06-26 02:51:42 +00:00
return pe
}
2022-10-16 10:53:04 +00:00
func newProxyMap(pe *ProxyEngine) proxyMap {
return proxyMap{
2022-12-27 12:47:37 +00:00
plot: cmap.New[*Proxy](),
2022-06-26 02:51:42 +00:00
parent: pe,
}
2021-09-13 08:30:49 +00:00
}
2022-10-16 10:53:04 +00:00
func (p5 *ProxyEngine) pondPanic(p interface{}) {
2022-06-28 02:27:52 +00:00
panic(p)
// pe.dbgPrint("Worker panic: " + fmt.Sprintf("%v", p))
}
2021-09-14 01:50:39 +00:00
// defaultUserAgents is a small list of user agents to use during validation.
var defaultUserAgents = []string{
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:60.0) Gecko/20100101 Firefox/60.0",
"Mozilla/5.0 (Windows NT 6.2; WOW64; rv:34.0) Gecko/20100101 Firefox/34.0",
"Mozilla/5.0 (Windows NT 6.2; Win64; x64; rv:24.0) Gecko/20140419 Firefox/24.0 PaleMoon/24.5.0",
"Mozilla/5.0 (X11; Ubuntu; Linux i686; rv:44.0) Gecko/20100101 Firefox/44.0",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.9; rv:49.0) Gecko/20100101 Firefox/49.0",
"Mozilla/5.0 (X11; Ubuntu; Linux i686; rv:55.0) Gecko/20100101 Firefox/55.0",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.11; rv:47.0) Gecko/20100101 Firefox/--.0",
"Mozilla/5.0 (Windows NT 6.0; rv:19.0) Gecko/20100101 Firefox/19.0",
"Mozilla/5.0 (X11; Ubuntu; Linux i686; rv:45.0) Gecko/20100101 Firefox/45.0",
"Mozilla/5.0 (Windows NT 6.0; WOW64; rv:45.0) Gecko/20100101 Firefox/45.0",
"Mozilla/5.0 (FreeBSD; Viera; rv:34.0) Gecko/20100101 Firefox/34.0",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.7; rv:20.0) Gecko/20100101 Firefox/20.0",
"Mozilla/5.0 (Android 6.0; Mobile; rv:60.0) Gecko/20100101 Firefox/60.0",
"Mozilla/5.0 (Windows NT 5.1; rv:37.0) Gecko/20100101 Firefox/37.0",
"Mozilla/5.0 (Windows NT 6.1; WOW64; rv:35.0) Gecko/20100101 Firefox/35.0 evaliant",
"Mozilla/5.0 (Windows NT 6.1; WOW64; rv:28.0) Gecko/20100101 Firefox/28.0",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:58.0) Gecko/20100101 Firefox/58.0",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:60.0) Gecko/20100101 Firefox/60.0",
"Mozilla/5.0 (Windows NT 10.0; WOW64; rv:45.0) Gecko/20100101 Firefox/45.0",
"Mozilla/5.0 (Windows NT 6.2; WOW64; rv:41.0) Gecko/20100101 Firefox/41.0",
}