package prox5 import ( "context" "strings" "sync" "sync/atomic" "time" "github.com/panjf2000/ants/v2" rl "github.com/yunginnanet/Rate5" "git.tcp.direct/kayos/prox5/internal/pools" "git.tcp.direct/kayos/prox5/logger" ) type ProxyChannels struct { // SOCKS5 is a constant stream of verified SOCKS5 proxies SOCKS5 chan *Proxy // SOCKS4 is a constant stream of verified SOCKS4 proxies SOCKS4 chan *Proxy // SOCKS4a is a constant stream of verified SOCKS5 proxies SOCKS4a chan *Proxy // HTTP is a constant stream of verified SOCKS5 proxies HTTP chan *Proxy } // Swamp represents a proxy pool type Swamp struct { Valids ProxyChannels DebugLogger logger.Logger // stats holds the statistics for our swamp stats *statistics Status uint32 // Pending is a constant stream of proxy strings to be verified Pending chan *Proxy // see: https://pkg.go.dev/github.com/yunginnanet/Rate5 useProx *rl.Limiter badProx *rl.Limiter dispenseMiddleware func(*Proxy) (*Proxy, bool) ctx context.Context quit context.CancelFunc swampmap swampMap // reaper sync.Pool mu *sync.RWMutex pool *ants.Pool swampopt *config runningdaemons int32 conductor chan bool } var ( defaultStaleTime = 30 * time.Minute defaultWorkerCount = 20 defaultBailout = 20 defaultRemoveAfter = 25 // Note: I've chosen to use https here exclusively assuring all validated proxies are SSL capable. defaultChecks = []string{ "https://wtfismyip.com/text", "https://myexternalip.com/raw", "https://ipinfo.io/ip", "https://api.ipify.org/", "https://icanhazip.com/", "https://ifconfig.me/ip", "https://www.trackip.net/ip", "https://checkip.amazonaws.com/", } ) // Returns a pointer to our default options (modified and accessed later through concurrent safe getters and setters) func defOpt() *config { sm := &config{ useProxConfig: defaultUseProxyRatelimiter, badProxConfig: defaultBadProxyRateLimiter, checkEndpoints: defaultChecks, userAgents: defaultUserAgents, RWMutex: &sync.RWMutex{}, removeafter: defaultRemoveAfter, recycle: true, debug: true, dialerBailout: defaultBailout, stale: defaultStaleTime, maxWorkers: defaultWorkerCount, redact: true, } sm.validationTimeout = time.Duration(18) * time.Second sm.serverTimeout = time.Duration(180) * time.Second return sm } // config holds our configuration for Swamp instances. // This is implemented as a pointer, and should be interacted with via the setter and getter functions. type config struct { // stale is the amount of time since verification that qualifies a proxy going stale. // if a stale proxy is drawn during the use of our getter functions, it will be skipped. stale time.Duration // userAgents contains a list of userAgents to be randomly drawn from for proxied requests, this should be supplied via SetUserAgents userAgents []string // debug when enabled will print results as they come in debug bool // checkEndpoints includes web services that respond with (just) the WAN IP of the connection for validation purposes checkEndpoints []string // maxWorkers determines the maximum amount of workers used for checking proxies maxWorkers int // validationTimeout defines the timeout for proxy validation operations. // This will apply for both the initial quick check (dial), and the second check (HTTP GET). validationTimeout time.Duration // serverTimeout defines the timeout for outgoing connections made with the MysteryDialer. serverTimeout time.Duration // dialerBailout defines the amount of times a dial atttempt can fail before giving up and returning an error. dialerBailout int // redact when enabled will redact the target string from the debug output redact bool // recycle determines whether or not we recycle proxies pack into the pending channel after we dispense them recycle bool // remove proxy from recycling after being marked bad this many times removeafter int // shuffle determines whether or not we shuffle proxies before we validate and dispense them. shuffle bool // TODO: make getters and setters for these useProxConfig rl.Policy badProxConfig rl.Policy *sync.RWMutex } // NewDefaultSwamp returns a new Swamp instance. // // Deprecated: use NewProxyEngine instead. func NewDefaultSwamp() *Swamp { return NewProxyEngine() } // NewProxyEngine returns a Swamp with default options. // After calling this you may use the various "setters" to change the options before calling Swamp.Start(). func NewProxyEngine() *Swamp { pe := &Swamp{ stats: &statistics{birthday: time.Now()}, DebugLogger: &basicPrinter{}, swampopt: defOpt(), conductor: make(chan bool), mu: &sync.RWMutex{}, Status: uint32(StateNew), } stats := []int64{pe.stats.Valid4, pe.stats.Valid4a, pe.stats.Valid5, pe.stats.ValidHTTP, pe.stats.Dispensed} for i := range stats { atomic.StoreInt64(&stats[i], 0) } chans := []*chan *Proxy{&pe.Valids.SOCKS5, &pe.Valids.SOCKS4, &pe.Valids.SOCKS4a, &pe.Valids.HTTP, &pe.Pending} for _, c := range chans { *c = make(chan *Proxy, 10000) } pe.dispenseMiddleware = func(p *Proxy) (*Proxy, bool) { return p, true } pe.ctx, pe.quit = context.WithCancel(context.Background()) pe.swampmap = newSwampMap(pe) atomic.StoreUint32(&pe.Status, uint32(StateNew)) atomic.StoreInt32(&pe.runningdaemons, 0) pe.useProx = rl.NewCustomLimiter(pe.swampopt.useProxConfig) pe.badProx = rl.NewCustomLimiter(pe.swampopt.badProxConfig) var err error pe.pool, err = ants.NewPool(pe.swampopt.maxWorkers, ants.WithOptions(ants.Options{ ExpiryDuration: 2 * time.Minute, PanicHandler: pe.pondPanic, })) if err != nil { buf := pools.CopABuffer.Get().(*strings.Builder) buf.WriteString("CRITICAL: ") buf.WriteString(err.Error()) pe.dbgPrint(buf) panic(err) } return pe } func newSwampMap(pe *Swamp) swampMap { return swampMap{ plot: make(map[string]*Proxy), mu: &sync.RWMutex{}, parent: pe, } } func (p5 *Swamp) pondPanic(p interface{}) { panic(p) // pe.dbgPrint("Worker panic: " + fmt.Sprintf("%v", p)) } // defaultUserAgents is a small list of user agents to use during validation. var defaultUserAgents = []string{ "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:60.0) Gecko/20100101 Firefox/60.0", "Mozilla/5.0 (Windows NT 6.2; WOW64; rv:34.0) Gecko/20100101 Firefox/34.0", "Mozilla/5.0 (Windows NT 6.2; Win64; x64; rv:24.0) Gecko/20140419 Firefox/24.0 PaleMoon/24.5.0", "Mozilla/5.0 (X11; Ubuntu; Linux i686; rv:44.0) Gecko/20100101 Firefox/44.0", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.9; rv:49.0) Gecko/20100101 Firefox/49.0", "Mozilla/5.0 (X11; Ubuntu; Linux i686; rv:55.0) Gecko/20100101 Firefox/55.0", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.11; rv:47.0) Gecko/20100101 Firefox/--.0", "Mozilla/5.0 (Windows NT 6.0; rv:19.0) Gecko/20100101 Firefox/19.0", "Mozilla/5.0 (X11; Ubuntu; Linux i686; rv:45.0) Gecko/20100101 Firefox/45.0", "Mozilla/5.0 (Windows NT 6.0; WOW64; rv:45.0) Gecko/20100101 Firefox/45.0", "Mozilla/5.0 (FreeBSD; Viera; rv:34.0) Gecko/20100101 Firefox/34.0", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.7; rv:20.0) Gecko/20100101 Firefox/20.0", "Mozilla/5.0 (Android 6.0; Mobile; rv:60.0) Gecko/20100101 Firefox/60.0", "Mozilla/5.0 (Windows NT 5.1; rv:37.0) Gecko/20100101 Firefox/37.0", "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:35.0) Gecko/20100101 Firefox/35.0 evaliant", "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:28.0) Gecko/20100101 Firefox/28.0", "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:58.0) Gecko/20100101 Firefox/58.0", "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:60.0) Gecko/20100101 Firefox/60.0", "Mozilla/5.0 (Windows NT 10.0; WOW64; rv:45.0) Gecko/20100101 Firefox/45.0", "Mozilla/5.0 (Windows NT 6.2; WOW64; rv:41.0) Gecko/20100101 Firefox/41.0", }