add: ability to set max workers and add useragent list
This commit is contained in:
parent
50c2df0752
commit
4ba04f5eac
26
defs.go
26
defs.go
@ -30,18 +30,22 @@ type Swamp struct {
|
||||
|
||||
scvm []string
|
||||
swampopt *SwampOptions
|
||||
started bool
|
||||
mu *sync.RWMutex
|
||||
}
|
||||
|
||||
var (
|
||||
// DefaultUserAgents representts our default list of useragents
|
||||
|
||||
// DefaultStaleTime represents our default setting for stale proxies
|
||||
DefaultStaleTime = time.Duration(1) * time.Hour
|
||||
defaultStaleTime = time.Duration(1) * time.Hour
|
||||
defWorkers = 100
|
||||
)
|
||||
|
||||
func defOpt() *SwampOptions {
|
||||
return &SwampOptions{UserAgents: DefaultUserAgents, Stale: DefaultStaleTime}
|
||||
return &SwampOptions{
|
||||
UserAgents: DefaultUserAgents,
|
||||
Stale: defaultStaleTime,
|
||||
MaxWorkers: defWorkers,
|
||||
Debug: false,
|
||||
}
|
||||
}
|
||||
|
||||
// SwampOptions holds our configuration for Swamp instances
|
||||
@ -53,6 +57,8 @@ type SwampOptions struct {
|
||||
Stale time.Duration
|
||||
// Debug when enabled will print results as they come in
|
||||
Debug bool
|
||||
// MaxWorkers determines the maximum amount of workers used for checking proxies
|
||||
MaxWorkers int
|
||||
}
|
||||
|
||||
var (
|
||||
@ -85,16 +91,16 @@ func init() {
|
||||
|
||||
// NewDefaultSwamp returns a Swamp with basic options.
|
||||
func NewDefaultSwamp() *Swamp {
|
||||
return &Swamp{
|
||||
s := &Swamp{
|
||||
Socks5: make(chan *Proxy, 500),
|
||||
Socks4: make(chan *Proxy, 500),
|
||||
Socks4a: make(chan *Proxy, 500),
|
||||
Pending: make(chan string, 1000),
|
||||
|
||||
Stats: &Statistics{
|
||||
validated4: 0,
|
||||
validated4a: 0,
|
||||
validated5: 0,
|
||||
Valid4: 0,
|
||||
Valid4a: 0,
|
||||
Valid5: 0,
|
||||
mu: &sync.Mutex{},
|
||||
},
|
||||
|
||||
@ -104,4 +110,6 @@ func NewDefaultSwamp() *Swamp {
|
||||
swampopt: defOpt(),
|
||||
mu: &sync.RWMutex{},
|
||||
}
|
||||
go s.feed()
|
||||
|
||||
}
|
||||
|
13
main.go
13
main.go
@ -37,11 +37,21 @@ func (s *Swamp) LoadProxyTXT(seedFile string) error {
|
||||
}
|
||||
|
||||
scan := bufio.NewScanner(f)
|
||||
|
||||
if !s.started {
|
||||
go s.tossUp()
|
||||
}
|
||||
|
||||
for scan.Scan() {
|
||||
s.scvm = append(s.scvm, scan.Text())
|
||||
}
|
||||
|
||||
if !s.started {
|
||||
go s.feed()
|
||||
}
|
||||
|
||||
s.started = true
|
||||
|
||||
if err := f.Close(); err != nil {
|
||||
s.dbgPrint(err.Error())
|
||||
return err
|
||||
@ -216,10 +226,13 @@ func (s *Swamp) tossUp() {
|
||||
|
||||
switch p.Proto {
|
||||
case "4":
|
||||
s.Stats.v4()
|
||||
s.Socks4 <- p
|
||||
case "4a":
|
||||
s.Stats.v4a()
|
||||
s.Socks4a <- p
|
||||
case "5":
|
||||
s.Stats.v5()
|
||||
s.Socks5 <- p
|
||||
}
|
||||
}
|
||||
|
13
setters.go
13
setters.go
@ -1,5 +1,7 @@
|
||||
package pxndscvm
|
||||
|
||||
import "errors"
|
||||
|
||||
// AddUserAgents appends to the list of useragents we randomly choose from during proxied requests
|
||||
func (s *Swamp) AddUserAgents(uagents []string) {
|
||||
// mutex lock so that RLock during proxy checking will block while we change this value
|
||||
@ -15,14 +17,25 @@ func (s *Swamp) SetUserAgents(uagents []string) {
|
||||
s.swampopt.UserAgents = append(s.swampopt.UserAgents, uagents...)
|
||||
}
|
||||
|
||||
// EnableDebug enables printing of verbose messages during operation
|
||||
func (s *Swamp) EnableDebug() {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
s.swampopt.Debug = true
|
||||
}
|
||||
|
||||
// DisableDebug enables printing of verbose messages during operation
|
||||
func (s *Swamp) DisableDebug() {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
s.swampopt.Debug = false
|
||||
}
|
||||
|
||||
// SetMaxWorkers set the maximum workers for proxy checking, this must be set before calling LoadProxyTXT fpr tje forst time
|
||||
func (s *Swamp) SetMaxWorkers(num int) error {
|
||||
if s.started {
|
||||
return errors.New("can't change max workers during proxypool operation, only before")
|
||||
}
|
||||
s.swampopt.MaxWorkers = num
|
||||
return nil
|
||||
}
|
||||
|
12
stats.go
12
stats.go
@ -4,9 +4,9 @@ import "sync"
|
||||
|
||||
// Statistics is used to encapsulate various swampy stats
|
||||
type Statistics struct {
|
||||
validated4 int
|
||||
validated4a int
|
||||
validated5 int
|
||||
Valid4 int
|
||||
Valid4a int
|
||||
Valid5 int
|
||||
|
||||
mu *sync.Mutex
|
||||
}
|
||||
@ -14,17 +14,17 @@ type Statistics struct {
|
||||
func (stats *Statistics) v4() {
|
||||
stats.mu.Lock()
|
||||
defer stats.mu.Unlock()
|
||||
stats.validated4++
|
||||
stats.Valid4++
|
||||
}
|
||||
|
||||
func (stats *Statistics) v4a() {
|
||||
stats.mu.Lock()
|
||||
defer stats.mu.Unlock()
|
||||
stats.validated4a++
|
||||
stats.Valid4a++
|
||||
}
|
||||
|
||||
func (stats *Statistics) v5() {
|
||||
stats.mu.Lock()
|
||||
defer stats.mu.Unlock()
|
||||
stats.validated5++
|
||||
stats.Valid5++
|
||||
}
|
||||
|
20006
useragents.go
20006
useragents.go
File diff suppressed because it is too large
Load Diff
Loading…
Reference in New Issue
Block a user