zgrab2/utility.go

142 lines
3.5 KiB
Go
Raw Normal View History

package zgrab2
import (
"errors"
"net"
Port FTP module from zgrab (#32) * remove unnecessary indirection on net.Conn * Ignore *.pyc * fix NPE on nil handshake * refactoring -- move status to status.go; add Open() methods for ScanTarget * cherry-pick .gitignore fix * pull in TLS fix * status.go comments * trim over-generalizations * use /usr/bin/env bash instead of absolute path * remove debug tcpwrap * add integration tests for postgres * hack for cleanup.sh to work on mingw -- use //var/lib instead of /var/lib * cleanup should actually stop the process though * comments / rearrange * Bump up timeout in postgres tests; only pass user if explicitly requested to do so * add schema stubs to new.sh * Integration test fixes -- use /usr/bin/env bash; log all validation failures * add postgres schemas * fill out zcrypto.client_hello schema * handle early get of TLSLog * postgres: return SCAN_SUCCESS on success * cleanup * fix new.sh * fix typo * postgres container cleanup * build.sh docs * standardize container/image names * add not to check for success * shift mysql's connection management to ScanTarget.Open(); wrap Read/Write methods returned by ScanTarget.Open() to enforce timeouts * catch schematically-valid but non-successful scans * postgres: clean up output format; more scanning * cleanup; better error handling; get detailed protocol version error * refactor modules * clean up dangling connections * split gigantic postgres.go * remove unused * ServerParams gets its own type * refactor integration tests: run zgrab2 in its own container, which is linked to the service containers, so that we don't need to keep track of unique ports on the host any more * rename entrypoint; remove duplicate postgres tests * comments for postgres schema * port over ftp support from the original zgrab; add schema / integration tests for FTP; fix log line in ssh test * Use param expansion to check for env variable [minor] This is a *very* minor change to `docker-runner/docker-run.sh` checks to see if the environment variable required to run the script has been set to a non-empty string. If not, the script exits with a non-zero status code and displays a default message: ``` ❯ docker-runner/docker-run.sh docker-runner/docker-run.sh: line 7: CONTAINER_NAME: parameter null or not set ``` This was the behavior before, but just uses a one-liner declarative bash idiom. For further reading on parameter expansion, see https://stackoverflow.com/a/307735. @justinbastress can tell me if I did something wrong and broke the intent of the script :-) * Add integration_test targets to makefile; use makefile instead of directly calling go build everywhere; run postgres schema through PEP8 linter * use make in docker-runner entrypoint * add .integration_test_setup to .gitignore * cleanup * add ftp schema * more .gitignore items * Makefile updates: Windows support; add docker-runner target; better cleanup. * docker-runner Dockerfile: start from zgrab2_runner_base image * cleanup postgres setup * make travis use make * add .gitattributes, try to prevent it from overriding lfs with crlfs in shell scripts at least * fix folder name in Makefile * update go (one of our dependencies now works only with >= 1.9) * From travis: `I don't have any idea what to do with '1.9.0'.` * explicit clean make * fix dep order * fix build.sh location * popd * use make to ensure zgrab2_runner exists * Make docker-runner an order-dependency for integration-test-cleanup; don't do a cleanup after each integration test * use explicit tag name for zgrab2_runner * Add container-clean target to Makefile, to remove cyclic dependency on docker; use .id files to track docker images; add servce-base image; use Make to build / track images * use LF in Makefiles; update .gitignore; use zgrab_service_base image in ssh container; fix line endings (?) * remove overzealous cleanup * let setup continue even if some containers are already running * ftp: use zgrab2_service_base for ftp container; restart container if already running * FTP: remove redundant prefix; add non-authtls test * zgrab depends on *.go * docker-runner depends on zgrab2 binary * clean output before running integration tests * address Zakir's comments: rename auth_*_resp -> auth_*, 'a FTP banner' -> 'an FTP banner' * fix schema * update/rebuild when necessary in container
2018-01-18 18:41:51 +00:00
"regexp"
2017-09-26 18:02:27 +00:00
"strconv"
2017-08-03 20:06:48 +00:00
"strings"
"github.com/ajholland/zflags"
)
var parser *flags.Parser
func init() {
parser = flags.NewParser(&config, flags.Default)
}
2017-10-04 03:57:56 +00:00
// NewIniParser creates and returns a ini parser initialized
// with the default parser
func NewIniParser() *flags.IniParser {
return flags.NewIniParser(parser)
}
2017-09-26 18:15:15 +00:00
// AddCommand adds a module to the parser and returns a pointer to
// a flags.command object or an error
2017-09-26 18:02:27 +00:00
func AddCommand(command string, shortDescription string, longDescription string, port int, m ScanModule) (*flags.Command, error) {
cmd, err := parser.AddCommand(command, shortDescription, longDescription, m)
if err != nil {
return nil, err
}
cmd.FindOptionByLongName("port").Default = []string{strconv.FormatUint(uint64(port), 10)}
cmd.FindOptionByLongName("name").Default = []string{command}
modules[command] = m
2017-09-26 18:02:27 +00:00
return cmd, nil
}
2017-10-04 03:57:56 +00:00
// ParseCommandLine parses the commands given on the command line
// and validates the framework configuration (global options)
// immediately after parsing
func ParseCommandLine(flags []string) ([]string, string, ScanFlags, error) {
posArgs, moduleType, f, err := parser.ParseCommandLine(flags)
if err == nil {
validateFrameworkConfiguration()
}
sf, _ := f.(ScanFlags)
return posArgs, moduleType, sf, err
}
2017-09-26 18:15:15 +00:00
// ParseTarget takes input as a string and parses it into either an IPNet
// (may have empty mask and just contain IP , domain name, or errors, may
// return both IPNet and domain name
func ParseTarget(s string) (*net.IPNet, string, error) {
i := strings.IndexByte(s, ',')
j := strings.IndexByte(s, '/')
switch {
case i == -1 && j == -1:
2017-09-26 18:15:15 +00:00
// just ip or domain
if ip := net.ParseIP(s); ip != nil {
return &net.IPNet{IP: ip}, "", nil
}
2017-10-04 03:57:56 +00:00
ips, err := net.LookupIP(s)
if err != nil {
return nil, "", err
}
return &net.IPNet{IP: ips[0]}, s, nil // only return first IP after a lookup
case i == -1:
2017-09-26 18:15:15 +00:00
// cidr block
_, ipnet, err := net.ParseCIDR(s)
2017-08-03 20:06:48 +00:00
if err != nil {
return nil, "", err
}
return ipnet, "", nil
case j == -1:
2017-09-26 18:15:15 +00:00
// ip,domain
str := strings.Split(s, ",")
if len(str) != 2 {
return nil, "", errors.New("malformed input")
}
2017-09-26 18:15:15 +00:00
d := strings.TrimSpace(str[1])
if ip := net.ParseIP(str[0]); ip != nil {
2017-09-26 18:15:15 +00:00
return &net.IPNet{IP: ip}, d, nil
}
2017-09-26 18:15:15 +00:00
return nil, d, nil
}
return nil, "", nil
}
func incrementIP(ip net.IP) {
2017-08-03 20:06:48 +00:00
for j := len(ip) - 1; j >= 0; j-- {
ip[j]++
if ip[j] > 0 {
break
}
}
}
2017-08-04 20:20:32 +00:00
func duplicateIP(ip net.IP) net.IP {
2017-08-04 20:20:32 +00:00
dup := make(net.IP, len(ip))
copy(dup, ip)
return dup
}
Port FTP module from zgrab (#32) * remove unnecessary indirection on net.Conn * Ignore *.pyc * fix NPE on nil handshake * refactoring -- move status to status.go; add Open() methods for ScanTarget * cherry-pick .gitignore fix * pull in TLS fix * status.go comments * trim over-generalizations * use /usr/bin/env bash instead of absolute path * remove debug tcpwrap * add integration tests for postgres * hack for cleanup.sh to work on mingw -- use //var/lib instead of /var/lib * cleanup should actually stop the process though * comments / rearrange * Bump up timeout in postgres tests; only pass user if explicitly requested to do so * add schema stubs to new.sh * Integration test fixes -- use /usr/bin/env bash; log all validation failures * add postgres schemas * fill out zcrypto.client_hello schema * handle early get of TLSLog * postgres: return SCAN_SUCCESS on success * cleanup * fix new.sh * fix typo * postgres container cleanup * build.sh docs * standardize container/image names * add not to check for success * shift mysql's connection management to ScanTarget.Open(); wrap Read/Write methods returned by ScanTarget.Open() to enforce timeouts * catch schematically-valid but non-successful scans * postgres: clean up output format; more scanning * cleanup; better error handling; get detailed protocol version error * refactor modules * clean up dangling connections * split gigantic postgres.go * remove unused * ServerParams gets its own type * refactor integration tests: run zgrab2 in its own container, which is linked to the service containers, so that we don't need to keep track of unique ports on the host any more * rename entrypoint; remove duplicate postgres tests * comments for postgres schema * port over ftp support from the original zgrab; add schema / integration tests for FTP; fix log line in ssh test * Use param expansion to check for env variable [minor] This is a *very* minor change to `docker-runner/docker-run.sh` checks to see if the environment variable required to run the script has been set to a non-empty string. If not, the script exits with a non-zero status code and displays a default message: ``` ❯ docker-runner/docker-run.sh docker-runner/docker-run.sh: line 7: CONTAINER_NAME: parameter null or not set ``` This was the behavior before, but just uses a one-liner declarative bash idiom. For further reading on parameter expansion, see https://stackoverflow.com/a/307735. @justinbastress can tell me if I did something wrong and broke the intent of the script :-) * Add integration_test targets to makefile; use makefile instead of directly calling go build everywhere; run postgres schema through PEP8 linter * use make in docker-runner entrypoint * add .integration_test_setup to .gitignore * cleanup * add ftp schema * more .gitignore items * Makefile updates: Windows support; add docker-runner target; better cleanup. * docker-runner Dockerfile: start from zgrab2_runner_base image * cleanup postgres setup * make travis use make * add .gitattributes, try to prevent it from overriding lfs with crlfs in shell scripts at least * fix folder name in Makefile * update go (one of our dependencies now works only with >= 1.9) * From travis: `I don't have any idea what to do with '1.9.0'.` * explicit clean make * fix dep order * fix build.sh location * popd * use make to ensure zgrab2_runner exists * Make docker-runner an order-dependency for integration-test-cleanup; don't do a cleanup after each integration test * use explicit tag name for zgrab2_runner * Add container-clean target to Makefile, to remove cyclic dependency on docker; use .id files to track docker images; add servce-base image; use Make to build / track images * use LF in Makefiles; update .gitignore; use zgrab_service_base image in ssh container; fix line endings (?) * remove overzealous cleanup * let setup continue even if some containers are already running * ftp: use zgrab2_service_base for ftp container; restart container if already running * FTP: remove redundant prefix; add non-authtls test * zgrab depends on *.go * docker-runner depends on zgrab2 binary * clean output before running integration tests * address Zakir's comments: rename auth_*_resp -> auth_*, 'a FTP banner' -> 'an FTP banner' * fix schema * update/rebuild when necessary in container
2018-01-18 18:41:51 +00:00
var InsufficientBufferError = errors.New("Not enough buffer space")
// ReadUntilRegex calls connection.Read() until it returns an error, or the cumulatively-read data matches the given regexp
func ReadUntilRegex(connection net.Conn, res []byte, expr *regexp.Regexp) (int, error) {
buf := res[0:]
length := 0
for finished := false; !finished; {
n, err := connection.Read(buf)
length += n
if err != nil {
return length, err
}
if expr.Match(res[0:length]) {
finished = true
}
if length == len(res) {
return length, InsufficientBufferError
}
buf = res[length:]
}
return length, nil
}
// TLDMatches checks for a strict TLD match
func TLDMatches(host1 string, host2 string) bool {
splitStr1 := strings.Split(stripPortNumber(host1), ".")
splitStr2 := strings.Split(stripPortNumber(host2), ".")
tld1 := splitStr1[len(splitStr1)-1]
tld2 := splitStr2[len(splitStr2)-1]
return tld1 == tld2
}
func stripPortNumber(host string) string {
return strings.Split(host, ":")[0]
}