Merge pull request #75 from zmap/feature/addSMTPModule

zgrab2: Port SMTP module from ZGrab

errors.go

@@ -2,4 +2,9 @@ package zgrab2
 
 import "errors"
 
+// ErrMismatchedFlags is thrown if the flags for one module type are 
+// passed to an incompatible module type.
 var ErrMismatchedFlags = errors.New("mismatched flag/module")
+
+// ErrInvalidArguments is thrown if the command-line arguments invalid.
+var ErrInvalidArguments = errors.New("invalid arguments")

integration_tests/smtp/cleanup.sh

@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+
+set +e
+
+echo "smtp/cleanup: Tests cleanup for smtp"
+
+CONTAINER_NAME=zgrab_smtp
+
+docker stop $CONTAINER_NAME

integration_tests/smtp/container/Dockerfile

@@ -0,0 +1,9 @@
+FROM zgrab2_service_base:latest
+
+RUN apt-get install -y qpsmtpd
+
+WORKDIR /
+COPY entrypoint.sh .
+RUN chmod a+x ./entrypoint.sh
+
+ENTRYPOINT ["/entrypoint.sh"]

integration_tests/smtp/container/entrypoint.sh

@@ -0,0 +1,10 @@
+#!/bin/sh
+
+set -x
+
+while true; do
+  if ! /usr/bin/qpsmtpd-prefork --debug --user root; then
+    echo "qpsmtpd exited unexpectedly ($?). Restarting..."
+    sleep 1
+  fi
+done

integration_tests/smtp/setup.sh

@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+
+echo "smtp/setup: Tests setup for smtp"
+
+CONTAINER_TAG="zgrab_smtp"
+
+CONTAINER_NAME="zgrab_smtp"
+
+# If the container is already running, use it.
+if docker ps --filter "name=$CONTAINER_NAME" | grep -q $CONTAINER_NAME; then
+    echo "smtp/setup: Container $CONTAINER_NAME already running -- nothing to setup"
+    exit 0
+fi
+
+# If it is not running, try launching it -- on success, use that. 
+echo "smtp/setup: Trying to launch $CONTAINER_NAME..."
+if ! docker run --rm --name $CONTAINER_NAME -td $CONTAINER_TAG; then
+    echo "smtp/setup: Building docker image $CONTAINER_TAG..."
+    # If it fails, build it from ./container/Dockerfile
+    docker build -t $CONTAINER_TAG ./container
+    # Try again
+    echo "smtp/setup: Launching $CONTAINER_NAME..."
+    docker run --rm --name $CONTAINER_NAME -td $CONTAINER_TAG
+fi

integration_tests/smtp/test.sh

@@ -0,0 +1,44 @@
+#!/usr/bin/env bash
+
+set -e
+MODULE_DIR=$(dirname $0)
+ZGRAB_ROOT=$MODULE_DIR/../..
+ZGRAB_OUTPUT=$ZGRAB_ROOT/zgrab-output
+
+mkdir -p $ZGRAB_OUTPUT/smtp
+
+CONTAINER_NAME=zgrab_smtp
+
+OUTPUT_ROOT="$ZGRAB_OUTPUT/smtp"
+
+echo "smtp/test: Tests runner for smtp"
+CONTAINER_NAME=$CONTAINER_NAME $ZGRAB_ROOT/docker-runner/docker-run.sh smtp > "$OUTPUT_ROOT/00.json"
+CONTAINER_NAME=$CONTAINER_NAME $ZGRAB_ROOT/docker-runner/docker-run.sh smtp --send-helo > "$OUTPUT_ROOT/helo.01.json"
+CONTAINER_NAME=$CONTAINER_NAME $ZGRAB_ROOT/docker-runner/docker-run.sh smtp --send-helo --helo-domain localhost > "$OUTPUT_ROOT/helo.02.json"
+CONTAINER_NAME=$CONTAINER_NAME $ZGRAB_ROOT/docker-runner/docker-run.sh smtp --send-ehlo > "$OUTPUT_ROOT/ehlo.03.json"
+CONTAINER_NAME=$CONTAINER_NAME $ZGRAB_ROOT/docker-runner/docker-run.sh smtp --ehlo-domain localhost > "$OUTPUT_ROOT/ehlo.04.json"
+CONTAINER_NAME=$CONTAINER_NAME $ZGRAB_ROOT/docker-runner/docker-run.sh smtp --send-ehlo --ehlo-domain localhost --send-quit > "$OUTPUT_ROOT/ehlo.quit.05.json"
+CONTAINER_NAME=$CONTAINER_NAME $ZGRAB_ROOT/docker-runner/docker-run.sh smtp --send-help --send-quit > "$OUTPUT_ROOT/help.quit.06.json"
+# TODO: the qpsmtpd container does not support STARTTLS.
+
+FIELDS="help quit helo ehlo"
+status=0
+for field in $FIELDS; do
+    for file in $(find $OUTPUT_ROOT -iname "*$field*.json"); do
+        echo "check $file for $field"
+        RESULT=$($ZGRAB_ROOT/jp data.smtp.result.$field < $file)
+        if [ "$RESULT" = "null" ]; then
+            echo "Did not find $field in $file [["
+            cat $file
+            echo "]]"
+            status=1
+        fi
+    done
+done
+
+# Dump the docker logs
+echo "smtp/test: BEGIN docker logs from $CONTAINER_NAME [{("
+docker logs --tail all $CONTAINER_NAME
+echo ")}] END docker logs from $CONTAINER_NAME"
+
+exit $status

modules/smtp.go

@@ -0,0 +1,7 @@
+package modules
+
+import "github.com/zmap/zgrab2/modules/smtp"
+
+func init() {
+	smtp.RegisterModule()
+}

modules/smtp/scanner.go

@@ -0,0 +1,265 @@
+// Package smtp provides a zgrab2 module that scans for SMTP mail
+// servers.
+// Default Port: 25 (TCP)
+//
+// The --send-ehlo and --send-helo flags tell the scanner to first send
+// the EHLO/HELO command; if a --ehlo-domain or --helo-domain is present
+// that domain will be used, otherwise it is omitted.
+// The EHLO and HELO flags are mutually exclusive.
+//
+// The --send-help flag tells the scanner to send a HELP command.
+//
+// The --starttls flag tells the scanner to send the STARTTLS command,
+// and then negotiate a TLS connection.
+// The scanner uses the standard TLS flags for the handshake.
+//
+// The --send-quit flag tells the scanner to send a QUIT command.
+// 
+// So, if no flags are specified, the scanner simply reads the banner
+// returned by the server and disconnects.
+//
+// The output contains the banner and the responses to any commands that
+// were sent, and if --starttls was sent, the standard TLS logs.
+package smtp
+
+import (
+	"errors"
+	"fmt"
+	"strconv"
+
+	log "github.com/sirupsen/logrus"
+	"github.com/zmap/zgrab2"
+)
+
+// ErrInvalidResponse is returned when the server returns an invalid or unexpected response.
+var ErrInvalidResponse = errors.New("invalid response")
+
+// ScanResults instances are returned by the module's Scan function.
+type ScanResults struct {
+	// Banner is the string sent by the server immediately after connecting.
+	Banner string `json:"banner,omitempty"`
+
+	// HELO is the server's response to the HELO command, if one is sent.
+	HELO string `json:"helo,omitempty"`
+
+	// EHLO is the server's response to the EHLO command, if one is sent.
+	EHLO string `json:"ehlo,omitempty"`
+
+	// HELP is the server's response to the HELP command, if it is sent.
+	HELP string `json:"help,omitempty"`
+
+	// StartTLS is the server's response to the STARTTLS command, if it is sent.
+	StartTLS string `json:"starttls,omitempty"`
+
+	// QUIT is the server's response to the QUIT command, if it is sent.
+	QUIT string `json:"quit,omitempty"`
+
+	// TLSLog is the standard TLS log, if STARTTLS is sent.
+	TLSLog *zgrab2.TLSLog `json:"tls,omitempty"`
+}
+
+// Flags holds the command-line configuration for the HTTP scan module.
+// Populated by the framework.
+type Flags struct {
+	zgrab2.BaseFlags
+	zgrab2.TLSFlags
+
+	// SendHELO indicates that the EHLO command should be set.
+	SendEHLO bool `long:"send-ehlo" description:"Send the EHLO command; use --ehlo-domain to set a domain."`
+
+	// SendEHLO indicates that the EHLO command should be set.
+	SendHELO bool `long:"send-helo" description:"Send the EHLO command; use --helo-domain to set a domain."`
+
+	// SendHELP indicates that the client should send the HELP command (after HELO/EHLO).
+	SendHELP bool `long:"send-help" description:"Send the HELP command"`
+
+	// SendQUIT indicates that the QUIT command should be set.
+	SendQUIT bool `long:"send-quit" description:"Send the QUIT command before closing."`
+
+	// HELODomain is the domain the client should send in the HELO command.
+	HELODomain string `long:"helo-domain" description:"Set the domain to use with the HELO command. Implies --send-helo."`
+
+	// EHLODomain is the domain the client should send in the HELO command.
+	EHLODomain string `long:"ehlo-domain" description:"Set the domain to use with the EHLO command. Implies --send-ehlo."`
+
+	// StartTLS indicates that the client should attempt to update the connection to TLS.
+	StartTLS bool `long:"starttls" description:"Send STARTTLS before negotiating"`
+
+	// Verbose indicates that there should be more verbose logging.
+	Verbose bool `long:"verbose" description:"More verbose logging, include debug fields in the scan results"`
+}
+
+// Module implements the zgrab2.Module interface.
+type Module struct {
+}
+
+// Scanner implements the zgrab2.Scanner interface.
+type Scanner struct {
+	config *Flags
+}
+
+// RegisterModule registers the zgrab2 module.
+func RegisterModule() {
+	var module Module
+	_, err := zgrab2.AddCommand("smtp", "smtp", "Probe for smtp", 25, &module)
+	if err != nil {
+		log.Fatal(err)
+	}
+}
+
+// NewFlags returns a default Flags object.
+func (module *Module) NewFlags() interface{} {
+	return new(Flags)
+}
+
+// NewScanner returns a new Scanner instance.
+func (module *Module) NewScanner() zgrab2.Scanner {
+	return new(Scanner)
+}
+
+// Validate checks that the flags are valid.
+// On success, returns nil.
+// On failure, returns an error instance describing the error.
+func (flags *Flags) Validate(args []string) error {
+	if flags.EHLODomain != "" {
+		flags.SendEHLO = true
+	}
+	if flags.HELODomain != "" {
+		flags.SendHELO = true
+	}
+	if flags.SendHELO && flags.SendEHLO {
+		log.Errorf("Cannot provide both EHLO and HELO")
+		return zgrab2.ErrInvalidArguments
+	}
+	return nil
+}
+
+// Help returns the module's help string.
+func (flags *Flags) Help() string {
+	return ""
+}
+
+// Init initializes the Scanner.
+func (scanner *Scanner) Init(flags zgrab2.ScanFlags) error {
+	f, _ := flags.(*Flags)
+	scanner.config = f
+	return nil
+}
+
+// InitPerSender initializes the scanner for a given sender.
+func (scanner *Scanner) InitPerSender(senderID int) error {
+	return nil
+}
+
+// GetName returns the Scanner name defined in the Flags.
+func (scanner *Scanner) GetName() string {
+	return scanner.config.Name
+}
+
+// Protocol returns the protocol identifier of the scan.
+func (scanner *Scanner) Protocol() string {
+	return "smtp"
+}
+
+// GetPort returns the port being scanned.
+func (scanner *Scanner) GetPort() uint {
+	return scanner.config.Port
+}
+
+func getSMTPCode(response string) (int, error) {
+	if len(response) < 5 {
+		return 0, ErrInvalidResponse
+	}
+	ret, err := strconv.Atoi(response[0:3])
+	if err != nil {
+		return 0, ErrInvalidResponse
+	}
+	return ret, nil
+}
+
+// Get a command with an optional argument (so if the argument is absent, there is no trailing space)
+func getCommand(cmd string, arg string) string {
+	if arg == "" {
+		return cmd
+	}
+	return cmd + " " + arg
+}
+
+// Scan performs the SMTP scan.
+// 1. Open a TCP connection to the target port (default 25).
+// 2. Read the banner.
+// 3. If --send-ehlo or --send-helo is sent, send the corresponding EHLO
+//    or HELO command.
+// 4. If --send-help is sent, send HELP, read the result.
+// 5. If --starttls is sent, send STARTTLS, read the result, negotiate a
+//    TLS connection.
+// 6. If --send-quit is sent, send QUIT and read the result.
+// 7. Close the connection.
+func (scanner *Scanner) Scan(target zgrab2.ScanTarget) (zgrab2.ScanStatus, interface{}, error) {
+	c, err := target.Open(&scanner.config.BaseFlags)
+	if err != nil {
+		return zgrab2.TryGetScanStatus(err), nil, err
+	}
+	defer c.Close()
+	conn := Connection{Conn: c}
+	banner, err := conn.ReadResponse()
+	if err != nil {
+		return zgrab2.TryGetScanStatus(err), nil, err
+	}
+	result := &ScanResults{}
+	result.Banner = banner
+	if scanner.config.SendHELO {
+		ret, err := conn.SendCommand(getCommand("HELO", scanner.config.HELODomain))
+		if err != nil {
+			return zgrab2.TryGetScanStatus(err), result, err
+		}
+		result.HELO = ret
+	}
+	if scanner.config.SendEHLO {
+		ret, err := conn.SendCommand(getCommand("EHLO", scanner.config.EHLODomain))
+		if err != nil {
+			return zgrab2.TryGetScanStatus(err), result, err
+		}
+		result.EHLO = ret
+	}
+	if scanner.config.SendHELP {
+		ret, err := conn.SendCommand("HELP")
+		if err != nil {
+			return zgrab2.TryGetScanStatus(err), result, err
+		}
+		result.HELP = ret
+	}
+	if scanner.config.StartTLS {
+		ret, err := conn.SendCommand("STARTTLS")
+		if err != nil {
+			return zgrab2.TryGetScanStatus(err), result, err
+		}
+		result.StartTLS = ret
+		code, err := getSMTPCode(ret)
+		if err != nil {
+			return zgrab2.TryGetScanStatus(err), result, err
+		}
+		if code < 200 || code >= 300 {
+			return zgrab2.SCAN_APPLICATION_ERROR, result, fmt.Errorf("SMTP error code %d returned from STARTTLS command (%s)", code, ret)
+		}
+		tlsConn, err := scanner.config.TLSFlags.GetTLSConnection(conn.Conn)
+		if err != nil {
+			return zgrab2.TryGetScanStatus(err), result, err
+		}
+		result.TLSLog = tlsConn.GetLog()
+		if err := tlsConn.Handshake(); err != nil {
+			return zgrab2.TryGetScanStatus(err), result, err
+		}
+		conn.Conn = tlsConn
+	}
+	if scanner.config.SendQUIT {
+		ret, err := conn.SendCommand("QUIT")
+		if err != nil {
+			if err != nil {
+				return zgrab2.TryGetScanStatus(err), nil, err
+			}
+		}
+		result.QUIT = ret
+	}
+	return zgrab2.SCAN_SUCCESS, result, nil
+}

modules/smtp/smtp.go

@@ -0,0 +1,38 @@
+package smtp
+
+import (
+	"net"
+	"regexp"
+
+	"github.com/zmap/zgrab2"
+)
+
+// This is the regex used in zgrab.
+// Corner cases like "200 OK\r\nthis is not valid at all\x00\x01\x02\x03\r\n" will be matched.
+var smtpEndRegex = regexp.MustCompile(`(?:^\d\d\d\s.*\r\n$)|(?:^\d\d\d-[\s\S]*\r\n\d\d\d\s.*\r\n$)`)
+
+const readBufferSize int = 0x10000
+
+// Connection wraps the state and access to the SMTP connection.
+type Connection struct {
+	Conn net.Conn
+}
+
+// ReadResponse reads from the connection until it matches the smtpEndRegex. Copied from the original zgrab.
+// TODO: Catch corner cases, parse out response code.
+func (conn *Connection) ReadResponse() (string, error) {
+	ret := make([]byte, readBufferSize)
+	n, err := zgrab2.ReadUntilRegex(conn.Conn, ret, smtpEndRegex)
+	if err != nil {
+		return "", nil
+	}
+	return string(ret[0:n]), nil
+}
+
+// SendCommand sends a command, followed by a CRLF, then wait for / read the server's response.
+func (conn *Connection) SendCommand(cmd string) (string, error) {
+	if _, err := conn.Conn.Write([]byte(cmd + "\r\n")); err != nil {
+		return "", err
+	}
+	return conn.ReadResponse()
+}

schemas/__init__.py

@@ -7,4 +7,5 @@ import schemas.ftp
 import schemas.ntp
 import schemas.mssql
 import schemas.redis
+import schemas.smtp
 import schemas.telnet

schemas/smtp.py

@@ -0,0 +1,24 @@
+# zschema sub-schema for zgrab2's smtp module
+# Registers zgrab2-smtp globally, and smtp with the main zgrab2 schema.
+from zschema.leaves import *
+from zschema.compounds import *
+import zschema.registry
+
+import schemas.zcrypto as zcrypto
+import schemas.zgrab2 as zgrab2
+
+smtp_scan_response = SubRecord({
+    "result": SubRecord({
+        "banner": String(),
+        "ehlo": String(),
+        "helo": String(),
+        "help": String(),
+        "starttls": String(),
+        "quit": String(),
+        "tls": zgrab2.tls_log,
+    })
+}, extends=zgrab2.base_scan_response)
+
+zschema.registry.register_schema("zgrab2-smtp", smtp_scan_response)
+
+zgrab2.register_scan_response_type("smtp", smtp_scan_response)