merge master

integration_tests/.template/module/scanner.go

@@ -15,7 +15,7 @@ type ScanResults struct {
 	// TLSLog      *zgrab2.TLSLog `json:"tls,omitempty"`
 }
 
-// Flags holds the command-line configuration for the HTTP scan module.
+// Flags holds the command-line configuration for the #{MODULE_NAME} scan module.
 // Populated by the framework.
 type Flags struct {
 	zgrab2.BaseFlags
@@ -36,7 +36,7 @@ type Scanner struct {
 	// TODO: Add scan state
 }
 
-// RegisterModule() registers the zgrab2 module.
+// RegisterModule registers the zgrab2 module.
 func RegisterModule() {
 	var module Module
 	// FIXME: Set default port
@@ -95,7 +95,7 @@ func (scanner *Scanner) GetPort() uint {
 	return scanner.config.Port
 }
 
-// Scan() TODO: describe what is scanned
+// Scan TODO: describe what is scanned
 func (scanner *Scanner) Scan(t zgrab2.ScanTarget) (status zgrab2.ScanStatus, result interface{}, thrown error) {
 	// TODO: implement
 	return zgrab2.SCAN_UNKNOWN_ERROR, nil, nil

integration_tests/.template/schema.py

@@ -11,7 +11,7 @@ import schemas.zgrab2 as zgrab2
     "result": SubRecord({
         # TODO FIXME IMPLEMENT SCHEMA
     })
-}, extends = zgrab2.base_scan_response)
+}, extends=zgrab2.base_scan_response)
 
 zschema.registry.register_schema("zgrab2-#{MODULE_NAME}", #{MODULE_NAME}_scan_response)
 

integration_tests/telnet/cleanup.sh

@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+
+set +e
+
+echo "telnet/cleanup: Tests cleanup for telnet"
+
+CONTAINER_NAME=zgrab_telnet
+
+docker stop $CONTAINER_NAME

integration_tests/telnet/container/Dockerfile

@@ -0,0 +1,9 @@
+FROM zgrab2_service_base:latest
+
+RUN apt-get install -y telnetd
+
+WORKDIR /
+COPY entrypoint.sh .
+RUN chmod a+x ./entrypoint.sh
+
+ENTRYPOINT ["/entrypoint.sh"]

integration_tests/telnet/container/entrypoint.sh

@@ -0,0 +1,10 @@
+#!/bin/sh
+
+set -x
+
+while true; do
+  if ! /usr/sbin/in.telnetd -debug 23; then
+    echo "in.telnetd exited unexpectedly. Restarting..."
+    sleep 1
+  fi
+done

integration_tests/telnet/setup.sh

@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+
+echo "telnet/setup: Tests setup for telnet"
+
+CONTAINER_TAG="zgrab_telnet"
+
+CONTAINER_NAME="zgrab_telnet"
+
+# If the container is already running, use it.
+if docker ps --filter "name=$CONTAINER_NAME" | grep -q $CONTAINER_NAME; then
+    echo "telnet/setup: Container $CONTAINER_NAME already running -- nothing to setup"
+    exit 0
+fi
+
+# If it is not running, try launching it -- on success, use that. 
+echo "telnet/setup: Trying to launch $CONTAINER_NAME..."
+if ! docker run --rm --name $CONTAINER_NAME -td $CONTAINER_TAG; then
+    echo "telnet/setup: Building docker image $CONTAINER_TAG..."
+    # If it fails, build it from ./container/Dockerfile
+    docker build -t $CONTAINER_TAG ./container
+    # Try again
+    echo "telnet/setup: Launching $CONTAINER_NAME..."
+    docker run --rm --name $CONTAINER_NAME -td $CONTAINER_TAG
+fi

integration_tests/telnet/test.sh

@@ -0,0 +1,22 @@
+#!/usr/bin/env bash
+
+set -e
+MODULE_DIR=$(dirname $0)
+ZGRAB_ROOT=$MODULE_DIR/../..
+ZGRAB_OUTPUT=$ZGRAB_ROOT/zgrab-output
+
+mkdir -p $ZGRAB_OUTPUT/telnet
+
+CONTAINER_NAME=zgrab_telnet
+
+OUTPUT_FILE=$ZGRAB_OUTPUT/telnet/telnet.json
+
+echo "telnet/test: Tests runner for telnet"
+CONTAINER_NAME=$CONTAINER_NAME $ZGRAB_ROOT/docker-runner/docker-run.sh telnet > $OUTPUT_FILE
+
+# Dump the docker logs
+echo "telnet/test: BEGIN docker logs from $CONTAINER_NAME [{("
+docker logs --tail all $CONTAINER_NAME
+echo ")}] END docker logs from $CONTAINER_NAME"
+
+# TODO: If there are any other relevant log files, dump those to stdout here.

modules/telnet.go

@@ -0,0 +1,7 @@
+package modules
+
+import "github.com/zmap/zgrab2/modules/telnet"
+
+func init() {
+	telnet.RegisterModule()
+}

modules/telnet/log.go

@@ -0,0 +1,33 @@
+/*
+ * ZGrab Copyright 2015 Regents of the University of Michigan
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not
+ * use this file except in compliance with the License. You may obtain a copy
+ * of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ * implied. See the License for the specific language governing
+ * permissions and limitations under the License.
+ */
+
+package telnet
+
+// TelnetLog is the output of the telnet grab.
+type TelnetLog struct {
+	// Banner is the telnet banner returned by the server.
+	Banner string `json:"banner,omitempty"`
+
+	// Will is the list of options that the server says that it will use.
+	Will []TelnetOption `json:"will,omitempty"`
+
+	// Do is the list of options that the server requests that the client use.
+	Do []TelnetOption `json:"do,omitempty"`
+
+	// Wont is the list of options that the server says it will *not* use.
+	Wont []TelnetOption `json:"wont,omitempty"`
+
+	// Dont is the list of options that the server requests the client *not* use.
+	Dont []TelnetOption `json:"dont,omitempty"`
+}

modules/telnet/names.go

@@ -0,0 +1,67 @@
+package telnet
+
+var optionToName map[int]string
+
+func init() {
+	optionToName = make(map[int]string, 256)
+	optionToName[0] = "Binary Transmission"
+	optionToName[1] = "Echo"
+	optionToName[2] = "Reconnection"
+	optionToName[3] = "Suppress Go Ahead"
+	optionToName[4] = "Approx Message Size Negotiation"
+	optionToName[5] = "Status"
+	optionToName[6] = "Timing Mark"
+	optionToName[7] = "Remote Controlled Trans and Echo"
+	optionToName[8] = "Output Line Width"
+	optionToName[9] = "Output Page Size"
+	optionToName[10] = "Output Carriage-Return Disposition"
+	optionToName[11] = "Output Horizontal Tab Stops"
+	optionToName[12] = "Output Horizontal Tab Disposition"
+	optionToName[13] = "Output Formfeed Disposition"
+	optionToName[14] = "Output Vertical Tabstops"
+	optionToName[15] = "Output Vertical Tab Disposition"
+	optionToName[16] = "Output Linefeed Disposition"
+	optionToName[17] = "Extended ASCII"
+	optionToName[18] = "Logout"
+	optionToName[19] = "Byte Macro"
+	optionToName[20] = "Data Entry Terminal"
+	optionToName[21] = "SUPDUP"
+	optionToName[22] = "SUPDUP Output"
+	optionToName[23] = "Send Location"
+	optionToName[24] = "Terminal Type"
+	optionToName[25] = "End of Record"
+	optionToName[26] = "TACACS User Identification"
+	optionToName[27] = "Output Marking"
+	optionToName[28] = "Terminal Location Number"
+	optionToName[29] = "Telnet 3270 Regime"
+	optionToName[30] = "X.3 PAD"
+	optionToName[31] = "Negotiate About Window Size"
+	optionToName[32] = "Terminal Speed"
+	optionToName[33] = "Remote Flow Control"
+	optionToName[34] = "Linemode"
+	optionToName[35] = "X Display Location"
+	optionToName[36] = "Environment Option"
+	optionToName[37] = "Authentication Option"
+	optionToName[38] = "Encryption Option"
+	optionToName[39] = "New Environment Option"
+	optionToName[40] = "TN3270E"
+	optionToName[41] = "XAUTH"
+	optionToName[42] = "CHARSET"
+	optionToName[43] = "Telnet Remote Serial Port (RSP)"
+	optionToName[44] = "Com Port Control Option"
+	optionToName[45] = "Telnet Suppress Local Echo"
+	optionToName[46] = "Telnet Start TLS"
+	optionToName[47] = "KERMIT"
+	optionToName[48] = "SEND-URL"
+	optionToName[49] = "FORWARD_X"
+	optionToName[138] = "TELOPT PRAGMA LOGON"
+	optionToName[139] = "TELOPT SSPI LOGON"
+	optionToName[140] = "TELOPT PRAGMA HEARTBEAT"
+	optionToName[255] = "Extended-Options-List"
+	for i := 50; i < 137; i++ {
+		optionToName[i] = "Unassigned"
+	}
+	for i := 141; i <= 254; i++ {
+		optionToName[i] = "Unassigned"
+	}
+}

modules/telnet/scanner.go

@@ -0,0 +1,105 @@
+// Package telnet provides a zgrab2 module that scans for telnet daemons.
+// Default Port: 23 (TCP)
+//
+// The --max-read-size flag allows setting a ceiling to the number of bytes
+// that will be read for the banner.
+//
+// The scan negotiates the options and attempts to grab the banner, using the
+// same behavior as the original zgrab.
+//
+// The output contains the banner and the negotiated options, in the same
+// format as the original zgrab.
+package telnet
+
+import (
+	log "github.com/sirupsen/logrus"
+	"github.com/zmap/zgrab2"
+)
+
+// Flags holds the command-line configuration for the Telnet scan module.
+// Populated by the framework.
+type Flags struct {
+	zgrab2.BaseFlags
+	MaxReadSize int  `long:"max-read-size" description:"Set the maximum number of bytes to read when grabbing the banner" default:"65536"`
+	Verbose     bool `long:"verbose" description:"More verbose logging, include debug fields in the scan results"`
+}
+
+// Module implements the zgrab2.Module interface.
+type Module struct {
+}
+
+// Scanner implements the zgrab2.Scanner interface.
+type Scanner struct {
+	config *Flags
+}
+
+// RegisterModule registers the zgrab2 module.
+func RegisterModule() {
+	var module Module
+	_, err := zgrab2.AddCommand("telnet", "telnet", "Probe for telnet", 23, &module)
+	if err != nil {
+		log.Fatal(err)
+	}
+}
+
+// NewFlags returns a default Flags object.
+func (module *Module) NewFlags() interface{} {
+	return new(Flags)
+}
+
+// NewScanner returns a new Scanner instance.
+func (module *Module) NewScanner() zgrab2.Scanner {
+	return new(Scanner)
+}
+
+// Validate checks that the flags are valid.
+// On success, returns nil.
+// On failure, returns an error instance describing the error.
+func (flags *Flags) Validate(args []string) error {
+	return nil
+}
+
+// Help returns the module's help string.
+func (flags *Flags) Help() string {
+	return ""
+}
+
+// Init initializes the Scanner.
+func (scanner *Scanner) Init(flags zgrab2.ScanFlags) error {
+	f, _ := flags.(*Flags)
+	scanner.config = f
+	return nil
+}
+
+// InitPerSender initializes the scanner for a given sender.
+func (scanner *Scanner) InitPerSender(senderID int) error {
+	return nil
+}
+
+// GetName returns the Scanner name defined in the Flags.
+func (scanner *Scanner) GetName() string {
+	return scanner.config.Name
+}
+
+// Protocol returns the protocol identifier of the scan.
+func (scanner *Scanner) Protocol() string {
+	return "telnet"
+}
+
+// GetPort returns the port being scanned.
+func (scanner *Scanner) GetPort() uint {
+	return scanner.config.Port
+}
+
+// Scan connects to the target (default port TCP 23) and attempts to grab the Telnet banner.
+func (scanner *Scanner) Scan(target zgrab2.ScanTarget) (zgrab2.ScanStatus, interface{}, error) {
+	conn, err := target.Open(&scanner.config.BaseFlags)
+	if err != nil {
+		return zgrab2.TryGetScanStatus(err), nil, err
+	}
+	result := new(TelnetLog)
+	if err := GetTelnetBanner(result, conn, scanner.config.MaxReadSize); err != nil {
+		return zgrab2.TryGetScanStatus(err), nil, err
+	}
+	return zgrab2.SCAN_SUCCESS, result, nil
+}

modules/telnet/telnet.go

@@ -0,0 +1,223 @@
+/*
+ * ZGrab Copyright 2015 Regents of the University of Michigan
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not
+ * use this file except in compliance with the License. You may obtain a copy
+ * of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ * implied. See the License for the specific language governing
+ * permissions and limitations under the License.
+ */
+
+package telnet
+
+import (
+	"bytes"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"math"
+	"net"
+)
+
+// RFC 854 - https://tools.ietf.org/html/rfc854
+const (
+	// IAC means INTERPRET AS COMMAND.
+	IAC                = byte(255)
+
+	// DONT means don't use these options.
+	DONT               = byte(254)
+
+	// DO means do use these options.
+	DO                 = byte(253)
+
+	// WONT means these options won't be used.
+	WONT               = byte(252)
+
+	// WILL means these options will be used.
+	WILL               = byte(251)
+
+	// GO_AHEAD is the special go ahead command.
+	GO_AHEAD           = byte(249)
+
+	// IAC_CMD_LENGTH gives the length of the special IAC command (inclusive).
+	IAC_CMD_LENGTH     = 3
+
+	// READ_BUFFER_LENGTH is the size of the read buffer.
+	READ_BUFFER_LENGTH = 8192
+)
+
+// TelnetOption provides mappings of telnet option enum values to/from their friendly names.
+type TelnetOption uint16
+
+// Name gets the friendly name of the TelnetOption (or "unknown" if it is not recognized).
+func (opt *TelnetOption) Name() string {
+	name, ok := optionToName[int(*opt)]
+	if !ok {
+		return "unknown"
+	}
+	return name
+}
+
+// MarshalJSON returns the JSON-encoded friendly name of the telnet option.
+func (opt *TelnetOption) MarshalJSON() ([]byte, error) {
+	out := struct {
+		Name  string `json:"name"`
+		Value int    `json:"value"`
+	}{
+		opt.Name(),
+		int(*opt),
+	}
+	return json.Marshal(&out)
+}
+
+// UnmarshalJSON returns the TelnetOption enum value from its JSON-encoded friendly name.
+func (opt *TelnetOption) UnmarshalJSON(b []byte) error {
+	aux := struct {
+		Value int `json:"value"`
+	}{}
+	if err := json.Unmarshal(b, &aux); err != nil {
+		return err
+	}
+	if aux.Value < 0 || aux.Value > 255 {
+		return errors.New("Invalid byte value")
+	}
+	*opt = TelnetOption(byte(aux.Value))
+	return nil
+}
+
+// GetTelnetBanner attempts to negotiate the options and fetch the telnet banner over the given connection, reading at
+// most maxReadSize bytes.
+func GetTelnetBanner(logStruct *TelnetLog, conn net.Conn, maxReadSize int) (err error) {
+	if err = NegotiateOptions(logStruct, conn); err != nil {
+		return err
+	}
+
+	var bannerSlice []byte
+	//grab banner
+	buffer := make([]byte, READ_BUFFER_LENGTH)
+
+	numBytes := len(buffer)
+	rounds := int(math.Ceil(float64(maxReadSize) / READ_BUFFER_LENGTH))
+	count := 0
+	for numBytes != 0 && count < rounds && numBytes == READ_BUFFER_LENGTH {
+
+		numBytes, err = conn.Read(buffer)
+		// ignore timeout errors if there is already banner content
+		if err, ok := err.(net.Error); ok && err.Timeout() {
+			if len(logStruct.Banner) == 0 {
+				return err
+			}
+			break
+		}
+
+		if err != nil && err != io.EOF {
+			return err
+		}
+
+		if containsIAC(buffer) {
+			continue
+		}
+
+		if count == rounds-1 {
+			bannerSlice = append(bannerSlice, buffer[0:maxReadSize%READ_BUFFER_LENGTH]...)
+		} else {
+			bannerSlice = append(bannerSlice, buffer[0:numBytes]...)
+		}
+		count++
+	}
+
+	logStruct.Banner += string(bannerSlice)
+
+	return nil
+}
+
+// NegotiateOptions attempts to negotiate the connection options over the given connection.
+func NegotiateOptions(logStruct *TelnetLog, conn net.Conn) error {
+	var readBuffer, retBuffer []byte
+	var option, optionType, returnOptionType byte
+	var iacIndex, firstUnreadIndex, numBytes, numDataBytes int
+	var err error
+
+	for finishedNegotiating := false; finishedNegotiating == false; {
+		readBuffer = make([]byte, READ_BUFFER_LENGTH)
+		retBuffer = nil
+		numBytes, err = conn.Read(readBuffer)
+		numDataBytes = numBytes
+
+		if err != nil {
+			return err
+		}
+
+		if numBytes == len(readBuffer) {
+			return errors.New("Not enough buffer space for telnet options")
+		}
+
+		// Negotiate options
+
+		for iacIndex = bytes.IndexByte(readBuffer, IAC); iacIndex != -1; iacIndex = bytes.IndexByte(readBuffer, IAC) {
+			firstUnreadIndex = 0
+			optionType = readBuffer[iacIndex+1]
+			option = readBuffer[iacIndex+2]
+
+			// ignore go ahead
+			if optionType == GO_AHEAD {
+				readBuffer = readBuffer[0:iacIndex]
+				numBytes = iacIndex
+				firstUnreadIndex = 0
+				break
+			}
+
+			// record all offered options
+			opt := TelnetOption(option)
+			if optionType == WILL {
+				logStruct.Will = append(logStruct.Will, opt)
+			} else if optionType == DO {
+				logStruct.Do = append(logStruct.Do, opt)
+			} else if optionType == WONT {
+				logStruct.Wont = append(logStruct.Wont, opt)
+			} else if optionType == DONT {
+				logStruct.Dont = append(logStruct.Dont, opt)
+			}
+
+			// reject all offered options
+			if optionType == WILL || optionType == WONT {
+				returnOptionType = DONT
+			} else if optionType == DO || optionType == DONT {
+				returnOptionType = WONT
+			} else {
+				return errors.New("Unsupported telnet IAC option type" + fmt.Sprintf("%d", optionType))
+			}
+
+			retBuffer = append(retBuffer, IAC)
+			retBuffer = append(retBuffer, returnOptionType)
+			retBuffer = append(retBuffer, option)
+
+			firstUnreadIndex = iacIndex + IAC_CMD_LENGTH
+			numDataBytes -= firstUnreadIndex
+			readBuffer = readBuffer[firstUnreadIndex:]
+		}
+
+		if _, err = conn.Write(retBuffer); err != nil {
+			return err
+		}
+
+		numIACBytes := numBytes - numDataBytes
+		finishedNegotiating = numBytes != numIACBytes
+	}
+
+	// no more IAC commands, just read the resulting data
+	if numDataBytes >= 0 {
+		logStruct.Banner = string(readBuffer[0:numDataBytes])
+	}
+
+	return nil
+}
+
+func containsIAC(buffer []byte) bool {
+	return bytes.IndexByte(buffer, IAC) != -1
+}

schemas/__init__.py

@@ -8,3 +8,4 @@ import schemas.ntp
 import schemas.mssql
 import schemas.redis
 import schemas.smtp
+import schemas.telnet

schemas/telnet.py

@@ -0,0 +1,27 @@
+# zschema sub-schema for zgrab2's telnet module
+# Registers zgrab2-telnet globally, and telnet with the main zgrab2 schema.
+from zschema.leaves import *
+from zschema.compounds import *
+import zschema.registry
+
+import schemas.zcrypto as zcrypto
+import schemas.zgrab2 as zgrab2
+
+telnet_option = SubRecord({
+    "name": String(),
+    "value": Unsigned16BitInteger(),
+})
+
+telnet_scan_response = SubRecord({
+    "result": SubRecord({
+        "banner": String(),
+        "will": ListOf(telnet_option),
+        "do": ListOf(telnet_option),
+        "wont": ListOf(telnet_option),
+        "dont": ListOf(telnet_option),
+    })
+}, extends=zgrab2.base_scan_response)
+
+zschema.registry.register_schema("zgrab2-telnet", telnet_scan_response)
+
+zgrab2.register_scan_response_type("telnet", telnet_scan_response)