diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..e994bba --- /dev/null +++ b/.gitattributes @@ -0,0 +1,2 @@ +*.sh eol=LF +Makefile eol=LF diff --git a/.gitignore b/.gitignore index 2502e0a..02738c9 100644 --- a/.gitignore +++ b/.gitignore @@ -2,8 +2,18 @@ .DS_Store # zgrab +zgrab2 cmd/zgrab2/zgrab2 +cmd/zgrab2/zgrab2.exe zgrab-output/ # CI dependencies jp +jp.exe + +# Compiled python modules +schemas/*.pyc + +# Marker files for make +.integration-test-setup +docker-runner/*.id diff --git a/.travis.yml b/.travis.yml index 343e2bc..0e2c96a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,6 @@ language: go go: -- 1.7.4 +- 1.9 services: - docker before_install: @@ -9,13 +9,13 @@ before_install: # JMESPath is used to do context-specific validation of results - go get github.com/jmespath/jp && go build github.com/jmespath/jp - pip install --user zschema -- ./integration_tests/setup.sh - docker ps -a install: -- pushd cmd/zgrab2 && go build && popd +- make clean zgrab2 script: -- ./integration_tests/test.sh +- make integration-test after_script: +# Cleanup even if the integration tests fail - ./integration_tests/cleanup.sh notifications: email: diff --git a/Makefile b/Makefile index 1262099..58d65e4 100644 --- a/Makefile +++ b/Makefile @@ -1,9 +1,44 @@ +ifeq ($(OS),Windows_NT) + EXECUTABLE_EXTENSION := .exe +else + EXECUTABLE_EXTENSION := +endif + +GO_FILES = $(shell find . -type f -name '*.go') + all: zgrab2 -.PHONY: clean zgrab2 +.PHONY: all clean integration-test integration-test-clean docker-runner container-clean -zgrab2: - cd cmd/zgrab2 && go build -o zgrab2 +zgrab2: $(GO_FILES) + cd cmd/zgrab2 && go build && cd ../.. + rm -f zgrab2 + ln -s cmd/zgrab2/zgrab2$(EXECUTABLE_EXTENSION) zgrab2 + +docker-runner: zgrab2 + make -C docker-runner + +.integration-test-setup: | docker-runner + ./integration_tests/setup.sh + touch .integration-test-setup + +integration-test: docker-runner .integration-test-setup + rm -rf zgrab-output + ./integration_tests/test.sh + +integration-test-clean: + rm -f .integration-test-setup + rm -rf zgrab-output + ./integration_tests/cleanup.sh + make -C docker-runner clean + +# This is the target for re-building from source in the container +container-clean: + rm -f zgrab2 + cd cmd/zgrab2 && go build && cd ../.. + ln -s cmd/zgrab2/zgrab2$(EXECUTABLE_EXTENSION) zgrab2 clean: - go clean + cd cmd/zgrab2 && go clean + rm -f .integration-test-setup + rm -f zgrab2 diff --git a/docker-runner/Dockerfile b/docker-runner/Dockerfile new file mode 100644 index 0000000..224414c --- /dev/null +++ b/docker-runner/Dockerfile @@ -0,0 +1,23 @@ +FROM zgrab2_runner_base:latest + +WORKDIR /go/src/github.com/zmap + +# Grab the currently-active version of the source +ADD . zgrab2 + +# This would instead grab it from the source repo +# RUN go-wrapper download github.com/zmap/zgrab2 + +WORKDIR /go/src/github.com/zmap/zgrab2 + +RUN go get -v ./... +RUN go get -v -t ./... + +# This should already be executable, but just in case... +RUN chmod a+x ./docker-runner/entrypoint.sh + +# Build on the container +RUN make container-clean + +CMD [] +ENTRYPOINT ["/go/src/github.com/zmap/zgrab2/docker-runner/entrypoint.sh"] diff --git a/docker-runner/Makefile b/docker-runner/Makefile new file mode 100644 index 0000000..57851a5 --- /dev/null +++ b/docker-runner/Makefile @@ -0,0 +1,25 @@ +ifeq ($(OS),Windows_NT) + EXECUTABLE_EXTENSION := .exe +else + EXECUTABLE_EXTENSION := +endif + +all: docker-runner.id + +.PHONY: clean clean-all + +service-base-image.id: + docker build -t zgrab2_service_base:latest -f service-base.Dockerfile -q . > service-base-image.id || rm -f service-base-image.id + +runner-base-image.id: + docker build -t zgrab2_runner_base:latest -f runner-base.Dockerfile -q . > runner-base-image.id || rm -f runner-base-image.id + +docker-runner.id: ../cmd/zgrab2/zgrab2$(EXECUTABLE_EXTENSION) runner-base-image.id service-base-image.id + docker build -t zgrab2_runner:latest -f Dockerfile -q .. > docker-runner.id || rm -f docker-runner.id + +clean: + if [ -f docker-runner.id ]; then docker rmi -f $$(cat docker-runner.id) && rm -f docker-runner.id; fi + +clean-all: clean + if [ -f service-base-image.id ]; then docker rmi -f $$(cat service-base-image.id) && rm -f service-base-image.id; fi + if [ -f runner-base-image.id ]; then docker rmi -f $$(cat runner-base-image.id) && rm -f runner-base-image.id; fi diff --git a/docker-runner/docker-run.sh b/docker-runner/docker-run.sh new file mode 100755 index 0000000..074a810 --- /dev/null +++ b/docker-runner/docker-run.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash + +# Runs the zgrab2_runner docker image (built with docker-runner/build-runner.sh) +# Links the runner image to the targetted container with the hostname alias "target", +# then scans target using the arguments to the script. + +: "${CONTAINER_NAME:?}" + +set -e +docker run --rm --link $CONTAINER_NAME:target -e ZGRAB_TARGET=target zgrab2_runner $@ diff --git a/docker-runner/entrypoint.sh b/docker-runner/entrypoint.sh new file mode 100755 index 0000000..85f5728 --- /dev/null +++ b/docker-runner/entrypoint.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash + +# This is the entrypoint for the zgrab2_runner container. +# Runs the zgrab2 binary, passing along any arguments, with stdin containing the single line: the ZGRAB_TARGET. + +set -e + +cd /go/src/github.com/zmap/zgrab2 + +if ! [ -x $ZGRAB_REBUILD ]; then + if ! [ -x $ZGRAB_BRANCH ]; then + git checkout $ZGRAB_BRANCH + fi + git pull + make +fi + +set -x +echo $ZGRAB_TARGET | /go/src/github.com/zmap/zgrab2/cmd/zgrab2/zgrab2 $* diff --git a/docker-runner/runner-base.Dockerfile b/docker-runner/runner-base.Dockerfile new file mode 100644 index 0000000..de71ca2 --- /dev/null +++ b/docker-runner/runner-base.Dockerfile @@ -0,0 +1,11 @@ +FROM golang:1.9 +# Base image that already has the pre-requisites downloaded. + +WORKDIR /go/src/github.com/zmap + +RUN go-wrapper download github.com/zmap/zgrab2 + +WORKDIR /go/src/github.com/zmap/zgrab2 + +RUN go get -v ./... +RUN go get -v -t ./... diff --git a/docker-runner/service-base.Dockerfile b/docker-runner/service-base.Dockerfile new file mode 100644 index 0000000..6c15600 --- /dev/null +++ b/docker-runner/service-base.Dockerfile @@ -0,0 +1,6 @@ +FROM ubuntu:16.04 + +# Base Ubuntu container with the apt cache already updated. +# Many containers will be able to use this as their base. + +RUN apt-get update diff --git a/integration_tests/cleanup.sh b/integration_tests/cleanup.sh index 6e193e7..a757938 100755 --- a/integration_tests/cleanup.sh +++ b/integration_tests/cleanup.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # Keep cleaning up, even if something fails set +e diff --git a/integration_tests/mysql/cleanup.sh b/integration_tests/mysql/cleanup.sh index ae99336..0904c3c 100755 --- a/integration_tests/mysql/cleanup.sh +++ b/integration_tests/mysql/cleanup.sh @@ -1,18 +1,15 @@ -#!/bin/bash +#!/usr/bin/env bash # Keep cleaning up even if something fails set +e -# Stop all MySQL containers, but first grab the logs from them +# Stop all MySQL containers. MYSQL_VERSIONS="5.5 5.6 5.7 8.0" for version in $MYSQL_VERSIONS; do - CONTAINER_NAME="testmysql-$version" - echo "BEGIN DOCKER LOGS FROM $CONTAINER_NAME [{(" - docker logs --tail all $CONTAINER_NAME - echo ")}] END DOCKER LOGS FROM $CONTAINER_NAME" - echo "Stopping $CONTAINER_NAME..." + CONTAINER_NAME="zgrab_mysql-$version" + echo "mysql/cleanup: Stopping $CONTAINER_NAME..." docker stop $CONTAINER_NAME - echo "...stopped." + echo "mysql/cleanup: ...stopped." done diff --git a/integration_tests/mysql/setup.sh b/integration_tests/mysql/setup.sh index c0a94d1..670b558 100755 --- a/integration_tests/mysql/setup.sh +++ b/integration_tests/mysql/setup.sh @@ -1,16 +1,35 @@ #!/bin/bash -e -# Start all of the MySQL docker containers, and wait for them start responding on port 3306 - -echo "Launching docker containers..." # NOTE: the 5.5 and 5.6 versions do not have SSL enabled -CONTAINER_NAME=testmysql-5.5 MYSQL_VERSION=5.5 MYSQL_PORT=13306 ./util/launch_mysql_container.sh -CONTAINER_NAME=testmysql-5.6 MYSQL_VERSION=5.6 MYSQL_PORT=23306 ./util/launch_mysql_container.sh -CONTAINER_NAME=testmysql-5.7 MYSQL_VERSION=5.7 MYSQL_PORT=33306 ./util/launch_mysql_container.sh -CONTAINER_NAME=testmysql-8.0 MYSQL_VERSION=8.0 MYSQL_PORT=43306 ./util/launch_mysql_container.sh +versions="5.5 5.6 5.7 8.0" -echo "Waiting for MySQL to start up on all containers..." -CONTAINER_NAME=testmysql-5.5 MYSQL_PORT=13306 ./util/wait_for_mysqld.sh -CONTAINER_NAME=testmysql-5.6 MYSQL_PORT=23306 ./util/wait_for_mysqld.sh -CONTAINER_NAME=testmysql-5.7 MYSQL_PORT=33306 ./util/wait_for_mysqld.sh -CONTAINER_NAME=testmysql-8.0 MYSQL_PORT=43306 ./util/wait_for_mysqld.sh +function launch() { + VERSION=$1 + CONTAINER_NAME="zgrab_mysql-$VERSION" + if docker ps --filter "name=$CONTAINER_NAME" | grep $CONTAINER_NAME; then + echo "mysql/setup: Container $CONTAINER_NAME already running -- stopping..." + docker stop $CONTAINER_NAME + echo "...stopped." + fi + docker run -itd --rm --name zgrab_mysql-$VERSION -e MYSQL_ALLOW_EMPTY_PASSWORD=true -e MYSQL_LOG_CONSOLE=true mysql:$VERSION +} + +function waitFor() { + VERSION=$1 + CONTAINER_NAME=zgrab_mysql-$VERSION + echo "mysql/setup: Waiting for mysqld process to come up on $CONTAINER_NAME..." + while ! (docker exec $CONTAINER_NAME ps -Af | grep mysqld > /dev/null); do + echo -n "*" + sleep 1 + done + echo "...ok." +} + +echo "mysql/setup: Launching docker containers..." +for version in $versions; do + launch $version +done + +for version in $versions; do + waitFor $version +done diff --git a/integration_tests/mysql/single_run.sh b/integration_tests/mysql/single_run.sh deleted file mode 100755 index 8272c6f..0000000 --- a/integration_tests/mysql/single_run.sh +++ /dev/null @@ -1,28 +0,0 @@ -#!/bin/bash -e - -# Run the MySQL-specific integration tests: -# 1. Run zgrab2 on localhost:$MYSQL_PORT -# 2. Check that data.mysql.result.handshake.parsed.server_version matches $MYSQL_VERSION - -if [ -z $MYSQL_PORT ] || [ -z $MYSQL_VERSION ]; then - echo "Must set MYSQL_PORT and MYSQL_VERSION" - exit 1 -fi - -mkdir -p $ZGRAB_OUTPUT/mysql - -CONTAINER_NAME="testmysql-$MYSQL_VERSION" -OUTPUT_FILE="$ZGRAB_OUTPUT/mysql/$MYSQL_VERSION.json" - -echo "Testing MySQL Version $MYSQL_VERSION on port $MYSQL_PORT..." -echo "127.0.0.1" | $ZGRAB_ROOT/cmd/zgrab2/zgrab2 mysql -p $MYSQL_PORT $* > $OUTPUT_FILE - -SERVER_VERSION=$($ZGRAB_ROOT/jp -u data.mysql.result.handshake.parsed.server_version < $OUTPUT_FILE) - -if [[ "$SERVER_VERSION" == "$MYSQL_VERSION."* ]]; then - echo "Server version matches expected version: $SERVER_VERSION == $MYSQL_VERSION.*" - exit 0 -else - echo "Server versiom mismatch: Got $SERVER_VERSION, expected $MYSQL_VERSION.*" - exit 1 -fi diff --git a/integration_tests/mysql/test.sh b/integration_tests/mysql/test.sh index 64d67ca..4af91ef 100755 --- a/integration_tests/mysql/test.sh +++ b/integration_tests/mysql/test.sh @@ -1,5 +1,41 @@ -#!/bin/sh -e -MYSQL_VERSION=5.5 MYSQL_PORT=13306 ./single_run.sh -MYSQL_VERSION=5.6 MYSQL_PORT=23306 ./single_run.sh -MYSQL_VERSION=5.7 MYSQL_PORT=33306 ./single_run.sh -MYSQL_VERSION=8.0 MYSQL_PORT=43306 ./single_run.sh +#!/usr/bin/env bash +set -e + +versions="5.5 5.6 5.7 8.0" + +# Run the MySQL-specific integration tests: +# 1. Run zgrab2 on the container +# 2. Check that data.mysql.result.handshake.parsed.server_version matches $MYSQL_VERSION + +MODULE_DIR=$(dirname $0) +TEST_ROOT=$MODULE_DIR/.. +ZGRAB_ROOT=$MODULE_DIR/../.. +ZGRAB_OUTPUT=$ZGRAB_ROOT/zgrab-output + +status=0 + +function doTest() { + MYSQL_VERSION=$1 + CONTAINER_NAME="zgrab_mysql-$MYSQL_VERSION" + OUTPUT_FILE="$ZGRAB_OUTPUT/mysql/$MYSQL_VERSION.json" + echo "mysql/test: Testing MySQL Version $MYSQL_VERSION..." + CONTAINER_NAME=$CONTAINER_NAME $ZGRAB_ROOT/docker-runner/docker-run.sh mysql --timeout 10 > $OUTPUT_FILE + SERVER_VERSION=$($ZGRAB_ROOT/jp -u data.mysql.result.handshake.parsed.server_version < $OUTPUT_FILE) + if [[ "$SERVER_VERSION" == "$MYSQL_VERSION."* ]]; then + echo "Server version matches expected version: $SERVER_VERSION == $MYSQL_VERSION.*" + else + echo "Server version mismatch: Got $SERVER_VERSION, expected $MYSQL_VERSION.*" + status=1 + fi + echo "mysql/test: BEGIN docker+mysql logs from $CONTAINER_NAME [{(" + docker logs --tail all $CONTAINER_NAME + echo ")}] END docker+mysql logs from $CONTAINER_NAME" +} + +mkdir -p $ZGRAB_OUTPUT/mysql + +for version in $versions; do + doTest $version +done + +exit $status diff --git a/integration_tests/mysql/util/launch_mysql_container.sh b/integration_tests/mysql/util/launch_mysql_container.sh deleted file mode 100755 index f3a7809..0000000 --- a/integration_tests/mysql/util/launch_mysql_container.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/bash - -# Launch a MySQL container with $MYSQL_VERSION, an root empty password, console logging, and host:$MYSQL_PORT forwarded to container:3306 - -if [ -z $CONTAINER_NAME ] || [ -z $MYSQL_PORT ] || [ -z $MYSQL_VERSION ]; then - echo "Must provide CONTAINER_NAME, MYSQL_PORT and MYSQL_VERSION" - exit 1 -fi - -set -x -docker run -itd -p $MYSQL_PORT:3306 --rm --name $CONTAINER_NAME -e MYSQL_ALLOW_EMPTY_PASSWORD=true -e MYSQL_LOG_CONSOLE=true $* mysql:$MYSQL_VERSION -set +x - -exit 0 diff --git a/integration_tests/mysql/util/wait_for_mysqld.sh b/integration_tests/mysql/util/wait_for_mysqld.sh deleted file mode 100755 index bae081c..0000000 --- a/integration_tests/mysql/util/wait_for_mysqld.sh +++ /dev/null @@ -1,28 +0,0 @@ -#!/bin/bash - -if [ -z $CONTAINER_NAME ] || [ -z $MYSQL_PORT ]; then - echo "Must provide CONTAINER_NAME and MYSQL_PORT" - exit 1 -fi - -echo "Waiting for mysqld process to come up on $CONTAINER_NAME..." -while ! (docker exec $CONTAINER_NAME ps -Af | grep mysqld > /dev/null); do - echo -n "." - sleep 1 -done - -echo "...mysqld is up, waiting for $MYSQL_PORT..." -while ! (nc -z localhost $MYSQL_PORT); do - echo -n "." - sleep 1 -done - -echo "...$MYSQL_PORT is up, waiting for data..." -while ! (output=$(nc -w 5 localhost $MYSQL_PORT) && [ ${#output} -gt 0 ]); do - echo -n "." - sleep 1 -done - -echo "Received data on port $MYSQL_PORT. Container $CONTAINER_NAME is ready." - -exit 0 diff --git a/integration_tests/new.sh b/integration_tests/new.sh index bf4aecc..9e23992 100755 --- a/integration_tests/new.sh +++ b/integration_tests/new.sh @@ -20,22 +20,59 @@ mkdir -p $module_path cat << EOF > $module_path/setup.sh #!/usr/bin/env bash -echo "Tests setup for $module_name" +echo "$module_name/setup: Tests setup for $module_name" EOF chmod +x $module_path/setup.sh cat << EOF > $module_path/test.sh #!/usr/bin/env bash -echo "Tests runner for $module_name" +set -e +MODULE_DIR=\$(dirname \$0) +TEST_ROOT=\$MODULE_DIR/.. +ZGRAB_ROOT=\$MODULE_DIR/../.. +ZGRAB_OUTPUT=\$ZGRAB_ROOT/zgrab-output + +mkdir -p \$ZGRAB_OUTPUT/$module_name + +# OUTPUT_FILE=[TODO].json + +echo "$module_name/test: Tests runner for $module_name" +# CONTAINER_NAME=[TODO] \$ZGRAB_ROOT/docker-runner/docker-run.sh $module_name > \$OUTPUT_FILE + EOF chmod +x $module_path/test.sh cat << EOF > $module_path/cleanup.sh #!/usr/bin/env bash -echo "Tests cleanup for $module_name" +set +e + +echo "$module_name/cleanup: Tests cleanup for $module_name" EOF chmod +x $module_path/cleanup.sh +cat << EOF > schemas/$module_name.py +# zschema sub-schema for zgrab2's $module_name module +# Registers zgrab2-$module_name globally, and $module_name with the main zgrab2 schema. +from zschema.leaves import * +from zschema.compounds import * +import zschema.registry + +import schemas.zcrypto as zcrypto +import schemas.zgrab2 as zgrab2 + +${module_name}_scan_response = SubRecord({ + "result": SubRecord({ + # TODO FIXME IMPLEMENT SCHEMA + }) +}, extends = zgrab2.base_scan_response) + +zschema.registry.register_schema("zgrab2-${module_name}", ${module_name}_scan_response) + +zgrab2.register_scan_response_type("${module_name}", ${module_name}_scan_response) +EOF + +echo "import schemas.$module_name" >> schemas/__init__.py + echo "Test files scaffolded in $module_path" diff --git a/integration_tests/postgres/cleanup.sh b/integration_tests/postgres/cleanup.sh new file mode 100755 index 0000000..e01a669 --- /dev/null +++ b/integration_tests/postgres/cleanup.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash + +set +e + +echo "postgres/cleanup: Begin tests cleanup for postgres" + +function clean() { + CONTAINER_NAME=$1 + docker stop $CONTAINER_NAME +} + +versions="9.3 9.4 9.5 9.6 10.1" +types="ssl nossl" +for version in $versions; do + for type in $types; do + clean "zgrab_postgres_${version}-${type}" + done +done + +echo "postgres/cleanup: Finished tests cleanup for postgres" diff --git a/integration_tests/postgres/container/Dockerfile.9.3 b/integration_tests/postgres/container/Dockerfile.9.3 new file mode 100644 index 0000000..776145e --- /dev/null +++ b/integration_tests/postgres/container/Dockerfile.9.3 @@ -0,0 +1,14 @@ +FROM postgres:9.3 + +ARG IMAGE_TYPE=ssl + +WORKDIR /tmp/postgres_setup +COPY postgresql.conf.9.3.$IMAGE_TYPE.partial postgresql.conf.partial +COPY setup_$IMAGE_TYPE.sh /docker-entrypoint-initdb.d/setup_$IMAGE_TYPE.sh + +RUN chown -R postgres:postgres . +RUN chown postgres:postgres /docker-entrypoint-initdb.d/setup_$IMAGE_TYPE.sh + +RUN chmod 0755 . +RUN chmod 0644 postgresql.conf.partial +RUN chmod 0755 /docker-entrypoint-initdb.d/setup_$IMAGE_TYPE.sh diff --git a/integration_tests/postgres/container/Dockerfile.template b/integration_tests/postgres/container/Dockerfile.template new file mode 100644 index 0000000..ac5b2b5 --- /dev/null +++ b/integration_tests/postgres/container/Dockerfile.template @@ -0,0 +1,18 @@ +FROM postgres:#{POSTGRES_VERSION} + +# Our custom postgres image -- based on the standard images (https://hub.docker.com/_/postgres/) +# On top of the base, we enable logging and (if IMAGE_TYPE == ssl) server-side SSL (with a self-signed certificate) +ARG IMAGE_TYPE=ssl + +WORKDIR /tmp/postgres_setup +# This gets catted to the end of $PGDATA/postgresql.conf +COPY postgresql.conf.$IMAGE_TYPE.partial postgresql.conf.partial +# The docker-entrypoint-initdb.d scripts get run after postgres is installed but before it is started. +COPY setup_$IMAGE_TYPE.sh /docker-entrypoint-initdb.d/setup_$IMAGE_TYPE.sh + +RUN chown -R postgres:postgres . +RUN chown postgres:postgres /docker-entrypoint-initdb.d/setup_$IMAGE_TYPE.sh + +RUN chmod 0755 . +RUN chmod 0644 postgresql.conf.partial +RUN chmod 0755 /docker-entrypoint-initdb.d/setup_$IMAGE_TYPE.sh diff --git a/integration_tests/postgres/container/README.md b/integration_tests/postgres/container/README.md new file mode 100644 index 0000000..1a66801 --- /dev/null +++ b/integration_tests/postgres/container/README.md @@ -0,0 +1,23 @@ +## About ## + +The integration_tests/postgres/container folder contains the config files for building the custom postgres docker images. +These are based on the standard postgres images (https://hub.docker.com/_/postgres/), but with two changes: + + 1. Enable logging (in $PGDATA/pg_log/postgres.log) + 2. Enable SSL (at least, if type = ssl) + +## Adding a new postgres version ## + +For most new versions, you can just add the new version tag to the `versions` list in setup.sh / test.sh and the new version will be pulled in. +If on the other hand you need a custom Dockerfile / setup script (as we did for 9.3, which doesn't support all of the SSL config options available in later versions), +you will need to add a custom `Dockerfile.[version]`. +The Dockerfile will receive a build-arg named IMAGE_TYPE, which can be `ssl` or `nossl`, which it can use to make the appropriate setup decisions. +See `Dockerfile.9.3` for an example. The only difference there is it uses the 9.3 versions of the conf files. + +## Details ## + + 1. `../setup.sh` calls `build.sh ssl [version]` and `build.sh nossl [version]` for each supported postgres version. + 2. `build.sh` creates a docker image tagged `zgrab_postgres:[version]-[type]`, where `[type]` is `ssl` or `nossl` + 3. The Dockerfile drops the `setup_[type].sh` and `postgresql.conf.[nossl].partial` into the image + 4. `../setup.sh` starts the containers, binding them to ports 3543x (ssl) and 4543x (nonssl). + 5. During startup, the `setup_[type].sh` script is run on the image, setting up logging (and, on SSL images, generating self-signed SSL certificates) diff --git a/integration_tests/postgres/container/build.sh b/integration_tests/postgres/container/build.sh new file mode 100755 index 0000000..af40b63 --- /dev/null +++ b/integration_tests/postgres/container/build.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash + +# This will build our custom postgres docker image for the requested type (ssl/nossl) and postgres version. +# + +set -e + +TYPE=$1 +VERSION=$2 + +if [ "$#" -ne 2 ] || ! ( [ "$TYPE" = "ssl" ] || [ "$TYPE" = "nossl" ] ); then + echo "integration_tests/postgres/container/build.sh: Build a zgrab_postgres docker image" + echo "" + echo "Usage:" + echo "" + echo " $0 [type] [version]" + echo "" + echo "...where [type] is \"ssl\" or \"nossl\", and [version] is the postgres server version." + echo "" + echo "On success, creates an image tagged zgrab_postgres:[version]-[type]". + echo "" + exit 1 +fi + +# If there is a Dockerfile specifically for this version, use that +if [ -f Dockerfile.$VERSION ]; then + cp Dockerfile.$VERSION Dockerfile +else + # TODO: There must be a better way to do this. + # The reason for the sed is, you cannot use build-args in the FROM directive. + # And, it doesn't seem that you can forward the version tag in the docker run command to the 'parent' image. + # So, it seems we're stuck creating a bunch of images whose only difference is the version tag in the FROM statement at build time. + # Or, using the base images, which don't have SSL or logging enabled. + + sed "s!#{POSTGRES_VERSION}!$VERSION!g" < Dockerfile.template > Dockerfile +fi + +docker build --build-arg IMAGE_TYPE=$TYPE -t zgrab_postgres:$VERSION-$TYPE . +rm Dockerfile diff --git a/integration_tests/postgres/container/postgresql.conf.9.3.nossl.partial b/integration_tests/postgres/container/postgresql.conf.9.3.nossl.partial new file mode 100644 index 0000000..fabc953 --- /dev/null +++ b/integration_tests/postgres/container/postgresql.conf.9.3.nossl.partial @@ -0,0 +1,7 @@ +logging_collector = on +log_directory = 'pg_log' +log_filename = 'postgres.log' +log_file_mode = '0666' +log_min_messages = 'DEBUG5' +log_connections = on +log_disconnections = on diff --git a/integration_tests/postgres/container/postgresql.conf.9.3.ssl.partial b/integration_tests/postgres/container/postgresql.conf.9.3.ssl.partial new file mode 100644 index 0000000..c3132d2 --- /dev/null +++ b/integration_tests/postgres/container/postgresql.conf.9.3.ssl.partial @@ -0,0 +1,14 @@ +ssl = on +ssl_ciphers = 'HIGH:MEDIUM:+3DES:!aNULL' +ssl_cert_file = 'server.crt' +ssl_key_file = 'server.key' +ssl_ca_file = '' +ssl_crl_file = '' + +logging_collector = on +log_directory = 'pg_log' +log_filename = 'postgres.log' +log_file_mode = '0666' +log_min_messages = 'DEBUG5' +log_connections = on +log_disconnections = on diff --git a/integration_tests/postgres/container/postgresql.conf.nossl.partial b/integration_tests/postgres/container/postgresql.conf.nossl.partial new file mode 100644 index 0000000..fabc953 --- /dev/null +++ b/integration_tests/postgres/container/postgresql.conf.nossl.partial @@ -0,0 +1,7 @@ +logging_collector = on +log_directory = 'pg_log' +log_filename = 'postgres.log' +log_file_mode = '0666' +log_min_messages = 'DEBUG5' +log_connections = on +log_disconnections = on diff --git a/integration_tests/postgres/container/postgresql.conf.ssl.partial b/integration_tests/postgres/container/postgresql.conf.ssl.partial new file mode 100644 index 0000000..e9d4009 --- /dev/null +++ b/integration_tests/postgres/container/postgresql.conf.ssl.partial @@ -0,0 +1,16 @@ +ssl = on +ssl_ciphers = 'HIGH:MEDIUM:+3DES:!aNULL' +ssl_prefer_server_ciphers = on +ssl_ecdh_curve = 'prime256v1' +ssl_cert_file = 'server.crt' +ssl_key_file = 'server.key' +ssl_ca_file = '' +ssl_crl_file = '' + +logging_collector = on +log_directory = 'pg_log' +log_filename = 'postgres.log' +log_file_mode = '0666' +log_min_messages = 'DEBUG5' +log_connections = on +log_disconnections = on diff --git a/integration_tests/postgres/container/setup_nossl.sh b/integration_tests/postgres/container/setup_nossl.sh new file mode 100755 index 0000000..f13324f --- /dev/null +++ b/integration_tests/postgres/container/setup_nossl.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash +set -e +set -x +WORKDIR=/tmp/postgres_setup +echo "" >> $PGDATA/postgresql.conf + +# Attach the SSL + Logging config to the end of the main postgresql.conf file +cat $WORKDIR/postgresql.conf.partial >> $PGDATA/postgresql.conf diff --git a/integration_tests/postgres/container/setup_ssl.sh b/integration_tests/postgres/container/setup_ssl.sh new file mode 100755 index 0000000..82b5446 --- /dev/null +++ b/integration_tests/postgres/container/setup_ssl.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash +set -e +set -x +WORKDIR=/tmp/postgres_setup + +# Generate a self-signed key/certificate +openssl req -new -x509 -nodes -keyout $WORKDIR/server.p8 -out $WORKDIR/server.crt -subj "/CN=localhost" + +# Get the private key in passwordless PEM format for use by postgres +openssl rsa -in $WORKDIR/server.p8 -out $WORKDIR/server.key + +chown postgres:postgres $WORKDIR/server.* +chmod 0600 $WORKDIR/server.key +chmod 0644 $WORKDIR/server.crt + +cp $WORKDIR/server.key $PGDATA +cp $WORKDIR/server.crt $PGDATA + +echo "" >> $PGDATA/postgresql.conf + +# Attach the SSL + Logging config to the end of the main postgresql.conf file +cat $WORKDIR/postgresql.conf.partial >> $PGDATA/postgresql.conf diff --git a/integration_tests/postgres/setup.sh b/integration_tests/postgres/setup.sh new file mode 100755 index 0000000..c7d3c1a --- /dev/null +++ b/integration_tests/postgres/setup.sh @@ -0,0 +1,62 @@ +#!/usr/bin/env bash + +set -e + +versions="9.3 9.4 9.5 9.6 10.1" +types="ssl nossl" + +function doSetup() { + VERSION=$1 + TYPE=$2 + CONTAINER_NAME="zgrab_postgres_${VERSION}-${TYPE}" + IMAGE_TAG="zgrab_postgres:${VERSION}-${TYPE}" + if grep "$IMAGE_TAG" < images.tmp > /dev/null && [ -x $REBUILD ]; then + echo "postgres/setup: docker image $IMAGE_TAG already exists -- skipping." + else + echo "postgres/setup: docker image $IMAGE_TAG does not exist -- creating..." + ./build.sh $TYPE $VERSION + fi + if docker ps --filter "name=$CONTAINER_NAME" | grep $CONTAINER_NAME; then + echo "postgres/setup: Container $CONTAINER_NAME already running -- stopping..." + docker stop $CONTAINER_NAME + sleep 2 + echo "...stopped." + fi + echo "postgres/setup: Starting container $CONTAINER_NAME on port local port $PORT..." + docker run -itd --rm --name $CONTAINER_NAME -e POSTGRES_PASSWORD=password $IMAGE_TAG + echo "...started." +} + +function waitFor() { + VERSION=$1 + TYPE=$2 + PORT=$3 + CONTAINER_NAME="zgrab_postgres_${VERSION}-${TYPE}" + echo "postgres/setup: Waiting for postgres process to come up on $CONTAINER_NAME..." + while ! (docker exec $CONTAINER_NAME ps -Af | grep "postgres: logger process" > /dev/null); do + echo -n "*" + sleep 1 + done + echo "...postgres is up." +} + +pushd container +docker images --format {{.Repository}}:{{.Tag}} > images.tmp + +for version in $versions; do + for type in $types; do + doSetup $version $type + done +done + +rm -f images.tmp +popd +echo "postgres/setup: Waiting for all postgres containers to start up..." + +for version in $versions; do + for type in $types; do + waitFor $version $type + done +done + +echo "postgres/setup: Containers started." diff --git a/integration_tests/postgres/test.sh b/integration_tests/postgres/test.sh new file mode 100755 index 0000000..dedb403 --- /dev/null +++ b/integration_tests/postgres/test.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash + +set -e + +versions="9.3 9.4 9.5 9.6 10.1" +types="ssl nossl" + +ZGRAB_ROOT=$(dirname $0)/../.. +ZGRAB_OUTPUT=$ZGRAB_ROOT/zgrab-output + +mkdir -p $ZGRAB_OUTPUT/postgres + +function doTest() { + VERSION=$1 + TYPE=$2 + CONTAINER_NAME=zgrab_postgres_$VERSION-$TYPE + CONTAINER_NAME=$CONTAINER_NAME $ZGRAB_ROOT/docker-runner/docker-run.sh postgres > $ZGRAB_OUTPUT/postgres/$VERSION-$TYPE.json + echo "BEGIN DOCKER LOGS FROM $CONTAINER_NAME [{(" + docker logs --tail all $CONTAINER_NAME + echo ")}] END DOCKER LOGS FROM $CONTAINER_NAME" + echo "BEGIN POSTGRES LOGS FROM $CONTAINER_NAME [{(" + # TODO: The "//var/lib" is a work-around for MinGW + docker exec -t $CONTAINER_NAME cat //var/lib/postgresql/data/pg_log/postgres.log + echo ")}] END POSTGRES LOGS FROM $CONTAINER_NAME" +} + +for version in $versions; do + for type in $types; do + doTest $version $type + done +done diff --git a/integration_tests/setup.sh b/integration_tests/setup.sh index fae5663..89de541 100755 --- a/integration_tests/setup.sh +++ b/integration_tests/setup.sh @@ -1,24 +1,30 @@ -#!/bin/bash -e +#!/usr/bin/env bash + +set -e # Set up the integration tests for all modules. # Drop your setup script(s) in integration_tests//setup(.*).sh # Run from root of project TEST_DIR=$(dirname "$0") -cd "$TEST_DIR/.." +ZGRAB_ROOT="$TEST_DIR/.." +cd "$ZGRAB_ROOT" + +echo "Building zgrab2_runner docker image..." +make -C ./docker-runner echo "Setting up integration tests..." pushd integration_tests for mod in $(ls); do - if [ -d "$mod" ]; then - pushd $mod - for setup in $(ls setup*.sh); do - echo "Setting up $mod (integration_tests/$mod/$setup)..." - ./$setup - done - popd - fi + if [ -d "$mod" ]; then + pushd $mod + for setup in $(ls setup*.sh); do + echo "Setting up $mod (integration_tests/$mod/$setup)..." + ./$setup + done + popd + fi done popd diff --git a/integration_tests/ssh/cleanup.sh b/integration_tests/ssh/cleanup.sh index 6ae12da..81e9d48 100755 --- a/integration_tests/ssh/cleanup.sh +++ b/integration_tests/ssh/cleanup.sh @@ -1,11 +1,7 @@ -#!/bin/bash +#!/usr/bin/env bash set +e -CONTAINER_NAME="sshtest" - -echo "BEGIN DOCKER LOGS FROM $CONTAINER_NAME [{(" -docker logs --tail all $CONTAINER_NAME -echo ")}] END DOCKER LOGS FROM $CONTAINER_NAME" +CONTAINER_NAME="zgrab_ssh" docker stop $CONTAINER_NAME diff --git a/integration_tests/ssh/container/Dockerfile b/integration_tests/ssh/container/Dockerfile index 15039e5..264692b 100644 --- a/integration_tests/ssh/container/Dockerfile +++ b/integration_tests/ssh/container/Dockerfile @@ -1,8 +1,8 @@ -FROM ubuntu:16.04 +FROM zgrab2_service_base:latest # Adapted from https://docs.docker.com/engine/examples/running_ssh_service/#run-a-test_sshd-container -RUN apt-get update && apt-get install -y openssh-server +RUN apt-get install -y openssh-server RUN mkdir /var/run/sshd RUN echo 'root:password' | chpasswd RUN sed -i 's/PermitRootLogin prohibit-password/PermitRootLogin yes/' /etc/ssh/sshd_config diff --git a/integration_tests/ssh/setup.sh b/integration_tests/ssh/setup.sh index 6f659e5..0b62c66 100755 --- a/integration_tests/ssh/setup.sh +++ b/integration_tests/ssh/setup.sh @@ -1,16 +1,22 @@ -#!/bin/bash -e -SSH_PORT=33022 -CONTAINER_TAG="sshtest" -CONTAINER_NAME="sshtest" +#!/usr/bin/env bash + +set -e + +CONTAINER_TAG="zgrab_ssh" +CONTAINER_NAME="zgrab_ssh" # TODO FIXME: find a pre-built container with sshd already running? This works, but if it has to build the container image, the apt-get update is very slow. +if docker ps --filter "name=$CONTAINER_NAME" | grep $CONTAINER_NAME; then + echo "ssh/setup: Container $CONTAINER_NAME already running -- stopping..." + docker stop $CONTAINER_NAME + echo "...stopped." +fi + # First attempt to just launch the container -if ! docker run --rm --name $CONTAINER_NAME -itd -p $SSH_PORT:22 $CONTAINER_TAG; then +if ! docker run --rm --name $CONTAINER_NAME -itd $CONTAINER_TAG; then # If it fails, build it from ./container/Dockerfile docker build -t $CONTAINER_TAG ./container # Try again - docker run --rm --name $CONTAINER_NAME -itd -p $SSH_PORT:22 $CONTAINER_TAG + docker run --rm --name $CONTAINER_NAME -itd $CONTAINER_TAG fi - -# TODO: Wait on port 22? diff --git a/integration_tests/ssh/test.sh b/integration_tests/ssh/test.sh index acc0af0..e960c9a 100755 --- a/integration_tests/ssh/test.sh +++ b/integration_tests/ssh/test.sh @@ -1,19 +1,23 @@ -#!/bin/bash -e +#!/usr/bin/env bash -SSH_PORT=33022 -CONTAINER_NAME="sshtest" +set -e +MODULE_DIR=$(dirname $0) +TEST_ROOT=$MODULE_DIR/.. +ZGRAB_ROOT=$MODULE_DIR/../.. +ZGRAB_OUTPUT=$ZGRAB_ROOT/zgrab-output + +CONTAINER_NAME="zgrab_ssh" # Run the SSH-specific integration tests: -# 1. Run zgrab2 on localhost:$SSH_PORT - -if [ -z $ZGRAB_ROOT ] || [ -z $ZGRAB_OUTPUT ]; then - echo "Must set ZGRAB_ROOT and ZGRAB_OUTPUT" - exit 1 -fi +# 1. Run zgrab2 on the container mkdir -p $ZGRAB_OUTPUT/ssh OUTPUT_FILE="$ZGRAB_OUTPUT/ssh/ssh.json" -echo "Testing SSH Version on local port $SSH_PORT..." -echo "127.0.0.1" | $ZGRAB_ROOT/cmd/zgrab2/zgrab2 ssh -p $SSH_PORT $* > $OUTPUT_FILE +echo "ssh/test: Testing SSH Version on local port $SSH_PORT..." +CONTAINER_NAME=$CONTAINER_NAME $ZGRAB_ROOT/docker-runner/docker-run.sh ssh > $OUTPUT_FILE + +echo "ssh/test: BEGIN docker logs from $CONTAINER_NAME [{(" +docker logs --tail all $CONTAINER_NAME +echo ")}] END docker logs from $CONTAINER_NAME" diff --git a/integration_tests/test.sh b/integration_tests/test.sh index 3c1d00e..aaac61b 100755 --- a/integration_tests/test.sh +++ b/integration_tests/test.sh @@ -1,19 +1,16 @@ -#!/bin/bash -e +#!/usr/bin/env bash + +set -e # Do all integration tests for all protocols # To add tests for a new protocol, run `./integration_tests/new.sh ` and implement the appropriate test scripts. # Run from root of project TEST_DIR=$(dirname "$0") -cd "$TEST_DIR/.." +ZGRAB_ROOT="$TEST_DIR/.." +cd "$ZGRAB_ROOT" -# _integration_tests.sh should drop its output into $ZGRAB_OUTPUT//* so that it can be validated -if [ -z $ZGRAB_OUTPUT ]; then - ZGRAB_OUTPUT="$(pwd)/zgrab-output" -fi - -export ZGRAB_OUTPUT=$ZGRAB_OUTPUT -export ZGRAB_ROOT=$(pwd) +ZGRAB_OUTPUT="$ZGRAB_ROOT/zgrab-output" pushd integration_tests for mod in $(ls); do @@ -28,6 +25,8 @@ for mod in $(ls); do done popd +status=0 +failures="" echo "Doing schema validation..." for protocol in $(ls $ZGRAB_OUTPUT); do for outfile in $(ls $ZGRAB_OUTPUT/$protocol); do @@ -35,7 +34,34 @@ for protocol in $(ls $ZGRAB_OUTPUT); do echo "Validating $target [{(" cat $target echo ")}]:" - python -m zschema validate schemas/__init__.py:zgrab2 $target - echo "validation of $target succeeded." + if ! python -m zschema validate schemas/__init__.py:zgrab2 $target; then + echo "Schema validation failed for $protocol/$outfile" + err="schema failure@$protocol/$outfile" + if [[ $status -eq 0 ]]; then + failures="$err" + else + failures="$failures, $err" + fi + status=1 + else + echo "validation of $target succeeded." + scan_status=$($ZGRAB_ROOT/jp -u data.${protocol}.status < $target) + if ! [ $scan_status = "success" ]; then + echo "Scan returned success=$scan_status for $protocol/$outfile" + err="scan failure(${scan_status})@$protocol/$outfile" + if [[ $status -eq 0 ]]; then + failures="$err" + else + failures="$failures, $err" + fi + status=1 + fi + fi done done + +if [ -n "$failures" ]; then + echo "One or more schema validations failed: $failures" +fi + +exit $status diff --git a/lib/mysql/mysql.go b/lib/mysql/mysql.go index e3e68c0..3876528 100644 --- a/lib/mysql/mysql.go +++ b/lib/mysql/mysql.go @@ -23,9 +23,6 @@ import ( "math" "net" "strings" - "time" - - "github.com/zmap/zcrypto/tls" log "github.com/sirupsen/logrus" ) @@ -98,12 +95,6 @@ const ( ) type Config struct { - // @TODO: Does it make sense to make Host/Port connection fields, so that Config can be shared across connections? - Host string - Port uint16 - - TLSConfig *tls.Config - Timeout time.Duration ClientCapabilities uint32 MaxPacketSize uint32 CharSet byte @@ -182,16 +173,6 @@ func InitConfig(base *Config) *Config { if base == nil { base = &Config{} } - if base.TLSConfig == nil { - // @TODO @FIXME Can this be pulled from a global zgrab config module? - base.TLSConfig = &tls.Config{InsecureSkipVerify: true} - } - if base.Port == 0 { - base.Port = DEFAULT_PORT - } - if base.Timeout == 0 { - base.Timeout = DEFAULT_TIMEOUT_SECS * time.Second - } if base.ClientCapabilities == 0 { base.ClientCapabilities = DEFAULT_CLIENT_CAPABILITIES } @@ -565,13 +546,10 @@ func (c *Connection) decodePacket(body []byte) (PacketInfo, error) { func (c *Connection) readPacket() (*ConnectionLogEntry, error) { // @TODO @FIXME Find/use conventional buffered packet-reading functions, handle timeouts / connection reset / etc reader := bufio.NewReader(c.Connection) - if terr := c.Connection.SetReadDeadline(time.Now().Add(c.Config.Timeout)); terr != nil { - return nil, fmt.Errorf("Error calling SetReadTimeout(): %s", terr) - } var header [4]byte n, err := reader.Read(header[:]) if err != nil { - return nil, fmt.Errorf("Error reading packet header (timeout=%s): %s", err, c.Config.Timeout) + return nil, fmt.Errorf("Error reading packet header: %s", err) } if n != 4 { return nil, fmt.Errorf("Wrong number of bytes returned (got %d, expected 4)", n) @@ -615,17 +593,6 @@ func (c *Connection) GetHandshake() *HandshakePacket { return nil } -// Perform a TLS handshake using the configured TLSConfig on the current connection -func (c *Connection) StartTLS() error { - client := tls.Client(c.Connection, c.Config.TLSConfig) - err := client.Handshake() - if err != nil { - return fmt.Errorf("TLS Handshake error: %s", err) - } - c.Connection = client - return nil -} - // Check if both the input client flags and the server capability flags support TLS func (c *Connection) SupportsTLS() bool { if handshake := c.GetHandshake(); handshake != nil { @@ -655,15 +622,7 @@ func (c *Connection) NegotiateTLS() error { } // Connect to the configured server and perform the initial handshake -func (c *Connection) Connect() error { - // Allow Scan on pre-connected / user-supplied connections? - dialer := net.Dialer{Timeout: c.Config.Timeout} - log.Debugf("Connecting to %s:%d", c.Config.Host, c.Config.Port) - conn, err := dialer.Dial("tcp", fmt.Sprintf("%s:%d", c.Config.Host, c.Config.Port)) - if err != nil { - log.Debugf("Error connecting: %v", err) - return fmt.Errorf("Connect error: %s", err) - } +func (c *Connection) Connect(conn net.Conn) error { c.Connection = conn c.State = STATE_CONNECTED c.ConnectionLog = ConnectionLog{ diff --git a/module.go b/module.go index 0bcfdb8..71ad420 100644 --- a/module.go +++ b/module.go @@ -16,22 +16,6 @@ type Scanner interface { Scan(t ScanTarget) (ScanStatus, interface{}, error) } -// ScanStatus is the enum value that states how the scan ended. -type ScanStatus string - -// TODO: Confirm to standard string const format (names, capitalization, hyphens/underscores, etc) -// TODO: Enumerate further status types -const ( - SCAN_SUCCESS = "success" // The protocol in question was positively identified and the scan encountered no errors - SCAN_CONNECTION_REFUSED = "connection-refused" // TCP connection was actively rejected - SCAN_CONNECTION_TIMEOUT = "connection-timeout" // No response to TCP connection request - SCAN_CONNECTION_CLOSED = "connection-closed" // The TCP connection was unexpectedly closed - SCAN_IO_TIMEOUT = "io-timeout" // Timed out waiting on data - SCAN_PROTOCOL_ERROR = "protocol-error" // Received data incompatible with the target protocol - SCAN_APPLICATION_ERROR = "application-error" // The application reported an error - SCAN_UNKNOWN_ERROR = "unknown-error" // Catch-all for unrecognized errors -) - // ScanResponse is the result of a scan on a single host type ScanResponse struct { // Status is required for all responses. Other fields are optional. diff --git a/modules/mysql.go b/modules/mysql.go index 16a6809..5b48856 100644 --- a/modules/mysql.go +++ b/modules/mysql.go @@ -69,10 +69,7 @@ func (s *MySQLScanner) GetPort() uint { } func (s *MySQLScanner) Scan(t zgrab2.ScanTarget) (status zgrab2.ScanStatus, result interface{}, thrown error) { - sql := mysql.NewConnection(&mysql.Config{ - Host: t.IP.String(), - Port: uint16(s.config.Port), - }) + sql := mysql.NewConnection(&mysql.Config{}) result = &MySQLScanResults{} defer func() { recovered := recover() @@ -85,7 +82,11 @@ func (s *MySQLScanner) Scan(t zgrab2.ScanTarget) (status zgrab2.ScanStatus, resu }() defer sql.Disconnect() var err error - if err = sql.Connect(); err != nil { + conn, err := t.Open(&s.config.BaseFlags) + if err != nil { + panic(err) + } + if err = sql.Connect(conn); err != nil { panic(err) } if sql.SupportsTLS() { diff --git a/modules/postgres.go b/modules/postgres.go new file mode 100644 index 0000000..89d16ca --- /dev/null +++ b/modules/postgres.go @@ -0,0 +1,7 @@ +package modules + +import "github.com/zmap/zgrab2/modules/postgres" + +func init() { + postgres.RegisterModule() +} diff --git a/modules/postgres/connection.go b/modules/postgres/connection.go new file mode 100644 index 0000000..57cf645 --- /dev/null +++ b/modules/postgres/connection.go @@ -0,0 +1,241 @@ +package postgres + +import ( + "encoding/binary" + "encoding/hex" + "fmt" + "io" + "net" + "strconv" + "strings" + + log "github.com/sirupsen/logrus" + + "github.com/zmap/zgrab2" +) + +// Connection wraps the state of a given connection to a server +type Connection struct { + Connection net.Conn + Config *PostgresFlags + IsSSL bool +} + +// ServerPacket is a direct representation of the response packet returned by the server (See e.g. https://www.postgresql.org/docs/9.6/static/protocol-message-formats.html) +// The first byte is a message type, an alphanumeric character. +// The following four bytes are the length of the message body. +// The following bytes are the message itself. +// In certain special cases, the Length can be 0; for instance, a response to an SSLRequest is only a S/N Type with no length / body, while pre-startup errors can be a E Type followed by a \n\0-terminated string. +type ServerPacket struct { + Type byte + Length uint32 + Body []byte +} + +// ServerPacket.ToString() is used in logging, to get a human-readable representation of the packet. +func (p *ServerPacket) ToString() string { + // TODO: Don't hex-encode human-readable bodies? + return fmt.Sprintf("{ ServerPacket(%p): { Type: '%c', Length: %d, Body: [[\n%s\n]] } }", &p, p.Type, p.Length, hex.Dump(p.Body)) +} + +// Connection.Send() sends a client packet: a big-endian uint32 length followed by the body. +func (c *Connection) Send(body []byte) error { + toSend := make([]byte, len(body)+4) + copy(toSend[4:], body) + // The length contains the length of the length, hence the +4. + binary.BigEndian.PutUint32(toSend[0:], uint32(len(body)+4)) + + // @TODO: Buffered send? + _, err := c.Connection.Write(toSend) + return err +} + +// Connection.SendU32() sends an uint32 packet to the server. +func (c *Connection) SendU32(val uint32) error { + toSend := make([]byte, 8) + binary.BigEndian.PutUint32(toSend[0:], uint32(8)) + binary.BigEndian.PutUint32(toSend[4:], val) + // @TODO: Buffered send? + _, err := c.Connection.Write(toSend) + return err +} + +// Connection.Close() closes out the underlying TCP connection to the server. +func (c *Connection) Close() error { + return c.Connection.Close() +} + +// Connection.tryReadPacket() attempts to read a packet length + body from the given connection. +func (c *Connection) tryReadPacket(header byte) (*ServerPacket, *zgrab2.ScanError) { + ret := ServerPacket{Type: header} + var length [4]byte + _, err := io.ReadFull(c.Connection, length[:]) + if err != nil && err != io.EOF { + return nil, zgrab2.DetectScanError(err) + } + ret.Length = binary.BigEndian.Uint32(length[:]) + if length[0] > 0x00 { + // For scanning purposes, there is no reason we want to read more than 2^24 bytes + // But in practice, it probably means we have a null-terminated error string + var buf [1024]byte + n, err := c.Connection.Read(buf[:]) + if err != nil && err != io.EOF { + return nil, zgrab2.DetectScanError(err) + } + ret.Body = buf[:n] + if string(buf[n-2:n]) == "\x0a\x00" { + ret.Length = 0 + ret.Body = append(length[:], ret.Body...) + return &ret, nil + } else { + return nil, zgrab2.NewScanError(zgrab2.SCAN_PROTOCOL_ERROR, fmt.Errorf("Server returned too much data: length = 0x%x; first %d bytes = %s", ret.Length, n, hex.EncodeToString(buf[:n]))) + } + } + ret.Body = make([]byte, ret.Length-4) // Length includes the length of the Length uint32 + _, err = io.ReadFull(c.Connection, ret.Body) + if err != nil && err != io.EOF { + return nil, zgrab2.DetectScanError(err) + } + return &ret, nil +} + +// Connection.RequestSSL() sends an SSLRequest packet to the server, and returns true iff the server reports that it is SSL-capable. Otherwise it returns false and possibly an error. +func (c *Connection) RequestSSL() (bool, *zgrab2.ScanError) { + // NOTE: The SSLRequest request type was introduced in version 7.2, released in 2002 (though the oldest supported version is 9.3, released 2013-09-09) + if err := c.SendU32(postgresSSLRequest); err != nil { + return false, zgrab2.DetectScanError(err) + } + var header [1]byte + _, err := io.ReadFull(c.Connection, header[0:1]) + if err != nil { + return false, zgrab2.DetectScanError(err) + } + if header[0] < '0' || header[0] > 'z' { + // Back-end messages always start with the alphanumeric Byte1 value + // We could further constrain this to currently-valid message types, but then we may incorrectly reject future versions + return false, zgrab2.NewScanError(zgrab2.SCAN_PROTOCOL_ERROR, fmt.Errorf("Response message type 0x%02x was not an alphanumeric character", header[0])) + } + switch header[0] { + case 'N': + return false, nil + case 'S': + return true, nil + } + // It was neither a single 'N' / 'S', so it's a failure -- at this point it's just a question of determining if it's an application error (valid packet) or a protocol error + packet, scanError := c.tryReadPacket(header[0]) + if scanError != nil { + return false, scanError + } + switch packet.Type { + case 'E': + return false, zgrab2.NewScanError(zgrab2.SCAN_APPLICATION_ERROR, fmt.Errorf("Application rejected SSLRequest packet -- response = %s", packet.ToString())) + default: + // Returning PROTOCOL_ERROR here since any garbage data that starts with a small-ish u32 could be a valid packet, and no known server versions return anything beyond S/N/E. + return false, zgrab2.NewScanError(zgrab2.SCAN_PROTOCOL_ERROR, fmt.Errorf("Unexpected response type '%c' from server (full response = %s)", packet.Type, packet.ToString())) + + } +} + +// Connection.ReadPacket() reads a ServerPacket from the server. +func (c *Connection) ReadPacket() (*ServerPacket, *zgrab2.ScanError) { + var header [1]byte + _, err := io.ReadFull(c.Connection, header[0:1]) + if err != nil { + return nil, zgrab2.DetectScanError(err) + } + if header[0] < '0' || header[0] > 'z' { + // Back-end messages always start with the alphanumeric Byte1 value + // We could further constrain this to currently-valid message types, but then we may incorrectly reject future versions + return nil, zgrab2.NewScanError(zgrab2.SCAN_PROTOCOL_ERROR, fmt.Errorf("Response message type 0x%02x was not an alphanumeric character", header[0])) + } + return c.tryReadPacket(header[0]) +} + +// Connection.GetTLSLog() gets the connection's TLSLog, or nil if the connection has not yet been set up as TLS +func (c *Connection) GetTLSLog() *zgrab2.TLSLog { + if !c.IsSSL { + return nil + } + return c.Connection.(*zgrab2.TLSConnection).GetLog() +} + +// encodeMap() encodes a map into a byte array of the form "key0\0value\0key1\0value1\0...keyN\0valueN\0\0" +func encodeMap(dict map[string]string) []byte { + var strs []string + for k, v := range dict { + strs = append(strs, k) + strs = append(strs, v) + } + return append([]byte(strings.Join(strs, "\x00")), 0x00, 0x00) +} + +// Connection.SendStartupMessage() creates and sends a StartupMessage: uint16 Major + uint16 Minor + (key/value pairs) +func (c *Connection) SendStartupMessage(version string, kvps map[string]string) error { + dict := encodeMap(kvps) + ret := make([]byte, len(dict)+4) + parts := strings.Split(version, ".") + if len(parts) == 1 { + parts = []string{parts[0], "0"} + } + major, err := strconv.ParseUint(parts[0], 0, 16) + if err != nil { + log.Fatalf("Error parsing major version %s as a uint16:", parts[0], err) + } + minor, err := strconv.ParseUint(parts[1], 0, 16) + if err != nil { + log.Fatalf("Error parsing minor version as a uint16:", parts[1], err) + } + binary.BigEndian.PutUint16(ret[0:2], uint16(major)) + binary.BigEndian.PutUint16(ret[2:4], uint16(minor)) + copy(ret[4:], dict) + + return c.Send(ret) +} + +// Connection.ReadAll() reads packets from the given connection until it hits a timeout, EOF, or a 'Z' packet. +func (c *Connection) ReadAll() ([]*ServerPacket, *zgrab2.ScanError) { + var ret []*ServerPacket = nil + for { + response, readError := c.ReadPacket() + if readError != nil { + if readError.Status == zgrab2.SCAN_IO_TIMEOUT || readError.Err == io.EOF { + return ret, nil + } else { + return ret, readError + } + } + ret = append(ret, response) + if response.Type == 'Z' { + return ret, nil + } + } +} + +// connectionManager is a utility for getting connections and ensuring that they all get closed +// TODO: Is there something like this in the standard libraries?? +type connectionManager struct { + connections []io.Closer +} + +// Add a connection to be cleaned up +func (m *connectionManager) addConnection(c io.Closer) { + m.connections = append(m.connections, c) +} + +// Close all managed connections +func (m *connectionManager) cleanUp() { + for _, v := range m.connections { + // Close them all even if there is a panic with one + defer func(c io.Closer) { + err := c.Close() + if err != nil { + log.Debugf("Got error closing connection: %v", err) + } + }(v) + } +} + +// Get a new connectionmanager instance +func newConnectionManager() *connectionManager { + return &connectionManager{} +} diff --git a/modules/postgres/scanner.go b/modules/postgres/scanner.go new file mode 100644 index 0000000..bac305b --- /dev/null +++ b/modules/postgres/scanner.go @@ -0,0 +1,443 @@ +package postgres + +import ( + "encoding/binary" + "fmt" + "net" + "strings" + + log "github.com/sirupsen/logrus" + "github.com/zmap/zgrab2" +) + +const ( + // From https://www.postgresql.org/docs/10/static/protocol-message-formats.html: "The SSL request code. The value is chosen to contain 1234 in the most significant 16 bits, and 5679 in the least significant 16 bits. (To avoid confusion, this code must not be the same as any protocol version number.)" + postgresSSLRequest = 80877103 +) + +const ( + KeyUnknownErrorTag = "_unknown_error_tag" + KeyBadParameters = "_bad_parameters" +) + +// PostgresResults is the information returned by the scanner to the caller. +// https://raw.githubusercontent.com/nmap/nmap/master/nmap-service-probes uses the line number of the error response (e.g. StartupError["line"]) to infer the version number +type PostgresResults struct { + TLSLog *zgrab2.TLSLog `json:"tls,omitempty"` + SupportedVersions string `json:"supported_versions,omitempty"` + ProtocolError *PostgresError `json:"protocol_error,omitempty"` + StartupError *PostgresError `json:"startup_error,omitempty"` + UserStartupError *PostgresError `json:"user_startup_error,omitempty"` + IsSSL bool `json:"is_ssl"` + AuthenticationMode *AuthenticationMode `json:"authentication_mode,omitempty"` + ServerParameters *ServerParameters `json:"server_parameters,omitempty"` + BackendKeyData *BackendKeyData `json:"backend_key_data,omitempty", zgrab:"debug"` + TransactionStatus string `json:"transaction_status,omitempty"` +} + +// PostgresError is parsed the payload of an 'E'-type packet, mapping the friendly names of the various fields to the values returned by the server +type PostgresError map[string]string + +// After authentication, the server sends a series of 'S' packets with key/value pairs. +// We keep track of them all -- but the golang postgres library only stores the server_version and TimeZone. +type ServerParameters map[string]string + +// BackendKeyData is the data returned by the 'K'-type packet +type BackendKeyData struct { + ProcessID uint32 `json:"process_id"` + SecretKey uint32 `json:"secret_key"` +} + +// AuthenticationMode abstracts the various 'R'-type packets +type AuthenticationMode struct { + Mode string `json:"mode"` + Payload []byte `json:"payload,omitempty"'` +} + +// PostgresFlags sets the module-specific flags that can be passed in from the command line +type PostgresFlags struct { + zgrab2.BaseFlags + zgrab2.TLSFlags + SkipSSL bool `long:"skip-ssl" description:"If set, do not attempt to negotiate an SSL connection"` + Verbose bool `long:"verbose" description:"More verbose logging, include debug fields in the scan results"` + ProtocolVersion string `long:"protocol_version" description:"The protocol to use in the StartupPacket" default:"3.0"` + User string `long:"user" description:"Username to pass to StartupMessage. If omitted, no user will be sent." default:""` + Database string `long:"database" description:"Database to pass to StartupMessage. If omitted, none will be sent." default:""` + ApplicationName string `long:"application_name" description:"application_name value to pass in StartupMessage. If omitted, none will be sent." default:""` +} + +// PostgresScanner is the zgrab2 scanner type for the postgres protocol +type PostgresScanner struct { + Config *PostgresFlags +} + +// PostgresModule is the zgrab2 module for the postgres protocol +type PostgresModule struct { +} + +// decodeAuthMode() decodes the body of an 'R'-type packet and returns a friendlier description of it +func decodeAuthMode(buf []byte) *AuthenticationMode { + // See the 'R' messages in https://www.postgresql.org/docs/10/static/protocol-message-formats.html + modeMap := map[uint32]string{ + 2: "kerberos_v5", + 3: "password_cleartext", + 5: "password_md5", + 6: "scm_credentials", + 7: "gss", + 9: "sspi", + 10: "sasl", + + // The following aren't actually authentication codes, but they are valid 'R'-type messages + 0: "ok", + 8: "gss-continue", + 11: "sasl-continue", + 12: "sasl-final", + } + + modeId := binary.BigEndian.Uint32(buf[0:4]) + mode, ok := modeMap[modeId] + if !ok { + mode = fmt.Sprintf("unknown (0x%x)", modeId) + } + return &AuthenticationMode{ + Mode: mode, + Payload: buf[4:], + } +} + +// decodeError() decodes an 'E'-type tag into a map of friendly name -> value; see https://www.postgresql.org/docs/10/static/protocol-error-fields.html +func decodeError(buf []byte) *PostgresError { + partMap := map[byte]string{ + 'S': "severity", + // Return both severity and severity_v -- they give the same content, but severity is localized, so it can leak some information about the server + 'V': "severity_v", + 'C': "code", + 'M': "message", + 'D': "detail", + 'H': "hint", + 'P': "position", + 'p': "internal_position", + 'q': "internal_query", + 'W': "where", + 's': "schema", + 't': "table", + 'd': "data", + 'n': "constraint", + 'F': "file", + 'L': "line", + 'R': "routine", + } + + ret := make(PostgresError) + parts := strings.Split(string(buf), "\x00") + for _, part := range parts { + if len(part) > 0 { + key, ok := partMap[part[0]] + if !ok { + ret[KeyUnknownErrorTag] = appendStringList(ret[KeyUnknownErrorTag], part) + } else { + value := part[1:] + ret[key] = value + } + } + } + return &ret +} + +// appendStringList() adds an entry to a semicolon-separated list; if the list is empty, no semicolon is added. +func appendStringList(dest string, val string) string { + if dest == "" { + return val + } else { + return dest + "; " + val + } +} + +// ServerParameters.appendBadParam() adds a packet to the list of bad/unexpected parameters +func (p *ServerParameters) appendBadParam(packet *ServerPacket) { + (*p)[KeyBadParameters] = appendStringList((*p)[KeyBadParameters], packet.ToString()) +} + +// PostgresResults.decodeServerResponse() fills out the results object with packets returned by the server. +func (results *PostgresResults) decodeServerResponse(packets []*ServerPacket) { + // Note: The only parameters the golang postgres library pays attention to are the server_version and the TimeZone. + serverParams := make(ServerParameters) + for _, packet := range packets { + switch packet.Type { + case 'S': + parts := strings.Split(string(packet.Body), "\x00") + if len(parts) == 2 || (len(parts) == 3 && len(parts[2]) == 0) { + serverParams[parts[0]] = parts[1] + } else { + log.Debugf("Unexpected format for ParameterStatus packet (%d parts)", len(parts)) + serverParams.appendBadParam(packet) + } + case 'K': + if packet.Length != 12 { + log.Debugf("Bad size for BackendKeyData (%d)", packet.Length) + serverParams.appendBadParam(packet) + } else { + pid := binary.BigEndian.Uint32(packet.Body[0:4]) + key := binary.BigEndian.Uint32(packet.Body[4:8]) + results.BackendKeyData = &BackendKeyData{ + ProcessID: pid, + SecretKey: key, + } + } + case 'Z': + if packet.Length != 5 { + log.Debugf("Bad size for ReadyForQuery (%d)", packet.Length) + serverParams.appendBadParam(packet) + } else { + results.TransactionStatus = string(packet.Body[0]) + } + case 'R': + results.AuthenticationMode = decodeAuthMode(packet.Body) + case 'E': + results.UserStartupError = decodeError(packet.Body) + default: + // Ignore other message types + } + } + // Merge the ServerParams, so that we can keep track of values across multiple connections + if len(serverParams) > 0 { + if results.ServerParameters == nil { + results.ServerParameters = &serverParams + } else { + for k, v := range serverParams { + (*results.ServerParameters)[k] = v + } + } + } +} + +func (m *PostgresModule) NewFlags() interface{} { + return new(PostgresFlags) +} + +func (m *PostgresModule) NewScanner() zgrab2.Scanner { + return new(PostgresScanner) +} + +func (f *PostgresFlags) Validate(args []string) error { + return nil +} + +func (f *PostgresFlags) Help() string { + return "" +} + +func (s *PostgresScanner) Init(flags zgrab2.ScanFlags) error { + f, _ := flags.(*PostgresFlags) + s.Config = f + return nil +} + +func (s *PostgresScanner) InitPerSender(senderID int) error { + return nil +} + +func (s *PostgresScanner) GetName() string { + return s.Config.Name +} + +func (s *PostgresScanner) GetPort() uint { + return s.Config.Port +} + +// PostgresScanner.DoSSL() attempts to upgrade the connection to SSL, returning an error on failure. +func (s *PostgresScanner) DoSSL(sql *Connection) error { + var conn *zgrab2.TLSConnection + var err error + if conn, err = s.Config.TLSFlags.GetTLSConnection(sql.Connection); err != nil { + return err + } + if err = conn.Handshake(); err != nil { + return err + } + // Replace sql.Connection to allow future calls to go over the secure connection + sql.Connection = conn + return nil +} + +// PostgresScanner.newConnection() opens up a new connection to the ScanTarget, and if necessary, attempts to update the connection to SSL +func (s *PostgresScanner) newConnection(t *zgrab2.ScanTarget, mgr *connectionManager, nossl bool) (*Connection, *zgrab2.ScanError) { + var conn net.Conn + var err error + // Open a managed connection to the ScanTarget, register it for automatic cleanup + if conn, err = t.Open(&s.Config.BaseFlags); err != nil { + return nil, zgrab2.DetectScanError(err) + } + mgr.addConnection(conn) + sql := Connection{Connection: conn, Config: s.Config} + sql.IsSSL = false + if !nossl && !s.Config.SkipSSL { + hasSSL, sslError := sql.RequestSSL() + if sslError != nil { + return nil, sslError + } + if hasSSL { + if err = s.DoSSL(&sql); err != nil { + return nil, zgrab2.NewScanError(zgrab2.SCAN_APPLICATION_ERROR, err) + } + sql.IsSSL = true + } + } + return &sql, nil +} + +// Return the default KVPs used for all Startup messages +func (s *PostgresScanner) getDefaultKVPs() map[string]string { + return map[string]string{ + "client_encoding": "UTF8", + "datestyle": "ISO, MDY", + } +} + +// PostgresScanner.Scan() does the actual scanning. It opens two connections: +// With the first it sends a bogus protocol version in hopes of getting a list of supported protcols back. +// With the second, it sends a standard StartupMessage, but without the required "user" field. +func (s *PostgresScanner) Scan(t zgrab2.ScanTarget) (status zgrab2.ScanStatus, result interface{}, thrown error) { + var results PostgresResults + + mgr := newConnectionManager() + defer mgr.cleanUp() + + // Send too-low protocol version (0.0) StartupMessage to get a simple supported-protocols error string + // Also do TLS handshake, if configured / supported + { + sql, connectErr := s.newConnection(&t, mgr, false) + if connectErr != nil { + return connectErr.Unpack(nil) + } + defer sql.Close() + if sql.IsSSL { + results.IsSSL = true + // This pointer will be populated as the connection is negotiated + results.TLSLog = sql.GetTLSLog() + } else { + results.IsSSL = false + results.TLSLog = nil + } + // Do SSL the first round, so that if we bail, we still have the TLS logs + + // Announce a (bogus) version 0.0 client, expect an 'E'-tagged response with just the error message + if err := sql.SendU32(0x00); err != nil { + return zgrab2.TryGetScanStatus(err), &results, err + } + response, readErr := sql.ReadPacket() + if readErr != nil { + return readErr.Unpack(&results) + } + + if response.Type != 'E' { + // No server should be allowing a 0.0 client...but if it does allow it, don't bail out + log.Debugf("Unexpected response from server: %s", response.ToString()) + results.SupportedVersions = response.ToString() + } else { + results.SupportedVersions = strings.Trim(string(response.Body), "\x00\r\n ") + } + + if _, err := sql.ReadAll(); err != nil { + return err.Unpack(&results) + } + sql.Close() + } + + // Send too-high protocol version (255.255) StartupMessage to get full error message (including line numbers, useful for probing server version) + { + sql, connectErr := s.newConnection(&t, mgr, true) + if connectErr != nil { + return connectErr.Unpack(&results) + } + + if err := sql.SendU32(0xff<<16 | 0xff); err != nil { + // Whatever the actual problem, a send error will be treated as a SCAN_PROTOCOL_ERROR since the scan got this far + return zgrab2.SCAN_PROTOCOL_ERROR, &results, err + } + response, readErr := sql.ReadPacket() + if readErr != nil { + return readErr.Unpack(&results) + } + + if response.Type != 'E' { + // No server should be allowing a 255.255 client...but if it does allow it, don't bail out + log.Debugf("Unexpected response from server: %s", response.ToString()) + results.ProtocolError = nil + } else { + results.ProtocolError = decodeError(response.Body) + } + + if _, err := sql.ReadAll(); err != nil { + return err.Unpack(&results) + } + sql.Close() + } + + // Send a StartupMessage with a valid protocol version number, but omit the user field + { + sql, connectErr := s.newConnection(&t, mgr, true) + if connectErr != nil { + return connectErr.Unpack(&results) + } + if err := sql.SendStartupMessage(s.Config.ProtocolVersion, s.getDefaultKVPs()); err != nil { + return zgrab2.SCAN_PROTOCOL_ERROR, &results, err + } + if response, err := sql.ReadPacket(); err != nil { + log.Debugf("Error reading response after StartupMessage: %v", err) + return err.Unpack(&results) + } else { + if response.Type == 'E' { + results.StartupError = decodeError(response.Body) + } else { + // No server should allow a missing User field -- but if it does, log and continue + log.Debugf("Unexpected response from server: %s", response.ToString()) + } + } + // TODO: use any packets returned to fill out results? There probably won't be any, and they will probably be overwritten if Config.User etc is set... + if _, err := sql.ReadAll(); err != nil { + return err.Unpack(&results) + } + sql.Close() + } + + // If user / database / application_name are provided, do a final scan with those + if s.Config.User != "" || s.Config.Database != "" || s.Config.ApplicationName != "" { + sql, connectErr := s.newConnection(&t, mgr, false) + if connectErr != nil { + return connectErr.Unpack(&results) + } + kvps := s.getDefaultKVPs() + if s.Config.User != "" { + kvps["user"] = s.Config.User + } + if s.Config.Database != "" { + kvps["database"] = s.Config.Database + } + if s.Config.ApplicationName != "" { + kvps["application_name"] = s.Config.ApplicationName + } + if err := sql.SendStartupMessage(s.Config.ProtocolVersion, kvps); err != nil { + return zgrab2.SCAN_PROTOCOL_ERROR, &results, err + } + packets, err := sql.ReadAll() + sql.Close() + if packets != nil { + results.decodeServerResponse(packets) + } + if err != nil { + return err.Unpack(&results) + } + } + return zgrab2.SCAN_SUCCESS, &results, thrown +} + +// Called by modules/postgres.go's init() +func RegisterModule() { + var module PostgresModule + _, err := zgrab2.AddCommand("postgres", "Postgres", "Grab a Postgres handshake", 5432, &module) + log.SetLevel(log.DebugLevel) + if err != nil { + log.Fatal(err) + } +} diff --git a/processing.go b/processing.go index e9bc62a..6c851bc 100644 --- a/processing.go +++ b/processing.go @@ -3,10 +3,12 @@ package zgrab2 import ( "bufio" "encoding/json" + "fmt" "io" "net" "strings" "sync" + "time" log "github.com/sirupsen/logrus" ) @@ -24,6 +26,53 @@ type ScanTarget struct { Domain string } +// scanTargetConnection wraps an existing net.Conn connection, overriding the Read/Write methods to use the configured timeouts +type scanTargetConnection struct { + net.Conn + Timeout time.Duration +} + +func (c *scanTargetConnection) Read(b []byte) (n int, err error) { + if c.Timeout > 0 { + if err = c.Conn.SetReadDeadline(time.Now().Add(c.Timeout)); err != nil { + return 0, err + } + } + return c.Conn.Read(b) +} + +func (c *scanTargetConnection) Write(b []byte) (n int, err error) { + if c.Timeout > 0 { + if err = c.Conn.SetWriteDeadline(time.Now().Add(c.Timeout)); err != nil { + return 0, err + } + } + return c.Conn.Write(b) +} + +// ScanTarget.Open connects to the ScanTarget using the configured flags, and returns a net.Conn that uses the configured timeouts for Read/Write operations. +func (t *ScanTarget) Open(flags *BaseFlags) (net.Conn, error) { + timeout := time.Second * time.Duration(flags.Timeout) + target := fmt.Sprintf("%s:%d", t.IP.String(), flags.Port) + var conn net.Conn + var err error + if timeout > 0 { + conn, err = net.DialTimeout("tcp", target, timeout) + } else { + conn, err = net.Dial("tcp", target) + } + if err != nil { + if conn != nil { + conn.Close() + } + return nil, err + } + return &scanTargetConnection{ + Conn: conn, + Timeout: timeout, + }, nil +} + // grabTarget calls handler for each action func grabTarget(input ScanTarget, m *Monitor) []byte { moduleResult := make(map[string]ScanResponse) diff --git a/schemas/__init__.py b/schemas/__init__.py index 072eeac..36307cc 100644 --- a/schemas/__init__.py +++ b/schemas/__init__.py @@ -1,3 +1,4 @@ # Ensure that all of the modules get executed so that they are registered import schemas.mysql import schemas.ssh +import schemas.postgres diff --git a/schemas/postgres.py b/schemas/postgres.py new file mode 100644 index 0000000..0a8c37a --- /dev/null +++ b/schemas/postgres.py @@ -0,0 +1,71 @@ +# zschema sub-schema for zgrab2's postgres module +# Registers zgrab2-postgres globally, and postgres with the main zgrab2 schema. +from zschema.leaves import * +from zschema.compounds import * +import zschema.registry + +import schemas.zcrypto as zcrypto +import schemas.zgrab2 as zgrab2 + +# modules/postgres/scanner.go - decodeError() (TODO: Currently an unconstrained +# map[string]string; it is possible to get "unknown (0x%x)" fields, but it +# would probably be proper to reject those at this point) + +# These are defined in detail at +# https://www.postgresql.org/docs/10/static/protocol-error-fields.html +postgres_error = SubRecord({ + "severity": String(required=True), + "severity_v": String(), + "code": String(required=True), + "message": String(), + "detail": String(), + "hint": String(), + "position": String(), + "internal_position": String(), + "internal_query": String(), + "where": String(), + "schema": String(), + "table": String(), + "data": String(), + "file": String(), + "line": String(), + "routine": String(), +}) + +# modules/postgres/scanner.go - decodeAuthMode() +AUTH_MODES = [ + "kerberos_v5", "password_cleartext", "password_md5", "scm_credentials", + "gss", "sspi", "sasl", "ok", "gss-continue", "sasl-continue", "sasl-final" +] + +# modules/postgres/scanner.go: AuthenticationMode +postgres_auth_mode = SubRecord({ + "mode": Enum(values=AUTH_MODES, required=True), + "Payload": Binary(), +}) + +# modules/postgres/scanner.go: BackendKeyData +postgres_key_data = SubRecord({ + "process_id": Unsigned32BitInteger(), + "secret_key": Unsigned32BitInteger(), +}) + +# modules/postgres/scanner.go: PostgresResults +postgres_scan_response = SubRecord({ + "result": SubRecord({ + "tls": zgrab2.tls_log, + "supported_versions": String(), + "protocol_error": postgres_error, + "startup_error": postgres_error, + "is_ssl": Boolean(required=True), + "authentication_mode": postgres_auth_mode, + # TODO FIXME: This is currendly an unconstrained map[string]string + "server_parameters": String(), + "backend_key_data": postgres_key_data, + "transaction_status": String(), + }) +}, extends=zgrab2.base_scan_response) + +zschema.registry.register_schema("zgrab2-postgres", postgres_scan_response) + +zgrab2.register_scan_response_type("postgres", postgres_scan_response) diff --git a/schemas/zcrypto.py b/schemas/zcrypto.py index c1a6cdc..4f1d657 100644 --- a/schemas/zcrypto.py +++ b/schemas/zcrypto.py @@ -206,11 +206,43 @@ server_certificate_valid = SubRecord({ "error":String() }) +hex_name_value = SubRecord({ + "hex":String(), + "name":String(), + # FIXME: Integer size? + "value":Integer(), +}) + +cipher_suite = hex_name_value + +signature_and_hash_type = SubRecord({ + "signature_algorithm":String(), + "hash_algorithm":String(), +}) + # zcrypto/tls/tls_handshake.go: ServerHandshake tls_handshake = SubRecord({ "client_hello":SubRecord({ - "random":Binary(), + "cipher_suites": ListOf(cipher_suite), + "compression_methods":ListOf(hex_name_value), + "extended_master_secret": Boolean(), "extended_random":Binary(), + "heartbeat":Boolean(), + "next_protocol_negotiation":Boolean(), + "ocsp_stapling":Boolean(), + "random":Binary(), + "sct_enabled":Boolean(), + "scts":Boolean(), + "secure_renegotiation":Boolean(), + "signature_and_hashes":ListOf(signature_and_hash_type), + "supported_curves": ListOf(hex_name_value), + "supported_point_formats": ListOf(hex_name_value), + "ticket": Boolean(), + "version":SubRecord({ + "name":String(), + # FIXME: Integer size? + "value":Integer() + }), }), "server_hello":SubRecord({ "version":SubRecord({ @@ -220,12 +252,7 @@ tls_handshake = SubRecord({ }), "random":Binary(), "session_id": Binary(), - "cipher_suite":SubRecord({ - "hex":String(), - "name":String(), - # FIXME: Integer size? - "value":Integer(), - }), + "cipher_suite":cipher_suite, # FIXME: Integer size? "compression_method":Integer(), "ocsp_stapling":Boolean(), @@ -309,10 +336,7 @@ tls_handshake = SubRecord({ "raw":Binary(), "type":String(), "valid":Boolean(), - "signature_and_hash_type":SubRecord({ - "signature_algorithm":String(), - "hash_algorithm":String(), - }), + "signature_and_hash_type":signature_and_hash_type, "tls_version":SubRecord({ "name":String(), # FIXME: Integer size diff --git a/schemas/zgrab2.py b/schemas/zgrab2.py index a081e6e..7dfa1a7 100644 --- a/schemas/zgrab2.py +++ b/schemas/zgrab2.py @@ -20,7 +20,8 @@ def DebugOnly(childType): # zgrab2/processing.go: Grab grab_result = Record({ - "ip": IPv4Address(required = True), + "ip": IPv4Address(required = False), + "domain": String(required = False), "data": SubRecord(scan_response_types, required = True), }) diff --git a/status.go b/status.go new file mode 100644 index 0000000..1e0f67a --- /dev/null +++ b/status.go @@ -0,0 +1,91 @@ +package zgrab2 + +import ( + "io" + "net" +) + +// ScanStatus is the enum value that states how the scan ended. +type ScanStatus string + +// TODO: Conform to standard string const format (names, capitalization, hyphens/underscores, etc) +// TODO: Enumerate further status types +const ( + SCAN_SUCCESS = "success" // The protocol in question was positively identified and the scan encountered no errors + SCAN_CONNECTION_REFUSED = "connection-refused" // TCP connection was actively rejected + SCAN_CONNECTION_TIMEOUT = "connection-timeout" // No response to TCP connection request + + // TODO: lump connection closed / io timeout? + SCAN_CONNECTION_CLOSED = "connection-closed" // The TCP connection was unexpectedly closed + SCAN_IO_TIMEOUT = "io-timeout" // Timed out waiting on data + SCAN_PROTOCOL_ERROR = "protocol-error" // Received data incompatible with the target protocol + // TODO: Add SCAN_TLS_PROTOCOL_ERROR? For purely TLS-wrapped protocols, SCAN_PROTOCOL_ERROR is fine -- but for protocols that have a non-TLS bootstrap (e.g. a STARTTLS procedure), SCAN_PROTOCOL_ERROR is misleading, since it did get far-enough into the application protocol to start TLS handshaking -- but a garbled TLS handshake is certainly not a SCAN_APPLICATION_ERROR + SCAN_APPLICATION_ERROR = "application-error" // The application reported an error + SCAN_UNKNOWN_ERROR = "unknown-error" // Catch-all for unrecognized errors +) + +// ScanError an error that also includes a ScanStatus. +type ScanError struct { + Status ScanStatus + Err error +} + +// Error is an implementation of the builtin.error interface -- just forward the wrapped error's Error() method +func (err *ScanError) Error() string { + if err.Err == nil { + return "" + } + return err.Err.Error() +} + +func (err *ScanError) Unpack(results interface{}) (ScanStatus, interface{}, error) { + return err.Status, results, err.Err +} + +// NewScanError returns a ScanError with the given status and error. +func NewScanError(status ScanStatus, err error) *ScanError { + return &ScanError{Status: status, Err: err} +} + +// DetectScanError returns a ScanError that attempts to detect the status from the given error. +func DetectScanError(err error) *ScanError { + return &ScanError{Status: TryGetScanStatus(err), Err: err} +} + +// TryGetScanStatus attempts to get the ScanStatus enum value corresponding to the given error. +// Mostly supports network errors. A nil error is interpreted as SCAN_SUCCESS. +// An unrecognized error is interpreted as SCAN_UNKNOWN_ERROR. +func TryGetScanStatus(err error) ScanStatus { + if err == nil { + return SCAN_SUCCESS + } + if err == io.EOF { + // Presumably the caller did not call TryGetScanStatus if the EOF was expected + return SCAN_IO_TIMEOUT + } + switch e := err.(type) { + case *ScanError: + return e.Status + case *net.OpError: + switch e.Op { + case "dial": + // TODO: Distinguish connection timeout / connection refused + // Windows examples: + // "dial tcp 192.168.30.3:22: connectex: A connection attempt failed because the connected party did not properly respond after a period of time, or established connection failed because connected host has failed to respond." + // "dial tcp 127.0.0.1:22: connectex: No connection could be made because the target machine actively refused it." + return SCAN_CONNECTION_TIMEOUT + case "read": + // TODO: Distinguish connection reset vs timeout + return SCAN_IO_TIMEOUT + case "write": + // TODO: Distinguish connection reset vs timeout + return SCAN_IO_TIMEOUT + default: + // TODO: Do we need a generic network error? + return SCAN_UNKNOWN_ERROR + } + // TODO: More error types + default: + return SCAN_UNKNOWN_ERROR + } +} diff --git a/test_mysql_all.sh b/test_mysql_all.sh deleted file mode 100755 index f195cff..0000000 --- a/test_mysql_all.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/bash -e - -# These are all of the versions supported by https://hub.docker.com/_/mysql/ -VERSIONS="5.5 5.6 5.7 8.0" - -# Unfortunately, the 5.5/5.6 containers do not have TLS support built in, so for now we are constrained to checking the version string - -for version in $VERSIONS; do - ./test_mysql_version.sh "$version" -done diff --git a/test_mysql_version.sh b/test_mysql_version.sh deleted file mode 100755 index b3656db..0000000 --- a/test_mysql_version.sh +++ /dev/null @@ -1,56 +0,0 @@ -#!/bin/bash -if [ "$#" -lt 1 ]; then - echo "Usage:" - echo "" - echo " $0 [version]" - echo "" - echo "[version] should be one of the supported tags listed at https://hub.docker.com/_/mysql/" - echo "" - echo "Example:" - echo "" - echo " $0 5.5" - echo "" - exit 1 -fi - -SQL_VERSION="$1" -CONTAINER_NAME="testmysql-$SQL_VERSION" -OUTPUT_FILE="out-$SQL_VERSION.json" - -shift - -echo "Starting the mysql $SQL_VERSION container (forwarding localhost:3306 -> container:3306)..." -set -x -if ! docker run -itd -p 3306:3306 --rm --name $CONTAINER_NAME -e MYSQL_ROOT_PASSWORD=rootPassword mysql:$SQL_VERSION; then - echo "Error running mysql:$SQL_VERSION docker instance: $?" - exit 1 -fi -set +x - -function cleanup { - echo "Stopping $CONTAINER_NAME..." - docker stop $CONTAINER_NAME - echo "Stopped." -} - -trap cleanup EXIT SIGINT - -set +e - -pushd cmd/zgrab2 -go build -o ../../zgrab2 -popd - -CONTAINER_NAME=$CONTAINER_NAME MYSQLD_PORT=3306 ./wait_for_mysqld.sh - -set -x -echo "127.0.0.1" | ./zgrab2 mysql $* > $OUTPUT_FILE -set +x -SERVER_VERSION=$(jp data.mysql.result.handshake.parsed.server_version < $OUTPUT_FILE) - -if [[ "$SERVER_VERSION" =~ "$SQL_VERSION\..*" ]]; then - echo "Server version matches expected version: $SERVER_VERSION =~ $SQL_VERSION" -else - echo "Server versiom mismatch: Got $SERVER_VERSION, expected $SQL_VERSION.*" - exit 1 -fi diff --git a/tls.go b/tls.go index 8978a64..9a366f7 100644 --- a/tls.go +++ b/tls.go @@ -222,6 +222,7 @@ func (t *TLSFlags) GetTLSConfig() (*tls.Config, error) { type TLSConnection struct { tls.Conn flags *TLSFlags + log *TLSLog } type TLSLog struct { @@ -232,31 +233,29 @@ type TLSLog struct { } func (z *TLSConnection) GetLog() *TLSLog { - handshake := z.Conn.GetHandshakeLog() - if !z.flags.KeepClientLogs { - handshake.ClientHello = nil - handshake.ClientKeyExchange = nil - handshake.ClientFinished = nil - } - var heartbleed *tls.Heartbleed - if z.flags.Heartbleed { - heartbleed = z.Conn.GetHeartbleedLog() - } else { - heartbleed = nil - } - return &TLSLog{ - HandshakeLog: handshake, - HeartbleedLog: heartbleed, + if z.log == nil { + z.log = &TLSLog{} } + + return z.log } func (z *TLSConnection) Handshake() error { + log := z.GetLog() if z.flags.Heartbleed { buf := make([]byte, 256) + defer func() { + log.HandshakeLog = z.Conn.GetHandshakeLog() + log.HeartbleedLog = z.Conn.GetHeartbleedLog() + }() // TODO - CheckHeartbleed does not bubble errors from Handshake _, err := z.CheckHeartbleed(buf) return err } else { + defer func() { + log.HandshakeLog = z.Conn.GetHandshakeLog() + log.HeartbleedLog = nil + }() return z.Conn.Handshake() } } diff --git a/utility.go b/utility.go index 0a0c242..4bfc982 100644 --- a/utility.go +++ b/utility.go @@ -100,33 +100,3 @@ func duplicateIP(ip net.IP) net.IP { copy(dup, ip) return dup } - -// Given an error object thrown by a scan, attempt to get the appropriate ScanStatus enum value -func TryGetScanStatus(err error) ScanStatus { - if err == nil { - return SCAN_SUCCESS - } - switch e := err.(type) { - case *net.OpError: - switch e.Op { - case "dial": - // TODO: Distinguish connection timeout / connection refused - // Windows examples: - // "dial tcp 192.168.30.3:22: connectex: A connection attempt failed because the connected party did not properly respond after a period of time, or established connection failed because connected host has failed to respond." - // "dial tcp 127.0.0.1:22: connectex: No connection could be made because the target machine actively refused it." - return SCAN_CONNECTION_TIMEOUT - case "read": - // TODO: Distinguish connection reset vs timeout - return SCAN_IO_TIMEOUT - case "write": - // TODO: Distinguish connection reset vs timeout - return SCAN_IO_TIMEOUT - default: - // TODO: Do we need a generic network error? - return SCAN_UNKNOWN_ERROR - } - // TODO: More error types - default: - return SCAN_UNKNOWN_ERROR - } -}