zgrab2/schemas/zcrypto.py

from zschema.leaves import *
from zschema.compounds import *
import zschema.registry

# Mostly copied from zmap/zgrab/zgrab_schema.py
# Since the struct -> json mappings are defined in zcrypto, it seems like it
# would make sense to have this schema defined there

# For items in x509/pkix/pkix.go, there is a corresponding struct in
# x509/pkix/json.go, prefixed with "aux" (e.g. Name -> auxName)

# x509/pkix/pkix.go: Name
distinguished_name = SubRecord({
    "serial_number": ListOf(String()),
    "common_name": ListOf(String()),
    "country": ListOf(String()),
    "locality": ListOf(String()),
    "province": ListOf(String()),
    "street_address": ListOf(String()),
    "organization": ListOf(String()),
    "organizational_unit": ListOf(String()),
    "postal_code": ListOf(String()),
    "domain_component": ListOf(String()),
})

# x509/pkix/pkix.go: Extension
unknown_extension = SubRecord({
    "id": String(),
    "critical": Boolean(),
    "value": Binary(),
})

# x509/pkix/pkix.go: type EDIPartyName struct
edi_party_name = SubRecord({
    "name_assigner": AnalyzedString(es_include_raw=True),
    "party_name": AnalyzedString(es_include_raw=True),
})

# x509/extensions.go: GeneralNames/jsonGeneralNames
alternate_name = SubRecord({
    "dns_names": ListOf(String()),
    "email_addresses": ListOf(String()),
    "ip_addresses": ListOf(String()),
    "directory_names": ListOf(distinguished_name),
    "edi_party_names": ListOf(SubRecord({
        "name_assigner": AnalyzedString(es_include_raw=True),
        "party_name": AnalyzedString(es_include_raw=True),
    })),
    "other_names": ListOf(SubRecord({
        "id": String(),
        "value": Binary(),
    })),
    "registered_ids": ListOf(String()),
    "uniform_resource_identifiers": ListOf(AnalyzedString(es_include_raw=True)),
})

# x509/json.go (mapped from crypto.rsa)
rsa_public_key = SubRecord({
    "exponent": Long(),
    "modulus": Binary(),
    "length": Unsigned32BitInteger(doc="Bit-length of modulus."),
})

# x509/json.go (mapped from crypto.dsa)
dsa_public_key = SubRecord({
    "p": Binary(),
    "q": Binary(),
    "g": Binary(),
    "y": Binary(),
})

# x509/json.go (mapped from crypto.ecdsa)
ecdsa_public_key = SubRecord({
    "pub": Binary(),
    "b": Binary(),
    "gx": Binary(),
    "gy": Binary(),
    "n": Binary(),
    "p": Binary(),
    "x": Binary(),
    "y": Binary(),
    "curve": String(),
    "length": Unsigned16BitInteger(),
    "asn1_oid": String(),
})

# /go/src/net/ip.go: type IPNet struct
ip_net = SubRecord({
    "IP": Binary(),
    "Mask": Binary(),
})

# x509/x509.go: type GeneralSubtreeIP struct
general_subtree_ip = SubRecord({
    "Data": ip_net,
    "Max": Integer(),
    "Min": Integer(),
})

# Generated by zcrypto/x509/extended_key_usage.sh, with a manual tweak on unknown
extended_key_usage = SubRecord({
    "eap_over_ppp": Boolean(),
    "ocsp_signing": Boolean(),
    "apple_software_update_signing": Boolean(),
    "apple_crypto_test_env": Boolean(),
    "microsoft_embedded_nt_crypto": Boolean(),
    "microsoft_drm_individualization": Boolean(),
    "microsoft_key_recovery_21": Boolean(),
    "apple_crypto_production_env": Boolean(),
    "apple_crypto_tier1_qos": Boolean(),
    "client_auth": Boolean(),
    "code_signing": Boolean(),
    "apple_crypto_development_env": Boolean(),
    "apple_crypto_tier0_qos": Boolean(),
    "apple_crypto_tier2_qos": Boolean(),
    "microsoft_mobile_device_software": Boolean(),
    "microsoft_nt5_crypto": Boolean(),
    "apple_code_signing_third_party": Boolean(),
    "microsoft_timestamp_signing": Boolean(),
    "microsoft_root_list_signer": Boolean(),
    "microsoft_system_health": Boolean(),
    "time_stamping": Boolean(),
    "apple_code_signing_development": Boolean(),
    "apple_crypto_qos": Boolean(),
    "microsoft_document_signing": Boolean(),
    "microsoft_encrypted_file_system": Boolean(),
    "microsoft_whql_crypto": Boolean(),
    "netscape_server_gated_crypto": Boolean(),
    "apple_crypto_tier3_qos": Boolean(),
    "microsoft_smart_display": Boolean(),
    "microsoft_efs_recovery": Boolean(),
    "microsoft_kernel_mode_code_signing": Boolean(),
    "server_auth": Boolean(),
    "ipsec_end_system": Boolean(),
    "ipsec_user": Boolean(),
    "microsoft_qualified_subordinate": Boolean(),
    "apple_resource_signing": Boolean(),
    "microsoft_oem_whql_crypto": Boolean(),
    "microsoft_smartcard_logon": Boolean(),
    "email_protection": Boolean(),
    "microsoft_server_gated_crypto": Boolean(),
    "microsoft_ca_exchange": Boolean(),
    "dvcs": Boolean(),
    "apple_ichat_signing": Boolean(),
    "apple_ichat_encryption": Boolean(),
    "apple_code_signing": Boolean(),
    "apple_crypto_env": Boolean(),
    "microsoft_system_health_loophole": Boolean(),
    "any": Boolean(),
    "apple_crypto_maintenance_env": Boolean(),
    "ipsec_tunnel": Boolean(),
    "microsoft_lifetime_signing": Boolean(),
    "microsoft_csp_signature": Boolean(),
    "microsoft_sgc_serialized": Boolean(),
    "sbgp_cert_aa_service_auth": Boolean(),
    "eap_over_lan": Boolean(),
    "microsoft_license_server": Boolean(),
    "microsoft_enrollment_agent": Boolean(),
    "apple_system_identity": Boolean(),
    "microsoft_key_recovery_3": Boolean(),
    "microsoft_cert_trust_list_signing": Boolean(),
    "microsoft_drm": Boolean(),
    "microsoft_licenses": Boolean(),
    "unknown": ListOf(OID()),
})

# x509/extensions.go: type NoticeReference struct
notice_reference = SubRecord({
    "organization": String(),
    "notice_numbers": ListOf(Integer()),
})

# x509/extensions.go: type UserNoticeData struct
user_notice_data = SubRecord({
    "explicit_text": String(),
    "notice_reference": ListOf(notice_reference),
})

# x509/extensions.go: type CertificatePoliciesJSON struct
certificate_policies_data = SubRecord({
    "id": String(),
    "cps": String(),
    "user_notice": ListOf(user_notice_data),
})

# x509/json.go jsonCertificate (mapped from x509.Certificate)
parsed_certificate = SubRecord({
    "subject": distinguished_name,
    # TODO FIXME: Added by jb 2017/12/11
    "subject_dn": String(),
    "issuer": distinguished_name,
    # TODO FIXME: Added by jb 2017/12/11
    "issuer_dn": String(),
    "version": Unsigned32BitInteger(),
    "serial_number": String(doc="Serial number as an unsigned decimal integer. Stored as string to support >uint lengths. Negative values are allowed."),
    "validity": SubRecord({
        "start": DateTime(doc="Timestamp of when certificate is first valid. Timezone is UTC."),
        "end": DateTime(doc="Timestamp of when certificate expires. Timezone is UTC."),
        "length": Unsigned32BitInteger(),
    }),
    "signature_algorithm": SubRecord({
        "name": String(),
        "oid": String(),
    }),
    "subject_key_info": SubRecord({
        "fingerprint_sha256": Binary(),
        "key_algorithm": SubRecord({
            "name": String(doc="Name of public key type, e.g., RSA or ECDSA. More information is available the named SubRecord (e.g., rsa_public_key)."),
         }),
        "rsa_public_key": rsa_public_key,
        "dsa_public_key": dsa_public_key,
        "ecdsa_public_key": ecdsa_public_key,
    }),
    "extensions": SubRecord({
        "key_usage": SubRecord({
            "digital_signature": Boolean(),
            "certificate_sign": Boolean(),
            "crl_sign": Boolean(),
            "content_commitment": Boolean(),
            "key_encipherment": Boolean(),
            "value": Unsigned32BitInteger(),
            "data_encipherment": Boolean(),
            "key_agreement": Boolean(),
            "decipher_only": Boolean(),
            "encipher_only": Boolean(),
        }),
        "basic_constraints": SubRecord({
            "is_ca": Boolean(),
            "max_path_len": Unsigned32BitInteger(),
        }),
        "subject_alt_name": alternate_name,
        "issuer_alt_name": alternate_name,
        "crl_distribution_points": ListOf(String()),
        "authority_key_id": Binary(),  # is this actually binary?
        "subject_key_id": Binary(),
        "extended_key_usage": extended_key_usage,
        "certificate_policies": ListOf(certificate_policies_data),
        "authority_info_access": SubRecord({
            "ocsp_urls": ListOf(String()),
            "issuer_urls": ListOf(String())
        }),
        "name_constraints": SubRecord({
            "critical": Boolean(),
            "permitted_names": ListOf(String()),
            "permitted_email_addresses": ListOf(String()),
            "permitted_ip_addresses": ListOf(String()),
            "permitted_directory_names": ListOf(distinguished_name),
            "permitted_edi_party_names": ListOf(edi_party_name),
            "permitted_registered_ids": ListOf(String()),
            "excluded_names": ListOf(String()),
            "excluded_email_addresses": ListOf(String()),
            "excluded_ip_addresses": ListOf(String()),
            "excluded_directory_names": ListOf(distinguished_name),
            "excluded_edi_party_names": ListOf(edi_party_name),
            "excluded_registered_ids": ListOf(String()),
        }),
        "signed_certificate_timestamps": ListOf(SubRecord({
            "version": Unsigned32BitInteger(),
            "log_id": Binary(es_index=True),
            "timestamp": DateTime(),
            "extensions": Binary(),
            "signature": Binary()
        })),
        "ct_poison": Boolean()
    }),
    "unknown_extensions": ListOf(unknown_extension),
    "signature": SubRecord({
        "signature_algorithm": SubRecord({
            "name": String(),
            "oid": String(),
        }),
        "value": Binary(),
        # TODO FIXME: valid was commented out...? uncommented by jb 2017/12/11
        "valid": Boolean(),
        "self_signed": Boolean(),
    }),
    "fingerprint_md5": Binary(),
    "fingerprint_sha1": Binary(),
    "fingerprint_sha256": Binary(),
    "spki_subject_fingerprint": Binary(),
    "tbs_fingerprint": Binary(),
    # TODO FIXME: added by jb 2017/12/11
    "tbs_noct_fingerprint": Binary(),
    "validation_level": String(),
    "redacted": Boolean(),
    "names": ListOf(String()),
})

# ???
certificate_trust = SubRecord({
    "type": String(doc="root, intermediate, or leaf certificate"),
    "trusted_path": Boolean(doc="Does certificate chain up to browser root store"),
    "valid": Boolean(doc="is this certificate currently valid in this browser"),
    "was_valid": Boolean(doc="was this certificate ever valid in this browser")
})

lint = SubRecord({})

# ???
certificate = SubRecord({
    "raw": Binary(),
    "parsed": parsed_certificate,
    "validation": SubRecord({
        "nss": certificate_trust,
        "apple": certificate_trust,
        "microsoft": certificate_trust,
        "android": certificate_trust,
        "java": certificate_trust,
    }),
    "lint": lint
})

# ???
server_certificate_valid = SubRecord({
    "complete_chain": Boolean(doc="does server provide a chain up to a root"),
    "valid": Boolean(doc="is this certificate currently valid in this browser"),
    "error": String()
})

hex_name_value = SubRecord({
    "hex": String(),
    "name": String(),
    # FIXME: Integer size?
    "value": Integer(),
})

cipher_suite = hex_name_value

signature_and_hash_type = SubRecord({
    "signature_algorithm": String(),
    "hash_algorithm": String(),
})

hex_name_value = SubRecord({
    "hex": String(),
    "name": String(),
    # FIXME: Integer size?
    "value": Integer(),
})

cipher_suite = hex_name_value

signature_and_hash_type = SubRecord({
    "signature_algorithm": String(),
    "hash_algorithm": String(),
})

# zcrypto/tls/tls_handshake.go: ServerHandshake
tls_handshake = SubRecord({
    "client_hello": SubRecord({
        "cipher_suites": ListOf(cipher_suite),
        "compression_methods": ListOf(hex_name_value),
        "extended_master_secret": Boolean(),
        "extended_random": Binary(),
        "heartbeat": Boolean(),
        "next_protocol_negotiation": Boolean(),
        "ocsp_stapling": Boolean(),
        "random": Binary(),
        "sct_enabled": Boolean(),
        "scts": Boolean(),
        "secure_renegotiation": Boolean(),
        "signature_and_hashes": ListOf(signature_and_hash_type),
        "supported_curves": ListOf(hex_name_value),
        "supported_point_formats": ListOf(hex_name_value),
        "ticket": Boolean(),
        "version": SubRecord({
            "name": String(),
            # FIXME: Integer size?
            "value": Integer()
        }),
    }),
    "server_hello": SubRecord({
        "version": SubRecord({
            "name": String(),
            # FIXME: Integer size?
            "value": Integer()
        }),
        "random": Binary(),
        "session_id": Binary(),
        "cipher_suite": cipher_suite,
        # FIXME: Integer size?
        "compression_method": Integer(),
        "ocsp_stapling": Boolean(),
        "ticket": Boolean(),
        "secure_renegotiation": Boolean(),
        "heartbeat": Boolean(),
        "extended_random": Binary(),
        "extended_master_secret": Boolean(),
        "scts": ListOf(SubRecord({
                "parsed": SubRecord({
                    "version": Unsigned16BitInteger(),
                    "log_id": IndexedBinary(),
                    "timestamp": Signed64BitInteger(),
                    "signature": Binary(),
                 }),
                "raw": Binary()
            })),
    }),
    "server_certificates": SubRecord({
        "certificate": certificate,
        "chain": ListOf(certificate),
        "validation": SubRecord({
            "matches_domain": Boolean(),
            "stores": SubRecord({
                "nss": server_certificate_valid,
                "microsoft": server_certificate_valid,
                "apple": server_certificate_valid,
                "java": server_certificate_valid,
                "android": server_certificate_valid,
            }),
            # TODO FIXME: ?? are the above applicable in zgrab2? I see the following    # TODO FIXME: Added by jb 2017/12/11
            # TODO FIXME: Added by jb 2017/12/11
            "browser_trusted": Boolean(),
            "browser_error": String()
        }),
    }),
    "server_key_exchange": SubRecord({
        "ecdh_params": SubRecord({
            "curve_id": SubRecord({
                "name": String(),
                # FIXME: Integer size (also -- not an OBJECT IDENTIFIER?)
                "id": Integer(),
            }),
            "server_public": SubRecord({
                "x": SubRecord({
                    "value": Binary(),
                    # FIXME: Integer size
                    "length": Integer(),
                }),
                "y": SubRecord({
                    "value": Binary(),
                    # FIXME: Integer size
                    "length": Integer(),
                }),
            }),
        }),
        "rsa_params": SubRecord({
            "exponent": Long(),
            "modulus": Binary(),
            # FIXME: Integer size
            "length": Integer(),
        }),
        "dh_params": SubRecord({
            "prime": SubRecord({
                "value": Binary(),
                # FIXME: Integer size
                "length": Integer(),
            }),
            "generator": SubRecord({
                "value": Binary(),
                # FIXME: Integer size
                "length": Integer(),
            }),
            "server_public": SubRecord({
                "value": Binary(),
                # FIXME: Integer size
                "length": Integer(),
            }),
        }),
        "signature": SubRecord({
            "raw": Binary(),
            "type": String(),
            "valid": Boolean(),
            "signature_and_hash_type": signature_and_hash_type,
            "tls_version": SubRecord({
                "name": String(),
                # FIXME: Integer size
                "value": Integer()
            }),
        }),
        "signature_error": String(),
    }),
    "server_finished": SubRecord({
        "verify_data": Binary()
    }),
    "session_ticket": SubRecord({
        "value": Binary(),
        # FIXME: Integer size
        "length": Integer(),
        "lifetime_hint": Long()
    }),
    "key_material": SubRecord({
        "pre_master_secret": SubRecord({
            "value": Binary(),
            # FIXME: Integer size
            "length": Integer()
        }),
        "master_secret": SubRecord({
            "value": Binary(),
            # FIXME: Integer size
            "length": Integer()
        }),
    }),
    "client_finished": SubRecord({
        "verify_data": Binary()
    }),
    "client_key_exchange": SubRecord({
        "dh_params": SubRecord({
            "prime": SubRecord({
                "value": Binary(),
                # FIXME: Integer size
                "length": Integer()
            }),
            "generator": SubRecord({
                "value": Binary(),
                # FIXME: Integer size
                "length": Integer()
            }),
            "client_public": SubRecord({
                # FIXME: Integer size
                "value": Binary(),
                "length": Integer()
            }),
            "client_private": SubRecord({
                # FIXME: Integer size
                "value": Binary(),
                "length": Integer()
            }),
        }),
        "ecdh_params": SubRecord({
            "curve_id": SubRecord({
                "name": String(),
                # FIXME: Integer size (and...not an OBJECT IDENTIFIER?)
                "id": Integer()
            }),
            "client_public": SubRecord({
                "x": SubRecord({
                    "value": Binary(),
                    # FIXME: Integer size
                    "length": Integer()
                }),
                "y": SubRecord({
                    "value": Binary(),
                    # FIXME: Integer size
                    "length": Integer()
                }),
            }),
            "client_private": SubRecord({
                "value": Binary(),
                # FIXME: Integer size
                "length": Integer()
            }),
        }),
        "rsa_params": SubRecord({
            # FIXME: Integer size
            "length": Integer(),
            "encrypted_pre_master_secret": Binary()
        }),
    }),
})

# zcrypto/tls/tls_heartbeat.go: Heartbleed
heartbleed_log = SubRecord({
    "heartbleed_enabled": Boolean(),
    "heartbleed_vulnerable": Boolean()
})

# zcrypto/x509/chain.go: type CertificateChain []*Certificate
certificate_chain = ListOf(parsed_certificate)

# zcrypto/tls/common.go: ConnectionState (note: no `json` tags)
tls_connection_state = SubRecord({
    "Version": Unsigned16BitInteger(),
    "HandshakeComplete": Boolean(),
    "DidResume": Boolean(),
    "CipherSuite": Unsigned16BitInteger(),
    "NegotiatedProtocol": String(),
    "NegotiatedProtocolIsMutual": Boolean(),
    "ServerName": String(),
    "PeerCertificate": parsed_certificate,
    "VerifiedChains": ListOf(certificate_chain),
})