Added tests and other cleanups resulting from merging PR #252.

README.md

@@ -190,6 +190,7 @@ For convenience, a web front-end on top of the command-line tool is available at
  - The built-in man page (`-m`, `--manual`) is now available on Docker, PyPI, and Snap builds, in addition to the Windows build.
  - Snap builds are now architecture-independent.
  - Changed Docker base image from `python:3-slim` to `python:3-alpine`, resulting in a 59% reduction in image size; credit [Daniel Thamdrup](https://github.com/dallemon).
+ - Custom policies now support the `allow_algorithm_subset_and_reordering` directive to allow targets to pass with a subset and/or re-ordered list of host keys, kex, ciphers, and MACs.  This allows for the creation of a baseline policy where targets can optionally implement stricter controls; partial credit [yannik1015](https://github.com/yannik1015).
  - Added 1 new key exchange algorithm: `gss-nistp384-sha384-*`.
 
 ### v3.1.0 (2023-12-20)

docker_test.sh

@@ -784,6 +784,12 @@ run_custom_policy_test "config2" "test13" "${PROGRAM_RETVAL_GOOD}"
 # Failing test with DH modulus test.
 run_custom_policy_test "config2" "test14" "${PROGRAM_RETVAL_FAILURE}"
 
+# Passing test with algorithm subset matching.
+run_custom_policy_test "config2" "test15" "${PROGRAM_RETVAL_GOOD}"
+
+# Failing test with algorithm subset matching.
+run_custom_policy_test "config2" "test16" "${PROGRAM_RETVAL_FAILURE}"
+
 # Failing test for built-in OpenSSH 8.0p1 server policy (RSA host key size is 3072 instead of 4096).
 run_builtin_policy_test "Hardened OpenSSH Server v8.0 (version 4)" "8.0p1" "test1" "-o HostKeyAlgorithms=rsa-sha2-512,rsa-sha2-256,ssh-ed25519 -o KexAlgorithms=curve25519-sha256,curve25519-sha256@libssh.org,diffie-hellman-group16-sha512,diffie-hellman-group18-sha512,diffie-hellman-group-exchange-sha256 -o Ciphers=chacha20-poly1305@openssh.com,aes256-gcm@openssh.com,aes128-gcm@openssh.com,aes256-ctr,aes192-ctr,aes128-ctr -o MACs=hmac-sha2-256-etm@openssh.com,hmac-sha2-512-etm@openssh.com,umac-128-etm@openssh.com" "${PROGRAM_RETVAL_FAILURE}"
 

src/ssh_audit/policy.py

@@ -55,6 +55,7 @@ class Policy:
         self._dh_modulus_sizes: Optional[Dict[str, int]] = None
         self._server_policy = True
         self._allow_algorithm_subset_and_reordering = False
+        self._errors: List[Any] = []
 
         self._name_and_version: str = ''
 
@@ -217,13 +218,12 @@ class Policy:
         self._name_and_version = "%s (version %s)" % (self._name, self._version)
 
 
-    @staticmethod
+    def _append_error(self, mismatched_field: str, expected_required: Optional[List[str]], expected_optional: Optional[List[str]], actual: List[str]) -> None:
-    def _append_error(errors: List[Any], mismatched_field: str, expected_required: Optional[List[str]], expected_optional: Optional[List[str]], actual: List[str]) -> None:
         if expected_required is None:
             expected_required = ['']
         if expected_optional is None:
             expected_optional = ['']
-        errors.append({'mismatched_field': mismatched_field, 'expected_required': expected_required, 'expected_optional': expected_optional, 'actual': actual})
+        self._errors.append({'mismatched_field': mismatched_field, 'expected_required': expected_required, 'expected_optional': expected_optional, 'actual': actual})
 
 
     def _normalize_hostkey_sizes(self) -> None:
@@ -296,6 +296,9 @@ name = "Custom Policy (based on %s on %s)"
 # The version of this policy (displayed in the output during scans).  Not parsed, and may be any value, including strings.
 version = 1
 
+# When false, host keys, kex, ciphers, and MAC lists must match exactly.  When true, the target host may support a subset of the specified algorithms and/or algorithms may appear in a different order; this is useful for specifying a baseline and allowing some hosts the option to implement stricter controls.
+allow_algorithm_subset_and_reordering = false
+
 # The banner that must match exactly.  Commented out to ignore banners, since minor variability in the banner is sometimes normal.
 # banner = "%s"
 
@@ -325,39 +328,41 @@ macs = %s
         '''Evaluates a server configuration against this policy.  Returns a tuple of a boolean (True if server adheres to policy) and an array of strings that holds error messages.'''
 
         ret = True
-        errors: List[Any] = []
 
         banner_str = str(banner)
         if (self._banner is not None) and (banner_str != self._banner):
             ret = False
-            self._append_error(errors, 'Banner', [self._banner], None, [banner_str])
+            self._append_error('Banner', [self._banner], None, [banner_str])
 
         # All subsequent tests require a valid kex, so end here if we don't have one.
         if kex is None:
-            return ret, errors, self._get_error_str(errors, self._allow_algorithm_subset_and_reordering)
+            error_list, error_str = self._get_errors()
+            return ret, error_list, error_str
 
         if (self._compressions is not None) and (kex.server.compression != self._compressions):
             ret = False
-            self._append_error(errors, 'Compression', self._compressions, None, kex.server.compression)
+            self._append_error('Compression', self._compressions, None, kex.server.compression)
 
         # If a list of optional host keys was given in the policy, remove any of its entries from the list retrieved from the server.  This allows us to do an exact comparison with the expected list below.
         pruned_host_keys = kex.key_algorithms
         if self._optional_host_keys is not None:
             pruned_host_keys = [x for x in kex.key_algorithms if x not in self._optional_host_keys]
 
-        # Checking Hostkeys
+        # Check host keys.
         if self._host_keys is not None:
+            # If the policy allows subsets and re-ordered algorithms...
             if self._allow_algorithm_subset_and_reordering:
                 for hostkey_t in kex.key_algorithms:
                     if hostkey_t not in self._host_keys:
                         ret = False
-                        self._append_error(errors, 'Host keys', self._host_keys, None, kex.key_algorithms)
+                        self._append_error('Host keys', self._host_keys, self._optional_host_keys, kex.key_algorithms)
                         break
+            # The policy requires exact matching of algorithms.
             elif pruned_host_keys != self._host_keys:
                 ret = False
-                self._append_error(errors, 'Host keys', self._host_keys, None, kex.key_algorithms)
+                self._append_error('Host keys', self._host_keys, self._optional_host_keys, kex.key_algorithms)
 
-        # Checking Host Key Sizes
+        # Check host key sizes.
         if self._hostkey_sizes is not None:
             hostkey_types = list(self._hostkey_sizes.keys())
             hostkey_types.sort()  # Sorted to make testing output repeatable.
@@ -368,7 +373,7 @@ macs = %s
                     actual_hostkey_size = server_host_keys[hostkey_type]['hostkey_size']
                     if actual_hostkey_size != expected_hostkey_size:
                         ret = False
-                        self._append_error(errors, 'Host key (%s) sizes' % hostkey_type, [str(expected_hostkey_size)], None, [str(actual_hostkey_size)])
+                        self._append_error('Host key (%s) sizes' % hostkey_type, [str(expected_hostkey_size)], None, [str(actual_hostkey_size)])
 
                     # If we have expected CA signatures set, check them against what the server returned.
                     if self._hostkey_sizes is not None and len(cast(str, self._hostkey_sizes[hostkey_type]['ca_key_type'])) > 0 and cast(int, self._hostkey_sizes[hostkey_type]['ca_key_size']) > 0:
@@ -380,47 +385,59 @@ macs = %s
                         # Ensure that the CA signature type is what's expected (i.e.: the server doesn't have an RSA sig when we're expecting an ED25519 sig).
                         if actual_ca_key_type != expected_ca_key_type:
                             ret = False
-                            self._append_error(errors, 'CA signature type', [expected_ca_key_type], None, [actual_ca_key_type])
+                            self._append_error('CA signature type', [expected_ca_key_type], None, [actual_ca_key_type])
                         # Ensure that the actual and expected signature sizes match.
                         elif actual_ca_key_size != expected_ca_key_size:
                             ret = False
-                            self._append_error(errors, 'CA signature size (%s)' % actual_ca_key_type, [str(expected_ca_key_size)], None, [str(actual_ca_key_size)])
+                            self._append_error('CA signature size (%s)' % actual_ca_key_type, [str(expected_ca_key_size)], None, [str(actual_ca_key_size)])
 
-        # Checking KEX
+        # Check key exchanges.
         if self._kex is not None:
+            # If the policy allows subsets and re-ordered algorithms...
             if self._allow_algorithm_subset_and_reordering:
                 for kex_t in kex.kex_algorithms:
                     if kex_t not in self._kex:
                         ret = False
-                        self._append_error(errors, 'Key exchanges', self._kex, None, kex.kex_algorithms)
+                        self._append_error('Key exchanges', self._kex, None, kex.kex_algorithms)
                         break
-            elif kex.kex_algorithms != self._kex: # Requires perfect match
+                # If kex-strict-?-v00@openssh.com is in the policy (i.e. the Terrapin vulnerability countermeasure), then it must appear in the server's list, regardless of the "allow_algorithm_subset_and_reordering" flag.
+                if ('kex-strict-s-v00@openssh.com' in self._kex and 'kex-strict-s-v00@openssh.com' not in kex.kex_algorithms) or \
+                   ('kex-strict-c-v00@openssh.com' in self._kex and 'kex-strict-c-v00@openssh.com' not in kex.kex_algorithms):
                     ret = False
-                self._append_error(errors, 'Key exchanges', self._kex, None, kex.kex_algorithms)
+                    self._append_error('Key exchanges', self._kex, None, kex.kex_algorithms)
+
+            # The policy requires exact matching of algorithms.
+            elif kex.kex_algorithms != self._kex:
+                ret = False
+                self._append_error('Key exchanges', self._kex, None, kex.kex_algorithms)
 
         # Checking Ciphers
         if self._ciphers is not None:
+            # If the policy allows subsets and re-ordered algorithms...
             if self._allow_algorithm_subset_and_reordering:
                 for cipher_t in kex.server.encryption:
                     if cipher_t not in self._ciphers:
                         ret = False
-                        self._append_error(errors, 'Ciphers', self._ciphers, None, kex.server.encryption)
+                        self._append_error('Ciphers', self._ciphers, None, kex.server.encryption)
                         break
-            elif kex.server.encryption != self._ciphers: # Requires perfect match
+            # The policy requires exact matching of algorithms.
+            elif kex.server.encryption != self._ciphers:
                 ret = False
-                self._append_error(errors, 'Ciphers', self._ciphers, None, kex.server.encryption)
+                self._append_error('Ciphers', self._ciphers, None, kex.server.encryption)
 
         # Checking MACs
         if self._macs is not None:
+            # If the policy allows subsets and re-ordered algorithms...
             if self._allow_algorithm_subset_and_reordering:
                 for mac_t in kex.server.mac:
                     if mac_t not in self._macs:
                         ret = False
-                        self._append_error(errors, 'MACs', self._macs, None, kex.server.mac)
+                        self._append_error('MACs', self._macs, None, kex.server.mac)
                         break
-            elif kex.server.mac != self._macs: # Requires perfect match
+            # The policy requires exact matching of algorithms.
+            elif kex.server.mac != self._macs:
                 ret = False
-                self._append_error(errors, 'MACs', self._macs, None, kex.server.mac)
+                self._append_error('MACs', self._macs, None, kex.server.mac)
 
         if self._dh_modulus_sizes is not None:
             dh_modulus_types = list(self._dh_modulus_sizes.keys())
@@ -431,30 +448,27 @@ macs = %s
                     actual_dh_modulus_size = kex.dh_modulus_sizes()[dh_modulus_type]
                     if expected_dh_modulus_size != actual_dh_modulus_size:
                         ret = False
-                        self._append_error(errors, 'Group exchange (%s) modulus sizes' % dh_modulus_type, [str(expected_dh_modulus_size)], None, [str(actual_dh_modulus_size)])
+                        self._append_error('Group exchange (%s) modulus sizes' % dh_modulus_type, [str(expected_dh_modulus_size)], None, [str(actual_dh_modulus_size)])
 
-        return ret, errors, self._get_error_str(errors, self._allow_algorithm_subset_and_reordering)
+        error_list, error_str = self._get_errors()
+        return ret, error_list, error_str
 
 
-    @staticmethod
+    def _get_errors(self) -> Tuple[List[Any], str]:
-    def _get_error_str(errors: List[Any], allow_algorithm_subset_and_reordering: bool = False) -> str:
+        '''Returns the list of errors, along with the string representation of those errors.'''
-        '''Transforms an error struct to a flat string of error messages.'''
-
-        if allow_algorithm_subset_and_reordering:
-            expected_str = 'allowed'
-        else:
-            expected_str = 'required'
 
+        subset_and_reordering_semicolon = "; subset and/or reordering allowed" if self._allow_algorithm_subset_and_reordering else "; exact match"
+        subset_and_reordering_parens = " (subset and/or reordering allowed)" if self._allow_algorithm_subset_and_reordering else ""
         error_list = []
         spacer = ''
-        for e in errors:
+        for e in self._errors:
             e_str = "  * %s did not match.\n" % e['mismatched_field']
 
             if ('expected_optional' in e) and (e['expected_optional'] != ['']):
-                e_str += "    - Expected (" + expected_str + "): %s\n    - Expected (optional): %s\n" % (Policy._normalize_error_field(e['expected_required']), Policy._normalize_error_field(e['expected_optional']))
+                e_str += "    - Expected (required%s): %s\n    - Expected (optional): %s\n" % (subset_and_reordering_semicolon, Policy._normalize_error_field(e['expected_required']), Policy._normalize_error_field(e['expected_optional']))
                 spacer = '              '
             else:
-                e_str += "    - Expected (" + expected_str + "): %s\n" % Policy._normalize_error_field(e['expected_required'])
+                e_str += "    - Expected%s: %s\n" % (subset_and_reordering_parens, Policy._normalize_error_field(e['expected_required']))
                 spacer = '   '
             e_str += "    - Actual:%s%s\n" % (spacer, Policy._normalize_error_field(e['actual']))
             error_list.append(e_str)
@@ -465,7 +479,7 @@ macs = %s
         if len(error_list) > 0:
             error_str = "\n".join(error_list)
 
-        return error_str
+        return self._errors, error_str
 
 
     def get_name_and_version(self) -> str:
@@ -581,4 +595,4 @@ macs = %s
         if self._dh_modulus_sizes is not None:
             dh_modulus_sizes_str = str(self._dh_modulus_sizes)
 
-        return "Name: %s\nVersion: %s\nBanner: %s\nCompressions: %s\nHost Keys: %s\nOptional Host Keys: %s\nKey Exchanges: %s\nCiphers: %s\nMACs: %s\nHost Key Sizes: %s\nDH Modulus Sizes: %s\nServer Policy: %r" % (name, version, banner, compressions_str, host_keys_str, optional_host_keys_str, kex_str, ciphers_str, macs_str, hostkey_sizes_str, dh_modulus_sizes_str, self._server_policy)
+        return "Name: %s\nVersion: %s\nAllow Algorithm Subset and/or Reordering: %r\nBanner: %s\nCompressions: %s\nHost Keys: %s\nOptional Host Keys: %s\nKey Exchanges: %s\nCiphers: %s\nMACs: %s\nHost Key Sizes: %s\nDH Modulus Sizes: %s\nServer Policy: %r" % (name, version, self._allow_algorithm_subset_and_reordering, banner, compressions_str, host_keys_str, optional_host_keys_str, kex_str, ciphers_str, macs_str, hostkey_sizes_str, dh_modulus_sizes_str, self._server_policy)

test/docker/expected_results/openssh_8.0p1_custom_policy_test15.json

@@ -0,0 +1,6 @@
+{
+    "errors": [],
+    "host": "localhost",
+    "passed": true,
+    "policy": "Docker policy: test15 (version 1)"
+}

test/docker/expected_results/openssh_8.0p1_custom_policy_test15.txt

@@ -0,0 +1,3 @@
+Host:   localhost:2222
+Policy: Docker policy: test15 (version 1)
+Result: ✔ Passed

test/docker/expected_results/openssh_8.0p1_custom_policy_test16.json

@@ -0,0 +1,86 @@
+{
+    "errors": [
+        {
+            "actual": [
+                "rsa-sha2-512",
+                "rsa-sha2-256",
+                "ssh-rsa",
+                "ecdsa-sha2-nistp256",
+                "ssh-ed25519"
+            ],
+            "expected_optional": [
+                ""
+            ],
+            "expected_required": [
+                "rsa-sha2-512",
+                "extra_hostkey_alg"
+            ],
+            "mismatched_field": "Host keys"
+        },
+        {
+            "actual": [
+                "curve25519-sha256",
+                "curve25519-sha256@libssh.org",
+                "ecdh-sha2-nistp256",
+                "ecdh-sha2-nistp384",
+                "ecdh-sha2-nistp521",
+                "diffie-hellman-group-exchange-sha256",
+                "diffie-hellman-group16-sha512",
+                "diffie-hellman-group18-sha512",
+                "diffie-hellman-group14-sha256",
+                "diffie-hellman-group14-sha1"
+            ],
+            "expected_optional": [
+                ""
+            ],
+            "expected_required": [
+                "curve25519-sha256",
+                "extra_kex_alg"
+            ],
+            "mismatched_field": "Key exchanges"
+        },
+        {
+            "actual": [
+                "chacha20-poly1305@openssh.com",
+                "aes128-ctr",
+                "aes192-ctr",
+                "aes256-ctr",
+                "aes128-gcm@openssh.com",
+                "aes256-gcm@openssh.com"
+            ],
+            "expected_optional": [
+                ""
+            ],
+            "expected_required": [
+                "chacha20-poly1305@openssh.com",
+                "extra_cipher_alg"
+            ],
+            "mismatched_field": "Ciphers"
+        },
+        {
+            "actual": [
+                "umac-64-etm@openssh.com",
+                "umac-128-etm@openssh.com",
+                "hmac-sha2-256-etm@openssh.com",
+                "hmac-sha2-512-etm@openssh.com",
+                "hmac-sha1-etm@openssh.com",
+                "umac-64@openssh.com",
+                "umac-128@openssh.com",
+                "hmac-sha2-256",
+                "hmac-sha2-512",
+                "hmac-sha1"
+            ],
+            "expected_optional": [
+                ""
+            ],
+            "expected_required": [
+                "umac-64-etm@openssh.com",
+                "extra_mac_alg"
+            ],
+            "mismatched_field": "MACs"
+        }
+    ],
+    "host": "localhost",
+    "passed": false,
+    "policy": "Docker policy: test16 (version 1)"
+}

test/docker/expected_results/openssh_8.0p1_custom_policy_test16.txt

@@ -0,0 +1,21 @@
+Host:   localhost:2222
+Policy: Docker policy: test16 (version 1)
+Result: ❌ Failed!
+
+Errors:
+  * Ciphers did not match.
+    - Expected (subset and/or reordering allowed): chacha20-poly1305@openssh.com, extra_cipher_alg
+    - Actual:   chacha20-poly1305@openssh.com, aes128-ctr, aes192-ctr, aes256-ctr, aes128-gcm@openssh.com, aes256-gcm@openssh.com
+
+  * Host keys did not match.
+    - Expected (subset and/or reordering allowed): rsa-sha2-512, extra_hostkey_alg
+    - Actual:   rsa-sha2-512, rsa-sha2-256, ssh-rsa, ecdsa-sha2-nistp256, ssh-ed25519
+
+  * Key exchanges did not match.
+    - Expected (subset and/or reordering allowed): curve25519-sha256, extra_kex_alg
+    - Actual:   curve25519-sha256, curve25519-sha256@libssh.org, ecdh-sha2-nistp256, ecdh-sha2-nistp384, ecdh-sha2-nistp521, diffie-hellman-group-exchange-sha256, diffie-hellman-group16-sha512, diffie-hellman-group18-sha512, diffie-hellman-group14-sha256, diffie-hellman-group14-sha1
+
+  * MACs did not match.
+    - Expected (subset and/or reordering allowed): umac-64-etm@openssh.com, extra_mac_alg
+    - Actual:   umac-64-etm@openssh.com, umac-128-etm@openssh.com, hmac-sha2-256-etm@openssh.com, hmac-sha2-512-etm@openssh.com, hmac-sha1-etm@openssh.com, umac-64@openssh.com, umac-128@openssh.com, hmac-sha2-256, hmac-sha2-512, hmac-sha1
+

test/docker/policies/policy_test15.txt

@@ -0,0 +1,13 @@
+#
+# Docker policy: test15
+#
+
+name = "Docker policy: test15"
+version = 1
+allow_algorithm_subset_and_reordering = true
+banner = "SSH-2.0-OpenSSH_8.0"
+compressions = none, zlib@openssh.com
+host keys = rsa-sha2-512, rsa-sha2-256, ssh-rsa, ecdsa-sha2-nistp256, ssh-ed25519, extra_hostkey_alg
+key exchanges = curve25519-sha256, curve25519-sha256@libssh.org, ecdh-sha2-nistp256, ecdh-sha2-nistp384, ecdh-sha2-nistp521, diffie-hellman-group-exchange-sha256, diffie-hellman-group16-sha512, diffie-hellman-group18-sha512, diffie-hellman-group14-sha256, diffie-hellman-group14-sha1, extra_kex_alg
+ciphers = chacha20-poly1305@openssh.com, aes128-ctr, aes192-ctr, aes256-ctr, aes128-gcm@openssh.com, aes256-gcm@openssh.com, extra_cipher_alg
+macs = umac-64-etm@openssh.com, umac-128-etm@openssh.com, hmac-sha2-256-etm@openssh.com, hmac-sha2-512-etm@openssh.com, hmac-sha1-etm@openssh.com, umac-64@openssh.com, umac-128@openssh.com, hmac-sha2-256, hmac-sha2-512, hmac-sha1, extra_mac_alg

test/docker/policies/policy_test16.txt

@@ -0,0 +1,13 @@
+#
+# Docker policy: test16
+#
+
+name = "Docker policy: test16"
+version = 1
+allow_algorithm_subset_and_reordering = true
+banner = "SSH-2.0-OpenSSH_8.0"
+compressions = none, zlib@openssh.com
+host keys = rsa-sha2-512, extra_hostkey_alg
+key exchanges = curve25519-sha256, extra_kex_alg
+ciphers = chacha20-poly1305@openssh.com, extra_cipher_alg
+macs = umac-64-etm@openssh.com, extra_mac_alg

test/test_policy.py

@@ -150,7 +150,7 @@ ciphers = cipher_alg1, cipher_alg2, cipher_alg3
 macs = mac_alg1, mac_alg2, mac_alg3'''
 
         policy = self.Policy(policy_data=policy_data)
-        assert str(policy) == "Name: [Test Policy]\nVersion: [1]\nBanner: {undefined}\nCompressions: comp_alg1\nHost Keys: key_alg1\nOptional Host Keys: {undefined}\nKey Exchanges: kex_alg1, kex_alg2\nCiphers: cipher_alg1, cipher_alg2, cipher_alg3\nMACs: mac_alg1, mac_alg2, mac_alg3\nHost Key Sizes: {undefined}\nDH Modulus Sizes: {undefined}\nServer Policy: True"
+        assert str(policy) == "Name: [Test Policy]\nVersion: [1]\nAllow Algorithm Subset and/or Reordering: False\nBanner: {undefined}\nCompressions: comp_alg1\nHost Keys: key_alg1\nOptional Host Keys: {undefined}\nKey Exchanges: kex_alg1, kex_alg2\nCiphers: cipher_alg1, cipher_alg2, cipher_alg3\nMACs: mac_alg1, mac_alg2, mac_alg3\nHost Key Sizes: {undefined}\nDH Modulus Sizes: {undefined}\nServer Policy: True"
 
 
     def test_policy_invalid_1(self):
@@ -297,7 +297,7 @@ macs = mac_alg1, mac_alg2, mac_alg3'''
         pol_data = pol_data.replace(date.today().strftime('%Y/%m/%d'), '[todays date]')
 
         # Instead of writing out the entire expected policy--line by line--just check that it has the expected hash.
-        assert hashlib.sha256(pol_data.encode('ascii')).hexdigest() == '4af7777fb57a1dad0cf438c899a11d4f625fd9276ea3bb5ef5c9fe8806cb47dc'
+        assert hashlib.sha256(pol_data.encode('ascii')).hexdigest() == '4b504b799f6b964a20ccbe8af7edd26c7b5f0e0b98070e754ea41dccdace33b4'
 
 
     def test_policy_evaluate_passing_1(self):
@@ -440,3 +440,96 @@ macs = mac_alg1, mac_alg2, XXXmismatchedXXX, mac_alg3'''
         assert len(errors) == 2
         assert error_str.find('Host keys did not match.') != -1
         assert error_str.find('MACs did not match.') != -1
+
+
+    def test_policy_evaluate_subset_passing_1(self):
+        '''Ensure that exact algorithm matches work even when subset parsing is enabled.'''
+
+        policy_data = '''name = "Test Policy"
+version = 1
+allow_algorithm_subset_and_reordering = true
+compressions = comp_alg1, comp_alg2
+host keys = key_alg1, key_alg2
+key exchanges = kex_alg1, kex_alg2
+ciphers = cipher_alg1, cipher_alg2, cipher_alg3
+macs = mac_alg1, mac_alg2, mac_alg3'''
+        policy = self.Policy(policy_data=policy_data)
+        ret, errors, error_str = policy.evaluate('SSH Server 1.0', self._get_kex())
+        assert ret is True
+        assert len(errors) == 0
+        assert error_str == ""
+
+
+    def test_policy_evaluate_subset_passing_2(self):
+        '''Ensure that subset parsing works.'''
+
+        policy_data = '''name = "Test Policy"
+version = 1
+allow_algorithm_subset_and_reordering = true
+compressions = comp_alg1, comp_alg2
+host keys = key_alg2, key_alg1, key_alg0
+key exchanges = kex_alg3, kex_alg1, kex_alg2
+ciphers = cipher_alg0, cipher_alg3, cipher_alg2, cipher_alg1
+macs = mac_alg2, mac_alg1, mac_alg3, mac_alg0'''
+        policy = self.Policy(policy_data=policy_data)
+        ret, errors, error_str = policy.evaluate('SSH Server 1.0', self._get_kex())
+        assert ret is True
+        assert len(errors) == 0
+        assert error_str == ""
+
+
+    def test_policy_evaluate_subset_failing_1(self):
+        '''Ensure that subset parsing returns a failure.'''
+
+        policy_data = '''name = "Test Policy"
+version = 1
+allow_algorithm_subset_and_reordering = true
+compressions = comp_alg1, comp_alg2
+host keys = key_alg7, key_alg8, key_alg9
+key exchanges = kex_alg7, kex_alg8, kex_alg9
+ciphers = cipher_alg7, cipher_alg8, cipher_alg9, cipher_alg10
+macs = mac_alg7, mac_alg8, mac_alg9, mac_alg10'''
+        policy = self.Policy(policy_data=policy_data)
+        ret, errors, error_str = policy.evaluate('SSH Server 1.0', self._get_kex())
+        assert ret is False
+        assert len(errors) == 4
+        assert error_str.find("Ciphers did not match.") != -1
+        assert error_str.find("Host keys did not match.") != -1
+        assert error_str.find("MACs did not match") != -1
+        assert error_str.find("Key exchanges did not match.") != -1
+
+
+    def test_policy_evaluate_subset_failing_2(self):
+        '''Ensure that subset parsing returns a failure when policy includes kex-strict-s-v00@openssh.com, but target does not.'''
+
+        policy_data = '''name = "Test Policy"
+version = 1
+allow_algorithm_subset_and_reordering = true
+compressions = comp_alg1, comp_alg2
+host keys = key_alg2, key_alg1, key_alg0
+key exchanges = kex_alg3, kex_alg1, kex_alg2, kex-strict-s-v00@openssh.com
+ciphers = cipher_alg0, cipher_alg3, cipher_alg2, cipher_alg1
+macs = mac_alg2, mac_alg1, mac_alg3, mac_alg0'''
+        policy = self.Policy(policy_data=policy_data)
+        ret, errors, error_str = policy.evaluate('SSH Server 1.0', self._get_kex())
+        assert ret is False
+        assert len(errors) == 1
+        assert error_str.find("Key exchanges did not match.") != -1
+
+
+    def test_policy_evaluate_subset_failing_3(self):
+        '''Ensure that subset parsing returns a failure when policy includes kex-strict-c-v00@openssh.com, but target does not.'''
+
+        policy_data = '''name = "Test Policy"
+version = 1
+allow_algorithm_subset_and_reordering = true
+compressions = comp_alg1, comp_alg2
+host keys = key_alg2, key_alg1, key_alg0
+key exchanges = kex_alg3, kex_alg1, kex_alg2, kex-strict-c-v00@openssh.com
+ciphers = cipher_alg0, cipher_alg3, cipher_alg2, cipher_alg1
+macs = mac_alg2, mac_alg1, mac_alg3, mac_alg0'''
+        policy = self.Policy(policy_data=policy_data)
+        ret, errors, error_str = policy.evaluate('SSH Server 1.0', self._get_kex())
+        assert ret is False
+        assert len(errors) == 1
+        assert error_str.find("Key exchanges did not match.") != -1