From c02ab8f170bddaace25002e5561ca3dc67f1a1f8 Mon Sep 17 00:00:00 2001
From: Joe Testa <jtesta@positronsecurity.com>
Date: Tue, 21 Mar 2023 11:28:52 -0400
Subject: [PATCH] Added --accept option to automatically update failed tests.

---
 docker_test.sh | 96 ++++++++++++++++++++++++++++++++++++++++++--------
 1 file changed, 82 insertions(+), 14 deletions(-)

diff --git a/docker_test.sh b/docker_test.sh
index dfb33d3..e2de550 100755
--- a/docker_test.sh
+++ b/docker_test.sh
@@ -4,6 +4,11 @@
 # This script will set up a docker image with multiple versions of OpenSSH, then
 # use it to run tests.
 #
+# Optional arguments:
+#   --accept: accepts test failures and overwrites expected results with actual results (useful for updating the tests themselves).
+#   --create: attempts to create a new docker image.
+#
+#
 # For debugging purposes, here is a cheat sheet for manually running the docker image:
 #
 # docker run -p 2222:22 -it ssh-audit-test:X /bin/bash
@@ -26,8 +31,9 @@ CLR="\033[0m"
 RED="\033[0;31m"
 YELLOW="\033[0;33m"
 GREEN="\033[0;32m"
-REDB="\033[1;31m"   # Red + bold
-GREENB="\033[1;32m" # Green + bold
+REDB="\033[1;31m"    # Red + bold
+YELLOWB="\033[1;33m" # Yellow + bold
+GREENB="\033[1;32m"  # Green + bold
 
 # Program return values.
 PROGRAM_RETVAL_FAILURE=3
@@ -39,6 +45,9 @@ PROGRAM_RETVAL_GOOD=0
 # Counts the number of test failures.
 num_failures=0
 
+# When set, if a failure is encountered, overwrite the expected output with the actual value (i.e.: the user validated the failures already and wants to update the tests themselves).
+accept=0
+
 
 # Returns 0 if current docker image exists.
 check_if_docker_image_exists() {
@@ -453,7 +462,12 @@ run_test() {
     actual_retval=$?
     if [[ $actual_retval != "$expected_retval" ]]; then
         echo -e "${REDB}Unexpected return value.  Expected: ${expected_retval}; Actual: ${actual_retval}${CLR}"
-	cat $test_result_stdout
+
+        if [[ $accept == 1 ]]; then
+            echo -e "\n${REDB}This failure cannot be automatically fixed; this script must be manually updated with the new expected return value.${CLR}"
+        fi
+
+        cat ${test_result_stdout}
         docker container stop -t 0 $cid > /dev/null
         exit 1
     fi
@@ -462,7 +476,12 @@ run_test() {
     actual_retval=$?
     if [[ $actual_retval != "$expected_retval" ]]; then
         echo -e "${REDB}Unexpected return value.  Expected: ${expected_retval}; Actual: ${actual_retval}${CLR}"
-	cat $test_result_json
+
+        if [[ $accept == 1 ]]; then
+            echo -e "\n${REDB}This failure cannot be automatically fixed; this script must be manually updated with the new expected return value.${CLR}"
+	fi
+
+        cat ${test_result_json}
         docker container stop -t 0 $cid > /dev/null
         exit 1
     fi
@@ -485,16 +504,32 @@ run_test() {
 
     diff=$(diff -u "${expected_result_stdout}" "${test_result_stdout}")
     if [[ $? != 0 ]]; then
-        echo -e "${test_name} ${REDB}FAILED${CLR}.\n\n${diff}\n"
-        failed=1
-        num_failures=$((num_failures+1))
+
+        # If the user wants to update the tests, then overwrite the expected results with the actual results.
+        if [[ $accept == 1 ]]; then
+            cp "${test_result_stdout}" "${expected_result_stdout}"
+            echo -e "${test_name} ${YELLOWB}UPDATED${CLR}\n"
+        else
+            echo -e "${test_name} ${REDB}FAILED${CLR}.\n\n${diff}\n"
+            failed=1
+            num_failures=$((num_failures+1))
+        fi
+
     fi
 
     diff=$(diff -u "${expected_result_json}" "${test_result_json}")
     if [[ $? != 0 ]]; then
-        echo -e "${test_name} ${REDB}FAILED${CLR}.\n\n${diff}\n"
-        failed=1
-        num_failures=$((num_failures+1))
+
+        # If the user wants to update the tests, then overwrite the expected results with the actual results.
+        if [[ $accept == 1 ]]; then
+            cp "${test_result_json}" "${expected_result_json}"
+            echo -e "${test_name} ${YELLOWB}UPDATED${CLR}\n"
+        else
+            echo -e "${test_name} ${REDB}FAILED${CLR}.\n\n${diff}\n"
+            failed=1
+            num_failures=$((num_failures+1))
+        fi
+
     fi
 
     if [[ $failed == 0 ]]; then
@@ -571,6 +606,11 @@ run_policy_test() {
     actual_exit_code=$?
     if [[ ${actual_exit_code} != "${expected_exit_code}" ]]; then
         echo -e "${test_name} ${REDB}FAILED${CLR} (expected exit code: ${expected_exit_code}; actual exit code: ${actual_exit_code}\n"
+
+        if [[ $accept == 1 ]]; then
+            echo -e "\n${REDB}This failure cannot be automatically fixed; this script must be manually updated with the new expected return value.${CLR}"
+        fi
+
         cat "${test_result_stdout}"
         docker container stop -t 0 $cid > /dev/null
         exit 1
@@ -581,6 +621,11 @@ run_policy_test() {
     actual_exit_code=$?
     if [[ ${actual_exit_code} != "${expected_exit_code}" ]]; then
         echo -e "${test_name} ${REDB}FAILED${CLR} (expected exit code: ${expected_exit_code}; actual exit code: ${actual_exit_code}\n"
+
+        if [[ $accept == 1 ]]; then
+            echo -e "\n${REDB}This failure cannot be automatically fixed; this script must be manually updated with the new expected return value.${CLR}"
+        fi
+
         cat "${test_result_json}"
         docker container stop -t 0 $cid > /dev/null
         exit 1
@@ -594,14 +639,30 @@ run_policy_test() {
 
     diff=$(diff -u "${expected_result_stdout}" "${test_result_stdout}")
     if [[ $? != 0 ]]; then
-        echo -e "${test_name} ${REDB}FAILED${CLR}.\n\n${diff}\n"
-        exit 1
+
+	# If the user wants to update the tests, then overwrite the expected results with the actual results.
+        if [[ $accept == 1 ]]; then
+            cp "${test_result_stdout}" "${expected_result_stdout}"
+            echo -e "${test_name} ${YELLOWB}UPDATED${CLR}\n"
+        else
+            echo -e "${test_name} ${REDB}FAILED${CLR}.\n\n${diff}\n"
+            exit 1
+        fi
+
     fi
 
     diff=$(diff -u "${expected_result_json}" "${test_result_json}")
     if [[ $? != 0 ]]; then
-        echo -e "${test_name} ${REDB}FAILED${CLR}.\n\n${diff}\n"
-        exit 1
+
+	# If the user wants to update the tests, then overwrite the expected results with the actual results.
+        if [[ $accept == 1 ]]; then
+            cp "${test_result_json}" "${expected_result_json}"
+            echo -e "${test_name} ${YELLOWB}UPDATED${CLR}\n"
+        else
+            echo -e "${test_name} ${REDB}FAILED${CLR}.\n\n${diff}\n"
+            exit 1
+        fi
+
     fi
 
     echo -e "${test_name} ${GREEN}passed${CLR}."
@@ -639,6 +700,13 @@ if [[ ($# == 1) && ($1 == "--create") ]]; then
 fi
 
 
+# If the user passes --accept, then the actual results will replace the expected results (meaning the user wants to update the tests themselves due to new functionality).
+if [[ ($# == 1) && ($1 == "--accept") ]]; then
+    accept=1
+    echo -e "\n${YELLOWB}Expected test results will be replaced with actual results.${CLR}"
+fi
+
+
 # If we weren't explicitly told to create a new image, and it doesn't exist, then pull it from Dockerhub.
 if [[ $docker_image_exists == 0 ]]; then
     echo -e "\nPulling docker image $IMAGE_NAME:$IMAGE_VERSION..."