refactor(makefile): consolidate the make targets used for bats tests (#1746)

New examples of running tests:

1. To run a specific bats file (with and without verbose output):
make run-blackbox-tests BATS_TEST_FILE_PATH=test/blackbox/delete_images.bats
make run-blackbox-tests BATS_TEST_FILE_PATH=test/blackbox/delete_images.bats BATS_VERBOSITY=2

2. To run the CI tests (with and without verbose output)
make run-blackbox-ci
make run-blackbox-ci BATS_VERBOSITY=2

BATS_TEST_FILE_PATH is used to pass on the test file to run using `run-blackbox-tests`
BATS_VERBOSITY controls the verbosity of the bats framework output, if unspecified the output only
contains test results and failure message in case of failures.
If BATS_VERBOSITY is 1, then also show commands as they are executed.
If BATS_VERBOSITY is 2, on top of the above it also shows output of passed tests.

Other changes in this PR:
- Update some of the tests to show logs after the run ends.
- Run the linters before the tests, as it saves time on failures when running in GH

Signed-off-by: Andrei Aaron <aaaron@luxoft.com>

test/blackbox/annotations.bats

@@ -1,4 +1,4 @@
-# Note: Intended to be run as "make test-annotations"
+# Note: Intended to be run as "make run-blackbox-tests" or "make run-blackbox-ci"
 #       Makefile target installs & checks all necessary tooling
 #       Extra tools that are not covered in Makefile target needs to be added in verify_prerequisites()
 
@@ -79,8 +79,12 @@ EOF
     wait_zot_reachable 8080
 }
 
+function teardown() {
+    # conditionally printing on failure is possible from teardown but not from from teardown_file
+    cat ${BATS_FILE_TMPDIR}/zot.log
+}
+
 function teardown_file() {
-    cat ${BATS_FILE_TMPDIR}/zot.log >&3
     zot_stop_all
     run rm -rf ${HOME}/.config/notation
 }

test/blackbox/anonymous_policy.bats

@@ -1,4 +1,4 @@
-# Note: Intended to be run as "make test-anonymous-push-pull"
+# Note: Intended to be run as "make run-blackbox-tests" or "make run-blackbox-ci"
 #       Makefile target installs & checks all necessary tooling
 #       Extra tools that are not covered in Makefile target needs to be added in verify_prerequisites()
 
@@ -59,7 +59,8 @@ function setup_file() {
         }
     },
     "log": {
-        "level": "debug"
+        "level": "debug",
+        "output": "${BATS_FILE_TMPDIR}/zot.log"
     }
 }
 EOF
@@ -67,6 +68,11 @@ EOF
     wait_zot_reachable 8080
 }
 
+function teardown() {
+    # conditionally printing on failure is possible from teardown but not from from teardown_file
+    cat ${BATS_FILE_TMPDIR}/zot.log
+}
+
 function teardown_file() {
     zot_stop_all
 }

test/blackbox/cloud_only.bats

@@ -1,3 +1,7 @@
+# Note: Intended to be run as "make run-blackbox-tests" or "make run-blackbox-cloud-ci"
+#       Makefile target installs & checks all necessary tooling
+#       Extra tools that are not covered in Makefile target needs to be added in verify_prerequisites()
+
 load helpers_cloud
 load helpers_wait
 
@@ -17,8 +21,8 @@ function setup() {
 
     cat > ${zot_config_file}<<EOF
 {
-	"distSpecVersion": "1.1.0-dev",
-	"storage": {
+    "distSpecVersion": "1.1.0-dev",
+    "storage": {
         "rootDirectory": "${zot_root_dir}",
         "dedupe": true,
         "remoteCache": true,
@@ -43,10 +47,10 @@ function setup() {
             "apiKeyTablename":"ApiKeyTable",
             "versionTablename": "Version"
         }
-	},
-	"http": {
-		"address": "127.0.0.1",
-		"port": "8080",
+    },
+    "http": {
+        "address": "127.0.0.1",
+        "port": "8080",
         "realm": "zot",
         "auth": {
             "openid": {
@@ -68,25 +72,25 @@ function setup() {
                 }
             }
         }
-	},
-	"log": {
-		"level": "debug"
-	},
-	"extensions": {
-		"metrics": {
+    },
+    "log": {
+        "level": "debug"
+    },
+    "extensions": {
+        "metrics": {
             "enable": true,
             "prometheus": {
                 "path": "/metrics"
             }
         },
-		"search": {
+        "search": {
             "enable": true
-		},
-		"scrub": {
-			"enable": true,
-			"interval": "24h"
-		}
-	}
+        },
+        "scrub": {
+            "enable": true,
+            "interval": "24h"
+        }
+    }
 }
 EOF
     awslocal s3 --region "us-east-2" mb s3://zot-storage

test/blackbox/cve.bats

@@ -1,4 +1,4 @@
-# Note: Intended to be run as "make test-bats-cve" or "make test-bats-cve-verbose"
+# Note: Intended to be run as "make run-blackbox-tests" or "make run-blackbox-ci"
 #       Makefile target installs & checks all necessary tooling
 #       Extra tools that are not covered in Makefile target needs to be added in verify_prerequisites()
 
@@ -42,7 +42,8 @@ function setup_file() {
         "port": "8080"
     },
     "log": {
-        "level": "debug"
+        "level": "debug",
+        "output": "${BATS_FILE_TMPDIR}/zot.log"
     },
     "extensions": {
         "search": {
@@ -63,6 +64,11 @@ EOF
     zli_add_config ${registry_name} ${registry_url}
 }
 
+function teardown() {
+    # conditionally printing on failure is possible from teardown but not from from teardown_file
+    cat ${BATS_FILE_TMPDIR}/zot.log
+}
+
 function teardown_file() {
     zot_stop_all
 }

test/blackbox/delete_images.bats

@@ -1,3 +1,7 @@
+# Note: Intended to be run as "make run-blackbox-tests" or "make run-blackbox-ci"
+#       Makefile target installs & checks all necessary tooling
+#       Extra tools that are not covered in Makefile target needs to be added in verify_prerequisites()
+
 load helpers_zot
 
 function verify_prerequisites {
@@ -14,7 +18,7 @@ function verify_prerequisites {
 
 function setup_file() {
     # Verify prerequisites are available
-    if ! (verify_prerequisites); then
+    if ! verify_prerequisites; then
         exit 1
     fi
 
@@ -22,10 +26,10 @@ function setup_file() {
     skopeo --insecure-policy copy --format=oci docker://ghcr.io/project-zot/test-images/alpine:3.17.3 oci:${TEST_DATA_DIR}/alpine:3.17.3
 
     # Setup zot server
-    ZOT_ROOT_DIR=${BATS_RUN_TMPDIR}/zot
+    ZOT_ROOT_DIR=${BATS_FILE_TMPDIR}/zot
     echo ${ZOT_ROOT_DIR}
-    local zot_log_file=${BATS_RUN_TMPDIR}/zot-log.json
-    local zot_config_file=${BATS_RUN_TMPDIR}/zot_config.json
+    local zot_log_file=${BATS_FILE_TMPDIR}/zot-log.json
+    local zot_config_file=${BATS_FILE_TMPDIR}/zot_config.json
     mkdir -p ${ZOT_ROOT_DIR}
     touch ${zot_log_file}
     cat >${zot_config_file} <<EOF
@@ -65,6 +69,11 @@ EOF
     echo ${MANIFEST_DIGEST}
 }
 
+function teardown() {
+    # conditionally printing on failure is possible from teardown but not from from teardown_file
+    cat ${BATS_FILE_TMPDIR}/zot-log.json
+}
+
 function teardown_file() {
     zot_stop_all
 }

test/blackbox/detect_manifest_collision.bats

@@ -1,4 +1,4 @@
-# Note: Intended to be run as "make test-detect-manifest-collision"
+# Note: Intended to be run as "make run-blackbox-tests" or "make run-blackbox-ci"
 #       Makefile target installs & checks all necessary tooling
 #       Extra tools that are not covered in Makefile target needs to be added in verify_prerequisites()
 
@@ -64,7 +64,8 @@ function setup_file() {
         }
     },
     "log": {
-        "level": "debug"
+        "level": "debug",
+        "output": "${BATS_FILE_TMPDIR}/zot.log"
     }
 }
 EOF
@@ -72,6 +73,11 @@ EOF
     wait_zot_reachable 8080
 }
 
+function teardown() {
+    # conditionally printing on failure is possible from teardown but not from from teardown_file
+    cat ${BATS_FILE_TMPDIR}/zot.log
+}
+
 function teardown_file() {
     zot_stop_all
 }

test/blackbox/garbage_collect.bats

@@ -45,7 +45,7 @@ function setup_file() {
     },
     "log": {
         "level": "debug",
-        "output": "/tmp/gc.log"
+        "output": "${BATS_FILE_TMPDIR}/gc.log"
     }
 }
 EOF
@@ -53,6 +53,11 @@ EOF
     wait_zot_reachable 8080
 }
 
+function teardown() {
+    # conditionally printing on failure is possible from teardown but not from from teardown_file
+    cat ${BATS_FILE_TMPDIR}/gc.log
+}
+
 function teardown_file() {
     zot_stop_all
 }

test/blackbox/metadata.bats

@@ -1,4 +1,4 @@
-# Note: Intended to be run as "make test-bats-metadata"
+# Note: Intended to be run as "make run-blackbox-tests" or "make run-blackbox-ci"
 #       Makefile target installs & checks all necessary tooling
 #       Extra tools that are not covered in Makefile target needs to be added in verify_prerequisites()
 
@@ -77,7 +77,8 @@ function setup_file() {
         }
     },
     "log": {
-        "level": "debug"
+        "level": "debug",
+        "output": "${BATS_FILE_TMPDIR}/zot.log"
     }
 }
 EOF
@@ -86,11 +87,15 @@ EOF
     wait_zot_reachable 8080
 }
 
+function teardown() {
+    # conditionally printing on failure is possible from teardown but not from from teardown_file
+    cat ${BATS_FILE_TMPDIR}/zot.log
+}
+
 function teardown_file() {
     zot_stop_all
 }
 
-
 @test "push image user policy" {
     run skopeo --insecure-policy copy --dest-creds test:test --dest-tls-verify=false \
         oci:${TEST_DATA_DIR}/golang:1.18 \

test/blackbox/metrics.bats

@@ -1,4 +1,4 @@
-# Note: Intended to be run as "make test-bats-metrics" or "make test-bats-metrics-verbose"
+# Note: Intended to be run as "make run-blackbox-tests" or "make run-blackbox-ci"
 #       Makefile target installs & checks all necessary tooling
 #       Extra tools that are not covered in Makefile target needs to be added in verify_prerequisites()
 
@@ -59,6 +59,11 @@ EOF
 
 }
 
+function teardown() {
+    # conditionally printing on failure is possible from teardown but not from from teardown_file
+    cat ${BATS_FILE_TMPDIR}/zot/zot-log.json
+}
+
 function teardown_file() {
     zot_stop_all
 }

test/blackbox/pushpull.bats

@@ -1,4 +1,4 @@
-# Note: Intended to be run as "make test-push-pull" or "test-push-pull-verbose"
+# Note: Intended to be run as "make run-blackbox-tests" or "make run-blackbox-ci"
 #       Makefile target installs & checks all necessary tooling
 #       Extra tools that are not covered in Makefile target needs to be added in verify_prerequisites()
 
@@ -42,7 +42,8 @@ function setup_file() {
         "port": "8080"
     },
     "log": {
-        "level": "debug"
+        "level": "debug",
+        "output": "${BATS_FILE_TMPDIR}/zot.log"
     }
 }
 EOF
@@ -51,6 +52,11 @@ EOF
     wait_zot_reachable 8080
 }
 
+function teardown() {
+    # conditionally printing on failure is possible from teardown but not from from teardown_file
+    cat ${BATS_FILE_TMPDIR}/zot.log
+}
+
 function teardown_file() {
     zot_stop_all
 }

test/blackbox/pushpull_authn.bats

@@ -69,7 +69,8 @@ function setup_file() {
     }
   },
   "log":{
-    "level":"debug"
+    "level":"debug",
+    "output": "${BATS_FILE_TMPDIR}/zot.log"
   }
 }
 EOF
@@ -77,6 +78,11 @@ EOF
     wait_zot_reachable 8080
 }
 
+function teardown() {
+    # conditionally printing on failure is possible from teardown but not from from teardown_file
+    cat ${BATS_FILE_TMPDIR}/zot.log
+}
+
 function teardown_file() {
     zot_stop_all
 }

test/blackbox/pushpull_running_dedupe.bats

@@ -1,4 +1,4 @@
-# Note: Intended to be run as "make test-push-pull-running-dedupe" or "test-push-pull-running-dedupe-verbose"
+# Note: Intended to be run as "make run-blackbox-tests" or "make run-blackbox-dedupe-nightly"
 #       Makefile target installs & checks all necessary tooling
 #       Extra tools that are not covered in Makefile target needs to be added in verify_prerequisites()
 
@@ -45,7 +45,8 @@ function setup_file() {
         "port": "8080"
     },
     "log": {
-        "level": "debug"
+        "level": "debug",
+        "output": "${BATS_FILE_TMPDIR}/zot.log"
     }
 }
 EOF
@@ -54,6 +55,11 @@ EOF
     wait_zot_reachable 8080
 }
 
+function teardown() {
+    # conditionally printing on failure is possible from teardown but not from from teardown_file
+    cat ${BATS_FILE_TMPDIR}/zot.log
+}
+
 function teardown_file() {
     zot_stop_all
 }

test/blackbox/referrers.bats

@@ -1,4 +1,4 @@
-# Note: Intended to be run as "make test-bats-referrers"
+# Note: Intended to be run as "make run-blackbox-ci"
 #       Makefile target installs & checks all necessary tooling
 #       Extra tools that are not covered in Makefile target needs to be added in verify_prerequisites()
 
@@ -87,6 +87,11 @@ EOF
     curl -X GET http://127.0.0.1:8080/v2/golang/referrers/${MANIFEST_DIGEST}?artifactType=image.artifact/type
 }
 
+function teardown() {
+    # conditionally printing on failure is possible from teardown but not from from teardown_file
+    cat ${BATS_FILE_TMPDIR}/zot/zot-log.json
+}
+
 function teardown_file() {
     zot_stop_all
 }

test/blackbox/restore_s3_blobs.bats

@@ -1,3 +1,7 @@
+# Note: Intended to be run as "make run-blackbox-tests" or "make run-blackbox-dedupe-nightly"
+#       Makefile target installs & checks all necessary tooling
+#       Extra tools that are not covered in Makefile target needs to be added in verify_prerequisites()
+
 load helpers_cloud
 load helpers_wait
 
@@ -81,6 +85,11 @@ EOF
     wait_zot_reachable 8080
 }
 
+function teardown() {
+    # conditionally printing on failure is possible from teardown but not from from teardown_file
+    cat ${BATS_FILE_TMPDIR}/zot/zot-log.json
+}
+
 function teardown_file() {
     local zot_root_dir=${BATS_FILE_TMPDIR}/zot
     zot_stop

test/blackbox/scrub.bats

@@ -1,4 +1,4 @@
-# Note: Intended to be run as "make test-bats-scrub" or "make test-bats-scrub-verbose"
+# Note: Intended to be run as "make run-blackbox-tests" or "make run-blackbox-ci"
 #       Makefile target installs & checks all necessary tooling
 #       Extra tools that are not covered in Makefile target needs to be added in verify_prerequisites()
 
@@ -53,11 +53,10 @@ EOF
 }
 
 function teardown() {
+    cat ${BATS_FILE_TMPDIR}/zot/zot-log.json
     zot_stop_all
 }
 
-
-
 @test "blobs/manifest integrity not affected" {
     add_test_files
     echo ${ZOT_CONFIG_FILE}

test/blackbox/sync.bats

@@ -1,4 +1,4 @@
-# Note: Intended to be run as "make test-bats-sync" or "make test-bats-sync-verbose"
+# Note: Intended to be run as "make run-blackbox-tests" or "make run-blackbox-ci"
 #       Makefile target installs & checks all necessary tooling
 #       Extra tools that are not covered in Makefile target needs to be added in verify_prerequisites()
 

test/blackbox/sync_docker.bats

@@ -1,4 +1,4 @@
-# Note: Intended to be run as "make test-bats-sync" or "make test-bats-sync-verbose"
+# Note: Intended to be run as "make run-blackbox-tests" or "make run-blackbox-ci"
 #       Makefile target installs & checks all necessary tooling
 #       Extra tools that are not covered in Makefile target needs to be added in verify_prerequisites()
 

test/blackbox/sync_harness.bats

@@ -1,4 +1,4 @@
-# Note: Intended to be run as "make test-sync-harness" or "make test-sync-harness-verbose"
+# Note: Intended to be run as "make run-blackbox-tests" or "make run-blackbox-sync-nightly"
 #       Makefile target installs & checks all necessary tooling
 #       Extra tools that are not covered in Makefile target needs to be added in verify_prerequisites()
 

test/blackbox/sync_replica_cluster.bats

@@ -1,4 +1,4 @@
-# Note: Intended to be run as "make test-bats-sync" or "make test-bats-sync-verbose"
+# Note: Intended to be run as "make run-blackbox-tests" or "make run-blackbox-ci"
 #       Makefile target installs & checks all necessary tooling
 #       Extra tools that are not covered in Makefile target needs to be added in verify_prerequisites()