From 8438c83a2392bc727f1af3c02edc977535520bee Mon Sep 17 00:00:00 2001 From: Andrei Aaron Date: Sun, 16 Mar 2025 08:42:37 +0000 Subject: [PATCH] test: add scale-out clustering tests using multiple zot servers with with redis and S3 integration Signed-off-by: Andrei Aaron --- .github/workflows/ecosystem-tools.yaml | 67 +++++++++++-- .github/workflows/nightly.yaml | 93 +++++++++++++++++++ Makefile | 10 ++ test/scale-out/cloud_scale_out_redis.bats | 86 +++++++++++++++++ .../cloud_scale_out_redis_scale.bats | 86 +++++++++++++++++ test/scale-out/helpers_redis.bash | 11 +++ test/scale-out/helpers_zot.bash | 55 +++++++++++ 7 files changed, 399 insertions(+), 9 deletions(-) create mode 100644 test/scale-out/cloud_scale_out_redis.bats create mode 100644 test/scale-out/cloud_scale_out_redis_scale.bats create mode 100644 test/scale-out/helpers_redis.bash diff --git a/.github/workflows/ecosystem-tools.yaml b/.github/workflows/ecosystem-tools.yaml index abae1422..6861e90c 100644 --- a/.github/workflows/ecosystem-tools.yaml +++ b/.github/workflows/ecosystem-tools.yaml @@ -91,36 +91,85 @@ jobs: env: AWS_ACCESS_KEY_ID: fake AWS_SECRET_ACCESS_KEY: fake - - name: Run cloud scale-out tests - id: scale + + # DynamoDB scale-out tests + - name: Run cloud scale-out DynamoDB tests + id: dynamodb_scale run: | make run-cloud-scale-out-tests env: AWS_ACCESS_KEY_ID: fake AWS_SECRET_ACCESS_KEY: fake continue-on-error: true - - name: print service logs for scale-out + - name: Print service logs for DynamoDB scale-out run: | find /tmp/zot-ft-logs -name '*.log' -print0 | xargs -0 cat - - name: multi-hop detection - id: multihop + - name: Upload DynamoDB zot logs as build artifact + uses: actions/upload-artifact@v4 + if: always() + with: + name: zot-scale-out-dynamodb-logs + path: /tmp/zot-ft-logs + if-no-files-found: error + - name: DynamoDB multi-hop detection + id: dynamodb_multihop run: | if find /tmp/zot-ft-logs -name '*.log' -print0 | xargs -0 cat | grep 'cannot proxy an already proxied request'; then - echo "detected multi-hop" + echo "detected multi-hop in DynamoDB tests" exit 1 else exit 0 fi continue-on-error: true - - name: clean up scale-out logs + - name: Clean up DynamoDB scale-out logs run: | rm -r /tmp/zot-ft-logs - - name: fail job if error - if: ${{ steps.scale.outcome != 'success' || steps.multihop.outcome != 'success' }} + - name: Fail job if DynamoDB error + if: ${{ steps.dynamodb_scale.outcome != 'success' || steps.dynamodb_multihop.outcome != 'success' }} run: | + echo "DynamoDB scale-out tests failed" + exit 1 + + # Redis scale-out tests + - name: Run cloud scale-out Redis tests + id: redis_scale + run: | + make run-cloud-scale-out-redis-tests + env: + AWS_ACCESS_KEY_ID: fake + AWS_SECRET_ACCESS_KEY: fake + continue-on-error: true + - name: Print service logs for Redis scale-out + run: | + find /tmp/zot-ft-logs/redis -name '*.log' -print0 | xargs -0 cat + - name: Upload Redis zot logs as build artifact + uses: actions/upload-artifact@v4 + if: always() + with: + name: zot-scale-out-redis-logs + path: /tmp/zot-ft-logs + if-no-files-found: ignore + - name: Redis multi-hop detection + id: redis_multihop + run: | + if find /tmp/zot-ft-logs/redis -name '*.log' -print0 | xargs -0 cat | grep 'cannot proxy an already proxied request'; then + echo "detected multi-hop in Redis tests" + exit 1 + else + exit 0 + fi + continue-on-error: true + - name: Clean up Redis scale-out logs + run: | + rm -rf /tmp/zot-ft-logs/redis + - name: Fail job if Redis error + if: ${{ steps.redis_scale.outcome != 'success' || steps.redis_multihop.outcome != 'success' }} + run: | + echo "Redis scale-out tests failed" exit 1 - name: Upload zb test results zip as build artifact uses: actions/upload-artifact@v4 + if: always() with: name: zb-cloud-scale-out-functional-results-${{ github.sha }} path: ./zb-results/ diff --git a/.github/workflows/nightly.yaml b/.github/workflows/nightly.yaml index 11d67722..43d50d63 100644 --- a/.github/workflows/nightly.yaml +++ b/.github/workflows/nightly.yaml @@ -249,6 +249,13 @@ jobs: AWS_ACCESS_KEY_ID: fake AWS_SECRET_ACCESS_KEY: fake continue-on-error: true + - name: Upload zot logs as build artifact + uses: actions/upload-artifact@v4 + if: always() + with: + name: zot-scale-out-dynamodb-logs + path: /tmp/zot-ft-logs + if-no-files-found: error - name: print service logs run: | sudo dmesg @@ -277,3 +284,89 @@ jobs: name: zb-cloud-scale-out-perf-results-${{ github.sha }} path: ./zb-results/ - uses: ./.github/actions/teardown-localstack + + cloud-scale-out-redis: + name: s3+redis scale-out + runs-on: ubuntu-latest-16-cores + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 + with: + cache: false + go-version: 1.23.x + - name: Install dependencies + run: | + cd $GITHUB_WORKSPACE + go install github.com/swaggo/swag/cmd/swag@v1.16.2 + go mod download + sudo apt-get update + sudo apt-get install libgpgme-dev libassuan-dev libbtrfs-dev libdevmapper-dev pkg-config rpm uidmap haproxy jq docker.io + # install skopeo + git clone -b v1.12.0 https://github.com/containers/skopeo.git + cd skopeo + make bin/skopeo + sudo cp bin/skopeo /usr/bin + skopeo -v + cd $GITHUB_WORKSPACE + - name: Log in to GitHub Docker Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ github.token }} + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + - name: Install localstack + run: | + pip install --upgrade pyopenssl + pip install localstack==3.3.0 awscli-local[ver1] # install LocalStack cli and awslocal + docker pull ghcr.io/project-zot/ci-images/localstack:3.3.0 # Make sure to pull a working version of the image + localstack start -d # Start LocalStack in the background + + echo "Waiting for LocalStack startup..." # Wait 30 seconds for the LocalStack container + localstack wait -t 30 # to become ready before timing out + echo "Startup complete" + - name: Run cloud scale-out Redis tests + id: scale + run: | + make run-cloud-scale-out-redis-high-scale-tests + env: + AWS_ACCESS_KEY_ID: fake + AWS_SECRET_ACCESS_KEY: fake + continue-on-error: true + - name: Upload zot logs as build artifact + uses: actions/upload-artifact@v4 + if: always() + with: + name: zot-scale-out-redis-logs + path: /tmp/zot-ft-logs + if-no-files-found: error + - name: print service logs + run: | + sudo dmesg + cat /tmp/zot-ft-logs/redis-scale/*.log + - name: multi-hop detection + id: multihop + run: | + if cat /tmp/zot-ft-logs/redis/*.log | grep 'cannot proxy an already proxied request'; then + echo "detected multi-hop" + exit 1 + else + exit 0 + fi + continue-on-error: true + - name: clean up logs + run: | + rm -r /tmp/zot-ft-logs/redis + - name: fail job if error + if: ${{ steps.scale.outcome != 'success' || steps.multihop.outcome != 'success' }} + run: | + exit 1 + - name: Upload zb test results zip as build artifact + if: steps.scale.outcome == 'success' + uses: actions/upload-artifact@v4 + with: + name: zb-cloud-scale-out-redis-results-${{ github.sha }} + path: ./zb-results/ + - uses: ./.github/actions/teardown-localstack diff --git a/Makefile b/Makefile index 0b1d8f37..70181eda 100644 --- a/Makefile +++ b/Makefile @@ -496,11 +496,21 @@ run-cloud-scale-out-tests: check-blackbox-prerequisites check-awslocal binary be $(BATS) $(BATS_FLAGS) test/scale-out/cloud_scale_out_no_auth.bats; \ $(BATS) $(BATS_FLAGS) test/scale-out/cloud_scale_out_basic_auth_tls.bats +.PHONY: run-cloud-scale-out-redis-tests +run-cloud-scale-out-redis-tests: check-blackbox-prerequisites check-awslocal binary bench test-prereq + echo running redis scale out bats test; \ + $(BATS) $(BATS_FLAGS) test/scale-out/cloud_scale_out_redis.bats + .PHONY: run-cloud-scale-out-high-scale-tests run-cloud-scale-out-high-scale-tests: check-blackbox-prerequisites check-awslocal binary bench test-prereq echo running cloud scale out bats high scale test; \ $(BATS) $(BATS_FLAGS) test/scale-out/cloud_scale_out_basic_auth_tls_scale.bats +.PHONY: run-cloud-scale-out-redis-high-scale-tests +run-cloud-scale-out-redis-high-scale-tests: check-blackbox-prerequisites check-awslocal binary bench test-prereq + echo running redis scale out high scale bats test; \ + $(BATS) $(BATS_FLAGS) test/scale-out/cloud_scale_out_redis_scale.bats + .PHONY: run-blackbox-ci run-blackbox-ci: check-blackbox-prerequisites binary binary-minimal cli echo running CI bats tests concurently diff --git a/test/scale-out/cloud_scale_out_redis.bats b/test/scale-out/cloud_scale_out_redis.bats new file mode 100644 index 00000000..9b80d07d --- /dev/null +++ b/test/scale-out/cloud_scale_out_redis.bats @@ -0,0 +1,86 @@ +# note: intended to be run as "make run-cloud-scale-out-redis-tests" +# makefile target installs & checks all necessary tooling +# extra tools that are not covered in Makefile target needs to be added in verify_prerequisites() + +NUM_ZOT_INSTANCES=6 +ZOT_LOG_DIR=/tmp/zot-ft-logs/redis + +load helpers_zot +load helpers_cloud +load helpers_haproxy +load helpers_redis + +function verify_prerequisites() { + if [ ! $(command -v docker) ]; then + echo "you need to install docker as a prerequisite to running the tests" >&3 + return 1 + fi + + return 0 +} + +function launch_zot_server() { + local zot_server_address=${1} + local zot_server_port=${2} + local zot_root_dir=${ZOT_ROOT_DIR} + local redis_url=${3} + + mkdir -p ${zot_root_dir} + mkdir -p ${ZOT_LOG_DIR} + + local zot_config_file="${BATS_FILE_TMPDIR}/zot_config_${zot_server_address}_${zot_server_port}.json" + local zot_log_file="${ZOT_LOG_DIR}/zot-${zot_server_address}-${zot_server_port}.log" + + create_zot_cloud_redis_config_file ${zot_server_address} ${zot_server_port} ${zot_root_dir} ${zot_config_file} ${zot_log_file} ${redis_url} + update_zot_cluster_member_list_in_config_file ${zot_config_file} ${ZOT_CLUSTER_MEMBERS_PATCH_FILE} + + echo "launching zot server ${zot_server_address}:${zot_server_port}" >&3 + echo "config file: ${zot_config_file}" >&3 + echo "log file: ${zot_log_file}" >&3 + + zot_serve ${zot_config_file} + wait_zot_reachable ${zot_server_port} +} + +function setup() { + # verify prerequisites are available + if ! $(verify_prerequisites); then + exit 1 + fi + + # setup Redis server + redis_port=$(get_free_port) + redis_start redis_server ${redis_port} + local redis_url="redis://127.0.0.1:${redis_port}" + + # setup S3 bucket and DynamoDB tables + setup_cloud_services + generate_zot_cluster_member_list ${NUM_ZOT_INSTANCES} ${ZOT_CLUSTER_MEMBERS_PATCH_FILE} + + for ((i=0;i<${NUM_ZOT_INSTANCES};i++)); do + launch_zot_server 127.0.0.1 $(( 10000 + $i )) ${redis_url} + done + + # list all zot processes that were started + ps -ef | grep ".*zot.*serve.*" | grep -v grep >&3 + + generate_haproxy_config ${HAPROXY_CFG_FILE} "http" + haproxy_start ${HAPROXY_CFG_FILE} + + # list HAproxy processes that were started + ps -ef | grep "haproxy" | grep -v grep >&3 +} + +function teardown() { + local zot_root_dir=${ZOT_ROOT_DIR} + haproxy_stop_all + zot_stop_all + redis_stop redis_server + rm -rf ${zot_root_dir} + teardown_cloud_services +} + +@test "Check for successful zb run on haproxy frontend with Redis cache" { + # zb_run + zb_run "cloud-scale-out-redis-bats" "http://127.0.0.1:8000" 3 5 +} \ No newline at end of file diff --git a/test/scale-out/cloud_scale_out_redis_scale.bats b/test/scale-out/cloud_scale_out_redis_scale.bats new file mode 100644 index 00000000..e52108ec --- /dev/null +++ b/test/scale-out/cloud_scale_out_redis_scale.bats @@ -0,0 +1,86 @@ +# note: intended to be run as "make run-cloud-scale-out-redis-high-scale-tests" +# makefile target installs & checks all necessary tooling +# extra tools that are not covered in Makefile target needs to be added in verify_prerequisites() + +NUM_ZOT_INSTANCES=6 +ZOT_LOG_DIR=/tmp/zot-ft-logs/redis-scale + +load helpers_zot +load helpers_cloud +load helpers_haproxy +load helpers_redis + +function verify_prerequisites() { + if [ ! $(command -v docker) ]; then + echo "you need to install docker as a prerequisite to running the tests" >&3 + return 1 + fi + + return 0 +} + +function launch_zot_server() { + local zot_server_address=${1} + local zot_server_port=${2} + local zot_root_dir=${ZOT_ROOT_DIR} + local redis_url=${3} + + mkdir -p ${zot_root_dir} + mkdir -p ${ZOT_LOG_DIR} + + local zot_config_file="${BATS_FILE_TMPDIR}/zot_config_${zot_server_address}_${zot_server_port}.json" + local zot_log_file="${ZOT_LOG_DIR}/zot-${zot_server_address}-${zot_server_port}.log" + + create_zot_cloud_redis_config_file ${zot_server_address} ${zot_server_port} ${zot_root_dir} ${zot_config_file} ${zot_log_file} ${redis_url} + update_zot_cluster_member_list_in_config_file ${zot_config_file} ${ZOT_CLUSTER_MEMBERS_PATCH_FILE} + + echo "launching zot server ${zot_server_address}:${zot_server_port}" >&3 + echo "config file: ${zot_config_file}" >&3 + echo "log file: ${zot_log_file}" >&3 + + zot_serve ${zot_config_file} + wait_zot_reachable ${zot_server_port} +} + +function setup() { + # verify prerequisites are available + if ! $(verify_prerequisites); then + exit 1 + fi + + # setup Redis server + redis_port=$(get_free_port) + redis_start redis_server ${redis_port} + local redis_url="redis://127.0.0.1:${redis_port}" + + # setup S3 bucket and DynamoDB tables + setup_cloud_services + generate_zot_cluster_member_list ${NUM_ZOT_INSTANCES} ${ZOT_CLUSTER_MEMBERS_PATCH_FILE} + + for ((i=0;i<${NUM_ZOT_INSTANCES};i++)); do + launch_zot_server 127.0.0.1 $(( 10000 + $i )) ${redis_url} + done + + # list all zot processes that were started + ps -ef | grep ".*zot.*serve.*" | grep -v grep >&3 + + generate_haproxy_config ${HAPROXY_CFG_FILE} "http" + haproxy_start ${HAPROXY_CFG_FILE} + + # list HAproxy processes that were started + ps -ef | grep "haproxy" | grep -v grep >&3 +} + +function teardown() { + local zot_root_dir=${ZOT_ROOT_DIR} + haproxy_stop_all + zot_stop_all + redis_stop redis_server + rm -rf ${zot_root_dir} + teardown_cloud_services +} + +@test "Check for successful zb run on haproxy frontend with Redis cache (high scale)" { + # zb_run + zb_run "cloud-scale-out-redis-scale-bats" "http://127.0.0.1:8000" 10 100 +} \ No newline at end of file diff --git a/test/scale-out/helpers_redis.bash b/test/scale-out/helpers_redis.bash new file mode 100644 index 00000000..437cd6de --- /dev/null +++ b/test/scale-out/helpers_redis.bash @@ -0,0 +1,11 @@ +function redis_start() { + local cname="$1" # container name + local free_port="$2" + docker run -d --name ${cname} -p ${free_port}:6379 redis +} + +function redis_stop() { + local cname="$1" + docker stop ${cname} + docker rm -f ${cname} +} \ No newline at end of file diff --git a/test/scale-out/helpers_zot.bash b/test/scale-out/helpers_zot.bash index dc701551..670012f9 100644 --- a/test/scale-out/helpers_zot.bash +++ b/test/scale-out/helpers_zot.bash @@ -237,6 +237,61 @@ function create_zot_cloud_base_config_file() { EOF } +# generates and saves a cloud config with Redis cache and shared storage +# given some basic parameters about the zot instance. +function create_zot_cloud_redis_config_file() { + local zot_server_address=${1} + local zot_server_port=${2} + local zot_root_dir=${3} + local zot_config_file=${4} + local zot_log_file=${5} + local redis_url=${6} + + cat > ${zot_config_file} <