mirror of
https://github.com/project-zot/zot.git
synced 2026-06-17 21:17:58 +08:00
test: add scale-out clustering tests using multiple zot servers with with redis and S3 integration
Signed-off-by: Andrei Aaron <aaaron@luxoft.com>
This commit is contained in:
Andrei Aaron
committed by
Andrei Aaron
Andrei Aaron
parent
2a4edde637
commit
8438c83a23
@@ -91,36 +91,85 @@ jobs:
|
||||
env:
|
||||
AWS_ACCESS_KEY_ID: fake
|
||||
AWS_SECRET_ACCESS_KEY: fake
|
||||
- name: Run cloud scale-out tests
|
||||
id: scale
|
||||
|
||||
# DynamoDB scale-out tests
|
||||
- name: Run cloud scale-out DynamoDB tests
|
||||
id: dynamodb_scale
|
||||
run: |
|
||||
make run-cloud-scale-out-tests
|
||||
env:
|
||||
AWS_ACCESS_KEY_ID: fake
|
||||
AWS_SECRET_ACCESS_KEY: fake
|
||||
continue-on-error: true
|
||||
- name: print service logs for scale-out
|
||||
- name: Print service logs for DynamoDB scale-out
|
||||
run: |
|
||||
find /tmp/zot-ft-logs -name '*.log' -print0 | xargs -0 cat
|
||||
- name: multi-hop detection
|
||||
id: multihop
|
||||
- name: Upload DynamoDB zot logs as build artifact
|
||||
uses: actions/upload-artifact@v4
|
||||
if: always()
|
||||
with:
|
||||
name: zot-scale-out-dynamodb-logs
|
||||
path: /tmp/zot-ft-logs
|
||||
if-no-files-found: error
|
||||
- name: DynamoDB multi-hop detection
|
||||
id: dynamodb_multihop
|
||||
run: |
|
||||
if find /tmp/zot-ft-logs -name '*.log' -print0 | xargs -0 cat | grep 'cannot proxy an already proxied request'; then
|
||||
echo "detected multi-hop"
|
||||
echo "detected multi-hop in DynamoDB tests"
|
||||
exit 1
|
||||
else
|
||||
exit 0
|
||||
fi
|
||||
continue-on-error: true
|
||||
- name: clean up scale-out logs
|
||||
- name: Clean up DynamoDB scale-out logs
|
||||
run: |
|
||||
rm -r /tmp/zot-ft-logs
|
||||
- name: fail job if error
|
||||
if: ${{ steps.scale.outcome != 'success' || steps.multihop.outcome != 'success' }}
|
||||
- name: Fail job if DynamoDB error
|
||||
if: ${{ steps.dynamodb_scale.outcome != 'success' || steps.dynamodb_multihop.outcome != 'success' }}
|
||||
run: |
|
||||
echo "DynamoDB scale-out tests failed"
|
||||
exit 1
|
||||
|
||||
# Redis scale-out tests
|
||||
- name: Run cloud scale-out Redis tests
|
||||
id: redis_scale
|
||||
run: |
|
||||
make run-cloud-scale-out-redis-tests
|
||||
env:
|
||||
AWS_ACCESS_KEY_ID: fake
|
||||
AWS_SECRET_ACCESS_KEY: fake
|
||||
continue-on-error: true
|
||||
- name: Print service logs for Redis scale-out
|
||||
run: |
|
||||
find /tmp/zot-ft-logs/redis -name '*.log' -print0 | xargs -0 cat
|
||||
- name: Upload Redis zot logs as build artifact
|
||||
uses: actions/upload-artifact@v4
|
||||
if: always()
|
||||
with:
|
||||
name: zot-scale-out-redis-logs
|
||||
path: /tmp/zot-ft-logs
|
||||
if-no-files-found: ignore
|
||||
- name: Redis multi-hop detection
|
||||
id: redis_multihop
|
||||
run: |
|
||||
if find /tmp/zot-ft-logs/redis -name '*.log' -print0 | xargs -0 cat | grep 'cannot proxy an already proxied request'; then
|
||||
echo "detected multi-hop in Redis tests"
|
||||
exit 1
|
||||
else
|
||||
exit 0
|
||||
fi
|
||||
continue-on-error: true
|
||||
- name: Clean up Redis scale-out logs
|
||||
run: |
|
||||
rm -rf /tmp/zot-ft-logs/redis
|
||||
- name: Fail job if Redis error
|
||||
if: ${{ steps.redis_scale.outcome != 'success' || steps.redis_multihop.outcome != 'success' }}
|
||||
run: |
|
||||
echo "Redis scale-out tests failed"
|
||||
exit 1
|
||||
- name: Upload zb test results zip as build artifact
|
||||
uses: actions/upload-artifact@v4
|
||||
if: always()
|
||||
with:
|
||||
name: zb-cloud-scale-out-functional-results-${{ github.sha }}
|
||||
path: ./zb-results/
|
||||
|
||||
@@ -249,6 +249,13 @@ jobs:
|
||||
AWS_ACCESS_KEY_ID: fake
|
||||
AWS_SECRET_ACCESS_KEY: fake
|
||||
continue-on-error: true
|
||||
- name: Upload zot logs as build artifact
|
||||
uses: actions/upload-artifact@v4
|
||||
if: always()
|
||||
with:
|
||||
name: zot-scale-out-dynamodb-logs
|
||||
path: /tmp/zot-ft-logs
|
||||
if-no-files-found: error
|
||||
- name: print service logs
|
||||
run: |
|
||||
sudo dmesg
|
||||
@@ -277,3 +284,89 @@ jobs:
|
||||
name: zb-cloud-scale-out-perf-results-${{ github.sha }}
|
||||
path: ./zb-results/
|
||||
- uses: ./.github/actions/teardown-localstack
|
||||
|
||||
cloud-scale-out-redis:
|
||||
name: s3+redis scale-out
|
||||
runs-on: ubuntu-latest-16-cores
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/setup-go@v5
|
||||
with:
|
||||
cache: false
|
||||
go-version: 1.23.x
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
cd $GITHUB_WORKSPACE
|
||||
go install github.com/swaggo/swag/cmd/swag@v1.16.2
|
||||
go mod download
|
||||
sudo apt-get update
|
||||
sudo apt-get install libgpgme-dev libassuan-dev libbtrfs-dev libdevmapper-dev pkg-config rpm uidmap haproxy jq docker.io
|
||||
# install skopeo
|
||||
git clone -b v1.12.0 https://github.com/containers/skopeo.git
|
||||
cd skopeo
|
||||
make bin/skopeo
|
||||
sudo cp bin/skopeo /usr/bin
|
||||
skopeo -v
|
||||
cd $GITHUB_WORKSPACE
|
||||
- name: Log in to GitHub Docker Registry
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ github.actor }}
|
||||
password: ${{ github.token }}
|
||||
- uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: '3.11'
|
||||
- name: Install localstack
|
||||
run: |
|
||||
pip install --upgrade pyopenssl
|
||||
pip install localstack==3.3.0 awscli-local[ver1] # install LocalStack cli and awslocal
|
||||
docker pull ghcr.io/project-zot/ci-images/localstack:3.3.0 # Make sure to pull a working version of the image
|
||||
localstack start -d # Start LocalStack in the background
|
||||
|
||||
echo "Waiting for LocalStack startup..." # Wait 30 seconds for the LocalStack container
|
||||
localstack wait -t 30 # to become ready before timing out
|
||||
echo "Startup complete"
|
||||
- name: Run cloud scale-out Redis tests
|
||||
id: scale
|
||||
run: |
|
||||
make run-cloud-scale-out-redis-high-scale-tests
|
||||
env:
|
||||
AWS_ACCESS_KEY_ID: fake
|
||||
AWS_SECRET_ACCESS_KEY: fake
|
||||
continue-on-error: true
|
||||
- name: Upload zot logs as build artifact
|
||||
uses: actions/upload-artifact@v4
|
||||
if: always()
|
||||
with:
|
||||
name: zot-scale-out-redis-logs
|
||||
path: /tmp/zot-ft-logs
|
||||
if-no-files-found: error
|
||||
- name: print service logs
|
||||
run: |
|
||||
sudo dmesg
|
||||
cat /tmp/zot-ft-logs/redis-scale/*.log
|
||||
- name: multi-hop detection
|
||||
id: multihop
|
||||
run: |
|
||||
if cat /tmp/zot-ft-logs/redis/*.log | grep 'cannot proxy an already proxied request'; then
|
||||
echo "detected multi-hop"
|
||||
exit 1
|
||||
else
|
||||
exit 0
|
||||
fi
|
||||
continue-on-error: true
|
||||
- name: clean up logs
|
||||
run: |
|
||||
rm -r /tmp/zot-ft-logs/redis
|
||||
- name: fail job if error
|
||||
if: ${{ steps.scale.outcome != 'success' || steps.multihop.outcome != 'success' }}
|
||||
run: |
|
||||
exit 1
|
||||
- name: Upload zb test results zip as build artifact
|
||||
if: steps.scale.outcome == 'success'
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: zb-cloud-scale-out-redis-results-${{ github.sha }}
|
||||
path: ./zb-results/
|
||||
- uses: ./.github/actions/teardown-localstack
|
||||
|
||||
@@ -496,11 +496,21 @@ run-cloud-scale-out-tests: check-blackbox-prerequisites check-awslocal binary be
|
||||
$(BATS) $(BATS_FLAGS) test/scale-out/cloud_scale_out_no_auth.bats; \
|
||||
$(BATS) $(BATS_FLAGS) test/scale-out/cloud_scale_out_basic_auth_tls.bats
|
||||
|
||||
.PHONY: run-cloud-scale-out-redis-tests
|
||||
run-cloud-scale-out-redis-tests: check-blackbox-prerequisites check-awslocal binary bench test-prereq
|
||||
echo running redis scale out bats test; \
|
||||
$(BATS) $(BATS_FLAGS) test/scale-out/cloud_scale_out_redis.bats
|
||||
|
||||
.PHONY: run-cloud-scale-out-high-scale-tests
|
||||
run-cloud-scale-out-high-scale-tests: check-blackbox-prerequisites check-awslocal binary bench test-prereq
|
||||
echo running cloud scale out bats high scale test; \
|
||||
$(BATS) $(BATS_FLAGS) test/scale-out/cloud_scale_out_basic_auth_tls_scale.bats
|
||||
|
||||
.PHONY: run-cloud-scale-out-redis-high-scale-tests
|
||||
run-cloud-scale-out-redis-high-scale-tests: check-blackbox-prerequisites check-awslocal binary bench test-prereq
|
||||
echo running redis scale out high scale bats test; \
|
||||
$(BATS) $(BATS_FLAGS) test/scale-out/cloud_scale_out_redis_scale.bats
|
||||
|
||||
.PHONY: run-blackbox-ci
|
||||
run-blackbox-ci: check-blackbox-prerequisites binary binary-minimal cli
|
||||
echo running CI bats tests concurently
|
||||
|
||||
@@ -0,0 +1,86 @@
|
||||
# note: intended to be run as "make run-cloud-scale-out-redis-tests"
|
||||
# makefile target installs & checks all necessary tooling
|
||||
# extra tools that are not covered in Makefile target needs to be added in verify_prerequisites()
|
||||
|
||||
NUM_ZOT_INSTANCES=6
|
||||
ZOT_LOG_DIR=/tmp/zot-ft-logs/redis
|
||||
|
||||
load helpers_zot
|
||||
load helpers_cloud
|
||||
load helpers_haproxy
|
||||
load helpers_redis
|
||||
|
||||
function verify_prerequisites() {
|
||||
if [ ! $(command -v docker) ]; then
|
||||
echo "you need to install docker as a prerequisite to running the tests" >&3
|
||||
return 1
|
||||
fi
|
||||
|
||||
return 0
|
||||
}
|
||||
|
||||
function launch_zot_server() {
|
||||
local zot_server_address=${1}
|
||||
local zot_server_port=${2}
|
||||
local zot_root_dir=${ZOT_ROOT_DIR}
|
||||
local redis_url=${3}
|
||||
|
||||
mkdir -p ${zot_root_dir}
|
||||
mkdir -p ${ZOT_LOG_DIR}
|
||||
|
||||
local zot_config_file="${BATS_FILE_TMPDIR}/zot_config_${zot_server_address}_${zot_server_port}.json"
|
||||
local zot_log_file="${ZOT_LOG_DIR}/zot-${zot_server_address}-${zot_server_port}.log"
|
||||
|
||||
create_zot_cloud_redis_config_file ${zot_server_address} ${zot_server_port} ${zot_root_dir} ${zot_config_file} ${zot_log_file} ${redis_url}
|
||||
update_zot_cluster_member_list_in_config_file ${zot_config_file} ${ZOT_CLUSTER_MEMBERS_PATCH_FILE}
|
||||
|
||||
echo "launching zot server ${zot_server_address}:${zot_server_port}" >&3
|
||||
echo "config file: ${zot_config_file}" >&3
|
||||
echo "log file: ${zot_log_file}" >&3
|
||||
|
||||
zot_serve ${zot_config_file}
|
||||
wait_zot_reachable ${zot_server_port}
|
||||
}
|
||||
|
||||
function setup() {
|
||||
# verify prerequisites are available
|
||||
if ! $(verify_prerequisites); then
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# setup Redis server
|
||||
redis_port=$(get_free_port)
|
||||
redis_start redis_server ${redis_port}
|
||||
local redis_url="redis://127.0.0.1:${redis_port}"
|
||||
|
||||
# setup S3 bucket and DynamoDB tables
|
||||
setup_cloud_services
|
||||
generate_zot_cluster_member_list ${NUM_ZOT_INSTANCES} ${ZOT_CLUSTER_MEMBERS_PATCH_FILE}
|
||||
|
||||
for ((i=0;i<${NUM_ZOT_INSTANCES};i++)); do
|
||||
launch_zot_server 127.0.0.1 $(( 10000 + $i )) ${redis_url}
|
||||
done
|
||||
|
||||
# list all zot processes that were started
|
||||
ps -ef | grep ".*zot.*serve.*" | grep -v grep >&3
|
||||
|
||||
generate_haproxy_config ${HAPROXY_CFG_FILE} "http"
|
||||
haproxy_start ${HAPROXY_CFG_FILE}
|
||||
|
||||
# list HAproxy processes that were started
|
||||
ps -ef | grep "haproxy" | grep -v grep >&3
|
||||
}
|
||||
|
||||
function teardown() {
|
||||
local zot_root_dir=${ZOT_ROOT_DIR}
|
||||
haproxy_stop_all
|
||||
zot_stop_all
|
||||
redis_stop redis_server
|
||||
rm -rf ${zot_root_dir}
|
||||
teardown_cloud_services
|
||||
}
|
||||
|
||||
@test "Check for successful zb run on haproxy frontend with Redis cache" {
|
||||
# zb_run <test_name> <zot_address> <concurrency> <num_requests> <credentials (optional)>
|
||||
zb_run "cloud-scale-out-redis-bats" "http://127.0.0.1:8000" 3 5
|
||||
}
|
||||
@@ -0,0 +1,86 @@
|
||||
# note: intended to be run as "make run-cloud-scale-out-redis-high-scale-tests"
|
||||
# makefile target installs & checks all necessary tooling
|
||||
# extra tools that are not covered in Makefile target needs to be added in verify_prerequisites()
|
||||
|
||||
NUM_ZOT_INSTANCES=6
|
||||
ZOT_LOG_DIR=/tmp/zot-ft-logs/redis-scale
|
||||
|
||||
load helpers_zot
|
||||
load helpers_cloud
|
||||
load helpers_haproxy
|
||||
load helpers_redis
|
||||
|
||||
function verify_prerequisites() {
|
||||
if [ ! $(command -v docker) ]; then
|
||||
echo "you need to install docker as a prerequisite to running the tests" >&3
|
||||
return 1
|
||||
fi
|
||||
|
||||
return 0
|
||||
}
|
||||
|
||||
function launch_zot_server() {
|
||||
local zot_server_address=${1}
|
||||
local zot_server_port=${2}
|
||||
local zot_root_dir=${ZOT_ROOT_DIR}
|
||||
local redis_url=${3}
|
||||
|
||||
mkdir -p ${zot_root_dir}
|
||||
mkdir -p ${ZOT_LOG_DIR}
|
||||
|
||||
local zot_config_file="${BATS_FILE_TMPDIR}/zot_config_${zot_server_address}_${zot_server_port}.json"
|
||||
local zot_log_file="${ZOT_LOG_DIR}/zot-${zot_server_address}-${zot_server_port}.log"
|
||||
|
||||
create_zot_cloud_redis_config_file ${zot_server_address} ${zot_server_port} ${zot_root_dir} ${zot_config_file} ${zot_log_file} ${redis_url}
|
||||
update_zot_cluster_member_list_in_config_file ${zot_config_file} ${ZOT_CLUSTER_MEMBERS_PATCH_FILE}
|
||||
|
||||
echo "launching zot server ${zot_server_address}:${zot_server_port}" >&3
|
||||
echo "config file: ${zot_config_file}" >&3
|
||||
echo "log file: ${zot_log_file}" >&3
|
||||
|
||||
zot_serve ${zot_config_file}
|
||||
wait_zot_reachable ${zot_server_port}
|
||||
}
|
||||
|
||||
function setup() {
|
||||
# verify prerequisites are available
|
||||
if ! $(verify_prerequisites); then
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# setup Redis server
|
||||
redis_port=$(get_free_port)
|
||||
redis_start redis_server ${redis_port}
|
||||
local redis_url="redis://127.0.0.1:${redis_port}"
|
||||
|
||||
# setup S3 bucket and DynamoDB tables
|
||||
setup_cloud_services
|
||||
generate_zot_cluster_member_list ${NUM_ZOT_INSTANCES} ${ZOT_CLUSTER_MEMBERS_PATCH_FILE}
|
||||
|
||||
for ((i=0;i<${NUM_ZOT_INSTANCES};i++)); do
|
||||
launch_zot_server 127.0.0.1 $(( 10000 + $i )) ${redis_url}
|
||||
done
|
||||
|
||||
# list all zot processes that were started
|
||||
ps -ef | grep ".*zot.*serve.*" | grep -v grep >&3
|
||||
|
||||
generate_haproxy_config ${HAPROXY_CFG_FILE} "http"
|
||||
haproxy_start ${HAPROXY_CFG_FILE}
|
||||
|
||||
# list HAproxy processes that were started
|
||||
ps -ef | grep "haproxy" | grep -v grep >&3
|
||||
}
|
||||
|
||||
function teardown() {
|
||||
local zot_root_dir=${ZOT_ROOT_DIR}
|
||||
haproxy_stop_all
|
||||
zot_stop_all
|
||||
redis_stop redis_server
|
||||
rm -rf ${zot_root_dir}
|
||||
teardown_cloud_services
|
||||
}
|
||||
|
||||
@test "Check for successful zb run on haproxy frontend with Redis cache (high scale)" {
|
||||
# zb_run <test_name> <zot_address> <concurrency> <num_requests> <credentials (optional)>
|
||||
zb_run "cloud-scale-out-redis-scale-bats" "http://127.0.0.1:8000" 10 100
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
function redis_start() {
|
||||
local cname="$1" # container name
|
||||
local free_port="$2"
|
||||
docker run -d --name ${cname} -p ${free_port}:6379 redis
|
||||
}
|
||||
|
||||
function redis_stop() {
|
||||
local cname="$1"
|
||||
docker stop ${cname}
|
||||
docker rm -f ${cname}
|
||||
}
|
||||
@@ -237,6 +237,61 @@ function create_zot_cloud_base_config_file() {
|
||||
EOF
|
||||
}
|
||||
|
||||
# generates and saves a cloud config with Redis cache and shared storage
|
||||
# given some basic parameters about the zot instance.
|
||||
function create_zot_cloud_redis_config_file() {
|
||||
local zot_server_address=${1}
|
||||
local zot_server_port=${2}
|
||||
local zot_root_dir=${3}
|
||||
local zot_config_file=${4}
|
||||
local zot_log_file=${5}
|
||||
local redis_url=${6}
|
||||
|
||||
cat > ${zot_config_file} <<EOF
|
||||
{
|
||||
"distSpecVersion": "1.1.1",
|
||||
"storage": {
|
||||
"rootDirectory": "${zot_root_dir}",
|
||||
"dedupe": true,
|
||||
"remoteCache": true,
|
||||
"storageDriver": {
|
||||
"name": "s3",
|
||||
"rootdirectory": "/zot",
|
||||
"region": "us-east-2",
|
||||
"regionendpoint": "localhost:4566",
|
||||
"bucket": "zot-storage-test",
|
||||
"secure": false,
|
||||
"skipverify": false
|
||||
},
|
||||
"cacheDriver": {
|
||||
"name": "redis",
|
||||
"url": "${redis_url}"
|
||||
}
|
||||
},
|
||||
"http": {
|
||||
"address": "${zot_server_address}",
|
||||
"port": "${zot_server_port}"
|
||||
},
|
||||
"cluster": {
|
||||
"members": [],
|
||||
"hashKey": "loremipsumdolors"
|
||||
},
|
||||
"log": {
|
||||
"level": "debug",
|
||||
"output": "${zot_log_file}"
|
||||
},
|
||||
"extensions": {
|
||||
"ui": {
|
||||
"enable": true
|
||||
},
|
||||
"search": {
|
||||
"enable": true
|
||||
}
|
||||
}
|
||||
}
|
||||
EOF
|
||||
}
|
||||
|
||||
# updates an existing zot config file that already has an HTTP config
|
||||
# to include htpasswd auth settings.
|
||||
# intended for use with create_zot_cloud_base_config_file() above.
|
||||
|
||||
Reference in New Issue
Block a user