github/zot
1
0
Fork 0
mirror of https://github.com/project-zot/zot.git synced 2026-06-17 21:17:58 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(tls): implement dynamic TLS certificate reloading with file watching (#3792)

Browse Source 
Fixes issue #3747

Signed-off-by: Ramkumar Chinchani <rchincha.dev@gmail.com>
This commit is contained in:
Ramkumar Chinchani
2026-02-15 13:01:50 -08:00
committed by GitHub
parent 2c110d2c20
commit 47659c11b2
9 changed files with 1372 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files
test/blackbox/ci.sh
+1 -1
View File
@@ -15,7 +15,7 @@ tests=("pushpull" "pushpull_authn" "delete_images" "referrers" "metadata" "anony
"annotations" "detect_manifest_collision" "cve" "sync" "sync_docker" "sync_replica_cluster"
"scrub" "garbage_collect" "metrics" "metrics_minimal" "multiarch_index" "docker_compat" "redis_local" "redis_session_store"
"events_nats" "events_http" "events_nats_lint_failure" "events_http_lint_failure" "events_sink_failure" "events_config_decoding"
"fips140" "fips140_authn" "openid_claim_mapping" "upgrade" "upgrade_minimal")
"fips140" "fips140_authn" "openid_claim_mapping" "upgrade" "upgrade_minimal" "dynamic_tls")
for test in ${tests[*]}; do
${BATS} ${BATS_FLAGS} ${SCRIPTPATH}/${test}.bats > ${test}.log & pids+=($!)
test/blackbox/dynamic_tls.bats
+275
View File
@@ -0,0 +1,275 @@
# Note: Intended to be run as "make run-blackbox-tests" or "make run-blackbox-ci"
# Makefile target installs & checks all necessary tooling
# Extra tools that are not covered in Makefile target needs to be added in verify_prerequisites()
load helpers_zot
load ../port_helper
function verify_prerequisites {
if ! command -v curl >/dev/null 2>&1; then
echo "you need to install curl as a prerequisite to running the tests" >&3
return 1
fi
if ! command -v jq >/dev/null 2>&1; then
echo "you need to install jq as a prerequisite to running the tests" >&3
return 1
fi
if ! command -v openssl >/dev/null 2>&1; then
echo "you need to install openssl as a prerequisite to running the tests" >&3
return 1
fi
return 0
}
# Generate a self-signed certificate with the given CN
function generate_self_signed_cert() {
local cert_path=${1}
local key_path=${2}
local common_name=${3:-"localhost"}
local days=${4:-365}
openssl req -x509 -newkey rsa:2048 -keyout "${key_path}" -out "${cert_path}" \
-days ${days} -nodes \
-subj "/C=US/ST=Test/L=Test/O=Zot/CN=${common_name}"
}
# Wait for a condition to be true, polling up to max_attempts times with interval_seconds between attempts
# Usage: wait_for_condition <max_attempts> <interval_seconds> "<command>"
# Returns 0 on success, 1 on timeout
function wait_for_condition() {
local max_attempts=${1}
local interval=${2}
local condition_cmd=${3}
local attempt=1
while [ $attempt -le $max_attempts ]; do
if eval "${condition_cmd}"; then
echo "Condition met after $attempt attempts" >&3
return 0
fi
if [ $attempt -lt $max_attempts ]; then
sleep "${interval}"
fi
((attempt++))
done
echo "Condition timed out after $max_attempts attempts" >&3
return 1
}
function setup_file() {
# Verify prerequisites are available
if ! verify_prerequisites; then
exit 1
fi
# Download test data to folder common for the entire suite, not just this file
skopeo --insecure-policy copy --format=oci docker://ghcr.io/project-zot/test-images/busybox:1.36 oci:${TEST_DATA_DIR}/busybox:1.36
# Setup zot server with TLS
local zot_root_dir=${BATS_FILE_TMPDIR}/zot
local zot_config_file=${BATS_FILE_TMPDIR}/zot_config.json
local zot_cert_file=${BATS_FILE_TMPDIR}/server.cert
local zot_key_file=${BATS_FILE_TMPDIR}/server.key
zot_port=$(get_free_port_for_service "zot")
echo ${zot_port} > ${BATS_FILE_TMPDIR}/zot.port
mkdir -p ${zot_root_dir}
# Generate initial TLS certificate
generate_self_signed_cert "${zot_cert_file}" "${zot_key_file}" "127.0.0.1" 365
# Create zot config with TLS enabled
cat > ${zot_config_file}<<EOF
{
"distSpecVersion":"1.1.1",
"storage":{
"dedupe": true,
"gc": true,
"gcDelay": "1h",
"gcInterval": "6h",
"rootDirectory": "${zot_root_dir}"
},
"http": {
"address": "127.0.0.1",
"port": "${zot_port}",
"tls": {
"cert": "${zot_cert_file}",
"key": "${zot_key_file}"
}
},
"log":{
"level":"debug",
"output": "${BATS_FILE_TMPDIR}/zot.log"
}
}
EOF
echo ${zot_root_dir} >&3
zot_serve ${ZOT_PATH} ${zot_config_file}
# Wait for server to be ready by polling for connectivity
wait_for_condition 30 0.2 "curl -k --max-time 5 --connect-timeout 3 https://127.0.0.1:${zot_port}/v2/_catalog >/dev/null 2>&1"
}
function teardown() {
# conditionally printing on failure is possible from teardown but not from teardown_file
cat ${BATS_FILE_TMPDIR}/zot.log 2>/dev/null || true
}
function teardown_file() {
zot_stop_all
}
@test "TLS connection succeeds with self-signed certificate" {
zot_port=$(cat ${BATS_FILE_TMPDIR}/zot.port)
# Test with curl using insecure flag since we're using self-signed cert
run curl -k --max-time 5 --connect-timeout 3 https://127.0.0.1:${zot_port}/v2/_catalog
[ "$status" -eq 0 ]
}
@test "push image with TLS enabled" {
zot_port=$(cat ${BATS_FILE_TMPDIR}/zot.port)
# Use skopeo to push image over HTTPS with insecure TLS verification
run skopeo --insecure-policy copy --dest-tls-verify=false \
oci:${TEST_DATA_DIR}/busybox:1.36 \
docker://127.0.0.1:${zot_port}/busybox:1.36
[ "$status" -eq 0 ]
}
@test "pull image with TLS enabled" {
zot_port=$(cat ${BATS_FILE_TMPDIR}/zot.port)
local temp_oci_dir=${BATS_FILE_TMPDIR}/busybox-pulled
mkdir -p ${temp_oci_dir}
# Pull the pushed image back
run skopeo --insecure-policy copy --src-tls-verify=false \
docker://127.0.0.1:${zot_port}/busybox:1.36 \
oci:${temp_oci_dir}
[ "$status" -eq 0 ]
# Verify OCI image was downloaded
[ -f "${temp_oci_dir}/oci-layout" ]
}
@test "dynamic certificate reload: verify server uses new certificate after update" {
zot_port=$(cat ${BATS_FILE_TMPDIR}/zot.port)
local zot_cert_file=${BATS_FILE_TMPDIR}/server.cert
local zot_key_file=${BATS_FILE_TMPDIR}/server.key
# Get the certificate fingerprint before update
cert_fingerprint_before=$(openssl x509 -fingerprint -sha256 -noout -in "${zot_cert_file}" 2>/dev/null | cut -d'=' -f2)
server_fingerprint_before=$(openssl s_client -connect 127.0.0.1:${zot_port} -servername 127.0.0.1 -showcerts </dev/null 2>/dev/null \
| openssl x509 -fingerprint -sha256 -noout 2>/dev/null | cut -d'=' -f2)
[ -n "${server_fingerprint_before}" ]
# Keep fetching catalog to ensure server is responsive before cert update
wait_for_condition 10 0.5 "curl -k --max-time 5 --connect-timeout 3 https://127.0.0.1:${zot_port}/v2/_catalog >/dev/null 2>&1"
# Update the certificate with a new one
# This simulates a real-world scenario where certificates are renewed
generate_self_signed_cert "${zot_cert_file}" "${zot_key_file}" "127.0.0.1" 365
# Wait for file system changes to be visible and stat cache to expire
# (allows time for inotify to detect changes or stat-based check to trigger)
wait_for_condition 10 0.1 "[ \"$(openssl x509 -fingerprint -sha256 -noout -in \"${zot_cert_file}\" 2>/dev/null | cut -d'=' -f2)\" != \"${cert_fingerprint_before}\" ]"
# Request a new fingerprint after expecting the server to reload
wait_for_condition 20 0.2 "[ \"$(openssl s_client -connect 127.0.0.1:${zot_port} -servername 127.0.0.1 -showcerts </dev/null 2>/dev/null | openssl x509 -fingerprint -sha256 -noout 2>/dev/null | cut -d'=' -f2)\" != \"${server_fingerprint_before}\" ]" || true
# Make several requests to ensure server picks up the new certificate
# The server should automatically reload it through the GetCertificate callback
server_fingerprint_after=""
for i in {1..10}; do
server_fingerprint_after=$(openssl s_client -connect 127.0.0.1:${zot_port} -servername 127.0.0.1 -showcerts </dev/null 2>/dev/null \
| openssl x509 -fingerprint -sha256 -noout 2>/dev/null | cut -d'=' -f2)
run curl -k --max-time 5 --connect-timeout 3 https://127.0.0.1:${zot_port}/v2/_catalog
if [ "$status" -eq 0 ] && [ -n "${server_fingerprint_after}" ] && \
[ "${server_fingerprint_before}" != "${server_fingerprint_after}" ]; then
# Server is using the new certificate
echo "Request $i succeeded with new certificate" >&3
break
fi
if [ $i -lt 10 ]; then
sleep 0.2
fi
done
[ -n "${server_fingerprint_after}" ]
[ "${server_fingerprint_before}" != "${server_fingerprint_after}" ]
[ "$status" -eq 0 ]
}
@test "TLS works with multiple concurrent connections after certificate reload" {
zot_port=$(cat ${BATS_FILE_TMPDIR}/zot.port)
local zot_cert_file=${BATS_FILE_TMPDIR}/server.cert
local zot_key_file=${BATS_FILE_TMPDIR}/server.key
# Regenerate certificate to trigger reload
generate_self_signed_cert "${zot_cert_file}" "${zot_key_file}" "127.0.0.1" 365
# Wait for certificate to be reloaded by making requests
wait_for_condition 20 0.2 "curl -k --max-time 5 --connect-timeout 3 https://127.0.0.1:${zot_port}/v2/_catalog >/dev/null 2>&1"
# Test multiple concurrent requests
local failed=0
local pids=()
for i in {1..5}; do
(curl -k --max-time 5 --connect-timeout 3 https://127.0.0.1:${zot_port}/v2/_catalog > /dev/null 2>&1) &
pids+=($!)
done
# Wait for all background requests to complete
for pid in "${pids[@]}"; do
if ! wait "$pid"; then
failed=$((failed + 1))
fi
done
[ "$failed" -eq 0 ]
# If any failed, the test will fail
# Check that at least one request succeeds by making one more
run curl -k --max-time 5 --connect-timeout 3 https://127.0.0.1:${zot_port}/v2/_catalog
[ "$status" -eq 0 ]
}
@test "certificate reload doesn't require server restart" {
zot_port=$(cat ${BATS_FILE_TMPDIR}/zot.port)
local zot_cert_file=${BATS_FILE_TMPDIR}/server.cert
local zot_key_file=${BATS_FILE_TMPDIR}/server.key
# Get initial server PID
local zot_pid=$(cat ${BATS_FILE_TMPDIR}/zot.pid | awk '{print $1}')
# Make a request to establish the server is running
run curl -k --max-time 5 --connect-timeout 3 https://127.0.0.1:${zot_port}/v2/_catalog
[ "$status" -eq 0 ]
# Verify server is still running with same PID
kill -0 ${zot_pid} 2>/dev/null
[ "$?" -eq 0 ]
# Update certificate multiple times
for iteration in {1..3}; do
generate_self_signed_cert "${zot_cert_file}" "${zot_key_file}" "127.0.0.1" 365
# Wait for server to reload the new certificate
wait_for_condition 20 0.2 "curl -k --max-time 5 --connect-timeout 3 https://127.0.0.1:${zot_port}/v2/_catalog >/dev/null 2>&1" || true
# Server should still be running with the same PID
kill -0 ${zot_pid} 2>/dev/null
[ "$?" -eq 0 ]
# Requests should still work
run curl -k --max-time 5 --connect-timeout 3 https://127.0.0.1:${zot_port}/v2/_catalog
[ "$status" -eq 0 ]
done
}
test/ports.json
+6
View File
@@ -448,5 +448,11 @@
"begin": 11510,
"end": 11519
}
},
"blackbox/dynamic_tls.bats": {
"zot": {
"begin": 11520,
"end": 11529
}
}
}