From 05823cd74f365a210eeff345b7471e3fe6347db5 Mon Sep 17 00:00:00 2001 From: Andrei Aaron Date: Thu, 30 Jan 2025 21:00:52 +0200 Subject: [PATCH] redis driver for blob cache information and metadb (#2865) * feat: add redis cache support https://github.com/project-zot/zot/pull/2005 Fixes https://github.com/project-zot/zot/issues/2004 * feat: add redis cache support Currently, we have dynamoDB as the remote shared cache but ideal only for the cloud use case. For on-prem use case, add support for redis. Signed-off-by: Ramkumar Chinchani * feat(redis): added blackbox tests for redis Signed-off-by: Petu Eusebiu * feat(redis): dummy implementation of MetaDB interface for redis cache Signed-off-by: Alexei Dodon * feat: check validity of driver configuration on metadb instantiation Signed-off-by: Andrei Aaron * feat: multiple fixes for redis cache driver implementation - add missing method GetAllBlobs - add redis cache tests, with and without mocking Signed-off-by: Andrei Aaron * feat(redis): redis implementation for MetaDB Signed-off-by: Andrei Aaron * feat(redis): use redsync to block concurrent write access to the redis DB Signed-off-by: Andrei Aaron * feat(redis): update .github/workflows/cluster.yaml to also test redis Signed-off-by: Andrei Aaron * feat(metadb): add keyPrefix parameter for redis and remove unneeded method meta.Crate() Signed-off-by: Andrei Aaron * feat(redis): support RedisCluster configuration and add unit tests Signed-off-by: Andrei Aaron * feat(redis): more tests for redis metadb implementation Signed-off-by: Andrei Aaron * feat(redis): add more examples and update examples/README.md Signed-off-by: Andrei Aaron * feat(redis): move option parsing and redis client initialization under pkg/api/config/redis Signed-off-by: Andrei Aaron * chore(cachedb): move Cache interface to pkg/storage/types Signed-off-by: Andrei Aaron * feat(redis): reorganize code in pkg/storage/cache.go Signed-off-by: Andrei Aaron * feat(redis): call redis.SetLogger() with the zot logger as parameter Signed-off-by: Andrei Aaron * feat(redis): rename pkg/meta/redisdb to pkg/meta/redis Signed-off-by: Andrei Aaron --------- Signed-off-by: Ramkumar Chinchani Signed-off-by: Petu Eusebiu Signed-off-by: Alexei Dodon Signed-off-by: Andrei Aaron Co-authored-by: a Co-authored-by: Ramkumar Chinchani Co-authored-by: Petu Eusebiu Co-authored-by: Alexei Dodon --- .github/workflows/cluster.yaml | 350 ++- Makefile | 1 + examples/README.md | 30 + examples/config-redis-all-options.json | 68 + examples/config-redis-cluster.json | 38 + examples/config-redis.json | 38 + go.mod | 9 +- go.sum | 30 +- pkg/api/config/redis/redis.go | 295 +++ pkg/api/config/redis/redis_test.go | 340 +++ pkg/api/controller_test.go | 126 +- pkg/cli/server/root.go | 7 +- pkg/extensions/extension_image_trust.go | 8 +- pkg/extensions/extension_image_trust_test.go | 16 +- pkg/extensions/imagetrust/image_trust_test.go | 24 +- pkg/extensions/search/search_test.go | 35 +- pkg/meta/meta.go | 128 +- pkg/meta/meta_test.go | 561 +++- pkg/meta/parse_test.go | 24 + pkg/meta/redis/redis.go | 2302 +++++++++++++++++ pkg/meta/redis/redis_internal_test.go | 61 + pkg/meta/redis/redis_test.go | 1532 +++++++++++ pkg/meta/types/types.go | 10 +- pkg/meta/version/patches.go | 5 + pkg/meta/version/version_test.go | 176 ++ pkg/storage/cache.go | 58 +- pkg/storage/cache/redis.go | 340 +++ pkg/storage/cache/redis_test.go | 712 +++++ pkg/storage/cache_benchmark_test.go | 11 +- pkg/storage/cache_test.go | 9 + pkg/storage/constants/constants.go | 2 + pkg/storage/imagestore/imagestore.go | 5 +- pkg/storage/local/local.go | 3 +- pkg/storage/s3/s3.go | 3 +- pkg/storage/s3/s3_test.go | 8 +- pkg/storage/scrub_test.go | 40 +- pkg/storage/storage_test.go | 624 +++-- .../cacheinterface.go => types/cache.go} | 3 +- test/blackbox/ci.sh | 2 +- test/blackbox/helpers_redis.bash | 12 + test/blackbox/redis_local.bats | 124 + test/blackbox/redis_s3.bats | 129 + test/cluster/config-minio-redis.json | 29 + 43 files changed, 7886 insertions(+), 442 deletions(-) create mode 100644 examples/config-redis-all-options.json create mode 100644 examples/config-redis-cluster.json create mode 100644 examples/config-redis.json create mode 100644 pkg/api/config/redis/redis.go create mode 100644 pkg/api/config/redis/redis_test.go create mode 100644 pkg/meta/redis/redis.go create mode 100644 pkg/meta/redis/redis_internal_test.go create mode 100644 pkg/meta/redis/redis_test.go create mode 100644 pkg/storage/cache/redis.go create mode 100644 pkg/storage/cache/redis_test.go rename pkg/storage/{cache/cacheinterface.go => types/cache.go} (91%) create mode 100644 test/blackbox/helpers_redis.bash create mode 100644 test/blackbox/redis_local.bats create mode 100644 test/blackbox/redis_s3.bats create mode 100644 test/cluster/config-minio-redis.json diff --git a/.github/workflows/cluster.yaml b/.github/workflows/cluster.yaml index fc904224..5f233f2a 100644 --- a/.github/workflows/cluster.yaml +++ b/.github/workflows/cluster.yaml @@ -12,20 +12,9 @@ on: permissions: read-all jobs: - client-tools: - name: Stateless zot with shared reliable storage + minio-bolt: + name: Stateless zot with minio and boltdb runs-on: ubuntu-latest - # services: - # minio: - # image: minio/minio:RELEASE.2024-07-16T23-46-41Z - # env: - # MINIO_ROOT_USER: minioadmin - # MINIO_ROOT_PASSWORD: minioadmin - # ports: - # - 9000:9000 - # volumes: - # - /tmp/data:/data - # options: --name=minio --health-cmd "curl http://localhost:9000/minio/health/live" steps: - uses: actions/checkout@v4 - uses: actions/setup-go@v5 @@ -101,12 +90,15 @@ jobs: cp test/cluster/config-minio.json test/cluster/config-minio1.json sed -i 's/8081/8081/g' test/cluster/config-minio1.json sed -i 's/\/tmp\/zot/\/tmp\/zot1/g' test/cluster/config-minio1.json + sed -i 's/\/dev\/null/\/tmp\/zot1.log/g' test/cluster/config-minio1.json cp test/cluster/config-minio.json test/cluster/config-minio2.json sed -i 's/8081/8082/g' test/cluster/config-minio2.json sed -i 's/\/tmp\/zot/\/tmp\/zot2/g' test/cluster/config-minio2.json + sed -i 's/\/dev\/null/\/tmp\/zot2.log/g' test/cluster/config-minio2.json cp test/cluster/config-minio.json test/cluster/config-minio3.json sed -i 's/8081/8083/g' test/cluster/config-minio3.json sed -i 's/\/tmp\/zot/\/tmp\/zot3/g' test/cluster/config-minio3.json + sed -i 's/\/dev\/null/\/tmp\/zot3.log/g' test/cluster/config-minio3.json - name: Free up disk space uses: jlumbroso/free-disk-space@main @@ -125,7 +117,13 @@ jobs: ./bin/zot-linux-amd64 serve test/cluster/config-minio1.json & ./bin/zot-linux-amd64 serve test/cluster/config-minio2.json & ./bin/zot-linux-amd64 serve test/cluster/config-minio3.json & - sleep 20 + sleep 10 + + # ensure the instances are online + curl --connect-timeout 3 --max-time 5 --retry 60 --retry-delay 1 --retry-max-time 120 --retry-connrefused http://localhost:8081/v2/ + curl --connect-timeout 3 --max-time 5 --retry 60 --retry-delay 1 --retry-max-time 120 --retry-connrefused http://localhost:8082/v2/ + curl --connect-timeout 3 --max-time 5 --retry 60 --retry-delay 1 --retry-max-time 120 --retry-connrefused http://localhost:8083/v2/ + # run tests skopeo --debug copy --format=oci --dest-tls-verify=false docker://ghcr.io/project-zot/golang:1.20 docker://localhost:8080/golang:1.20 skopeo --debug copy --src-tls-verify=false docker://localhost:8080/golang:1.20 oci:golang:1.20 @@ -139,46 +137,105 @@ jobs: oras pull --plain-http localhost:8080/hello-artifact:v2 -d -v grep -q "hello world" artifact.txt # should print "hello world" if [ $? -ne 0 ]; then \ - killall -r zot-*; \ + killall --wait -r zot-*; \ exit 1; \ fi - killall -r zot-* + echo "killing zot instances" + killall --wait -r zot-* + + # archive logs + zip logs-push-pull-bolt.zip /tmp/*.log -r + + # clean zot storage + sudo rm -rf /tmp/data/zot-storage/zot + # clean zot cache and metadb + sudo rm -rf /tmp/zot*/ + # clean zot logs + rm /tmp/*.log env: AWS_ACCESS_KEY_ID: minioadmin AWS_SECRET_ACCESS_KEY: minioadmin + - name: Upload zot logs for push-pull tests + uses: actions/upload-artifact@v4 + if: always() + with: + name: logs-push-pull-bolt + path: logs-push-pull-bolt.zip + if-no-files-found: error + - name: Run benchmark with --src-cidr arg run: | ./bin/zot-linux-amd64 serve test/cluster/config-minio1.json & ./bin/zot-linux-amd64 serve test/cluster/config-minio2.json & ./bin/zot-linux-amd64 serve test/cluster/config-minio3.json & - sleep 20 + sleep 10 + + # ensure the instances are online + curl --connect-timeout 3 --max-time 5 --retry 60 --retry-delay 1 --retry-max-time 120 --retry-connrefused http://localhost:8081/v2/ + curl --connect-timeout 3 --max-time 5 --retry 60 --retry-delay 1 --retry-max-time 120 --retry-connrefused http://localhost:8082/v2/ + curl --connect-timeout 3 --max-time 5 --retry 60 --retry-delay 1 --retry-max-time 120 --retry-connrefused http://localhost:8083/v2/ + # run zb with --src-cidr bin/zb-linux-amd64 -c 10 -n 50 -o ci-cd --src-cidr 127.0.0.0/8 http://localhost:8080 - killall -r zot-* + echo "killing zot instances" + killall --wait -r zot-* + + # archive logs + zip logs-src-cidr-bolt.zip /tmp/*.log -r # clean zot storage sudo rm -rf /tmp/data/zot-storage/zot + # clean zot cache and metadb + sudo rm -rf /tmp/zot*/ + # clean zot logs + rm /tmp/*.log env: AWS_ACCESS_KEY_ID: minioadmin AWS_SECRET_ACCESS_KEY: minioadmin + - name: Upload zot logs for cidr tests + uses: actions/upload-artifact@v4 + if: always() + with: + name: logs-src-cidr-bolt + path: logs-src-cidr-bolt.zip + if-no-files-found: error + - name: Run benchmark with --src-ips arg run: | ./bin/zot-linux-amd64 serve test/cluster/config-minio1.json & ./bin/zot-linux-amd64 serve test/cluster/config-minio2.json & ./bin/zot-linux-amd64 serve test/cluster/config-minio3.json & - sleep 20 + sleep 10 + + # ensure the instances are online + curl --connect-timeout 3 --max-time 5 --retry 60 --retry-delay 1 --retry-max-time 120 --retry-connrefused http://localhost:8081/v2/ + curl --connect-timeout 3 --max-time 5 --retry 60 --retry-delay 1 --retry-max-time 120 --retry-connrefused http://localhost:8082/v2/ + curl --connect-timeout 3 --max-time 5 --retry 60 --retry-delay 1 --retry-max-time 120 --retry-connrefused http://localhost:8083/v2/ + # run zb with --src-ips bin/zb-linux-amd64 -c 10 -n 50 -o ci-cd --src-ips 127.0.0.2,127.0.0.3,127.0.0.4,127.0.0.5,127.0.0.6,127.0.12.5,127.0.12.6 http://localhost:8080 - killall -r zot-* + echo "killing zot instances" + killall --wait -r zot-* + + # archive logs + zip logs-src-ips-bolt.zip /tmp/*.log -r env: AWS_ACCESS_KEY_ID: minioadmin AWS_SECRET_ACCESS_KEY: minioadmin + - name: Upload zot logs for src-ips tests + uses: actions/upload-artifact@v4 + if: always() + with: + name: logs-src-ips-bolt + path: logs-src-ips-bolt.zip + if-no-files-found: error + # Download previous benchmark result from cache (if exists) - name: Download previous benchmark data uses: actions/cache@v4 @@ -198,3 +255,256 @@ jobs: # Workflow will fail when an alert happens fail-on-alert: true # Upload the updated cache file for the next job by actions/cache + + minio-redis: + name: Stateless zot with minio and redis + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 + with: + cache: false + go-version: 1.23.x + - name: Install dependencies + run: | + cd $GITHUB_WORKSPACE + go install github.com/swaggo/swag/cmd/swag@v1.16.2 + go mod download + sudo apt-get update + sudo apt-get -y install rpm uidmap + # install skopeo + sudo apt-get -y install skopeo + + # install haproxy + sudo apt-get install haproxy + + - name: Build binaries + run: | + cd $GITHUB_WORKSPACE + make binary + make bench + make $PWD/hack/tools/bin/oras + + - name: Setup minio service + run: | + docker run -d -p 9000:9000 --name minio \ + -e "MINIO_ACCESS_KEY=minioadmin" \ + -e "MINIO_SECRET_KEY=minioadmin" \ + -v /tmp/data:/data \ + -v /tmp/config:/root/.minio \ + --health-cmd "curl http://localhost:9000/minio/health/live" \ + minio/minio:RELEASE.2024-07-16T23-46-41Z server /data + - name: Install py minio + run: pip3 install minio + + - name: Wait for minio to come up + run: | + curl --connect-timeout 5 \ + --max-time 120 \ + --retry 12 \ + --retry-max-time 120 \ + 'http://localhost:9000/minio/health/live' + + - name: Setup redis service + run: | + docker run -d -p 6379:6379 --name redis \ + --health-cmd "redis-cli ping" \ + --health-interval 10s \ + --health-timeout 5s \ + --health-retries 5 \ + redis:7.4.2 + + - name: Create minio bucket + run: | + python3 - <<'EOF' + from minio import Minio + + try: + minio = Minio( + 'localhost:9000', + access_key='minioadmin', + secret_key='minioadmin', + secure=False + ) + except Exception as ex: + raise + + minio.make_bucket('zot-storage') + print(f'{minio.list_buckets()}') + EOF + + - name: Run haproxy + run: | + sudo haproxy -d -f examples/cluster/haproxy.cfg -D + sleep 10 + + - name: Prepare configuration files + run: | + cp test/cluster/config-minio-redis.json test/cluster/config-minio1.json + sed -i 's/8081/8081/g' test/cluster/config-minio1.json + sed -i 's/\/tmp\/zot/\/tmp\/zot1/g' test/cluster/config-minio1.json + sed -i 's/\/dev\/null/\/tmp\/zot1.log/g' test/cluster/config-minio1.json + cp test/cluster/config-minio-redis.json test/cluster/config-minio2.json + sed -i 's/8081/8082/g' test/cluster/config-minio2.json + sed -i 's/\/tmp\/zot/\/tmp\/zot2/g' test/cluster/config-minio2.json + sed -i 's/\/dev\/null/\/tmp\/zot2.log/g' test/cluster/config-minio2.json + cp test/cluster/config-minio-redis.json test/cluster/config-minio3.json + sed -i 's/8081/8083/g' test/cluster/config-minio3.json + sed -i 's/\/tmp\/zot/\/tmp\/zot3/g' test/cluster/config-minio3.json + sed -i 's/\/dev\/null/\/tmp\/zot3.log/g' test/cluster/config-minio3.json + + - name: Free up disk space + uses: jlumbroso/free-disk-space@main + with: + # This might remove tools that are actually needed, if set to "true" but frees about 6 GB + tool-cache: true + # All of these default to true, but feel free to set to "false" if necessary for your workflow + android: true + dotnet: true + haskell: true + large-packages: true + swap-storage: true + + - name: Run push-pull tests + run: | + ./bin/zot-linux-amd64 serve test/cluster/config-minio1.json & + ./bin/zot-linux-amd64 serve test/cluster/config-minio2.json & + ./bin/zot-linux-amd64 serve test/cluster/config-minio3.json & + sleep 10 + + # ensure the instances are online + curl --connect-timeout 3 --max-time 5 --retry 60 --retry-delay 1 --retry-max-time 120 --retry-connrefused http://localhost:8081/v2/ + curl --connect-timeout 3 --max-time 5 --retry 60 --retry-delay 1 --retry-max-time 120 --retry-connrefused http://localhost:8082/v2/ + curl --connect-timeout 3 --max-time 5 --retry 60 --retry-delay 1 --retry-max-time 120 --retry-connrefused http://localhost:8083/v2/ + + # run tests + skopeo --debug copy --format=oci --dest-tls-verify=false docker://ghcr.io/project-zot/golang:1.20 docker://localhost:8080/golang:1.20 + skopeo --debug copy --src-tls-verify=false docker://localhost:8080/golang:1.20 oci:golang:1.20 + echo "{\"name\":\"foo\",\"value\":\"bar\"}" > config.json + echo "hello world" > artifact.txt + export PATH=$PATH:$PWD/hack/tools/bin + oras push --plain-http localhost:8080/hello-artifact:v2 \ + --config config.json:application/vnd.acme.rocket.config.v1+json \ + artifact.txt:text/plain -d -v + rm -f artifact.txt # first delete the file + oras pull --plain-http localhost:8080/hello-artifact:v2 -d -v + grep -q "hello world" artifact.txt # should print "hello world" + if [ $? -ne 0 ]; then \ + killall --wait -r zot-*; \ + exit 1; \ + fi + + echo "killing zot instances" + killall --wait -r zot-* + + # archive logs + zip logs-push-pull-redis.zip /tmp/*.log -r + + # clean zot storage + sudo rm -rf /tmp/data/zot-storage/zot + # clean zot cache and metadb + docker exec redis redis-cli FLUSHDB + # clean zot logs + rm /tmp/*.log + env: + AWS_ACCESS_KEY_ID: minioadmin + AWS_SECRET_ACCESS_KEY: minioadmin + + - name: Upload zot logs for push-pull tests + uses: actions/upload-artifact@v4 + if: always() + with: + name: logs-push-pull-redis + path: logs-push-pull-redis.zip + if-no-files-found: error + + - name: Run benchmark with --src-cidr arg + run: | + ./bin/zot-linux-amd64 serve test/cluster/config-minio1.json & + ./bin/zot-linux-amd64 serve test/cluster/config-minio2.json & + ./bin/zot-linux-amd64 serve test/cluster/config-minio3.json & + sleep 10 + + # ensure the instances are online + curl --connect-timeout 3 --max-time 5 --retry 60 --retry-delay 1 --retry-max-time 120 --retry-connrefused http://localhost:8081/v2/ + curl --connect-timeout 3 --max-time 5 --retry 60 --retry-delay 1 --retry-max-time 120 --retry-connrefused http://localhost:8082/v2/ + curl --connect-timeout 3 --max-time 5 --retry 60 --retry-delay 1 --retry-max-time 120 --retry-connrefused http://localhost:8083/v2/ + + # run zb with --src-cidr + bin/zb-linux-amd64 -c 10 -n 50 -o ci-cd --src-cidr 127.0.0.0/8 http://localhost:8080 + + echo "killing zot instances" + killall --wait -r zot-* + + # archive logs + zip logs-src-cidr-redis.zip /tmp/*.log -r + + # clean zot storage + sudo rm -rf /tmp/data/zot-storage/zot + # clean zot cache and metadb + docker exec redis redis-cli FLUSHDB + # clean zot logs + rm /tmp/*.log + env: + AWS_ACCESS_KEY_ID: minioadmin + AWS_SECRET_ACCESS_KEY: minioadmin + + - name: Upload zot logs for cidr tests + uses: actions/upload-artifact@v4 + if: always() + with: + name: logs-src-cidr-redis + path: logs-src-cidr-redis.zip + if-no-files-found: error + + - name: Run benchmark with --src-ips arg + run: | + ./bin/zot-linux-amd64 serve test/cluster/config-minio1.json & + ./bin/zot-linux-amd64 serve test/cluster/config-minio2.json & + ./bin/zot-linux-amd64 serve test/cluster/config-minio3.json & + sleep 10 + + # ensure the instances are online + curl --connect-timeout 3 --max-time 5 --retry 60 --retry-delay 1 --retry-max-time 120 --retry-connrefused http://localhost:8081/v2/ + curl --connect-timeout 3 --max-time 5 --retry 60 --retry-delay 1 --retry-max-time 120 --retry-connrefused http://localhost:8082/v2/ + curl --connect-timeout 3 --max-time 5 --retry 60 --retry-delay 1 --retry-max-time 120 --retry-connrefused http://localhost:8083/v2/ + + # run zb with --src-ips + bin/zb-linux-amd64 -c 10 -n 50 -o ci-cd --src-ips 127.0.0.2,127.0.0.3,127.0.0.4,127.0.0.5,127.0.0.6,127.0.12.5,127.0.12.6 http://localhost:8080 + + echo "killing zot instances" + killall --wait -r zot-* + + # archive logs + zip logs-src-ips-redis.zip /tmp/*.log -r + env: + AWS_ACCESS_KEY_ID: minioadmin + AWS_SECRET_ACCESS_KEY: minioadmin + + - name: Upload zot logs for src-ips tests + uses: actions/upload-artifact@v4 + if: always() + with: + name: logs-src-ips-redis + path: logs-src-ips-redis.zip + if-no-files-found: error + + # Download previous benchmark result from cache (if exists) + - name: Download previous benchmark data + uses: actions/cache@v4 + with: + path: ./cache + key: ${{ runner.os }}-gen1-benchmark-stateless-cluster-redis + # Run `github-action-benchmark` action + - name: Store benchmark result + uses: benchmark-action/github-action-benchmark@v1.20.4 + with: + # What benchmark tool the output.txt came from + tool: 'customBiggerIsBetter' + # Where the output from the benchmark tool is stored + output-file-path: ci-cd.json + # Where the previous data file is stored + external-data-json-path: ./cache/benchmark-data.json + # Workflow will fail when an alert happens + fail-on-alert: true + # Upload the updated cache file for the next job by actions/cache diff --git a/Makefile b/Makefile index 51395c90..0b1d8f37 100644 --- a/Makefile +++ b/Makefile @@ -511,6 +511,7 @@ run-blackbox-cloud-ci: check-blackbox-prerequisites check-awslocal binary $(BATS echo running cloud CI bats tests; \ $(BATS) $(BATS_FLAGS) test/blackbox/cloud_only.bats $(BATS) $(BATS_FLAGS) test/blackbox/sync_cloud.bats + $(BATS) $(BATS_FLAGS) test/blackbox/redis_s3.bats .PHONY: run-blackbox-dedupe-nightly run-blackbox-dedupe-nightly: check-blackbox-prerequisites check-awslocal binary binary-minimal diff --git a/examples/README.md b/examples/README.md index f817e078..a5f2f733 100644 --- a/examples/README.md +++ b/examples/README.md @@ -900,6 +900,36 @@ The following AWS policy is required by zot for caching blobs. Make sure to repl ] } +### Redis + +Redis is an alternative to BoltDB (which cannot be shared by multiple zot instances) and DynamoDB (requires access to AWS). +Redis can be set up using a configuration similar to the one below: + +```json + "storage": { + "rootDirectory": "/tmp/zot", + "remoteCache": true, + "cacheDriver": { + "name": "redis", + "url": "redis://localhost:6379", + "keyprefix": "zot" + } + } +``` + +The "name" setting selects the Redis driver implementation. +The "keyprefix" is a string prepended to all Redis keys created by this zot instance. +The "url" setting points to the Redis server (or servers in the case of a Redis cluster). +More details on how this is parsed are available at: +- https://github.com/redis/go-redis/blob/v9.7.0/options.go#L247 +- https://github.com/redis/go-redis/blob/v9.7.0/osscluster.go#L144 + +If the "url" setting is missing, the parameters need to be passed individually as keys in the same "cacheDriver" map. +The keys are the same as the attributes that would otherwise be included in the "url". +Note that at this time the library we import only supports "url" parsing in the case of a Redis single instance, or cluster configuration. +In the case of a Redis Sentinel setup, you would need to add each key manually in the "cacheDriver" map and make sure to specify +a "master_name" key, see https://github.com/redis/go-redis/blob/v9.7.0/universal.go#L240 + ## Sync Enable and configure sync with: diff --git a/examples/config-redis-all-options.json b/examples/config-redis-all-options.json new file mode 100644 index 00000000..403cc6ec --- /dev/null +++ b/examples/config-redis-all-options.json @@ -0,0 +1,68 @@ +{ + "distSpecVersion": "1.1.0", + "storage": { + "dedupe": true, + "gc": true, + "rootDirectory": "/tmp/zot", + "cacheDriver": { + "name": "redis", + "keyprefix": "zot", + "addr": ["host1:6379", "host2:6379", "host3:6379"], + "client_name": "client", + "db": 1, + "protocol": 3, + "username": "user1", + "password": "pass1", + "sentinel_username": "user2", + "sentinel_password": "pass2", + "max_retries": 3, + "min_retry_backoff": "5s", + "max_retry_backoff": "5s", + "dial_timeout": "5s", + "read_timeout": "5s", + "write_timeout": "5s", + "context_timeout_enabled": false, + "pool_fifo": false, + "pool_size": 3, + "pool_timeout": "5s", + "min_idle_conns": 1, + "max_idle_conns": 3, + "max_active_conns": 3, + "conn_max_idle_time": "5s", + "conn_max_lifetime": "5s", + "max_redirects": 3, + "read_only": false, + "route_by_latency": false, + "route_randomly": false, + "master_name": "zotmeta", + "disable_identity": false, + "identity_suffix": "zotmeta", + "unstable_resp3": false + }, + "storageDriver": { + "name": "s3", + "rootdirectory": "/zot", + "region": "us-east-2", + "regionendpoint": "localhost:4566", + "bucket": "zot-storage", + "forcepathstyle": true, + "secure": false, + "skipverify": false + } + }, + "http": { + "address": "0.0.0.0", + "port": "8484" + }, + "log": { + "level": "debug" + }, + "extensions": { + "ui": { + "enable": true + }, + "search": { + "enable": true + } + } +} diff --git a/examples/config-redis-cluster.json b/examples/config-redis-cluster.json new file mode 100644 index 00000000..27349565 --- /dev/null +++ b/examples/config-redis-cluster.json @@ -0,0 +1,38 @@ +{ + "distSpecVersion": "1.1.0", + "storage": { + "dedupe": true, + "gc": true, + "rootDirectory": "/tmp/zot", + "cacheDriver": { + "name": "redis", + "url": "redis://user:password@host1:6379?dial_timeout=3&read_timeout=6s&addr=host2:6379&addr=host3:6379", + "keyprefix": "zot" + }, + "storageDriver": { + "name": "s3", + "rootdirectory": "/zot", + "region": "us-east-2", + "regionendpoint": "localhost:4566", + "bucket": "zot-storage", + "forcepathstyle": true, + "secure": false, + "skipverify": false + } + }, + "http": { + "address": "0.0.0.0", + "port": "8484" + }, + "log": { + "level": "debug" + }, + "extensions": { + "ui": { + "enable": true + }, + "search": { + "enable": true + } + } +} diff --git a/examples/config-redis.json b/examples/config-redis.json new file mode 100644 index 00000000..0e0e5e87 --- /dev/null +++ b/examples/config-redis.json @@ -0,0 +1,38 @@ +{ + "distSpecVersion": "1.1.0", + "storage": { + "dedupe": true, + "gc": true, + "rootDirectory": "/tmp/zot", + "cacheDriver": { + "name": "redis", + "url": "redis://localhost:6379", + "keyprefix": "zot" + }, + "storageDriver": { + "name": "s3", + "rootdirectory": "/zot", + "region": "us-east-2", + "regionendpoint": "localhost:4566", + "bucket": "zot-storage", + "forcepathstyle": true, + "secure": false, + "skipverify": false + } + }, + "http": { + "address": "0.0.0.0", + "port": "8484" + }, + "log": { + "level": "debug" + }, + "extensions": { + "ui": { + "enable": true + }, + "search": { + "enable": true + } + } +} diff --git a/go.mod b/go.mod index 9b186fca..8c83721b 100644 --- a/go.mod +++ b/go.mod @@ -7,6 +7,7 @@ toolchain go1.23.4 require ( github.com/99designs/gqlgen v0.17.63 github.com/Masterminds/semver v1.5.0 + github.com/alicebob/miniredis/v2 v2.34.0 github.com/aquasecurity/trivy v0.58.2 github.com/aquasecurity/trivy-db v0.0.0-20241209111357-8c398f13db0e github.com/aws/aws-sdk-go v1.55.6 @@ -29,6 +30,8 @@ require ( github.com/dustin/go-humanize v1.0.1 github.com/fsnotify/fsnotify v1.8.0 github.com/go-ldap/ldap/v3 v3.4.10 + github.com/go-redis/redismock/v9 v9.2.0 + github.com/go-redsync/redsync/v4 v4.13.0 github.com/gofrs/uuid v4.4.0+incompatible github.com/google/go-containerregistry v0.20.3 github.com/google/go-github/v62 v62.0.0 @@ -51,10 +54,12 @@ require ( github.com/project-zot/mockoidc v0.0.0-20240610203808-d69d9e02020a github.com/prometheus/client_golang v1.20.5 github.com/prometheus/client_model v0.6.1 + github.com/redis/go-redis/v9 v9.7.0 github.com/rs/zerolog v1.33.0 github.com/sigstore/cosign/v2 v2.4.1 github.com/sigstore/sigstore v1.8.12 github.com/smartystreets/goconvey v1.8.1 + github.com/spf13/cast v1.7.0 github.com/spf13/cobra v1.8.1 github.com/spf13/viper v1.19.0 github.com/stretchr/testify v1.10.0 @@ -140,6 +145,7 @@ require ( github.com/alibabacloud-go/tea-utils v1.4.5 // indirect github.com/alibabacloud-go/tea-utils/v2 v2.0.6 // indirect github.com/alibabacloud-go/tea-xml v1.1.3 // indirect + github.com/alicebob/gopher-json v0.0.0-20230218143504-906a9b012302 // indirect github.com/aliyun/credentials-go v1.3.6 // indirect github.com/anchore/go-struct-converter v0.0.0-20230627203149-c72ef8859ca9 // indirect github.com/apparentlymart/go-cidr v1.1.0 // indirect @@ -270,6 +276,7 @@ require ( github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.4 // indirect github.com/golang/snappy v0.0.4 // indirect + github.com/gomodule/redigo v2.0.0+incompatible // indirect github.com/google/btree v1.1.2 // indirect github.com/google/certificate-transparency-go v1.2.1 // indirect github.com/google/gnostic-models v0.6.9-0.20230804172637-c7be7c783f49 // indirect @@ -426,7 +433,6 @@ require ( github.com/sourcegraph/conc v0.3.0 // indirect github.com/spdx/tools-golang v0.5.5 // indirect github.com/spf13/afero v1.11.0 // indirect - github.com/spf13/cast v1.7.0 // indirect github.com/spf13/pflag v1.0.5 // indirect github.com/spiffe/go-spiffe/v2 v2.3.0 // indirect github.com/stefanberger/go-pkcs11uri v0.0.0-20230803200340-78284954bff6 // indirect @@ -462,6 +468,7 @@ require ( github.com/xlab/treeprint v1.2.0 // indirect github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 // indirect github.com/yashtewari/glob-intersection v0.2.0 // indirect + github.com/yuin/gopher-lua v1.1.1 // indirect github.com/zclconf/go-cty v1.15.0 // indirect github.com/zclconf/go-cty-yaml v1.1.0 // indirect github.com/zeebo/errs v1.3.0 // indirect diff --git a/go.sum b/go.sum index 954a86d7..3a0f9543 100644 --- a/go.sum +++ b/go.sum @@ -369,10 +369,10 @@ github.com/alibabacloud-go/tea-utils/v2 v2.0.6/go.mod h1:qxn986l+q33J5VkialKMqT/ github.com/alibabacloud-go/tea-xml v1.1.2/go.mod h1:Rq08vgCcCAjHyRi/M7xlHKUykZCEtyBy9+DPF6GgEu8= github.com/alibabacloud-go/tea-xml v1.1.3 h1:7LYnm+JbOq2B+T/B0fHC4Ies4/FofC4zHzYtqw7dgt0= github.com/alibabacloud-go/tea-xml v1.1.3/go.mod h1:Rq08vgCcCAjHyRi/M7xlHKUykZCEtyBy9+DPF6GgEu8= -github.com/alicebob/gopher-json v0.0.0-20200520072559-a9ecdc9d1d3a h1:HbKu58rmZpUGpz5+4FfNmIU+FmZg2P3Xaj2v2bfNWmk= -github.com/alicebob/gopher-json v0.0.0-20200520072559-a9ecdc9d1d3a/go.mod h1:SGnFV6hVsYE877CKEZ6tDNTjaSXYUk6QqoIK6PrAtcc= -github.com/alicebob/miniredis/v2 v2.33.0 h1:uvTF0EDeu9RLnUEG27Db5I68ESoIxTiXbNUiji6lZrA= -github.com/alicebob/miniredis/v2 v2.33.0/go.mod h1:MhP4a3EU7aENRi9aO+tHfTBZicLqQevyi/DJpoj6mi0= +github.com/alicebob/gopher-json v0.0.0-20230218143504-906a9b012302 h1:uvdUDbHQHO85qeSydJtItA4T55Pw6BtAejd0APRJOCE= +github.com/alicebob/gopher-json v0.0.0-20230218143504-906a9b012302/go.mod h1:SGnFV6hVsYE877CKEZ6tDNTjaSXYUk6QqoIK6PrAtcc= +github.com/alicebob/miniredis/v2 v2.34.0 h1:mBFWMaJSNL9RwdGRyEDoAAv8OQc5UlEhLDQggTglU/0= +github.com/alicebob/miniredis/v2 v2.34.0/go.mod h1:kWShP4b58T1CW0Y5dViCd5ztzrDqRWqM3nksiyXk5s8= github.com/aliyun/credentials-go v1.1.2/go.mod h1:ozcZaMR5kLM7pwtCMEpVmQ242suV6qTJya2bDq4X1Tw= github.com/aliyun/credentials-go v1.3.6 h1:K5STbhaWjoj5Ht0juOj9mWE2lGelShHLzu5QR3cQ5X8= github.com/aliyun/credentials-go v1.3.6/go.mod h1:1LxUuX7L5YrZUWzBrRyk0SwSdH4OmPrib8NVePL3fxM= @@ -497,6 +497,10 @@ github.com/briandowns/spinner v1.23.2 h1:Zc6ecUnI+YzLmJniCfDNaMbW0Wid1d5+qcTq4L2 github.com/briandowns/spinner v1.23.2/go.mod h1:LaZeM4wm2Ywy6vO571mvhQNRcWfRUnXOs0RcKV0wYKM= github.com/bshuster-repo/logrus-logstash-hook v1.0.0 h1:e+C0SB5R1pu//O4MQ3f9cFuPGoOVeF2fE4Og9otCc70= github.com/bshuster-repo/logrus-logstash-hook v1.0.0/go.mod h1:zsTqEiSzDgAa/8GZR7E1qaXrhYNDKBYy5/dWPTIflbk= +github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs= +github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c= +github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA= +github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0= github.com/buildkite/agent/v3 v3.81.0 h1:JVfkng2XnsXesFXwiFwLJFkuzVu4zvoJCvedfoIXD6E= github.com/buildkite/agent/v3 v3.81.0/go.mod h1:edJeyycODRxaFvpT22rDGwaQ5oa4eB8GjtbjgX5VpFw= github.com/buildkite/go-pipeline v0.13.1 h1:Y9p8pQIwPtauVwNrcmTDH6+XK7jE1nLuvWVaK8oymA8= @@ -776,8 +780,16 @@ github.com/go-pkgz/expirable-cache/v3 v3.0.0 h1:u3/gcu3sabLYiTCevoRKv+WzjIn5oo7P github.com/go-pkgz/expirable-cache/v3 v3.0.0/go.mod h1:2OQiDyEGQalYecLWmXprm3maPXeVb5/6/X7yRPYTzec= github.com/go-quicktest/qt v1.101.0 h1:O1K29Txy5P2OK0dGo59b7b0LR6wKfIhttaAhHUyn7eI= github.com/go-quicktest/qt v1.101.0/go.mod h1:14Bz/f7NwaXPtdYEgzsx46kqSxVwTbzVZsDC26tQJow= +github.com/go-redis/redis v6.15.9+incompatible h1:K0pv1D7EQUjfyoMql+r/jZqCLizCGKFlFgcHWWmHQjg= +github.com/go-redis/redis v6.15.9+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA= +github.com/go-redis/redis/v7 v7.4.1 h1:PASvf36gyUpr2zdOUS/9Zqc80GbM+9BDyiJSJDDOrTI= +github.com/go-redis/redis/v7 v7.4.1/go.mod h1:JDNMw23GTyLNC4GZu9njt15ctBQVn7xjRfnwdHj/Dcg= github.com/go-redis/redis/v8 v8.11.5 h1:AcZZR7igkdvfVmQTPnu9WE37LRrO/YrBH5zWyjDC0oI= github.com/go-redis/redis/v8 v8.11.5/go.mod h1:gREzHqY1hg6oD9ngVRbLStwAWKhA0FEgq8Jd4h5lpwo= +github.com/go-redis/redismock/v9 v9.2.0 h1:ZrMYQeKPECZPjOj5u9eyOjg8Nnb0BS9lkVIZ6IpsKLw= +github.com/go-redis/redismock/v9 v9.2.0/go.mod h1:18KHfGDK4Y6c2R0H38EUGWAdc7ZQS9gfYxc94k7rWT0= +github.com/go-redsync/redsync/v4 v4.13.0 h1:49X6GJfnbLGaIpBBREM/zA4uIMDXKAh1NDkvQ1EkZKA= +github.com/go-redsync/redsync/v4 v4.13.0/go.mod h1:HMW4Q224GZQz6x1Xc7040Yfgacukdzu7ifTDAKiyErQ= github.com/go-rod/rod v0.116.2 h1:A5t2Ky2A+5eD/ZJQr1EfsQSe5rms5Xof/qj296e+ZqA= github.com/go-rod/rod v0.116.2/go.mod h1:H+CMO9SCNc2TJ2WfrG+pKhITz57uGNYU43qYHh438Mg= github.com/go-sql-driver/mysql v1.8.1 h1:LedoTUt/eveggdHS9qUFC1EFSa8bU2+1pZjSRpvNJ1Y= @@ -854,6 +866,8 @@ github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6 github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/gomodule/redigo v2.0.0+incompatible h1:K/R+8tc58AaqLkqG2Ol3Qk+DR/TlNuhuh457pBFPtt0= +github.com/gomodule/redigo v2.0.0+incompatible/go.mod h1:B4C85qUVwatsJoIUNIfCRsp7qO0iAmpGFZ4EELWSbC4= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.1.2 h1:xf4v41cLI2Z6FxbKm+8Bu+m8ifhj15JuZ9sa0jZCMUU= @@ -1353,8 +1367,10 @@ github.com/redis/go-redis/extra/rediscmd/v9 v9.0.5 h1:EaDatTxkdHG+U3Bk4EUr+DZ7fO github.com/redis/go-redis/extra/rediscmd/v9 v9.0.5/go.mod h1:fyalQWdtzDBECAQFBJuQe5bzQ02jGd5Qcbgb97Flm7U= github.com/redis/go-redis/extra/redisotel/v9 v9.0.5 h1:EfpWLLCyXw8PSM2/XNJLjI3Pb27yVE+gIAfeqp8LUCc= github.com/redis/go-redis/extra/redisotel/v9 v9.0.5/go.mod h1:WZjPDy7VNzn77AAfnAfVjZNvfJTYfPetfZk5yoSTLaQ= -github.com/redis/go-redis/v9 v9.6.1 h1:HHDteefn6ZkTtY5fGUE8tj8uy85AHk6zP7CpzIAM0y4= -github.com/redis/go-redis/v9 v9.6.1/go.mod h1:0C0c6ycQsdpVNQpxb1njEQIqkx5UcsM8FJCQLgE9+RA= +github.com/redis/go-redis/v9 v9.7.0 h1:HhLSs+B6O021gwzl+locl0zEDnyNkxMtf/Z3NNBMa9E= +github.com/redis/go-redis/v9 v9.7.0/go.mod h1:f6zhXITC7JUJIlPEiBOTXxJgPLdZcA93GewI7inzyWw= +github.com/redis/rueidis v1.0.19 h1:s65oWtotzlIFN8eMPhyYwxlwLR1lUdhza2KtWprKYSo= +github.com/redis/rueidis v1.0.19/go.mod h1:8B+r5wdnjwK3lTFml5VtxjzGOQAC+5UmujoD12pDrEo= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= @@ -1494,6 +1510,8 @@ github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXl github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/stvp/tempredis v0.0.0-20181119212430-b82af8480203 h1:QVqDTf3h2WHt08YuiTGPZLls0Wq99X9bWd0Q5ZSBesM= +github.com/stvp/tempredis v0.0.0-20181119212430-b82af8480203/go.mod h1:oqN97ltKNihBbwlX8dLpwxCl3+HnXKV/R0e+sRLd9C8= github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= github.com/swaggo/files v1.0.1 h1:J1bVJ4XHZNq0I46UU90611i9/YzdrF7x92oX1ig5IdE= diff --git a/pkg/api/config/redis/redis.go b/pkg/api/config/redis/redis.go new file mode 100644 index 00000000..7dc76cd0 --- /dev/null +++ b/pkg/api/config/redis/redis.go @@ -0,0 +1,295 @@ +package rediscfg + +import ( + "context" + "fmt" + "strings" + "sync" + "time" + + "github.com/redis/go-redis/v9" + "github.com/spf13/cast" + + "zotregistry.dev/zot/errors" + "zotregistry.dev/zot/pkg/log" +) + +var once sync.Once //nolint: gochecknoglobals // redis.SetLogger modifies an unprotected global variable + +type redisLogger struct { + log log.Logger +} + +func (r redisLogger) Printf(ctx context.Context, format string, v ...interface{}) { + r.log.Debug().Msgf(format, v...) +} + +func GetRedisClient(redisConfig map[string]interface{}, log log.Logger) (redis.UniversalClient, error) { + once.Do(func() { redis.SetLogger(redisLogger{log}) }) // call redis.SetLogger only once + + // go-redis supports connecting via the redis uri specification (more convenient than parameter parsing) + // Note failover/Sentinel cannot be configured via URL parsing at the moment + if val, ok := redisConfig["url"]; ok { + str, ok := val.(string) + if !ok { + return nil, fmt.Errorf("%w: cachedriver %s has invalid value for url", errors.ErrBadConfig, redisConfig) + } + + // The cluster URL has additional addresses in query parameters + if strings.Count(str, "addr") > 0 { + opts, err := redis.ParseClusterURL(str) + if err != nil { + return nil, err + } + + return redis.NewClusterClient(opts), nil + } + + opts, err := redis.ParseURL(str) + if err != nil { + return nil, err + } + + return redis.NewClient(opts), nil + } + + // URL configuration not provided by the user, we need to initialize UniversalOptions based on the provided parameters + opts := ParseRedisUniversalOptions(redisConfig, log) + + return redis.NewUniversalClient(opts), nil +} + +func ParseRedisUniversalOptions(redisConfig map[string]interface{}, //nolint: gocyclo + log log.Logger, +) *redis.UniversalOptions { + opts := redis.UniversalOptions{} + sanitizedConfig := map[string]interface{}{} + + for key, val := range redisConfig { + if key == "password" || key == "sentinel_password" { + sanitizedConfig[key] = "******" + + continue + } + + sanitizedConfig[key] = val + } + + log.Info().Interface("redisConfig", sanitizedConfig).Msg("parsing redis universal options") + + if val, ok := getStringSlice(redisConfig, "addr", log); ok { + opts.Addrs = val + } + + if val, ok := getString(redisConfig, "client_name", false, log); ok { + opts.ClientName = val + } + + if val, ok := getInt(redisConfig, "db", log); ok { + opts.DB = val + } + + if val, ok := getInt(redisConfig, "protocol", log); ok { + opts.Protocol = val + } + + if val, ok := getString(redisConfig, "username", false, log); ok { + opts.Username = val + } + + if val, ok := getString(redisConfig, "password", true, log); ok { + opts.Password = val + } + + if val, ok := getString(redisConfig, "sentinel_username", false, log); ok { + opts.SentinelUsername = val + } + + if val, ok := getString(redisConfig, "sentinel_password", true, log); ok { + opts.SentinelPassword = val + } + + if val, ok := getInt(redisConfig, "max_retries", log); ok { + opts.MaxRetries = val + } + + if val, ok := getDuration(redisConfig, "min_retry_backoff", log); ok { + opts.MinRetryBackoff = val + } + + if val, ok := getDuration(redisConfig, "max_retry_backoff", log); ok { + opts.MaxRetryBackoff = val + } + + if val, ok := getDuration(redisConfig, "dial_timeout", log); ok { + opts.DialTimeout = val + } + + if val, ok := getDuration(redisConfig, "read_timeout", log); ok { + opts.ReadTimeout = val + } + + if val, ok := getDuration(redisConfig, "write_timeout", log); ok { + opts.WriteTimeout = val + } + + if val, ok := getBool(redisConfig, "context_timeout_enabled", log); ok { + opts.ContextTimeoutEnabled = val + } + + if val, ok := getBool(redisConfig, "pool_fifo", log); ok { + opts.PoolFIFO = val + } + + if val, ok := getInt(redisConfig, "pool_size", log); ok { + opts.PoolSize = val + } + + if val, ok := getDuration(redisConfig, "pool_timeout", log); ok { + opts.PoolTimeout = val + } + + if val, ok := getInt(redisConfig, "min_idle_conns", log); ok { + opts.MinIdleConns = val + } + + if val, ok := getInt(redisConfig, "max_idle_conns", log); ok { + opts.MaxIdleConns = val + } + + if val, ok := getInt(redisConfig, "max_active_conns", log); ok { + opts.MaxActiveConns = val + } + + if val, ok := getDuration(redisConfig, "conn_max_idle_time", log); ok { + opts.ConnMaxIdleTime = val + } + + if val, ok := getDuration(redisConfig, "conn_max_lifetime", log); ok { + opts.ConnMaxLifetime = val + } + + if val, ok := getInt(redisConfig, "max_redirects", log); ok { + opts.MaxRedirects = val + } + + if val, ok := getBool(redisConfig, "read_only", log); ok { + opts.ReadOnly = val + } + + if val, ok := getBool(redisConfig, "route_by_latency", log); ok { + opts.RouteByLatency = val + } + + if val, ok := getBool(redisConfig, "route_randomly", log); ok { + opts.RouteRandomly = val + } + + if val, ok := getString(redisConfig, "master_name", false, log); ok { + opts.MasterName = val + } + + if val, ok := getBool(redisConfig, "disable_identity", log); ok { + opts.DisableIndentity = val + } + + if val, ok := getString(redisConfig, "identity_suffix", false, log); ok { + opts.IdentitySuffix = val + } + + if val, ok := getBool(redisConfig, "unstable_resp3", log); ok { + opts.UnstableResp3 = val + } + + log.Info().Msg("finished parsing redis universal options") + + return &opts +} + +func logCastWarning(key string, value interface{}, hideValue bool, log log.Logger) { + if hideValue { + log.Warn().Str("key", key).Msg("failed to cast parameter to intended type") + } else { + log.Warn().Str("key", key).Interface("value", value).Msg("failed to cast parameter to intended type") + } +} + +func getBool(dict map[string]interface{}, key string, log log.Logger) (bool, bool) { + value, ok := dict[key] + if !ok { + return false, false + } + + ret, err := cast.ToBoolE(value) + if err != nil { + logCastWarning(key, value, false, log) + + return false, false + } + + return ret, true +} + +func getInt(dict map[string]interface{}, key string, log log.Logger) (int, bool) { + value, ok := dict[key] + if !ok { + return 0, false + } + + ret, err := cast.ToIntE(value) + if err != nil { + logCastWarning(key, value, false, log) + + return 0, false + } + + return ret, true +} + +func getString(dict map[string]interface{}, key string, hideValue bool, log log.Logger) (string, bool) { + value, ok := dict[key] + if !ok { + return "", false + } + + ret, err := cast.ToStringE(value) + if err != nil { + logCastWarning(key, value, hideValue, log) + + return "", false + } + + return ret, true +} + +func getStringSlice(dict map[string]interface{}, key string, log log.Logger) ([]string, bool) { + value, ok := dict[key] + if !ok { + return []string{}, false + } + + ret, err := cast.ToStringSliceE(value) + if err != nil { + logCastWarning(key, value, false, log) + + return []string{}, false + } + + return ret, true +} + +func getDuration(dict map[string]interface{}, key string, log log.Logger) (time.Duration, bool) { + value, ok := dict[key] + if !ok { + return 0, false + } + + ret, err := cast.ToDurationE(value) + if err != nil { + logCastWarning(key, value, false, log) + + return 0, false + } + + return ret, true +} diff --git a/pkg/api/config/redis/redis_test.go b/pkg/api/config/redis/redis_test.go new file mode 100644 index 00000000..ffb7e294 --- /dev/null +++ b/pkg/api/config/redis/redis_test.go @@ -0,0 +1,340 @@ +package rediscfg_test + +import ( + "os" + "path" + "testing" + "time" + + "github.com/redis/go-redis/v9" + . "github.com/smartystreets/goconvey/convey" + + "zotregistry.dev/zot/pkg/api/config" + rediscfg "zotregistry.dev/zot/pkg/api/config/redis" + "zotregistry.dev/zot/pkg/cli/server" + "zotregistry.dev/zot/pkg/log" +) + +func TestRedisOptions(t *testing.T) { + Convey("Test redis initialization", t, func() { + log := log.NewLogger("debug", "") + So(log, ShouldNotBeNil) + + Convey("Test redis url parsing", func() { + // Errors + config := map[string]interface{}{"url": false} + + clientIntf, err := rediscfg.GetRedisClient(config, log) + So(err, ShouldNotBeNil) + So(clientIntf, ShouldBeNil) + + config = map[string]interface{}{"url": ""} + + clientIntf, err = rediscfg.GetRedisClient(config, log) + So(err, ShouldNotBeNil) + So(clientIntf, ShouldBeNil) + + config = map[string]interface{}{"url": "qwerty@localhost:6379/1?dial_timeout=5s"} + + clientIntf, err = rediscfg.GetRedisClient(config, log) + So(err, ShouldNotBeNil) + So(clientIntf, ShouldBeNil) + + config = map[string]interface{}{"url": "http://:qwerty@localhost:6379/1?dial_timeout=5s"} + + clientIntf, err = rediscfg.GetRedisClient(config, log) + So(err, ShouldNotBeNil) + So(clientIntf, ShouldBeNil) + + config = map[string]interface{}{"url": "http://localhost:6379/1?addr=host2:6379&addr=host1:6379"} + + clientIntf, err = rediscfg.GetRedisClient(config, log) + So(err, ShouldNotBeNil) + So(clientIntf, ShouldBeNil) + + // Success + config = map[string]interface{}{"url": "redis://user:password@localhost:6379/1?dial_timeout=5s"} + + clientIntf, err = rediscfg.GetRedisClient(config, log) + So(err, ShouldBeNil) + So(clientIntf, ShouldNotBeNil) + + _, ok := clientIntf.(*redis.Client) + So(ok, ShouldBeTrue) + + config = map[string]interface{}{"url": "redis://user:password@host1:6379?addr=host2:6379&addr=host1:6379"} + + clientIntf, err = rediscfg.GetRedisClient(config, log) + So(err, ShouldBeNil) + So(clientIntf, ShouldNotBeNil) + + _, ok = clientIntf.(*redis.ClusterClient) + So(ok, ShouldBeTrue) + }) + + Convey("Test empty redis options from struct successfully", func() { + config := map[string]interface{}{} + + // All attributes will have zero values + options := rediscfg.ParseRedisUniversalOptions(config, log) + So(options, ShouldNotBeNil) + So(options.Addrs, ShouldEqual, []string(nil)) + So(options.DB, ShouldEqual, 0) + So(options.MasterName, ShouldEqual, "") + So(options.ClientName, ShouldEqual, "") + So(options.Protocol, ShouldEqual, 0) + So(options.Username, ShouldEqual, "") + So(options.Password, ShouldEqual, "") + So(options.SentinelUsername, ShouldEqual, "") + So(options.SentinelPassword, ShouldEqual, "") + So(options.DialTimeout, ShouldEqual, 0) + So(options.MaxRetries, ShouldEqual, 0) + So(options.MinRetryBackoff, ShouldEqual, 0) + So(options.MaxRetryBackoff, ShouldEqual, 0) + So(options.ReadTimeout, ShouldEqual, 0) + So(options.WriteTimeout, ShouldEqual, 0) + So(options.ContextTimeoutEnabled, ShouldEqual, false) + So(options.PoolFIFO, ShouldEqual, false) + So(options.PoolSize, ShouldEqual, 0) + So(options.PoolTimeout, ShouldEqual, 0) + So(options.MinIdleConns, ShouldEqual, 0) + So(options.MaxIdleConns, ShouldEqual, 0) + So(options.MaxActiveConns, ShouldEqual, 0) + So(options.ConnMaxIdleTime, ShouldEqual, 0) + So(options.ConnMaxLifetime, ShouldEqual, 0) + So(options.MaxRedirects, ShouldEqual, 0) + So(options.ReadOnly, ShouldEqual, false) + So(options.RouteByLatency, ShouldEqual, false) + So(options.RouteRandomly, ShouldEqual, false) + So(options.DisableIndentity, ShouldEqual, false) + So(options.IdentitySuffix, ShouldEqual, "") + So(options.UnstableResp3, ShouldEqual, false) + + clientIntf, err := rediscfg.GetRedisClient(config, log) + So(err, ShouldBeNil) + So(clientIntf, ShouldNotBeNil) + + _, ok := clientIntf.(*redis.Client) + So(ok, ShouldBeTrue) + }) + + Convey("Test redis options from struct successfully", func() { + config := map[string]interface{}{ + "addr": []string{ + "a.repo:26379", + "b.repo:26379", + "c.repo:26379", + }, + "db": 1, + "master_name": "zotmeta", + "client_name": "client", + "protocol": 3, + "username": "redis", + "password": "**secret**", + "sentinel_username": "sentinel", + "sentinel_password": "**secret**", + "dial_timeout": 5 * time.Second, + "max_retries": 5, + "min_retry_backoff": 1 * time.Second, + "max_retry_backoff": 3 * time.Second, + "read_timeout": 1 * time.Second, + "write_timeout": 1 * time.Second, + "context_timeout_enabled": true, + "pool_fifo": false, + "pool_size": 2, + "pool_timeout": 10 * time.Second, + "min_idle_conns": 1, + "max_idle_conns": 2, + "max_active_conns": 3, + "conn_max_idle_time": 20 * time.Second, + "conn_max_lifetime": 50 * time.Second, + "max_redirects": 3, + "read_only": true, + "route_by_latency": false, + "route_randomly": true, + "disable_identity": false, + "identity_suffix": "test", + "unstable_resp3": true, + } + + // All attribute values are taken from config + options := rediscfg.ParseRedisUniversalOptions(config, log) + So(options, ShouldNotBeNil) + So(options.Addrs, ShouldEqual, []string{"a.repo:26379", "b.repo:26379", "c.repo:26379"}) + So(options.DB, ShouldEqual, 1) + So(options.MasterName, ShouldEqual, "zotmeta") + So(options.ClientName, ShouldEqual, "client") + So(options.Protocol, ShouldEqual, 3) + So(options.Username, ShouldEqual, "redis") + So(options.Password, ShouldEqual, "**secret**") + So(options.SentinelUsername, ShouldEqual, "sentinel") + So(options.SentinelPassword, ShouldEqual, "**secret**") + So(options.DialTimeout, ShouldEqual, 5*time.Second) + So(options.MaxRetries, ShouldEqual, 5) + So(options.MinRetryBackoff, ShouldEqual, 1*time.Second) + So(options.MaxRetryBackoff, ShouldEqual, 3*time.Second) + So(options.ReadTimeout, ShouldEqual, 1*time.Second) + So(options.WriteTimeout, ShouldEqual, 1*time.Second) + So(options.ContextTimeoutEnabled, ShouldEqual, true) + So(options.PoolFIFO, ShouldEqual, false) + So(options.PoolSize, ShouldEqual, 2) + So(options.PoolTimeout, ShouldEqual, 10*time.Second) + So(options.MinIdleConns, ShouldEqual, 1) + So(options.MaxIdleConns, ShouldEqual, 2) + So(options.MaxActiveConns, ShouldEqual, 3) + So(options.ConnMaxIdleTime, ShouldEqual, 20*time.Second) + So(options.ConnMaxLifetime, ShouldEqual, 50*time.Second) + So(options.MaxRedirects, ShouldEqual, 3) + So(options.ReadOnly, ShouldEqual, true) + So(options.RouteByLatency, ShouldEqual, false) + So(options.RouteRandomly, ShouldEqual, true) + So(options.DisableIndentity, ShouldEqual, false) + So(options.IdentitySuffix, ShouldEqual, "test") + So(options.UnstableResp3, ShouldEqual, true) + + clientIntf, err := rediscfg.GetRedisClient(config, log) + So(err, ShouldBeNil) + So(clientIntf, ShouldNotBeNil) + + _, ok := clientIntf.(*redis.Client) + So(ok, ShouldBeTrue) + }) + + Convey("Test redis options from struct with warnings", func() { + config := map[string]interface{}{ + "addr": map[string]int{}, + "db": "somestring", + "master_name": map[string]int{}, + "client_name": map[string]int{}, + "protocol": "somestring", + "username": map[string]int{}, + "password": map[string]int{}, + "sentinel_username": map[string]int{}, + "sentinel_password": map[string]int{}, + "dial_timeout": "somestring", + "max_retries": "somestring", + "min_retry_backoff": "somestring", + "max_retry_backoff": "somestring", + "read_timeout": false, + "write_timeout": true, + "context_timeout_enabled": "somestring", + "pool_fifo": "somestring", + "pool_size": "somestring", + "pool_timeout": "somestring", + "min_idle_conns": map[string]int{}, + "max_idle_conns": map[string]int{}, + "max_active_conns": "somestring", + "conn_max_idle_time": "somestring", + "conn_max_lifetime": "somestring", + "max_redirects": map[string]int{}, + "read_only": map[string]int{}, + "route_by_latency": "somestring", + "route_randomly": map[string]int{}, + "disable_identity": "somestring", + "identity_suffix": map[string]int{}, + "unstable_resp3": "somestring", + } + + // All attributes remain with default values + options := rediscfg.ParseRedisUniversalOptions(config, log) + So(options, ShouldNotBeNil) + So(options.Addrs, ShouldEqual, []string(nil)) + So(options.DB, ShouldEqual, 0) + So(options.MasterName, ShouldEqual, "") + So(options.ClientName, ShouldEqual, "") + So(options.Protocol, ShouldEqual, 0) + So(options.Username, ShouldEqual, "") + So(options.Password, ShouldEqual, "") + So(options.SentinelUsername, ShouldEqual, "") + So(options.SentinelPassword, ShouldEqual, "") + So(options.DialTimeout, ShouldEqual, 0) + So(options.MaxRetries, ShouldEqual, 0) + So(options.MinRetryBackoff, ShouldEqual, 0) + So(options.MaxRetryBackoff, ShouldEqual, 0) + So(options.ReadTimeout, ShouldEqual, 0) + So(options.WriteTimeout, ShouldEqual, 0) + So(options.ContextTimeoutEnabled, ShouldEqual, false) + So(options.PoolFIFO, ShouldEqual, false) + So(options.PoolSize, ShouldEqual, 0) + So(options.PoolTimeout, ShouldEqual, 0) + So(options.MinIdleConns, ShouldEqual, 0) + So(options.MaxIdleConns, ShouldEqual, 0) + So(options.MaxActiveConns, ShouldEqual, 0) + So(options.ConnMaxIdleTime, ShouldEqual, 0) + So(options.ConnMaxLifetime, ShouldEqual, 0) + So(options.MaxRedirects, ShouldEqual, 0) + So(options.ReadOnly, ShouldEqual, false) + So(options.RouteByLatency, ShouldEqual, false) + So(options.RouteRandomly, ShouldEqual, false) + So(options.DisableIndentity, ShouldEqual, false) + So(options.IdentitySuffix, ShouldEqual, "") + So(options.UnstableResp3, ShouldEqual, false) + + clientIntf, err := rediscfg.GetRedisClient(config, log) + So(err, ShouldBeNil) + So(clientIntf, ShouldNotBeNil) + + _, ok := clientIntf.(*redis.Client) + So(ok, ShouldBeTrue) + }) + + Convey("Test redis options from json", func(c C) { + fileContent := []byte(`{ + "distSpecVersion": "1.1.0", + "storage": { + "remoteCache": true, + "cacheDriver": { + "name": "redis", + "addr": [ + "a.repo:26379", + "b.repo:26379", + "c.repo:26379" + ], + "db": 1, + "master_name": "zotmeta", + "username": "redis", + "password": "**secret**", + "dial_timeout": "5s" + }, + "commit": false, + "dedupe": false, + "gc": true, + "rootDirectory": "/data/zot-cache/dev" + }, + "http": { + "address": "127.0.0.1", + "port": "8080" + }, + "log": { + "level": "debug" + } + }`) + + dir := t.TempDir() + configPath := path.Join(dir, "test-config.json") + + err := os.WriteFile(configPath, fileContent, 0o600) + So(err, ShouldBeNil) + + conf := config.New() + err = server.LoadConfiguration(conf, configPath) + So(err, ShouldBeNil) + + options := rediscfg.ParseRedisUniversalOptions(conf.Storage.CacheDriver, log) + So(options, ShouldNotBeNil) + So(options.Addrs, ShouldEqual, []string{"a.repo:26379", "b.repo:26379", "c.repo:26379"}) + So(options.DB, ShouldEqual, 1) + So(options.MasterName, ShouldEqual, "zotmeta") + So(options.Username, ShouldEqual, "redis") + So(options.Password, ShouldEqual, "**secret**") + So(options.DialTimeout, ShouldEqual, 5*time.Second) + + clientIntf, err := rediscfg.GetRedisClient(conf.Storage.CacheDriver, log) + So(err, ShouldBeNil) + So(clientIntf, ShouldNotBeNil) + + _, ok := clientIntf.(*redis.Client) + So(ok, ShouldBeTrue) + }) + }) +} diff --git a/pkg/api/controller_test.go b/pkg/api/controller_test.go index 600ccf13..a61de5fb 100644 --- a/pkg/api/controller_test.go +++ b/pkg/api/controller_test.go @@ -26,6 +26,7 @@ import ( "testing" "time" + "github.com/alicebob/miniredis/v2" "github.com/google/go-github/v62/github" "github.com/gorilla/mux" "github.com/gorilla/securecookie" @@ -154,6 +155,55 @@ func TestCreateCacheDatabaseDriver(t *testing.T) { So(err, ShouldBeNil) So(driver, ShouldBeNil) }) + Convey("Test CreateCacheDatabaseDriver redisdb", t, func() { + miniRedis := miniredis.RunT(t) + + log := log.NewLogger("debug", "") + + dir := t.TempDir() + conf := config.New() + conf.Storage.RootDirectory = dir + conf.Storage.Dedupe = true + conf.Storage.RemoteCache = true + + // test error on invalid redis client config + conf.Storage.CacheDriver = map[string]interface{}{ + "name": "redis", + "url": false, + } + + driver, err := storage.CreateCacheDatabaseDriver(conf.Storage.StorageConfig, log) + So(err, ShouldNotBeNil) + So(driver, ShouldBeNil) + + // test valid redis client config + conf.Storage.CacheDriver = map[string]interface{}{ + "name": "redis", + "url": "redis://" + miniRedis.Addr(), + } + + // test initialization for S3 storage + conf.Storage.StorageDriver = map[string]interface{}{ + "name": "s3", + "rootdirectory": "/zot", + "url": "us-east-2", + } + + driver, err = storage.CreateCacheDatabaseDriver(conf.Storage.StorageConfig, log) + So(err, ShouldBeNil) + So(driver, ShouldNotBeNil) + So(driver.Name(), ShouldEqual, "redis") + So(driver.UsesRelativePaths(), ShouldEqual, false) + + // test initialization for local storage + conf.Storage.StorageDriver = nil + + driver, err = storage.CreateCacheDatabaseDriver(conf.Storage.StorageConfig, log) + So(err, ShouldBeNil) + So(driver, ShouldNotBeNil) + So(driver.Name(), ShouldEqual, "redis") + So(driver.UsesRelativePaths(), ShouldEqual, true) + }) tskip.SkipDynamo(t) tskip.SkipS3(t) Convey("Test CreateCacheDatabaseDriver dynamodb", t, func() { @@ -226,7 +276,7 @@ func TestCreateCacheDatabaseDriver(t *testing.T) { } func TestCreateMetaDBDriver(t *testing.T) { - Convey("Test CreateCacheDatabaseDriver dynamo", t, func() { + Convey("Test create MetaDB dynamo", t, func() { log := log.NewLogger("debug", "") dir := t.TempDir() conf := config.New() @@ -253,11 +303,26 @@ func TestCreateMetaDBDriver(t *testing.T) { "userdatatablename": "UserDatatable", } + metaDB, err := meta.New(conf.Storage.StorageConfig, log) + So(err, ShouldNotBeNil) + So(metaDB, ShouldBeNil) + + conf.Storage.CacheDriver = map[string]interface{}{ + "name": "dynamodb", + "endpoint": "http://localhost:4566", + "region": "us-east-2", + "cachetablename": "BlobTable", + "repometatablename": "RepoMetadataTable", + "imageMetaTablename": "ZotImageMetaTable", + "repoBlobsInfoTablename": "ZotRepoBlobsInfoTable", + "userdatatablename": "UserDatatable", + } + testFunc := func() { _, _ = meta.New(conf.Storage.StorageConfig, log) } So(testFunc, ShouldPanic) conf.Storage.CacheDriver = map[string]interface{}{ - "name": "dummy", + "name": "dynamodb", "endpoint": "http://localhost:4566", "region": "us-east-2", "cachetablename": "", @@ -272,7 +337,7 @@ func TestCreateMetaDBDriver(t *testing.T) { So(testFunc, ShouldPanic) conf.Storage.CacheDriver = map[string]interface{}{ - "name": "dummy", + "name": "dynamodb", "endpoint": "http://localhost:4566", "region": "us-east-2", "cachetablename": "test", @@ -288,7 +353,60 @@ func TestCreateMetaDBDriver(t *testing.T) { So(testFunc, ShouldNotPanic) }) - Convey("Test CreateCacheDatabaseDriver bolt", t, func() { + Convey("Test create MetaDB redis", t, func() { + miniRedis := miniredis.RunT(t) + + log := log.NewLogger("debug", "") + dir := t.TempDir() + conf := config.New() + conf.Storage.RootDirectory = dir + conf.Storage.Dedupe = true + conf.Storage.RemoteCache = true + conf.Storage.StorageDriver = map[string]interface{}{ + "name": "s3", + "rootdirectory": "/zot", + "region": "us-east-2", + "bucket": "zot-storage", + "secure": true, + "skipverify": false, + } + + conf.Storage.CacheDriver = map[string]interface{}{ + "name": "dummy", + } + + metaDB, err := meta.New(conf.Storage.StorageConfig, log) + So(err, ShouldNotBeNil) + So(metaDB, ShouldBeNil) + + conf.Storage.CacheDriver = map[string]interface{}{ + "name": "redis", + } + + metaDB, err = meta.New(conf.Storage.StorageConfig, log) + So(err, ShouldNotBeNil) + So(metaDB, ShouldBeNil) + + conf.Storage.CacheDriver = map[string]interface{}{ + "name": "redis", + "url": "url", + } + + metaDB, err = meta.New(conf.Storage.StorageConfig, log) + So(err, ShouldNotBeNil) + So(metaDB, ShouldBeNil) + + conf.Storage.CacheDriver = map[string]interface{}{ + "name": "redis", + "url": "redis://" + miniRedis.Addr(), + } + + metaDB, err = meta.New(conf.Storage.StorageConfig, log) + So(err, ShouldBeNil) + So(metaDB, ShouldNotBeNil) + }) + + Convey("Test create MetaDB bolt", t, func() { log := log.NewLogger("debug", "") dir := t.TempDir() conf := config.New() diff --git a/pkg/cli/server/root.go b/pkg/cli/server/root.go index 6eebd11f..c0db1bb3 100644 --- a/pkg/cli/server/root.go +++ b/pkg/cli/server/root.go @@ -257,7 +257,9 @@ func validateCacheConfig(cfg *config.Config, log zlog.Logger) error { if cfg.Storage.CacheDriver != nil && cfg.Storage.RemoteCache { // local storage with remote database - if cfg.Storage.StorageDriver == nil { + // redis is supported with both local and S3 storage, while dynamodb is only supported with S3 + // redis is only supported with local storage in a non-clustering scenario with a single zot instance, + if cfg.Storage.StorageDriver == nil && cfg.Storage.CacheDriver["name"] != storageConstants.RedisDriverName { msg := "invalid database config, cannot have local storage driver with remote database!" log.Error().Err(zerr.ErrBadConfig).Msg(msg) @@ -265,7 +267,8 @@ func validateCacheConfig(cfg *config.Config, log zlog.Logger) error { } // unsupported database driver - if cfg.Storage.CacheDriver["name"] != storageConstants.DynamoDBDriverName { + if cfg.Storage.CacheDriver["name"] != storageConstants.DynamoDBDriverName && + cfg.Storage.CacheDriver["name"] != storageConstants.RedisDriverName { msg := "invalid database config, unsupported database driver" log.Error().Err(zerr.ErrBadConfig).Interface("cacheDriver", cfg.Storage.CacheDriver["name"]).Msg(msg) diff --git a/pkg/extensions/extension_image_trust.go b/pkg/extensions/extension_image_trust.go index cdfcfb65..709a6765 100644 --- a/pkg/extensions/extension_image_trust.go +++ b/pkg/extensions/extension_image_trust.go @@ -19,6 +19,7 @@ import ( "zotregistry.dev/zot/pkg/log" mTypes "zotregistry.dev/zot/pkg/meta/types" "zotregistry.dev/zot/pkg/scheduler" + sconstants "zotregistry.dev/zot/pkg/storage/constants" ) func IsBuiltWithImageTrustExtension() bool { @@ -172,7 +173,11 @@ func SetupImageTrustExtension(conf *config.Config, metaDB mTypes.MetaDB, log log var err error - if conf.Storage.RemoteCache { + if conf.Storage.RemoteCache && conf.Storage.CacheDriver["name"] == sconstants.DynamoDBDriverName { + // AWS secrets manager + // In case of AWS let's assume if dynamodDB is used, the AWS secrets manager is also used + // we use the CacheDriver settings as opposed to the storage settings because we want to + // be able to use S3/minio and redis in the same configuration endpoint, _ := conf.Storage.CacheDriver["endpoint"].(string) region, _ := conf.Storage.CacheDriver["region"].(string) @@ -181,6 +186,7 @@ func SetupImageTrustExtension(conf *config.Config, metaDB mTypes.MetaDB, log log return err } } else { + // Store secrets on the local disk imgTrustStore, err = imagetrust.NewLocalImageTrustStore(conf.Storage.RootDirectory) if err != nil { return err diff --git a/pkg/extensions/extension_image_trust_test.go b/pkg/extensions/extension_image_trust_test.go index a7de46bc..17e7465b 100644 --- a/pkg/extensions/extension_image_trust_test.go +++ b/pkg/extensions/extension_image_trust_test.go @@ -17,6 +17,7 @@ import ( "testing" "time" + "github.com/alicebob/miniredis/v2" guuid "github.com/gofrs/uuid" "github.com/sigstore/cosign/v2/cmd/cosign/cli/generate" "github.com/sigstore/cosign/v2/cmd/cosign/cli/options" @@ -123,6 +124,19 @@ func TestSignatureUploadAndVerificationLocal(t *testing.T) { }) } +func TestSignatureUploadAndVerificationRedis(t *testing.T) { + Convey("test with local storage and redis metadb", t, func() { + miniRedis := miniredis.RunT(t) + + cacheDriverParams := map[string]interface{}{ + "name": "redis", + "url": "redis://" + miniRedis.Addr(), + } + + RunSignatureUploadAndVerificationTests(t, cacheDriverParams) + }) +} + func TestSignatureUploadAndVerificationAWS(t *testing.T) { tskip.SkipDynamo(t) @@ -139,7 +153,7 @@ func TestSignatureUploadAndVerificationAWS(t *testing.T) { repoBlobsInfoTablename := "repoBlobsInfoTable" + uuid.String() cacheDriverParams := map[string]interface{}{ - "name": "dynamoDB", + "name": "dynamodb", "endpoint": os.Getenv("DYNAMODBMOCK_ENDPOINT"), "region": "us-east-2", "cacheTablename": cacheTablename, diff --git a/pkg/extensions/imagetrust/image_trust_test.go b/pkg/extensions/imagetrust/image_trust_test.go index 7fb307cf..bfad641b 100644 --- a/pkg/extensions/imagetrust/image_trust_test.go +++ b/pkg/extensions/imagetrust/image_trust_test.go @@ -16,6 +16,7 @@ import ( "testing" "time" + "github.com/alicebob/miniredis/v2" awshttp "github.com/aws/aws-sdk-go-v2/aws/transport/http" "github.com/aws/aws-sdk-go-v2/service/secretsmanager" "github.com/aws/aws-sdk-go-v2/service/secretsmanager/types" @@ -653,6 +654,25 @@ func TestLocalTrustStore(t *testing.T) { }) } +func TestLocalTrustStoreRedis(t *testing.T) { + miniRedis := miniredis.RunT(t) + + Convey("test local storage and redis", t, func() { + rootDir := t.TempDir() + + imageTrustStore, err := imagetrust.NewLocalImageTrustStore(rootDir) + So(err, ShouldBeNil) + + dbDriverParams := map[string]interface{}{ + "name": "redis", + "url": "redis://" + miniRedis.Addr(), + } + + RunUploadTests(t, *imageTrustStore) + RunVerificationTests(t, dbDriverParams) + }) +} + func TestAWSTrustStore(t *testing.T) { tskip.SkipDynamo(t) @@ -1065,6 +1085,7 @@ func TestAWSTrustStore(t *testing.T) { panic(err) } + cacheTablename := "BlobTable" + uuid.String() repoMetaTablename := "RepoMetadataTable" + uuid.String() versionTablename := "Version" + uuid.String() userDataTablename := "UserDataTable" + uuid.String() @@ -1073,9 +1094,10 @@ func TestAWSTrustStore(t *testing.T) { repoBlobsInfoTablename := "repoBlobsInfoTable" + uuid.String() dynamoDBDriverParams := map[string]interface{}{ - "name": "dynamoDB", + "name": "dynamodb", "endpoint": os.Getenv("DYNAMODBMOCK_ENDPOINT"), "region": "us-east-2", + "cachetablename": cacheTablename, "repometatablename": repoMetaTablename, "imagemetatablename": imageMetaTablename, "repoblobsinfotablename": repoBlobsInfoTablename, diff --git a/pkg/extensions/search/search_test.go b/pkg/extensions/search/search_test.go index 234c6300..0e3a3f8c 100644 --- a/pkg/extensions/search/search_test.go +++ b/pkg/extensions/search/search_test.go @@ -18,6 +18,7 @@ import ( "testing" "time" + "github.com/alicebob/miniredis/v2" guuid "github.com/gofrs/uuid" regTypes "github.com/google/go-containerregistry/pkg/v1/types" notreg "github.com/notaryproject/notation-go/registry" @@ -3905,7 +3906,7 @@ func TestGlobalSearch(t *testing.T) { //nolint: gocyclo So(len(results.Repos), ShouldEqual, 0) }) - Convey("test nested indexes", t, func() { + Convey("test nested indexes CVE scanning disabled", t, func() { log := log.NewLogger("debug", "") rootDir := t.TempDir() port := GetFreePort() @@ -3913,6 +3914,20 @@ func TestGlobalSearch(t *testing.T) { //nolint: gocyclo conf := config.New() conf.HTTP.Port = port conf.Storage.RootDirectory = rootDir + + Convey("test with boltdb", func() { + conf.Storage.CacheDriver = nil + }) + + Convey("test with redis", func() { + miniRedis := miniredis.RunT(t) + + conf.Storage.CacheDriver = map[string]interface{}{ + "name": "redis", + "url": "redis://" + miniRedis.Addr(), + } + }) + defaultVal := true conf.Extensions = &extconf.ExtensionConfig{ Search: &extconf.SearchConfig{BaseConfig: extconf.BaseConfig{Enable: &defaultVal}}, @@ -4054,7 +4069,7 @@ func TestGlobalSearch(t *testing.T) { //nolint: gocyclo } }) - Convey("test nested indexes", t, func() { + Convey("test nested indexes CVE scanning enabled", t, func() { log := log.NewLogger("debug", "") rootDir := t.TempDir() port := GetFreePort() @@ -4062,6 +4077,20 @@ func TestGlobalSearch(t *testing.T) { //nolint: gocyclo conf := config.New() conf.HTTP.Port = port conf.Storage.RootDirectory = rootDir + + Convey("test with boltdb", func() { + conf.Storage.CacheDriver = nil + }) + + Convey("test with redis", func() { + miniRedis := miniredis.RunT(t) + + conf.Storage.CacheDriver = map[string]interface{}{ + "name": "redis", + "url": "redis://" + miniRedis.Addr(), + } + }) + defaultVal := true updateDuration, _ := time.ParseDuration("1h") @@ -6838,7 +6867,7 @@ func TestReadUploadDeleteDynamoDB(t *testing.T) { repoBlobsTablename := "RepoBlobs" + uuid.String() cacheDriverParams := map[string]interface{}{ - "name": "dynamoDB", + "name": "dynamodb", "endpoint": os.Getenv("DYNAMODBMOCK_ENDPOINT"), "region": "us-east-2", "cachetablename": cacheTablename, diff --git a/pkg/meta/meta.go b/pkg/meta/meta.go index c4ab4b53..5eabf7dd 100644 --- a/pkg/meta/meta.go +++ b/pkg/meta/meta.go @@ -1,27 +1,52 @@ package meta import ( - "github.com/aws/aws-sdk-go-v2/service/dynamodb" - "go.etcd.io/bbolt" + "fmt" "zotregistry.dev/zot/errors" "zotregistry.dev/zot/pkg/api/config" + rediscfg "zotregistry.dev/zot/pkg/api/config/redis" "zotregistry.dev/zot/pkg/log" "zotregistry.dev/zot/pkg/meta/boltdb" mdynamodb "zotregistry.dev/zot/pkg/meta/dynamodb" + "zotregistry.dev/zot/pkg/meta/redis" mTypes "zotregistry.dev/zot/pkg/meta/types" + sconstants "zotregistry.dev/zot/pkg/storage/constants" ) func New(storageConfig config.StorageConfig, log log.Logger) (mTypes.MetaDB, error) { if storageConfig.RemoteCache { - dynamoParams := getDynamoParams(storageConfig.CacheDriver, log) + if storageConfig.CacheDriver["name"] == sconstants.DynamoDBDriverName { + dynamoParams := getDynamoParams(storageConfig.CacheDriver, log) - client, err := mdynamodb.GetDynamoClient(dynamoParams) - if err != nil { - return nil, err + client, err := mdynamodb.GetDynamoClient(dynamoParams) + if err != nil { + return nil, err + } + + return mdynamodb.New(client, dynamoParams, log) //nolint:contextcheck } - return Create("dynamodb", client, dynamoParams, log) //nolint:contextcheck + if storageConfig.CacheDriver["name"] == sconstants.RedisDriverName { + redisParams := getRedisParams(storageConfig.CacheDriver, log) + + client, err := rediscfg.GetRedisClient(storageConfig.CacheDriver, log) + if err != nil { //nolint:wsl + return nil, err + } + + return redis.New(client, redisParams, log) //nolint:contextcheck + } + + // this behavior is also mentioned in the configuration validation logic inside the cli package + return nil, fmt.Errorf("%w: cachedriver %s and remotecache %t", errors.ErrBadConfig, + storageConfig.CacheDriver["name"], storageConfig.RemoteCache) + } + + if driverName, ok := storageConfig.CacheDriver["name"]; ok && driverName != sconstants.BoltdbName { + // this behavior is also mentioned in the configuration validation logic inside the cli package + log.Warn().Interface("cachedriver", driverName).Bool("remotecache", storageConfig.RemoteCache). + Msg("unsupported cachedriver for remotecache disabled, will default to boltdb") } params := boltdb.DBParameters{} @@ -32,78 +57,34 @@ func New(storageConfig config.StorageConfig, log log.Logger) (mTypes.MetaDB, err return nil, err } - return Create("boltdb", driver, params, log) //nolint:contextcheck -} - -func Create(dbtype string, dbDriver, parameters interface{}, log log.Logger, //nolint:contextcheck -) (mTypes.MetaDB, error, -) { - switch dbtype { - case "boltdb": - { - properDriver, ok := dbDriver.(*bbolt.DB) - if !ok { - log.Error().Err(errors.ErrTypeAssertionFailed). - Msgf("failed to cast type, expected type '%T' but got '%T'", &bbolt.DB{}, dbDriver) - - return nil, errors.ErrTypeAssertionFailed - } - - return boltdb.New(properDriver, log) - } - case "dynamodb": - { - properDriver, ok := dbDriver.(*dynamodb.Client) - if !ok { - log.Error().Err(errors.ErrTypeAssertionFailed). - Msgf("failed to cast type, expected type '%T' but got '%T'", &dynamodb.Client{}, dbDriver) - - return nil, errors.ErrTypeAssertionFailed - } - - properParameters, ok := parameters.(mdynamodb.DBDriverParameters) - if !ok { - log.Error().Err(errors.ErrTypeAssertionFailed). - Msgf("failed to cast type, expected type '%T' but got '%T'", mdynamodb.DBDriverParameters{}, - parameters) - - return nil, errors.ErrTypeAssertionFailed - } - - return mdynamodb.New(properDriver, properParameters, log) - } - default: - { - return nil, errors.ErrBadConfig - } - } + return boltdb.New(driver, log) //nolint:contextcheck } func getDynamoParams(cacheDriverConfig map[string]interface{}, log log.Logger) mdynamodb.DBDriverParameters { allParametersOk := true - endpoint, ok := toStringIfOk(cacheDriverConfig, "endpoint", log) + endpoint, ok := toStringIfOk(cacheDriverConfig, "endpoint", "", log) allParametersOk = allParametersOk && ok - region, ok := toStringIfOk(cacheDriverConfig, "region", log) + region, ok := toStringIfOk(cacheDriverConfig, "region", "", log) allParametersOk = allParametersOk && ok - repoMetaTablename, ok := toStringIfOk(cacheDriverConfig, "repometatablename", log) + repoMetaTablename, ok := toStringIfOk(cacheDriverConfig, "repometatablename", "", log) allParametersOk = allParametersOk && ok - repoBlobsInfoTablename, ok := toStringIfOk(cacheDriverConfig, "repoblobsinfotablename", log) + repoBlobsInfoTablename, ok := toStringIfOk(cacheDriverConfig, "repoblobsinfotablename", "", log) allParametersOk = allParametersOk && ok - imageMetaTablename, ok := toStringIfOk(cacheDriverConfig, "imagemetatablename", log) + imageMetaTablename, ok := toStringIfOk(cacheDriverConfig, "imagemetatablename", "", log) allParametersOk = allParametersOk && ok - apiKeyTablename, ok := toStringIfOk(cacheDriverConfig, "apikeytablename", log) + apiKeyTablename, ok := toStringIfOk(cacheDriverConfig, "apikeytablename", "", log) allParametersOk = allParametersOk && ok - versionTablename, ok := toStringIfOk(cacheDriverConfig, "versiontablename", log) + versionTablename, ok := toStringIfOk(cacheDriverConfig, "versiontablename", "", log) allParametersOk = allParametersOk && ok - userDataTablename, ok := toStringIfOk(cacheDriverConfig, "userdatatablename", log) + userDataTablename, ok := toStringIfOk(cacheDriverConfig, "userdatatablename", "", log) allParametersOk = allParametersOk && ok if !allParametersOk { @@ -122,17 +103,36 @@ func getDynamoParams(cacheDriverConfig map[string]interface{}, log log.Logger) m } } -func toStringIfOk(cacheDriverConfig map[string]interface{}, param string, log log.Logger) (string, bool) { +func getRedisParams(cacheDriverConfig map[string]interface{}, log log.Logger) redis.DBDriverParameters { + keyPrefix, ok := toStringIfOk(cacheDriverConfig, "keyprefix", "zot", log) + if !ok { + log.Panic().Msg("redis parameters are not specified correctly, can't proceed") + } + + return redis.DBDriverParameters{ + KeyPrefix: keyPrefix, + } +} + +func toStringIfOk(cacheDriverConfig map[string]interface{}, + param string, + defaultVal string, + log log.Logger, +) (string, bool) { val, ok := cacheDriverConfig[param] - if !ok { + if !ok && defaultVal != "" { + log.Info().Str("field", param).Str("default", defaultVal). + Msg("field is not present in CacheDriver config, using default value") + + return defaultVal, true + } else if !ok { log.Error().Str("field", param).Msg("failed to parse CacheDriver config, field is not present") return "", false } str, ok := val.(string) - if !ok { log.Error().Str("parameter", param).Msg("failed to parse CacheDriver config, parameter isn't a string") @@ -145,5 +145,5 @@ func toStringIfOk(cacheDriverConfig map[string]interface{}, param string, log lo return "", false } - return str, ok + return str, true } diff --git a/pkg/meta/meta_test.go b/pkg/meta/meta_test.go index c918bf4b..46fe39dc 100644 --- a/pkg/meta/meta_test.go +++ b/pkg/meta/meta_test.go @@ -5,21 +5,26 @@ package meta_test import ( "context" + "encoding/json" "fmt" "os" "path" "testing" "time" - "github.com/aws/aws-sdk-go-v2/service/dynamodb" + "github.com/alicebob/miniredis/v2" guuid "github.com/gofrs/uuid" "github.com/notaryproject/notation-core-go/signature/jws" "github.com/notaryproject/notation-go" "github.com/notaryproject/notation-go/signer" godigest "github.com/opencontainers/go-digest" + "github.com/opencontainers/image-spec/specs-go" ispec "github.com/opencontainers/image-spec/specs-go/v1" . "github.com/smartystreets/goconvey/convey" + zerr "zotregistry.dev/zot/errors" + "zotregistry.dev/zot/pkg/api/config" + rediscfg "zotregistry.dev/zot/pkg/api/config/redis" zcommon "zotregistry.dev/zot/pkg/common" "zotregistry.dev/zot/pkg/extensions/imagetrust" "zotregistry.dev/zot/pkg/extensions/search/convert" @@ -28,6 +33,7 @@ import ( "zotregistry.dev/zot/pkg/meta/boltdb" "zotregistry.dev/zot/pkg/meta/common" mdynamodb "zotregistry.dev/zot/pkg/meta/dynamodb" + "zotregistry.dev/zot/pkg/meta/redis" mTypes "zotregistry.dev/zot/pkg/meta/types" reqCtx "zotregistry.dev/zot/pkg/requestcontext" tCommon "zotregistry.dev/zot/pkg/test/common" @@ -164,7 +170,39 @@ func TestDynamoDBWrapper(t *testing.T) { }) } -func RunMetaDBTests(t *testing.T, metaDB mTypes.MetaDB, preparationFuncs ...func() error) { //nolint: thelper +func TestRedisDB(t *testing.T) { + miniRedis := miniredis.RunT(t) + + Convey("RedisDB Wrapper", t, func() { + rootDir := t.TempDir() + log := log.NewLogger("debug", "") + + params := redis.DBDriverParameters{KeyPrefix: "zot"} + driverConfig := map[string]interface{}{"url": "redis://" + miniRedis.Addr()} + + redisDriver, err := rediscfg.GetRedisClient(driverConfig, log) + So(err, ShouldBeNil) + + metaDB, err := redis.New(redisDriver, params, log) + So(metaDB, ShouldNotBeNil) + So(err, ShouldBeNil) + + imgTrustStore, err := imagetrust.NewLocalImageTrustStore(rootDir) + So(err, ShouldBeNil) + + metaDB.SetImageTrustStore(imgTrustStore) + + defer func() { + metaDB.ResetDB() //nolint: errcheck + os.RemoveAll(path.Join(rootDir, "_cosign")) + os.RemoveAll(path.Join(rootDir, "_notation")) + }() + + RunMetaDBTests(t, metaDB) + }) +} + +func RunMetaDBTests(t *testing.T, metaDB mTypes.MetaDB, preparationFuncs ...func() error) { //nolint: thelper,gocyclo ctx := context.Background() Convey("Test MetaDB Interface implementation", func() { @@ -376,6 +414,12 @@ func RunMetaDBTests(t *testing.T, metaDB mTypes.MetaDB, preparationFuncs ...func err = metaDB.SetUserData(ctx, userProfileSrc) So(err, ShouldNotBeNil) + + err = metaDB.SetUserGroups(ctx, []string{"group1", "groups2"}) + So(err, ShouldNotBeNil) + + err = metaDB.DeleteUserData(ctx) + So(err, ShouldNotBeNil) }) Convey("Test API keys operations with empty userid", func() { @@ -411,6 +455,12 @@ func RunMetaDBTests(t *testing.T, metaDB mTypes.MetaDB, preparationFuncs ...func err = metaDB.SetUserData(ctx, userProfileSrc) So(err, ShouldNotBeNil) + + err = metaDB.SetUserGroups(ctx, []string{"group1", "groups2"}) + So(err, ShouldNotBeNil) + + err = metaDB.DeleteUserData(ctx) + So(err, ShouldNotBeNil) }) Convey("Test API keys with short expiration date", func() { @@ -823,10 +873,22 @@ func RunMetaDBTests(t *testing.T, metaDB mTypes.MetaDB, preparationFuncs ...func var ( repo1 = "repo1" repo2 = "repo2" + repo3 = "repo3" tag1 = "0.0.1" tag2 = "0.0.2" + tag3 = "0.0.3" ) + userAc := reqCtx.NewUserAccessControl() + userAc.SetUsername("username") + userAc.SetGlobPatterns("read", map[string]bool{ + repo1: true, + repo2: true, + repo3: false, + }) + + ctx := userAc.DeriveContext(context.Background()) + image1 := CreateImageWith(). RandomLayers(2, 10). ImageConfig(ispec.Image{Platform: ispec.Platform{OS: "os1", Architecture: "arch1"}}). @@ -841,6 +903,13 @@ func RunMetaDBTests(t *testing.T, metaDB mTypes.MetaDB, preparationFuncs ...func Build() imageMeta2 := image2.AsImageMeta() + image3 := CreateImageWith(). + LayerBlobs(image1.Layers). + ImageConfig(ispec.Image{Platform: ispec.Platform{OS: "os3", Architecture: "arch3"}}). + Annotations(map[string]string{ispec.AnnotationVendor: "vendor3"}). + Build() + imageMeta3 := image3.AsImageMeta() + err := metaDB.SetRepoReference(ctx, repo1, tag1, imageMeta1) So(err, ShouldBeNil) @@ -850,11 +919,22 @@ func RunMetaDBTests(t *testing.T, metaDB mTypes.MetaDB, preparationFuncs ...func err = metaDB.SetRepoReference(ctx, repo2, tag2, imageMeta2) So(err, ShouldBeNil) + err = metaDB.SetRepoReference(ctx, repo3, tag3, imageMeta3) + So(err, ShouldBeNil) + Convey("Get all RepoMeta", func() { repoMetaSlice, err := metaDB.GetMultipleRepoMeta(context.TODO(), func(repoMeta mTypes.RepoMeta) bool { return true }) So(err, ShouldBeNil) + So(len(repoMetaSlice), ShouldEqual, 3) + }) + + Convey("Get all RepoMeta for user with restricted permissions", func() { + repoMetaSlice, err := metaDB.GetMultipleRepoMeta(ctx, func(repoMeta mTypes.RepoMeta) bool { + return true + }) + So(err, ShouldBeNil) So(len(repoMetaSlice), ShouldEqual, 2) }) @@ -975,6 +1055,65 @@ func RunMetaDBTests(t *testing.T, metaDB mTypes.MetaDB, preparationFuncs ...func So(repoMeta.StarCount, ShouldEqual, 3) }) + Convey("Test bookmarked/starred behavior when user context is not present/invalid", func() { + // Context has no user access control data + ctx := context.Background() + + toggleState, err := metaDB.ToggleBookmarkRepo(ctx, repo) + So(err, ShouldNotBeNil) + So(toggleState, ShouldEqual, mTypes.NotChanged) + + toggleState, err = metaDB.ToggleStarRepo(ctx, repo) + So(err, ShouldNotBeNil) + So(toggleState, ShouldEqual, mTypes.NotChanged) + + // Context has invalid user access control data + var invalid struct{} + + key := reqCtx.GetContextKey() + ctx = context.WithValue(context.Background(), key, invalid) + + toggleState, err = metaDB.ToggleBookmarkRepo(ctx, repo) + So(err, ShouldNotBeNil) + So(toggleState, ShouldEqual, mTypes.NotChanged) + + toggleState, err = metaDB.ToggleStarRepo(ctx, repo) + So(err, ShouldNotBeNil) + So(toggleState, ShouldEqual, mTypes.NotChanged) + + // Context has user access control data, but it is not completely initialized + userAc := reqCtx.NewUserAccessControl() + ctx = userAc.DeriveContext(context.Background()) + + toggleState, err = metaDB.ToggleBookmarkRepo(ctx, repo) + So(err, ShouldNotBeNil) + So(toggleState, ShouldEqual, mTypes.NotChanged) + + toggleState, err = metaDB.ToggleStarRepo(ctx, repo) + So(err, ShouldNotBeNil) + So(toggleState, ShouldEqual, mTypes.NotChanged) + + // Context has user access control data with all fields initialized + userAc.SetUsername("user1") + userAc.SetGlobPatterns("read", map[string]bool{ + repo: true, + }) + + ctx = userAc.DeriveContext(context.Background()) + + toggleState, err = metaDB.ToggleBookmarkRepo(ctx, repo) + So(err, ShouldBeNil) + So(toggleState, ShouldEqual, mTypes.Added) + + toggleState, err = metaDB.ToggleBookmarkRepo(ctx, repo) + So(err, ShouldBeNil) + So(toggleState, ShouldEqual, mTypes.Removed) + + toggleState, err = metaDB.ToggleStarRepo(ctx, repo) + So(err, ShouldEqual, zerr.ErrRepoMetaNotFound) + So(toggleState, ShouldEqual, mTypes.NotChanged) + }) + Convey("Test repo stars for user", func() { var ( repo1 = "repo1" @@ -1616,9 +1755,51 @@ func RunMetaDBTests(t *testing.T, metaDB mTypes.MetaDB, preparationFuncs ...func So(err, ShouldBeNil) So(len(repoMetaList), ShouldEqual, 2) - So(repoMetaList[0].Tags[tag1].Digest, ShouldResemble, image1.DigestStr()) - So(repoMetaList[0].Tags[tag2].Digest, ShouldResemble, image2.DigestStr()) - So(repoMetaList[1].Tags[tag3].Digest, ShouldResemble, image3.DigestStr()) + repos := map[string]map[string]string{} + for _, repoMeta := range repoMetaList { + if _, exists := repos[repoMeta.Name]; !exists { + repos[repoMeta.Name] = map[string]string{} + } + + for tag, descriptor := range repoMeta.Tags { + repos[repoMeta.Name][tag] = descriptor.Digest + } + } + + So(repos[repo1][tag1], ShouldEqual, image1.DigestStr()) + So(repos[repo1][tag2], ShouldEqual, image2.DigestStr()) + So(repos[repo2][tag3], ShouldEqual, image3.DigestStr()) + }) + + Convey("Search repos including empty ones", func() { + err := metaDB.SetRepoReference(ctx, repo1, tag1, image1.AsImageMeta()) + So(err, ShouldBeNil) + err = metaDB.SetRepoReference(ctx, repo1, tag2, image2.AsImageMeta()) + So(err, ShouldBeNil) + // We need to add a new reference and then remove it in order to obtain the empty repo + err = metaDB.SetRepoReference(ctx, repo2, tag3, image3.AsImageMeta()) + So(err, ShouldBeNil) + err = metaDB.RemoveRepoReference(repo2, tag3, image3.Digest()) + So(err, ShouldBeNil) + + repoMetaList, err := metaDB.SearchRepos(ctx, "") + So(err, ShouldBeNil) + So(len(repoMetaList), ShouldEqual, 1) // Empty repos are not returned + + repos := map[string]map[string]string{} + for _, repoMeta := range repoMetaList { + if _, exists := repos[repoMeta.Name]; !exists { + repos[repoMeta.Name] = map[string]string{} + } + + for tag, descriptor := range repoMeta.Tags { + repos[repoMeta.Name][tag] = descriptor.Digest + } + } + + So(repos[repo1][tag1], ShouldEqual, image1.DigestStr()) + So(repos[repo1][tag2], ShouldEqual, image2.DigestStr()) + So(len(repos[repo2]), ShouldEqual, 0) }) Convey("Search a repo by name", func() { @@ -2296,12 +2477,26 @@ func RunMetaDBTests(t *testing.T, metaDB mTypes.MetaDB, preparationFuncs ...func Convey("FilterRepos", func() { repo := "repoFilter" + repoUnmatched := "repoExcluded" tag1 := "tag1" tag2 := "tag22" + tag3 := "tag3" + tag4 := "tag44" + + userAc := reqCtx.NewUserAccessControl() + userAc.SetUsername("username") + userAc.SetGlobPatterns("read", map[string]bool{ + repo: true, + repoUnmatched: false, + }) + + ctx := userAc.DeriveContext(context.Background()) image := CreateImageWith().DefaultLayers().PlatformConfig("image-platform", "image-os").Build() err := metaDB.SetRepoReference(ctx, repo, tag1, image.AsImageMeta()) So(err, ShouldBeNil) + err = metaDB.SetRepoReference(ctx, repoUnmatched, tag3, image.AsImageMeta()) + So(err, ShouldBeNil) multiarch := CreateMultiarchWith(). Images([]Image{ @@ -2316,9 +2511,11 @@ func RunMetaDBTests(t *testing.T, metaDB mTypes.MetaDB, preparationFuncs ...func err = metaDB.SetRepoReference(ctx, repo, tag2, multiarch.AsImageMeta()) So(err, ShouldBeNil) + err = metaDB.SetRepoReference(ctx, repoUnmatched, tag4, multiarch.AsImageMeta()) + So(err, ShouldBeNil) //nolint: contextcheck - repoMetaList, err := metaDB.FilterRepos(context.Background(), mTypes.AcceptAllRepoNames, + repoMetaList, err := metaDB.FilterRepos(ctx, mTypes.AcceptAllRepoNames, mTypes.AcceptAllRepoMeta) So(err, ShouldBeNil) So(len(repoMetaList), ShouldEqual, 1) @@ -2490,6 +2687,186 @@ func RunMetaDBTests(t *testing.T, metaDB mTypes.MetaDB, preparationFuncs ...func So(repos, ShouldNotContain, repo2) So(repos, ShouldNotContain, repo3) }) + + Convey("Test Nested Indexes", func() { + // nested manifest/indexes: + // image111 -> multiArchBottom11 -> multiArchMiddle1 -> multiArchTop + // image112 -> multiArchBottom11 -> multiArchMiddle1 -> multiArchTop + // image121 -> multiArchBottom12 -> multiArchMiddle1 -> multiArchTop + // image122 -> multiArchBottom12 -> multiArchMiddle1 -> multiArchTop + // image211 -> multiArchBottom21 -> multiArchMiddle2 -> multiArchTop + // image212 -> multiArchBottom21 -> multiArchMiddle2 -> multiArchTop + // image31 -> multiArchMiddle3 -> multiArchTop + // image32 -> multiArchMiddle3 -> multiArchTop + repoName := "nested" + + image111 := CreateRandomImage() + image112 := CreateRandomImage() + multiArchBottom11 := CreateMultiarchWith().Images([]Image{image111, image112}).Build() + + err := metaDB.SetRepoReference(ctx, repoName, image111.DigestStr(), image111.AsImageMeta()) + So(err, ShouldBeNil) + err = metaDB.SetRepoReference(ctx, repoName, image112.DigestStr(), image112.AsImageMeta()) + So(err, ShouldBeNil) + err = metaDB.SetRepoReference(ctx, repoName, multiArchBottom11.DigestStr(), multiArchBottom11.AsImageMeta()) + So(err, ShouldBeNil) + + image121 := CreateRandomImage() + image122 := CreateRandomImage() + multiArchBottom12 := CreateMultiarchWith().Images([]Image{image121, image122}).Build() + + err = metaDB.SetRepoReference(ctx, repoName, image121.DigestStr(), image121.AsImageMeta()) + So(err, ShouldBeNil) + err = metaDB.SetRepoReference(ctx, repoName, image122.DigestStr(), image122.AsImageMeta()) + So(err, ShouldBeNil) + err = metaDB.SetRepoReference(ctx, repoName, multiArchBottom12.DigestStr(), multiArchBottom12.AsImageMeta()) + So(err, ShouldBeNil) + + indexMultiArchMiddle1 := ispec.Index{ + Versioned: specs.Versioned{SchemaVersion: 2}, + MediaType: ispec.MediaTypeImageIndex, + Manifests: []ispec.Descriptor{ + { + Digest: multiArchBottom11.IndexDescriptor.Digest, + Size: multiArchBottom11.IndexDescriptor.Size, + MediaType: ispec.MediaTypeImageIndex, + }, + { + Digest: multiArchBottom12.IndexDescriptor.Digest, + Size: multiArchBottom12.IndexDescriptor.Size, + MediaType: ispec.MediaTypeImageIndex, + }, + }, + } + + indexMultiArchMiddle1Blob, err := json.Marshal(indexMultiArchMiddle1) + So(err, ShouldBeNil) + indexMultiArchMiddle1Digest := godigest.FromBytes(indexMultiArchMiddle1Blob) + + indexMultiArchMiddle1Meta := mTypes.ImageMeta{ + MediaType: ispec.MediaTypeImageIndex, + Digest: indexMultiArchMiddle1Digest, + Size: int64(len(indexMultiArchMiddle1Blob)), + Index: &indexMultiArchMiddle1, + } + + err = metaDB.SetRepoReference(ctx, repoName, "multiArchMiddle1", indexMultiArchMiddle1Meta) + So(err, ShouldBeNil) + + image211 := CreateRandomImage() + image212 := CreateRandomImage() + multiArchBottom21 := CreateMultiarchWith().Images([]Image{image211, image212}).Build() + + err = metaDB.SetRepoReference(ctx, repoName, image211.DigestStr(), image211.AsImageMeta()) + So(err, ShouldBeNil) + err = metaDB.SetRepoReference(ctx, repoName, image212.DigestStr(), image212.AsImageMeta()) + So(err, ShouldBeNil) + err = metaDB.SetRepoReference(ctx, repoName, multiArchBottom21.DigestStr(), multiArchBottom21.AsImageMeta()) + So(err, ShouldBeNil) + + indexMultiArchMiddle2 := ispec.Index{ + Versioned: specs.Versioned{SchemaVersion: 2}, + MediaType: ispec.MediaTypeImageIndex, + Manifests: []ispec.Descriptor{ + { + Digest: multiArchBottom21.IndexDescriptor.Digest, + Size: multiArchBottom21.IndexDescriptor.Size, + MediaType: ispec.MediaTypeImageIndex, + }, + }, + } + + indexMultiArchMiddle2Blob, err := json.Marshal(indexMultiArchMiddle2) + So(err, ShouldBeNil) + + indexMultiArchMiddle2Digest := godigest.FromBytes(indexMultiArchMiddle2Blob) + + indexMultiArchMiddle2Meta := mTypes.ImageMeta{ + MediaType: ispec.MediaTypeImageIndex, + Digest: indexMultiArchMiddle2Digest, + Size: int64(len(indexMultiArchMiddle2Blob)), + Index: &indexMultiArchMiddle2, + } + + err = metaDB.SetRepoReference(ctx, repoName, "multiArchMiddle2", indexMultiArchMiddle2Meta) + So(err, ShouldBeNil) + + image31 := CreateRandomImage() + image32 := CreateRandomImage() + multiArchBottom3 := CreateMultiarchWith().Images([]Image{image31, image32}).Build() + + err = metaDB.SetRepoReference(ctx, repoName, image31.DigestStr(), image31.AsImageMeta()) + So(err, ShouldBeNil) + err = metaDB.SetRepoReference(ctx, repoName, image32.DigestStr(), image32.AsImageMeta()) + So(err, ShouldBeNil) + err = metaDB.SetRepoReference(ctx, repoName, multiArchBottom3.DigestStr(), multiArchBottom3.AsImageMeta()) + So(err, ShouldBeNil) + + indexMultiArchTop := ispec.Index{ + Versioned: specs.Versioned{SchemaVersion: 2}, + MediaType: ispec.MediaTypeImageIndex, + Manifests: []ispec.Descriptor{ + { + Digest: indexMultiArchMiddle1Digest, + Size: int64(len(indexMultiArchMiddle1Blob)), + MediaType: ispec.MediaTypeImageIndex, + }, + { + Digest: indexMultiArchMiddle2Digest, + Size: int64(len(indexMultiArchMiddle2Blob)), + MediaType: ispec.MediaTypeImageIndex, + }, + { + Digest: multiArchBottom3.IndexDescriptor.Digest, + Size: multiArchBottom3.IndexDescriptor.Size, + MediaType: ispec.MediaTypeImageIndex, + }, + }, + } + + indexMultiArchTopBlob, err := json.Marshal(indexMultiArchTop) + So(err, ShouldBeNil) + + indexMultiArchTopBlobDigest := godigest.FromBytes(indexMultiArchTopBlob) + + indexMultiArchTopMeta := mTypes.ImageMeta{ + MediaType: ispec.MediaTypeImageIndex, + Digest: indexMultiArchTopBlobDigest, + Size: int64(len(indexMultiArchTopBlob)), + Index: &indexMultiArchTop, + } + + err = metaDB.SetRepoReference(ctx, repoName, "multiArchTop", indexMultiArchTopMeta) + So(err, ShouldBeNil) + + Convey("Test searchTags", func() { + fullImageMetaList, err := metaDB.SearchTags(ctx, repoName+":"+"multiArch") + So(err, ShouldBeNil) + + So(len(fullImageMetaList), ShouldEqual, 3) + + tags := map[string][]string{} + + for _, imageMeta := range fullImageMetaList { + So(imageMeta.MediaType, ShouldEqual, ispec.MediaTypeImageIndex) + + digests := []string{} + for _, manifest := range imageMeta.Manifests { + digests = append(digests, manifest.Digest.String()) + } + + tags[imageMeta.Tag] = digests + } + + So(tags, ShouldContainKey, "multiArchMiddle1") + So(tags, ShouldContainKey, "multiArchMiddle2") + So(tags, ShouldContainKey, "multiArchTop") + + So(len(tags["multiArchMiddle1"]), ShouldEqual, 4) + So(len(tags["multiArchMiddle2"]), ShouldEqual, 2) + So(len(tags["multiArchTop"]), ShouldEqual, 8) + }) + }) }) } @@ -2514,66 +2891,134 @@ func TestRelevanceSorting(t *testing.T) { }) } -func TestCreateDynamo(t *testing.T) { - tskip.SkipDynamo(t) - - Convey("Create", t, func() { - dynamoDBDriverParams := mdynamodb.DBDriverParameters{ - Endpoint: os.Getenv("DYNAMODBMOCK_ENDPOINT"), - RepoMetaTablename: "RepoMetadataTable", - RepoBlobsInfoTablename: "RepoBlobs", - ImageMetaTablename: "ImageMeta", - UserDataTablename: "UserDataTable", - APIKeyTablename: "ApiKeyTable", - VersionTablename: "Version", - Region: "us-east-2", - } - - client, err := mdynamodb.GetDynamoClient(dynamoDBDriverParams) - So(err, ShouldBeNil) - - log := log.NewLogger("debug", "") - - metaDB, err := meta.Create("dynamodb", client, dynamoDBDriverParams, log) - So(metaDB, ShouldNotBeNil) - So(err, ShouldBeNil) - }) - - Convey("Fails", t, func() { - log := log.NewLogger("debug", "") - - _, err := meta.Create("dynamodb", nil, boltdb.DBParameters{RootDir: "root"}, log) - So(err, ShouldNotBeNil) - - _, err = meta.Create("dynamodb", &dynamodb.Client{}, "bad", log) - So(err, ShouldNotBeNil) - - metaDB, err := meta.Create("random", nil, boltdb.DBParameters{RootDir: "root"}, log) - So(metaDB, ShouldBeNil) - So(err, ShouldNotBeNil) - }) -} - func TestCreateBoltDB(t *testing.T) { - Convey("Create", t, func() { + Convey("New() succeeds", t, func() { rootDir := t.TempDir() - params := boltdb.DBParameters{ - RootDir: rootDir, - } - boltDriver, err := boltdb.GetBoltDriver(params) - So(err, ShouldBeNil) + + conf := config.New() + conf.Storage.RootDirectory = rootDir log := log.NewLogger("debug", "") + So(log, ShouldNotBeNil) - metaDB, err := meta.Create("boltdb", boltDriver, params, log) + Convey("Test New() with unspecified driver", func() { + conf.Storage.CacheDriver = map[string]interface{}{} + }) + + Convey("Test New() with bad driver", func() { + // we default to bolt in case of misconfiguration + conf.Storage.CacheDriver = map[string]interface{}{"name": "somedriver"} + }) + + Convey("Test New() with specified driver", func() { + conf.Storage.CacheDriver = map[string]interface{}{"name": "cache"} + }) + + repoDBPath := path.Join(rootDir, "meta.db") + defer os.Remove(repoDBPath) + + metaDB, err := meta.New(conf.Storage.StorageConfig, log) + So(err, ShouldBeNil) So(metaDB, ShouldNotBeNil) + + err = os.Chmod(repoDBPath, 0o200) So(err, ShouldBeNil) - }) - Convey("fails", t, func() { - log := log.NewLogger("debug", "") - - _, err := meta.Create("boltdb", nil, mdynamodb.DBDriverParameters{}, log) + metaDB, err = meta.New(conf.Storage.StorageConfig, log) So(err, ShouldNotBeNil) + So(metaDB, ShouldBeNil) + + err = os.Chmod(repoDBPath, 0o600) + So(err, ShouldBeNil) + }) +} + +func TestCreateRedisDB(t *testing.T) { + Convey("Test New()", t, func() { + conf := config.New() + conf.Storage.RemoteCache = true + + log := log.NewLogger("debug", "") + So(log, ShouldNotBeNil) + + Convey("Succeeds with default key prefix", func() { + miniRedis := miniredis.RunT(t) + + cacheDriverParams := map[string]interface{}{ + "name": "redis", + "url": "redis://" + miniRedis.Addr(), + } + + conf.Storage.CacheDriver = cacheDriverParams + + metaDB, err := meta.New(conf.Storage.StorageConfig, log) + So(err, ShouldBeNil) + So(metaDB, ShouldNotBeNil) + }) + + Convey("Succeeds with specific key prefix", func() { + miniRedis := miniredis.RunT(t) + + cacheDriverParams := map[string]interface{}{ + "name": "redis", + "url": "redis://" + miniRedis.Addr(), + "key": "keyPrefix", + } + + conf.Storage.CacheDriver = cacheDriverParams + + metaDB, err := meta.New(conf.Storage.StorageConfig, log) + So(err, ShouldBeNil) + So(metaDB, ShouldNotBeNil) + }) + + Convey("Fails on Ping()", func() { + // Redis client will not be responding + cacheDriverParams := map[string]interface{}{ + "name": "redis", + "url": "redis://127.0.0.1:" + tCommon.GetFreePort(), + } + + conf.Storage.CacheDriver = cacheDriverParams + + _, err := meta.New(conf.Storage.StorageConfig, log) + So(err, ShouldNotBeNil) + }) + + Convey("Fail on invalid parameters", func() { + // Bad key types + cacheDriverParams := map[string]interface{}{ + "name": "redis", + "url": "redis://127.0.0.1:" + tCommon.GetFreePort(), + "keyprefix": true, + } + + conf.Storage.CacheDriver = cacheDriverParams + + testFunc := func() { _, _ = meta.New(conf.Storage.StorageConfig, log) } + So(testFunc, ShouldPanic) + + cacheDriverParams = map[string]interface{}{ + "name": "redis", + "url": "redis://127.0.0.1:" + tCommon.GetFreePort(), + "keyprefix": "", + } + + conf.Storage.CacheDriver = cacheDriverParams + + testFunc = func() { _, _ = meta.New(conf.Storage.StorageConfig, log) } + So(testFunc, ShouldPanic) + + cacheDriverParams = map[string]interface{}{ + "name": "redis", + "url": false, + "keyprefix": "zot", + } + + conf.Storage.CacheDriver = cacheDriverParams + + _, err := meta.New(conf.Storage.StorageConfig, log) + So(err, ShouldNotBeNil) + }) }) } diff --git a/pkg/meta/parse_test.go b/pkg/meta/parse_test.go index 9fc0c64f..ad8409f7 100644 --- a/pkg/meta/parse_test.go +++ b/pkg/meta/parse_test.go @@ -11,16 +11,19 @@ import ( "testing" "time" + "github.com/alicebob/miniredis/v2" godigest "github.com/opencontainers/go-digest" ispec "github.com/opencontainers/image-spec/specs-go/v1" . "github.com/smartystreets/goconvey/convey" + rediscfg "zotregistry.dev/zot/pkg/api/config/redis" zcommon "zotregistry.dev/zot/pkg/common" "zotregistry.dev/zot/pkg/extensions/monitoring" "zotregistry.dev/zot/pkg/log" "zotregistry.dev/zot/pkg/meta" "zotregistry.dev/zot/pkg/meta/boltdb" "zotregistry.dev/zot/pkg/meta/dynamodb" + "zotregistry.dev/zot/pkg/meta/redis" mTypes "zotregistry.dev/zot/pkg/meta/types" "zotregistry.dev/zot/pkg/storage" "zotregistry.dev/zot/pkg/storage/local" @@ -302,6 +305,27 @@ func TestParseStorageWithBoltDB(t *testing.T) { }) } +func TestParseStorageWithRedisDB(t *testing.T) { + Convey("Redis", t, func() { + miniRedis := miniredis.RunT(t) + + rootDir := t.TempDir() + log := log.NewLogger("debug", "") + + params := redis.DBDriverParameters{KeyPrefix: "zot"} + driverConfig := map[string]interface{}{"url": "redis://" + miniRedis.Addr()} + + redisDriver, err := rediscfg.GetRedisClient(driverConfig, log) + So(err, ShouldBeNil) + + metaDB, err := redis.New(redisDriver, params, log) + So(metaDB, ShouldNotBeNil) + So(err, ShouldBeNil) + + RunParseStorageTests(rootDir, metaDB, log) + }) +} + func TestParseStorageDynamoWrapper(t *testing.T) { tskip.SkipDynamo(t) diff --git a/pkg/meta/redis/redis.go b/pkg/meta/redis/redis.go new file mode 100644 index 00000000..50aefa6f --- /dev/null +++ b/pkg/meta/redis/redis.go @@ -0,0 +1,2302 @@ +package redis + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "strings" + "time" + + "github.com/go-redsync/redsync/v4" + gors "github.com/go-redsync/redsync/v4/redis/goredis/v9" + godigest "github.com/opencontainers/go-digest" + ispec "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/redis/go-redis/v9" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/known/timestamppb" + + zerr "zotregistry.dev/zot/errors" + "zotregistry.dev/zot/pkg/api/constants" + zcommon "zotregistry.dev/zot/pkg/common" + "zotregistry.dev/zot/pkg/log" + "zotregistry.dev/zot/pkg/meta/common" + mConvert "zotregistry.dev/zot/pkg/meta/convert" + proto_go "zotregistry.dev/zot/pkg/meta/proto/gen" + mTypes "zotregistry.dev/zot/pkg/meta/types" + "zotregistry.dev/zot/pkg/meta/version" + reqCtx "zotregistry.dev/zot/pkg/requestcontext" +) + +const ( + ImageMetaBucket = "ImageMeta" + RepoMetaBucket = "RepoMeta" + RepoBlobsBucket = "RepoBlobsMeta" + RepoLastUpdatedBucket = "RepoLastUpdated" + UserDataBucket = "UserData" + VersionBucket = "Version" + UserAPIKeysBucket = "UserAPIKeys" + LocksBucket = "Locks" +) + +type RedisDB struct { + Client redis.UniversalClient + imgTrustStore mTypes.ImageTrustStore + Patches []func(client redis.UniversalClient) error + Version string + Log log.Logger + RS *redsync.Redsync + ImageMetaKey string + RepoMetaKey string + RepoBlobsKey string + RepoLastUpdatedKey string + UserDataKey string + VersionKey string + UserAPIKeysKey string + LocksKey string +} + +type DBDriverParameters struct { + KeyPrefix string +} + +func New(client redis.UniversalClient, params DBDriverParameters, log log.Logger) (*RedisDB, error) { + redisWrapper := RedisDB{ + Client: client, + Log: log, + Patches: version.GetRedisDBPatches(), + Version: version.CurrentVersion, + imgTrustStore: nil, + ImageMetaKey: join(params.KeyPrefix, ImageMetaBucket), + RepoMetaKey: join(params.KeyPrefix, RepoMetaBucket), + RepoBlobsKey: join(params.KeyPrefix, RepoBlobsBucket), + RepoLastUpdatedKey: join(params.KeyPrefix, RepoLastUpdatedBucket), + UserDataKey: join(params.KeyPrefix, UserDataBucket), + VersionKey: join(params.KeyPrefix, VersionBucket), + UserAPIKeysKey: join(params.KeyPrefix, UserAPIKeysBucket), + LocksKey: join(params.KeyPrefix, LocksBucket), + } + + if err := client.Ping(context.Background()).Err(); err != nil { + log.Error().Err(err).Msg("failed to ping redis DB") + + return nil, err + } + + // Create an instance of redisync to be used to obtain locks + // these locks would be used only for writes in the DB + // Depending on what resource/ bucket we want to lock, + // the key used for locking can be: + // - repo name + // - image digest + // - user ID + // - version + pool := gors.NewPool(client) + redisWrapper.RS = redsync.New(pool) + + return &redisWrapper, nil +} + +// GetStarredRepos returns starred repos and takes current user in consideration. +func (rc *RedisDB) GetStarredRepos(ctx context.Context) ([]string, error) { + userData, err := rc.GetUserData(ctx) + if errors.Is(err, zerr.ErrUserDataNotFound) || errors.Is(err, zerr.ErrUserDataNotAllowed) { + return []string{}, nil + } + + return userData.StarredRepos, err +} + +// GetBookmarkedRepos returns bookmarked repos and takes current user in consideration. +func (rc *RedisDB) GetBookmarkedRepos(ctx context.Context) ([]string, error) { + userData, err := rc.GetUserData(ctx) + if errors.Is(err, zerr.ErrUserDataNotFound) || errors.Is(err, zerr.ErrUserDataNotAllowed) { + return []string{}, nil + } + + return userData.BookmarkedRepos, err +} + +// ToggleStarRepo adds/removes stars on repos. +func (rc *RedisDB) ToggleStarRepo(ctx context.Context, repo string) (mTypes.ToggleState, error) { + userAc, err := reqCtx.UserAcFromContext(ctx) + if err != nil { + return mTypes.NotChanged, err + } + + if userAc.IsAnonymous() || !userAc.Can(constants.ReadPermission, repo) { + return mTypes.NotChanged, zerr.ErrUserDataNotAllowed + } + + userid := userAc.GetUsername() + + var res mTypes.ToggleState + + err = rc.withRSLocks(ctx, []string{rc.getRepoLockKey(repo), rc.getUserLockKey(userid)}, func() error { + userData, err := rc.GetUserData(ctx) + if err != nil && !errors.Is(err, zerr.ErrUserDataNotFound) { + res = mTypes.NotChanged + + return err + } + + isRepoStarred := zcommon.Contains(userData.StarredRepos, repo) + + if isRepoStarred { + res = mTypes.Removed + + userData.StarredRepos = zcommon.RemoveFrom(userData.StarredRepos, repo) + } else { + res = mTypes.Added + + userData.StarredRepos = append(userData.StarredRepos, repo) + } + + userDataBlob, err := json.Marshal(userData) + if err != nil { + res = mTypes.NotChanged + + return err + } + + protoRepoMeta, err := rc.getProtoRepoMeta(ctx, repo) + if err != nil { + res = mTypes.NotChanged + + return err + } + + switch res { + case mTypes.Added: + protoRepoMeta.Stars++ + case mTypes.Removed: + protoRepoMeta.Stars-- + } + + repoMetaBlob, err := proto.Marshal(protoRepoMeta) + if err != nil { + res = mTypes.NotChanged + + return err + } + + _, err = rc.Client.TxPipelined(ctx, func(txrp redis.Pipeliner) error { + if err = txrp.HSet(ctx, rc.UserDataKey, userid, userDataBlob).Err(); err != nil { + rc.Log.Error().Err(err).Str("hset", rc.UserDataKey).Str("userid", userid). + Msg("failed to set user data record") + + return fmt.Errorf("failed to set user data for identity %s: %w", userid, err) + } + + if err := txrp.HSet(ctx, rc.RepoMetaKey, repo, repoMetaBlob).Err(); err != nil { + rc.Log.Error().Err(err).Str("hset", rc.RepoMetaKey).Str("repo", repo). + Msg("failed to put repo meta record") + + return fmt.Errorf("failed to set repometa for repo %s: %w", repo, err) + } + + return nil + }) + + return err + }) + + return res, err +} + +// ToggleBookmarkRepo adds/removes bookmarks on repos. +func (rc *RedisDB) ToggleBookmarkRepo(ctx context.Context, repo string) (mTypes.ToggleState, error) { + userAc, err := reqCtx.UserAcFromContext(ctx) + if err != nil { + return mTypes.NotChanged, err + } + + if userAc.IsAnonymous() || !userAc.Can(constants.ReadPermission, repo) { + return mTypes.NotChanged, zerr.ErrUserDataNotAllowed + } + + userid := userAc.GetUsername() + + var res mTypes.ToggleState + + err = rc.withRSLocks(ctx, []string{rc.getUserLockKey(userid)}, func() error { + userData, err := rc.GetUserData(ctx) + if err != nil && !errors.Is(err, zerr.ErrUserDataNotFound) { + res = mTypes.NotChanged + + return err + } + + isRepoBookmarked := zcommon.Contains(userData.BookmarkedRepos, repo) + + if isRepoBookmarked { + res = mTypes.Removed + + userData.BookmarkedRepos = zcommon.RemoveFrom(userData.BookmarkedRepos, repo) + } else { + res = mTypes.Added + + userData.BookmarkedRepos = append(userData.BookmarkedRepos, repo) + } + + userDataBlob, err := json.Marshal(userData) + if err != nil { + res = mTypes.NotChanged + + return err + } + + err = rc.Client.HSet(ctx, rc.UserDataKey, userid, userDataBlob).Err() + if err != nil { + rc.Log.Error().Err(err).Str("hset", rc.UserDataKey).Str("userid", userid). + Msg("failed to set user data record") + + res = mTypes.NotChanged + + return fmt.Errorf("failed to set user data for identity %s: %w", userid, err) + } + + return err + }) + + return res, err +} + +// UserDB profile/api key CRUD. +func (rc *RedisDB) GetUserData(ctx context.Context) (mTypes.UserData, error) { + userData := mTypes.UserData{} + userData.APIKeys = make(map[string]mTypes.APIKeyDetails) + + userAc, err := reqCtx.UserAcFromContext(ctx) + if err != nil { + return userData, err + } + + if userAc.IsAnonymous() { + return userData, zerr.ErrUserDataNotAllowed + } + + userid := userAc.GetUsername() + + userDataBlob, err := rc.Client.HGet(ctx, rc.UserDataKey, userid).Bytes() + if err != nil && !errors.Is(err, redis.Nil) { + rc.Log.Error().Err(err).Str("hget", rc.UserDataKey).Str("userid", userid). + Msg("failed to get user data record") + + return userData, fmt.Errorf("failed to get user data record for identity %s: %w", userid, err) + } + + if errors.Is(err, redis.Nil) { + return userData, zerr.ErrUserDataNotFound + } + + err = json.Unmarshal(userDataBlob, &userData) + + if userData.APIKeys == nil { + // Unmarshal may have reset the value + userData.APIKeys = make(map[string]mTypes.APIKeyDetails) + } + + return userData, err +} + +// SetUserData should NEVER be used in production as both GetUserData and SetUserData +// should be locked for the duration of the entire transaction at a higher level in the app. +func (rc *RedisDB) SetUserData(ctx context.Context, userData mTypes.UserData) error { + userAc, err := reqCtx.UserAcFromContext(ctx) + if err != nil { + return err + } + + if userAc.IsAnonymous() { + return zerr.ErrUserDataNotAllowed + } + + userid := userAc.GetUsername() + + userDataBlob, err := json.Marshal(userData) + if err != nil { + return err + } + + err = rc.withRSLocks(ctx, []string{rc.getUserLockKey(userid)}, func() error { + err = rc.Client.HSet(ctx, rc.UserDataKey, userid, userDataBlob).Err() + if err != nil { + rc.Log.Error().Err(err).Str("hset", rc.UserDataKey).Str("userid", userid). + Msg("failed to set user data record") + + return fmt.Errorf("failed to set user data for identity %s: %w", userid, err) + } + + return nil + }) + + return err +} + +func (rc *RedisDB) SetUserGroups(ctx context.Context, groups []string) error { + userAc, err := reqCtx.UserAcFromContext(ctx) + if err != nil { + return err + } + + if userAc.IsAnonymous() { + return zerr.ErrUserDataNotAllowed + } + + userid := userAc.GetUsername() + + err = rc.withRSLocks(ctx, []string{rc.getUserLockKey(userid)}, func() error { + userData, err := rc.GetUserData(ctx) + if err != nil && !errors.Is(err, zerr.ErrUserDataNotFound) { + return err + } + + userData.Groups = groups + + userDataBlob, err := json.Marshal(userData) + if err != nil { + return err + } + + err = rc.Client.HSet(ctx, rc.UserDataKey, userid, userDataBlob).Err() + if err != nil { + rc.Log.Error().Err(err).Str("hset", rc.UserDataKey).Str("userid", userid). + Msg("failed to set user data record") + + return fmt.Errorf("failed to set user data for identity %s: %w", userid, err) + } + + return nil + }) + + return err +} + +func (rc *RedisDB) GetUserGroups(ctx context.Context) ([]string, error) { + userData, err := rc.GetUserData(ctx) + + return userData.Groups, err +} + +func (rc *RedisDB) DeleteUserData(ctx context.Context) error { + userAc, err := reqCtx.UserAcFromContext(ctx) + if err != nil { + return err + } + + if userAc.IsAnonymous() { + return zerr.ErrUserDataNotAllowed + } + + userid := userAc.GetUsername() + + err = rc.withRSLocks(ctx, []string{rc.getUserLockKey(userid)}, func() error { + _, err = rc.GetUserData(ctx) + if err != nil && errors.Is(err, zerr.ErrUserDataNotFound) { + return zerr.ErrBucketDoesNotExist + } + + err = rc.Client.HDel(ctx, rc.UserDataKey, userid).Err() + if err != nil { + rc.Log.Error().Err(err).Str("hdel", rc.UserDataKey).Str("userid", userid). + Msg("failed to delete user data record") + + return fmt.Errorf("failed to delete user data for identity %s: %w", userid, err) + } + + return nil + }) + + return err +} + +func (rc *RedisDB) GetUserAPIKeyInfo(hashedKey string) (string, error) { + ctx := context.Background() + + userid, err := rc.Client.HGet(ctx, rc.UserAPIKeysKey, hashedKey).Result() + if err != nil && !errors.Is(err, redis.Nil) { + rc.Log.Error().Err(err).Str("hget", rc.UserAPIKeysKey).Str("userid", userid). + Msg("failed to get api key record") + + return userid, fmt.Errorf("failed to get api key record for identity %s: %w", userid, err) + } + + if len(userid) == 0 || errors.Is(err, redis.Nil) { + return userid, zerr.ErrUserAPIKeyNotFound + } + + return userid, err +} + +func (rc *RedisDB) GetUserAPIKeys(ctx context.Context) ([]mTypes.APIKeyDetails, error) { + apiKeys := make([]mTypes.APIKeyDetails, 0) + + userAc, err := reqCtx.UserAcFromContext(ctx) + if err != nil { + return nil, err + } + + if userAc.IsAnonymous() { + return nil, zerr.ErrUserDataNotAllowed + } + + userid := userAc.GetUsername() + + // Lock used because getting API keys also updates their expired flag in the DB + err = rc.withRSLocks(ctx, []string{rc.getUserLockKey(userid)}, func() error { + userData, err := rc.GetUserData(ctx) + if err != nil && !errors.Is(err, zerr.ErrUserDataNotFound) { + return err + } + + changed := false + + for hashedKey, apiKeyDetails := range userData.APIKeys { + // if expiresAt is not nil value + if !apiKeyDetails.ExpirationDate.Equal(time.Time{}) && time.Now().After(apiKeyDetails.ExpirationDate) { + apiKeyDetails.IsExpired = true + + changed = true + } + + userData.APIKeys[hashedKey] = apiKeyDetails + + apiKeys = append(apiKeys, apiKeyDetails) + } + + if !changed { + // return early, no need to make a call to update key expiry in the DB + return nil + } + + userDataBlob, err := json.Marshal(userData) + if err != nil { + return err + } + + err = rc.Client.HSet(ctx, rc.UserDataKey, userid, userDataBlob).Err() + if err != nil { + rc.Log.Error().Err(err).Str("hset", rc.UserDataKey).Str("userid", userid). + Msg("failed to set user data record") + + return fmt.Errorf("failed to set user data for identity %s: %w", userid, err) + } + + return nil + }) + + return apiKeys, err +} + +func (rc *RedisDB) AddUserAPIKey(ctx context.Context, hashedKey string, apiKeyDetails *mTypes.APIKeyDetails) error { + userAc, err := reqCtx.UserAcFromContext(ctx) + if err != nil { + return err + } + + if userAc.IsAnonymous() { + return zerr.ErrUserDataNotAllowed + } + + userid := userAc.GetUsername() + + err = rc.withRSLocks(ctx, []string{rc.getUserLockKey(userid)}, func() error { + userData, err := rc.GetUserData(ctx) + if err != nil && !errors.Is(err, zerr.ErrUserDataNotFound) { + return err + } + + userData.APIKeys[hashedKey] = *apiKeyDetails + + userDataBlob, err := json.Marshal(userData) + if err != nil { + return err + } + + _, err = rc.Client.TxPipelined(ctx, func(txrp redis.Pipeliner) error { + if err := txrp.HSet(ctx, rc.UserDataKey, userid, userDataBlob).Err(); err != nil { + rc.Log.Error().Err(err).Str("hset", rc.UserDataKey).Str("userid", userid). + Msg("failed to set user data record") + + return fmt.Errorf("failed to set user data for identity %s: %w", userid, err) + } + + if err := txrp.HSet(ctx, rc.UserAPIKeysKey, hashedKey, userid).Err(); err != nil { + rc.Log.Error().Err(err).Str("hset", rc.UserAPIKeysKey).Str("userid", userid). + Msg("failed to set api key record") + + return fmt.Errorf("failed to set api key for identity %s: %w", userid, err) + } + + return nil + }) + + return err + }) + + return err +} + +func (rc *RedisDB) IsAPIKeyExpired(ctx context.Context, hashedKey string) (bool, error) { + userAc, err := reqCtx.UserAcFromContext(ctx) + if err != nil { + return false, err + } + + if userAc.IsAnonymous() { + return false, zerr.ErrUserDataNotAllowed + } + + userid := userAc.GetUsername() + + var isExpired bool + + // Lock used because getting API keys also updates their expired flag in the DB + err = rc.withRSLocks(ctx, []string{rc.getUserLockKey(userid)}, func() error { + userData, err := rc.GetUserData(ctx) + if err != nil && !errors.Is(err, zerr.ErrUserDataNotFound) { + return err + } + + apiKeyDetails := userData.APIKeys[hashedKey] + if apiKeyDetails.IsExpired { + isExpired = true + + return nil + } + + // if expiresAt is not nil value + if !apiKeyDetails.ExpirationDate.Equal(time.Time{}) && time.Now().After(apiKeyDetails.ExpirationDate) { + isExpired = true + apiKeyDetails.IsExpired = true + } + + userData.APIKeys[hashedKey] = apiKeyDetails + + userDataBlob, err := json.Marshal(userData) + if err != nil { + return err + } + + err = rc.Client.HSet(ctx, rc.UserDataKey, userid, userDataBlob).Err() + if err != nil { + rc.Log.Error().Err(err).Str("hset", rc.UserDataKey).Str("userid", userid). + Msg("failed to set user data record") + + return fmt.Errorf("failed to set user data for identity %s: %w", userid, err) + } + + return nil + }) + + return isExpired, err +} + +func (rc *RedisDB) UpdateUserAPIKeyLastUsed(ctx context.Context, hashedKey string) error { + userAc, err := reqCtx.UserAcFromContext(ctx) + if err != nil { + return err + } + + if userAc.IsAnonymous() { + return zerr.ErrUserDataNotAllowed + } + + userid := userAc.GetUsername() + + err = rc.withRSLocks(ctx, []string{rc.getUserLockKey(userid)}, func() error { + userData, err := rc.GetUserData(ctx) + if err != nil && !errors.Is(err, zerr.ErrUserDataNotFound) { + return err + } + + apiKeyDetails := userData.APIKeys[hashedKey] + apiKeyDetails.LastUsed = time.Now() + + userData.APIKeys[hashedKey] = apiKeyDetails + + userDataBlob, err := json.Marshal(userData) + if err != nil { + return err + } + + err = rc.Client.HSet(ctx, rc.UserDataKey, userid, userDataBlob).Err() + if err != nil { + rc.Log.Error().Err(err).Str("hset", rc.UserDataKey).Str("userid", userid). + Msg("failed to set user data record") + + return fmt.Errorf("failed to set user data for identity %s: %w", userid, err) + } + + return nil + }) + + return err +} + +func (rc *RedisDB) DeleteUserAPIKey(ctx context.Context, keyID string) error { + userAc, err := reqCtx.UserAcFromContext(ctx) + if err != nil { + return err + } + + if userAc.IsAnonymous() { + return zerr.ErrUserDataNotAllowed + } + + userid := userAc.GetUsername() + + err = rc.withRSLocks(ctx, []string{rc.getUserLockKey(userid)}, func() error { + userData, err := rc.GetUserData(ctx) + if err != nil { + return err + } + + for hash, apiKeyDetails := range userData.APIKeys { + if apiKeyDetails.UUID != keyID { + continue + } + + delete(userData.APIKeys, hash) + + userDataBlob, err := json.Marshal(userData) + if err != nil { + return err + } + + _, err = rc.Client.TxPipelined(ctx, func(txrp redis.Pipeliner) error { + if err = txrp.HSet(ctx, rc.UserDataKey, userid, userDataBlob).Err(); err != nil { + rc.Log.Error().Err(err).Str("hset", rc.UserDataKey).Str("userid", userid). + Msg("failed to set user data record") + + return fmt.Errorf("failed to set user data for identity %s: %w", userid, err) + } + + if err = txrp.HDel(ctx, rc.UserAPIKeysKey, hash).Err(); err != nil { + rc.Log.Error().Err(err).Str("hdel", rc.UserAPIKeysKey).Str("userid", userid). + Msg("failed to delete api key record") + + return fmt.Errorf("failed to delete api key record for identity %s: %w", userid, err) + } + + return nil + }) + } + + return nil + }) + + return err +} + +// SetImageMeta should NEVER be used in production as both GetImageMeta and SetImageMeta +// should be locked for the duration of the entire transaction at a higher level in the app. +func (rc *RedisDB) SetImageMeta(digest godigest.Digest, imageMeta mTypes.ImageMeta) error { + protoImageMeta := &proto_go.ImageMeta{} + ctx := context.Background() + + switch imageMeta.MediaType { + case ispec.MediaTypeImageManifest: + manifest := imageMeta.Manifests[0] + + protoImageMeta = mConvert.GetProtoImageManifestData(manifest.Manifest, manifest.Config, + manifest.Size, manifest.Digest.String()) + case ispec.MediaTypeImageIndex: + protoImageMeta = mConvert.GetProtoImageIndexMeta(*imageMeta.Index, imageMeta.Size, imageMeta.Digest.String()) + } + + pImageMetaBlob, err := proto.Marshal(protoImageMeta) + if err != nil { + return fmt.Errorf("failed to calculate blob for manifest with digest %s %w", digest, err) + } + + err = rc.withRSLocks(ctx, []string{rc.getImageLockKey(digest.String())}, func() error { + err = rc.Client.HSet(ctx, rc.ImageMetaKey, digest.String(), pImageMetaBlob).Err() + if err != nil { + rc.Log.Error().Err(err).Str("hset", rc.ImageMetaKey).Str("digest", digest.String()). + Msg("failed to set image meta record") + + return fmt.Errorf("failed to set image meta record for digest %s: %w", digest.String(), err) + } + + return nil + }) + + return err +} + +// SetRepoReference sets the given image data to the repo metadata. +func (rc *RedisDB) SetRepoReference(ctx context.Context, repo string, + reference string, imageMeta mTypes.ImageMeta, +) error { + if err := common.ValidateRepoReferenceInput(repo, reference, imageMeta.Digest); err != nil { + return err + } + + var userid string + + userAc, err := reqCtx.UserAcFromContext(ctx) + if err == nil { + userid = userAc.GetUsername() + } + + // 1. Add image data to db if needed + protoImageMeta := mConvert.GetProtoImageMeta(imageMeta) + + imageMetaBlob, err := proto.Marshal(protoImageMeta) + if err != nil { + return err + } + + locks := []string{rc.getImageLockKey(imageMeta.Digest.String()), rc.getRepoLockKey(repo)} + err = rc.withRSLocks(ctx, locks, func() error { + err := rc.Client.HSet(ctx, rc.ImageMetaKey, imageMeta.Digest.String(), imageMetaBlob).Err() + if err != nil { + rc.Log.Error().Err(err).Str("hset", rc.ImageMetaKey).Str("digest", imageMeta.Digest.String()). + Msg("failed to set image meta record") + + return fmt.Errorf("failed to set image meta record for digest %s: %w", imageMeta.Digest.String(), err) + } + + protoRepoMeta, err := rc.getProtoRepoMeta(ctx, repo) + if err != nil && !errors.Is(err, zerr.ErrRepoMetaNotFound) { + return err + } + + // 2. Referrers + if subject := mConvert.GetImageSubject(protoImageMeta); subject != nil { + refInfo := &proto_go.ReferrersInfo{} + if protoRepoMeta.Referrers[subject.Digest.String()] != nil { + refInfo = protoRepoMeta.Referrers[subject.Digest.String()] + } + + foundReferrer := false + + for i := range refInfo.List { + if refInfo.List[i].Digest == mConvert.GetImageDigestStr(protoImageMeta) { + foundReferrer = true + refInfo.List[i].Count += 1 + + break + } + } + + if !foundReferrer { + refInfo.List = append(refInfo.List, &proto_go.ReferrerInfo{ + Count: 1, + MediaType: protoImageMeta.MediaType, + Digest: mConvert.GetImageDigestStr(protoImageMeta), + ArtifactType: mConvert.GetImageArtifactType(protoImageMeta), + Size: mConvert.GetImageManifestSize(protoImageMeta), + Annotations: mConvert.GetImageAnnotations(protoImageMeta), + }) + } + + protoRepoMeta.Referrers[subject.Digest.String()] = refInfo + } + + // 3. Update tag + if !common.ReferenceIsDigest(reference) { + protoRepoMeta.Tags[reference] = &proto_go.TagDescriptor{ + Digest: imageMeta.Digest.String(), + MediaType: imageMeta.MediaType, + } + } + + if _, ok := protoRepoMeta.Statistics[imageMeta.Digest.String()]; !ok { + protoRepoMeta.Statistics[imageMeta.Digest.String()] = &proto_go.DescriptorStatistics{ + DownloadCount: 0, + LastPullTimestamp: ×tamppb.Timestamp{}, + PushTimestamp: timestamppb.Now(), + PushedBy: userid, + } + } else if protoRepoMeta.Statistics[imageMeta.Digest.String()].PushTimestamp.AsTime().IsZero() { + protoRepoMeta.Statistics[imageMeta.Digest.String()].PushTimestamp = timestamppb.Now() + } + + if _, ok := protoRepoMeta.Signatures[imageMeta.Digest.String()]; !ok { + protoRepoMeta.Signatures[imageMeta.Digest.String()] = &proto_go.ManifestSignatures{ + Map: map[string]*proto_go.SignaturesInfo{"": {}}, + } + } + + if _, ok := protoRepoMeta.Referrers[imageMeta.Digest.String()]; !ok { + protoRepoMeta.Referrers[imageMeta.Digest.String()] = &proto_go.ReferrersInfo{ + List: []*proto_go.ReferrerInfo{}, + } + } + + // 4. Blobs + repoBlobsBytes, err := rc.Client.HGet(ctx, rc.RepoBlobsKey, repo).Bytes() + if err != nil && !errors.Is(err, redis.Nil) { + rc.Log.Error().Err(err).Str("hget", rc.RepoBlobsKey).Str("repo", repo). + Msg("failed to get repo blobs record") + + return fmt.Errorf("failed to get repo blobs record for repo %s: %w", repo, err) + } + + repoBlobs, err := unmarshalProtoRepoBlobs(repo, repoBlobsBytes) + if err != nil { + return err + } + + protoRepoMeta, repoBlobs = common.AddImageMetaToRepoMeta(protoRepoMeta, repoBlobs, reference, imageMeta) + protoTime := timestamppb.New(time.Now()) + + protoTimeBlob, err := proto.Marshal(protoTime) + if err != nil { + return err + } + + repoBlobsBytes, err = proto.Marshal(repoBlobs) + if err != nil { + return err + } + + repoMetaBlob, err := proto.Marshal(protoRepoMeta) + if err != nil { + return err + } + + _, err = rc.Client.TxPipelined(ctx, func(txrp redis.Pipeliner) error { + if err := txrp.HSet(ctx, rc.RepoLastUpdatedKey, repo, protoTimeBlob).Err(); err != nil { + rc.Log.Error().Err(err).Str("hset", rc.RepoLastUpdatedKey).Str("repo", repo). + Msg("failed to put repo last updated timestamp") + + return fmt.Errorf("failed to put repo last updated record for repo %s: %w", repo, err) + } + + if err := txrp.HSet(ctx, rc.RepoBlobsKey, repo, repoBlobsBytes).Err(); err != nil { + rc.Log.Error().Err(err).Str("hset", rc.RepoBlobsKey).Str("repo", repo). + Msg("failed to put repo blobs record") + + return fmt.Errorf("failed to set repo blobs record for repo %s: %w", repo, err) + } + + if err := txrp.HSet(ctx, rc.RepoMetaKey, repo, repoMetaBlob).Err(); err != nil { + rc.Log.Error().Err(err).Str("hset", rc.RepoMetaKey).Str("repo", repo). + Msg("failed to put repo meta record") + + return fmt.Errorf("failed to put repometa record for repo %s: %w", repo, err) + } + + return nil + }) + + return err + }) + + return err +} + +// SearchRepos searches for repos given a search string. +func (rc *RedisDB) SearchRepos(ctx context.Context, searchText string) ([]mTypes.RepoMeta, error) { + foundRepos := []mTypes.RepoMeta{} + + repoMetaEntries, err := rc.Client.HGetAll(ctx, rc.RepoMetaKey).Result() + if err != nil { + rc.Log.Error().Err(err).Str("hgetall", rc.RepoMetaKey).Msg("failed to get all repo meta records") + + return foundRepos, fmt.Errorf("failed to get all repo meta records: %w", err) + } + + userBookmarks, userStars := rc.getUserBookmarksAndStarsNoError(ctx) + + for repo, repoMetaBlob := range repoMetaEntries { + if ok, err := reqCtx.RepoIsUserAvailable(ctx, repo); !ok || err != nil { + continue + } + + rank := common.RankRepoName(searchText, repo) + if rank == -1 { + continue + } + + protoRepoMeta, err := unmarshalProtoRepoMeta(repo, []byte(repoMetaBlob)) + if err != nil { + // similarly with other metadb implementations, do not return a partial result on error + return []mTypes.RepoMeta{}, err + } + + delete(protoRepoMeta.Tags, "") + + if len(protoRepoMeta.Tags) == 0 { + continue + } + + protoRepoMeta.Rank = int32(rank) //nolint:gosec // ignore overflow + protoRepoMeta.IsBookmarked = zcommon.Contains(userBookmarks, protoRepoMeta.Name) + protoRepoMeta.IsStarred = zcommon.Contains(userStars, protoRepoMeta.Name) + + repoMeta := mConvert.GetRepoMeta(protoRepoMeta) + foundRepos = append(foundRepos, repoMeta) + } + + return foundRepos, err +} + +// SearchTags searches for images(repo:tag) given a search string. +func (rc *RedisDB) SearchTags(ctx context.Context, searchText string) ([]mTypes.FullImageMeta, error) { + images := []mTypes.FullImageMeta{} + + searchedRepo, searchedTag, err := common.GetRepoTag(searchText) + if err != nil { + return images, fmt.Errorf("failed to parse search text, invalid format %w", err) + } + + repoMetaEntries, err := rc.Client.HGetAll(ctx, rc.RepoMetaKey).Result() + if err != nil { + rc.Log.Error().Err(err).Str("hgetall", rc.RepoMetaKey).Msg("failed to get all repo meta records") + + return images, fmt.Errorf("failed to get all repo meta records: %w", err) + } + + userBookmarks, userStars := rc.getUserBookmarksAndStarsNoError(ctx) + + for repo, repoMetaBlob := range repoMetaEntries { + if repo != searchedRepo { + continue + } + + if ok, err := reqCtx.RepoIsUserAvailable(ctx, repo); !ok || err != nil { + return images, err + } + + protoRepoMeta, err := unmarshalProtoRepoMeta(repo, []byte(repoMetaBlob)) + if err != nil { + return images, err + } + + delete(protoRepoMeta.Tags, "") + + protoRepoMeta.IsBookmarked = zcommon.Contains(userBookmarks, protoRepoMeta.Name) + protoRepoMeta.IsStarred = zcommon.Contains(userStars, protoRepoMeta.Name) + + for tag, descriptor := range protoRepoMeta.Tags { + if !strings.HasPrefix(tag, searchedTag) || tag == "" { + continue + } + + var protoImageMeta *proto_go.ImageMeta + + switch descriptor.MediaType { + case ispec.MediaTypeImageManifest: + manifestDigest := descriptor.Digest + + imageManifestData, err := rc.getProtoImageMeta(ctx, manifestDigest) + if err != nil { + return images, fmt.Errorf("failed to fetch manifest meta for manifest with digest %s %w", + manifestDigest, err) + } + + protoImageMeta = imageManifestData + case ispec.MediaTypeImageIndex: + indexDigest := descriptor.Digest + + imageIndexData, err := rc.getProtoImageMeta(ctx, indexDigest) + if err != nil { + return images, fmt.Errorf("failed to fetch manifest meta for manifest with digest %s %w", + indexDigest, err) + } + + _, manifestDataList, err := rc.getAllContainedMeta(ctx, imageIndexData) + if err != nil { + return images, err + } + + imageIndexData.Manifests = manifestDataList + + protoImageMeta = imageIndexData + default: + rc.Log.Error().Str("mediaType", descriptor.MediaType).Msg("unsupported media type") + + continue + } + + images = append(images, mConvert.GetFullImageMetaFromProto(tag, protoRepoMeta, protoImageMeta)) + } + } + + return images, err +} + +// FilterTags filters for images given a filter function. +func (rc *RedisDB) FilterTags(ctx context.Context, filterRepoTag mTypes.FilterRepoTagFunc, + filterFunc mTypes.FilterFunc, +) ([]mTypes.FullImageMeta, error) { + images := []mTypes.FullImageMeta{} + + repoMetaEntries, err := rc.Client.HGetAll(ctx, rc.RepoMetaKey).Result() + if err != nil { + rc.Log.Error().Err(err).Str("hgetall", rc.RepoMetaKey).Msg("failed to get all repo meta records") + + return images, fmt.Errorf("failed to get all repo meta records: %w", err) + } + + userBookmarks, userStars := rc.getUserBookmarksAndStarsNoError(ctx) + + var unifiedErr error + + for repo, repoMetaBlob := range repoMetaEntries { + if ok, err := reqCtx.RepoIsUserAvailable(ctx, repo); !ok || err != nil { + continue + } + + protoRepoMeta, err := unmarshalProtoRepoMeta(repo, []byte(repoMetaBlob)) + if err != nil { + unifiedErr = errors.Join(unifiedErr, err) + + continue + } + + delete(protoRepoMeta.Tags, "") + protoRepoMeta.IsBookmarked = zcommon.Contains(userBookmarks, protoRepoMeta.Name) + protoRepoMeta.IsStarred = zcommon.Contains(userStars, protoRepoMeta.Name) + repoMeta := mConvert.GetRepoMeta(protoRepoMeta) + + for tag, descriptor := range protoRepoMeta.Tags { + if !filterRepoTag(repo, tag) { + continue + } + + switch descriptor.MediaType { + case ispec.MediaTypeImageManifest: + manifestDigest := descriptor.Digest + + imageManifestData, err := rc.getProtoImageMeta(ctx, manifestDigest) + if err != nil { + unifiedErr = errors.Join(unifiedErr, err) + + continue + } + + imageMeta := mConvert.GetImageMeta(imageManifestData) + + if filterFunc(repoMeta, imageMeta) { + images = append(images, mConvert.GetFullImageMetaFromProto(tag, protoRepoMeta, imageManifestData)) + } + case ispec.MediaTypeImageIndex: + indexDigest := descriptor.Digest + + protoImageIndexMeta, err := rc.getProtoImageMeta(ctx, indexDigest) + if err != nil { + unifiedErr = errors.Join(unifiedErr, err) + + continue + } + + imageIndexMeta := mConvert.GetImageMeta(protoImageIndexMeta) + matchedManifests := []*proto_go.ManifestMeta{} + + imageManifestDataList, _, err := rc.getAllContainedMeta(ctx, protoImageIndexMeta) + if err != nil { + unifiedErr = errors.Join(unifiedErr, err) + + continue + } + + for _, imageManifestData := range imageManifestDataList { + imageMeta := mConvert.GetImageMeta(imageManifestData) + partialImageMeta := common.GetPartialImageMeta(imageIndexMeta, imageMeta) + + if filterFunc(repoMeta, partialImageMeta) { + matchedManifests = append(matchedManifests, imageManifestData.Manifests[0]) + } + } + + if len(matchedManifests) > 0 { + protoImageIndexMeta.Manifests = matchedManifests + + images = append(images, mConvert.GetFullImageMetaFromProto(tag, protoRepoMeta, protoImageIndexMeta)) + } + default: + rc.Log.Error().Str("mediaType", descriptor.MediaType).Msg("unsupported media type") + + continue + } + } + } + + return images, unifiedErr +} + +// FilterRepos filters for repos given a filter function. +func (rc *RedisDB) FilterRepos(ctx context.Context, acceptName mTypes.FilterRepoNameFunc, + filterFunc mTypes.FilterFullRepoFunc, +) ([]mTypes.RepoMeta, error) { + foundRepos := []mTypes.RepoMeta{} + + repoMetaEntries, err := rc.Client.HGetAll(ctx, rc.RepoMetaKey).Result() + if err != nil { + rc.Log.Error().Err(err).Str("hgetall", rc.RepoMetaKey).Msg("failed to get all repo meta records") + + return foundRepos, fmt.Errorf("failed to get all repo meta records: %w", err) + } + + userBookmarks, userStars := rc.getUserBookmarksAndStarsNoError(ctx) + + for repo, repoMetaBlob := range repoMetaEntries { + if ok, err := reqCtx.RepoIsUserAvailable(ctx, repo); !ok || err != nil { + continue + } + + if !acceptName(repo) { + continue + } + + protoRepoMeta, err := unmarshalProtoRepoMeta(repo, []byte(repoMetaBlob)) + if err != nil { + // similarly with other metadb implementations, do not return a partial result on error + return []mTypes.RepoMeta{}, err + } + + protoRepoMeta.IsBookmarked = zcommon.Contains(userBookmarks, protoRepoMeta.Name) + protoRepoMeta.IsStarred = zcommon.Contains(userStars, protoRepoMeta.Name) + + repoMeta := mConvert.GetRepoMeta(protoRepoMeta) + + if filterFunc(repoMeta) { + foundRepos = append(foundRepos, repoMeta) + } + } + + return foundRepos, nil +} + +// GetRepoMeta returns the full information about a repo. +func (rc *RedisDB) GetRepoMeta(ctx context.Context, repo string) (mTypes.RepoMeta, error) { + protoRepoMeta, err := rc.getProtoRepoMeta(ctx, repo) + if err != nil { + return mTypes.RepoMeta{}, err + } + + userBookmarks, userStars := rc.getUserBookmarksAndStarsNoError(ctx) + + delete(protoRepoMeta.Tags, "") + protoRepoMeta.IsBookmarked = zcommon.Contains(userBookmarks, repo) + protoRepoMeta.IsStarred = zcommon.Contains(userStars, repo) + + return mConvert.GetRepoMeta(protoRepoMeta), err +} + +// GetFullImageMeta returns the full information about an image. +func (rc *RedisDB) GetFullImageMeta(ctx context.Context, repo string, tag string) (mTypes.FullImageMeta, error) { + protoImageMeta := &proto_go.ImageMeta{} + + protoRepoMeta, err := rc.getProtoRepoMeta(ctx, repo) + if err != nil { + return mConvert.GetFullImageMetaFromProto(tag, protoRepoMeta, protoImageMeta), err + } + + userBookmarks, userStars := rc.getUserBookmarksAndStarsNoError(ctx) + + delete(protoRepoMeta.Tags, "") + protoRepoMeta.IsBookmarked = zcommon.Contains(userBookmarks, repo) + protoRepoMeta.IsStarred = zcommon.Contains(userStars, repo) + + descriptor, ok := protoRepoMeta.Tags[tag] + if !ok { + return mConvert.GetFullImageMetaFromProto(tag, protoRepoMeta, protoImageMeta), + fmt.Errorf("%w for tag %s in repo %s", zerr.ErrImageMetaNotFound, tag, repo) + } + + protoImageMeta, err = rc.getProtoImageMeta(ctx, descriptor.Digest) + if err != nil { + return mConvert.GetFullImageMetaFromProto(tag, protoRepoMeta, protoImageMeta), err + } + + if protoImageMeta.MediaType == ispec.MediaTypeImageIndex { + _, manifestDataList, err := rc.getAllContainedMeta(ctx, protoImageMeta) + if err != nil { + return mConvert.GetFullImageMetaFromProto(tag, protoRepoMeta, protoImageMeta), err + } + + protoImageMeta.Manifests = manifestDataList + } + + return mConvert.GetFullImageMetaFromProto(tag, protoRepoMeta, protoImageMeta), nil +} + +// GetImageMeta returns the raw information about an image. +func (rc *RedisDB) GetImageMeta(digest godigest.Digest) (mTypes.ImageMeta, error) { + imageMeta := mTypes.ImageMeta{} + ctx := context.Background() + + protoImageMeta, err := rc.getProtoImageMeta(ctx, digest.String()) + if err != nil { + return imageMeta, err + } + + if protoImageMeta.MediaType == ispec.MediaTypeImageIndex { + _, manifestDataList, err := rc.getAllContainedMeta(ctx, protoImageMeta) + if err != nil { + return imageMeta, err + } + + protoImageMeta.Manifests = manifestDataList + } + + imageMeta = mConvert.GetImageMeta(protoImageMeta) + + return imageMeta, err +} + +// GetMultipleRepoMeta returns a list of all repos that match the given filter function. +func (rc *RedisDB) GetMultipleRepoMeta(ctx context.Context, filter func(repoMeta mTypes.RepoMeta) bool) ( + []mTypes.RepoMeta, error, +) { + foundRepos := []mTypes.RepoMeta{} + + repoMetaEntries, err := rc.Client.HGetAll(ctx, rc.RepoMetaKey).Result() + if err != nil { + rc.Log.Error().Err(err).Str("hgetall", rc.RepoMetaKey).Msg("failed to get all repo meta records") + + return foundRepos, fmt.Errorf("failed to get all repometa records: %w", err) + } + + for repo, repoMetaBlob := range repoMetaEntries { + if ok, err := reqCtx.RepoIsUserAvailable(ctx, repo); !ok || err != nil { + continue + } + + protoRepoMeta, err := unmarshalProtoRepoMeta(repo, []byte(repoMetaBlob)) + if err != nil { + // similarly with other metadb implementations, return a partial result on error + return foundRepos, err + } + + delete(protoRepoMeta.Tags, "") + + repoMeta := mConvert.GetRepoMeta(protoRepoMeta) + + if filter(repoMeta) { + foundRepos = append(foundRepos, repoMeta) + } + } + + return foundRepos, err +} + +// AddManifestSignature adds signature metadata to a given manifest in the database. +func (rc *RedisDB) AddManifestSignature(repo string, signedManifestDigest godigest.Digest, + sigMeta mTypes.SignatureMetadata, +) error { + ctx := context.Background() + + err := rc.withRSLocks(ctx, []string{rc.getRepoLockKey(repo)}, func() error { + protoRepoMeta, err := rc.getProtoRepoMeta(ctx, repo) + if err != nil && !errors.Is(err, zerr.ErrRepoMetaNotFound) { + return err + } + + if errors.Is(err, zerr.ErrRepoMetaNotFound) { + var err error + // create a new object + repoMeta := proto_go.RepoMeta{ + Name: repo, + Tags: map[string]*proto_go.TagDescriptor{"": {}}, + Signatures: map[string]*proto_go.ManifestSignatures{ + signedManifestDigest.String(): { + Map: map[string]*proto_go.SignaturesInfo{ + sigMeta.SignatureType: { + List: []*proto_go.SignatureInfo{ + { + SignatureManifestDigest: sigMeta.SignatureDigest, + LayersInfo: mConvert.GetProtoLayersInfo(sigMeta.LayersInfo), + }, + }, + }, + }, + }, + }, + Referrers: map[string]*proto_go.ReferrersInfo{"": {}}, + Statistics: map[string]*proto_go.DescriptorStatistics{"": {}}, + } + + repoMetaBlob, err := proto.Marshal(&repoMeta) + if err != nil { + return err + } + + if err := rc.Client.HSet(ctx, rc.RepoMetaKey, repo, repoMetaBlob).Err(); err != nil { + rc.Log.Error().Err(err).Str("hset", rc.RepoMetaKey).Str("repo", repo). + Msg("failed to put repo meta record") + + return fmt.Errorf("failed to put repometa record for repo %s: %w", repo, err) + } + + return nil + } + + var ( + manifestSignatures *proto_go.ManifestSignatures + found bool + ) + + if manifestSignatures, found = protoRepoMeta.Signatures[signedManifestDigest.String()]; !found { + manifestSignatures = &proto_go.ManifestSignatures{Map: map[string]*proto_go.SignaturesInfo{"": {}}} + } + + signatureSlice := &proto_go.SignaturesInfo{List: []*proto_go.SignatureInfo{}} + if sigSlice, found := manifestSignatures.Map[sigMeta.SignatureType]; found { + signatureSlice = sigSlice + } + + if !common.ProtoSignatureAlreadyExists(signatureSlice.List, sigMeta) { + switch sigMeta.SignatureType { + case zcommon.NotationSignature: + signatureSlice.List = append(signatureSlice.List, &proto_go.SignatureInfo{ + SignatureManifestDigest: sigMeta.SignatureDigest, + LayersInfo: mConvert.GetProtoLayersInfo(sigMeta.LayersInfo), + }) + case zcommon.CosignSignature: + newCosignSig := &proto_go.SignatureInfo{ + SignatureManifestDigest: sigMeta.SignatureDigest, + LayersInfo: mConvert.GetProtoLayersInfo(sigMeta.LayersInfo), + } + + if zcommon.IsCosignTag(sigMeta.SignatureTag) { + // the entry for "sha256-{digest}.sig" signatures should be overwritten if + // it exists or added on the first position if it doesn't exist + if len(signatureSlice.GetList()) == 0 { + signatureSlice.List = []*proto_go.SignatureInfo{newCosignSig} + } else { + signatureSlice.List[0] = newCosignSig + } + } else { + // the first position should be reserved for "sha256-{digest}.sig" signatures + if len(signatureSlice.GetList()) == 0 { + signatureSlice.List = []*proto_go.SignatureInfo{{ + SignatureManifestDigest: "", + LayersInfo: []*proto_go.LayersInfo{}, + }} + } + + signatureSlice.List = append(signatureSlice.List, newCosignSig) + } + } + } + + manifestSignatures.Map[sigMeta.SignatureType] = signatureSlice + protoRepoMeta.Signatures[signedManifestDigest.String()] = manifestSignatures + + repoMetaBlob, err := proto.Marshal(protoRepoMeta) + if err != nil { + return err + } + + err = rc.Client.HSet(ctx, rc.RepoMetaKey, repo, repoMetaBlob).Err() + if err != nil { + rc.Log.Error().Err(err).Str("hset", rc.RepoMetaKey).Str("repo", repo). + Msg("failed to put repo meta record") + + return fmt.Errorf("failed to put repometa record for repo %s: %w", repo, err) + } + + return nil + }) + + return err +} + +// DeleteSignature deletes signature metadata to a given manifest from the database. +func (rc *RedisDB) DeleteSignature(repo string, signedManifestDigest godigest.Digest, + sigMeta mTypes.SignatureMetadata, +) error { + ctx := context.Background() + + err := rc.withRSLocks(ctx, []string{rc.getRepoLockKey(repo)}, func() error { + protoRepoMeta, err := rc.getProtoRepoMeta(ctx, repo) + if err != nil { + return err + } + + manifestSignatures, found := protoRepoMeta.Signatures[signedManifestDigest.String()] + if !found { + return zerr.ErrImageMetaNotFound + } + + signatureSlice := manifestSignatures.Map[sigMeta.SignatureType] + + newSignatureSlice := make([]*proto_go.SignatureInfo, 0, len(signatureSlice.List)) + + for _, sigInfo := range signatureSlice.List { + if sigInfo.SignatureManifestDigest != sigMeta.SignatureDigest { + newSignatureSlice = append(newSignatureSlice, sigInfo) + } + } + + manifestSignatures.Map[sigMeta.SignatureType] = &proto_go.SignaturesInfo{List: newSignatureSlice} + protoRepoMeta.Signatures[signedManifestDigest.String()] = manifestSignatures + + repoMetaBlob, err := proto.Marshal(protoRepoMeta) + if err != nil { + return err + } + + err = rc.Client.HSet(ctx, rc.RepoMetaKey, repo, repoMetaBlob).Err() + if err != nil { + rc.Log.Error().Err(err).Str("hset", rc.RepoMetaKey).Str("repo", repo). + Msg("failed to put repo meta record") + + return fmt.Errorf("failed to put repometa record for repo %s: %w", repo, err) + } + + return nil + }) + + return err +} + +// UpdateSignaturesValidity checks and updates signatures validity of a given manifest. +func (rc *RedisDB) UpdateSignaturesValidity(ctx context.Context, repo string, manifestDigest godigest.Digest) error { + imgTrustStore := rc.ImageTrustStore() + + if imgTrustStore == nil { + return nil + } + + err := rc.withRSLocks(ctx, []string{rc.getRepoLockKey(repo)}, func() error { + // get ManifestData of signed manifest + protoImageMeta, err := rc.getProtoImageMeta(ctx, manifestDigest.String()) + if err != nil { + if errors.Is(err, zerr.ErrImageMetaNotFound) { + // manifest meta not found, updating signatures with details about validity and author will not be performed + return nil + } + + return err + } + + // update signatures with details about validity and author + protoRepoMeta, err := rc.getProtoRepoMeta(ctx, repo) + if err != nil { + return err + } + + manifestSignatures := proto_go.ManifestSignatures{Map: map[string]*proto_go.SignaturesInfo{"": {}}} + + for sigType, sigs := range protoRepoMeta.Signatures[manifestDigest.String()].Map { + if zcommon.IsContextDone(ctx) { + return ctx.Err() + } + + signaturesInfo := []*proto_go.SignatureInfo{} + + for _, sigInfo := range sigs.List { + layersInfo := []*proto_go.LayersInfo{} + + for _, layerInfo := range sigInfo.LayersInfo { + author, date, isTrusted, _ := imgTrustStore.VerifySignature(sigType, layerInfo.LayerContent, + layerInfo.SignatureKey, manifestDigest, mConvert.GetImageMeta(protoImageMeta), repo) + + if isTrusted { + layerInfo.Signer = author + } + + if !date.IsZero() { + layerInfo.Signer = author + layerInfo.Date = timestamppb.New(date) + } + + layersInfo = append(layersInfo, layerInfo) + } + + signaturesInfo = append(signaturesInfo, &proto_go.SignatureInfo{ + SignatureManifestDigest: sigInfo.SignatureManifestDigest, + LayersInfo: layersInfo, + }) + } + + manifestSignatures.Map[sigType] = &proto_go.SignaturesInfo{List: signaturesInfo} + } + + protoRepoMeta.Signatures[manifestDigest.String()] = &manifestSignatures + + repoMetaBlob, err := proto.Marshal(protoRepoMeta) + if err != nil { + return err + } + + err = rc.Client.HSet(ctx, rc.RepoMetaKey, repo, repoMetaBlob).Err() + if err != nil { + rc.Log.Error().Err(err).Str("hset", rc.RepoMetaKey).Str("repo", repo). + Msg("failed to put repo meta record") + + return fmt.Errorf("failed to put repometa record for repo %s: %w", repo, err) + } + + return nil + }) + + return err +} + +// IncrementRepoStars adds 1 to the star count of an image. +func (rc *RedisDB) IncrementRepoStars(repo string) error { + ctx := context.Background() + + err := rc.withRSLocks(ctx, []string{rc.getRepoLockKey(repo)}, func() error { + protoRepoMeta, err := rc.getProtoRepoMeta(ctx, repo) + if err != nil { + return err + } + + protoRepoMeta.Stars++ + + repoMetaBlob, err := proto.Marshal(protoRepoMeta) + if err != nil { + return err + } + + err = rc.Client.HSet(ctx, rc.RepoMetaKey, repo, repoMetaBlob).Err() + if err != nil { + rc.Log.Error().Err(err).Str("hset", rc.RepoMetaKey).Str("repo", repo). + Msg("failed to put repo meta record") + + return fmt.Errorf("failed to put repometa record for repo %s: %w", repo, err) + } + + return nil + }) + + return err +} + +// DecrementRepoStars subtracts 1 from the star count of an image. +func (rc *RedisDB) DecrementRepoStars(repo string) error { + ctx := context.Background() + + err := rc.withRSLocks(ctx, []string{rc.getRepoLockKey(repo)}, func() error { + protoRepoMeta, err := rc.getProtoRepoMeta(ctx, repo) + if err != nil { + return err + } + + if protoRepoMeta.Stars == 0 { + return nil + } + + protoRepoMeta.Stars-- + + repoMetaBlob, err := proto.Marshal(protoRepoMeta) + if err != nil { + return err + } + + err = rc.Client.HSet(ctx, rc.RepoMetaKey, repo, repoMetaBlob).Err() + if err != nil { + rc.Log.Error().Err(err).Str("hset", rc.RepoMetaKey).Str("repo", repo). + Msg("failed to put repo meta record") + + return fmt.Errorf("failed to put repometa record for repo %s: %w", repo, err) + } + + return nil + }) + + return err +} + +// SetRepoMeta should NEVER be used in production as both GetRepoMeta and SetRepoMeta +// should be locked for the duration of the entire transaction at a higher level in the app. +func (rc *RedisDB) SetRepoMeta(repo string, repoMeta mTypes.RepoMeta) error { + repoMeta.Name = repo + + repoMetaBlob, err := proto.Marshal(mConvert.GetProtoRepoMeta(repoMeta)) + if err != nil { + return err + } + + // The last update time is set to 0 in order to force an update in case of a next storage parsing + protoTime := timestamppb.New(time.Time{}) + + protoTimeBlob, err := proto.Marshal(protoTime) + if err != nil { + return err + } + + ctx := context.Background() + + err = rc.withRSLocks(ctx, []string{rc.getRepoLockKey(repo)}, func() error { + _, err := rc.Client.TxPipelined(ctx, func(txrp redis.Pipeliner) error { + if err := txrp.HSet(ctx, rc.RepoMetaKey, repo, repoMetaBlob).Err(); err != nil { + rc.Log.Error().Err(err).Str("hset", rc.RepoMetaKey).Str("repo", repo). + Msg("failed to put repo meta record") + + return fmt.Errorf("failed to put repometa record for repo %s: %w", repo, err) + } + + if err := txrp.HSet(ctx, rc.RepoLastUpdatedKey, repo, protoTimeBlob).Err(); err != nil { + rc.Log.Error().Err(err).Str("hset", rc.RepoLastUpdatedKey).Str("repo", repo). + Msg("failed to put repo last updated timestamp") + + return fmt.Errorf("failed to put repo last updated record for repo %s: %w", repo, err) + } + + return nil + }) + + return err + }) + + return err +} + +func (rc *RedisDB) DeleteRepoMeta(repo string) error { + ctx := context.Background() + + err := rc.withRSLocks(ctx, []string{rc.getRepoLockKey(repo)}, func() error { + _, err := rc.Client.TxPipelined(ctx, func(txrp redis.Pipeliner) error { + if err := txrp.HDel(ctx, rc.RepoMetaKey, repo).Err(); err != nil { + rc.Log.Error().Err(err).Str("hdel", rc.RepoMetaKey).Str("repo", repo). + Msg("failed to delete repo meta record") + + return fmt.Errorf("failed to delete repometa record for repo %s: %w", repo, err) + } + + if err := txrp.HDel(ctx, rc.RepoBlobsKey, repo).Err(); err != nil { + rc.Log.Error().Err(err).Str("hdel", rc.RepoBlobsKey).Str("repo", repo). + Msg("failed to put repo blobs record") + + return fmt.Errorf("failed to delete repo blobs record for repo %s: %w", repo, err) + } + + if err := txrp.HDel(ctx, rc.RepoLastUpdatedKey, repo).Err(); err != nil { + rc.Log.Error().Err(err).Str("hdel", rc.RepoLastUpdatedKey).Str("repo", repo). + Msg("failed to put repo last updated timestamp") + + return fmt.Errorf("failed to delete repo last updated record for repo %s: %w", repo, err) + } + + return nil + }) + + return err + }) + + return err +} + +// GetReferrersInfo returns a list of for all referrers of the given digest that match one of the +// artifact types. +func (rc *RedisDB) GetReferrersInfo(repo string, referredDigest godigest.Digest, + artifactTypes []string, +) ([]mTypes.ReferrerInfo, error) { + referrersInfoResult := []mTypes.ReferrerInfo{} + ctx := context.Background() + + protoRepoMeta, err := rc.getProtoRepoMeta(ctx, repo) + if err != nil { + return referrersInfoResult, err + } + + referrersInfo := protoRepoMeta.Referrers[referredDigest.String()].List + + for i := range referrersInfo { + if !common.MatchesArtifactTypes(referrersInfo[i].ArtifactType, artifactTypes) { + continue + } + + referrersInfoResult = append(referrersInfoResult, mTypes.ReferrerInfo{ + Digest: referrersInfo[i].Digest, + MediaType: referrersInfo[i].MediaType, + ArtifactType: referrersInfo[i].ArtifactType, + Size: int(referrersInfo[i].Size), + Annotations: referrersInfo[i].Annotations, + }) + } + + return referrersInfoResult, err +} + +// UpdateStatsOnDownload adds 1 to the download count of an image and sets the timestamp of download. +func (rc *RedisDB) UpdateStatsOnDownload(repo string, reference string) error { + ctx := context.Background() + + err := rc.withRSLocks(ctx, []string{rc.getRepoLockKey(repo)}, func() error { + protoRepoMeta, err := rc.getProtoRepoMeta(ctx, repo) + if err != nil { + return err + } + + manifestDigest := reference + + if common.ReferenceIsTag(reference) { + descriptor, found := protoRepoMeta.Tags[reference] + + if !found { + return zerr.ErrImageMetaNotFound + } + + manifestDigest = descriptor.Digest + } + + manifestStatistics, ok := protoRepoMeta.Statistics[manifestDigest] + if !ok { + return zerr.ErrImageMetaNotFound + } + + manifestStatistics.DownloadCount++ + manifestStatistics.LastPullTimestamp = timestamppb.Now() + protoRepoMeta.Statistics[manifestDigest] = manifestStatistics + + repoMetaBlob, err := proto.Marshal(protoRepoMeta) + if err != nil { + return err + } + + err = rc.Client.HSet(ctx, rc.RepoMetaKey, repo, repoMetaBlob).Err() + if err != nil { + rc.Log.Error().Err(err).Str("hset", rc.RepoMetaKey).Str("repo", repo). + Msg("failed to put repo meta record") + + return fmt.Errorf("failed to put repometa record for repo %s: %w", repo, err) + } + + return nil + }) + + return err +} + +// FilterImageMeta returns the image data for the given digests. +func (rc *RedisDB) FilterImageMeta(ctx context.Context, + digests []string, +) (map[mTypes.ImageDigest]mTypes.ImageMeta, error) { + imageMetaMap := map[string]mTypes.ImageMeta{} + + for _, digest := range digests { + protoImageMeta, err := rc.getProtoImageMeta(ctx, digest) + if err != nil { + return imageMetaMap, err + } + + if protoImageMeta.MediaType == ispec.MediaTypeImageIndex { + _, manifestDataList, err := rc.getAllContainedMeta(ctx, protoImageMeta) + if err != nil { + return imageMetaMap, err + } + + protoImageMeta.Manifests = manifestDataList + } + + imageMetaMap[digest] = mConvert.GetImageMeta(protoImageMeta) + } + + return imageMetaMap, nil +} + +/* + RemoveRepoReference removes the tag from RepoMetadata if the reference is a tag, + +it also removes its corresponding digest from Statistics, Signatures and Referrers if there are no tags +pointing to it. +If the reference is a digest then it will remove the digest from Statistics, Signatures and Referrers only +if there are no tags pointing to the digest, otherwise it's noop. +*/ +func (rc *RedisDB) RemoveRepoReference(repo, reference string, manifestDigest godigest.Digest) error { + ctx := context.Background() + + locks := []string{rc.getImageLockKey(manifestDigest.String()), rc.getRepoLockKey(repo)} + err := rc.withRSLocks(ctx, locks, func() error { + protoRepoMeta, err := rc.getProtoRepoMeta(ctx, repo) + if err != nil { + if errors.Is(err, zerr.ErrRepoMetaNotFound) { + return nil + } + + return err + } + + protoImageMeta, err := rc.getProtoImageMeta(ctx, manifestDigest.String()) + if err != nil { + if errors.Is(err, zerr.ErrImageMetaNotFound) { + return nil + } + + return err + } + + // Remove Referrers + if subject := mConvert.GetImageSubject(protoImageMeta); subject != nil { + referredDigest := subject.Digest.String() + refInfo := &proto_go.ReferrersInfo{} + + if protoRepoMeta.Referrers[referredDigest] != nil { + refInfo = protoRepoMeta.Referrers[referredDigest] + } + + referrers := refInfo.List + + for i := range referrers { + if referrers[i].Digest == manifestDigest.String() { + referrers[i].Count -= 1 + + if referrers[i].Count == 0 || common.ReferenceIsDigest(reference) { + referrers = append(referrers[:i], referrers[i+1:]...) + } + + break + } + } + + refInfo.List = referrers + + protoRepoMeta.Referrers[referredDigest] = refInfo + } + + if !common.ReferenceIsDigest(reference) { + delete(protoRepoMeta.Tags, reference) + } else { + // remove all tags pointing to this digest + for tag, desc := range protoRepoMeta.Tags { + if desc.Digest == reference { + delete(protoRepoMeta.Tags, tag) + } + } + } + + /* try to find at least one tag pointing to manifestDigest + if not found then we can also remove everything related to this digest */ + var foundTag bool + + for _, desc := range protoRepoMeta.Tags { + if desc.Digest == manifestDigest.String() { + foundTag = true + } + } + + if !foundTag { + delete(protoRepoMeta.Statistics, manifestDigest.String()) + delete(protoRepoMeta.Signatures, manifestDigest.String()) + delete(protoRepoMeta.Referrers, manifestDigest.String()) + } + + repoBlobsBytes, err := rc.Client.HGet(ctx, rc.RepoBlobsKey, repo).Bytes() + if err != nil && !errors.Is(err, redis.Nil) { + rc.Log.Error().Err(err).Str("hget", rc.RepoBlobsKey).Str("repo", repo). + Msg("failed to get repo blobs record") + + return fmt.Errorf("failed to get repo blobs record for repo %s: %w", repo, err) + } + + repoBlobs, err := unmarshalProtoRepoBlobs(repo, repoBlobsBytes) + if err != nil { + return err + } + + protoRepoMeta, repoBlobs = common.RemoveImageFromRepoMeta(protoRepoMeta, repoBlobs, reference) + protoTime := timestamppb.New(time.Now()) + + protoTimeBlob, err := proto.Marshal(protoTime) + if err != nil { + return err + } + + repoBlobsBytes, err = proto.Marshal(repoBlobs) + if err != nil { + return err + } + + repoMetaBlob, err := proto.Marshal(protoRepoMeta) + if err != nil { + return err + } + + _, err = rc.Client.TxPipelined(ctx, func(txrp redis.Pipeliner) error { + if err := txrp.HSet(ctx, rc.RepoLastUpdatedKey, repo, protoTimeBlob).Err(); err != nil { + rc.Log.Error().Err(err).Str("hset", rc.RepoLastUpdatedKey).Str("repo", repo). + Msg("failed to put repo last updated timestamp") + + return fmt.Errorf("failed to put repo last updated record for repo %s: %w", repo, err) + } + + if err := txrp.HSet(ctx, rc.RepoBlobsKey, repo, repoBlobsBytes).Err(); err != nil { + rc.Log.Error().Err(err).Str("hset", rc.RepoBlobsKey).Str("repo", repo). + Msg("failed to put repo blobs record") + + return fmt.Errorf("failed to set repo blobs record for repo %s: %w", repo, err) + } + + if err := txrp.HSet(ctx, rc.RepoMetaKey, repo, repoMetaBlob).Err(); err != nil { + rc.Log.Error().Err(err).Str("hset", rc.RepoMetaKey).Str("repo", repo). + Msg("failed to put repo meta record") + + return fmt.Errorf("failed to put repometa record for repo %s: %w", repo, err) + } + + return nil + }) + + return err + }) + + return err +} + +// ResetRepoReferences resets all layout specific data (tags, signatures, referrers, etc.) but keep user and image +// specific metadata such as star count, downloads other statistics. +func (rc *RedisDB) ResetRepoReferences(repo string) error { + ctx := context.Background() + + err := rc.withRSLocks(ctx, []string{rc.getRepoLockKey(repo)}, func() error { + protoRepoMeta, err := rc.getProtoRepoMeta(ctx, repo) + if err != nil && !errors.Is(err, zerr.ErrRepoMetaNotFound) { + return err + } + + repoMetaBlob, err := proto.Marshal(&proto_go.RepoMeta{ + Name: repo, + Statistics: protoRepoMeta.Statistics, + Stars: protoRepoMeta.Stars, + Tags: map[string]*proto_go.TagDescriptor{"": {}}, + Signatures: map[string]*proto_go.ManifestSignatures{"": {Map: map[string]*proto_go.SignaturesInfo{"": {}}}}, + Referrers: map[string]*proto_go.ReferrersInfo{"": {}}, + }) + if err != nil { + return err + } + + if err := rc.Client.HSet(ctx, rc.RepoMetaKey, repo, repoMetaBlob).Err(); err != nil { + rc.Log.Error().Err(err).Str("hset", rc.RepoMetaKey).Str("repo", repo). + Msg("failed to put repo meta record") + + return fmt.Errorf("failed to put repometa record for repo %s: %w", repo, err) + } + + return nil + }) + + return err +} + +func (rc *RedisDB) GetRepoLastUpdated(repo string) time.Time { + ctx := context.Background() + + lastUpdatedBlob, err := rc.Client.HGet(ctx, rc.RepoLastUpdatedKey, repo).Bytes() + if err != nil { + rc.Log.Error().Err(err).Str("hget", rc.RepoLastUpdatedKey).Str("repo", repo). + Msg("failed to get repo last updated timestamp") + + return time.Time{} + } + + if len(lastUpdatedBlob) == 0 { + return time.Time{} + } + + protoTime := ×tamppb.Timestamp{} + + err = proto.Unmarshal(lastUpdatedBlob, protoTime) + if err != nil { + return time.Time{} + } + + lastUpdated := *mConvert.GetTime(protoTime) + + return lastUpdated +} + +func (rc *RedisDB) GetAllRepoNames() ([]string, error) { + foundRepos := []string{} + ctx := context.Background() + + repoMetaEntries, err := rc.Client.HGetAll(ctx, rc.RepoMetaKey).Result() + if err != nil { + rc.Log.Error().Err(err).Str("hgetall", rc.RepoMetaKey).Msg("failed to get all repo meta records") + + return foundRepos, fmt.Errorf("failed to get all repometa records %w", err) + } + + for repo := range repoMetaEntries { + foundRepos = append(foundRepos, repo) + } + + return foundRepos, nil +} + +// ResetDB will delete all data in the DB. +// Ideally we would use locks here, but it would require a more complex logic to lock/unlock +// everything, and this function is only used in testing, so let's not add that complexity. +func (rc *RedisDB) ResetDB() error { + ctx := context.Background() + + _, err := rc.Client.TxPipelined(ctx, func(txrp redis.Pipeliner) error { + if err := txrp.Del(ctx, rc.RepoMetaKey).Err(); err != nil { + rc.Log.Error().Err(err).Str("del", rc.RepoMetaKey).Msg("failed to delete repo meta bucket") + + return fmt.Errorf("failed to delete repo meta bucket: %w", err) + } + + if err := txrp.Del(ctx, rc.ImageMetaKey).Err(); err != nil { + rc.Log.Error().Err(err).Str("del", rc.ImageMetaKey).Msg("failed to delete image meta bucket") + + return fmt.Errorf("failed to delete image meta bucket: %w", err) + } + + if err := txrp.Del(ctx, rc.RepoBlobsKey).Err(); err != nil { + rc.Log.Error().Err(err).Str("del", rc.RepoBlobsKey).Msg("failed to delete repo blobs bucket") + + return fmt.Errorf("failed to delete repo blobs bucket: %w", err) + } + + if err := txrp.Del(ctx, rc.RepoLastUpdatedKey).Err(); err != nil { + rc.Log.Error().Err(err).Str("del", rc.RepoLastUpdatedKey).Msg("failed to delete repo last updated bucket") + + return fmt.Errorf("failed to delete repo last updated bucket: %w", err) + } + + if err := txrp.Del(ctx, rc.UserDataKey).Err(); err != nil { + rc.Log.Error().Err(err).Str("del", rc.UserDataKey).Msg("failed to delete user data bucket") + + return fmt.Errorf("failed to delete user data bucket: %w", err) + } + + if err := txrp.Del(ctx, rc.UserAPIKeysKey).Err(); err != nil { + rc.Log.Error().Err(err).Str("del", rc.UserAPIKeysKey).Msg("failed to delete user api key bucket") + + return fmt.Errorf("failed to delete user api key bucket: %w", err) + } + + if err := txrp.Del(ctx, rc.VersionKey).Err(); err != nil { + rc.Log.Error().Err(err).Str("del", rc.VersionKey).Msg("failed to delete version bucket") + + return fmt.Errorf("failed to delete version bucket: %w", err) + } + + return nil + }) + + return err +} + +func (rc *RedisDB) PatchDB() error { + ctx := context.Background() + + err := rc.withRSLocks(ctx, []string{rc.getVersionLockKey()}, func() error { + var DBVersion string + + DBVersion, err := rc.Client.Get(ctx, rc.VersionKey).Result() + if err != nil { + if !errors.Is(err, redis.Nil) { + rc.Log.Error().Err(err).Str("get", rc.VersionKey).Msg("failed to get db version") + + return fmt.Errorf("patching the database failed, can't read db version: %w", err) + } + + // this is a new DB, we need to initialize the version + if err := rc.Client.Set(ctx, rc.VersionKey, rc.Version, 0).Err(); err != nil { + rc.Log.Error().Err(err).Str("set", rc.VersionKey). + Str("value", version.CurrentVersion).Msg("failed to set db version") + + return fmt.Errorf("patching the database failed, can't set db version: %w", err) + } + + // No need to apply patches on a new DB + return nil + } + + if version.GetVersionIndex(DBVersion) == -1 { + return fmt.Errorf("%w: %s could not identify patches", zerr.ErrInvalidMetaDBVersion, DBVersion) + } + + for patchIndex, patch := range rc.Patches { + if patchIndex < version.GetVersionIndex(DBVersion) { + continue + } + + err := patch(rc.Client) + if err != nil { + return err + } + } + + return nil + }) + + return err +} + +func (rc *RedisDB) ImageTrustStore() mTypes.ImageTrustStore { + return rc.imgTrustStore +} + +func (rc *RedisDB) SetImageTrustStore(imgTrustStore mTypes.ImageTrustStore) { + rc.imgTrustStore = imgTrustStore +} + +// getUserBookmarksAndStarsNoError is used in several calls where we don't want +// to fail if the user data is unavailable, such as the case of getting all repos for +// anonymous users, or using metaDB internaly for CVE scanning repos. +func (rc *RedisDB) getUserBookmarksAndStarsNoError(ctx context.Context) ([]string, []string) { + userData, err := rc.GetUserData(ctx) + if err != nil { + return []string{}, []string{} + } + + return userData.BookmarkedRepos, userData.StarredRepos +} + +func (rc *RedisDB) getProtoImageMeta(ctx context.Context, digest string) (*proto_go.ImageMeta, error) { + imageMetaBlob, err := rc.Client.HGet(ctx, rc.ImageMetaKey, digest).Bytes() + if err != nil && !errors.Is(err, redis.Nil) { + rc.Log.Error().Err(err).Str("hget", rc.ImageMetaKey).Str("digest", digest). + Msg("failed to get image meta record") + + return nil, fmt.Errorf("failed to get image meta record for digest %s: %w", digest, err) + } + + if len(imageMetaBlob) == 0 || errors.Is(err, redis.Nil) { + return nil, fmt.Errorf("%w for digest %s", zerr.ErrImageMetaNotFound, digest) + } + + imageMeta := proto_go.ImageMeta{} + + err = proto.Unmarshal(imageMetaBlob, &imageMeta) + if err != nil { + return nil, err + } + + return &imageMeta, nil +} + +func (rc *RedisDB) getAllContainedMeta(ctx context.Context, imageIndexData *proto_go.ImageMeta, +) ([]*proto_go.ImageMeta, []*proto_go.ManifestMeta, error) { + manifestDataList := make([]*proto_go.ManifestMeta, 0, len(imageIndexData.Index.Index.Manifests)) + imageMetaList := make([]*proto_go.ImageMeta, 0, len(imageIndexData.Index.Index.Manifests)) + + for _, manifest := range imageIndexData.Index.Index.Manifests { + imageManifestData, err := rc.getProtoImageMeta(ctx, manifest.Digest) + if err != nil { + return imageMetaList, manifestDataList, err + } + + switch imageManifestData.MediaType { + case ispec.MediaTypeImageManifest: + imageMetaList = append(imageMetaList, imageManifestData) + manifestDataList = append(manifestDataList, imageManifestData.Manifests[0]) + case ispec.MediaTypeImageIndex: + partialImageDataList, partialManifestDataList, err := rc.getAllContainedMeta(ctx, imageManifestData) + if err != nil { + return imageMetaList, manifestDataList, err + } + + imageMetaList = append(imageMetaList, partialImageDataList...) + manifestDataList = append(manifestDataList, partialManifestDataList...) + } + } + + return imageMetaList, manifestDataList, nil +} + +func (rc *RedisDB) getProtoRepoMeta(ctx context.Context, repo string) (*proto_go.RepoMeta, error) { + repoMetaBlob, err := rc.Client.HGet(ctx, rc.RepoMetaKey, repo).Bytes() + if err != nil && !errors.Is(err, redis.Nil) { + rc.Log.Error().Err(err).Str("hget", rc.RepoMetaKey).Str("repo", repo). + Msg("failed to get repo meta record") + + return nil, fmt.Errorf("failed to get repo meta record for repo %s: %w", repo, err) + } + + return unmarshalProtoRepoMeta(repo, repoMetaBlob) +} + +func (rc *RedisDB) withRSLocks(ctx context.Context, lockNames []string, wrappedFunc func() error) error { + for _, lockName := range lockNames { + lock := rc.RS.NewMutex(lockName) + + if err := lock.LockContext(ctx); err != nil { + rc.Log.Error().Err(err).Str("lockName", lockName).Msg("failed to acquire redis lock") + + return err + } + + defer func() { + if _, err := lock.UnlockContext(ctx); err != nil { + rc.Log.Error().Err(err).Str("lockName", lockName).Msg("failed to release redis lock") + } + }() + } + + return wrappedFunc() +} + +func (rc *RedisDB) getRepoLockKey(name string) string { + return strings.Join([]string{rc.LocksKey, "Repo", name}, ":") +} + +func (rc *RedisDB) getImageLockKey(name string) string { + return strings.Join([]string{rc.LocksKey, "Image", name}, ":") +} + +func (rc *RedisDB) getUserLockKey(name string) string { + return strings.Join([]string{rc.LocksKey, "User", name}, ":") +} + +func (rc *RedisDB) getVersionLockKey() string { + return strings.Join([]string{rc.LocksKey, "Version"}, ":") +} + +// unmarshalProtoRepoMeta will unmarshal the repoMeta blob and initialize nil maps. If the blob is empty +// an empty initialized object is returned. +func unmarshalProtoRepoMeta(repo string, repoMetaBlob []byte) (*proto_go.RepoMeta, error) { + protoRepoMeta := &proto_go.RepoMeta{ + Name: repo, + } + + if len(repoMetaBlob) > 0 { + err := proto.Unmarshal(repoMetaBlob, protoRepoMeta) + if err != nil { + return protoRepoMeta, err + } + } + + if protoRepoMeta.Tags == nil { + protoRepoMeta.Tags = map[string]*proto_go.TagDescriptor{"": {}} + } + + if protoRepoMeta.Statistics == nil { + protoRepoMeta.Statistics = map[string]*proto_go.DescriptorStatistics{"": {}} + } + + if protoRepoMeta.Signatures == nil { + protoRepoMeta.Signatures = map[string]*proto_go.ManifestSignatures{"": {}} + } + + if protoRepoMeta.Referrers == nil { + protoRepoMeta.Referrers = map[string]*proto_go.ReferrersInfo{"": {}} + } + + if len(repoMetaBlob) == 0 { + return protoRepoMeta, zerr.ErrRepoMetaNotFound + } + + return protoRepoMeta, nil +} + +// unmarshalProtoRepoBlobs will unmarshal the repoBlobs blob and initialize nil maps. If the blob is empty +// an empty initialized object is returned. +func unmarshalProtoRepoBlobs(repo string, repoBlobsBytes []byte) (*proto_go.RepoBlobs, error) { + repoBlobs := &proto_go.RepoBlobs{ + Name: repo, + } + + if len(repoBlobsBytes) > 0 { + err := proto.Unmarshal(repoBlobsBytes, repoBlobs) + if err != nil { + return nil, err + } + } + + if repoBlobs.Blobs == nil { + repoBlobs.Blobs = map[string]*proto_go.BlobInfo{"": {}} + } + + return repoBlobs, nil +} + +func join(xs ...string) string { + return strings.Join(xs, ":") +} diff --git a/pkg/meta/redis/redis_internal_test.go b/pkg/meta/redis/redis_internal_test.go new file mode 100644 index 00000000..2c9d9981 --- /dev/null +++ b/pkg/meta/redis/redis_internal_test.go @@ -0,0 +1,61 @@ +package redis + +import ( + "testing" + + "github.com/alicebob/miniredis/v2" + "github.com/redis/go-redis/v9" + . "github.com/smartystreets/goconvey/convey" + + "zotregistry.dev/zot/pkg/log" +) + +func Test(t *testing.T) { + Convey("Test redis metadb key generation", t, func() { + miniRedis := miniredis.RunT(t) + + log := log.NewLogger("debug", "") + So(log, ShouldNotBeNil) + + opts, err := redis.ParseURL("redis://" + miniRedis.Addr()) + So(err, ShouldBeNil) + + client := redis.NewClient(opts) + + params := DBDriverParameters{KeyPrefix: "zot"} + + metaDB, err := New(client, params, log) + So(err, ShouldBeNil) + So(metaDB.ImageMetaKey, ShouldEqual, "zot:ImageMeta") + So(metaDB.RepoMetaKey, ShouldEqual, "zot:RepoMeta") + So(metaDB.RepoLastUpdatedKey, ShouldEqual, "zot:RepoLastUpdated") + So(metaDB.RepoBlobsKey, ShouldEqual, "zot:RepoBlobsMeta") + So(metaDB.UserDataKey, ShouldEqual, "zot:UserData") + So(metaDB.UserAPIKeysKey, ShouldEqual, "zot:UserAPIKeys") + So(metaDB.VersionKey, ShouldEqual, "zot:Version") + So(metaDB.LocksKey, ShouldEqual, "zot:Locks") + + So(metaDB.getUserLockKey("user1"), ShouldEqual, "zot:Locks:User:user1") + So(metaDB.getRepoLockKey("repo1"), ShouldEqual, "zot:Locks:Repo:repo1") + So(metaDB.getImageLockKey("image1"), ShouldEqual, "zot:Locks:Image:image1") + So(metaDB.getVersionLockKey(), ShouldEqual, "zot:Locks:Version") + + params = DBDriverParameters{KeyPrefix: "someprefix"} + + metaDB, err = New(client, params, log) + So(err, ShouldBeNil) + So(metaDB.ImageMetaKey, ShouldEqual, "someprefix:ImageMeta") + So(metaDB.RepoMetaKey, ShouldEqual, "someprefix:RepoMeta") + So(metaDB.RepoLastUpdatedKey, ShouldEqual, "someprefix:RepoLastUpdated") + So(metaDB.RepoBlobsKey, ShouldEqual, "someprefix:RepoBlobsMeta") + So(metaDB.UserDataKey, ShouldEqual, "someprefix:UserData") + So(metaDB.UserAPIKeysKey, ShouldEqual, "someprefix:UserAPIKeys") + So(metaDB.VersionKey, ShouldEqual, "someprefix:Version") + So(metaDB.LocksKey, ShouldEqual, "someprefix:Locks") + + So(metaDB.getUserLockKey("user1"), ShouldEqual, "someprefix:Locks:User:user1") + So(metaDB.getRepoLockKey("repo1"), ShouldEqual, "someprefix:Locks:Repo:repo1") + So(metaDB.getImageLockKey("image1"), ShouldEqual, "someprefix:Locks:Image:image1") + So(metaDB.getVersionLockKey(), ShouldEqual, "someprefix:Locks:Version") + }) +} diff --git a/pkg/meta/redis/redis_test.go b/pkg/meta/redis/redis_test.go new file mode 100644 index 00000000..36c27089 --- /dev/null +++ b/pkg/meta/redis/redis_test.go @@ -0,0 +1,1532 @@ +package redis_test + +import ( + "context" + "errors" + "fmt" + "strings" + "testing" + "time" + + "github.com/alicebob/miniredis/v2" + "github.com/go-redis/redismock/v9" + godigest "github.com/opencontainers/go-digest" + ispec "github.com/opencontainers/image-spec/specs-go/v1" + goredis "github.com/redis/go-redis/v9" + . "github.com/smartystreets/goconvey/convey" + "google.golang.org/protobuf/proto" + + "zotregistry.dev/zot/pkg/log" + proto_go "zotregistry.dev/zot/pkg/meta/proto/gen" + "zotregistry.dev/zot/pkg/meta/redis" + mTypes "zotregistry.dev/zot/pkg/meta/types" + reqCtx "zotregistry.dev/zot/pkg/requestcontext" + test "zotregistry.dev/zot/pkg/test/common" + . "zotregistry.dev/zot/pkg/test/image-utils" +) + +const keyPrefix = "zot" + +var ErrTestError = errors.New("TestError") + +type imgTrustStore struct{} + +func (its imgTrustStore) VerifySignature( + signatureType string, rawSignature []byte, sigKey string, manifestDigest godigest.Digest, imageMeta mTypes.ImageMeta, + repo string, +) (mTypes.Author, mTypes.ExpiryDate, mTypes.Validity, error) { + return "", time.Time{}, false, nil +} + +func TestRedisMocked(t *testing.T) { + Convey("Test redis metadb implementation", t, func() { + log := log.NewLogger("debug", "") + So(log, ShouldNotBeNil) + + client, mock := redismock.NewClientMock() + defer DumpKeys(t, client) // Troubleshoot test failures + + mock.ExpectPing().SetVal("PONG") + + params := redis.DBDriverParameters{KeyPrefix: "zot"} + + metaDB, err := redis.New(client, params, log) + So(err, ShouldBeNil) + + Convey("GetAllRepoNames HGetAll error", func() { + mock.ExpectHGetAll(metaDB.RepoMetaKey). + SetErr(ErrTestError) + + repoNames, err := metaDB.GetAllRepoNames() + So(errors.Is(err, ErrTestError), ShouldEqual, true) + So(len(repoNames), ShouldEqual, 0) + + err = mock.ExpectationsWereMet() + So(err, ShouldBeNil) + }) + + Convey("GetAllRepoNames HGetAll succeeds", func() { + mock.ExpectHGetAll(metaDB.RepoMetaKey).SetVal( + map[string]string{ + "repo1": "meta1", + "repo2": "meta2", + "repo3": "meta3", + }, + ) + + repoNames, err := metaDB.GetAllRepoNames() + So(err, ShouldBeNil) + So(len(repoNames), ShouldEqual, 3) + + err = mock.ExpectationsWereMet() + So(err, ShouldBeNil) + }) + + Convey("ResetDB Del RepoMetaKey error", func() { + mock.ExpectTxPipeline() + mock.ExpectDel(metaDB.RepoMetaKey).SetErr(ErrTestError) + + err := metaDB.ResetDB() + So(err, ShouldNotBeNil) + + err = mock.ExpectationsWereMet() + So(err, ShouldBeNil) + }) + + Convey("ResetDB Del ImageMetaKey error", func() { + mock.ExpectTxPipeline() + mock.ExpectDel(metaDB.RepoMetaKey).SetVal(0) + mock.ExpectDel(metaDB.ImageMetaKey).SetErr(ErrTestError) + + err := metaDB.ResetDB() + So(err, ShouldNotBeNil) + + err = mock.ExpectationsWereMet() + So(err, ShouldBeNil) + }) + + Convey("ResetDB Del RepoBlobsKey error", func() { + mock.ExpectTxPipeline() + mock.ExpectDel(metaDB.RepoMetaKey).SetVal(0) + mock.ExpectDel(metaDB.ImageMetaKey).SetVal(0) + mock.ExpectDel(metaDB.RepoBlobsKey).SetErr(ErrTestError) + + err := metaDB.ResetDB() + So(err, ShouldNotBeNil) + + err = mock.ExpectationsWereMet() + So(err, ShouldBeNil) + }) + + Convey("ResetDB Del RepoLastUpdatedKey error", func() { + mock.ExpectTxPipeline() + mock.ExpectDel(metaDB.RepoMetaKey).SetVal(0) + mock.ExpectDel(metaDB.ImageMetaKey).SetVal(0) + mock.ExpectDel(metaDB.RepoBlobsKey).SetVal(0) + mock.ExpectDel(metaDB.RepoLastUpdatedKey).SetErr(ErrTestError) + + err := metaDB.ResetDB() + So(err, ShouldNotBeNil) + + err = mock.ExpectationsWereMet() + So(err, ShouldBeNil) + }) + + Convey("ResetDB Del UserDataKey error", func() { + mock.ExpectTxPipeline() + mock.ExpectDel(metaDB.RepoMetaKey).SetVal(0) + mock.ExpectDel(metaDB.ImageMetaKey).SetVal(0) + mock.ExpectDel(metaDB.RepoBlobsKey).SetVal(0) + mock.ExpectDel(metaDB.RepoLastUpdatedKey).SetVal(0) + mock.ExpectDel(metaDB.UserDataKey).SetErr(ErrTestError) + + err := metaDB.ResetDB() + So(err, ShouldNotBeNil) + + err = mock.ExpectationsWereMet() + So(err, ShouldBeNil) + }) + + Convey("ResetDB Del UserAPIKeysKey error", func() { + mock.ExpectTxPipeline() + mock.ExpectDel(metaDB.RepoMetaKey).SetVal(0) + mock.ExpectDel(metaDB.ImageMetaKey).SetVal(0) + mock.ExpectDel(metaDB.RepoBlobsKey).SetVal(0) + mock.ExpectDel(metaDB.RepoLastUpdatedKey).SetVal(0) + mock.ExpectDel(metaDB.UserDataKey).SetVal(0) + mock.ExpectDel(metaDB.UserAPIKeysKey).SetErr(ErrTestError) + + err := metaDB.ResetDB() + So(err, ShouldNotBeNil) + + err = mock.ExpectationsWereMet() + So(err, ShouldBeNil) + }) + + Convey("ResetDB Del VersionKey error", func() { + mock.ExpectTxPipeline() + mock.ExpectDel(metaDB.RepoMetaKey).SetVal(0) + mock.ExpectDel(metaDB.ImageMetaKey).SetVal(0) + mock.ExpectDel(metaDB.RepoBlobsKey).SetVal(0) + mock.ExpectDel(metaDB.RepoLastUpdatedKey).SetVal(0) + mock.ExpectDel(metaDB.UserDataKey).SetVal(0) + mock.ExpectDel(metaDB.UserAPIKeysKey).SetVal(0) + mock.ExpectDel(metaDB.VersionKey).SetErr(ErrTestError) + + err := metaDB.ResetDB() + So(err, ShouldNotBeNil) + + err = mock.ExpectationsWereMet() + So(err, ShouldBeNil) + }) + + Convey("DeleteRepoMeta Del RepoMetaKey error", func() { + mock.Regexp().ExpectSetNX(metaDB.LocksKey+":Repo:repo", `.*`, 8*time.Second). + SetVal(true) + mock.ExpectTxPipeline() + mock.ExpectHDel(metaDB.RepoMetaKey, "repo").SetErr(ErrTestError) + + err := metaDB.DeleteRepoMeta("repo") + So(err, ShouldNotBeNil) + + err = mock.ExpectationsWereMet() + So(err, ShouldBeNil) + }) + + Convey("DeleteRepoMeta Del RepoBlobsKey error", func() { + mock.Regexp().ExpectSetNX(metaDB.LocksKey+":Repo:repo", `.*`, 8*time.Second). + SetVal(true) + mock.ExpectTxPipeline() + mock.ExpectHDel(metaDB.RepoMetaKey, "repo").SetVal(0) + mock.ExpectHDel(metaDB.RepoBlobsKey, "repo").SetErr(ErrTestError) + + err := metaDB.DeleteRepoMeta("repo") + So(err, ShouldNotBeNil) + + err = mock.ExpectationsWereMet() + So(err, ShouldBeNil) + }) + + Convey("DeleteRepoMeta Del RepoLastUpdatedKey error", func() { + mock.Regexp().ExpectSetNX(metaDB.LocksKey+":Repo:repo", `.*`, 8*time.Second). + SetVal(true) + mock.ExpectTxPipeline() + mock.ExpectHDel(metaDB.RepoMetaKey, "repo").SetVal(0) + mock.ExpectHDel(metaDB.RepoBlobsKey, "repo").SetVal(0) + mock.ExpectHDel(metaDB.RepoLastUpdatedKey, "repo").SetErr(ErrTestError) + + err := metaDB.DeleteRepoMeta("repo") + So(err, ShouldNotBeNil) + + err = mock.ExpectationsWereMet() + So(err, ShouldBeNil) + }) + }) +} + +func TestRedisRepoMeta(t *testing.T) { + miniRedis := miniredis.RunT(t) + + Convey("Test repometa implementation", t, func() { + log := log.NewLogger("debug", "") + So(log, ShouldNotBeNil) + + opts, err := goredis.ParseURL("redis://" + miniRedis.Addr()) + So(err, ShouldBeNil) + + client := goredis.NewClient(opts) + defer DumpKeys(t, client) // Troubleshoot test failures + + params := redis.DBDriverParameters{KeyPrefix: "zot"} + + metaDB, err := redis.New(client, params, log) + So(err, ShouldBeNil) + + Convey("Test repoMeta ops", func() { + ctx := context.Background() + + // Create/Get repo meta + for i := range 5 { + repoName := fmt.Sprintf("repo%d", i+1) + digest := fmt.Sprintf("dig%d", i+1) + + initialRepoMeta := mTypes.RepoMeta{ + Name: repoName, + Tags: map[mTypes.Tag]mTypes.Descriptor{"tag": {Digest: digest}}, + + Statistics: map[mTypes.ImageDigest]mTypes.DescriptorStatistics{}, + Signatures: map[mTypes.ImageDigest]mTypes.ManifestSignatures{}, + Referrers: map[mTypes.ImageDigest][]mTypes.ReferrerInfo{"digest": {{Digest: digest}}}, + } + + err = metaDB.SetRepoMeta(repoName, initialRepoMeta) + So(err, ShouldBeNil) + + expectedRepoMeta, err := metaDB.GetRepoMeta(ctx, repoName) + So(err, ShouldBeNil) + + So(expectedRepoMeta.Name, ShouldEqual, initialRepoMeta.Name) + So(expectedRepoMeta.Tags["tag"].Digest, ShouldEqual, initialRepoMeta.Tags["tag"].Digest) + } + + // Get Multiple, Filter and Delete repo meta + repoMetas, err := metaDB.GetMultipleRepoMeta(ctx, func(repoMeta mTypes.RepoMeta) bool { + if strings.Contains(repoMeta.Name, "repo1") || strings.Contains(repoMeta.Name, "repo4") { + return true + } + + return false + }) + So(err, ShouldBeNil) + So(len(repoMetas), ShouldEqual, 2) + So(repoMetas[0].Name, ShouldNotEqual, repoMetas[1].Name) + + for _, repoMeta := range repoMetas { + So(repoMeta.Name, ShouldBeIn, []string{"repo1", "repo4"}) + } + + repoMetas, err = metaDB.FilterRepos(ctx, + func(repo string) bool { + return true + }, + func(repoMeta mTypes.RepoMeta) bool { + if strings.Contains(repoMeta.Tags["tag"].Digest, "dig2") || + strings.Contains(repoMeta.Tags["tag"].Digest, "dig5") { + return true + } + + return false + }, + ) + So(err, ShouldBeNil) + So(len(repoMetas), ShouldEqual, 2) + So(repoMetas[0].Tags["tag"].Digest, ShouldNotEqual, repoMetas[1].Tags["tag"].Digest) + + for _, repoMeta := range repoMetas { + So(repoMeta.Tags["tag"].Digest, ShouldBeIn, []string{"dig2", "dig5"}) + } + + repoNames, err := metaDB.GetAllRepoNames() + So(err, ShouldBeNil) + So(len(repoNames), ShouldEqual, 5) + + err = metaDB.DeleteRepoMeta("repo2") + So(err, ShouldBeNil) + + repoMeta, err := metaDB.GetRepoMeta(ctx, "repo2") + So(err, ShouldNotBeNil) + So(repoMeta.Name, ShouldBeEmpty) + + repoNames, err = metaDB.GetAllRepoNames() + So(err, ShouldBeNil) + So(len(repoNames), ShouldEqual, 4) + + repoMetas, err = metaDB.GetMultipleRepoMeta(ctx, func(repoMeta mTypes.RepoMeta) bool { return true }) + So(err, ShouldBeNil) + So(len(repoMetas), ShouldEqual, 4) + + repoMetas, err = metaDB.GetMultipleRepoMeta(ctx, func(repoMeta mTypes.RepoMeta) bool { return false }) + So(err, ShouldBeNil) + So(len(repoMetas), ShouldEqual, 0) + + repoMetas, err = metaDB.FilterRepos(ctx, + func(repo string) bool { + result := strings.Contains(repo, "repo5") + + return result + }, + func(repoMeta mTypes.RepoMeta) bool { + if strings.Contains(repoMeta.Tags["tag"].Digest, "dig3") || + strings.Contains(repoMeta.Tags["tag"].Digest, "dig5") { + return true + } + + return false + }, + ) + So(err, ShouldBeNil) + So(len(repoMetas), ShouldEqual, 1) + So(repoMetas[0].Tags["tag"].Digest, ShouldEqual, "dig5") + + repoMetas, err = metaDB.SearchRepos(ctx, "repo") + So(err, ShouldBeNil) + So(len(repoMetas), ShouldEqual, 4) + + repoMetas, err = metaDB.SearchRepos(ctx, "epo3") + So(err, ShouldBeNil) + So(len(repoMetas), ShouldEqual, 1) + + // Stars + repoMeta, err = metaDB.GetRepoMeta(ctx, "repo1") + So(err, ShouldBeNil) + So(repoMeta, ShouldNotBeNil) + So(repoMeta.StarCount, ShouldEqual, 0) + + err = metaDB.IncrementRepoStars("repo1") + So(err, ShouldBeNil) + err = metaDB.IncrementRepoStars("repo1") + So(err, ShouldBeNil) + + repoMeta, err = metaDB.GetRepoMeta(ctx, "repo1") + So(err, ShouldBeNil) + So(repoMeta, ShouldNotBeNil) + So(repoMeta.StarCount, ShouldEqual, 2) + + err = metaDB.DecrementRepoStars("repo1") + So(err, ShouldBeNil) + err = metaDB.DecrementRepoStars("repo1") + So(err, ShouldBeNil) + + repoMeta, err = metaDB.GetRepoMeta(ctx, "repo1") + So(err, ShouldBeNil) + So(repoMeta, ShouldNotBeNil) + So(repoMeta.StarCount, ShouldEqual, 0) + + err = metaDB.DecrementRepoStars("repo1") + So(err, ShouldBeNil) + + repoMeta, err = metaDB.GetRepoMeta(ctx, "repo1") + So(err, ShouldBeNil) + So(repoMeta, ShouldNotBeNil) + So(repoMeta.StarCount, ShouldEqual, 0) + }) + }) +} + +func TestRedisUnreachable(t *testing.T) { + Convey("Redis unreachable", t, func() { + miniRedis := miniredis.RunT(t) + + log := log.NewLogger("debug", "") + So(log, ShouldNotBeNil) + + connOpts, err := goredis.ParseURL("redis://" + miniRedis.Addr()) + So(err, ShouldBeNil) + workingClient := goredis.NewClient(connOpts) + + params := redis.DBDriverParameters{KeyPrefix: "zot"} + + metaDB, err := redis.New(workingClient, params, log) + So(err, ShouldBeNil) + So(metaDB, ShouldNotBeNil) + + connOpts, err = goredis.ParseURL("redis://127.0.0.1:" + test.GetFreePort()) + So(err, ShouldBeNil) + brokenClient := goredis.NewClient(connOpts) + + // Replace connection with the unreachable server + metaDB.Client = brokenClient + + metaDB.SetImageTrustStore(imgTrustStore{}) + + userAc := reqCtx.NewUserAccessControl() + userAc.SetUsername("test") + + ctx := userAc.DeriveContext(context.Background()) + + repo := "repo" + reference := "tag" + digest := godigest.FromString("SomeString") + image := CreateDefaultImage() + imageMeta := image.AsImageMeta() + + err = metaDB.SetImageMeta(digest, imageMeta) + So(err, ShouldNotBeNil) + + err = metaDB.SetRepoReference(ctx, repo, reference, imageMeta) + So(err, ShouldNotBeNil) + + _, err = metaDB.SearchRepos(ctx, repo) + So(err, ShouldNotBeNil) + + _, err = metaDB.SearchTags(ctx, repo+":"+reference) + So(err, ShouldNotBeNil) + + _, err = metaDB.FilterTags(ctx, mTypes.AcceptAllRepoTag, mTypes.AcceptAllImageMeta) + So(err, ShouldNotBeNil) + + _, err = metaDB.FilterRepos(ctx, mTypes.AcceptAllRepoNames, mTypes.AcceptAllRepoMeta) + So(err, ShouldNotBeNil) + + _, err = metaDB.GetRepoMeta(ctx, repo) + So(err, ShouldNotBeNil) + + _, err = metaDB.GetFullImageMeta(ctx, repo, reference) + So(err, ShouldNotBeNil) + + _, err = metaDB.GetImageMeta(digest) + So(err, ShouldNotBeNil) + + _, err = metaDB.GetMultipleRepoMeta(ctx, func(repoMeta mTypes.RepoMeta) bool { return true }) + So(err, ShouldNotBeNil) + + err = metaDB.AddManifestSignature(repo, digest, mTypes.SignatureMetadata{}) + So(err, ShouldNotBeNil) + + err = metaDB.DeleteSignature(repo, digest, mTypes.SignatureMetadata{}) + So(err, ShouldNotBeNil) + + err = metaDB.UpdateSignaturesValidity(ctx, repo, digest) + So(err, ShouldNotBeNil) + + err = metaDB.IncrementRepoStars(repo) + So(err, ShouldNotBeNil) + + err = metaDB.DecrementRepoStars(repo) + So(err, ShouldNotBeNil) + + err = metaDB.SetRepoMeta(repo, mTypes.RepoMeta{}) + So(err, ShouldNotBeNil) + + err = metaDB.DeleteRepoMeta(repo) + So(err, ShouldNotBeNil) + + _, err = metaDB.GetReferrersInfo(repo, digest, []string{}) + So(err, ShouldNotBeNil) + + err = metaDB.UpdateStatsOnDownload(repo, reference) + So(err, ShouldNotBeNil) + + _, err = metaDB.FilterImageMeta(ctx, []string{digest.String()}) + So(err, ShouldNotBeNil) + + err = metaDB.RemoveRepoReference(repo, reference, digest) + So(err, ShouldNotBeNil) + + err = metaDB.ResetRepoReferences(repo) + So(err, ShouldNotBeNil) + + t := metaDB.GetRepoLastUpdated(repo) + So(t, ShouldEqual, time.Time{}) + + _, err = metaDB.GetAllRepoNames() + So(err, ShouldNotBeNil) + + err = metaDB.ResetDB() + So(err, ShouldNotBeNil) + + err = metaDB.PatchDB() + So(err, ShouldNotBeNil) + + _, err = metaDB.GetStarredRepos(ctx) + So(err, ShouldNotBeNil) + + _, err = metaDB.GetBookmarkedRepos(ctx) + So(err, ShouldNotBeNil) + + _, err = metaDB.ToggleStarRepo(ctx, repo) + So(err, ShouldNotBeNil) + + _, err = metaDB.ToggleBookmarkRepo(ctx, repo) + So(err, ShouldNotBeNil) + + _, err = metaDB.GetUserData(ctx) + So(err, ShouldNotBeNil) + + err = metaDB.SetUserData(ctx, mTypes.UserData{}) + So(err, ShouldNotBeNil) + + err = metaDB.SetUserGroups(ctx, []string{}) + So(err, ShouldNotBeNil) + + _, err = metaDB.GetUserGroups(ctx) + So(err, ShouldNotBeNil) + + err = metaDB.DeleteUserData(ctx) + So(err, ShouldNotBeNil) + + _, err = metaDB.GetUserAPIKeyInfo("hash") + So(err, ShouldNotBeNil) + + _, err = metaDB.GetUserAPIKeys(ctx) + So(err, ShouldNotBeNil) + + err = metaDB.AddUserAPIKey(ctx, "hash", &mTypes.APIKeyDetails{}) + So(err, ShouldNotBeNil) + + _, err = metaDB.IsAPIKeyExpired(ctx, "hash") + So(err, ShouldNotBeNil) + + err = metaDB.UpdateUserAPIKeyLastUsed(ctx, "hash") + So(err, ShouldNotBeNil) + + err = metaDB.DeleteUserAPIKey(ctx, "test") + So(err, ShouldNotBeNil) + }) +} + +func TestWrapperErrors(t *testing.T) { + image := CreateDefaultImage() + imageMeta := image.AsImageMeta() + multiarchImageMeta := CreateMultiarchWith().Images([]Image{image}).Build().AsImageMeta() + + badProtoBlob := []byte("bad-repo-meta") + + goodRepoMetaBlob, err := proto.Marshal(&proto_go.RepoMeta{Name: "repo"}) + if err != nil { + t.FailNow() + } + + Convey("Errors", t, func() { + miniRedis := miniredis.RunT(t) + + log := log.NewLogger("debug", "") + So(log, ShouldNotBeNil) + + opts, err := goredis.ParseURL("redis://" + miniRedis.Addr()) + So(err, ShouldBeNil) + + client := goredis.NewClient(opts) + params := redis.DBDriverParameters{KeyPrefix: keyPrefix} + + metaDB, err := redis.New(client, params, log) + So(metaDB, ShouldNotBeNil) + So(err, ShouldBeNil) + + metaDB.SetImageTrustStore(imgTrustStore{}) + + userAc := reqCtx.NewUserAccessControl() + userAc.SetUsername("test") + + ctx := userAc.DeriveContext(context.Background()) + + Convey("RemoveRepoReference", func() { + Convey("getProtoRepoMeta errors", func() { + err := setRepoMeta("repo", badProtoBlob, client) + So(err, ShouldBeNil) + + err = metaDB.RemoveRepoReference("repo", "ref", imageMeta.Digest) + So(err, ShouldNotBeNil) + }) + + Convey("getProtoImageMeta errors", func() { + err := metaDB.SetRepoMeta("repo", mTypes.RepoMeta{ + Name: "repo", + Tags: map[mTypes.Tag]mTypes.Descriptor{ + "tag": { + MediaType: ispec.MediaTypeImageManifest, + Digest: imageMeta.Digest.String(), + }, + }, + }) + So(err, ShouldBeNil) + + err = setImageMeta(imageMeta.Digest, badProtoBlob, client) + So(err, ShouldBeNil) + + err = metaDB.RemoveRepoReference("repo", "ref", imageMeta.Digest) + So(err, ShouldNotBeNil) + }) + + Convey("unmarshalProtoRepoBlobs errors", func() { + err := metaDB.SetRepoReference(ctx, "repo", "tag", imageMeta) + So(err, ShouldBeNil) + + err = setRepoBlobInfo("repo", badProtoBlob, client) + So(err, ShouldBeNil) + + err = metaDB.RemoveRepoReference("repo", "ref", imageMeta.Digest) + So(err, ShouldNotBeNil) + }) + }) + + Convey("UpdateSignaturesValidity", func() { + metaDB.SetImageTrustStore(imgTrustStore{}) + + digest := image.Digest() + + ctx := context.Background() + + Convey("image meta blob not found", func() { + err := metaDB.UpdateSignaturesValidity(ctx, "repo", digest) + So(err, ShouldBeNil) + }) + + Convey("image meta unmarshal fail", func() { + err := setImageMeta(digest, badProtoBlob, client) + So(err, ShouldBeNil) + + err = metaDB.UpdateSignaturesValidity(ctx, "repo", digest) + So(err, ShouldNotBeNil) + }) + + Convey("repo meta blob not found", func() { + err := metaDB.SetImageMeta(digest, imageMeta) + So(err, ShouldBeNil) + + err = metaDB.UpdateSignaturesValidity(ctx, "repo", digest) + So(err, ShouldNotBeNil) + }) + + Convey("repo meta unmarshal fail", func() { + err := metaDB.SetImageMeta(digest, imageMeta) + So(err, ShouldBeNil) + + err = setRepoMeta("repo", badProtoBlob, client) + So(err, ShouldBeNil) + + err = metaDB.UpdateSignaturesValidity(ctx, "repo", digest) + So(err, ShouldNotBeNil) + }) + }) + + Convey("GetRepoLastUpdated", func() { + Convey("bad blob in db", func() { + err := setRepoLastUpdated("repo", []byte("bad-blob"), client) + So(err, ShouldBeNil) + + lastUpdated := metaDB.GetRepoLastUpdated("repo") + So(lastUpdated, ShouldEqual, time.Time{}) + }) + + Convey("empty blob in db", func() { + err := setRepoLastUpdated("repo", []byte(""), client) + So(err, ShouldBeNil) + + lastUpdated := metaDB.GetRepoLastUpdated("repo") + So(lastUpdated, ShouldEqual, time.Time{}) + }) + }) + + Convey("UpdateStatsOnDownload", func() { + Convey("repo meta not found", func() { + err = metaDB.UpdateStatsOnDownload("repo", "ref") + So(err, ShouldNotBeNil) + }) + + Convey("unmarshalProtoRepoMeta error", func() { + err := setRepoMeta("repo", badProtoBlob, client) + So(err, ShouldBeNil) + + err = metaDB.UpdateStatsOnDownload("repo", "ref") + So(err, ShouldNotBeNil) + }) + + Convey("ref is tag and tag is not found", func() { + err := metaDB.SetRepoReference(ctx, "repo", "tag", imageMeta) + So(err, ShouldBeNil) + + err = metaDB.UpdateStatsOnDownload("repo", "not-found-tag") + So(err, ShouldNotBeNil) + }) + + Convey("digest not found in statistics", func() { + err := metaDB.SetRepoReference(ctx, "repo", "tag", imageMeta) + So(err, ShouldBeNil) + + err = metaDB.UpdateStatsOnDownload("repo", godigest.FromString("not-found").String()) + So(err, ShouldNotBeNil) + }) + }) + + Convey("GetReferrersInfo", func() { + Convey("unmarshalProtoRepoMeta error", func() { + err := setRepoMeta("repo", badProtoBlob, client) + So(err, ShouldBeNil) + + _, err = metaDB.GetReferrersInfo("repo", "refDig", []string{}) + So(err, ShouldNotBeNil) + }) + }) + + Convey("ResetRepoReferences", func() { + Convey("unmarshalProtoRepoMeta error", func() { + err := setRepoMeta("repo", badProtoBlob, client) + So(err, ShouldBeNil) + + err = metaDB.ResetRepoReferences("repo") + So(err, ShouldNotBeNil) + }) + }) + + Convey("DecrementRepoStars", func() { + Convey("unmarshalProtoRepoMeta error", func() { + err := setRepoMeta("repo", badProtoBlob, client) + So(err, ShouldBeNil) + + err = metaDB.DecrementRepoStars("repo") + So(err, ShouldNotBeNil) + }) + }) + + Convey("IncrementRepoStars", func() { + Convey("unmarshalProtoRepoMeta error", func() { + err := setRepoMeta("repo", badProtoBlob, client) + So(err, ShouldBeNil) + + err = metaDB.IncrementRepoStars("repo") + So(err, ShouldNotBeNil) + }) + }) + + Convey("DeleteSignature", func() { + Convey("repo meta not found", func() { + err = metaDB.DeleteSignature("repo", godigest.FromString("dig"), mTypes.SignatureMetadata{}) + So(err, ShouldNotBeNil) + }) + + Convey("unmarshalProtoRepoMeta error", func() { + err := setRepoMeta("repo", badProtoBlob, client) + So(err, ShouldBeNil) + + err = metaDB.DeleteSignature("repo", godigest.FromString("dig"), mTypes.SignatureMetadata{}) + So(err, ShouldNotBeNil) + }) + }) + + Convey("AddManifestSignature", func() { + Convey("unmarshalProtoRepoMeta error", func() { + err := setRepoMeta("repo", badProtoBlob, client) + So(err, ShouldBeNil) + + err = metaDB.AddManifestSignature("repo", godigest.FromString("dig"), mTypes.SignatureMetadata{}) + So(err, ShouldNotBeNil) + }) + }) + + Convey("GetMultipleRepoMeta", func() { + Convey("unmarshalProtoRepoMeta error", func() { + err := setRepoMeta("repo", badProtoBlob, client) + So(err, ShouldBeNil) + + _, err = metaDB.GetMultipleRepoMeta(ctx, func(repoMeta mTypes.RepoMeta) bool { return true }) + So(err, ShouldNotBeNil) + }) + }) + + Convey("GetImageMeta", func() { + Convey("malformed image manifest", func() { + badImageDigest := godigest.FromString("bad-image-manifest") + + err = setImageMeta(badImageDigest, badProtoBlob, client) + So(err, ShouldBeNil) + + _, err := metaDB.GetImageMeta(badImageDigest) + So(err, ShouldNotBeNil) + }) + + Convey("good image index, malformed inside manifest", func() { + goodIndexBadManifestDigest := godigest.FromString("good-index-bad-manifests") + + err = metaDB.SetImageMeta(goodIndexBadManifestDigest, multiarchImageMeta) + So(err, ShouldBeNil) + + err = setImageMeta(image.Digest(), badProtoBlob, client) + So(err, ShouldBeNil) + + _, err := metaDB.GetImageMeta(image.Digest()) + So(err, ShouldNotBeNil) + }) + }) + + Convey("GetFullImageMeta", func() { + Convey("repo meta not found", func() { + _, err := metaDB.GetFullImageMeta(ctx, "repo", "tag") + So(err, ShouldNotBeNil) + }) + + Convey("unmarshalProtoRepoMeta fails", func() { + err := setRepoMeta("repo", badProtoBlob, client) + So(err, ShouldBeNil) + + _, err = metaDB.GetFullImageMeta(ctx, "repo", "tag") + So(err, ShouldNotBeNil) + }) + + Convey("tag not found", func() { + err := setRepoMeta("repo", goodRepoMetaBlob, client) + So(err, ShouldBeNil) + + _, err = metaDB.GetFullImageMeta(ctx, "repo", "tag-not-found") + So(err, ShouldNotBeNil) + }) + + Convey("getProtoImageMeta fails", func() { + err := metaDB.SetRepoMeta("repo", mTypes.RepoMeta{ + Name: "repo", + Tags: map[mTypes.Tag]mTypes.Descriptor{ + "tag": { + MediaType: ispec.MediaTypeImageManifest, + Digest: godigest.FromString("not-found").String(), + }, + }, + }) + So(err, ShouldBeNil) + + _, err = metaDB.GetFullImageMeta(ctx, "repo", "tag") + So(err, ShouldNotBeNil) + }) + + Convey("image is index, fail to get manifests", func() { + err := metaDB.SetImageMeta(multiarchImageMeta.Digest, multiarchImageMeta) + So(err, ShouldBeNil) + + err = metaDB.SetRepoMeta("repo", mTypes.RepoMeta{ + Name: "repo", + Tags: map[mTypes.Tag]mTypes.Descriptor{ + "tag": { + MediaType: ispec.MediaTypeImageIndex, + Digest: multiarchImageMeta.Digest.String(), + }, + }, + }) + So(err, ShouldBeNil) + + _, err = metaDB.GetFullImageMeta(ctx, "repo", "tag") + So(err, ShouldNotBeNil) + }) + }) + + Convey("FilterRepos", func() { + err := setRepoMeta("repo", badProtoBlob, client) + So(err, ShouldBeNil) + + _, err = metaDB.FilterRepos(ctx, mTypes.AcceptAllRepoNames, mTypes.AcceptAllRepoMeta) + So(err, ShouldNotBeNil) + }) + + Convey("SearchTags", func() { + Convey("unmarshalProtoRepoMeta fails", func() { + err := setRepoMeta("repo", badProtoBlob, client) + So(err, ShouldBeNil) + + // manifests are missing + _, err = metaDB.SearchTags(ctx, "repo:") + So(err, ShouldNotBeNil) + }) + + Convey("found repo meta", func() { + Convey("bad image manifest", func() { + badImageDigest := godigest.FromString("bad-image-manifest") + err := metaDB.SetRepoMeta("repo", mTypes.RepoMeta{ + Name: "repo", + Tags: map[mTypes.Tag]mTypes.Descriptor{ + "bad-image-manifest": { + MediaType: ispec.MediaTypeImageManifest, + Digest: badImageDigest.String(), + }, + }, + }) + So(err, ShouldBeNil) + + err = setImageMeta(badImageDigest, badProtoBlob, client) + So(err, ShouldBeNil) + + _, err = metaDB.SearchTags(ctx, "repo:") + So(err, ShouldNotBeNil) + }) + Convey("bad image index", func() { + badIndexDigest := godigest.FromString("bad-image-manifest") + err := metaDB.SetRepoMeta("repo", mTypes.RepoMeta{ + Name: "repo", + Tags: map[mTypes.Tag]mTypes.Descriptor{ + "bad-image-index": { + MediaType: ispec.MediaTypeImageIndex, + Digest: badIndexDigest.String(), + }, + }, + }) + So(err, ShouldBeNil) + + err = setImageMeta(badIndexDigest, badProtoBlob, client) + So(err, ShouldBeNil) + + _, err = metaDB.SearchTags(ctx, "repo:") + So(err, ShouldNotBeNil) + }) + Convey("good image index, bad inside manifest", func() { + goodIndexBadManifestDigest := godigest.FromString("good-index-bad-manifests") + err := metaDB.SetRepoMeta("repo", mTypes.RepoMeta{ + Name: "repo", + Tags: map[mTypes.Tag]mTypes.Descriptor{ + "good-index-bad-manifests": { + MediaType: ispec.MediaTypeImageIndex, + Digest: goodIndexBadManifestDigest.String(), + }, + }, + }) + So(err, ShouldBeNil) + + err = metaDB.SetImageMeta(goodIndexBadManifestDigest, multiarchImageMeta) + So(err, ShouldBeNil) + + err = setImageMeta(image.Digest(), badProtoBlob, client) + So(err, ShouldBeNil) + + _, err = metaDB.SearchTags(ctx, "repo:") + So(err, ShouldNotBeNil) + }) + Convey("bad media type", func() { + err := metaDB.SetRepoMeta("repo", mTypes.RepoMeta{ + Name: "repo", + Tags: map[mTypes.Tag]mTypes.Descriptor{ + "mad-media-type": { + MediaType: "bad media type", + Digest: godigest.FromString("dig").String(), + }, + }, + }) + So(err, ShouldBeNil) + + _, err = metaDB.SearchTags(ctx, "repo:") + So(err, ShouldBeNil) + }) + }) + }) + + Convey("FilterTags", func() { + Convey("unmarshalProtoRepoMeta fails", func() { + err := setRepoMeta("repo", badProtoBlob, client) + So(err, ShouldBeNil) + + _, err = metaDB.FilterTags(ctx, mTypes.AcceptAllRepoTag, mTypes.AcceptAllImageMeta) + So(err, ShouldNotBeNil) + }) + + Convey("bad media Type fails", func() { + err := metaDB.SetRepoMeta("repo", mTypes.RepoMeta{ + Name: "repo", + Tags: map[mTypes.Tag]mTypes.Descriptor{ + "bad-repo-meta": { + MediaType: "bad media type", + Digest: godigest.FromString("dig").String(), + }, + }, + }) + So(err, ShouldBeNil) + + _, err = metaDB.FilterTags(ctx, mTypes.AcceptAllRepoTag, mTypes.AcceptAllImageMeta) + So(err, ShouldBeNil) + }) + }) + + Convey("SearchRepos", func() { + Convey("unmarshalProtoRepoMeta fails", func() { + err := setRepoMeta("repo", badProtoBlob, client) + So(err, ShouldBeNil) + + // manifests are missing + _, err = metaDB.SearchRepos(ctx, "repo") + So(err, ShouldNotBeNil) + }) + }) + + Convey("FilterImageMeta", func() { + Convey("MediaType ImageIndex, getProtoImageMeta fails", func() { + err := metaDB.SetImageMeta(multiarchImageMeta.Digest, multiarchImageMeta) + So(err, ShouldBeNil) + + err = setImageMeta(image.Digest(), badProtoBlob, client) + So(err, ShouldBeNil) + + // manifests are missing + _, err = metaDB.FilterImageMeta(ctx, []string{multiarchImageMeta.Digest.String()}) + So(err, ShouldNotBeNil) + }) + }) + + Convey("SetRepoReference", func() { + Convey("getProtoRepoMeta errors", func() { + err := setRepoMeta("repo", badProtoBlob, client) + So(err, ShouldBeNil) + + err = metaDB.SetRepoReference(ctx, "repo", "tag", imageMeta) + So(err, ShouldNotBeNil) + }) + + Convey("unmarshalProtoRepoBlobs errors", func() { + err := setRepoMeta("repo", goodRepoMetaBlob, client) + So(err, ShouldBeNil) + + err = setRepoBlobInfo("repo", badProtoBlob, client) + So(err, ShouldBeNil) + + err = metaDB.SetRepoReference(ctx, "repo", "tag", imageMeta) + So(err, ShouldNotBeNil) + }) + }) + + Convey("AddUserAPIKey", func() { + // no userid found + userAc := reqCtx.NewUserAccessControl() + ctx := userAc.DeriveContext(context.Background()) + + err = metaDB.AddUserAPIKey(ctx, "", &mTypes.APIKeyDetails{}) + So(err, ShouldNotBeNil) + }) + + Convey("UpdateUserAPIKey", func() { + // no userid found + userAc := reqCtx.NewUserAccessControl() + ctx := userAc.DeriveContext(context.Background()) + + err = metaDB.UpdateUserAPIKeyLastUsed(ctx, "") //nolint: contextcheck + So(err, ShouldNotBeNil) + }) + + Convey("DeleteUserAPIKey", func() { + err = metaDB.SetUserData(ctx, mTypes.UserData{}) + So(err, ShouldBeNil) + + err = metaDB.AddUserAPIKey(ctx, "hashedKey", &mTypes.APIKeyDetails{}) + So(err, ShouldBeNil) + + Convey("userdata not found", func() { + userAc := reqCtx.NewUserAccessControl() + userAc.SetUsername("test") + ctx := userAc.DeriveContext(context.Background()) + + err := metaDB.DeleteUserData(ctx) + So(err, ShouldBeNil) + + err = metaDB.DeleteUserAPIKey(ctx, "") + So(err, ShouldNotBeNil) + }) + + userAc := reqCtx.NewUserAccessControl() + ctx := userAc.DeriveContext(context.Background()) + + err = metaDB.DeleteUserAPIKey(ctx, "test") //nolint: contextcheck + So(err, ShouldNotBeNil) + + err = deleteUserDataBucket(client) + So(err, ShouldBeNil) + + err = metaDB.DeleteUserAPIKey(ctx, "") //nolint: contextcheck + So(err, ShouldNotBeNil) + }) + + Convey("GetUserAPIKeyInfo", func() { + err = deleteUserAPIKeysBucket(client) + So(err, ShouldBeNil) + + _, err = metaDB.GetUserAPIKeyInfo("") + So(err, ShouldNotBeNil) + }) + + Convey("GetUserData", func() { + err = setUserData("test", []byte("dsa8"), client) + So(err, ShouldBeNil) + + _, err = metaDB.GetUserData(ctx) + So(err, ShouldNotBeNil) + + err = deleteUserAPIKeysBucket(client) + So(err, ShouldBeNil) + + _, err = metaDB.GetUserData(ctx) + So(err, ShouldNotBeNil) + }) + + Convey("SetUserData", func() { + userAc := reqCtx.NewUserAccessControl() + ctx := userAc.DeriveContext(context.Background()) + + err = metaDB.SetUserData(ctx, mTypes.UserData{}) + So(err, ShouldNotBeNil) + + err = deleteUserDataBucket(client) + So(err, ShouldBeNil) + + userAc = reqCtx.NewUserAccessControl() + userAc.SetUsername("test") + ctx = userAc.DeriveContext(context.Background()) + + err = metaDB.SetUserData(ctx, mTypes.UserData{}) //nolint: contextcheck + So(err, ShouldBeNil) + }) + + Convey("DeleteUserData", func() { + userAc = reqCtx.NewUserAccessControl() + ctx = userAc.DeriveContext(context.Background()) //nolint:fatcontext // test code + + err = metaDB.DeleteUserData(ctx) + So(err, ShouldNotBeNil) + + err = deleteUserDataBucket(client) + So(err, ShouldBeNil) + + userAc = reqCtx.NewUserAccessControl() + userAc.SetUsername("test") + ctx = userAc.DeriveContext(context.Background()) + + err = metaDB.DeleteUserData(ctx) + So(err, ShouldNotBeNil) + }) + + Convey("GetUserGroups and SetUserGroups", func() { + userAc = reqCtx.NewUserAccessControl() + ctx = userAc.DeriveContext(context.Background()) //nolint:fatcontext // test code + + _, err := metaDB.GetUserGroups(ctx) + So(err, ShouldNotBeNil) + + err = metaDB.SetUserGroups(ctx, []string{}) + So(err, ShouldNotBeNil) + }) + + Convey("ToggleStarRepo bad context errors", func() { + uacKey := reqCtx.GetContextKey() + ctx := context.WithValue(context.Background(), uacKey, "bad context") + + _, err := metaDB.ToggleStarRepo(ctx, "repo") + So(err, ShouldNotBeNil) + }) + + Convey("ToggleStarRepo, no repoMeta found", func() { + userAc := reqCtx.NewUserAccessControl() + userAc.SetUsername("username") + userAc.SetGlobPatterns("read", map[string]bool{ + "repo": true, + }) + + ctx := userAc.DeriveContext(context.Background()) + + err = setRepoMeta("repo", []byte("bad repo"), client) + So(err, ShouldBeNil) + + _, err = metaDB.ToggleStarRepo(ctx, "repo") + So(err, ShouldNotBeNil) + }) + + Convey("ToggleStarRepo bad UserData found", func() { + userAc := reqCtx.NewUserAccessControl() + userAc.SetUsername("username") + userAc.SetGlobPatterns("read", map[string]bool{ + "repo": true, + }) + + ctx := userAc.DeriveContext(context.Background()) + + err = setUserData("username", []byte("dsa8"), client) + So(err, ShouldBeNil) + + _, err = metaDB.ToggleStarRepo(ctx, "repo") + So(err, ShouldNotBeNil) + }) + + Convey("ToggleStarRepo, bad repoMeta found", func() { + userAc := reqCtx.NewUserAccessControl() + userAc.SetUsername("username") + userAc.SetGlobPatterns("read", map[string]bool{ + "repo": true, + }) + + ctx := userAc.DeriveContext(context.Background()) + + _, err = metaDB.ToggleStarRepo(ctx, "repo") + So(err, ShouldNotBeNil) + }) + + Convey("ToggleBookmarkRepo bad context errors", func() { + uacKey := reqCtx.GetContextKey() + ctx := context.WithValue(context.Background(), uacKey, "bad context") + + _, err := metaDB.ToggleBookmarkRepo(ctx, "repo") + So(err, ShouldNotBeNil) + }) + + Convey("ToggleBookmarkRepo bad UserData found", func() { + userAc := reqCtx.NewUserAccessControl() + userAc.SetUsername("username") + userAc.SetGlobPatterns("read", map[string]bool{ + "repo": true, + }) + + ctx := userAc.DeriveContext(context.Background()) + + err = setUserData("username", []byte("dsa8"), client) + So(err, ShouldBeNil) + + _, err = metaDB.ToggleBookmarkRepo(ctx, "repo") + So(err, ShouldNotBeNil) + }) + + Convey("GetUserData bad context errors", func() { + uacKey := reqCtx.GetContextKey() + ctx := context.WithValue(context.Background(), uacKey, "bad context") + + _, err := metaDB.GetUserData(ctx) + So(err, ShouldNotBeNil) + }) + + Convey("SetUserData bad context errors", func() { + uacKey := reqCtx.GetContextKey() + ctx := context.WithValue(context.Background(), uacKey, "bad context") + + err := metaDB.SetUserData(ctx, mTypes.UserData{}) + So(err, ShouldNotBeNil) + }) + + Convey("GetUserGroups bad context errors", func() { + _, err := metaDB.GetUserGroups(ctx) + So(err, ShouldNotBeNil) + + uacKey := reqCtx.GetContextKey() + ctx := context.WithValue(context.Background(), uacKey, "bad context") + + _, err = metaDB.GetUserGroups(ctx) //nolint: contextcheck + So(err, ShouldNotBeNil) + }) + + Convey("SetUserGroups bad context errors", func() { + uacKey := reqCtx.GetContextKey() + ctx := context.WithValue(context.Background(), uacKey, "bad context") + + err := metaDB.SetUserGroups(ctx, []string{}) + So(err, ShouldNotBeNil) + }) + + Convey("SetUserGroups bad UserData found", func() { + userAc := reqCtx.NewUserAccessControl() + userAc.SetUsername("username") + userAc.SetGlobPatterns("read", map[string]bool{ + "repo": true, + }) + + ctx := userAc.DeriveContext(context.Background()) + + err = setUserData("username", []byte("dsa8"), client) + So(err, ShouldBeNil) + + err = metaDB.SetUserGroups(ctx, []string{}) + So(err, ShouldNotBeNil) + }) + + Convey("AddUserAPIKey bad context errors", func() { + uacKey := reqCtx.GetContextKey() + ctx := context.WithValue(context.Background(), uacKey, "bad context") + + err := metaDB.AddUserAPIKey(ctx, "", &mTypes.APIKeyDetails{}) + So(err, ShouldNotBeNil) + }) + + Convey("AddUserAPIKey bad UserData found", func() { + userAc := reqCtx.NewUserAccessControl() + userAc.SetUsername("username") + userAc.SetGlobPatterns("read", map[string]bool{ + "repo": true, + }) + + ctx := userAc.DeriveContext(context.Background()) + + err = setUserData("username", []byte("dsa8"), client) + So(err, ShouldBeNil) + + err = metaDB.AddUserAPIKey(ctx, "", &mTypes.APIKeyDetails{}) + So(err, ShouldNotBeNil) + }) + + Convey("DeleteUserAPIKey bad context errors", func() { + uacKey := reqCtx.GetContextKey() + ctx := context.WithValue(context.Background(), uacKey, "bad context") + + err := metaDB.DeleteUserAPIKey(ctx, "") + So(err, ShouldNotBeNil) + }) + + Convey("GetUserAPIKeys bad UserData found", func() { + userAc := reqCtx.NewUserAccessControl() + userAc.SetUsername("username") + userAc.SetGlobPatterns("read", map[string]bool{ + "repo": true, + }) + + ctx := userAc.DeriveContext(context.Background()) + + err = setUserData("username", []byte("dsa8"), client) + So(err, ShouldBeNil) + + _, err = metaDB.GetUserAPIKeys(ctx) + So(err, ShouldNotBeNil) + }) + + Convey("IsAPIKeyExpired bad UserData found", func() { + userAc := reqCtx.NewUserAccessControl() + userAc.SetUsername("username") + userAc.SetGlobPatterns("read", map[string]bool{ + "repo": true, + }) + + ctx := userAc.DeriveContext(context.Background()) + + err = setUserData("username", []byte("dsa8"), client) + So(err, ShouldBeNil) + + _, err = metaDB.IsAPIKeyExpired(ctx, "") + So(err, ShouldNotBeNil) + }) + + Convey("UpdateUserAPIKeyLastUsed bad context errors", func() { + uacKey := reqCtx.GetContextKey() + ctx := context.WithValue(context.Background(), uacKey, "bad context") + + err := metaDB.UpdateUserAPIKeyLastUsed(ctx, "") + So(err, ShouldNotBeNil) + }) + + Convey("UpdateUserAPIKeyLastUsed bad UserData found", func() { + userAc := reqCtx.NewUserAccessControl() + userAc.SetUsername("username") + userAc.SetGlobPatterns("read", map[string]bool{ + "repo": true, + }) + + ctx := userAc.DeriveContext(context.Background()) + + err = setUserData("username", []byte("dsa8"), client) + So(err, ShouldBeNil) + + err = metaDB.UpdateUserAPIKeyLastUsed(ctx, "") + So(err, ShouldNotBeNil) + }) + + Convey("DeleteUserData bad context errors", func() { + uacKey := reqCtx.GetContextKey() + ctx := context.WithValue(context.Background(), uacKey, "bad context") + + err := metaDB.DeleteUserData(ctx) + So(err, ShouldNotBeNil) + }) + + Convey("GetStarredRepos bad context errors", func() { + uacKey := reqCtx.GetContextKey() + ctx := context.WithValue(context.Background(), uacKey, "bad context") + + _, err := metaDB.GetStarredRepos(ctx) + So(err, ShouldNotBeNil) + }) + + Convey("GetBookmarkedRepos bad context errors", func() { + uacKey := reqCtx.GetContextKey() + ctx := context.WithValue(context.Background(), uacKey, "bad context") + + _, err := metaDB.GetBookmarkedRepos(ctx) + So(err, ShouldNotBeNil) + }) + + Convey("GetUserRepoMeta unmarshal error", func() { + userAc := reqCtx.NewUserAccessControl() + userAc.SetUsername("username") + userAc.SetGlobPatterns("read", map[string]bool{ + "repo": true, + }) + + ctx := userAc.DeriveContext(context.Background()) + + err = setRepoMeta("repo", []byte("bad repo"), client) + So(err, ShouldBeNil) + + _, err := metaDB.GetRepoMeta(ctx, "repo") + So(err, ShouldNotBeNil) + }) + }) +} + +func setRepoMeta(repo string, blob []byte, client *goredis.Client) error { //nolint: unparam + ctx := context.Background() + key := keyPrefix + ":" + redis.RepoMetaBucket + + return client.HSet(ctx, key, repo, blob).Err() +} + +func setRepoLastUpdated(repo string, blob []byte, client *goredis.Client) error { + ctx := context.Background() + key := keyPrefix + ":" + redis.RepoLastUpdatedBucket + + return client.HSet(ctx, key, repo, blob).Err() +} + +func setImageMeta(digest godigest.Digest, blob []byte, client *goredis.Client) error { + ctx := context.Background() + key := keyPrefix + ":" + redis.ImageMetaBucket + + return client.HSet(ctx, key, digest.String(), blob).Err() +} + +func setRepoBlobInfo(repo string, blob []byte, client *goredis.Client) error { + ctx := context.Background() + key := keyPrefix + ":" + redis.RepoBlobsBucket + + return client.HSet(ctx, key, repo, blob).Err() +} + +func setUserData(userID string, blob []byte, client *goredis.Client) error { + ctx := context.Background() + key := keyPrefix + ":" + redis.UserDataBucket + + return client.HSet(ctx, key, userID, blob).Err() +} + +func deleteUserDataBucket(client *goredis.Client) error { + ctx := context.Background() + key := keyPrefix + ":" + redis.UserDataBucket + + return client.Del(ctx, key).Err() +} + +func deleteUserAPIKeysBucket(client *goredis.Client) error { + ctx := context.Background() + key := keyPrefix + ":" + redis.UserAPIKeysBucket + + return client.Del(ctx, key).Err() +} + +func DumpKeys(t *testing.T, client goredis.UniversalClient) { + t.Helper() + + // Retrieve all keys + keys, err := client.Keys(context.Background(), "*").Result() + if err != nil { + t.Log("Error retrieving keys:", err) + + return + } + + // Print the keys + t.Log("Keys in Redis:") + + for _, key := range keys { + keyType, err := client.Type(context.Background(), key).Result() + if err != nil { + t.Logf("Error retrieving type for key %s: %v\n", key, err) + + continue + } + + var value string + + switch keyType { + case "string": + value, err = client.Get(context.Background(), key).Result() + case "list": + values, err := client.LRange(context.Background(), key, 0, -1).Result() + if err == nil { + value = fmt.Sprintf("%v", values) + } + case "hash": + values, err := client.HGetAll(context.Background(), key).Result() + if err == nil { + value = fmt.Sprintf("%v", values) + } + case "set": + values, err := client.SMembers(context.Background(), key).Result() + if err == nil { + value = fmt.Sprintf("%v", values) + } + case "zset": + values, err := client.ZRange(context.Background(), key, 0, -1).Result() + if err == nil { + value = fmt.Sprintf("%v", values) + } + default: + value = "Unsupported type" + } + + if err != nil { + t.Logf("Error retrieving value for key %s: %v\n", key, err) + } else { + t.Logf("Key: %s, Type: %s, Value: %s\n", key, keyType, value) + } + } +} diff --git a/pkg/meta/types/types.go b/pkg/meta/types/types.go index e6374e91..3b53ba63 100644 --- a/pkg/meta/types/types.go +++ b/pkg/meta/types/types.go @@ -61,6 +61,9 @@ func GetLatestImageDigests(repoMetaList []RepoMeta) []string { type MetaDB interface { //nolint:interfacebloat UserDB + // SetImageMeta sets ImageMeta for a given image in the database + // should NEVER be used in production as both GetImageMeta and SetImageMeta + // should be locked for the duration of the entire transaction at a higher level in the app SetImageMeta(digest godigest.Digest, imageMeta ImageMeta) error // SetRepoReference sets the given image data to the repo metadata. @@ -109,7 +112,9 @@ type MetaDB interface { //nolint:interfacebloat // DecrementRepoStars subtracts 1 from the star count of an image DecrementRepoStars(repo string) error - // SetRepoMeta returns RepoMetadata of a repo from the database + // SetRepoMeta sets RepoMetadata for a given repo in the database + // should NEVER be used in production as both GetRepoMeta and SetRepoMeta + // should be locked for the duration of the entire transaction at a higher level in the app SetRepoMeta(repo string, repoMeta RepoMeta) error // DeleteRepoMeta @@ -169,6 +174,9 @@ type UserDB interface { //nolint:interfacebloat // UserDB profile/api key CRUD GetUserData(ctx context.Context) (UserData, error) + // SetUserData sets UserData for a given user in the database + // SetUserData should NEVER be used in production as both GetUserData and SetUserData + // should be locked for the duration of the entire transaction at a higher level in the app SetUserData(ctx context.Context, userData UserData) error SetUserGroups(ctx context.Context, groups []string) error diff --git a/pkg/meta/version/patches.go b/pkg/meta/version/patches.go index bc125a05..70d05061 100644 --- a/pkg/meta/version/patches.go +++ b/pkg/meta/version/patches.go @@ -2,6 +2,7 @@ package version import ( "github.com/aws/aws-sdk-go-v2/service/dynamodb" + "github.com/redis/go-redis/v9" "go.etcd.io/bbolt" ) @@ -12,3 +13,7 @@ func GetBoltDBPatches() []func(DB *bbolt.DB) error { func GetDynamoDBPatches() []func(client *dynamodb.Client, tableNames map[string]string) error { return []func(client *dynamodb.Client, tableNames map[string]string) error{} } + +func GetRedisDBPatches() []func(client redis.UniversalClient) error { + return []func(client redis.UniversalClient) error{} +} diff --git a/pkg/meta/version/version_test.go b/pkg/meta/version/version_test.go index e3bb26bd..38bcd92b 100644 --- a/pkg/meta/version/version_test.go +++ b/pkg/meta/version/version_test.go @@ -3,21 +3,25 @@ package version_test import ( "context" "errors" + "fmt" "os" "path" "testing" + "github.com/alicebob/miniredis/v2" "github.com/aws/aws-sdk-go-v2/feature/dynamodb/attributevalue" "github.com/aws/aws-sdk-go-v2/service/dynamodb" "github.com/aws/aws-sdk-go-v2/service/dynamodb/types" "github.com/aws/aws-sdk-go/aws" guuid "github.com/gofrs/uuid" + goredis "github.com/redis/go-redis/v9" . "github.com/smartystreets/goconvey/convey" "go.etcd.io/bbolt" "zotregistry.dev/zot/pkg/log" "zotregistry.dev/zot/pkg/meta/boltdb" mdynamodb "zotregistry.dev/zot/pkg/meta/dynamodb" + "zotregistry.dev/zot/pkg/meta/redis" "zotregistry.dev/zot/pkg/meta/version" tskip "zotregistry.dev/zot/pkg/test/skip" ) @@ -211,3 +215,175 @@ func setDynamoDBVersion(client *dynamodb.Client, versTable, vers string) error { return err } + +func TestVersioningRedisDB(t *testing.T) { + miniRedis := miniredis.RunT(t) + + Convey("Tests", t, func() { + opts, err := goredis.ParseURL("redis://" + miniRedis.Addr()) + So(err, ShouldBeNil) + + client := goredis.NewClient(opts) + defer dumpRedisKeys(t, client) // Troubleshoot test failures + + log := log.NewLogger("debug", "") + + params := redis.DBDriverParameters{KeyPrefix: "zot"} + + metaDB, err := redis.New(client, params, log) + So(err, ShouldBeNil) + + So(metaDB.ResetDB(), ShouldBeNil) + + ctx := context.Background() + + Convey("empty initial version triggers setting the default", func() { + // Check no value is initially set + actualVersion, err := client.Get(ctx, metaDB.VersionKey).Result() + So(err, ShouldEqual, goredis.Nil) + So(actualVersion, ShouldEqual, "") + + err = metaDB.PatchDB() + So(err, ShouldBeNil) + + // Check default version is added in the DB + actualVersion, err = client.Get(ctx, metaDB.VersionKey).Result() + So(err, ShouldBeNil) + So(actualVersion, ShouldEqual, version.CurrentVersion) + }) + + Convey("initial version with a bad value raises an error", func() { + // Set invalid initial value + err = client.Set(ctx, metaDB.VersionKey, "VInvalid", 0).Err() + So(err, ShouldBeNil) + + // Check error when attempting to patch + err = metaDB.PatchDB() + So(err, ShouldNotBeNil) + }) + + Convey("skip iterating patches", func() { + // Initialize DB version + metaDB.Version = version.Version1 + + // Patches have errors so we can check bad upgrade logic + metaDB.Patches = []func(client goredis.UniversalClient) error{ + func(client goredis.UniversalClient) error { // V1 to V2 + return ErrTestError + }, + func(client goredis.UniversalClient) error { // V2 to V3 + return ErrTestError + }, + } + + // No patch should be applied for V1 so no error is expected + err = metaDB.PatchDB() + So(err, ShouldBeNil) + }) + + Convey("iterate over patches without any errors", func() { + // Initialize DB version with a lower version + metaDB.Version = version.Version1 + + err = metaDB.PatchDB() + So(err, ShouldBeNil) + + // Now change to a newer DB version and apply patches + metaDB.Version = version.Version3 + + metaDB.Patches = []func(goredis.UniversalClient) error{ + func(client goredis.UniversalClient) error { // V1 to V2 + return nil + }, + func(client goredis.UniversalClient) error { // V2 to V3 + return nil + }, + } + + err = metaDB.PatchDB() + So(err, ShouldBeNil) + }) + + Convey("iterate over patches with errors", func() { + // initialize DB version with a lower version + metaDB.Version = version.Version1 + + err = metaDB.PatchDB() + So(err, ShouldBeNil) + + // now change to a newer DB version and apply patches + metaDB.Version = version.Version3 + + metaDB.Patches = []func(client goredis.UniversalClient) error{ + func(client goredis.UniversalClient) error { // V1 to V2 + return nil + }, + func(client goredis.UniversalClient) error { // V2 to V3 + return ErrTestError + }, + } + + err = metaDB.PatchDB() + So(err, ShouldNotBeNil) + }) + }) +} + +func dumpRedisKeys(t *testing.T, client goredis.UniversalClient) { + t.Helper() + + // Retrieve all keys + keys, err := client.Keys(context.Background(), "*").Result() + if err != nil { + t.Log("Error retrieving keys:", err) + + return + } + + // Print the keys + t.Log("Keys in Redis:") + + for _, key := range keys { + keyType, err := client.Type(context.Background(), key).Result() + if err != nil { + t.Logf("Error retrieving type for key %s: %v\n", key, err) + + continue + } + + var value string + + switch keyType { + case "string": + value, err = client.Get(context.Background(), key).Result() + case "list": + values, err := client.LRange(context.Background(), key, 0, -1).Result() + if err == nil { + value = fmt.Sprintf("%v", values) + } + case "hash": + values, err := client.HGetAll(context.Background(), key).Result() + if err == nil { + value = fmt.Sprintf("%v", values) + } + case "set": + values, err := client.SMembers(context.Background(), key).Result() + if err == nil { + value = fmt.Sprintf("%v", values) + } + case "zset": + values, err := client.ZRange(context.Background(), key, 0, -1).Result() + if err == nil { + value = fmt.Sprintf("%v", values) + } + default: + value = "Unsupported type" + } + + if err != nil { + t.Logf("Error retrieving value for key %s: %v\n", key, err) + } else { + t.Logf("Key: %s, Type: %s, Value: %s\n", key, keyType, value) + } + } +} diff --git a/pkg/storage/cache.go b/pkg/storage/cache.go index eed2ab4d..ec80f259 100644 --- a/pkg/storage/cache.go +++ b/pkg/storage/cache.go @@ -3,12 +3,14 @@ package storage import ( zerr "zotregistry.dev/zot/errors" "zotregistry.dev/zot/pkg/api/config" + rediscfg "zotregistry.dev/zot/pkg/api/config/redis" zlog "zotregistry.dev/zot/pkg/log" "zotregistry.dev/zot/pkg/storage/cache" "zotregistry.dev/zot/pkg/storage/constants" + storageTypes "zotregistry.dev/zot/pkg/storage/types" ) -func CreateCacheDatabaseDriver(storageConfig config.StorageConfig, log zlog.Logger) (cache.Cache, error) { +func CreateCacheDatabaseDriver(storageConfig config.StorageConfig, log zlog.Logger) (storageTypes.Cache, error) { if !storageConfig.Dedupe && storageConfig.StorageDriver == nil { return nil, nil //nolint:nilnil } @@ -32,25 +34,33 @@ func CreateCacheDatabaseDriver(storageConfig config.StorageConfig, log zlog.Logg return nil, nil //nolint:nilnil } - if name != constants.DynamoDBDriverName { + if name != constants.DynamoDBDriverName && + name != constants.RedisDriverName { log.Warn().Str("driver", name).Msg("remote cache driver unsupported!") return nil, nil //nolint:nilnil } - // dynamodb - dynamoParams := cache.DynamoDBDriverParameters{} - dynamoParams.Endpoint, _ = storageConfig.CacheDriver["endpoint"].(string) - dynamoParams.Region, _ = storageConfig.CacheDriver["region"].(string) - dynamoParams.TableName, _ = storageConfig.CacheDriver["cachetablename"].(string) + if name == constants.DynamoDBDriverName { + // dynamodb + return Create(name, getDynamoParams(&storageConfig), log) + } - return Create("dynamodb", dynamoParams, log) + if name == constants.RedisDriverName { + // redis + redisParams, err := getRedisParams(&storageConfig, log) + if err != nil { + return nil, err + } + + return Create(name, redisParams, log) + } } return nil, nil //nolint:nilnil } -func Create(dbtype string, parameters interface{}, log zlog.Logger) (cache.Cache, error) { +func Create(dbtype string, parameters interface{}, log zlog.Logger) (storageTypes.Cache, error) { switch dbtype { case "boltdb": { @@ -60,6 +70,10 @@ func Create(dbtype string, parameters interface{}, log zlog.Logger) (cache.Cache { return cache.NewDynamoDBCache(parameters, log) } + case "redis": + { + return cache.NewRedisCache(parameters, log) + } default: { return nil, zerr.ErrBadConfig @@ -68,5 +82,31 @@ func Create(dbtype string, parameters interface{}, log zlog.Logger) (cache.Cache } func getUseRelPaths(storageConfig *config.StorageConfig) bool { + // In case of local storage we use rel paths, in case of S3 we don't return storageConfig.StorageDriver == nil } + +func getDynamoParams(storageConfig *config.StorageConfig) cache.DynamoDBDriverParameters { + dynamoParams := cache.DynamoDBDriverParameters{} + dynamoParams.Endpoint, _ = storageConfig.CacheDriver["endpoint"].(string) + dynamoParams.Region, _ = storageConfig.CacheDriver["region"].(string) + dynamoParams.TableName, _ = storageConfig.CacheDriver["cachetablename"].(string) + + return dynamoParams +} + +func getRedisParams(storageConfig *config.StorageConfig, log zlog.Logger) (cache.RedisDriverParameters, error) { + redisParams := cache.RedisDriverParameters{} + + client, err := rediscfg.GetRedisClient(storageConfig.CacheDriver, log) + if err != nil { + return redisParams, err + } + + redisParams.RootDir = storageConfig.RootDirectory + redisParams.Client = client + redisParams.KeyPrefix, _ = storageConfig.CacheDriver["keyprefix"].(string) + redisParams.UseRelPaths = getUseRelPaths(storageConfig) + + return redisParams, nil +} diff --git a/pkg/storage/cache/redis.go b/pkg/storage/cache/redis.go new file mode 100644 index 00000000..ae37e6e2 --- /dev/null +++ b/pkg/storage/cache/redis.go @@ -0,0 +1,340 @@ +package cache + +import ( + "context" + goerrors "errors" + "path/filepath" + "strings" + + "github.com/go-redsync/redsync/v4" + gors "github.com/go-redsync/redsync/v4/redis/goredis/v9" + godigest "github.com/opencontainers/go-digest" + "github.com/redis/go-redis/v9" + + zerr "zotregistry.dev/zot/errors" + zlog "zotregistry.dev/zot/pkg/log" + "zotregistry.dev/zot/pkg/storage/constants" +) + +type RedisDriver struct { + rootDir string + db redis.UniversalClient + log zlog.Logger + keyPrefix string // prepended to all keys, logically separating cache drivers accessing the same DB + useRelPaths bool // whether or not to use relative paths, should be true for filesystem and false for s3 + rs *redsync.Redsync // used for locks, at the moment we are locking only for calls writing to the DB +} + +type RedisDriverParameters struct { + Client redis.UniversalClient + RootDir string + UseRelPaths bool + KeyPrefix string +} + +func NewRedisCache(parameters interface{}, log zlog.Logger) (*RedisDriver, error) { + properParameters, ok := parameters.(RedisDriverParameters) + if !ok { + log.Error().Err(zerr.ErrTypeAssertionFailed).Msgf("failed to cast type, expected type '%T' but got '%T'", + RedisDriverParameters{}, parameters) + + return nil, zerr.ErrTypeAssertionFailed + } + + keyPrefix := properParameters.KeyPrefix + if len(keyPrefix) == 0 { + keyPrefix = "zot" + } + + cacheDB := properParameters.Client + + if _, err := cacheDB.Ping(context.Background()).Result(); err != nil { + log.Error().Err(err).Msg("failed to ping redis cache") + + return nil, err + } + + // Create an instance of redisync to be used to obtain locks + pool := gors.NewPool(cacheDB) + + // note for integration with local storage we need relative paths + // while for integration with s3 storage we need absolute paths + driver := &RedisDriver{ + db: cacheDB, + log: log, + rootDir: properParameters.RootDir, + useRelPaths: properParameters.UseRelPaths, + keyPrefix: keyPrefix, + rs: redsync.New(pool), + } + + return driver, nil +} + +func (d *RedisDriver) join(xs ...string) string { + return d.keyPrefix + ":" + strings.Join(xs, ":") +} + +func (d *RedisDriver) UsesRelativePaths() bool { + return d.useRelPaths +} + +func (d *RedisDriver) Name() string { + return "redis" +} + +// SetClient is supposed to be used only for testing purposes. +func (d *RedisDriver) SetClient(client redis.UniversalClient) { + d.db = client +} + +func (d *RedisDriver) PutBlob(digest godigest.Digest, path string) error { + ctx := context.TODO() + + if path == "" { + d.log.Error().Err(zerr.ErrEmptyValue).Str("digest", digest.String()).Msg("failed to provide non-empty path") + + return zerr.ErrEmptyValue + } + + // use only relative (to rootDir) paths on blobs + var err error + if d.useRelPaths { + path, err = filepath.Rel(d.rootDir, path) + if err != nil { + d.log.Error().Err(err).Str("path", path).Msg("failed to get relative path") + } + } + + if len(path) == 0 { + return zerr.ErrEmptyValue + } + + lock := d.rs.NewMutex(d.join(constants.RedisLocksBucket, digest.String())) + err = lock.Lock() + if err != nil { + d.log.Error().Err(err).Str("digest", digest.String()).Msg("failed to acquire redis lock") + + return err + } + + defer func() { + if _, err := lock.Unlock(); err != nil { + d.log.Error().Err(err).Str("digest", digest.String()).Msg("failed to release redis lock") + } + }() + + // see if the blob digest exists. + exists, err := d.db.HExists(ctx, d.join(constants.BlobsCache, constants.OriginalBucket), digest.String()).Result() + if err != nil { + return err + } + + if _, err := d.db.TxPipelined(ctx, func(txrp redis.Pipeliner) error { + if !exists { + // add the key value pair [digest, path] to blobs:origin if not + // exist already. the path becomes the canonical blob we do this in + // a transaction to make sure that if something is in the set, then + // it is guaranteed to always have a path + // note that there is a race, but the worst case is that a different + // origin path that is still valid is used. + if err := txrp.HSet(ctx, d.join(constants.BlobsCache, constants.OriginalBucket), + digest.String(), path).Err(); err != nil { + d.log.Error().Err(err).Str("hset", d.join(constants.BlobsCache, constants.OriginalBucket)). + Str("value", path).Msg("unable to put record") + + return err + } + } + // add path to the set of paths which the digest represents + if err := txrp.SAdd(ctx, d.join(constants.BlobsCache, constants.DuplicatesBucket, + digest.String()), path).Err(); err != nil { + d.log.Error().Err(err).Str("sadd", d.join(constants.BlobsCache, constants.DuplicatesBucket, digest.String())). + Str("value", path).Msg("unable to put record") + + return err + } + + return nil + }); err != nil { + return err + } + + return nil +} + +func (d *RedisDriver) GetBlob(digest godigest.Digest) (string, error) { + ctx := context.TODO() + + path, err := d.db.HGet(ctx, d.join(constants.BlobsCache, constants.OriginalBucket), digest.String()).Result() + if err != nil { + if goerrors.Is(err, redis.Nil) { + return "", zerr.ErrCacheMiss + } + + d.log.Error().Err(err).Str("hget", d.join(constants.BlobsCache, constants.OriginalBucket)). + Str("digest", digest.String()).Msg("unable to get record") + + return "", err + } + + return path, nil +} + +func (d *RedisDriver) GetAllBlobs(digest godigest.Digest) ([]string, error) { + blobPaths := []string{} + + ctx := context.TODO() + + originalPath, err := d.db.HGet(ctx, d.join(constants.BlobsCache, constants.OriginalBucket), digest.String()).Result() + if err != nil { + if goerrors.Is(err, redis.Nil) { + return nil, zerr.ErrCacheMiss + } + + d.log.Error().Err(err).Str("hget", d.join(constants.BlobsCache, constants.OriginalBucket)). + Str("digest", digest.String()).Msg("unable to get record") + + return nil, err + } + + blobPaths = append(blobPaths, originalPath) + + // see if we are in the set + duplicateBlobPaths, err := d.db.SMembers(ctx, d.join(constants.BlobsCache, constants.DuplicatesBucket, + digest.String())).Result() + if err != nil { + d.log.Error().Err(err).Str("smembers", d.join(constants.BlobsCache, constants.DuplicatesBucket, digest.String())). + Str("digest", digest.String()).Msg("unable to get record") + + return nil, err + } + + for _, item := range duplicateBlobPaths { + if item != originalPath { + blobPaths = append(blobPaths, item) + } + } + + return blobPaths, nil +} + +func (d *RedisDriver) HasBlob(digest godigest.Digest, path string) bool { + var err error + + if d.useRelPaths { + path, err = filepath.Rel(d.rootDir, path) + if err != nil { + d.log.Error().Err(err).Str("path", path).Msg("failed to get relative path") + } + } + + if len(path) == 0 { + return false + } + + ctx := context.TODO() + // see if we are in the set + exists, err := d.db.SIsMember(ctx, d.join(constants.BlobsCache, constants.DuplicatesBucket, + digest.String()), path).Result() + if err != nil { + d.log.Error().Err(err).Str("sismember", d.join(constants.BlobsCache, constants.DuplicatesBucket, digest.String())). + Str("digest", digest.String()).Msg("unable to get record") + + return false + } + + if !exists { + return false + } + + // see if the path entry exists. is this actually needed? i guess it doesn't really hurt (it is fast) + exists, err = d.db.HExists(ctx, d.join(constants.BlobsCache, constants.OriginalBucket), digest.String()).Result() + + d.log.Error().Err(err).Str("hexists", d.join(constants.BlobsCache, constants.OriginalBucket)). + Str("digest", digest.String()).Msg("unable to get record") + + if err != nil { + return false + } + + if !exists { + return false + } + + return true +} + +func (d *RedisDriver) DeleteBlob(digest godigest.Digest, path string) error { + ctx := context.TODO() + + // use only relative (to rootDir) paths on blobs + var err error + if d.useRelPaths { + path, err = filepath.Rel(d.rootDir, path) + if err != nil { + d.log.Error().Err(err).Str("path", path).Msg("failed to get relative path") + } + } + + lock := d.rs.NewMutex(d.join(constants.RedisLocksBucket, digest.String())) + err = lock.Lock() + if err != nil { + d.log.Error().Err(err).Str("digest", digest.String()).Msg("failed to acquire redis lock") + + return err + } + + defer func() { + if _, err := lock.Unlock(); err != nil { + d.log.Error().Err(err).Str("digest", digest.String()).Msg("failed to release redis lock") + } + }() + + pathSet := d.join(constants.BlobsCache, constants.DuplicatesBucket, digest.String()) + + // delete path from the set of paths which the digest represents + _, err = d.db.SRem(ctx, pathSet, path).Result() + if err != nil { + d.log.Error().Err(err).Str("srem", pathSet).Str("value", path).Msg("failed to delete record") + + return err + } + + currentPath, err := d.GetBlob(digest) + if err != nil { + return err + } + + if currentPath != path { + // nothing we need to do, return nil yay + return nil + } + + // we need to set a new path + newPath, err := d.db.SRandMember(ctx, pathSet).Result() + if err != nil { + if goerrors.Is(err, redis.Nil) { + _, err := d.db.HDel(ctx, d.join(constants.BlobsCache, constants.OriginalBucket), digest.String()).Result() + if err != nil { + return err + } + + return nil + } + + d.log.Error().Err(err).Str("srandmember", pathSet).Msg("failed to get new path") + + return err + } + + if _, err := d.db.HSet(ctx, d.join(constants.BlobsCache, constants.OriginalBucket), + digest.String(), newPath).Result(); err != nil { + d.log.Error().Err(err).Str("hset", d.join(constants.BlobsCache, constants.OriginalBucket)).Str("value", newPath). + Msg("unable to put record") + + return err + } + + return nil +} diff --git a/pkg/storage/cache/redis_test.go b/pkg/storage/cache/redis_test.go new file mode 100644 index 00000000..e242cfb6 --- /dev/null +++ b/pkg/storage/cache/redis_test.go @@ -0,0 +1,712 @@ +package cache_test + +import ( + "errors" + "fmt" + "path" + "testing" + "time" + + "github.com/alicebob/miniredis/v2" + "github.com/go-redis/redismock/v9" + "github.com/redis/go-redis/v9" + . "github.com/smartystreets/goconvey/convey" + + zerr "zotregistry.dev/zot/errors" + "zotregistry.dev/zot/pkg/log" + "zotregistry.dev/zot/pkg/storage" + "zotregistry.dev/zot/pkg/storage/cache" + "zotregistry.dev/zot/pkg/storage/constants" + test "zotregistry.dev/zot/pkg/test/common" +) + +var ErrTestError = errors.New("TestError") + +func TestRedisCache(t *testing.T) { + miniRedis := miniredis.RunT(t) + + Convey("Make a new cache", t, func() { + dir := t.TempDir() + + log := log.NewLogger("debug", "") + So(log, ShouldNotBeNil) + + cacheDriver, err := storage.Create("redis", "failTypeAssertion", log) + So(cacheDriver, ShouldBeNil) + So(err, ShouldNotBeNil) + + connOpts, _ := redis.ParseURL("redis://" + miniRedis.Addr()) + client := redis.NewClient(connOpts) + + cacheDriver, err = storage.Create("redis", + cache.RedisDriverParameters{client, dir, true, "zot"}, log) + So(cacheDriver, ShouldNotBeNil) + So(err, ShouldBeNil) + + name := cacheDriver.Name() + So(name, ShouldEqual, "redis") + + val, err := cacheDriver.GetBlob("key") + So(err, ShouldEqual, zerr.ErrCacheMiss) + So(val, ShouldBeEmpty) + + exists := cacheDriver.HasBlob("key", path.Join(dir, "value")) + So(exists, ShouldBeFalse) + + exists = cacheDriver.HasBlob("key", "value") + So(exists, ShouldBeFalse) + + err = cacheDriver.PutBlob("key", path.Join(dir, "value")) + So(err, ShouldBeNil) + + err = cacheDriver.PutBlob("key", "value") + So(err, ShouldNotBeNil) + + exists = cacheDriver.HasBlob("key", path.Join(dir, "value")) + So(exists, ShouldBeTrue) + + val, err = cacheDriver.GetBlob("key") + So(err, ShouldBeNil) + So(val, ShouldNotBeEmpty) + + err = cacheDriver.DeleteBlob("bogusKey", "bogusValue") + So(err, ShouldEqual, zerr.ErrCacheMiss) + + err = cacheDriver.DeleteBlob("key", "bogusValue") + So(err, ShouldBeNil) + + // try to insert empty path + err = cacheDriver.PutBlob("key", "") + So(err, ShouldNotBeNil) + So(err, ShouldEqual, zerr.ErrEmptyValue) + + connOpts, _ = redis.ParseURL("redis://" + miniRedis.Addr() + "/5") + client = redis.NewClient(connOpts) + + cacheDriver, err = storage.Create("redis", + cache.RedisDriverParameters{client, t.TempDir(), false, "zot"}, log) + So(cacheDriver, ShouldNotBeNil) + So(err, ShouldBeNil) + + err = cacheDriver.PutBlob("key1", "originalBlobPath") + So(err, ShouldBeNil) + + err = cacheDriver.PutBlob("key1", "duplicateBlobPath") + So(err, ShouldBeNil) + + val, err = cacheDriver.GetBlob("key1") + So(val, ShouldEqual, "originalBlobPath") + So(err, ShouldBeNil) + + err = cacheDriver.DeleteBlob("key1", "duplicateBlobPath") + So(err, ShouldBeNil) + + val, err = cacheDriver.GetBlob("key1") + So(val, ShouldEqual, "originalBlobPath") + So(err, ShouldBeNil) + + err = cacheDriver.PutBlob("key1", "duplicateBlobPath") + So(err, ShouldBeNil) + + err = cacheDriver.DeleteBlob("key1", "originalBlobPath") + So(err, ShouldBeNil) + + val, err = cacheDriver.GetBlob("key1") + So(val, ShouldEqual, "duplicateBlobPath") + So(err, ShouldBeNil) + + err = cacheDriver.DeleteBlob("key1", "duplicateBlobPath") + So(err, ShouldBeNil) + + // should be empty + val, err = cacheDriver.GetBlob("key1") + So(err, ShouldNotBeNil) + So(val, ShouldBeEmpty) + + // try to add three same values + err = cacheDriver.PutBlob("key2", "duplicate") + So(err, ShouldBeNil) + + err = cacheDriver.PutBlob("key2", "duplicate") + So(err, ShouldBeNil) + + err = cacheDriver.PutBlob("key2", "duplicate") + So(err, ShouldBeNil) + + val, err = cacheDriver.GetBlob("key2") + So(val, ShouldEqual, "duplicate") + So(err, ShouldBeNil) + + err = cacheDriver.DeleteBlob("key2", "duplicate") + So(err, ShouldBeNil) + + // should be empty + val, err = cacheDriver.GetBlob("key2") + So(err, ShouldNotBeNil) + So(val, ShouldBeEmpty) + }) + + Convey("Test cache.GetAllBlos()", t, func() { + dir := t.TempDir() + + log := log.NewLogger("debug", "") + So(log, ShouldNotBeNil) + + connOpts, _ := redis.ParseURL("redis://" + miniRedis.Addr()) + client := redis.NewClient(connOpts) + + cacheDriver, err := storage.Create("redis", + cache.RedisDriverParameters{client, dir, true, "zot"}, log) + So(cacheDriver, ShouldNotBeNil) + So(err, ShouldBeNil) + + name := cacheDriver.Name() + So(name, ShouldEqual, "redis") + + blobs, err := cacheDriver.GetAllBlobs("digest") + So(err, ShouldEqual, zerr.ErrCacheMiss) + So(blobs, ShouldBeNil) + + err = cacheDriver.PutBlob("digest", path.Join(dir, "first")) + So(err, ShouldBeNil) + + err = cacheDriver.PutBlob("digest", path.Join(dir, "second")) + So(err, ShouldBeNil) + + err = cacheDriver.PutBlob("digest", path.Join(dir, "third")) + So(err, ShouldBeNil) + + blobs, err = cacheDriver.GetAllBlobs("digest") + So(err, ShouldBeNil) + + So(blobs, ShouldResemble, []string{"first", "second", "third"}) + + err = cacheDriver.DeleteBlob("digest", path.Join(dir, "first")) + So(err, ShouldBeNil) + + blobs, err = cacheDriver.GetAllBlobs("digest") + So(err, ShouldBeNil) + So(len(blobs), ShouldEqual, 2) + So(blobs, ShouldContain, "second") + So(blobs, ShouldContain, "third") + + err = cacheDriver.DeleteBlob("digest", path.Join(dir, "third")) + So(err, ShouldBeNil) + + blobs, err = cacheDriver.GetAllBlobs("digest") + So(err, ShouldBeNil) + + So(blobs, ShouldResemble, []string{"second"}) + }) +} + +func TestRedisCacheError(t *testing.T) { + Convey("Make a new cache", t, func() { + dir := t.TempDir() + redisURL := "redis://127.0.0.1:" + test.GetFreePort() + connOpts, _ := redis.ParseURL(redisURL) + brokenClient := redis.NewClient(connOpts) + + log := log.NewLogger("debug", "") + So(log, ShouldNotBeNil) + + // redis server is not running + cacheDriver, err := storage.Create("redis", + cache.RedisDriverParameters{brokenClient, dir, true, "zot"}, log) + So(err, ShouldNotBeNil) + So(cacheDriver, ShouldBeNil) + }) + + Convey("Redis unreachable", t, func() { + miniRedis := miniredis.RunT(t) + dir := t.TempDir() + + log := log.NewLogger("debug", "") + So(log, ShouldNotBeNil) + + connOpts, _ := redis.ParseURL("redis://" + miniRedis.Addr()) + workingClient := redis.NewClient(connOpts) + + redisURL := "redis://127.0.0.1:" + test.GetFreePort() // must not match miniRedis.Addr() + connOpts, _ = redis.ParseURL(redisURL) + brokenClient := redis.NewClient(connOpts) + + cacheDriver, err := cache.NewRedisCache( + cache.RedisDriverParameters{workingClient, dir, false, "zot"}, log) + So(cacheDriver, ShouldNotBeNil) + So(err, ShouldBeNil) + + // replace the working driver with the broken one + cacheDriver.SetClient(brokenClient) + + err = cacheDriver.PutBlob("key", "val") + So(err, ShouldNotBeNil) + + found := cacheDriver.HasBlob("key", "val") + So(found, ShouldEqual, false) + + _, err = cacheDriver.GetBlob("key") + So(err, ShouldNotBeNil) + + _, err = cacheDriver.GetAllBlobs("key") + So(err, ShouldNotBeNil) + + err = cacheDriver.DeleteBlob("key", "val") + So(err, ShouldNotBeNil) + }) +} + +func TestRedisMocked(t *testing.T) { + Convey("Redis tests using mocks", t, func() { + dir := t.TempDir() + + log := log.NewLogger("debug", "") + So(log, ShouldNotBeNil) + + tests := []cache.RedisDriverParameters{ + { + RootDir: dir, + UseRelPaths: true, + }, { + RootDir: dir, + UseRelPaths: false, + }, { + RootDir: dir, + UseRelPaths: true, + KeyPrefix: "someprefix", + }, { + RootDir: dir, + UseRelPaths: true, + KeyPrefix: "zot", + }, + } + + for i, redisDriverParams := range tests { + testID := fmt.Sprintf(" %d", i) + + keyPrefix := redisDriverParams.KeyPrefix + if len(keyPrefix) == 0 { + // check default + keyPrefix = "zot" + } + keyPrefix += ":" + + // depending on UseRelPaths value we check the relative or absolute value + // in results using path.Join(pathPrefix, path) in both cases + pathPrefix := "" + if !redisDriverParams.UseRelPaths { + pathPrefix = redisDriverParams.RootDir + } + + Convey("PutBlob HExists error"+testID, func() { + // initialize mock client + cacheDB, mock := redismock.NewClientMock() + redisDriverParams.Client = cacheDB + + mock.ExpectPing().SetVal("OK") + cacheDriver, err := cache.NewRedisCache(redisDriverParams, log) + So(cacheDriver, ShouldNotBeNil) + So(err, ShouldBeNil) + + mock.Regexp().ExpectSetNX(keyPrefix+"locks:key", `.*`, 8*time.Second).SetVal(true) + mock.ExpectHExists(keyPrefix+constants.BlobsCache+":"+constants.OriginalBucket, "key"). + SetErr(ErrTestError) + + err = cacheDriver.PutBlob("key", path.Join(dir, "val")) + So(err, ShouldEqual, ErrTestError) + + err = mock.ExpectationsWereMet() + So(err, ShouldBeNil) + }) + + Convey("PutBlob HSet error"+testID, func() { + // initialize mock client + cacheDB, mock := redismock.NewClientMock() + redisDriverParams.Client = cacheDB + + mock.ExpectPing().SetVal("OK") + cacheDriver, err := cache.NewRedisCache(redisDriverParams, log) + So(cacheDriver, ShouldNotBeNil) + So(err, ShouldBeNil) + + mock.Regexp().ExpectSetNX(keyPrefix+"locks:key", `.*`, 8*time.Second).SetVal(true) + mock.ExpectHExists(keyPrefix+constants.BlobsCache+":"+constants.OriginalBucket, "key"). + SetVal(false) + mock.ExpectTxPipeline() + mock.ExpectHSet(keyPrefix+constants.BlobsCache+":"+constants.OriginalBucket, "key", + path.Join(pathPrefix, "val")).SetErr(ErrTestError) + + err = cacheDriver.PutBlob("key", path.Join(dir, "val")) + So(err, ShouldEqual, ErrTestError) + + err = mock.ExpectationsWereMet() + So(err, ShouldBeNil) + }) + + Convey("PutBlob SAdd error"+testID, func() { + // initialize mock client + cacheDB, mock := redismock.NewClientMock() + redisDriverParams.Client = cacheDB + + mock.ExpectPing().SetVal("OK") + cacheDriver, err := cache.NewRedisCache(redisDriverParams, log) + So(cacheDriver, ShouldNotBeNil) + So(err, ShouldBeNil) + + mock.Regexp().ExpectSetNX(keyPrefix+"locks:key", `.*`, 8*time.Second).SetVal(true) + mock.ExpectHExists(keyPrefix+constants.BlobsCache+":"+constants.OriginalBucket, "key"). + SetVal(false) + mock.ExpectTxPipeline() + mock.ExpectHSet(keyPrefix+constants.BlobsCache+":"+constants.OriginalBucket, "key", + path.Join(pathPrefix, "val")).SetVal(1) + mock.ExpectSAdd(keyPrefix+constants.BlobsCache+":"+constants.DuplicatesBucket+":key", + path.Join(pathPrefix, "val")).SetErr(ErrTestError) + + err = cacheDriver.PutBlob("key", path.Join(dir, "val")) + So(err, ShouldEqual, ErrTestError) + + err = mock.ExpectationsWereMet() + So(err, ShouldBeNil) + }) + + Convey("PutBlob succeeds original bucket is created"+testID, func() { + // initialize mock client + cacheDB, mock := redismock.NewClientMock() + redisDriverParams.Client = cacheDB + + mock.ExpectPing().SetVal("OK") + cacheDriver, err := cache.NewRedisCache(redisDriverParams, log) + So(cacheDriver, ShouldNotBeNil) + So(err, ShouldBeNil) + + mock.Regexp().ExpectSetNX(keyPrefix+"locks:key", `.*`, 8*time.Second).SetVal(true) + mock.ExpectHExists(keyPrefix+constants.BlobsCache+":"+constants.OriginalBucket, "key"). + SetVal(false) + mock.ExpectTxPipeline() + mock.ExpectHSet(keyPrefix+constants.BlobsCache+":"+constants.OriginalBucket, "key", + path.Join(pathPrefix, "val")).SetVal(1) + mock.ExpectSAdd(keyPrefix+constants.BlobsCache+":"+constants.DuplicatesBucket+":key", + path.Join(pathPrefix, "val")).SetVal(1) + mock.ExpectTxPipelineExec() + + err = cacheDriver.PutBlob("key", path.Join(dir, "val")) + So(err, ShouldBeNil) + + err = mock.ExpectationsWereMet() + So(err, ShouldBeNil) + }) + + Convey("PutBlob succeeds original bucket is reused"+testID, func() { + // initialize mock client + cacheDB, mock := redismock.NewClientMock() + redisDriverParams.Client = cacheDB + + mock.ExpectPing().SetVal("OK") + cacheDriver, err := cache.NewRedisCache(redisDriverParams, log) + So(cacheDriver, ShouldNotBeNil) + So(err, ShouldBeNil) + + mock.Regexp().ExpectSetNX(keyPrefix+"locks:key", `.*`, 8*time.Second).SetVal(true) + mock.ExpectHExists(keyPrefix+constants.BlobsCache+":"+constants.OriginalBucket, "key"). + SetVal(true) + mock.ExpectTxPipeline() + mock.ExpectSAdd(keyPrefix+constants.BlobsCache+":"+constants.DuplicatesBucket+":key", + path.Join(pathPrefix, "val")).SetVal(1) + mock.ExpectTxPipelineExec() + + err = cacheDriver.PutBlob("key", path.Join(dir, "val")) + So(err, ShouldBeNil) + + err = mock.ExpectationsWereMet() + So(err, ShouldBeNil) + }) + + Convey("SMembers error in GetAllBlobs"+testID, func() { + // initialize mock client + cacheDB, mock := redismock.NewClientMock() + redisDriverParams.Client = cacheDB + + mock.ExpectPing().SetVal("OK") + cacheDriver, err := cache.NewRedisCache(redisDriverParams, log) + So(cacheDriver, ShouldNotBeNil) + So(err, ShouldBeNil) + + mock.ExpectHGet(keyPrefix+constants.BlobsCache+":"+constants.OriginalBucket, "key"). + SetVal(path.Join(pathPrefix, "val")) + mock.ExpectSMembers(keyPrefix + constants.BlobsCache + ":" + constants.DuplicatesBucket + ":key"). + SetErr(ErrTestError) + + _, err = cacheDriver.GetAllBlobs("key") + So(err, ShouldEqual, ErrTestError) + + err = mock.ExpectationsWereMet() + So(err, ShouldBeNil) + }) + + Convey("GetAllBlobs succeeds"+testID, func() { + // initialize mock client + cacheDB, mock := redismock.NewClientMock() + redisDriverParams.Client = cacheDB + + mock.ExpectPing().SetVal("OK") + cacheDriver, err := cache.NewRedisCache(redisDriverParams, log) + So(cacheDriver, ShouldNotBeNil) + So(err, ShouldBeNil) + + mock.Regexp().ExpectSetNX(keyPrefix+"locks:key", `.*`, 8*time.Second).SetVal(true) + mock.ExpectHExists(keyPrefix+constants.BlobsCache+":"+constants.OriginalBucket, "key"). + SetVal(false) + mock.ExpectTxPipeline() + mock.ExpectHSet(keyPrefix+constants.BlobsCache+":"+constants.OriginalBucket, "key", + path.Join(pathPrefix, "val1")).SetVal(1) + mock.ExpectSAdd(keyPrefix+constants.BlobsCache+":"+constants.DuplicatesBucket+":key", + path.Join(pathPrefix, "val1")).SetVal(1) + mock.ExpectTxPipelineExec() + + err = cacheDriver.PutBlob("key", path.Join(dir, "val1")) + So(err, ShouldBeNil) + + mock.Regexp().ExpectSetNX(keyPrefix+"locks:key", `.*`, 8*time.Second).SetVal(true) + mock.ExpectHExists(keyPrefix+constants.BlobsCache+":"+constants.OriginalBucket, "key"). + SetVal(false) + mock.ExpectTxPipeline() + mock.ExpectHSet(keyPrefix+constants.BlobsCache+":"+constants.OriginalBucket, "key", + path.Join(pathPrefix, "val2")).SetVal(1) + mock.ExpectSAdd(keyPrefix+constants.BlobsCache+":"+constants.DuplicatesBucket+":key", + path.Join(pathPrefix, "val2")).SetVal(1) + mock.ExpectTxPipelineExec() + + err = cacheDriver.PutBlob("key", path.Join(dir, "val2")) + So(err, ShouldBeNil) + + mock.ExpectHGet(keyPrefix+constants.BlobsCache+":"+constants.OriginalBucket, "key"). + SetVal(path.Join(pathPrefix, "val1")) + mock.ExpectSMembers(keyPrefix + constants.BlobsCache + ":" + constants.DuplicatesBucket + ":key"). + SetVal([]string{path.Join(pathPrefix, "val1"), path.Join(pathPrefix, "val2")}) + + allBlobs, err := cacheDriver.GetAllBlobs("key") + So(err, ShouldBeNil) + So(allBlobs, ShouldResemble, []string{path.Join(pathPrefix, "val1"), path.Join(pathPrefix, "val2")}) + + err = mock.ExpectationsWereMet() + So(err, ShouldBeNil) + }) + + Convey("HasBlob HExists returns error"+testID, func() { + // initialize mock client + cacheDB, mock := redismock.NewClientMock() + redisDriverParams.Client = cacheDB + + mock.ExpectPing().SetVal("OK") + cacheDriver, err := cache.NewRedisCache(redisDriverParams, log) + So(cacheDriver, ShouldNotBeNil) + So(err, ShouldBeNil) + + mock.ExpectSIsMember(keyPrefix+constants.BlobsCache+":"+constants.DuplicatesBucket+":key", + path.Join(pathPrefix, "val")).SetVal(true) + mock.ExpectHExists(keyPrefix+constants.BlobsCache+":"+constants.OriginalBucket, "key"). + SetErr(ErrTestError) + + ok := cacheDriver.HasBlob("key", path.Join(dir, "val")) + So(ok, ShouldBeFalse) + + err = mock.ExpectationsWereMet() + So(err, ShouldBeNil) + }) + + Convey("HasBlob SIsMember returns error"+testID, func() { + // initialize mock client + cacheDB, mock := redismock.NewClientMock() + redisDriverParams.Client = cacheDB + + mock.ExpectPing().SetVal("OK") + cacheDriver, err := cache.NewRedisCache(redisDriverParams, log) + So(cacheDriver, ShouldNotBeNil) + So(err, ShouldBeNil) + + mock.ExpectSIsMember(keyPrefix+constants.BlobsCache+":"+constants.DuplicatesBucket+":key", + path.Join(pathPrefix, "val")).SetErr(ErrTestError) + + ok := cacheDriver.HasBlob("key", path.Join(dir, "val")) + So(ok, ShouldBeFalse) + + err = mock.ExpectationsWereMet() + So(err, ShouldBeNil) + }) + + Convey("HasBlob HExists returns false"+testID, func() { + // initialize mock client + cacheDB, mock := redismock.NewClientMock() + redisDriverParams.Client = cacheDB + + mock.ExpectPing().SetVal("OK") + cacheDriver, err := cache.NewRedisCache(redisDriverParams, log) + So(cacheDriver, ShouldNotBeNil) + So(err, ShouldBeNil) + + mock.ExpectSIsMember(keyPrefix+constants.BlobsCache+":"+constants.DuplicatesBucket+":key", + path.Join(pathPrefix, "val")).SetVal(true) + mock.ExpectHExists(keyPrefix+constants.BlobsCache+":"+constants.OriginalBucket, "key"). + SetVal(false) + + ok := cacheDriver.HasBlob("key", path.Join(dir, "val")) + So(ok, ShouldBeFalse) + + err = mock.ExpectationsWereMet() + So(err, ShouldBeNil) + }) + + Convey("DeleteBlob tests"+testID, func() { + // initialize mock client + cacheDB, mock := redismock.NewClientMock() + redisDriverParams.Client = cacheDB + + mock.ExpectPing().SetVal("OK") + cacheDriver, err := cache.NewRedisCache(redisDriverParams, log) + So(cacheDriver, ShouldNotBeNil) + So(err, ShouldBeNil) + + // Create entry for 1st path + mock.Regexp().ExpectSetNX(keyPrefix+"locks:key", `.*`, 8*time.Second).SetVal(true) + mock.ExpectHExists(keyPrefix+constants.BlobsCache+":"+constants.OriginalBucket, "key"). + SetVal(false) + mock.ExpectTxPipeline() + mock.ExpectHSet(keyPrefix+constants.BlobsCache+":"+constants.OriginalBucket, "key", + path.Join(pathPrefix, "val1")).SetVal(1) + mock.ExpectSAdd(keyPrefix+constants.BlobsCache+":"+constants.DuplicatesBucket+":key", + path.Join(pathPrefix, "val1")).SetVal(1) + mock.ExpectTxPipelineExec() + + err = cacheDriver.PutBlob("key", path.Join(dir, "val1")) + So(err, ShouldBeNil) + + Convey("DeleteBlob error in HDel"+testID, func() { + // If the 2nd path does not exist, HDel is callled + // Error switching to new path + mock.Regexp().ExpectSetNX(keyPrefix+"locks:key", `.*`, 8*time.Second).SetVal(true) + mock.ExpectSRem(keyPrefix+constants.BlobsCache+":"+constants.DuplicatesBucket+":key", + path.Join(pathPrefix, "val1")).SetVal(1) + mock.ExpectHGet(keyPrefix+constants.BlobsCache+":"+constants.OriginalBucket, "key"). + SetVal(path.Join(pathPrefix, "val1")) + // failed to get new path + mock.ExpectSRandMember(keyPrefix + constants.BlobsCache + ":" + constants.DuplicatesBucket + ":key"). + RedisNil() + mock.ExpectHDel(keyPrefix+constants.BlobsCache+":"+constants.OriginalBucket, "key"). + SetErr(ErrTestError) + + err = cacheDriver.DeleteBlob("key", path.Join(dir, "val1")) + So(err, ShouldEqual, ErrTestError) + }) + + Convey("DeleteBlob succeeds in deleting all data for original blob"+testID, func() { + // If the 2nd path does not exist, HDel is callled + // Error switching to new path + mock.Regexp().ExpectSetNX(keyPrefix+"locks:key", `.*`, 8*time.Second).SetVal(true) + mock.ExpectSRem(keyPrefix+constants.BlobsCache+":"+constants.DuplicatesBucket+":key", + path.Join(pathPrefix, "val1")).SetVal(1) + mock.ExpectHGet(keyPrefix+constants.BlobsCache+":"+constants.OriginalBucket, "key"). + SetVal(path.Join(pathPrefix, "val1")) + // failed to get new path + mock.ExpectSRandMember(keyPrefix + constants.BlobsCache + ":" + constants.DuplicatesBucket + ":key"). + RedisNil() + mock.ExpectHDel(keyPrefix+constants.BlobsCache+":"+constants.OriginalBucket, "key"). + SetVal(1) + + err = cacheDriver.DeleteBlob("key", path.Join(dir, "val1")) + So(err, ShouldBeNil) + }) + + Convey("DeleteBlob error in SRandMember"+testID, func() { + // Create entry for 2nd path + mock.Regexp().ExpectSetNX(keyPrefix+"locks:key", `.*`, 8*time.Second).SetVal(true) + mock.ExpectHExists(keyPrefix+constants.BlobsCache+":"+constants.OriginalBucket, "key"). + SetVal(false) + mock.ExpectTxPipeline() + mock.ExpectHSet(keyPrefix+constants.BlobsCache+":"+constants.OriginalBucket, "key", + path.Join(pathPrefix, "val2")).SetVal(1) + mock.ExpectSAdd(keyPrefix+constants.BlobsCache+":"+constants.DuplicatesBucket+":key", + path.Join(pathPrefix, "val2")).SetVal(1) + mock.ExpectTxPipelineExec() + + err = cacheDriver.PutBlob("key", path.Join(dir, "val2")) + So(err, ShouldBeNil) + + // Error switching to new path + mock.Regexp().ExpectSetNX(keyPrefix+"locks:key", `.*`, 8*time.Second).SetVal(true) + mock.ExpectSRem(keyPrefix+constants.BlobsCache+":"+constants.DuplicatesBucket+":key", + path.Join(pathPrefix, "val1")).SetVal(1) + mock.ExpectHGet(keyPrefix+constants.BlobsCache+":"+constants.OriginalBucket, "key"). + SetVal(path.Join(pathPrefix, "val1")) + // failed to get new path + mock.ExpectSRandMember(keyPrefix + constants.BlobsCache + ":" + constants.DuplicatesBucket + ":key"). + SetErr(ErrTestError) + + err = cacheDriver.DeleteBlob("key", path.Join(dir, "val1")) + So(err, ShouldEqual, ErrTestError) + }) + + Convey("DeleteBlob error in HSet"+testID, func() { + // Create entry for 2nd path + mock.Regexp().ExpectSetNX(keyPrefix+"locks:key", `.*`, 8*time.Second).SetVal(true) + mock.ExpectHExists(keyPrefix+constants.BlobsCache+":"+constants.OriginalBucket, "key"). + SetVal(false) + mock.ExpectTxPipeline() + mock.ExpectHSet(keyPrefix+constants.BlobsCache+":"+constants.OriginalBucket, "key", + path.Join(pathPrefix, "val2")).SetVal(1) + mock.ExpectSAdd(keyPrefix+constants.BlobsCache+":"+constants.DuplicatesBucket+":key", + path.Join(pathPrefix, "val2")).SetVal(1) + mock.ExpectTxPipelineExec() + + err = cacheDriver.PutBlob("key", path.Join(dir, "val2")) + So(err, ShouldBeNil) + + // Error switching to new path + mock.Regexp().ExpectSetNX(keyPrefix+"locks:key", `.*`, 8*time.Second).SetVal(true) + mock.ExpectSRem(keyPrefix+constants.BlobsCache+":"+constants.DuplicatesBucket+":key", + path.Join(pathPrefix, "val1")).SetVal(1) + mock.ExpectHGet(keyPrefix+constants.BlobsCache+":"+constants.OriginalBucket, "key"). + SetVal(path.Join(pathPrefix, "val1")) + mock.ExpectSRandMember(keyPrefix + constants.BlobsCache + ":" + constants.DuplicatesBucket + ":key"). + SetVal(path.Join(pathPrefix, "val2")) + mock.ExpectHSet(keyPrefix+constants.BlobsCache+":"+constants.OriginalBucket, "key", + path.Join(pathPrefix, "val2")).SetErr(ErrTestError) + + err = cacheDriver.DeleteBlob("key", path.Join(dir, "val1")) + So(err, ShouldEqual, ErrTestError) + }) + + Convey("DeleteBlob succeeds in switching original blob path"+testID, func() { + // Create entry for 2nd path + mock.Regexp().ExpectSetNX(keyPrefix+"locks:key", `.*`, 8*time.Second).SetVal(true) + mock.ExpectHExists(keyPrefix+constants.BlobsCache+":"+constants.OriginalBucket, "key"). + SetVal(false) + mock.ExpectTxPipeline() + mock.ExpectHSet(keyPrefix+constants.BlobsCache+":"+constants.OriginalBucket, "key", + path.Join(pathPrefix, "val2")).SetVal(1) + mock.ExpectSAdd(keyPrefix+constants.BlobsCache+":"+constants.DuplicatesBucket+":key", + path.Join(pathPrefix, "val2")).SetVal(1) + mock.ExpectTxPipelineExec() + + err = cacheDriver.PutBlob("key", path.Join(dir, "val2")) + So(err, ShouldBeNil) + + mock.Regexp().ExpectSetNX(keyPrefix+"locks:key", `.*`, 8*time.Second).SetVal(true) + mock.ExpectSRem(keyPrefix+constants.BlobsCache+":"+constants.DuplicatesBucket+":key", + path.Join(pathPrefix, "val1")).SetVal(1) + mock.ExpectHGet(keyPrefix+constants.BlobsCache+":"+constants.OriginalBucket, "key"). + SetVal(path.Join(pathPrefix, "val1")) + mock.ExpectSRandMember(keyPrefix + constants.BlobsCache + ":" + constants.DuplicatesBucket + ":key"). + SetVal(path.Join(pathPrefix, "val2")) + mock.ExpectHSet(keyPrefix+constants.BlobsCache+":"+constants.OriginalBucket, "key", + path.Join(pathPrefix, "val2")).SetVal(1) + + err = cacheDriver.DeleteBlob("key", path.Join(dir, "val1")) + So(err, ShouldBeNil) + }) + + err = mock.ExpectationsWereMet() + So(err, ShouldBeNil) + }) + } + }) +} diff --git a/pkg/storage/cache_benchmark_test.go b/pkg/storage/cache_benchmark_test.go index 0b149b59..7288369a 100644 --- a/pkg/storage/cache_benchmark_test.go +++ b/pkg/storage/cache_benchmark_test.go @@ -11,6 +11,7 @@ import ( "zotregistry.dev/zot/pkg/log" "zotregistry.dev/zot/pkg/storage" "zotregistry.dev/zot/pkg/storage/cache" + storageTypes "zotregistry.dev/zot/pkg/storage/types" test "zotregistry.dev/zot/pkg/test/common" ) @@ -44,31 +45,31 @@ func generateData() map[godigest.Digest]string { return dataMap } -func helperPutAll(cache cache.Cache, testData map[godigest.Digest]string) { +func helperPutAll(cache storageTypes.Cache, testData map[godigest.Digest]string) { for digest, path := range testData { _ = cache.PutBlob(digest, path) } } -func helperDeleteAll(cache cache.Cache, testData map[godigest.Digest]string) { +func helperDeleteAll(cache storageTypes.Cache, testData map[godigest.Digest]string) { for digest, path := range testData { _ = cache.DeleteBlob(digest, path) } } -func helperHasAll(cache cache.Cache, testData map[godigest.Digest]string) { +func helperHasAll(cache storageTypes.Cache, testData map[godigest.Digest]string) { for digest, path := range testData { _ = cache.HasBlob(digest, path) } } -func helperGetAll(cache cache.Cache, testData map[godigest.Digest]string) { +func helperGetAll(cache storageTypes.Cache, testData map[godigest.Digest]string) { for digest := range testData { _, _ = cache.GetBlob(digest) } } -func helperMix(cache cache.Cache, testData map[godigest.Digest]string, digestSlice []godigest.Digest) { +func helperMix(cache storageTypes.Cache, testData map[godigest.Digest]string, digestSlice []godigest.Digest) { // The test data contains datasetSize entries by default, and each set of operations uses 5 entries for i := 0; i < 1000; i++ { _ = cache.PutBlob(digestSlice[i*5], testData[digestSlice[i*5]]) diff --git a/pkg/storage/cache_test.go b/pkg/storage/cache_test.go index 16d06374..ae1534fc 100644 --- a/pkg/storage/cache_test.go +++ b/pkg/storage/cache_test.go @@ -72,4 +72,13 @@ func TestCache(t *testing.T) { So(err, ShouldNotBeNil) So(err, ShouldEqual, errors.ErrEmptyValue) }) + + Convey("Invalid cache driver dbtype", t, func() { + log := log.NewLogger("debug", "") + So(log, ShouldNotBeNil) + + cacheDriver, err := storage.Create("sometype", map[string]interface{}{}, log) + So(err, ShouldEqual, errors.ErrBadConfig) + So(cacheDriver, ShouldBeNil) + }) } diff --git a/pkg/storage/constants/constants.go b/pkg/storage/constants/constants.go index 905178bd..0aba30a2 100644 --- a/pkg/storage/constants/constants.go +++ b/pkg/storage/constants/constants.go @@ -19,6 +19,8 @@ const ( DBCacheLockCheckTimeout = 10 * time.Second BoltdbName = "cache" DynamoDBDriverName = "dynamodb" + RedisDriverName = "redis" + RedisLocksBucket = "locks" DefaultGCDelay = 1 * time.Hour DefaultRetentionDelay = 24 * time.Hour DefaultGCInterval = 1 * time.Hour diff --git a/pkg/storage/imagestore/imagestore.go b/pkg/storage/imagestore/imagestore.go index d37842e0..ab753f18 100644 --- a/pkg/storage/imagestore/imagestore.go +++ b/pkg/storage/imagestore/imagestore.go @@ -27,7 +27,6 @@ import ( zlog "zotregistry.dev/zot/pkg/log" zreg "zotregistry.dev/zot/pkg/regexp" "zotregistry.dev/zot/pkg/scheduler" - "zotregistry.dev/zot/pkg/storage/cache" common "zotregistry.dev/zot/pkg/storage/common" storageConstants "zotregistry.dev/zot/pkg/storage/constants" storageTypes "zotregistry.dev/zot/pkg/storage/types" @@ -46,7 +45,7 @@ type ImageStore struct { lock *sync.RWMutex log zlog.Logger metrics monitoring.MetricServer - cache cache.Cache + cache storageTypes.Cache dedupe bool linter common.Lint commit bool @@ -70,7 +69,7 @@ func (is *ImageStore) DirExists(d string) bool { // Use the last argument to properly set a cache database, or it will default to boltDB local storage. func NewImageStore(rootDir string, cacheDir string, dedupe, commit bool, log zlog.Logger, metrics monitoring.MetricServer, linter common.Lint, storeDriver storageTypes.Driver, - cacheDriver cache.Cache, compat []compat.MediaCompatibility, + cacheDriver storageTypes.Cache, compat []compat.MediaCompatibility, ) storageTypes.ImageStore { if err := storeDriver.EnsureDir(rootDir); err != nil { log.Error().Err(err).Str("rootDir", rootDir).Msg("failed to create root dir") diff --git a/pkg/storage/local/local.go b/pkg/storage/local/local.go index 3309415c..3c2e372d 100644 --- a/pkg/storage/local/local.go +++ b/pkg/storage/local/local.go @@ -4,7 +4,6 @@ import ( "zotregistry.dev/zot/pkg/compat" "zotregistry.dev/zot/pkg/extensions/monitoring" zlog "zotregistry.dev/zot/pkg/log" - "zotregistry.dev/zot/pkg/storage/cache" common "zotregistry.dev/zot/pkg/storage/common" "zotregistry.dev/zot/pkg/storage/imagestore" storageTypes "zotregistry.dev/zot/pkg/storage/types" @@ -13,7 +12,7 @@ import ( // NewImageStore returns a new image store backed by a file storage. // Use the last argument to properly set a cache database, or it will default to boltDB local storage. func NewImageStore(rootDir string, dedupe, commit bool, log zlog.Logger, - metrics monitoring.MetricServer, linter common.Lint, cacheDriver cache.Cache, + metrics monitoring.MetricServer, linter common.Lint, cacheDriver storageTypes.Cache, compat []compat.MediaCompatibility, ) storageTypes.ImageStore { return imagestore.NewImageStore( diff --git a/pkg/storage/s3/s3.go b/pkg/storage/s3/s3.go index 41a8915c..685e2a3a 100644 --- a/pkg/storage/s3/s3.go +++ b/pkg/storage/s3/s3.go @@ -9,7 +9,6 @@ import ( "zotregistry.dev/zot/pkg/compat" "zotregistry.dev/zot/pkg/extensions/monitoring" zlog "zotregistry.dev/zot/pkg/log" - "zotregistry.dev/zot/pkg/storage/cache" common "zotregistry.dev/zot/pkg/storage/common" "zotregistry.dev/zot/pkg/storage/imagestore" storageTypes "zotregistry.dev/zot/pkg/storage/types" @@ -20,7 +19,7 @@ import ( // Use the last argument to properly set a cache database, or it will default to boltDB local storage. func NewImageStore(rootDir string, cacheDir string, dedupe, commit bool, log zlog.Logger, metrics monitoring.MetricServer, linter common.Lint, store driver.StorageDriver, - cacheDriver cache.Cache, compat []compat.MediaCompatibility, + cacheDriver storageTypes.Cache, compat []compat.MediaCompatibility, ) storageTypes.ImageStore { return imagestore.NewImageStore( rootDir, diff --git a/pkg/storage/s3/s3_test.go b/pkg/storage/s3/s3_test.go index f21b3ffc..c9f75eec 100644 --- a/pkg/storage/s3/s3_test.go +++ b/pkg/storage/s3/s3_test.go @@ -63,7 +63,7 @@ func createMockStorage(rootDir string, cacheDir string, dedupe bool, store drive log := log.Logger{Logger: zerolog.New(os.Stdout)} metrics := monitoring.NewMetricsServer(true, log) - var cacheDriver cache.Cache + var cacheDriver storageTypes.Cache // from pkg/cli/server/root.go/applyDefaultValues, s3 magic if _, err := os.Stat(path.Join(cacheDir, @@ -81,7 +81,7 @@ func createMockStorage(rootDir string, cacheDir string, dedupe bool, store drive } func createMockStorageWithMockCache(rootDir string, dedupe bool, store driver.StorageDriver, - cacheDriver cache.Cache, + cacheDriver storageTypes.Cache, ) storageTypes.ImageStore { log := log.Logger{Logger: zerolog.New(os.Stdout)} metrics := monitoring.NewMetricsServer(false, log) @@ -133,7 +133,7 @@ func createObjectsStore(rootDir string, cacheDir string, dedupe bool) ( log := log.Logger{Logger: zerolog.New(os.Stdout)} metrics := monitoring.NewMetricsServer(false, log) - var cacheDriver cache.Cache + var cacheDriver storageTypes.Cache var err error @@ -162,7 +162,7 @@ func createObjectsStoreDynamo(rootDir string, cacheDir string, dedupe bool, tabl log := log.Logger{Logger: zerolog.New(os.Stdout)} metrics := monitoring.NewMetricsServer(false, log) - var cacheDriver cache.Cache + var cacheDriver storageTypes.Cache // from pkg/cli/server/root.go/applyDefaultValues, s3 magic tableName = strings.ReplaceAll(tableName, "/", "") diff --git a/pkg/storage/scrub_test.go b/pkg/storage/scrub_test.go index 468b7a00..d7e1a895 100644 --- a/pkg/storage/scrub_test.go +++ b/pkg/storage/scrub_test.go @@ -11,20 +11,21 @@ import ( "strings" "testing" - "github.com/distribution/distribution/v3/registry/storage/driver" + "github.com/alicebob/miniredis/v2" guuid "github.com/gofrs/uuid" godigest "github.com/opencontainers/go-digest" ispec "github.com/opencontainers/image-spec/specs-go/v1" . "github.com/smartystreets/goconvey/convey" zerr "zotregistry.dev/zot/errors" + rediscfg "zotregistry.dev/zot/pkg/api/config/redis" "zotregistry.dev/zot/pkg/extensions/monitoring" "zotregistry.dev/zot/pkg/log" "zotregistry.dev/zot/pkg/storage" "zotregistry.dev/zot/pkg/storage/cache" common "zotregistry.dev/zot/pkg/storage/common" + storageConstants "zotregistry.dev/zot/pkg/storage/constants" "zotregistry.dev/zot/pkg/storage/local" - "zotregistry.dev/zot/pkg/storage/s3" storageTypes "zotregistry.dev/zot/pkg/storage/types" . "zotregistry.dev/zot/pkg/test/image-utils" "zotregistry.dev/zot/pkg/test/mocks" @@ -55,6 +56,29 @@ func TestLocalCheckAllBlobsIntegrity(t *testing.T) { }) } +func TestRedisCheckAllBlobsIntegrity(t *testing.T) { + miniRedis := miniredis.RunT(t) + + Convey("test with local storage", t, func() { + tdir := t.TempDir() + log := log.NewLogger("debug", "") + + metrics := monitoring.NewMetricsServer(false, log) + + client, _ := rediscfg.GetRedisClient(map[string]interface{}{"url": "redis://" + miniRedis.Addr()}, log) + + cacheDriver, _ := storage.Create("redis", cache.RedisDriverParameters{ + Client: client, + RootDir: tdir, + UseRelPaths: false, + }, log) + driver := local.New(true) + imgStore := local.NewImageStore(tdir, true, true, log, metrics, nil, cacheDriver, nil) + + RunCheckAllBlobsIntegrityTests(t, imgStore, driver, log) + }) +} + func TestS3CheckAllBlobsIntegrity(t *testing.T) { tskip.SkipS3(t) @@ -68,11 +92,15 @@ func TestS3CheckAllBlobsIntegrity(t *testing.T) { tdir := t.TempDir() log := log.NewLogger("debug", "") - var store driver.StorageDriver - store, imgStore, _ := createObjectsStore(testDir, tdir) - defer cleanupStorage(store, testDir) + opts := createObjectStoreOpts{ + rootDir: testDir, + cacheDir: tdir, + cacheType: storageConstants.BoltdbName, + storageType: storageConstants.S3StorageDriverName, + } - driver := s3.New(store) + driver, imgStore, _, _ := createObjectsStore(opts) + defer cleanupStorage(driver, testDir) RunCheckAllBlobsIntegrityTests(t, imgStore, driver, log) }) diff --git a/pkg/storage/storage_test.go b/pkg/storage/storage_test.go index 5a88621f..7a1cb814 100644 --- a/pkg/storage/storage_test.go +++ b/pkg/storage/storage_test.go @@ -16,19 +16,20 @@ import ( "testing" "time" - // Add s3 support. - "github.com/distribution/distribution/v3/registry/storage/driver" + "github.com/alicebob/miniredis/v2" "github.com/distribution/distribution/v3/registry/storage/driver/factory" _ "github.com/distribution/distribution/v3/registry/storage/driver/s3-aws" guuid "github.com/gofrs/uuid" godigest "github.com/opencontainers/go-digest" ispec "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/redis/go-redis/v9" "github.com/rs/zerolog" . "github.com/smartystreets/goconvey/convey" "gopkg.in/resty.v1" zerr "zotregistry.dev/zot/errors" "zotregistry.dev/zot/pkg/api/config" + rediscfg "zotregistry.dev/zot/pkg/api/config/redis" "zotregistry.dev/zot/pkg/extensions/monitoring" zlog "zotregistry.dev/zot/pkg/log" "zotregistry.dev/zot/pkg/storage" @@ -58,17 +59,65 @@ var DeleteReferrers = config.ImageRetention{ //nolint: gochecknoglobals }, } -func cleanupStorage(store driver.StorageDriver, name string) { - _ = store.Delete(context.Background(), name) +func cleanupStorage(store storageTypes.Driver, name string) { + _ = store.Delete(name) } -func createObjectsStore(rootDir string, cacheDir string) ( - driver.StorageDriver, storageTypes.ImageStore, error, +type createObjectStoreOpts struct { + storageType string + rootDir string + cacheDir string + cacheType string + miniRedisAddr string +} + +func createObjectsStore(options createObjectStoreOpts) ( + storageTypes.Driver, storageTypes.ImageStore, storageTypes.Cache, error, ) { + var ( + cacheDriver storageTypes.Cache + useRelPaths bool + ) + + log := zlog.Logger{Logger: zerolog.New(os.Stdout)} + + if options.storageType == storageConstants.S3StorageDriverName { + useRelPaths = false + } else { + useRelPaths = true + } + + if options.cacheType == storageConstants.RedisDriverName { + client, _ := rediscfg.GetRedisClient(map[string]interface{}{"url": options.miniRedisAddr}, log) + + cacheDriver, _ = storage.Create("redis", cache.RedisDriverParameters{ + Client: client, + RootDir: options.cacheDir, + UseRelPaths: useRelPaths, + }, log) + } else { + cacheDriver, _ = storage.Create("boltdb", cache.BoltDBDriverParameters{ + RootDir: options.cacheDir, + Name: "cache", + UseRelPaths: useRelPaths, + }, log) + } + + metrics := monitoring.NewMetricsServer(false, log) + + if options.storageType != storageConstants.S3StorageDriverName { + storeDriver := local.New(true) + + imgStore := imagestore.NewImageStore(options.rootDir, options.cacheDir, true, + true, log, metrics, nil, storeDriver, cacheDriver, nil) + + return storeDriver, imgStore, cacheDriver, nil + } + bucket := "zot-storage-test" endpoint := os.Getenv("S3MOCK_ENDPOINT") storageDriverParams := map[string]interface{}{ - "rootDir": rootDir, + "rootDir": options.rootDir, "name": "s3", "region": "us-east-2", "bucket": bucket, @@ -82,7 +131,7 @@ func createObjectsStore(rootDir string, cacheDir string) ( storeName := fmt.Sprintf("%v", storageDriverParams["name"]) - store, err := factory.Create(context.Background(), storeName, storageDriverParams) + s3Driver, err := factory.Create(context.Background(), storeName, storageDriverParams) if err != nil { panic(err) } @@ -93,32 +142,37 @@ func createObjectsStore(rootDir string, cacheDir string) ( panic(err) } - log := zlog.Logger{Logger: zerolog.New(os.Stdout)} - metrics := monitoring.NewMetricsServer(false, log) + imgStore := s3.NewImageStore(options.rootDir, options.cacheDir, true, false, log, + metrics, nil, s3Driver, cacheDriver, nil) - cacheDriver, _ := storage.Create("boltdb", cache.BoltDBDriverParameters{ - RootDir: cacheDir, - Name: "cache", - UseRelPaths: false, - }, log) - - il := s3.NewImageStore(rootDir, cacheDir, true, false, log, metrics, nil, store, cacheDriver, nil) - - return store, il, err + return s3.New(s3Driver), imgStore, cacheDriver, err } //nolint:gochecknoglobals var testCases = []struct { testCaseName string storageType string + cacheType string }{ { - testCaseName: "S3APIs", + testCaseName: "S3APIs_BoltDB", storageType: storageConstants.S3StorageDriverName, + cacheType: storageConstants.BoltdbName, }, { - testCaseName: "FileSystemAPIs", + testCaseName: "FileSystemAPIs_BoltDB", storageType: storageConstants.LocalStorageDriverName, + cacheType: storageConstants.BoltdbName, + }, + { + testCaseName: "S3APIs_Redis", + storageType: storageConstants.S3StorageDriverName, + cacheType: storageConstants.RedisDriverName, + }, + { + testCaseName: "FileSystemAPIs_Redis", + storageType: storageConstants.LocalStorageDriverName, + cacheType: storageConstants.RedisDriverName, }, } @@ -140,6 +194,21 @@ func TestGetAllDedupeReposCandidates(t *testing.T) { t.Run(testcase.testCaseName, func(t *testing.T) { var imgStore storageTypes.ImageStore + cacheDir := t.TempDir() + + opts := createObjectStoreOpts{ + rootDir: cacheDir, + cacheDir: cacheDir, + cacheType: testcase.cacheType, + storageType: testcase.storageType, + } + + if testcase.cacheType == storageConstants.RedisDriverName { + miniRedis := miniredis.RunT(t) + opts.miniRedisAddr = "redis://" + miniRedis.Addr() + defer DumpKeys(t, opts.miniRedisAddr) + } + if testcase.storageType == storageConstants.S3StorageDriverName { tskip.SkipS3(t) @@ -149,25 +218,13 @@ func TestGetAllDedupeReposCandidates(t *testing.T) { } testDir := path.Join("/oci-repo-test", uuid.String()) - tdir := t.TempDir() + opts.rootDir = testDir - var store driver.StorageDriver - store, imgStore, _ = createObjectsStore(testDir, tdir) + var store storageTypes.Driver + store, imgStore, _, _ = createObjectsStore(opts) defer cleanupStorage(store, testDir) } else { - dir := t.TempDir() - - log := zlog.Logger{Logger: zerolog.New(os.Stdout)} - metrics := monitoring.NewMetricsServer(false, log) - cacheDriver, _ := storage.Create("boltdb", cache.BoltDBDriverParameters{ - RootDir: dir, - Name: "cache", - UseRelPaths: true, - }, log) - - driver := local.New(true) - - imgStore = imagestore.NewImageStore(dir, dir, true, true, log, metrics, nil, driver, cacheDriver, nil) + _, imgStore, _, _ = createObjectsStore(opts) } Convey("Push repos with deduped blobs", t, func(c C) { @@ -210,6 +267,21 @@ func TestStorageAPIs(t *testing.T) { t.Run(testcase.testCaseName, func(t *testing.T) { var imgStore storageTypes.ImageStore + cacheDir := t.TempDir() + + opts := createObjectStoreOpts{ + rootDir: cacheDir, + cacheDir: cacheDir, + cacheType: testcase.cacheType, + storageType: testcase.storageType, + } + + if testcase.cacheType == storageConstants.RedisDriverName { + miniRedis := miniredis.RunT(t) + opts.miniRedisAddr = "redis://" + miniRedis.Addr() + defer DumpKeys(t, opts.miniRedisAddr) + } + if testcase.storageType == storageConstants.S3StorageDriverName { tskip.SkipS3(t) @@ -219,25 +291,13 @@ func TestStorageAPIs(t *testing.T) { } testDir := path.Join("/oci-repo-test", uuid.String()) - tdir := t.TempDir() + opts.rootDir = testDir - var store driver.StorageDriver - store, imgStore, _ = createObjectsStore(testDir, tdir) + var store storageTypes.Driver + store, imgStore, _, _ = createObjectsStore(opts) defer cleanupStorage(store, testDir) } else { - dir := t.TempDir() - - log := zlog.Logger{Logger: zerolog.New(os.Stdout)} - metrics := monitoring.NewMetricsServer(false, log) - cacheDriver, _ := storage.Create("boltdb", cache.BoltDBDriverParameters{ - RootDir: dir, - Name: "cache", - UseRelPaths: true, - }, log) - - driver := local.New(true) - - imgStore = imagestore.NewImageStore(dir, dir, true, true, log, metrics, nil, driver, cacheDriver, nil) + _, imgStore, _, _ = createObjectsStore(opts) } Convey("Repo layout", t, func(c C) { @@ -948,15 +1008,30 @@ func TestMandatoryAnnotations(t *testing.T) { for _, testcase := range testCases { testcase := testcase t.Run(testcase.testCaseName, func(t *testing.T) { - var imgStore storageTypes.ImageStore - - var testDir, tdir string - - var store driver.StorageDriver + var ( + imgStore storageTypes.ImageStore + store storageTypes.Driver + testDir string + ) log := zlog.Logger{Logger: zerolog.New(os.Stdout)} metrics := monitoring.NewMetricsServer(false, log) + cacheDir := t.TempDir() + + opts := createObjectStoreOpts{ + rootDir: cacheDir, + cacheDir: cacheDir, + cacheType: testcase.cacheType, + storageType: testcase.storageType, + } + + if testcase.cacheType == storageConstants.RedisDriverName { + miniRedis := miniredis.RunT(t) + opts.miniRedisAddr = "redis://" + miniRedis.Addr() + defer DumpKeys(t, opts.miniRedisAddr) + } + if testcase.storageType == storageConstants.S3StorageDriverName { tskip.SkipS3(t) @@ -966,32 +1041,29 @@ func TestMandatoryAnnotations(t *testing.T) { } testDir = path.Join("/oci-repo-test", uuid.String()) - tdir = t.TempDir() + opts.rootDir = testDir - store, _, _ = createObjectsStore(testDir, tdir) - driver := s3.New(store) - imgStore = imagestore.NewImageStore(testDir, tdir, false, false, log, metrics, + var cacheDriver storageTypes.Cache + store, _, cacheDriver, _ = createObjectsStore(opts) + + imgStore = imagestore.NewImageStore(testDir, cacheDir, false, false, log, metrics, &mocks.MockedLint{ LintFn: func(repo string, manifestDigest godigest.Digest, imageStore storageTypes.ImageStore) (bool, error) { return false, nil }, - }, driver, nil, nil) + }, store, cacheDriver, nil) defer cleanupStorage(store, testDir) } else { - tdir = t.TempDir() - cacheDriver, _ := storage.Create("boltdb", cache.BoltDBDriverParameters{ - RootDir: tdir, - Name: "cache", - UseRelPaths: true, - }, log) - driver := local.New(true) - imgStore = imagestore.NewImageStore(tdir, tdir, true, - true, log, metrics, &mocks.MockedLint{ + var cacheDriver storageTypes.Cache + store, _, cacheDriver, _ = createObjectsStore(opts) + + imgStore = imagestore.NewImageStore(cacheDir, cacheDir, true, true, log, metrics, + &mocks.MockedLint{ LintFn: func(repo string, manifestDigest godigest.Digest, imageStore storageTypes.ImageStore) (bool, error) { return false, nil }, - }, driver, cacheDriver, nil) + }, store, cacheDriver, nil) } Convey("Setup manifest", t, func() { @@ -1038,28 +1110,24 @@ func TestMandatoryAnnotations(t *testing.T) { Convey("Error on mandatory annotations", func() { if testcase.storageType == storageConstants.S3StorageDriverName { - driver := s3.New(store) - imgStore = imagestore.NewImageStore(testDir, tdir, false, false, log, metrics, + imgStore = imagestore.NewImageStore(testDir, cacheDir, false, false, log, metrics, &mocks.MockedLint{ LintFn: func(repo string, manifestDigest godigest.Digest, imageStore storageTypes.ImageStore) (bool, error) { //nolint: goerr113 return false, errors.New("linter error") }, - }, driver, nil, nil) + }, store, nil, nil) } else { - cacheDriver, _ := storage.Create("boltdb", cache.BoltDBDriverParameters{ - RootDir: tdir, - Name: "cache", - UseRelPaths: true, - }, log) - driver := local.New(true) - imgStore = imagestore.NewImageStore(tdir, tdir, true, - true, log, metrics, &mocks.MockedLint{ + var cacheDriver storageTypes.Cache + store, _, cacheDriver, _ = createObjectsStore(opts) + + imgStore = imagestore.NewImageStore(cacheDir, cacheDir, true, true, log, metrics, + &mocks.MockedLint{ LintFn: func(repo string, manifestDigest godigest.Digest, imageStore storageTypes.ImageStore) (bool, error) { //nolint: goerr113 return false, errors.New("linter error") }, - }, driver, cacheDriver, nil) + }, store, cacheDriver, nil) } _, _, err = imgStore.PutImageManifest("test", "1.0.0", ispec.MediaTypeImageManifest, manifestBuf) @@ -1143,12 +1211,22 @@ func TestDeleteBlobsInUse(t *testing.T) { t.Run(testcase.testCaseName, func(t *testing.T) { var imgStore storageTypes.ImageStore - var testDir, tdir string - - var store driver.StorageDriver - log := zlog.Logger{Logger: zerolog.New(os.Stdout)} - metrics := monitoring.NewMetricsServer(false, log) + + cacheDir := t.TempDir() + + opts := createObjectStoreOpts{ + rootDir: cacheDir, + cacheDir: cacheDir, + cacheType: testcase.cacheType, + storageType: testcase.storageType, + } + + if testcase.cacheType == storageConstants.RedisDriverName { + miniRedis := miniredis.RunT(t) + opts.miniRedisAddr = "redis://" + miniRedis.Addr() + defer DumpKeys(t, opts.miniRedisAddr) + } if testcase.storageType == storageConstants.S3StorageDriverName { tskip.SkipS3(t) @@ -1158,22 +1236,15 @@ func TestDeleteBlobsInUse(t *testing.T) { panic(err) } - testDir = path.Join("/oci-repo-test", uuid.String()) - tdir = t.TempDir() + testDir := path.Join("/oci-repo-test", uuid.String()) + opts.rootDir = testDir - store, imgStore, _ = createObjectsStore(testDir, tdir) + var store storageTypes.Driver + store, imgStore, _, _ = createObjectsStore(opts) defer cleanupStorage(store, testDir) } else { - tdir = t.TempDir() - cacheDriver, _ := storage.Create("boltdb", cache.BoltDBDriverParameters{ - RootDir: tdir, - Name: "cache", - UseRelPaths: true, - }, log) - driver := local.New(true) - imgStore = imagestore.NewImageStore(tdir, tdir, true, - true, log, metrics, nil, driver, cacheDriver, nil) + _, imgStore, _, _ = createObjectsStore(opts) } Convey("Setup manifest", t, func() { @@ -1447,16 +1518,25 @@ func TestReuploadCorruptedBlob(t *testing.T) { for _, testcase := range testCases { testcase := testcase t.Run(testcase.testCaseName, func(t *testing.T) { - var imgStore storageTypes.ImageStore + var ( + imgStore storageTypes.ImageStore + driver storageTypes.Driver + ) - var testDir, tdir string + cacheDir := t.TempDir() - var store driver.StorageDriver + opts := createObjectStoreOpts{ + rootDir: cacheDir, + cacheDir: cacheDir, + cacheType: testcase.cacheType, + storageType: testcase.storageType, + } - var driver storageTypes.Driver - - log := zlog.Logger{Logger: zerolog.New(os.Stdout)} - metrics := monitoring.NewMetricsServer(false, log) + if testcase.cacheType == storageConstants.RedisDriverName { + miniRedis := miniredis.RunT(t) + opts.miniRedisAddr = "redis://" + miniRedis.Addr() + defer DumpKeys(t, opts.miniRedisAddr) + } if testcase.storageType == storageConstants.S3StorageDriverName { tskip.SkipS3(t) @@ -1466,22 +1546,13 @@ func TestReuploadCorruptedBlob(t *testing.T) { panic(err) } - testDir = path.Join("/oci-repo-test", uuid.String()) - tdir = t.TempDir() + testDir := path.Join("/oci-repo-test", uuid.String()) + opts.rootDir = testDir - store, imgStore, _ = createObjectsStore(testDir, tdir) - driver = s3.New(store) - defer cleanupStorage(store, testDir) + driver, imgStore, _, _ = createObjectsStore(opts) + defer cleanupStorage(driver, testDir) } else { - tdir = t.TempDir() - cacheDriver, _ := storage.Create("boltdb", cache.BoltDBDriverParameters{ - RootDir: tdir, - Name: "cache", - UseRelPaths: true, - }, log) - driver = local.New(true) - imgStore = imagestore.NewImageStore(tdir, tdir, true, - true, log, metrics, nil, driver, cacheDriver, nil) + driver, imgStore, _, _ = createObjectsStore(opts) } Convey("Test errors paths", t, func() { @@ -1582,59 +1653,73 @@ func TestStorageHandler(t *testing.T) { for _, testcase := range testCases { testcase := testcase t.Run(testcase.testCaseName, func(t *testing.T) { - var firstStore storageTypes.ImageStore + var ( + firstStore storageTypes.ImageStore + secondStore storageTypes.ImageStore + thirdStore storageTypes.ImageStore + firstRootDir string + secondRootDir string + thirdRootDir string + ) - var secondStore storageTypes.ImageStore + opts := createObjectStoreOpts{ + cacheType: testcase.cacheType, + storageType: testcase.storageType, + } - var thirdStore storageTypes.ImageStore - - var firstRootDir string - - var secondRootDir string - - var thirdRootDir string + if testcase.cacheType == storageConstants.RedisDriverName { + miniRedis := miniredis.RunT(t) + opts.miniRedisAddr = "redis://" + miniRedis.Addr() + defer DumpKeys(t, opts.miniRedisAddr) + } if testcase.storageType == storageConstants.S3StorageDriverName { tskip.SkipS3(t) - var firstStorageDriver driver.StorageDriver - - var secondStorageDriver driver.StorageDriver - - var thirdStorageDriver driver.StorageDriver + var ( + firstStorageDriver storageTypes.Driver + secondStorageDriver storageTypes.Driver + thirdStorageDriver storageTypes.Driver + ) firstRootDir = "/util_test1" - firstStorageDriver, firstStore, _ = createObjectsStore(firstRootDir, t.TempDir()) + opts.rootDir = firstRootDir + opts.cacheDir = t.TempDir() + + firstStorageDriver, firstStore, _, _ = createObjectsStore(opts) defer cleanupStorage(firstStorageDriver, firstRootDir) secondRootDir = "/util_test2" - secondStorageDriver, secondStore, _ = createObjectsStore(secondRootDir, t.TempDir()) + opts.rootDir = secondRootDir + opts.cacheDir = t.TempDir() + + secondStorageDriver, secondStore, _, _ = createObjectsStore(opts) defer cleanupStorage(secondStorageDriver, secondRootDir) thirdRootDir = "/util_test3" - thirdStorageDriver, thirdStore, _ = createObjectsStore(thirdRootDir, t.TempDir()) + opts.rootDir = thirdRootDir + opts.cacheDir = t.TempDir() + + thirdStorageDriver, thirdStore, _, _ = createObjectsStore(opts) defer cleanupStorage(thirdStorageDriver, thirdRootDir) } else { - // Create temporary directory firstRootDir = t.TempDir() + opts.rootDir = firstRootDir + opts.cacheDir = firstRootDir + + _, firstStore, _, _ = createObjectsStore(opts) + secondRootDir = t.TempDir() + opts.rootDir = secondRootDir + opts.cacheDir = secondRootDir + + _, secondStore, _, _ = createObjectsStore(opts) + thirdRootDir = t.TempDir() + opts.rootDir = thirdRootDir + opts.cacheDir = thirdRootDir - log := zlog.NewLogger("debug", "") - - metrics := monitoring.NewMetricsServer(false, log) - - driver := local.New(true) - - // Create ImageStore - firstStore = imagestore.NewImageStore(firstRootDir, firstRootDir, false, false, - log, metrics, nil, driver, nil, nil) - - secondStore = imagestore.NewImageStore(secondRootDir, secondRootDir, false, false, - log, metrics, nil, driver, nil, nil) - - thirdStore = imagestore.NewImageStore(thirdRootDir, thirdRootDir, false, false, log, - metrics, nil, driver, nil, nil) + _, thirdStore, _, _ = createObjectsStore(opts) } Convey("Test storage handler", t, func() { @@ -1685,12 +1770,25 @@ func TestGarbageCollectImageManifest(t *testing.T) { log := zlog.NewLogger("debug", "") audit := zlog.NewAuditLogger("debug", "") - metrics := monitoring.NewMetricsServer(false, log) - ctx := context.Background() //nolint: contextcheck Convey("Repo layout", t, func(c C) { + cacheDir := t.TempDir() + + opts := createObjectStoreOpts{ + rootDir: cacheDir, + cacheDir: cacheDir, + cacheType: testcase.cacheType, + storageType: testcase.storageType, + } + + if testcase.cacheType == storageConstants.RedisDriverName { + miniRedis := miniredis.RunT(t) + opts.miniRedisAddr = "redis://" + miniRedis.Addr() + defer DumpKeys(t, opts.miniRedisAddr) + } + Convey("Garbage collect with default/long delay", func() { var imgStore storageTypes.ImageStore @@ -1703,23 +1801,13 @@ func TestGarbageCollectImageManifest(t *testing.T) { } testDir := path.Join("/oci-repo-test", uuid.String()) - tdir := t.TempDir() + opts.rootDir = testDir - var store driver.StorageDriver - store, imgStore, _ = createObjectsStore(testDir, tdir) + var store storageTypes.Driver + store, imgStore, _, _ = createObjectsStore(opts) defer cleanupStorage(store, testDir) } else { - dir := t.TempDir() - - cacheDriver, _ := storage.Create("boltdb", cache.BoltDBDriverParameters{ - RootDir: dir, - Name: "cache", - UseRelPaths: true, - }, log) - - driver := local.New(true) - - imgStore = imagestore.NewImageStore(dir, dir, true, true, log, metrics, nil, driver, cacheDriver, nil) + _, imgStore, _, _ = createObjectsStore(opts) } gc := gc.NewGarbageCollect(imgStore, mocks.MetaDBMock{}, gc.Options{ @@ -1874,24 +1962,13 @@ func TestGarbageCollectImageManifest(t *testing.T) { } testDir := path.Join("/oci-repo-test", uuid.String()) - tdir := t.TempDir() + opts.rootDir = testDir - var store driver.StorageDriver - store, imgStore, _ = createObjectsStore(testDir, tdir) + var store storageTypes.Driver + store, imgStore, _, _ = createObjectsStore(opts) defer cleanupStorage(store, testDir) } else { - dir := t.TempDir() - - cacheDriver, _ := storage.Create("boltdb", cache.BoltDBDriverParameters{ - RootDir: dir, - Name: "cache", - UseRelPaths: true, - }, log) - - driver := local.New(true) - - imgStore = imagestore.NewImageStore(dir, dir, true, - true, log, metrics, nil, driver, cacheDriver, nil) + _, imgStore, _, _ = createObjectsStore(opts) } gc := gc.NewGarbageCollect(imgStore, mocks.MetaDBMock{}, gc.Options{ @@ -2160,23 +2237,13 @@ func TestGarbageCollectImageManifest(t *testing.T) { } testDir := path.Join("/oci-repo-test", uuid.String()) - tdir := t.TempDir() + opts.rootDir = testDir - var store driver.StorageDriver - store, imgStore, _ = createObjectsStore(testDir, tdir) + var store storageTypes.Driver + store, imgStore, _, _ = createObjectsStore(opts) defer cleanupStorage(store, testDir) } else { - dir := t.TempDir() - - cacheDriver, _ := storage.Create("boltdb", cache.BoltDBDriverParameters{ - RootDir: dir, - Name: "cache", - UseRelPaths: true, - }, log) - - driver := local.New(true) - - imgStore = imagestore.NewImageStore(dir, dir, true, true, log, metrics, nil, driver, cacheDriver, nil) + _, imgStore, _, _ = createObjectsStore(opts) } gc := gc.NewGarbageCollect(imgStore, mocks.MetaDBMock{}, gc.Options{ @@ -2387,12 +2454,25 @@ func TestGarbageCollectImageIndex(t *testing.T) { log := zlog.NewLogger("debug", "") audit := zlog.NewAuditLogger("debug", "") - metrics := monitoring.NewMetricsServer(false, log) - ctx := context.Background() //nolint: contextcheck Convey("Repo layout", t, func(c C) { + cacheDir := t.TempDir() + + opts := createObjectStoreOpts{ + rootDir: cacheDir, + cacheDir: cacheDir, + cacheType: testcase.cacheType, + storageType: testcase.storageType, + } + + if testcase.cacheType == storageConstants.RedisDriverName { + miniRedis := miniredis.RunT(t) + opts.miniRedisAddr = "redis://" + miniRedis.Addr() + defer DumpKeys(t, opts.miniRedisAddr) + } + Convey("Garbage collect with default/long delay", func() { var imgStore storageTypes.ImageStore @@ -2405,23 +2485,13 @@ func TestGarbageCollectImageIndex(t *testing.T) { } testDir := path.Join("/oci-repo-test", uuid.String()) - tdir := t.TempDir() + opts.rootDir = testDir - var store driver.StorageDriver - store, imgStore, _ = createObjectsStore(testDir, tdir) + var store storageTypes.Driver + store, imgStore, _, _ = createObjectsStore(opts) defer cleanupStorage(store, testDir) } else { - dir := t.TempDir() - - cacheDriver, _ := storage.Create("boltdb", cache.BoltDBDriverParameters{ - RootDir: dir, - Name: "cache", - UseRelPaths: true, - }, log) - - driver := local.New(true) - - imgStore = imagestore.NewImageStore(dir, dir, true, true, log, metrics, nil, driver, cacheDriver, nil) + _, imgStore, _, _ = createObjectsStore(opts) } gc := gc.NewGarbageCollect(imgStore, mocks.MetaDBMock{}, gc.Options{ @@ -2534,23 +2604,13 @@ func TestGarbageCollectImageIndex(t *testing.T) { } testDir := path.Join("/oci-repo-test", uuid.String()) - tdir := t.TempDir() + opts.rootDir = testDir - var store driver.StorageDriver - store, imgStore, _ = createObjectsStore(testDir, tdir) + var store storageTypes.Driver + store, imgStore, _, _ = createObjectsStore(opts) defer cleanupStorage(store, testDir) } else { - dir := t.TempDir() - - cacheDriver, _ := storage.Create("boltdb", cache.BoltDBDriverParameters{ - RootDir: dir, - Name: "cache", - UseRelPaths: true, - }, log) - - driver := local.New(true) - - imgStore = imagestore.NewImageStore(dir, dir, true, true, log, metrics, nil, driver, cacheDriver, nil) + _, imgStore, _, _ = createObjectsStore(opts) } gc := gc.NewGarbageCollect(imgStore, mocks.MetaDBMock{}, gc.Options{ @@ -2802,8 +2862,6 @@ func TestGarbageCollectChainedImageIndexes(t *testing.T) { log := zlog.NewLogger("debug", "") audit := zlog.NewAuditLogger("debug", "") - metrics := monitoring.NewMetricsServer(false, log) - ctx := context.Background() //nolint: contextcheck @@ -2813,6 +2871,21 @@ func TestGarbageCollectChainedImageIndexes(t *testing.T) { gcDelay := 5 * time.Second imageRetentionDelay := 5 * time.Second + cacheDir := t.TempDir() + + opts := createObjectStoreOpts{ + rootDir: cacheDir, + cacheDir: cacheDir, + cacheType: testcase.cacheType, + storageType: testcase.storageType, + } + + if testcase.cacheType == storageConstants.RedisDriverName { + miniRedis := miniredis.RunT(t) + opts.miniRedisAddr = "redis://" + miniRedis.Addr() + defer DumpKeys(t, opts.miniRedisAddr) + } + if testcase.storageType == storageConstants.S3StorageDriverName { tskip.SkipS3(t) @@ -2822,23 +2895,13 @@ func TestGarbageCollectChainedImageIndexes(t *testing.T) { } testDir := path.Join("/oci-repo-test", uuid.String()) - tdir := t.TempDir() + opts.rootDir = testDir - var store driver.StorageDriver - store, imgStore, _ = createObjectsStore(testDir, tdir) + var store storageTypes.Driver + store, imgStore, _, _ = createObjectsStore(opts) defer cleanupStorage(store, testDir) } else { - dir := t.TempDir() - - cacheDriver, _ := storage.Create("boltdb", cache.BoltDBDriverParameters{ - RootDir: dir, - Name: "cache", - UseRelPaths: true, - }, log) - - driver := local.New(true) - - imgStore = imagestore.NewImageStore(dir, dir, true, true, log, metrics, nil, driver, cacheDriver, nil) + _, imgStore, _, _ = createObjectsStore(opts) } gc := gc.NewGarbageCollect(imgStore, mocks.MetaDBMock{}, gc.Options{ @@ -3347,3 +3410,70 @@ func pushRandomImageIndex(imgStore storageTypes.ImageStore, repoName string, return bdgst, digest, indexDigest, int64(len(indexContent)) } + +func DumpKeys(t *testing.T, redisURL string) { + t.Helper() + + // Initialize redis client + connOpts, err := redis.ParseURL(redisURL) + if err != nil { + return + } + + client := redis.NewClient(connOpts) + + // Retrieve all keys + keys, err := client.Keys(context.Background(), "*").Result() + if err != nil { + t.Log("Error retrieving keys:", err) + + return + } + + // Print the keys + t.Log("Keys in Redis:") + + for _, key := range keys { + keyType, err := client.Type(context.Background(), key).Result() + if err != nil { + t.Logf("Error retrieving type for key %s: %v\n", key, err) + + continue + } + + var value string + + switch keyType { + case "string": + value, err = client.Get(context.Background(), key).Result() + case "list": + values, err := client.LRange(context.Background(), key, 0, -1).Result() + if err == nil { + value = fmt.Sprintf("%v", values) + } + case "hash": + values, err := client.HGetAll(context.Background(), key).Result() + if err == nil { + value = fmt.Sprintf("%v", values) + } + case "set": + values, err := client.SMembers(context.Background(), key).Result() + if err == nil { + value = fmt.Sprintf("%v", values) + } + case "zset": + values, err := client.ZRange(context.Background(), key, 0, -1).Result() + if err == nil { + value = fmt.Sprintf("%v", values) + } + default: + value = "Unsupported type" + } + + if err != nil { + t.Logf("Error retrieving value for key %s: %v\n", key, err) + } else { + t.Logf("Key: %s, Type: %s, Value: %s\n", key, keyType, value) + } + } +} diff --git a/pkg/storage/cache/cacheinterface.go b/pkg/storage/types/cache.go similarity index 91% rename from pkg/storage/cache/cacheinterface.go rename to pkg/storage/types/cache.go index 58044f6e..35bd6703 100644 --- a/pkg/storage/cache/cacheinterface.go +++ b/pkg/storage/types/cache.go @@ -1,4 +1,4 @@ -package cache +package types import ( godigest "github.com/opencontainers/go-digest" @@ -11,6 +11,7 @@ type Cache interface { // Retrieves the blob matching provided digest. GetBlob(digest godigest.Digest) (string, error) + // Retrieves all blobs matching provided digest. GetAllBlobs(digest godigest.Digest) ([]string, error) // Uploads blob to cachedb. diff --git a/test/blackbox/ci.sh b/test/blackbox/ci.sh index 3820d195..ae97a81c 100755 --- a/test/blackbox/ci.sh +++ b/test/blackbox/ci.sh @@ -9,7 +9,7 @@ PATH=$PATH:${SCRIPTPATH}/../../hack/tools/bin tests=("pushpull" "pushpull_authn" "delete_images" "referrers" "metadata" "anonymous_policy" "annotations" "detect_manifest_collision" "cve" "sync" "sync_docker" "sync_replica_cluster" - "scrub" "garbage_collect" "metrics" "metrics_minimal" "multiarch_index" "docker_compat") + "scrub" "garbage_collect" "metrics" "metrics_minimal" "multiarch_index" "docker_compat" "redis_local") for test in ${tests[*]}; do ${BATS} ${BATS_FLAGS} ${SCRIPTPATH}/${test}.bats > ${test}.log & pids+=($!) diff --git a/test/blackbox/helpers_redis.bash b/test/blackbox/helpers_redis.bash new file mode 100644 index 00000000..460c85ba --- /dev/null +++ b/test/blackbox/helpers_redis.bash @@ -0,0 +1,12 @@ + +function redis_start() { + local cname="$1" # container name + local free_port="$2" + docker run -d --name ${cname} -p ${free_port}:6379 redis +} + +function redis_stop() { + local cname="$1" + docker stop ${cname} + docker rm -f ${cname} +} diff --git a/test/blackbox/redis_local.bats b/test/blackbox/redis_local.bats new file mode 100644 index 00000000..8f619c9a --- /dev/null +++ b/test/blackbox/redis_local.bats @@ -0,0 +1,124 @@ +# Note: Intended to be run as "make run-blackbox-tests" or "make run-blackbox-ci" +# Makefile target installs & checks all necessary tooling +# Extra tools that are not covered in Makefile target needs to be added in verify_prerequisites() + +load helpers_zot +load helpers_redis + +function verify_prerequisites() { + if [ ! $(command -v curl) ]; then + echo "you need to install curl as a prerequisite to running the tests" >&3 + return 1 + fi + + if [ ! $(command -v jq) ]; then + echo "you need to install jq as a prerequisite to running the tests" >&3 + return 1 + fi + + if [ ! $(command -v docker) ]; then + echo "you need to install docker as a prerequisite to running the tests" >&3 + return 1 + fi + + return 0 +} + +function setup_file() { + # Verify prerequisites are available + if ! $(verify_prerequisites); then + exit 1 + fi + + # Download test data to folder common for the entire suite, not just this file + skopeo --insecure-policy copy --format=oci docker://ghcr.io/project-zot/golang:1.20 oci:${TEST_DATA_DIR}/golang:1.20 + + # Setup redis server + redis_port=$(get_free_port) + redis_start redis_server_local ${redis_port} + + # Setup zot server + local zot_root_dir=${BATS_FILE_TMPDIR}/zot + local zot_sync_ondemand_config_file=${BATS_FILE_TMPDIR}/zot_sync_ondemand_config.json + zot_port=$(get_free_port) + echo ${zot_port} > ${BATS_FILE_TMPDIR}/zot.port + + mkdir -p ${zot_root_dir} + + cat >${zot_sync_ondemand_config_file} <&3 + return 1 + fi + + return 0 +} + +function setup_file() { + # Verify prerequisites are available + if ! $(verify_prerequisites); then + exit 1 + fi + + # Download test data to folder common for the entire suite, not just this file + skopeo --insecure-policy copy --format=oci docker://ghcr.io/project-zot/test-images/alpine:3.17.3 oci:${TEST_DATA_DIR}/alpine:1 + + # Setup redis server + redis_port=$(get_free_port) + redis_start redis_server ${redis_port} + + # Setup zot server + local zot_root_dir=${BATS_FILE_TMPDIR}/zot + local zot_sync_ondemand_config_file=${BATS_FILE_TMPDIR}/zot_sync_ondemand_config.json + zot_port=$(get_free_port) + echo ${zot_port} > ${BATS_FILE_TMPDIR}/zot.port + + mkdir -p ${zot_root_dir} + + cat >${zot_sync_ondemand_config_file} <