mirror of
https://github.com/project-zot/zot.git
synced 2026-06-16 04:17:55 +08:00
peusebiu
b80deb9927
unified both local and s3 ImageStore logic into a single ImageStore added a new driver interface for common file/dirs manipulations to be implemented by different storage types refactor(gc): drop umoci dependency, implemented internal gc added retentionDelay config option that specifies the garbage collect delay for images without tags this will also clean manifests which are part of an index image (multiarch) that no longer exist. fix(dedupe): skip blobs under .sync/ directory if startup dedupe is running while also syncing is running ignore blobs under sync's temporary storage fix(storage): do not allow image indexes modifications when deleting a manifest verify that it is not part of a multiarch image and throw a MethodNotAllowed error to the client if it is. we don't want to modify multiarch images Signed-off-by: Petu Eusebiu <peusebiu@cisco.com>
158 lines
6.5 KiB
YAML
158 lines
6.5 KiB
YAML
name: 'Nightly jobs'
|
|
on:
|
|
schedule:
|
|
- cron: '30 1 * * *'
|
|
workflow_dispatch:
|
|
|
|
permissions: read-all
|
|
|
|
# Here we are running two tests:
|
|
# 1. run zot with local storage and dedupe disabled, push images, restart zot with dedupe enabled
|
|
# task scheduler will start a dedupe all blobs process at zot startup and it shouldn't interfere with clients.
|
|
# 2. run zot with s3 storage and dynamodb and dedupe enabled, push images, restart zot with dedupe false and no cache
|
|
# task scheduler will start a restore all blobs process at zot startup, after it finishes all blobs should be restored to their original state (have content)
|
|
jobs:
|
|
dedupe:
|
|
name: Dedupe/restore blobs
|
|
runs-on: ubuntu-latest
|
|
steps:
|
|
- uses: actions/checkout@v3
|
|
- uses: ./.github/actions/clean-runner
|
|
- uses: actions/setup-go@v3
|
|
with:
|
|
go-version: 1.20.x
|
|
- name: Install dependencies
|
|
run: |
|
|
cd $GITHUB_WORKSPACE
|
|
go install github.com/swaggo/swag/cmd/swag@v1.8.12
|
|
go mod download
|
|
sudo apt-get update
|
|
sudo apt-get install libgpgme-dev libassuan-dev libbtrfs-dev libdevmapper-dev pkg-config rpm uidmap
|
|
# install skopeo
|
|
git clone -b v1.12.0 https://github.com/containers/skopeo.git
|
|
cd skopeo
|
|
make bin/skopeo
|
|
sudo cp bin/skopeo /usr/bin
|
|
skopeo -v
|
|
- name: Install localstack
|
|
run: |
|
|
pip install --upgrade pyopenssl
|
|
pip install localstack awscli-local[ver1] # install LocalStack cli and awslocal
|
|
docker pull localstack/localstack # Make sure to pull the latest version of the image
|
|
localstack start -d # Start LocalStack in the background
|
|
|
|
echo "Waiting for LocalStack startup..." # Wait 30 seconds for the LocalStack container
|
|
localstack wait -t 30 # to become ready before timing out
|
|
echo "Startup complete"
|
|
- name: Run restore s3 blobs after cache is deleted
|
|
run: |
|
|
make test-restore-s3-blobs
|
|
env:
|
|
AWS_ACCESS_KEY_ID: fake
|
|
AWS_SECRET_ACCESS_KEY: fake
|
|
- name: Run dedupe filesystem blobs after switching dedupe to enable.
|
|
run: |
|
|
make test-push-pull-running-dedupe
|
|
env:
|
|
AWS_ACCESS_KEY_ID: fake
|
|
AWS_SECRET_ACCESS_KEY: fake
|
|
sync:
|
|
name: Sync harness
|
|
runs-on: ubuntu-latest
|
|
steps:
|
|
- name: Check out source code
|
|
uses: actions/checkout@v3
|
|
- uses: ./.github/actions/clean-runner
|
|
- uses: actions/setup-go@v3
|
|
with:
|
|
go-version: 1.20.x
|
|
- name: Install dependencies
|
|
run: |
|
|
cd $GITHUB_WORKSPACE
|
|
go install github.com/swaggo/swag/cmd/swag@v1.8.12
|
|
go mod download
|
|
- name: Run sync harness
|
|
run: |
|
|
make test-sync-harness
|
|
gc-referrers-stress-s3:
|
|
name: GC(with referrers) on S3 with short interval
|
|
runs-on: ubuntu-latest
|
|
steps:
|
|
- uses: actions/checkout@v3
|
|
- uses: ./.github/actions/clean-runner
|
|
- uses: actions/setup-go@v4
|
|
with:
|
|
cache: false
|
|
go-version: 1.20.x
|
|
- name: Setup localstack service
|
|
run: |
|
|
pip install localstack # Install LocalStack cli
|
|
docker pull localstack/localstack:1.3 # Make sure to pull the latest version of the image
|
|
localstack start -d # Start LocalStack in the background
|
|
|
|
echo "Waiting for LocalStack startup..." # Wait 30 seconds for the LocalStack container
|
|
localstack wait -t 30 # to become ready before timing out
|
|
echo "Startup complete"
|
|
|
|
aws --endpoint-url=http://localhost:4566 s3api create-bucket --bucket zot-storage --region us-east-2 --create-bucket-configuration="{\"LocationConstraint\": \"us-east-2\"}"
|
|
aws dynamodb --endpoint-url http://localhost:4566 --region "us-east-2" create-table --table-name BlobTable --attribute-definitions AttributeName=Digest,AttributeType=S --key-schema AttributeName=Digest,KeyType=HASH --provisioned-throughput ReadCapacityUnits=10,WriteCapacityUnits=5
|
|
env:
|
|
AWS_ACCESS_KEY_ID: fake
|
|
AWS_SECRET_ACCESS_KEY: fake
|
|
- name: Run zb
|
|
run: |
|
|
make binary
|
|
make bench
|
|
./bin/zot-linux-amd64 serve test/gc-stress/config-gc-referrers-bench-s3.json &
|
|
sleep 10
|
|
bin/zb-linux-amd64 -c 10 -n 100 -o ci-cd http://localhost:8080
|
|
|
|
killall -r zot-*
|
|
|
|
# clean zot storage
|
|
sudo rm -rf /tmp/zot
|
|
env:
|
|
AWS_ACCESS_KEY_ID: fake
|
|
AWS_SECRET_ACCESS_KEY: fake
|
|
|
|
gc-stress-s3:
|
|
name: GC(without referrers) on S3 with short interval
|
|
runs-on: ubuntu-latest
|
|
steps:
|
|
- uses: actions/checkout@v3
|
|
- uses: ./.github/actions/clean-runner
|
|
- uses: actions/setup-go@v4
|
|
with:
|
|
cache: false
|
|
go-version: 1.20.x
|
|
- name: Setup localstack service
|
|
run: |
|
|
pip install localstack # Install LocalStack cli
|
|
docker pull localstack/localstack:1.3 # Make sure to pull the latest version of the image
|
|
localstack start -d # Start LocalStack in the background
|
|
|
|
echo "Waiting for LocalStack startup..." # Wait 30 seconds for the LocalStack container
|
|
localstack wait -t 30 # to become ready before timing out
|
|
echo "Startup complete"
|
|
|
|
aws --endpoint-url=http://localhost:4566 s3api create-bucket --bucket zot-storage --region us-east-2 --create-bucket-configuration="{\"LocationConstraint\": \"us-east-2\"}"
|
|
aws dynamodb --endpoint-url http://localhost:4566 --region "us-east-2" create-table --table-name BlobTable --attribute-definitions AttributeName=Digest,AttributeType=S --key-schema AttributeName=Digest,KeyType=HASH --provisioned-throughput ReadCapacityUnits=10,WriteCapacityUnits=5
|
|
env:
|
|
AWS_ACCESS_KEY_ID: fake
|
|
AWS_SECRET_ACCESS_KEY: fake
|
|
- name: Run zb
|
|
run: |
|
|
make binary
|
|
make bench
|
|
./bin/zot-linux-amd64 serve test/gc-stress/config-gc-bench-s3.json &
|
|
sleep 10
|
|
bin/zb-linux-amd64 -c 10 -n 100 -o ci-cd http://localhost:8080
|
|
|
|
killall -r zot-*
|
|
|
|
# clean zot storage
|
|
sudo rm -rf /tmp/zot
|
|
env:
|
|
AWS_ACCESS_KEY_ID: fake
|
|
AWS_SECRET_ACCESS_KEY: fake
|