refactor(storage): refactor storage into a single ImageStore (#1656)

unified both local and s3 ImageStore logic into a single ImageStore
added a new driver interface for common file/dirs manipulations
to be implemented by different storage types

refactor(gc): drop umoci dependency, implemented internal gc

added retentionDelay config option that specifies
the garbage collect delay for images without tags

this will also clean manifests which are part of an index image
(multiarch) that no longer exist.

fix(dedupe): skip blobs under .sync/ directory

if startup dedupe is running while also syncing is running
ignore blobs under sync's temporary storage

fix(storage): do not allow image indexes modifications

when deleting a manifest verify that it is not part of a multiarch image
and throw a MethodNotAllowed error to the client if it is.
we don't want to modify multiarch images

Signed-off-by: Petu Eusebiu <peusebiu@cisco.com>

.github/workflows/ecosystem-tools.yaml

@@ -17,6 +17,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
+      - uses: ./.github/actions/clean-runner
       - uses: actions/setup-go@v4
         with:
           cache: false
@@ -98,6 +99,9 @@ jobs:
       - name: Run annotations tests
         run: |
             make test-annotations
+      - name: Run garbage collect tests
+        run: |
+            make test-garbage-collect
       - name: Install localstack
         run: |
           pip install --upgrade pyopenssl
@@ -129,4 +133,4 @@ jobs:
           sudo du -sh /var/
           sudo du -sh /var/lib/docker/
           du -sh /home/runner/work/
-          set +x
+          set +x

.github/workflows/gc-stress-test.yaml

@@ -12,8 +12,8 @@ on:
 permissions: read-all
 
 jobs:
-  client-tools:
-    name: GC with short interval
+  gc-referrers-stress-local:
+    name: GC(with referrers) on filesystem with short interval
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
@@ -27,7 +27,31 @@ jobs:
         run: |
             make binary
             make bench
-            ./bin/zot-linux-amd64 serve examples/config-gc-bench.json &
+            ./bin/zot-linux-amd64 serve test/gc-stress/config-gc-referrers-bench-local.json &
+            sleep 10
+            bin/zb-linux-amd64 -c 10 -n 100 -o ci-cd http://localhost:8080
+
+            killall -r zot-*
+
+            # clean zot storage
+            sudo rm -rf /tmp/zot
+
+  gc-stress-local:
+    name: GC(without referrers) on filesystem with short interval
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: ./.github/actions/clean-runner
+      - uses: actions/setup-go@v4
+        with:
+          cache: false
+          go-version: 1.20.x
+
+      - name: Run zb
+        run: |
+            make binary
+            make bench
+            ./bin/zot-linux-amd64 serve test/gc-stress/config-gc-bench-local.json &
             sleep 10
             bin/zb-linux-amd64 -c 10 -n 100 -o ci-cd http://localhost:8080
 

.github/workflows/nightly.yaml

@@ -74,3 +74,84 @@ jobs:
       - name: Run sync harness
         run: |
             make test-sync-harness
+  gc-referrers-stress-s3:
+    name: GC(with referrers) on S3 with short interval
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: ./.github/actions/clean-runner
+      - uses: actions/setup-go@v4
+        with:
+          cache: false
+          go-version: 1.20.x
+      - name: Setup localstack service
+        run: |
+            pip install localstack                    # Install LocalStack cli
+            docker pull localstack/localstack:1.3     # Make sure to pull the latest version of the image
+            localstack start -d                       # Start LocalStack in the background
+            
+            echo "Waiting for LocalStack startup..."  # Wait 30 seconds for the LocalStack container
+            localstack wait -t 30                     # to become ready before timing out 
+            echo "Startup complete"
+            
+            aws --endpoint-url=http://localhost:4566 s3api create-bucket --bucket zot-storage --region us-east-2 --create-bucket-configuration="{\"LocationConstraint\": \"us-east-2\"}"
+            aws dynamodb --endpoint-url http://localhost:4566 --region "us-east-2" create-table --table-name BlobTable --attribute-definitions AttributeName=Digest,AttributeType=S --key-schema AttributeName=Digest,KeyType=HASH --provisioned-throughput ReadCapacityUnits=10,WriteCapacityUnits=5
+        env:
+          AWS_ACCESS_KEY_ID: fake
+          AWS_SECRET_ACCESS_KEY: fake
+      - name: Run zb
+        run: |
+            make binary
+            make bench
+            ./bin/zot-linux-amd64 serve test/gc-stress/config-gc-referrers-bench-s3.json &
+            sleep 10
+            bin/zb-linux-amd64 -c 10 -n 100 -o ci-cd http://localhost:8080
+
+            killall -r zot-*
+
+            # clean zot storage
+            sudo rm -rf /tmp/zot
+        env:
+          AWS_ACCESS_KEY_ID: fake
+          AWS_SECRET_ACCESS_KEY: fake
+
+  gc-stress-s3:
+    name: GC(without referrers) on S3 with short interval
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: ./.github/actions/clean-runner
+      - uses: actions/setup-go@v4
+        with:
+          cache: false
+          go-version: 1.20.x
+      - name: Setup localstack service
+        run: |
+            pip install localstack                    # Install LocalStack cli
+            docker pull localstack/localstack:1.3     # Make sure to pull the latest version of the image
+            localstack start -d                       # Start LocalStack in the background
+            
+            echo "Waiting for LocalStack startup..."  # Wait 30 seconds for the LocalStack container
+            localstack wait -t 30                     # to become ready before timing out 
+            echo "Startup complete"
+            
+            aws --endpoint-url=http://localhost:4566 s3api create-bucket --bucket zot-storage --region us-east-2 --create-bucket-configuration="{\"LocationConstraint\": \"us-east-2\"}"
+            aws dynamodb --endpoint-url http://localhost:4566 --region "us-east-2" create-table --table-name BlobTable --attribute-definitions AttributeName=Digest,AttributeType=S --key-schema AttributeName=Digest,KeyType=HASH --provisioned-throughput ReadCapacityUnits=10,WriteCapacityUnits=5
+        env:
+          AWS_ACCESS_KEY_ID: fake
+          AWS_SECRET_ACCESS_KEY: fake
+      - name: Run zb
+        run: |
+            make binary
+            make bench
+            ./bin/zot-linux-amd64 serve test/gc-stress/config-gc-bench-s3.json &
+            sleep 10
+            bin/zb-linux-amd64 -c 10 -n 100 -o ci-cd http://localhost:8080
+
+            killall -r zot-*
+
+            # clean zot storage
+            sudo rm -rf /tmp/zot
+        env:
+          AWS_ACCESS_KEY_ID: fake
+          AWS_SECRET_ACCESS_KEY: fake