refactor(storage): refactor storage into a single ImageStore (#1656)

unified both local and s3 ImageStore logic into a single ImageStore
added a new driver interface for common file/dirs manipulations
to be implemented by different storage types

refactor(gc): drop umoci dependency, implemented internal gc

added retentionDelay config option that specifies
the garbage collect delay for images without tags

this will also clean manifests which are part of an index image
(multiarch) that no longer exist.

fix(dedupe): skip blobs under .sync/ directory

if startup dedupe is running while also syncing is running
ignore blobs under sync's temporary storage

fix(storage): do not allow image indexes modifications

when deleting a manifest verify that it is not part of a multiarch image
and throw a MethodNotAllowed error to the client if it is.
we don't want to modify multiarch images

Signed-off-by: Petu Eusebiu <peusebiu@cisco.com>

pkg/storage/cache/boltdb.go

@@ -77,6 +77,10 @@ func NewBoltDBCache(parameters interface{}, log zlog.Logger) Cache {
 	}
 }
 
+func (d *BoltDBDriver) UsesRelativePaths() bool {
+	return d.useRelPaths
+}
+
 func (d *BoltDBDriver) Name() string {
 	return "boltdb"
 }

pkg/storage/cache/cacheinterface.go

@@ -19,4 +19,7 @@ type Cache interface {
 
 	// Delete a blob from the cachedb.
 	DeleteBlob(digest godigest.Digest, path string) error
+
+	// UsesRelativePaths returns if cache is storing blobs relative to cache rootDir
+	UsesRelativePaths() bool
 }

pkg/storage/cache/dynamodb.go

@@ -99,6 +99,10 @@ func NewDynamoDBCache(parameters interface{}, log zlog.Logger) Cache {
 	return driver
 }
 
+func (d *DynamoDBDriver) UsesRelativePaths() bool {
+	return false
+}
+
 func (d *DynamoDBDriver) Name() string {
 	return "dynamodb"
 }

pkg/storage/cache_test.go

@@ -35,6 +35,8 @@ func TestCache(t *testing.T) {
 		}, log)
 		So(cacheDriver, ShouldNotBeNil)
 
+		So(cacheDriver.UsesRelativePaths(), ShouldBeTrue)
+
 		name := cacheDriver.Name()
 		So(name, ShouldEqual, "boltdb")
 

pkg/storage/common/common.go

@@ -5,22 +5,22 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
-	"io"
 	"math/rand"
 	"path"
 	"strings"
 	"time"
 
+	"github.com/docker/distribution/registry/storage/driver"
 	notreg "github.com/notaryproject/notation-go/registry"
 	godigest "github.com/opencontainers/go-digest"
 	"github.com/opencontainers/image-spec/schema"
 	imeta "github.com/opencontainers/image-spec/specs-go"
 	ispec "github.com/opencontainers/image-spec/specs-go/v1"
 	oras "github.com/oras-project/artifacts-spec/specs-go/v1"
-	"github.com/rs/zerolog"
 
 	zerr "zotregistry.io/zot/errors"
 	zcommon "zotregistry.io/zot/pkg/common"
+	zlog "zotregistry.io/zot/pkg/log"
 	"zotregistry.io/zot/pkg/scheduler"
 	storageConstants "zotregistry.io/zot/pkg/storage/constants"
 	storageTypes "zotregistry.io/zot/pkg/storage/types"
@@ -62,7 +62,7 @@ func GetManifestDescByReference(index ispec.Index, reference string) (ispec.Desc
 }
 
 func ValidateManifest(imgStore storageTypes.ImageStore, repo, reference, mediaType string, body []byte,
-	log zerolog.Logger,
+	log zlog.Logger,
 ) (godigest.Digest, error) {
 	// validate the manifest
 	if !IsSupportedMediaType(mediaType) {
@@ -105,7 +105,7 @@ func ValidateManifest(imgStore storageTypes.ImageStore, repo, reference, mediaTy
 					continue
 				}
 
-				ok, _, err := imgStore.StatBlob(repo, layer.Digest)
+				ok, _, _, err := imgStore.StatBlob(repo, layer.Digest)
 				if !ok || err != nil {
 					log.Error().Err(err).Str("digest", layer.Digest.String()).Msg("missing layer blob")
 
@@ -136,7 +136,7 @@ func ValidateManifest(imgStore storageTypes.ImageStore, repo, reference, mediaTy
 		}
 
 		for _, manifest := range indexManifest.Manifests {
-			if ok, _, err := imgStore.StatBlob(repo, manifest.Digest); !ok || err != nil {
+			if ok, _, _, err := imgStore.StatBlob(repo, manifest.Digest); !ok || err != nil {
 				log.Error().Err(err).Str("digest", manifest.Digest.String()).Msg("missing manifest blob")
 
 				return "", zerr.ErrBadManifest
@@ -147,7 +147,7 @@ func ValidateManifest(imgStore storageTypes.ImageStore, repo, reference, mediaTy
 	return "", nil
 }
 
-func GetAndValidateRequestDigest(body []byte, digestStr string, log zerolog.Logger) (godigest.Digest, error) {
+func GetAndValidateRequestDigest(body []byte, digestStr string, log zlog.Logger) (godigest.Digest, error) {
 	bodyDigest := godigest.FromBytes(body)
 
 	d, err := godigest.Parse(digestStr)
@@ -169,7 +169,7 @@ CheckIfIndexNeedsUpdate verifies if an index needs to be updated given a new man
 Returns whether or not index needs update, in the latter case it will also return the previous digest.
 */
 func CheckIfIndexNeedsUpdate(index *ispec.Index, desc *ispec.Descriptor,
-	log zerolog.Logger,
+	log zlog.Logger,
 ) (bool, godigest.Digest, error) {
 	var oldDgst godigest.Digest
 
@@ -242,11 +242,15 @@ func CheckIfIndexNeedsUpdate(index *ispec.Index, desc *ispec.Descriptor,
 }
 
 // GetIndex returns the contents of index.json.
-func GetIndex(imgStore storageTypes.ImageStore, repo string, log zerolog.Logger) (ispec.Index, error) {
+func GetIndex(imgStore storageTypes.ImageStore, repo string, log zlog.Logger) (ispec.Index, error) {
 	var index ispec.Index
 
 	buf, err := imgStore.GetIndexContent(repo)
 	if err != nil {
+		if errors.As(err, &driver.PathNotFoundError{}) {
+			return index, zerr.ErrRepoNotFound
+		}
+
 		return index, err
 	}
 
@@ -260,7 +264,7 @@ func GetIndex(imgStore storageTypes.ImageStore, repo string, log zerolog.Logger)
 }
 
 // GetImageIndex returns a multiarch type image.
-func GetImageIndex(imgStore storageTypes.ImageStore, repo string, digest godigest.Digest, log zerolog.Logger,
+func GetImageIndex(imgStore storageTypes.ImageStore, repo string, digest godigest.Digest, log zlog.Logger,
 ) (ispec.Index, error) {
 	var imageIndex ispec.Index
 
@@ -285,7 +289,7 @@ func GetImageIndex(imgStore storageTypes.ImageStore, repo string, digest godiges
 	return imageIndex, nil
 }
 
-func GetImageManifest(imgStore storageTypes.ImageStore, repo string, digest godigest.Digest, log zerolog.Logger,
+func GetImageManifest(imgStore storageTypes.ImageStore, repo string, digest godigest.Digest, log zlog.Logger,
 ) (ispec.Manifest, error) {
 	var manifestContent ispec.Manifest
 
@@ -352,7 +356,7 @@ index, ensure that they do not have a name or they are not in other
 manifest indexes else GC can never clean them.
 */
 func UpdateIndexWithPrunedImageManifests(imgStore storageTypes.ImageStore, index *ispec.Index, repo string,
-	desc ispec.Descriptor, oldDgst godigest.Digest, log zerolog.Logger,
+	desc ispec.Descriptor, oldDgst godigest.Digest, log zlog.Logger,
 ) error {
 	if (desc.MediaType == ispec.MediaTypeImageIndex) && (oldDgst != "") {
 		otherImgIndexes := []ispec.Descriptor{}
@@ -385,7 +389,7 @@ same constitutent manifests so that they can be garbage-collected correctly
 PruneImageManifestsFromIndex is a helper routine to achieve this.
 */
 func PruneImageManifestsFromIndex(imgStore storageTypes.ImageStore, repo string, digest godigest.Digest, //nolint:gocyclo,lll
-	outIndex ispec.Index, otherImgIndexes []ispec.Descriptor, log zerolog.Logger,
+	outIndex ispec.Index, otherImgIndexes []ispec.Descriptor, log zlog.Logger,
 ) ([]ispec.Descriptor, error) {
 	dir := path.Join(imgStore.RootDir(), repo)
 
@@ -459,8 +463,8 @@ func PruneImageManifestsFromIndex(imgStore storageTypes.ImageStore, repo string,
 	return prunedManifests, nil
 }
 
-func isBlobReferencedInManifest(imgStore storageTypes.ImageStore, repo string,
-	bdigest, mdigest godigest.Digest, log zerolog.Logger,
+func isBlobReferencedInImageManifest(imgStore storageTypes.ImageStore, repo string,
+	bdigest, mdigest godigest.Digest, log zlog.Logger,
 ) (bool, error) {
 	if bdigest == mdigest {
 		return true, nil
@@ -487,16 +491,14 @@ func isBlobReferencedInManifest(imgStore storageTypes.ImageStore, repo string,
 	return false, nil
 }
 
-func isBlobReferencedInImageIndex(imgStore storageTypes.ImageStore, repo string,
-	digest godigest.Digest, index ispec.Index, log zerolog.Logger,
+func IsBlobReferencedInImageIndex(imgStore storageTypes.ImageStore, repo string,
+	digest godigest.Digest, index ispec.Index, log zlog.Logger,
 ) (bool, error) {
 	for _, desc := range index.Manifests {
 		var found bool
 
 		switch desc.MediaType {
 		case ispec.MediaTypeImageIndex:
-			/* this branch is not needed, because every manifests in index is already checked
-			when this one is hit, all manifests are referenced in index.json */
 			indexImage, err := GetImageIndex(imgStore, repo, desc.Digest, log)
 			if err != nil {
 				log.Error().Err(err).Str("repository", repo).Str("digest", desc.Digest.String()).
@@ -505,9 +507,9 @@ func isBlobReferencedInImageIndex(imgStore storageTypes.ImageStore, repo string,
 				return false, err
 			}
 
-			found, _ = isBlobReferencedInImageIndex(imgStore, repo, digest, indexImage, log)
+			found, _ = IsBlobReferencedInImageIndex(imgStore, repo, digest, indexImage, log)
 		case ispec.MediaTypeImageManifest:
-			found, _ = isBlobReferencedInManifest(imgStore, repo, digest, desc.Digest, log)
+			found, _ = isBlobReferencedInImageManifest(imgStore, repo, digest, desc.Digest, log)
 		}
 
 		if found {
@@ -519,7 +521,7 @@ func isBlobReferencedInImageIndex(imgStore storageTypes.ImageStore, repo string,
 }
 
 func IsBlobReferenced(imgStore storageTypes.ImageStore, repo string,
-	digest godigest.Digest, log zerolog.Logger,
+	digest godigest.Digest, log zlog.Logger,
 ) (bool, error) {
 	dir := path.Join(imgStore.RootDir(), repo)
 	if !imgStore.DirExists(dir) {
@@ -531,7 +533,133 @@ func IsBlobReferenced(imgStore storageTypes.ImageStore, repo string,
 		return false, err
 	}
 
-	return isBlobReferencedInImageIndex(imgStore, repo, digest, index, log)
+	return IsBlobReferencedInImageIndex(imgStore, repo, digest, index, log)
+}
+
+/* Garbage Collection */
+
+func AddImageManifestBlobsToReferences(imgStore storageTypes.ImageStore,
+	repo string, mdigest godigest.Digest, refBlobs map[string]bool, log zlog.Logger,
+) error {
+	manifestContent, err := GetImageManifest(imgStore, repo, mdigest, log)
+	if err != nil {
+		log.Error().Err(err).Str("repository", repo).Str("digest", mdigest.String()).
+			Msg("gc: failed to read manifest image")
+
+		return err
+	}
+
+	refBlobs[mdigest.String()] = true
+	refBlobs[manifestContent.Config.Digest.String()] = true
+
+	// if there is a Subject, it may not exist yet and that is ok
+	if manifestContent.Subject != nil {
+		refBlobs[manifestContent.Subject.Digest.String()] = true
+	}
+
+	for _, layer := range manifestContent.Layers {
+		refBlobs[layer.Digest.String()] = true
+	}
+
+	return nil
+}
+
+func AddORASImageManifestBlobsToReferences(imgStore storageTypes.ImageStore,
+	repo string, mdigest godigest.Digest, refBlobs map[string]bool, log zlog.Logger,
+) error {
+	manifestContent, err := GetOrasManifestByDigest(imgStore, repo, mdigest, log)
+	if err != nil {
+		log.Error().Err(err).Str("repository", repo).Str("digest", mdigest.String()).
+			Msg("gc: failed to read manifest image")
+
+		return err
+	}
+
+	refBlobs[mdigest.String()] = true
+
+	// if there is a Subject, it may not exist yet and that is ok
+	if manifestContent.Subject != nil {
+		refBlobs[manifestContent.Subject.Digest.String()] = true
+	}
+
+	for _, blob := range manifestContent.Blobs {
+		refBlobs[blob.Digest.String()] = true
+	}
+
+	return nil
+}
+
+func AddImageIndexBlobsToReferences(imgStore storageTypes.ImageStore,
+	repo string, mdigest godigest.Digest, refBlobs map[string]bool, log zlog.Logger,
+) error {
+	index, err := GetImageIndex(imgStore, repo, mdigest, log)
+	if err != nil {
+		log.Error().Err(err).Str("repository", repo).Str("digest", mdigest.String()).
+			Msg("gc: failed to read manifest image")
+
+		return err
+	}
+
+	refBlobs[mdigest.String()] = true
+
+	// if there is a Subject, it may not exist yet and that is ok
+	if index.Subject != nil {
+		refBlobs[index.Subject.Digest.String()] = true
+	}
+
+	for _, manifest := range index.Manifests {
+		refBlobs[manifest.Digest.String()] = true
+	}
+
+	return nil
+}
+
+func AddIndexBlobToReferences(imgStore storageTypes.ImageStore,
+	repo string, index ispec.Index, refBlobs map[string]bool, log zlog.Logger,
+) error {
+	for _, desc := range index.Manifests {
+		switch desc.MediaType {
+		case ispec.MediaTypeImageIndex:
+			if err := AddImageIndexBlobsToReferences(imgStore, repo, desc.Digest, refBlobs, log); err != nil {
+				log.Error().Err(err).Str("repository", repo).Str("digest", desc.Digest.String()).
+					Msg("failed to read blobs in multiarch(index) image")
+
+				return err
+			}
+		case ispec.MediaTypeImageManifest:
+			if err := AddImageManifestBlobsToReferences(imgStore, repo, desc.Digest, refBlobs, log); err != nil {
+				log.Error().Err(err).Str("repository", repo).Str("digest", desc.Digest.String()).
+					Msg("failed to read blobs in image manifest")
+
+				return err
+			}
+		case oras.MediaTypeArtifactManifest:
+			if err := AddORASImageManifestBlobsToReferences(imgStore, repo, desc.Digest, refBlobs, log); err != nil {
+				log.Error().Err(err).Str("repository", repo).Str("digest", desc.Digest.String()).
+					Msg("failed to read blobs in image manifest")
+
+				return err
+			}
+		}
+	}
+
+	return nil
+}
+
+func AddRepoBlobsToReferences(imgStore storageTypes.ImageStore,
+	repo string, refBlobs map[string]bool, log zlog.Logger,
+) error {
+	dir := path.Join(imgStore.RootDir(), repo)
+	if !imgStore.DirExists(dir) {
+		return zerr.ErrRepoNotFound
+	}
+
+	index, err := GetIndex(imgStore, repo, log)
+	if err != nil {
+		return err
+	}
+
+	return AddIndexBlobToReferences(imgStore, repo, index, refBlobs, log)
 }
 
 func ApplyLinter(imgStore storageTypes.ImageStore, linter Lint, repo string, descriptor ispec.Descriptor,
@@ -580,7 +708,7 @@ func IsSignature(descriptor ispec.Descriptor) bool {
 }
 
 func GetOrasReferrers(imgStore storageTypes.ImageStore, repo string, gdigest godigest.Digest, artifactType string,
-	log zerolog.Logger,
+	log zlog.Logger,
 ) ([]oras.Descriptor, error) {
 	if err := gdigest.Validate(); err != nil {
 		return nil, err
@@ -638,7 +766,7 @@ func GetOrasReferrers(imgStore storageTypes.ImageStore, repo string, gdigest god
 }
 
 func GetReferrers(imgStore storageTypes.ImageStore, repo string, gdigest godigest.Digest, artifactTypes []string,
-	log zerolog.Logger,
+	log zlog.Logger,
 ) (ispec.Index, error) {
 	nilIndex := ispec.Index{}
 
@@ -741,7 +869,7 @@ func GetReferrers(imgStore storageTypes.ImageStore, repo string, gdigest godiges
 	return index, nil
 }
 
-func GetOrasManifestByDigest(imgStore storageTypes.ImageStore, repo string, digest godigest.Digest, log zerolog.Logger,
+func GetOrasManifestByDigest(imgStore storageTypes.ImageStore, repo string, digest godigest.Digest, log zlog.Logger,
 ) (oras.Manifest, error) {
 	var artManifest oras.Manifest
 
@@ -827,7 +955,7 @@ type DedupeTaskGenerator struct {
 	and generating a task for each unprocessed one*/
 	lastDigests []godigest.Digest
 	done        bool
-	Log         zerolog.Logger
+	Log         zlog.Logger
 }
 
 func (gen *DedupeTaskGenerator) Next() (scheduler.Task, error) {
@@ -879,11 +1007,11 @@ type dedupeTask struct {
 	// blobs paths with the same digest ^
 	duplicateBlobs []string
 	dedupe         bool
-	log            zerolog.Logger
+	log            zlog.Logger
 }
 
 func newDedupeTask(imgStore storageTypes.ImageStore, digest godigest.Digest, dedupe bool,
-	duplicateBlobs []string, log zerolog.Logger,
+	duplicateBlobs []string, log zlog.Logger,
 ) *dedupeTask {
 	return &dedupeTask{imgStore, digest, duplicateBlobs, dedupe, log}
 }
@@ -929,8 +1057,7 @@ func (gen *GCTaskGenerator) Next() (scheduler.Task, error) {
 	gen.nextRun = time.Now().Add(time.Duration(delay) * time.Second)
 
 	repo, err := gen.ImgStore.GetNextRepository(gen.lastRepo)
-
-	if err != nil && !errors.Is(err, io.EOF) {
+	if err != nil {
 		return nil, err
 	}
 

pkg/storage/common/common_test.go

@@ -5,6 +5,7 @@ import (
 	"encoding/json"
 	"errors"
 	"os"
+	"path"
 	"testing"
 
 	godigest "github.com/opencontainers/go-digest"
@@ -36,8 +37,8 @@ func TestValidateManifest(t *testing.T) {
 			Name:        "cache",
 			UseRelPaths: true,
 		}, log)
-		imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, true,
-			true, log, metrics, nil, cacheDriver)
+		imgStore := local.NewImageStore(dir, true, true, storageConstants.DefaultGCDelay,
+			storageConstants.DefaultUntaggedImgeRetentionDelay, true, true, log, metrics, nil, cacheDriver)
 
 		content := []byte("this is a blob")
 		digest := godigest.FromBytes(content)
@@ -81,6 +82,37 @@ func TestValidateManifest(t *testing.T) {
 			So(internalErr.GetDetails()["jsonSchemaValidation"], ShouldEqual, "[schemaVersion: Must be less than or equal to 2]")
 		})
 
+		Convey("bad config blob", func() {
+			manifest := ispec.Manifest{
+				Config: ispec.Descriptor{
+					MediaType: ispec.MediaTypeImageConfig,
+					Digest:    cdigest,
+					Size:      int64(len(cblob)),
+				},
+				Layers: []ispec.Descriptor{
+					{
+						MediaType: ispec.MediaTypeImageLayer,
+						Digest:    digest,
+						Size:      int64(len(content)),
+					},
+				},
+			}
+
+			manifest.SchemaVersion = 2
+
+			configBlobPath := imgStore.BlobPath("test", cdigest)
+
+			err := os.WriteFile(configBlobPath, []byte("bad config blob"), 0o000)
+			So(err, ShouldBeNil)
+
+			body, err := json.Marshal(manifest)
+			So(err, ShouldBeNil)
+
+			// this was actually an umoci error on config blob
+			_, _, err = imgStore.PutImageManifest("test", "1.0", ispec.MediaTypeImageManifest, body)
+			So(err, ShouldBeNil)
+		})
+
 		Convey("manifest with non-distributable layers", func() {
 			content := []byte("this blob doesn't exist")
 			digest := godigest.FromBytes(content)
@@ -124,29 +156,29 @@ func TestGetReferrersErrors(t *testing.T) {
 			UseRelPaths: true,
 		}, log)
 
-		imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, false,
-			true, log, metrics, nil, cacheDriver)
+		imgStore := local.NewImageStore(dir, true, true, storageConstants.DefaultGCDelay,
+			storageConstants.DefaultUntaggedImgeRetentionDelay, false, true, log, metrics, nil, cacheDriver)
 
 		artifactType := "application/vnd.example.icecream.v1"
 		validDigest := godigest.FromBytes([]byte("blob"))
 
 		Convey("Trigger invalid digest error", func(c C) {
 			_, err := common.GetReferrers(imgStore, "zot-test", "invalidDigest",
-				[]string{artifactType}, log.With().Caller().Logger())
+				[]string{artifactType}, log)
 			So(err, ShouldNotBeNil)
 
 			_, err = common.GetOrasReferrers(imgStore, "zot-test", "invalidDigest",
-				artifactType, log.With().Caller().Logger())
+				artifactType, log)
 			So(err, ShouldNotBeNil)
 		})
 
 		Convey("Trigger repo not found error", func(c C) {
 			_, err := common.GetReferrers(imgStore, "zot-test", validDigest,
-				[]string{artifactType}, log.With().Caller().Logger())
+				[]string{artifactType}, log)
 			So(err, ShouldNotBeNil)
 
 			_, err = common.GetOrasReferrers(imgStore, "zot-test", validDigest,
-				artifactType, log.With().Caller().Logger())
+				artifactType, log)
 			So(err, ShouldNotBeNil)
 		})
 
@@ -179,11 +211,11 @@ func TestGetReferrersErrors(t *testing.T) {
 			}
 
 			_, err = common.GetReferrers(imgStore, "zot-test", validDigest,
-				[]string{artifactType}, log.With().Caller().Logger())
+				[]string{artifactType}, log)
 			So(err, ShouldNotBeNil)
 
 			_, err = common.GetOrasReferrers(imgStore, "zot-test", validDigest,
-				artifactType, log.With().Caller().Logger())
+				artifactType, log)
 			So(err, ShouldNotBeNil)
 		})
 
@@ -198,11 +230,11 @@ func TestGetReferrersErrors(t *testing.T) {
 			}
 
 			_, err = common.GetReferrers(imgStore, "zot-test", validDigest,
-				[]string{artifactType}, log.With().Caller().Logger())
+				[]string{artifactType}, log)
 			So(err, ShouldNotBeNil)
 
 			_, err = common.GetOrasReferrers(imgStore, "zot-test", validDigest,
-				artifactType, log.With().Caller().Logger())
+				artifactType, log)
 			So(err, ShouldNotBeNil)
 		})
 
@@ -227,11 +259,11 @@ func TestGetReferrersErrors(t *testing.T) {
 			}
 
 			_, err = common.GetOrasReferrers(imgStore, "zot-test", validDigest,
-				artifactType, log.With().Caller().Logger())
+				artifactType, log)
 			So(err, ShouldNotBeNil)
 
 			_, err = common.GetOrasReferrers(imgStore, "zot-test", digest,
-				artifactType, log.With().Caller().Logger())
+				artifactType, log)
 			So(err, ShouldNotBeNil)
 		})
 
@@ -245,7 +277,7 @@ func TestGetReferrersErrors(t *testing.T) {
 				},
 			}
 
-			_, err = common.GetOrasReferrers(imgStore, "zot-test", validDigest, artifactType, log.With().Caller().Logger())
+			_, err = common.GetOrasReferrers(imgStore, "zot-test", validDigest, artifactType, log)
 			So(err, ShouldNotBeNil)
 		})
 
@@ -272,7 +304,7 @@ func TestGetReferrersErrors(t *testing.T) {
 			}
 
 			_, err = common.GetReferrers(imgStore, "zot-test", validDigest,
-				[]string{artifactType}, log.With().Caller().Logger())
+				[]string{artifactType}, log)
 			So(err, ShouldNotBeNil)
 		})
 
@@ -306,7 +338,7 @@ func TestGetReferrersErrors(t *testing.T) {
 			}
 
 			_, err = common.GetReferrers(imgStore, "zot-test", validDigest,
-				[]string{artifactType}, log.With().Caller().Logger())
+				[]string{artifactType}, log)
 			So(err, ShouldBeNil)
 		})
 
@@ -326,7 +358,7 @@ func TestGetReferrersErrors(t *testing.T) {
 			}
 
 			_, err = common.GetReferrers(imgStore, "zot-test", validDigest,
-				[]string{}, log.With().Caller().Logger())
+				[]string{}, log)
 			So(err, ShouldNotBeNil)
 		})
 
@@ -348,7 +380,7 @@ func TestGetReferrersErrors(t *testing.T) {
 			}
 
 			ref, err := common.GetReferrers(imgStore, "zot-test", validDigest,
-				[]string{"art.type"}, log.With().Caller().Logger())
+				[]string{"art.type"}, log)
 			So(err, ShouldBeNil)
 			So(len(ref.Manifests), ShouldEqual, 0)
 		})
@@ -356,7 +388,7 @@ func TestGetReferrersErrors(t *testing.T) {
 }
 
 func TestGetImageIndexErrors(t *testing.T) {
-	log := zerolog.New(os.Stdout)
+	log := log.Logger{Logger: zerolog.New(os.Stdout)}
 
 	Convey("Trigger invalid digest error", t, func(c C) {
 		imgStore := &mocks.MockedImageStore{}
@@ -400,3 +432,193 @@ func TestIsSignature(t *testing.T) {
 		So(isSingature, ShouldBeFalse)
 	})
 }
+
+func TestGarbageCollectManifestErrors(t *testing.T) {
+	Convey("Make imagestore and upload manifest", t, func(c C) {
+		dir := t.TempDir()
+
+		repoName := "test"
+
+		log := log.Logger{Logger: zerolog.New(os.Stdout)}
+		metrics := monitoring.NewMetricsServer(false, log)
+		cacheDriver, _ := storage.Create("boltdb", cache.BoltDBDriverParameters{
+			RootDir:     dir,
+			Name:        "cache",
+			UseRelPaths: true,
+		}, log)
+		imgStore := local.NewImageStore(dir, true, true, storageConstants.DefaultGCDelay,
+			storageConstants.DefaultUntaggedImgeRetentionDelay, true, true, log, metrics, nil, cacheDriver)
+
+		Convey("trigger repo not found in GetReferencedBlobs()", func() {
+			err := common.AddRepoBlobsToReferences(imgStore, repoName, map[string]bool{}, log)
+			So(err, ShouldNotBeNil)
+		})
+
+		content := []byte("this is a blob")
+		digest := godigest.FromBytes(content)
+		So(digest, ShouldNotBeNil)
+
+		_, blen, err := imgStore.FullBlobUpload(repoName, bytes.NewReader(content), digest)
+		So(err, ShouldBeNil)
+		So(blen, ShouldEqual, len(content))
+
+		cblob, cdigest := test.GetRandomImageConfig()
+		_, clen, err := imgStore.FullBlobUpload(repoName, bytes.NewReader(cblob), cdigest)
+		So(err, ShouldBeNil)
+		So(clen, ShouldEqual, len(cblob))
+
+		manifest := ispec.Manifest{
+			Config: ispec.Descriptor{
+				MediaType: ispec.MediaTypeImageConfig,
+				Digest:    cdigest,
+				Size:      int64(len(cblob)),
+			},
+			Layers: []ispec.Descriptor{
+				{
+					MediaType: ispec.MediaTypeImageLayer,
+					Digest:    digest,
+					Size:      int64(len(content)),
+				},
+			},
+		}
+
+		manifest.SchemaVersion = 2
+
+		body, err := json.Marshal(manifest)
+		So(err, ShouldBeNil)
+
+		manifestDigest := godigest.FromBytes(body)
+
+		_, _, err = imgStore.PutImageManifest(repoName, "1.0", ispec.MediaTypeImageManifest, body)
+		So(err, ShouldBeNil)
+
+		Convey("trigger GetIndex error in GetReferencedBlobs", func() {
+			err := os.Chmod(path.Join(imgStore.RootDir(), repoName), 0o000)
+			So(err, ShouldBeNil)
+
+			defer func() {
+				err := os.Chmod(path.Join(imgStore.RootDir(), repoName), 0o755)
+				So(err, ShouldBeNil)
+			}()
+
+			err = common.AddRepoBlobsToReferences(imgStore, repoName, map[string]bool{}, log)
+			So(err, ShouldNotBeNil)
+		})
+
+		Convey("trigger GetImageManifest error in GetReferencedBlobsInImageManifest", func() {
+			err := os.Chmod(path.Join(imgStore.RootDir(), repoName, "blobs", "sha256", manifestDigest.Encoded()), 0o000)
+			So(err, ShouldBeNil)
+
+			defer func() {
+				err := os.Chmod(path.Join(imgStore.RootDir(), repoName, "blobs", "sha256", manifestDigest.Encoded()), 0o755)
+				So(err, ShouldBeNil)
+			}()
+
+			err = common.AddRepoBlobsToReferences(imgStore, repoName, map[string]bool{}, log)
+			So(err, ShouldNotBeNil)
+		})
+	})
+}
+
+func TestGarbageCollectIndexErrors(t *testing.T) {
+	Convey("Make imagestore and upload manifest", t, func(c C) {
+		dir := t.TempDir()
+
+		repoName := "test"
+
+		log := log.Logger{Logger: zerolog.New(os.Stdout)}
+		metrics := monitoring.NewMetricsServer(false, log)
+		cacheDriver, _ := storage.Create("boltdb", cache.BoltDBDriverParameters{
+			RootDir:     dir,
+			Name:        "cache",
+			UseRelPaths: true,
+		}, log)
+		imgStore := local.NewImageStore(dir, true, true, storageConstants.DefaultGCDelay,
+			storageConstants.DefaultUntaggedImgeRetentionDelay, true, true, log, metrics, nil, cacheDriver)
+
+		content := []byte("this is a blob")
+		bdgst := godigest.FromBytes(content)
+		So(bdgst, ShouldNotBeNil)
+
+		_, bsize, err := imgStore.FullBlobUpload(repoName, bytes.NewReader(content), bdgst)
+		So(err, ShouldBeNil)
+		So(bsize, ShouldEqual, len(content))
+
+		var index ispec.Index
+		index.SchemaVersion = 2
+		index.MediaType = ispec.MediaTypeImageIndex
+
+		var digest godigest.Digest
+		for i := 0; i < 4; i++ {
+			// upload image config blob
+			upload, err := imgStore.NewBlobUpload(repoName)
+			So(err, ShouldBeNil)
+			So(upload, ShouldNotBeEmpty)
+
+			cblob, cdigest := test.GetRandomImageConfig()
+			buf := bytes.NewBuffer(cblob)
+			buflen := buf.Len()
+			blob, err := imgStore.PutBlobChunkStreamed(repoName, upload, buf)
+			So(err, ShouldBeNil)
+			So(blob, ShouldEqual, buflen)
+
+			err = imgStore.FinishBlobUpload(repoName, upload, buf, cdigest)
+			So(err, ShouldBeNil)
+			So(blob, ShouldEqual, buflen)
+
+			// create a manifest
+			manifest := ispec.Manifest{
+				Config: ispec.Descriptor{
+					MediaType: ispec.MediaTypeImageConfig,
+					Digest:    cdigest,
+					Size:      int64(len(cblob)),
+				},
+				Layers: []ispec.Descriptor{
+					{
+						MediaType: ispec.MediaTypeImageLayer,
+						Digest:    bdgst,
+						Size:      bsize,
+					},
+				},
+			}
+			manifest.SchemaVersion = 2
+			content, err = json.Marshal(manifest)
+			So(err, ShouldBeNil)
+			digest = godigest.FromBytes(content)
+			So(digest, ShouldNotBeNil)
+			_, _, err = imgStore.PutImageManifest(repoName, digest.String(), ispec.MediaTypeImageManifest, content)
+			So(err, ShouldBeNil)
+
+			index.Manifests = append(index.Manifests, ispec.Descriptor{
+				Digest:    digest,
+				MediaType: ispec.MediaTypeImageManifest,
+				Size:      int64(len(content)),
+			})
+		}
+
+		// upload index image
+		indexContent, err := json.Marshal(index)
+		So(err, ShouldBeNil)
+		indexDigest := godigest.FromBytes(indexContent)
+		So(indexDigest, ShouldNotBeNil)
+
+		_, _, err = imgStore.PutImageManifest(repoName, "1.0", ispec.MediaTypeImageIndex, indexContent)
+		So(err, ShouldBeNil)
+
+		err = common.AddRepoBlobsToReferences(imgStore, repoName, map[string]bool{}, log)
+		So(err, ShouldBeNil)
+
+		Convey("trigger GetImageIndex error in GetReferencedBlobsInImageIndex", func() {
+			err := os.Chmod(path.Join(imgStore.RootDir(), repoName, "blobs", "sha256", indexDigest.Encoded()), 0o000)
+			So(err, ShouldBeNil)
+
+			defer func() {
+				err := os.Chmod(path.Join(imgStore.RootDir(), repoName, "blobs", "sha256", indexDigest.Encoded()), 0o755)
+				So(err, ShouldBeNil)
+			}()
+
+			err = common.AddRepoBlobsToReferences(imgStore, repoName, map[string]bool{}, log)
+			So(err, ShouldNotBeNil)
+		})
+	})
+}

pkg/storage/constants/constants.go

@@ -6,20 +6,22 @@ import (
 
 const (
 	// BlobUploadDir defines the upload directory for blob uploads.
-	BlobUploadDir           = ".uploads"
-	SchemaVersion           = 2
-	DefaultFilePerms        = 0o600
-	DefaultDirPerms         = 0o700
-	RLOCK                   = "RLock"
-	RWLOCK                  = "RWLock"
-	BlobsCache              = "blobs"
-	DuplicatesBucket        = "duplicates"
-	OriginalBucket          = "original"
-	DBExtensionName         = ".db"
-	DBCacheLockCheckTimeout = 10 * time.Second
-	BoltdbName              = "cache"
-	DynamoDBDriverName      = "dynamodb"
-	DefaultGCDelay          = 1 * time.Hour
-	DefaultGCInterval       = 1 * time.Hour
-	S3StorageDriverName     = "s3"
+	BlobUploadDir                     = ".uploads"
+	SchemaVersion                     = 2
+	DefaultFilePerms                  = 0o600
+	DefaultDirPerms                   = 0o700
+	RLOCK                             = "RLock"
+	RWLOCK                            = "RWLock"
+	BlobsCache                        = "blobs"
+	DuplicatesBucket                  = "duplicates"
+	OriginalBucket                    = "original"
+	DBExtensionName                   = ".db"
+	DBCacheLockCheckTimeout           = 10 * time.Second
+	BoltdbName                        = "cache"
+	DynamoDBDriverName                = "dynamodb"
+	DefaultGCDelay                    = 1 * time.Hour
+	DefaultUntaggedImgeRetentionDelay = 24 * time.Hour
+	DefaultGCInterval                 = 1 * time.Hour
+	S3StorageDriverName               = "s3"
+	LocalStorageDriverName            = "local"
 )

pkg/storage/local/driver.go

@@ -0,0 +1,481 @@
+package local
+
+import (
+	"bufio"
+	"bytes"
+	"errors"
+	"io"
+	"io/fs"
+	"os"
+	"path"
+	"sort"
+	"syscall"
+	"time"
+	"unicode/utf8"
+
+	storagedriver "github.com/docker/distribution/registry/storage/driver"
+
+	zerr "zotregistry.io/zot/errors"
+	storageConstants "zotregistry.io/zot/pkg/storage/constants"
+	"zotregistry.io/zot/pkg/test/inject"
+)
+
+type Driver struct {
+	commit bool
+}
+
+func New(commit bool) *Driver {
+	return &Driver{commit: commit}
+}
+
+func (driver *Driver) Name() string {
+	return storageConstants.LocalStorageDriverName
+}
+
+func (driver *Driver) EnsureDir(path string) error {
+	err := os.MkdirAll(path, storageConstants.DefaultDirPerms)
+
+	return driver.formatErr(err)
+}
+
+func (driver *Driver) DirExists(path string) bool {
+	if !utf8.ValidString(path) {
+		return false
+	}
+
+	fileInfo, err := os.Stat(path)
+	if err != nil {
+		if e, ok := err.(*fs.PathError); ok && errors.Is(e.Err, syscall.ENAMETOOLONG) || //nolint: errorlint
+			errors.Is(e.Err, syscall.EINVAL) {
+			return false
+		}
+	}
+
+	if err != nil && os.IsNotExist(err) {
+		return false
+	}
+
+	if !fileInfo.IsDir() {
+		return false
+	}
+
+	return true
+}
+
+func (driver *Driver) Reader(path string, offset int64) (io.ReadCloser, error) {
+	file, err := os.OpenFile(path, os.O_RDONLY, storageConstants.DefaultFilePerms)
+	if err != nil {
+		if os.IsNotExist(err) {
+			return nil, storagedriver.PathNotFoundError{Path: path}
+		}
+
+		return nil, driver.formatErr(err)
+	}
+
+	seekPos, err := file.Seek(offset, io.SeekStart)
+	if err != nil {
+		file.Close()
+
+		return nil, driver.formatErr(err)
+	} else if seekPos < offset {
+		file.Close()
+
+		return nil, storagedriver.InvalidOffsetError{Path: path, Offset: offset}
+	}
+
+	return file, nil
+}
+
+func (driver *Driver) ReadFile(path string) ([]byte, error) {
+	reader, err := driver.Reader(path, 0)
+	if err != nil {
+		return nil, err
+	}
+
+	defer reader.Close()
+
+	buf, err := io.ReadAll(reader)
+	if err != nil {
+		return nil, driver.formatErr(err)
+	}
+
+	return buf, nil
+}
+
+func (driver *Driver) Delete(path string) error {
+	_, err := os.Stat(path)
+	if err != nil && !os.IsNotExist(err) {
+		return driver.formatErr(err)
+	} else if err != nil {
+		return storagedriver.PathNotFoundError{Path: path}
+	}
+
+	return os.RemoveAll(path)
+}
+
+func (driver *Driver) Stat(path string) (storagedriver.FileInfo, error) {
+	fi, err := os.Stat(path) //nolint: varnamelen
+	if err != nil {
+		if os.IsNotExist(err) {
+			return nil, storagedriver.PathNotFoundError{Path: path}
+		}
+
+		return nil, driver.formatErr(err)
+	}
+
+	return fileInfo{
+		path:     path,
+		FileInfo: fi,
+	}, nil
+}
+
+func (driver *Driver) Writer(filepath string, append bool) (storagedriver.FileWriter, error) { //nolint:predeclared
+	if append {
+		_, err := os.Stat(filepath)
+		if err != nil {
+			if os.IsNotExist(err) {
+				return nil, storagedriver.PathNotFoundError{Path: filepath}
+			}
+
+			return nil, driver.formatErr(err)
+		}
+	}
+
+	parentDir := path.Dir(filepath)
+	if err := os.MkdirAll(parentDir, storageConstants.DefaultDirPerms); err != nil {
+		return nil, driver.formatErr(err)
+	}
+
+	file, err := os.OpenFile(filepath, os.O_WRONLY|os.O_CREATE, storageConstants.DefaultFilePerms)
+	if err != nil {
+		return nil, driver.formatErr(err)
+	}
+
+	var offset int64
+
+	if !append {
+		err := file.Truncate(0)
+		if err != nil {
+			file.Close()
+
+			return nil, driver.formatErr(err)
+		}
+	} else {
+		n, err := file.Seek(0, io.SeekEnd) //nolint: varnamelen
+		if err != nil {
+			file.Close()
+
+			return nil, driver.formatErr(err)
+		}
+		offset = n
+	}
+
+	return newFileWriter(file, offset, driver.commit), nil
+}
+
+func (driver *Driver) WriteFile(filepath string, content []byte) (int, error) {
+	writer, err := driver.Writer(filepath, false)
+	if err != nil {
+		return -1, err
+	}
+
+	nbytes, err := io.Copy(writer, bytes.NewReader(content))
+	if err != nil {
+		_ = writer.Cancel()
+
+		return -1, driver.formatErr(err)
+	}
+
+	return int(nbytes), writer.Close()
+}
+
+func (driver *Driver) Walk(path string, walkFn storagedriver.WalkFn) error {
+	children, err := driver.List(path)
+	if err != nil {
+		return err
+	}
+
+	sort.Stable(sort.StringSlice(children))
+
+	for _, child := range children {
+		// Calling driver.Stat for every entry is quite
+		// expensive when running against backends with a slow Stat
+		// implementation, such as s3. This is very likely a serious
+		// performance bottleneck.
+		fileInfo, err := driver.Stat(child)
+		if err != nil {
+			switch errors.As(err, &storagedriver.PathNotFoundError{}) {
+			case true:
+				// repository was removed in between listing and enumeration. Ignore it.
+				continue
+			default:
+				return err
+			}
+		}
+		err = walkFn(fileInfo)
+		//nolint: gocritic
+		if err == nil && fileInfo.IsDir() {
+			if err := driver.Walk(child, walkFn); err != nil {
+				return err
+			}
+		} else if errors.Is(err, storagedriver.ErrSkipDir) {
+			// Stop iteration if it's a file, otherwise noop if it's a directory
+			if !fileInfo.IsDir() {
+				return nil
+			}
+		} else if err != nil {
+			return driver.formatErr(err)
+		}
+	}
+
+	return nil
+}
+
+func (driver *Driver) List(fullpath string) ([]string, error) {
+	dir, err := os.Open(fullpath)
+	if err != nil {
+		if os.IsNotExist(err) {
+			return nil, storagedriver.PathNotFoundError{Path: fullpath}
+		}
+
+		return nil, driver.formatErr(err)
+	}
+
+	defer dir.Close()
+
+	fileNames, err := dir.Readdirnames(0)
+	if err != nil {
+		return nil, driver.formatErr(err)
+	}
+
+	keys := make([]string, 0, len(fileNames))
+	for _, fileName := range fileNames {
+		keys = append(keys, path.Join(fullpath, fileName))
+	}
+
+	return keys, nil
+}
+
+func (driver *Driver) Move(sourcePath string, destPath string) error {
+	if _, err := os.Stat(sourcePath); os.IsNotExist(err) {
+		return storagedriver.PathNotFoundError{Path: sourcePath}
+	}
+
+	if err := os.MkdirAll(path.Dir(destPath), storageConstants.DefaultDirPerms); err != nil {
+		return driver.formatErr(err)
+	}
+
+	return driver.formatErr(os.Rename(sourcePath, destPath))
+}
+
+func (driver *Driver) SameFile(path1, path2 string) bool {
+	file1, err := os.Stat(path1)
+	if err != nil {
+		return false
+	}
+
+	file2, err := os.Stat(path2)
+	if err != nil {
+		return false
+	}
+
+	return os.SameFile(file1, file2)
+}
+
+func (driver *Driver) Link(src, dest string) error {
+	if err := os.Remove(dest); err != nil && !os.IsNotExist(err) {
+		return err
+	}
+
+	return driver.formatErr(os.Link(src, dest))
+}
+
+func (driver *Driver) formatErr(err error) error {
+	switch actual := err.(type) { //nolint: errorlint
+	case nil:
+		return nil
+	case storagedriver.PathNotFoundError:
+		actual.DriverName = driver.Name()
+
+		return actual
+	case storagedriver.InvalidPathError:
+		actual.DriverName = driver.Name()
+
+		return actual
+	case storagedriver.InvalidOffsetError:
+		actual.DriverName = driver.Name()
+
+		return actual
+	default:
+		storageError := storagedriver.Error{
+			DriverName: driver.Name(),
+			Enclosed:   err,
+		}
+
+		return storageError
+	}
+}
+
+type fileInfo struct {
+	os.FileInfo
+	path string
+}
+
+// asserts fileInfo implements storagedriver.FileInfo.
+var _ storagedriver.FileInfo = fileInfo{}
+
+// Path provides the full path of the target of this file info.
+func (fi fileInfo) Path() string {
+	return fi.path
+}
+
+// Size returns current length in bytes of the file. The return value can
+// be used to write to the end of the file at path. The value is
+// meaningless if IsDir returns true.
+func (fi fileInfo) Size() int64 {
+	if fi.IsDir() {
+		return 0
+	}
+
+	return fi.FileInfo.Size()
+}
+
+// ModTime returns the modification time for the file. For backends that
+// don't have a modification time, the creation time should be returned.
+func (fi fileInfo) ModTime() time.Time {
+	return fi.FileInfo.ModTime()
+}
+
+// IsDir returns true if the path is a directory.
+func (fi fileInfo) IsDir() bool {
+	return fi.FileInfo.IsDir()
+}
+
+type fileWriter struct {
+	file      *os.File
+	size      int64
+	bw        *bufio.Writer
+	closed    bool
+	committed bool
+	cancelled bool
+	commit    bool
+}
+
+func newFileWriter(file *os.File, size int64, commit bool) *fileWriter {
+	return &fileWriter{
+		file:   file,
+		size:   size,
+		commit: commit,
+		bw:     bufio.NewWriter(file),
+	}
+}
+
+func (fw *fileWriter) Write(buf []byte) (int, error) {
+	//nolint: gocritic
+	if fw.closed {
+		return 0, zerr.ErrFileAlreadyClosed
+	} else if fw.committed {
+		return 0, zerr.ErrFileAlreadyCommitted
+	} else if fw.cancelled {
+		return 0, zerr.ErrFileAlreadyCancelled
+	}
+
+	n, err := fw.bw.Write(buf)
+	fw.size += int64(n)
+
+	return n, err
+}
+
+func (fw *fileWriter) Size() int64 {
+	return fw.size
+}
+
+func (fw *fileWriter) Close() error {
+	if fw.closed {
+		return zerr.ErrFileAlreadyClosed
+	}
+
+	if err := fw.bw.Flush(); err != nil {
+		return err
+	}
+
+	if fw.commit {
+		if err := inject.Error(fw.file.Sync()); err != nil {
+			return err
+		}
+	}
+
+	if err := inject.Error(fw.file.Close()); err != nil {
+		return err
+	}
+
+	fw.closed = true
+
+	return nil
+}
+
+func (fw *fileWriter) Cancel() error {
+	if fw.closed {
+		return zerr.ErrFileAlreadyClosed
+	}
+
+	fw.cancelled = true
+	fw.file.Close()
+
+	return os.Remove(fw.file.Name())
+}
+
+func (fw *fileWriter) Commit() error {
+	//nolint: gocritic
+	if fw.closed {
+		return zerr.ErrFileAlreadyClosed
+	} else if fw.committed {
+		return zerr.ErrFileAlreadyCommitted
+	} else if fw.cancelled {
+		return zerr.ErrFileAlreadyCancelled
+	}
+
+	if err := fw.bw.Flush(); err != nil {
+		return err
+	}
+
+	if fw.commit {
+		if err := fw.file.Sync(); err != nil {
+			return err
+		}
+	}
+
+	fw.committed = true
+
+	return nil
+}
+
+func ValidateHardLink(rootDir string) error {
+	if err := os.MkdirAll(rootDir, storageConstants.DefaultDirPerms); err != nil {
+		return err
+	}
+
+	err := os.WriteFile(path.Join(rootDir, "hardlinkcheck.txt"),
+		[]byte("check whether hardlinks work on filesystem"), storageConstants.DefaultFilePerms)
+	if err != nil {
+		return err
+	}
+
+	err = os.Link(path.Join(rootDir, "hardlinkcheck.txt"), path.Join(rootDir, "duphardlinkcheck.txt"))
+	if err != nil {
+		// Remove hardlinkcheck.txt if hardlink fails
+		zerr := os.RemoveAll(path.Join(rootDir, "hardlinkcheck.txt"))
+		if zerr != nil {
+			return zerr
+		}
+
+		return err
+	}
+
+	err = os.RemoveAll(path.Join(rootDir, "hardlinkcheck.txt"))
+	if err != nil {
+		return err
+	}
+
+	return os.RemoveAll(path.Join(rootDir, "duphardlinkcheck.txt"))
+}

pkg/storage/local/local_elevated_test.go

@@ -36,8 +36,8 @@ func TestElevatedPrivilegesInvalidDedupe(t *testing.T) {
 			Name:        "cache",
 			UseRelPaths: true,
 		}, log)
-		imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, true, true, log,
-			metrics, nil, cacheDriver)
+		imgStore := local.NewImageStore(dir, true, true, storageConstants.DefaultGCDelay,
+			storageConstants.DefaultUntaggedImgeRetentionDelay, true, true, log, metrics, nil, cacheDriver)
 
 		upload, err := imgStore.NewBlobUpload("dedupe1")
 		So(err, ShouldBeNil)

pkg/storage/s3/driver.go

@@ -0,0 +1,115 @@
+package s3
+
+import (
+	"context"
+	"io"
+
+	// Add s3 support.
+	"github.com/docker/distribution/registry/storage/driver"
+	_ "github.com/docker/distribution/registry/storage/driver/s3-aws"
+
+	storageConstants "zotregistry.io/zot/pkg/storage/constants"
+)
+
+type Driver struct {
+	store driver.StorageDriver
+}
+
+func New(storeDriver driver.StorageDriver) *Driver {
+	return &Driver{store: storeDriver}
+}
+
+func (driver *Driver) Name() string {
+	return storageConstants.S3StorageDriverName
+}
+
+func (driver *Driver) EnsureDir(path string) error {
+	return nil
+}
+
+func (driver *Driver) DirExists(path string) bool {
+	if fi, err := driver.store.Stat(context.Background(), path); err == nil && fi.IsDir() {
+		return true
+	}
+
+	return false
+}
+
+func (driver *Driver) Reader(path string, offset int64) (io.ReadCloser, error) {
+	return driver.store.Reader(context.Background(), path, offset)
+}
+
+func (driver *Driver) ReadFile(path string) ([]byte, error) {
+	return driver.store.GetContent(context.Background(), path)
+}
+
+func (driver *Driver) Delete(path string) error {
+	return driver.store.Delete(context.Background(), path)
+}
+
+func (driver *Driver) Stat(path string) (driver.FileInfo, error) {
+	return driver.store.Stat(context.Background(), path)
+}
+
+func (driver *Driver) Writer(filepath string, append bool) (driver.FileWriter, error) { //nolint:predeclared
+	return driver.store.Writer(context.Background(), filepath, append)
+}
+
+func (driver *Driver) WriteFile(filepath string, content []byte) (int, error) {
+	var n int
+
+	if stwr, err := driver.store.Writer(context.Background(), filepath, false); err == nil {
+		defer stwr.Close()
+
+		if n, err = stwr.Write(content); err != nil {
+			return -1, err
+		}
+
+		if err := stwr.Commit(); err != nil {
+			return -1, err
+		}
+	} else {
+		return -1, err
+	}
+
+	return n, nil
+}
+
+func (driver *Driver) Walk(path string, f driver.WalkFn) error {
+	return driver.store.Walk(context.Background(), path, f)
+}
+
+func (driver *Driver) List(fullpath string) ([]string, error) {
+	return driver.store.List(context.Background(), fullpath)
+}
+
+func (driver *Driver) Move(sourcePath string, destPath string) error {
+	return driver.store.Move(context.Background(), sourcePath, destPath)
+}
+
+func (driver *Driver) SameFile(path1, path2 string) bool {
+	fi1, _ := driver.store.Stat(context.Background(), path1)
+
+	fi2, _ := driver.store.Stat(context.Background(), path2)
+
+	if fi1 != nil && fi2 != nil {
+		if fi1.IsDir() == fi2.IsDir() &&
+			fi1.ModTime() == fi2.ModTime() &&
+			fi1.Path() == fi2.Path() &&
+			fi1.Size() == fi2.Size() {
+			return true
+		}
+	}
+
+	return false
+}
+
+/*
+	Link put an empty file that will act like a link between the original file and deduped one
+
+because s3 doesn't support symlinks, wherever the storage will encounter an empty file, it will get the original one
+from cache.
+*/
+func (driver *Driver) Link(src, dest string) error {
+	return driver.store.PutContent(context.Background(), dest, []byte{})
+}

pkg/storage/s3/s3_test.go

@@ -77,15 +77,17 @@ func createMockStorage(rootDir string, cacheDir string, dedupe bool, store drive
 	var cacheDriver cache.Cache
 
 	// from pkg/cli/root.go/applyDefaultValues, s3 magic
-	if _, err := os.Stat(path.Join(cacheDir, "s3_cache.db")); dedupe || (!dedupe && err == nil) {
+	if _, err := os.Stat(path.Join(cacheDir,
+		storageConstants.BoltdbName+storageConstants.DBExtensionName)); dedupe || (!dedupe && err == nil) {
 		cacheDriver, _ = storage.Create("boltdb", cache.BoltDBDriverParameters{
 			RootDir:     cacheDir,
-			Name:        "s3_cache",
+			Name:        "cache",
 			UseRelPaths: false,
 		}, log)
 	}
-	il := s3.NewImageStore(rootDir, cacheDir, false, storageConstants.DefaultGCDelay,
-		dedupe, false, log, metrics, nil, store, cacheDriver,
+
+	il := s3.NewImageStore(rootDir, cacheDir, true, true, storageConstants.DefaultGCDelay,
+		storageConstants.DefaultUntaggedImgeRetentionDelay, dedupe, false, log, metrics, nil, store, cacheDriver,
 	)
 
 	return il
@@ -97,8 +99,8 @@ func createMockStorageWithMockCache(rootDir string, dedupe bool, store driver.St
 	log := log.Logger{Logger: zerolog.New(os.Stdout)}
 	metrics := monitoring.NewMetricsServer(false, log)
 
-	il := s3.NewImageStore(rootDir, "", false, storageConstants.DefaultGCDelay,
-		dedupe, false, log, metrics, nil, store, cacheDriver,
+	il := s3.NewImageStore(rootDir, "", true, true, storageConstants.DefaultGCDelay,
+		storageConstants.DefaultUntaggedImgeRetentionDelay, dedupe, false, log, metrics, nil, store, cacheDriver,
 	)
 
 	return il
@@ -150,17 +152,17 @@ func createObjectsStore(rootDir string, cacheDir string, dedupe bool) (
 	var err error
 
 	// from pkg/cli/root.go/applyDefaultValues, s3 magic
-	s3CacheDBPath := path.Join(cacheDir, s3.CacheDBName+storageConstants.DBExtensionName)
+	s3CacheDBPath := path.Join(cacheDir, storageConstants.BoltdbName+storageConstants.DBExtensionName)
 	if _, err = os.Stat(s3CacheDBPath); dedupe || (!dedupe && err == nil) {
 		cacheDriver, _ = storage.Create("boltdb", cache.BoltDBDriverParameters{
 			RootDir:     cacheDir,
-			Name:        "s3_cache",
+			Name:        "cache",
 			UseRelPaths: false,
 		}, log)
 	}
 
-	il := s3.NewImageStore(rootDir, cacheDir, false, storageConstants.DefaultGCDelay,
-		dedupe, false, log, metrics, nil, store, cacheDriver)
+	il := s3.NewImageStore(rootDir, cacheDir, true, true, storageConstants.DefaultGCDelay,
+		storageConstants.DefaultUntaggedImgeRetentionDelay, dedupe, false, log, metrics, nil, store, cacheDriver)
 
 	return store, il, err
 }
@@ -194,8 +196,8 @@ func createObjectsStoreDynamo(rootDir string, cacheDir string, dedupe bool, tabl
 		panic(err)
 	}
 
-	il := s3.NewImageStore(rootDir, cacheDir, false, storageConstants.DefaultGCDelay,
-		dedupe, false, log, metrics, nil, store, cacheDriver)
+	il := s3.NewImageStore(rootDir, cacheDir, true, true, storageConstants.DefaultGCDelay,
+		storageConstants.DefaultUntaggedImgeRetentionDelay, dedupe, false, log, metrics, nil, store, cacheDriver)
 
 	return store, il, err
 }
@@ -893,7 +895,7 @@ func TestNegativeCasesObjectsStorage(t *testing.T) {
 			_, _, err = imgStore.CheckBlob(testImage, digest)
 			So(err, ShouldNotBeNil)
 
-			_, _, err = imgStore.StatBlob(testImage, digest)
+			_, _, _, err = imgStore.StatBlob(testImage, digest)
 			So(err, ShouldNotBeNil)
 		})
 
@@ -1050,7 +1052,7 @@ func TestNegativeCasesObjectsStorage(t *testing.T) {
 				WriterFn: func(ctx context.Context, path string, isAppend bool) (driver.FileWriter, error) {
 					return &FileWriterMock{WriteFn: func(b []byte) (int, error) {
 						return 0, errS3
-					}}, nil
+					}}, errS3
 				},
 			})
 			_, err := imgStore.PutBlobChunkStreamed(testImage, "uuid", io.NopCloser(strings.NewReader("")))
@@ -1091,7 +1093,7 @@ func TestNegativeCasesObjectsStorage(t *testing.T) {
 						WriteFn: func(b []byte) (int, error) {
 							return 0, errS3
 						},
-					}, nil
+					}, errS3
 				},
 			})
 			_, err := imgStore.PutBlobChunk(testImage, "uuid", 12, 100, io.NopCloser(strings.NewReader("")))
@@ -1280,7 +1282,7 @@ func TestS3Dedupe(t *testing.T) {
 		So(checkBlobSize1, ShouldBeGreaterThan, 0)
 		So(err, ShouldBeNil)
 
-		ok, checkBlobSize1, err = imgStore.StatBlob("dedupe1", digest)
+		ok, checkBlobSize1, _, err = imgStore.StatBlob("dedupe1", digest)
 		So(ok, ShouldBeTrue)
 		So(checkBlobSize1, ShouldBeGreaterThan, 0)
 		So(err, ShouldBeNil)
@@ -1466,12 +1468,12 @@ func TestS3Dedupe(t *testing.T) {
 		Convey("Check backward compatibility - switch dedupe to false", func() {
 			/* copy cache to the new storage with dedupe false (doing this because we
 			already have a cache object holding the lock on cache db file) */
-			input, err := os.ReadFile(path.Join(tdir, s3.CacheDBName+storageConstants.DBExtensionName))
+			input, err := os.ReadFile(path.Join(tdir, storageConstants.BoltdbName+storageConstants.DBExtensionName))
 			So(err, ShouldBeNil)
 
 			tdir = t.TempDir()
 
-			err = os.WriteFile(path.Join(tdir, s3.CacheDBName+storageConstants.DBExtensionName), input, 0o600)
+			err = os.WriteFile(path.Join(tdir, storageConstants.BoltdbName+storageConstants.DBExtensionName), input, 0o600)
 			So(err, ShouldBeNil)
 
 			storeDriver, imgStore, _ := createObjectsStore(testDir, tdir, false)
@@ -3306,6 +3308,7 @@ func TestS3ManifestImageIndex(t *testing.T) {
 
 				err = imgStore.DeleteImageManifest("index", "test:index1", false)
 				So(err, ShouldBeNil)
+
 				_, _, _, err = imgStore.GetImageManifest("index", "test:index1")
 				So(err, ShouldNotBeNil)
 
@@ -3599,7 +3602,7 @@ func TestS3DedupeErr(t *testing.T) {
 
 		imgStore = createMockStorage(testDir, tdir, true, &StorageDriverMock{})
 
-		err = os.Remove(path.Join(tdir, s3.CacheDBName+storageConstants.DBExtensionName))
+		err = os.Remove(path.Join(tdir, storageConstants.BoltdbName+storageConstants.DBExtensionName))
 		digest := godigest.NewDigestFromEncoded(godigest.SHA256, "digest")
 
 		// trigger unable to insert blob record
@@ -3640,8 +3643,9 @@ func TestS3DedupeErr(t *testing.T) {
 		err := imgStore.DedupeBlob("", digest, "dst")
 		So(err, ShouldBeNil)
 
+		// error will be triggered in driver.SameFile()
 		err = imgStore.DedupeBlob("", digest, "dst2")
-		So(err, ShouldNotBeNil)
+		So(err, ShouldBeNil)
 	})
 
 	Convey("Test DedupeBlob - error on store.PutContent()", t, func(c C) {
@@ -3776,12 +3780,12 @@ func TestS3DedupeErr(t *testing.T) {
 		So(err, ShouldBeNil)
 
 		// copy cache db to the new imagestore
-		input, err := os.ReadFile(path.Join(tdir, s3.CacheDBName+storageConstants.DBExtensionName))
+		input, err := os.ReadFile(path.Join(tdir, storageConstants.BoltdbName+storageConstants.DBExtensionName))
 		So(err, ShouldBeNil)
 
 		tdir = t.TempDir()
 
-		err = os.WriteFile(path.Join(tdir, s3.CacheDBName+storageConstants.DBExtensionName), input, 0o600)
+		err = os.WriteFile(path.Join(tdir, storageConstants.BoltdbName+storageConstants.DBExtensionName), input, 0o600)
 		So(err, ShouldBeNil)
 
 		imgStore = createMockStorage(testDir, tdir, true, &StorageDriverMock{
@@ -3797,12 +3801,14 @@ func TestS3DedupeErr(t *testing.T) {
 		_, _, err = imgStore.GetBlob("repo2", digest, "application/vnd.oci.image.layer.v1.tar+gzip")
 		So(err, ShouldNotBeNil)
 
+		// now it should move content from /repo1/dst1 to /repo2/dst2
 		_, err = imgStore.GetBlobContent("repo2", digest)
-		So(err, ShouldNotBeNil)
+		So(err, ShouldBeNil)
 
-		_, _, err = imgStore.StatBlob("repo2", digest)
-		So(err, ShouldNotBeNil)
+		_, _, _, err = imgStore.StatBlob("repo2", digest)
+		So(err, ShouldBeNil)
 
+		// it errors out because of bad range, as mock store returns a driver.FileInfo with 0 size
 		_, _, _, err = imgStore.GetBlobPartial("repo2", digest, "application/vnd.oci.image.layer.v1.tar+gzip", 0, 1)
 		So(err, ShouldNotBeNil)
 	})
@@ -3822,12 +3828,12 @@ func TestS3DedupeErr(t *testing.T) {
 		So(err, ShouldBeNil)
 
 		// copy cache db to the new imagestore
-		input, err := os.ReadFile(path.Join(tdir, s3.CacheDBName+storageConstants.DBExtensionName))
+		input, err := os.ReadFile(path.Join(tdir, storageConstants.BoltdbName+storageConstants.DBExtensionName))
 		So(err, ShouldBeNil)
 
 		tdir = t.TempDir()
 
-		err = os.WriteFile(path.Join(tdir, s3.CacheDBName+storageConstants.DBExtensionName), input, 0o600)
+		err = os.WriteFile(path.Join(tdir, storageConstants.BoltdbName+storageConstants.DBExtensionName), input, 0o600)
 		So(err, ShouldBeNil)
 
 		imgStore = createMockStorage(testDir, tdir, true, &StorageDriverMock{
@@ -3887,7 +3893,7 @@ func TestS3DedupeErr(t *testing.T) {
 		_, err = imgStore.GetBlobContent("repo2", digest)
 		So(err, ShouldNotBeNil)
 
-		_, _, err = imgStore.StatBlob("repo2", digest)
+		_, _, _, err = imgStore.StatBlob("repo2", digest)
 		So(err, ShouldNotBeNil)
 
 		_, _, _, err = imgStore.GetBlobPartial("repo2", digest, "application/vnd.oci.image.layer.v1.tar+gzip", 0, 1)

pkg/storage/scrub_test.go

@@ -39,8 +39,8 @@ func TestCheckAllBlobsIntegrity(t *testing.T) {
 		Name:        "cache",
 		UseRelPaths: true,
 	}, log)
-	imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay,
-		true, true, log, metrics, nil, cacheDriver)
+	imgStore := local.NewImageStore(dir, true, true, storageConstants.DefaultGCDelay,
+		storageConstants.DefaultUntaggedImgeRetentionDelay, true, true, log, metrics, nil, cacheDriver)
 
 	Convey("Scrub only one repo", t, func(c C) {
 		// initialize repo
@@ -113,7 +113,7 @@ func TestCheckAllBlobsIntegrity(t *testing.T) {
 			// verify error message
 			So(actual, ShouldContainSubstring, "test 1.0 affected parse application/vnd.oci.image.manifest.v1+json")
 
-			index, err := common.GetIndex(imgStore, repoName, log.With().Caller().Logger())
+			index, err := common.GetIndex(imgStore, repoName, log)
 			So(err, ShouldBeNil)
 
 			So(len(index.Manifests), ShouldEqual, 1)
@@ -193,7 +193,7 @@ func TestCheckAllBlobsIntegrity(t *testing.T) {
 			err = os.Chmod(layerFile, 0x0200)
 			So(err, ShouldBeNil)
 
-			index, err := common.GetIndex(imgStore, repoName, log.With().Caller().Logger())
+			index, err := common.GetIndex(imgStore, repoName, log)
 			So(err, ShouldBeNil)
 
 			So(len(index.Manifests), ShouldEqual, 1)
@@ -327,7 +327,7 @@ func TestCheckAllBlobsIntegrity(t *testing.T) {
 			So(actual, ShouldContainSubstring, "test 1.0 affected")
 			So(actual, ShouldContainSubstring, "no such file or directory")
 
-			index, err := common.GetIndex(imgStore, repoName, log.With().Caller().Logger())
+			index, err := common.GetIndex(imgStore, repoName, log)
 			So(err, ShouldBeNil)
 
 			So(len(index.Manifests), ShouldEqual, 2)

pkg/storage/storage.go

@@ -51,8 +51,9 @@ func New(config *config.Config, linter common.Lint, metrics monitoring.MetricSer
 	if config.Storage.StorageDriver == nil {
 		// false positive lint - linter does not implement Lint method
 		//nolint:typecheck,contextcheck
-		defaultStore = local.NewImageStore(config.Storage.RootDirectory,
-			config.Storage.GC, config.Storage.GCDelay,
+		rootDir := config.Storage.RootDirectory
+		defaultStore = local.NewImageStore(rootDir,
+			config.Storage.GC, config.Storage.GCReferrers, config.Storage.GCDelay, config.Storage.UntaggedImageRetentionDelay,
 			config.Storage.Dedupe, config.Storage.Commit, log, metrics, linter,
 			CreateCacheDatabaseDriver(config.Storage.StorageConfig, log),
 		)
@@ -80,7 +81,8 @@ func New(config *config.Config, linter common.Lint, metrics monitoring.MetricSer
 		// false positive lint - linter does not implement Lint method
 		//nolint: typecheck,contextcheck
 		defaultStore = s3.NewImageStore(rootDir, config.Storage.RootDirectory,
-			config.Storage.GC, config.Storage.GCDelay, config.Storage.Dedupe,
+			config.Storage.GC, config.Storage.GCReferrers, config.Storage.GCDelay,
+			config.Storage.UntaggedImageRetentionDelay, config.Storage.Dedupe,
 			config.Storage.Commit, log, metrics, linter, store,
 			CreateCacheDatabaseDriver(config.Storage.StorageConfig, log))
 	}
@@ -152,9 +154,13 @@ func getSubStore(cfg *config.Config, subPaths map[string]config.StorageConfig,
 			// add it to uniqueSubFiles
 			// Create a new image store and assign it to imgStoreMap
 			if isUnique {
-				imgStoreMap[storageConfig.RootDirectory] = local.NewImageStore(storageConfig.RootDirectory,
-					storageConfig.GC, storageConfig.GCDelay, storageConfig.Dedupe,
-					storageConfig.Commit, log, metrics, linter, CreateCacheDatabaseDriver(storageConfig, log))
+				rootDir := storageConfig.RootDirectory
+				imgStoreMap[storageConfig.RootDirectory] = local.NewImageStore(rootDir,
+					storageConfig.GC, storageConfig.GCReferrers, storageConfig.GCDelay,
+					storageConfig.UntaggedImageRetentionDelay, storageConfig.Dedupe,
+					storageConfig.Commit, log, metrics, linter,
+					CreateCacheDatabaseDriver(storageConfig, log),
+				)
 
 				subImageStore[route] = imgStoreMap[storageConfig.RootDirectory]
 			}
@@ -183,8 +189,9 @@ func getSubStore(cfg *config.Config, subPaths map[string]config.StorageConfig,
 			// false positive lint - linter does not implement Lint method
 			//nolint: typecheck
 			subImageStore[route] = s3.NewImageStore(rootDir, storageConfig.RootDirectory,
-				storageConfig.GC, storageConfig.GCDelay,
-				storageConfig.Dedupe, storageConfig.Commit, log, metrics, linter, store,
+				storageConfig.GC, storageConfig.GCReferrers, storageConfig.GCDelay,
+				storageConfig.UntaggedImageRetentionDelay, storageConfig.Dedupe,
+				storageConfig.Commit, log, metrics, linter, store,
 				CreateCacheDatabaseDriver(storageConfig, log),
 			)
 		}

pkg/storage/types/types.go

@@ -4,6 +4,7 @@ import (
 	"io"
 	"time"
 
+	storagedriver "github.com/docker/distribution/registry/storage/driver"
 	godigest "github.com/opencontainers/go-digest"
 	ispec "github.com/opencontainers/image-spec/specs-go/v1"
 	artifactspec "github.com/oras-project/artifacts-spec/specs-go/v1"
@@ -38,7 +39,7 @@ type ImageStore interface { //nolint:interfacebloat
 	DeleteBlobUpload(repo, uuid string) error
 	BlobPath(repo string, digest godigest.Digest) string
 	CheckBlob(repo string, digest godigest.Digest) (bool, int64, error)
-	StatBlob(repo string, digest godigest.Digest) (bool, int64, error)
+	StatBlob(repo string, digest godigest.Digest) (bool, int64, time.Time, error)
 	GetBlob(repo string, digest godigest.Digest, mediaType string) (io.ReadCloser, int64, error)
 	GetBlobPartial(repo string, digest godigest.Digest, mediaType string, from, to int64,
 	) (io.ReadCloser, int64, int64, error)
@@ -52,4 +53,22 @@ type ImageStore interface { //nolint:interfacebloat
 	RunDedupeBlobs(interval time.Duration, sch *scheduler.Scheduler)
 	RunDedupeForDigest(digest godigest.Digest, dedupe bool, duplicateBlobs []string) error
 	GetNextDigestWithBlobPaths(lastDigests []godigest.Digest) (godigest.Digest, []string, error)
+	GetAllBlobs(repo string) ([]string, error)
+}
+
+type Driver interface { //nolint:interfacebloat
+	Name() string
+	EnsureDir(path string) error
+	DirExists(path string) bool
+	Reader(path string, offset int64) (io.ReadCloser, error)
+	ReadFile(path string) ([]byte, error)
+	Delete(path string) error
+	Stat(path string) (storagedriver.FileInfo, error)
+	Writer(filepath string, append bool) (storagedriver.FileWriter, error) //nolint: predeclared
+	WriteFile(filepath string, content []byte) (int, error)
+	Walk(path string, f storagedriver.WalkFn) error
+	List(fullpath string) ([]string, error)
+	Move(sourcePath string, destPath string) error
+	SameFile(path1, path2 string) bool
+	Link(src, dest string) error
 }