Merge pull request from GHSA-55r9-5mx9-qq7r

when a client pushes an image zot's inline dedupe
will try to find the blob path corresponding with the blob digest
that it's currently pushed and if it's found in the cache
then zot will make a symbolic link to that cache entry and report
to the client that the blob already exists on the location.

Before this patch authorization was not applied on this process meaning
that a user could copy blobs without having permissions on the source repo.

Added a rule which says that the client should have read permissions on the source repo
before deduping, otherwise just Stat() the blob and return the corresponding status code.

Signed-off-by: Petu Eusebiu <peusebiu@cisco.com>
Co-authored-by: Petu Eusebiu <peusebiu@cisco.com>

pkg/storage/cache/boltdb.go

@@ -174,6 +174,57 @@ func (d *BoltDBDriver) PutBlob(digest godigest.Digest, path string) error {
 	return nil
 }
 
+func (d *BoltDBDriver) GetAllBlobs(digest godigest.Digest) ([]string, error) {
+	var blobPath strings.Builder
+
+	blobPaths := []string{}
+
+	if err := d.db.View(func(tx *bbolt.Tx) error {
+		root := tx.Bucket([]byte(constants.BlobsCache))
+		if root == nil {
+			// this is a serious failure
+			err := zerr.ErrCacheRootBucket
+			d.log.Error().Err(err).Msg("failed to access root bucket")
+
+			return err
+		}
+
+		bucket := root.Bucket([]byte(digest.String()))
+		if bucket != nil {
+			origin := bucket.Bucket([]byte(constants.OriginalBucket))
+			blobPath.Write(d.getOne(origin))
+			originBlob := blobPath.String()
+
+			blobPaths = append(blobPaths, originBlob)
+
+			deduped := bucket.Bucket([]byte(constants.DuplicatesBucket))
+			if deduped != nil {
+				cursor := deduped.Cursor()
+
+				for k, _ := cursor.First(); k != nil; k, _ = cursor.Next() {
+					var blobPath strings.Builder
+
+					blobPath.Write(k)
+
+					duplicateBlob := blobPath.String()
+
+					if duplicateBlob != originBlob {
+						blobPaths = append(blobPaths, duplicateBlob)
+					}
+				}
+
+				return nil
+			}
+		}
+
+		return zerr.ErrCacheMiss
+	}); err != nil {
+		return nil, err
+	}
+
+	return blobPaths, nil
+}
+
 func (d *BoltDBDriver) GetBlob(digest godigest.Digest) (string, error) {
 	var blobPath strings.Builder
 

pkg/storage/cache/boltdb_test.go

@@ -122,4 +122,47 @@ func TestBoltDBCache(t *testing.T) {
 		So(err, ShouldNotBeNil)
 		So(val, ShouldBeEmpty)
 	})
+
+	Convey("Test cache.GetAllBlos()", t, func() {
+		dir := t.TempDir()
+
+		log := log.NewLogger("debug", "")
+		So(log, ShouldNotBeNil)
+
+		_, err := storage.Create("boltdb", "failTypeAssertion", log)
+		So(err, ShouldNotBeNil)
+
+		cacheDriver, _ := storage.Create("boltdb", cache.BoltDBDriverParameters{dir, "cache_test", false}, log)
+		So(cacheDriver, ShouldNotBeNil)
+
+		err = cacheDriver.PutBlob("digest", "first")
+		So(err, ShouldBeNil)
+
+		err = cacheDriver.PutBlob("digest", "second")
+		So(err, ShouldBeNil)
+
+		err = cacheDriver.PutBlob("digest", "third")
+		So(err, ShouldBeNil)
+
+		blobs, err := cacheDriver.GetAllBlobs("digest")
+		So(err, ShouldBeNil)
+
+		So(blobs, ShouldResemble, []string{"first", "second", "third"})
+
+		err = cacheDriver.DeleteBlob("digest", "first")
+		So(err, ShouldBeNil)
+
+		blobs, err = cacheDriver.GetAllBlobs("digest")
+		So(err, ShouldBeNil)
+
+		So(blobs, ShouldResemble, []string{"second", "third"})
+
+		err = cacheDriver.DeleteBlob("digest", "third")
+		So(err, ShouldBeNil)
+
+		blobs, err = cacheDriver.GetAllBlobs("digest")
+		So(err, ShouldBeNil)
+
+		So(blobs, ShouldResemble, []string{"second"})
+	})
 }

pkg/storage/cache/cacheinterface.go

@@ -11,6 +11,8 @@ type Cache interface {
 	// Retrieves the blob matching provided digest.
 	GetBlob(digest godigest.Digest) (string, error)
 
+	GetAllBlobs(digest godigest.Digest) ([]string, error)
+
 	// Uploads blob to cachedb.
 	PutBlob(digest godigest.Digest, path string) error
 

pkg/storage/cache/dynamodb.go

@@ -141,6 +141,42 @@ func (d *DynamoDBDriver) GetBlob(digest godigest.Digest) (string, error) {
 	return out.OriginalBlobPath, nil
 }
 
+func (d *DynamoDBDriver) GetAllBlobs(digest godigest.Digest) ([]string, error) {
+	blobPaths := []string{}
+
+	resp, err := d.client.GetItem(context.TODO(), &dynamodb.GetItemInput{
+		TableName: aws.String(d.tableName),
+		Key: map[string]types.AttributeValue{
+			"Digest": &types.AttributeValueMemberS{Value: digest.String()},
+		},
+	})
+	if err != nil {
+		d.log.Error().Err(err).Str("tableName", d.tableName).Msg("failed to get blob")
+
+		return nil, err
+	}
+
+	out := Blob{}
+
+	if resp.Item == nil {
+		d.log.Debug().Err(zerr.ErrCacheMiss).Str("digest", string(digest)).Msg("failed to find blob in cache")
+
+		return nil, zerr.ErrCacheMiss
+	}
+
+	_ = attributevalue.UnmarshalMap(resp.Item, &out)
+
+	blobPaths = append(blobPaths, out.OriginalBlobPath)
+
+	for _, item := range out.DuplicateBlobPath {
+		if item != out.OriginalBlobPath {
+			blobPaths = append(blobPaths, item)
+		}
+	}
+
+	return blobPaths, nil
+}
+
 func (d *DynamoDBDriver) PutBlob(digest godigest.Digest, path string) error {
 	if path == "" {
 		d.log.Error().Err(zerr.ErrEmptyValue).Str("digest", digest.String()).

pkg/storage/cache/dynamodb_test.go

@@ -144,6 +144,48 @@ func TestDynamoDB(t *testing.T) {
 		So(err, ShouldNotBeNil)
 		So(val, ShouldBeEmpty)
 	})
+
+	Convey("Test dynamoDB", t, func(c C) {
+		log := log.NewLogger("debug", "")
+
+		cacheDriver, err := storage.Create("dynamodb", cache.DynamoDBDriverParameters{
+			Endpoint:  os.Getenv("DYNAMODBMOCK_ENDPOINT"),
+			TableName: "BlobTable",
+			Region:    "us-east-2",
+		}, log)
+		So(cacheDriver, ShouldNotBeNil)
+		So(err, ShouldBeNil)
+
+		err = cacheDriver.PutBlob("digest", "first")
+		So(err, ShouldBeNil)
+
+		err = cacheDriver.PutBlob("digest", "second")
+		So(err, ShouldBeNil)
+
+		err = cacheDriver.PutBlob("digest", "third")
+		So(err, ShouldBeNil)
+
+		blobs, err := cacheDriver.GetAllBlobs("digest")
+		So(err, ShouldBeNil)
+
+		So(blobs, ShouldResemble, []string{"first", "second", "third"})
+
+		err = cacheDriver.DeleteBlob("digest", "first")
+		So(err, ShouldBeNil)
+
+		blobs, err = cacheDriver.GetAllBlobs("digest")
+		So(err, ShouldBeNil)
+
+		So(blobs, ShouldResemble, []string{"second", "third"})
+
+		err = cacheDriver.DeleteBlob("digest", "third")
+		So(err, ShouldBeNil)
+
+		blobs, err = cacheDriver.GetAllBlobs("digest")
+		So(err, ShouldBeNil)
+
+		So(blobs, ShouldResemble, []string{"second"})
+	})
 }
 
 func TestDynamoDBError(t *testing.T) {

pkg/storage/imagestore/imagestore.go

@@ -1114,6 +1114,37 @@ func (is *ImageStore) BlobPath(repo string, digest godigest.Digest) string {
 	return path.Join(is.rootDir, repo, "blobs", digest.Algorithm().String(), digest.Encoded())
 }
 
+func (is *ImageStore) GetAllDedupeReposCandidates(digest godigest.Digest) ([]string, error) {
+	var lockLatency time.Time
+
+	if err := digest.Validate(); err != nil {
+		return nil, err
+	}
+
+	is.RLock(&lockLatency)
+	defer is.RUnlock(&lockLatency)
+
+	blobsPaths, err := is.cache.GetAllBlobs(digest)
+	if err != nil {
+		return nil, err
+	}
+
+	repos := []string{}
+
+	for _, blobPath := range blobsPaths {
+		// these can be both full paths or relative paths depending on the cache options
+		if !is.cache.UsesRelativePaths() && path.IsAbs(blobPath) {
+			blobPath, _ = filepath.Rel(is.rootDir, blobPath)
+		}
+
+		blobsDirIndex := strings.LastIndex(blobPath, "/blobs/")
+
+		repos = append(repos, blobPath[:blobsDirIndex])
+	}
+
+	return repos, nil
+}
+
 /*
 	CheckBlob verifies a blob and returns true if the blob is correct
 

pkg/storage/storage_test.go

@@ -10,6 +10,7 @@ import (
 	"io"
 	"os"
 	"path"
+	"slices"
 	"strings"
 	"sync"
 	"testing"
@@ -132,6 +133,75 @@ func TestStorageNew(t *testing.T) {
 	})
 }
 
+func TestGetAllDedupeReposCandidates(t *testing.T) {
+	for _, testcase := range testCases {
+		testcase := testcase
+		t.Run(testcase.testCaseName, func(t *testing.T) {
+			var imgStore storageTypes.ImageStore
+			if testcase.storageType == storageConstants.S3StorageDriverName {
+				tskip.SkipS3(t)
+
+				uuid, err := guuid.NewV4()
+				if err != nil {
+					panic(err)
+				}
+
+				testDir := path.Join("/oci-repo-test", uuid.String())
+				tdir := t.TempDir()
+
+				var store driver.StorageDriver
+				store, imgStore, _ = createObjectsStore(testDir, tdir)
+				defer cleanupStorage(store, testDir)
+			} else {
+				dir := t.TempDir()
+
+				log := zlog.Logger{Logger: zerolog.New(os.Stdout)}
+				metrics := monitoring.NewMetricsServer(false, log)
+				cacheDriver, _ := storage.Create("boltdb", cache.BoltDBDriverParameters{
+					RootDir:     dir,
+					Name:        "cache",
+					UseRelPaths: true,
+				}, log)
+
+				driver := local.New(true)
+
+				imgStore = imagestore.NewImageStore(dir, dir, true, true, log, metrics, nil, driver, cacheDriver)
+			}
+
+			Convey("Push repos with deduped blobs", t, func(c C) {
+				repoNames := []string{
+					"first",
+					"second",
+					"repo/a",
+					"repo/a/b/c/d/e/f",
+					"repo/repo-b/blobs",
+					"foo/bar/baz",
+					"blobs/foo/bar/blobs",
+					"blobs",
+					"blobs/foo",
+				}
+
+				storeController := storage.StoreController{DefaultStore: imgStore}
+
+				image := CreateRandomImage()
+
+				for _, repoName := range repoNames {
+					err := WriteImageToFileSystem(image, repoName, tag, storeController)
+					So(err, ShouldBeNil)
+				}
+
+				randomBlobDigest := image.Manifest.Layers[0].Digest
+
+				repos, err := imgStore.GetAllDedupeReposCandidates(randomBlobDigest)
+				So(err, ShouldBeNil)
+				slices.Sort(repoNames)
+				slices.Sort(repos)
+				So(repoNames, ShouldResemble, repos)
+			})
+		})
+	}
+}
+
 func TestStorageAPIs(t *testing.T) {
 	for _, testcase := range testCases {
 		testcase := testcase

pkg/storage/types/types.go

@@ -63,6 +63,7 @@ type ImageStore interface { //nolint:interfacebloat
 	GetAllBlobs(repo string) ([]string, error)
 	PopulateStorageMetrics(interval time.Duration, sch *scheduler.Scheduler)
 	VerifyBlobDigestValue(repo string, digest godigest.Digest) error
+	GetAllDedupeReposCandidates(digest godigest.Digest) ([]string, error)
 }
 
 type Driver interface { //nolint:interfacebloat