fix: deduplicate entries in referrers responses (#3524)

See: https://github.com/project-zot/zot/issues/2506

Note we are not loosing anything functionality-wise by making this change.
Initially we considered the tags are in the annotations present in the referrers
but the only annotations we set on referrers are the ones inside the manifests themselves,
not the ones in the manifest descriptors, so the tags were not presetn anyway.

Signed-off-by: Andrei Aaron <andreifdaaron@gmail.com>

pkg/storage/common/common.go

@@ -666,12 +666,21 @@ func GetReferrers(imgStore storageTypes.ImageStore, repo string, gdigest godiges
 	}
 
 	result := []ispec.Descriptor{}
+	seenDigests := make(map[godigest.Digest]struct{})
 
 	for _, descriptor := range index.Manifests {
 		if descriptor.Digest == gdigest {
 			continue
 		}
 
+		// Skip if we've already processed this digest
+		if _, seen := seenDigests[descriptor.Digest]; seen {
+			continue
+		}
+
+		// Mark as seen early to avoid processing duplicates
+		seenDigests[descriptor.Digest] = struct{}{}
+
 		buf, err := imgStore.GetBlobContent(repo, descriptor.Digest)
 		if err != nil {
 			log.Error().Err(err).Str("blob", imgStore.BlobPath(repo, descriptor.Digest)).Msg("failed to read manifest")

pkg/storage/common/common_test.go

@@ -373,6 +373,86 @@ func TestGetReferrersErrors(t *testing.T) {
 	})
 }
 
+func TestGetReferrersDeduplication(t *testing.T) {
+	Convey("Test GetReferrers deduplication", t, func(c C) {
+		dir := t.TempDir()
+
+		log := log.NewTestLogger()
+		metrics := monitoring.NewMetricsServer(false, log)
+
+		defer metrics.Stop() // Clean up metrics server to prevent resource leaks
+		cacheDriver, _ := storage.Create("boltdb", cache.BoltDBDriverParameters{
+			RootDir:     dir,
+			Name:        "cache",
+			UseRelPaths: true,
+		}, log)
+
+		imgStore := local.NewImageStore(dir, false, true, log, metrics, nil, cacheDriver, nil, nil)
+		storageCtlr := storage.StoreController{DefaultStore: imgStore}
+
+		// Create subject image
+		subjectImage := CreateDefaultImage()
+		err := WriteImageToFileSystem(subjectImage, "test-repo", "subject-tag", storageCtlr)
+		So(err, ShouldBeNil)
+
+		subjectDigest := subjectImage.Digest()
+
+		// Create referrer image using builder pattern
+		referrerImage := CreateImageWith().
+			DefaultLayers().
+			DefaultConfig().
+			Subject(subjectImage.DescriptorRef()).
+			Annotations(map[string]string{
+				"test": "referrer",
+			}).
+			Build()
+
+		// Write referrer image to filesystem (this will add it to index once)
+		err = WriteImageToFileSystem(referrerImage, "test-repo", referrerImage.DigestStr(), storageCtlr)
+		So(err, ShouldBeNil)
+
+		referrerDigest := referrerImage.Digest()
+
+		// Add referrer manifest to index multiple times (simulating tagging)
+		index, err := common.GetIndex(imgStore, "test-repo", log)
+		So(err, ShouldBeNil)
+
+		// Add same referrer with different tags (simulating duplicates)
+		desc1 := ispec.Descriptor{
+			MediaType: ispec.MediaTypeImageManifest,
+			Digest:    referrerDigest,
+			Size:      referrerImage.ManifestDescriptor.Size,
+			Annotations: map[string]string{
+				ispec.AnnotationRefName: "tag1",
+			},
+		}
+		desc2 := ispec.Descriptor{
+			MediaType: ispec.MediaTypeImageManifest,
+			Digest:    referrerDigest,
+			Size:      referrerImage.ManifestDescriptor.Size,
+			Annotations: map[string]string{
+				ispec.AnnotationRefName: "tag2",
+			},
+		}
+		desc3 := ispec.Descriptor{
+			MediaType: ispec.MediaTypeImageManifest,
+			Digest:    referrerDigest,
+			Size:      referrerImage.ManifestDescriptor.Size,
+		}
+
+		index.Manifests = append(index.Manifests, desc1, desc2, desc3)
+
+		err = imgStore.PutIndexContent("test-repo", index)
+		So(err, ShouldBeNil)
+
+		// Get referrers - should return only one instance despite multiple entries
+		referrers, err := common.GetReferrers(imgStore, "test-repo", subjectDigest, []string{}, log)
+		So(err, ShouldBeNil)
+		So(len(referrers.Manifests), ShouldEqual, 1)
+		So(referrers.Manifests[0].Digest, ShouldEqual, referrerDigest)
+	})
+}
+
 func TestGetImageIndexErrors(t *testing.T) {
 	log := log.NewTestLogger()