fix: deduplicate entries in referrers responses (#3524)

See: https://github.com/project-zot/zot/issues/2506

Note we are not loosing anything functionality-wise by making this change.
Initially we considered the tags are in the annotations present in the referrers
but the only annotations we set on referrers are the ones inside the manifests themselves,
not the ones in the manifest descriptors, so the tags were not presetn anyway.

Signed-off-by: Andrei Aaron <andreifdaaron@gmail.com>

pkg/meta/boltdb/boltdb.go

@@ -1187,12 +1187,19 @@ func (bdw *BoltDB) GetReferrersInfo(repo string, referredDigest godigest.Digest,
 		}
 
 		referrersInfo := protoRepoMeta.Referrers[referredDigest.String()].List
+		seenDigests := make(map[string]struct{})
 
 		for i := range referrersInfo {
 			if !common.MatchesArtifactTypes(referrersInfo[i].ArtifactType, artifactTypes) {
 				continue
 			}
 
+			if _, seen := seenDigests[referrersInfo[i].Digest]; seen {
+				continue
+			}
+
+			seenDigests[referrersInfo[i].Digest] = struct{}{}
+
 			referrersInfoResult = append(referrersInfoResult, mTypes.ReferrerInfo{
 				Digest:       referrersInfo[i].Digest,
 				MediaType:    referrersInfo[i].MediaType,

pkg/meta/dynamodb/dynamodb.go

@@ -1117,12 +1117,19 @@ func (dwr *DynamoDB) GetReferrersInfo(repo string, referredDigest godigest.Diges
 	referrersInfo := repoMeta.Referrers[referredDigest.String()]
 
 	filteredResults := make([]mTypes.ReferrerInfo, 0, len(referrersInfo))
+	seenDigests := make(map[string]struct{})
 
 	for _, referrerInfo := range referrersInfo {
 		if !common.MatchesArtifactTypes(referrerInfo.ArtifactType, artifactTypes) {
 			continue
 		}
 
+		if _, seen := seenDigests[referrerInfo.Digest]; seen {
+			continue
+		}
+
+		seenDigests[referrerInfo.Digest] = struct{}{}
+
 		filteredResults = append(filteredResults, referrerInfo)
 	}
 

pkg/meta/meta_test.go

@@ -2410,6 +2410,58 @@ func RunMetaDBTests(t *testing.T, metaDB mTypes.MetaDB, preparationFuncs ...func
 			So(referrerInfo[0].Digest, ShouldResemble, referrerWantedType.DigestStr())
 		})
 
+		Convey("GetReferrersInfo deduplication", func() {
+			image := CreateRandomImage()
+			referrer := CreateRandomImageWith().Subject(image.DescriptorRef()).Build()
+
+			err := metaDB.SetRepoReference(ctx, "repo", "tag", image.AsImageMeta())
+			So(err, ShouldBeNil)
+
+			// Add same referrer multiple times with different tags (simulating duplicates)
+			err = metaDB.SetRepoReference(ctx, "repo", "ref-tag1", referrer.AsImageMeta())
+			So(err, ShouldBeNil)
+
+			err = metaDB.SetRepoReference(ctx, "repo", "ref-tag2", referrer.AsImageMeta())
+			So(err, ShouldBeNil)
+
+			err = metaDB.SetRepoReference(ctx, "repo", referrer.DigestStr(), referrer.AsImageMeta())
+			So(err, ShouldBeNil)
+
+			// GetReferrersInfo should return only one instance despite multiple tags
+			referrers, err := metaDB.GetReferrersInfo("repo", image.Digest(), []string{})
+			So(err, ShouldBeNil)
+			So(len(referrers), ShouldEqual, 1)
+			So(referrers[0].Digest, ShouldEqual, referrer.DigestStr())
+
+			// Now manually add duplicate entries to test the deduplication logic in GetReferrersInfo
+			// This simulates a scenario where duplicates might exist (e.g., from migration or data corruption)
+			repoMeta, err := metaDB.GetRepoMeta(ctx, "repo")
+			So(err, ShouldBeNil)
+
+			// Add duplicate referrer entries manually
+			duplicateReferrer := mTypes.ReferrerInfo{
+				Digest:       referrer.DigestStr(),
+				MediaType:    referrer.Manifest.MediaType,
+				ArtifactType: zcommon.GetManifestArtifactType(referrer.Manifest),
+				Size:         int(referrer.ManifestDescriptor.Size),
+				Annotations:  referrer.Manifest.Annotations,
+			}
+			repoMeta.Referrers[image.Digest().String()] = append(
+				repoMeta.Referrers[image.Digest().String()],
+				duplicateReferrer,
+				duplicateReferrer,
+			)
+
+			err = metaDB.SetRepoMeta("repo", repoMeta)
+			So(err, ShouldBeNil)
+
+			// GetReferrersInfo should still return only one instance despite duplicates in the data
+			referrers, err = metaDB.GetReferrersInfo("repo", image.Digest(), []string{})
+			So(err, ShouldBeNil)
+			So(len(referrers), ShouldEqual, 1)
+			So(referrers[0].Digest, ShouldEqual, referrer.DigestStr())
+		})
+
 		Convey("FilterImageMeta", func() {
 			repo := "repo"
 			tag := "tag"

pkg/meta/redis/redis.go

@@ -1704,12 +1704,19 @@ func (rc *RedisDB) GetReferrersInfo(repo string, referredDigest godigest.Digest,
 	}
 
 	referrersInfo := protoRepoMeta.Referrers[referredDigest.String()].List
+	seenDigests := make(map[string]struct{})
 
 	for i := range referrersInfo {
 		if !common.MatchesArtifactTypes(referrersInfo[i].ArtifactType, artifactTypes) {
 			continue
 		}
 
+		if _, seen := seenDigests[referrersInfo[i].Digest]; seen {
+			continue
+		}
+
+		seenDigests[referrersInfo[i].Digest] = struct{}{}
+
 		referrersInfoResult = append(referrersInfoResult, mTypes.ReferrerInfo{
 			Digest:       referrersInfo[i].Digest,
 			MediaType:    referrersInfo[i].MediaType,