feat(repodb): update referrers api to use repodb (#1230)

Signed-off-by: Laurentiu Niculae <niculae.laurentiu1@gmail.com>

pkg/extensions/search/common/common_test.go

@@ -1008,6 +1008,7 @@ func TestGetReferrersGQL(t *testing.T) {
 			},
 			Subject:      subjectDescriptor,
 			ArtifactType: artifactType,
+			MediaType:    ispec.MediaTypeArtifactManifest,
 			Annotations: map[string]string{
 				"com.artifact.format": "test",
 			},
@@ -1017,14 +1018,15 @@ func TestGetReferrersGQL(t *testing.T) {
 		So(err, ShouldBeNil)
 		artifactManifestDigest := godigest.FromBytes(artifactManifestBlob)
 
-		err = UploadArtifact(baseURL, repo, artifact)
+		err = UploadArtifactManifest(artifact, baseURL, repo)
 		So(err, ShouldBeNil)
 
 		gqlQuery := `
-			{Referrers(
-				repo: "%s",
-				digest: "%s",
-				type: ""
+			{
+				Referrers(
+					repo: "%s",
+					digest: "%s",
+					type: ""
 		   		){
 					ArtifactType,
 					Digest,
@@ -5356,20 +5358,15 @@ func TestRepoDBWhenReadingImages(t *testing.T) {
 
 func TestRepoDBWhenDeletingImages(t *testing.T) {
 	Convey("Setting up zot repo with test images", t, func() {
-		subpath := "/a"
-
 		dir := t.TempDir()
-		subDir := t.TempDir()
-
-		subRootDir := path.Join(subDir, subpath)
-
 		port := GetFreePort()
 		baseURL := GetBaseURL(port)
+
 		conf := config.New()
 		conf.HTTP.Port = port
 		conf.Storage.RootDirectory = dir
-		conf.Storage.SubPaths = make(map[string]config.StorageConfig)
-		conf.Storage.SubPaths[subpath] = config.StorageConfig{RootDirectory: subRootDir}
+		conf.Storage.GC = false
+
 		defaultVal := true
 		conf.Extensions = &extconf.ExtensionConfig{
 			Search: &extconf.SearchConfig{BaseConfig: extconf.BaseConfig{Enable: &defaultVal}},
@@ -5384,46 +5381,32 @@ func TestRepoDBWhenDeletingImages(t *testing.T) {
 		defer ctlrManager.StopServer()
 
 		// push test images to repo 1 image 1
-		config1, layers1, manifest1, err := GetImageComponents(100)
+		image1, err := GetRandomImage("1.0.1")
 		So(err, ShouldBeNil)
 
-		layersSize1 := 0
-		for _, l := range layers1 {
-			layersSize1 += len(l)
-		}
-
 		err = UploadImage(
-			Image{
-				Manifest:  manifest1,
-				Config:    config1,
-				Layers:    layers1,
-				Reference: "1.0.1",
-			},
+			image1,
 			baseURL,
 			"repo1",
 		)
 		So(err, ShouldBeNil)
 
 		// push test images to repo 1 image 2
-		config2, layers2, manifest2, err := GetImageComponents(200)
-		So(err, ShouldBeNil)
 		createdTime2 := time.Date(2009, 1, 1, 12, 0, 0, 0, time.UTC)
-		config2.History = append(config2.History, ispec.History{Created: &createdTime2})
-		manifest2, err = updateManifestConfig(manifest2, config2)
+		image2, err := GetImageWithConfig(ispec.Image{
+			Created: &createdTime2,
+			History: []ispec.History{
+				{
+					Created: &createdTime2,
+				},
+			},
+		})
 		So(err, ShouldBeNil)
 
-		layersSize2 := 0
-		for _, l := range layers2 {
-			layersSize2 += len(l)
-		}
+		image2.Reference = "1.0.2"
 
 		err = UploadImage(
-			Image{
-				Manifest:  manifest2,
-				Config:    config2,
-				Layers:    layers2,
-				Reference: "1.0.2",
-			},
+			image2,
 			baseURL,
 			"repo1",
 		)
@@ -5439,22 +5422,6 @@ func TestRepoDBWhenDeletingImages(t *testing.T) {
 						LastUpdated Size 
 					}
 				}
-				Repos {
-					Name LastUpdated Size
-					Platforms { Os Arch }
-					Vendors Score
-					NewestImage { 
-						RepoName Tag LastUpdated Size IsSigned
-						Manifests{
-							Platform { Os Arch }
-							LastUpdated Size 
-						}
-					}
-				}
-				Layers {
-					Digest
-					Size
-				}
 			}
 		}`
 
@@ -5627,7 +5594,7 @@ func TestRepoDBWhenDeletingImages(t *testing.T) {
 
 			So(sigManifestContent, ShouldNotBeZeroValue)
 			// check notation signature
-			manifest1Blob, err := json.Marshal(manifest1)
+			manifest1Blob, err := json.Marshal(image1.Manifest)
 			So(err, ShouldBeNil)
 			manifest1Digest := godigest.FromBytes(manifest1Blob)
 			So(sigManifestContent.Subject, ShouldNotBeNil)
@@ -5653,6 +5620,65 @@ func TestRepoDBWhenDeletingImages(t *testing.T) {
 			So(responseStruct.GlobalSearchResult.GlobalSearch.Images[0].IsSigned, ShouldBeFalse)
 		})
 
+		Convey("Delete a referrer", func() {
+			referredImageDigest, err := image1.Digest()
+			So(err, ShouldBeNil)
+
+			referrerImage, err := GetImageWithSubject(referredImageDigest, ispec.MediaTypeImageManifest)
+			So(err, ShouldBeNil)
+
+			err = UploadImage(
+				referrerImage,
+				baseURL,
+				"repo1",
+			)
+			So(err, ShouldBeNil)
+
+			// ------- check referrers for this image
+
+			query := fmt.Sprintf(`
+			{
+				Referrers(repo:"repo1", digest:"%s"){
+					MediaType
+					Digest
+				}
+			}`, referredImageDigest.String())
+
+			resp, err = resty.R().Get(baseURL + graphqlQueryPrefix + "?query=" + url.QueryEscape(query))
+			So(resp, ShouldNotBeNil)
+			So(err, ShouldBeNil)
+			So(resp.StatusCode(), ShouldEqual, 200)
+
+			responseStruct := &ReferrersResp{}
+
+			err = json.Unmarshal(resp.Body(), responseStruct)
+			So(err, ShouldBeNil)
+
+			So(len(responseStruct.ReferrersResult.Referrers), ShouldEqual, 1)
+			So(responseStruct.ReferrersResult.Referrers[0].Digest, ShouldResemble, referrerImage.Reference)
+
+			statusCode, err := DeleteImage("repo1", referrerImage.Reference, "badURL")
+			So(err, ShouldNotBeNil)
+			So(statusCode, ShouldEqual, -1)
+
+			// ------- Delete the referrer and see if it disappears from repoDB also
+			statusCode, err = DeleteImage("repo1", referrerImage.Reference, baseURL)
+			So(err, ShouldBeNil)
+			So(statusCode, ShouldEqual, http.StatusAccepted)
+
+			resp, err = resty.R().Get(baseURL + graphqlQueryPrefix + "?query=" + url.QueryEscape(query))
+			So(resp, ShouldNotBeNil)
+			So(err, ShouldBeNil)
+			So(resp.StatusCode(), ShouldEqual, 200)
+
+			responseStruct = &ReferrersResp{}
+
+			err = json.Unmarshal(resp.Body(), responseStruct)
+			So(err, ShouldBeNil)
+
+			So(len(responseStruct.ReferrersResult.Referrers), ShouldEqual, 0)
+		})
+
 		Convey("Deleting causes errors", func() {
 			Convey("error while backing up the manifest", func() {
 				ctlr.StoreController.DefaultStore = mocks.MockedImageStore{

pkg/extensions/search/convert/repodb.go

@@ -627,6 +627,22 @@ func RepoMeta2ExpandedRepoInfo(ctx context.Context, repoMeta repodb.RepoMetadata
 	return summary, imageSummaries
 }
 
+func StringMap2Annotations(strMap map[string]string) []*gql_generated.Annotation {
+	annotations := make([]*gql_generated.Annotation, 0, len(strMap))
+
+	for key, value := range strMap {
+		key := key
+		value := value
+
+		annotations = append(annotations, &gql_generated.Annotation{
+			Key:   &key,
+			Value: &value,
+		})
+	}
+
+	return annotations
+}
+
 func GetPreloads(ctx context.Context) map[string]bool {
 	if !graphql.HasOperationContext(ctx) {
 		return map[string]bool{}

pkg/extensions/search/cve/cve_test.go

@@ -317,7 +317,7 @@ func TestImageFormat(t *testing.T) {
 		})
 		So(err, ShouldBeNil)
 
-		err = repodb.SyncRepoDB(repoDB, storeController, log)
+		err = repodb.ParseStorage(repoDB, storeController, log)
 		So(err, ShouldBeNil)
 
 		cveInfo := cveinfo.NewCVEInfo(storeController, repoDB, "", log)

pkg/extensions/search/cve/trivy/scanner_internal_test.go

@@ -88,7 +88,7 @@ func TestMultipleStoragePath(t *testing.T) {
 		})
 		So(err, ShouldBeNil)
 
-		err = repodb.SyncRepoDB(repoDB, storeController, log)
+		err = repodb.ParseStorage(repoDB, storeController, log)
 		So(err, ShouldBeNil)
 
 		scanner := NewScanner(storeController, repoDB, "ghcr.io/project-zot/trivy-db", log)
@@ -178,7 +178,7 @@ func TestTrivyLibraryErrors(t *testing.T) {
 		})
 		So(err, ShouldBeNil)
 
-		err = repodb.SyncRepoDB(repoDB, storeController, log)
+		err = repodb.ParseStorage(repoDB, storeController, log)
 		So(err, ShouldBeNil)
 
 		scanner := NewScanner(storeController, repoDB, "ghcr.io/project-zot/trivy-db", log)
@@ -424,7 +424,7 @@ func TestDefaultTrivyDBUrl(t *testing.T) {
 		})
 		So(err, ShouldBeNil)
 
-		err = repodb.SyncRepoDB(repoDB, storeController, log)
+		err = repodb.ParseStorage(repoDB, storeController, log)
 		So(err, ShouldBeNil)
 
 		// Use empty string for DB repository, the default url would be used internally

pkg/extensions/search/resolver.go

@@ -1193,41 +1193,33 @@ func getImageList(ctx context.Context, repo string, repoDB repodb.RepoDB, cveInf
 	}, nil
 }
 
-func getReferrers(store storage.ImageStore, repoName string, digest string, artifactTypes []string, log log.Logger) (
-	[]*gql_generated.Referrer, error,
-) {
-	results := make([]*gql_generated.Referrer, 0)
+func getReferrers(repoDB repodb.RepoDB, repo string, referredDigest string, artifactTypes []string,
+	log log.Logger,
+) ([]*gql_generated.Referrer, error) {
+	refDigest := godigest.Digest(referredDigest)
+	if err := refDigest.Validate(); err != nil {
+		log.Error().Err(err).Msgf("graphql: bad digest string from request '%s'", referredDigest)
 
-	index, err := store.GetReferrers(repoName, godigest.Digest(digest), artifactTypes)
-	if err != nil {
-		log.Error().Err(err).Msg("error extracting referrers list")
-
-		return results, err
+		return []*gql_generated.Referrer{}, errors.Wrapf(err, "graphql: bad digest string from request '%s'",
+			referredDigest)
 	}
 
-	for _, manifest := range index.Manifests {
-		size := int(manifest.Size)
-		digest := manifest.Digest.String()
-		annotations := make([]*gql_generated.Annotation, 0)
-		artifactType := manifest.ArtifactType
-		mediaType := manifest.MediaType
+	referrers, err := repoDB.GetFilteredReferrersInfo(repo, refDigest, artifactTypes)
+	if err != nil {
+		return nil, err
+	}
 
-		for k, v := range manifest.Annotations {
-			key := k
-			value := v
+	results := make([]*gql_generated.Referrer, 0, len(referrers))
 
-			annotations = append(annotations, &gql_generated.Annotation{
-				Key:   &key,
-				Value: &value,
-			})
-		}
+	for _, referrer := range referrers {
+		referrer := referrer
 
 		results = append(results, &gql_generated.Referrer{
-			MediaType:    &mediaType,
-			ArtifactType: &artifactType,
-			Digest:       &digest,
-			Size:         &size,
-			Annotations:  annotations,
+			MediaType:    &referrer.MediaType,
+			ArtifactType: &referrer.ArtifactType,
+			Digest:       &referrer.Digest,
+			Size:         &referrer.Size,
+			Annotations:  convert.StringMap2Annotations(referrer.Annotations),
 		})
 	}
 

pkg/extensions/search/resolver_test.go

@@ -1362,15 +1362,25 @@ func TestImageList(t *testing.T) {
 
 func TestGetReferrers(t *testing.T) {
 	Convey("getReferrers", t, func() {
+		referredDigest := godigest.FromString("t").String()
+
+		Convey("referredDigest is empty", func() {
+			testLogger := log.NewLogger("debug", "")
+
+			_, err := getReferrers(mocks.RepoDBMock{}, "test", "", nil, testLogger)
+			So(err, ShouldNotBeNil)
+		})
+
 		Convey("GetReferrers returns error", func() {
 			testLogger := log.NewLogger("debug", "")
-			mockedStore := mocks.MockedImageStore{
-				GetReferrersFn: func(repo string, digest godigest.Digest, artifactType []string) (ispec.Index, error) {
-					return ispec.Index{}, ErrTestError
+			mockedStore := mocks.RepoDBMock{
+				GetFilteredReferrersInfoFn: func(repo string, referredDigest godigest.Digest, artifactTypes []string,
+				) ([]repodb.ReferrerInfo, error) {
+					return nil, ErrTestError
 				},
 			}
 
-			_, err := getReferrers(mockedStore, "test", "", nil, testLogger)
+			_, err := getReferrers(mockedStore, "test", referredDigest, nil, testLogger)
 			So(err, ShouldNotBeNil)
 		})
 
@@ -1385,17 +1395,22 @@ func TestGetReferrers(t *testing.T) {
 					"key": "value",
 				},
 			}
-			mockedStore := mocks.MockedImageStore{
-				GetReferrersFn: func(repo string, digest godigest.Digest, artifactTypes []string) (ispec.Index, error) {
-					return ispec.Index{
-						Manifests: []ispec.Descriptor{
-							referrerDescriptor,
+			mockedStore := mocks.RepoDBMock{
+				GetFilteredReferrersInfoFn: func(repo string, referredDigest godigest.Digest, artifactTypes []string,
+				) ([]repodb.ReferrerInfo, error) {
+					return []repodb.ReferrerInfo{
+						{
+							Digest:       referrerDescriptor.Digest.String(),
+							MediaType:    referrerDescriptor.MediaType,
+							ArtifactType: referrerDescriptor.ArtifactType,
+							Size:         int(referrerDescriptor.Size),
+							Annotations:  referrerDescriptor.Annotations,
 						},
 					}, nil
 				},
 			}
 
-			referrers, err := getReferrers(mockedStore, "test", "", nil, testLogger)
+			referrers, err := getReferrers(mockedStore, "test", referredDigest, nil, testLogger)
 			So(err, ShouldBeNil)
 			So(*referrers[0].ArtifactType, ShouldEqual, referrerDescriptor.ArtifactType)
 			So(*referrers[0].MediaType, ShouldEqual, referrerDescriptor.MediaType)

pkg/extensions/search/schema.resolvers.go

@@ -130,9 +130,7 @@ func (r *queryResolver) Image(ctx context.Context, image string) (*gql_generated
 
 // Referrers is the resolver for the Referrers field.
 func (r *queryResolver) Referrers(ctx context.Context, repo string, digest string, typeArg []string) ([]*gql_generated.Referrer, error) {
-	store := r.storeController.GetImageStore(repo)
-
-	referrers, err := getReferrers(store, repo, digest, typeArg, r.log)
+	referrers, err := getReferrers(r.repoDB, repo, digest, typeArg, r.log)
 	if err != nil {
 		r.log.Error().Err(err).Msg("unable to get referrers from default store")