diff --git a/.github/workflows/ci-cd.yml b/.github/workflows/ci-cd.yml index 5f907d67..1182be34 100644 --- a/.github/workflows/ci-cd.yml +++ b/.github/workflows/ci-cd.yml @@ -89,11 +89,10 @@ jobs: echo "Waiting for LocalStack startup..." # Wait 30 seconds for the LocalStack container localstack wait -t 30 # to become ready before timing out - echo "Startup complete" - + echo "Startup complete" + + aws --endpoint-url=http://localhost:4566 s3api create-bucket --bucket zot-storage --region us-east-2 --create-bucket-configuration="{\"LocationConstraint\": \"us-east-2\"}" aws dynamodb --endpoint-url http://localhost:4566 --region "us-east-2" create-table --table-name BlobTable --attribute-definitions AttributeName=Digest,AttributeType=S --key-schema AttributeName=Digest,KeyType=HASH --provisioned-throughput ReadCapacityUnits=10,WriteCapacityUnits=5 - aws dynamodb --endpoint-url http://localhost:4566 --region "us-east-2" create-table --table-name RepoMetadataTable --attribute-definitions AttributeName=RepoName,AttributeType=S --key-schema AttributeName=RepoName,KeyType=HASH --provisioned-throughput ReadCapacityUnits=10,WriteCapacityUnits=5 - aws dynamodb --endpoint-url http://localhost:4566 --region "us-east-2" create-table --table-name ManifestDataTable --attribute-definitions AttributeName=Digest,AttributeType=S --key-schema AttributeName=Digest,KeyType=HASH --provisioned-throughput ReadCapacityUnits=10,WriteCapacityUnits=5 env: AWS_ACCESS_KEY_ID: fake AWS_SECRET_ACCESS_KEY: fake diff --git a/.github/workflows/ecosystem-tools.yaml b/.github/workflows/ecosystem-tools.yaml index 1c03246f..d40d7245 100644 --- a/.github/workflows/ecosystem-tools.yaml +++ b/.github/workflows/ecosystem-tools.yaml @@ -33,6 +33,9 @@ jobs: make bin/skopeo sudo cp bin/skopeo /usr/bin skopeo -v + - name: Run referrers tests + run: | + make test-bats-referrers - name: Run push-pull tests run: | make test-push-pull diff --git a/Makefile b/Makefile index 98744df7..dae6e9a2 100644 --- a/Makefile +++ b/Makefile @@ -298,6 +298,11 @@ test-push-pull: binary check-skopeo $(BATS) $(REGCLIENT) $(ORAS) $(HELM) test-push-pull-verbose: binary check-skopeo $(BATS) $(BATS) --trace --verbose-run --print-output-on-failure --show-output-of-passing-tests test/blackbox/pushpull.bats +.PHONY: test-bats-referrers +test-bats-referrers: EXTENSIONS=search +test-bats-referrers: binary check-skopeo $(BATS) $(ORAS) + $(BATS) --trace --print-output-on-failure test/blackbox/referrers.bats + .PHONY: test-cloud-only test-cloud-only: binary check-skopeo $(BATS) $(BATS) --trace --print-output-on-failure test/blackbox/cloud-only.bats diff --git a/errors/errors.go b/errors/errors.go index d1099a0b..9e2228b8 100644 --- a/errors/errors.go +++ b/errors/errors.go @@ -63,6 +63,7 @@ var ( ErrManifestConflict = errors.New("manifest: multiple manifests found") ErrManifestMetaNotFound = errors.New("repodb: image metadata not found for given manifest reference") ErrManifestDataNotFound = errors.New("repodb: image data not found for given manifest digest") + ErrArtifactDataNotFound = errors.New("repodb: artifact data not found for given digest") ErrIndexDataNotFount = errors.New("repodb: index data not found for given digest") ErrRepoMetaNotFound = errors.New("repodb: repo metadata not found for given repo name") ErrTagMetaNotFound = errors.New("repodb: tag metadata not found for given repo and tag names") diff --git a/examples/config-dynamodb.json b/examples/config-dynamodb.json index 1134727a..b1ca64bc 100644 --- a/examples/config-dynamodb.json +++ b/examples/config-dynamodb.json @@ -20,6 +20,7 @@ "cacheTablename": "ZotBlobTable", "repoMetaTablename": "ZotRepoMetadataTable", "manifestDataTablename": "ZotManifestDataTable", + "artifactDataTablename": "ZotArtifactDataTable", "versionTablename": "ZotVersion" } }, diff --git a/pkg/api/controller.go b/pkg/api/controller.go index dfd734e2..c6a55807 100644 --- a/pkg/api/controller.go +++ b/pkg/api/controller.go @@ -512,7 +512,7 @@ func (c *Controller) InitRepoDB(reloadCtx context.Context) error { return err } - err = repodb.SyncRepoDB(driver, c.StoreController, c.Log) + err = repodb.ParseStorage(driver, c.StoreController, c.Log) if err != nil { return err } @@ -554,6 +554,9 @@ func getDynamoParams(cacheDriverConfig map[string]interface{}, log log.Logger) d indexDataTablename, ok := toStringIfOk(cacheDriverConfig, "indexdatatablename", log) allParametersOk = allParametersOk && ok + artifactDataTablename, ok := toStringIfOk(cacheDriverConfig, "artifactdatatablename", log) + allParametersOk = allParametersOk && ok + versionTablename, ok := toStringIfOk(cacheDriverConfig, "versiontablename", log) allParametersOk = allParametersOk && ok @@ -567,6 +570,7 @@ func getDynamoParams(cacheDriverConfig map[string]interface{}, log log.Logger) d RepoMetaTablename: repoMetaTablename, ManifestDataTablename: manifestDataTablename, IndexDataTablename: indexDataTablename, + ArtifactDataTablename: artifactDataTablename, VersionTablename: versionTablename, } } diff --git a/pkg/api/controller_test.go b/pkg/api/controller_test.go index fb8ec62e..06df2739 100644 --- a/pkg/api/controller_test.go +++ b/pkg/api/controller_test.go @@ -170,6 +170,7 @@ func TestCreateCacheDatabaseDriver(t *testing.T) { "cacheTablename": "BlobTable", "repoMetaTablename": "RepoMetadataTable", "manifestDataTablename": "ManifestDataTable", + "artifactDataTablename": "ArtifactDataTable", "versionTablename": "Version", } @@ -184,6 +185,7 @@ func TestCreateCacheDatabaseDriver(t *testing.T) { "cacheTablename": "BlobTable", "repoMetaTablename": "RepoMetadataTable", "manifestDataTablename": "ManifestDataTable", + "artifactDataTablename": "ArtifactDataTable", "versionTablename": "Version", } @@ -197,6 +199,7 @@ func TestCreateCacheDatabaseDriver(t *testing.T) { "cacheTablename": "BlobTable", "repoMetaTablename": "RepoMetadataTable", "manifestDataTablename": "ManifestDataTable", + "artifactDataTablename": "ArtifactDataTable", "versionTablename": "Version", } @@ -229,6 +232,7 @@ func TestCreateRepoDBDriver(t *testing.T) { "cachetablename": "BlobTable", "repometatablename": "RepoMetadataTable", "manifestdatatablename": "ManifestDataTable", + "artifactDataTablename": "ArtifactDataTable", } testFunc := func() { _, _ = api.CreateRepoDBDriver(conf.Storage.StorageConfig, log) } @@ -241,6 +245,7 @@ func TestCreateRepoDBDriver(t *testing.T) { "cachetablename": "", "repometatablename": "RepoMetadataTable", "manifestdatatablename": "ManifestDataTable", + "artifactDataTablename": "ArtifactDataTable", "versiontablename": 1, } @@ -2574,6 +2579,7 @@ func TestAuthorizationWithBasicAuth(t *testing.T) { manifestBlob := resp.Body() var manifest ispec.Manifest + err = json.Unmarshal(manifestBlob, &manifest) So(err, ShouldBeNil) diff --git a/pkg/extensions/search/common/common_test.go b/pkg/extensions/search/common/common_test.go index 99c0e5e9..358c4a17 100644 --- a/pkg/extensions/search/common/common_test.go +++ b/pkg/extensions/search/common/common_test.go @@ -1008,6 +1008,7 @@ func TestGetReferrersGQL(t *testing.T) { }, Subject: subjectDescriptor, ArtifactType: artifactType, + MediaType: ispec.MediaTypeArtifactManifest, Annotations: map[string]string{ "com.artifact.format": "test", }, @@ -1017,14 +1018,15 @@ func TestGetReferrersGQL(t *testing.T) { So(err, ShouldBeNil) artifactManifestDigest := godigest.FromBytes(artifactManifestBlob) - err = UploadArtifact(baseURL, repo, artifact) + err = UploadArtifactManifest(artifact, baseURL, repo) So(err, ShouldBeNil) gqlQuery := ` - {Referrers( - repo: "%s", - digest: "%s", - type: "" + { + Referrers( + repo: "%s", + digest: "%s", + type: "" ){ ArtifactType, Digest, @@ -5356,20 +5358,15 @@ func TestRepoDBWhenReadingImages(t *testing.T) { func TestRepoDBWhenDeletingImages(t *testing.T) { Convey("Setting up zot repo with test images", t, func() { - subpath := "/a" - dir := t.TempDir() - subDir := t.TempDir() - - subRootDir := path.Join(subDir, subpath) - port := GetFreePort() baseURL := GetBaseURL(port) + conf := config.New() conf.HTTP.Port = port conf.Storage.RootDirectory = dir - conf.Storage.SubPaths = make(map[string]config.StorageConfig) - conf.Storage.SubPaths[subpath] = config.StorageConfig{RootDirectory: subRootDir} + conf.Storage.GC = false + defaultVal := true conf.Extensions = &extconf.ExtensionConfig{ Search: &extconf.SearchConfig{BaseConfig: extconf.BaseConfig{Enable: &defaultVal}}, @@ -5384,46 +5381,32 @@ func TestRepoDBWhenDeletingImages(t *testing.T) { defer ctlrManager.StopServer() // push test images to repo 1 image 1 - config1, layers1, manifest1, err := GetImageComponents(100) + image1, err := GetRandomImage("1.0.1") So(err, ShouldBeNil) - layersSize1 := 0 - for _, l := range layers1 { - layersSize1 += len(l) - } - err = UploadImage( - Image{ - Manifest: manifest1, - Config: config1, - Layers: layers1, - Reference: "1.0.1", - }, + image1, baseURL, "repo1", ) So(err, ShouldBeNil) // push test images to repo 1 image 2 - config2, layers2, manifest2, err := GetImageComponents(200) - So(err, ShouldBeNil) createdTime2 := time.Date(2009, 1, 1, 12, 0, 0, 0, time.UTC) - config2.History = append(config2.History, ispec.History{Created: &createdTime2}) - manifest2, err = updateManifestConfig(manifest2, config2) + image2, err := GetImageWithConfig(ispec.Image{ + Created: &createdTime2, + History: []ispec.History{ + { + Created: &createdTime2, + }, + }, + }) So(err, ShouldBeNil) - layersSize2 := 0 - for _, l := range layers2 { - layersSize2 += len(l) - } + image2.Reference = "1.0.2" err = UploadImage( - Image{ - Manifest: manifest2, - Config: config2, - Layers: layers2, - Reference: "1.0.2", - }, + image2, baseURL, "repo1", ) @@ -5439,22 +5422,6 @@ func TestRepoDBWhenDeletingImages(t *testing.T) { LastUpdated Size } } - Repos { - Name LastUpdated Size - Platforms { Os Arch } - Vendors Score - NewestImage { - RepoName Tag LastUpdated Size IsSigned - Manifests{ - Platform { Os Arch } - LastUpdated Size - } - } - } - Layers { - Digest - Size - } } }` @@ -5627,7 +5594,7 @@ func TestRepoDBWhenDeletingImages(t *testing.T) { So(sigManifestContent, ShouldNotBeZeroValue) // check notation signature - manifest1Blob, err := json.Marshal(manifest1) + manifest1Blob, err := json.Marshal(image1.Manifest) So(err, ShouldBeNil) manifest1Digest := godigest.FromBytes(manifest1Blob) So(sigManifestContent.Subject, ShouldNotBeNil) @@ -5653,6 +5620,65 @@ func TestRepoDBWhenDeletingImages(t *testing.T) { So(responseStruct.GlobalSearchResult.GlobalSearch.Images[0].IsSigned, ShouldBeFalse) }) + Convey("Delete a referrer", func() { + referredImageDigest, err := image1.Digest() + So(err, ShouldBeNil) + + referrerImage, err := GetImageWithSubject(referredImageDigest, ispec.MediaTypeImageManifest) + So(err, ShouldBeNil) + + err = UploadImage( + referrerImage, + baseURL, + "repo1", + ) + So(err, ShouldBeNil) + + // ------- check referrers for this image + + query := fmt.Sprintf(` + { + Referrers(repo:"repo1", digest:"%s"){ + MediaType + Digest + } + }`, referredImageDigest.String()) + + resp, err = resty.R().Get(baseURL + graphqlQueryPrefix + "?query=" + url.QueryEscape(query)) + So(resp, ShouldNotBeNil) + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, 200) + + responseStruct := &ReferrersResp{} + + err = json.Unmarshal(resp.Body(), responseStruct) + So(err, ShouldBeNil) + + So(len(responseStruct.ReferrersResult.Referrers), ShouldEqual, 1) + So(responseStruct.ReferrersResult.Referrers[0].Digest, ShouldResemble, referrerImage.Reference) + + statusCode, err := DeleteImage("repo1", referrerImage.Reference, "badURL") + So(err, ShouldNotBeNil) + So(statusCode, ShouldEqual, -1) + + // ------- Delete the referrer and see if it disappears from repoDB also + statusCode, err = DeleteImage("repo1", referrerImage.Reference, baseURL) + So(err, ShouldBeNil) + So(statusCode, ShouldEqual, http.StatusAccepted) + + resp, err = resty.R().Get(baseURL + graphqlQueryPrefix + "?query=" + url.QueryEscape(query)) + So(resp, ShouldNotBeNil) + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, 200) + + responseStruct = &ReferrersResp{} + + err = json.Unmarshal(resp.Body(), responseStruct) + So(err, ShouldBeNil) + + So(len(responseStruct.ReferrersResult.Referrers), ShouldEqual, 0) + }) + Convey("Deleting causes errors", func() { Convey("error while backing up the manifest", func() { ctlr.StoreController.DefaultStore = mocks.MockedImageStore{ diff --git a/pkg/extensions/search/convert/repodb.go b/pkg/extensions/search/convert/repodb.go index f50b740d..019eb35e 100644 --- a/pkg/extensions/search/convert/repodb.go +++ b/pkg/extensions/search/convert/repodb.go @@ -627,6 +627,22 @@ func RepoMeta2ExpandedRepoInfo(ctx context.Context, repoMeta repodb.RepoMetadata return summary, imageSummaries } +func StringMap2Annotations(strMap map[string]string) []*gql_generated.Annotation { + annotations := make([]*gql_generated.Annotation, 0, len(strMap)) + + for key, value := range strMap { + key := key + value := value + + annotations = append(annotations, &gql_generated.Annotation{ + Key: &key, + Value: &value, + }) + } + + return annotations +} + func GetPreloads(ctx context.Context) map[string]bool { if !graphql.HasOperationContext(ctx) { return map[string]bool{} diff --git a/pkg/extensions/search/cve/cve_test.go b/pkg/extensions/search/cve/cve_test.go index d12bdf76..097535cc 100644 --- a/pkg/extensions/search/cve/cve_test.go +++ b/pkg/extensions/search/cve/cve_test.go @@ -317,7 +317,7 @@ func TestImageFormat(t *testing.T) { }) So(err, ShouldBeNil) - err = repodb.SyncRepoDB(repoDB, storeController, log) + err = repodb.ParseStorage(repoDB, storeController, log) So(err, ShouldBeNil) cveInfo := cveinfo.NewCVEInfo(storeController, repoDB, "", log) diff --git a/pkg/extensions/search/cve/trivy/scanner_internal_test.go b/pkg/extensions/search/cve/trivy/scanner_internal_test.go index c84d1f25..961adb6b 100644 --- a/pkg/extensions/search/cve/trivy/scanner_internal_test.go +++ b/pkg/extensions/search/cve/trivy/scanner_internal_test.go @@ -88,7 +88,7 @@ func TestMultipleStoragePath(t *testing.T) { }) So(err, ShouldBeNil) - err = repodb.SyncRepoDB(repoDB, storeController, log) + err = repodb.ParseStorage(repoDB, storeController, log) So(err, ShouldBeNil) scanner := NewScanner(storeController, repoDB, "ghcr.io/project-zot/trivy-db", log) @@ -178,7 +178,7 @@ func TestTrivyLibraryErrors(t *testing.T) { }) So(err, ShouldBeNil) - err = repodb.SyncRepoDB(repoDB, storeController, log) + err = repodb.ParseStorage(repoDB, storeController, log) So(err, ShouldBeNil) scanner := NewScanner(storeController, repoDB, "ghcr.io/project-zot/trivy-db", log) @@ -424,7 +424,7 @@ func TestDefaultTrivyDBUrl(t *testing.T) { }) So(err, ShouldBeNil) - err = repodb.SyncRepoDB(repoDB, storeController, log) + err = repodb.ParseStorage(repoDB, storeController, log) So(err, ShouldBeNil) // Use empty string for DB repository, the default url would be used internally diff --git a/pkg/extensions/search/resolver.go b/pkg/extensions/search/resolver.go index 428fad28..034d80b0 100644 --- a/pkg/extensions/search/resolver.go +++ b/pkg/extensions/search/resolver.go @@ -1193,41 +1193,33 @@ func getImageList(ctx context.Context, repo string, repoDB repodb.RepoDB, cveInf }, nil } -func getReferrers(store storage.ImageStore, repoName string, digest string, artifactTypes []string, log log.Logger) ( - []*gql_generated.Referrer, error, -) { - results := make([]*gql_generated.Referrer, 0) +func getReferrers(repoDB repodb.RepoDB, repo string, referredDigest string, artifactTypes []string, + log log.Logger, +) ([]*gql_generated.Referrer, error) { + refDigest := godigest.Digest(referredDigest) + if err := refDigest.Validate(); err != nil { + log.Error().Err(err).Msgf("graphql: bad digest string from request '%s'", referredDigest) - index, err := store.GetReferrers(repoName, godigest.Digest(digest), artifactTypes) - if err != nil { - log.Error().Err(err).Msg("error extracting referrers list") - - return results, err + return []*gql_generated.Referrer{}, errors.Wrapf(err, "graphql: bad digest string from request '%s'", + referredDigest) } - for _, manifest := range index.Manifests { - size := int(manifest.Size) - digest := manifest.Digest.String() - annotations := make([]*gql_generated.Annotation, 0) - artifactType := manifest.ArtifactType - mediaType := manifest.MediaType + referrers, err := repoDB.GetFilteredReferrersInfo(repo, refDigest, artifactTypes) + if err != nil { + return nil, err + } - for k, v := range manifest.Annotations { - key := k - value := v + results := make([]*gql_generated.Referrer, 0, len(referrers)) - annotations = append(annotations, &gql_generated.Annotation{ - Key: &key, - Value: &value, - }) - } + for _, referrer := range referrers { + referrer := referrer results = append(results, &gql_generated.Referrer{ - MediaType: &mediaType, - ArtifactType: &artifactType, - Digest: &digest, - Size: &size, - Annotations: annotations, + MediaType: &referrer.MediaType, + ArtifactType: &referrer.ArtifactType, + Digest: &referrer.Digest, + Size: &referrer.Size, + Annotations: convert.StringMap2Annotations(referrer.Annotations), }) } diff --git a/pkg/extensions/search/resolver_test.go b/pkg/extensions/search/resolver_test.go index 2bc82aa3..8b7d0a2d 100644 --- a/pkg/extensions/search/resolver_test.go +++ b/pkg/extensions/search/resolver_test.go @@ -1362,15 +1362,25 @@ func TestImageList(t *testing.T) { func TestGetReferrers(t *testing.T) { Convey("getReferrers", t, func() { + referredDigest := godigest.FromString("t").String() + + Convey("referredDigest is empty", func() { + testLogger := log.NewLogger("debug", "") + + _, err := getReferrers(mocks.RepoDBMock{}, "test", "", nil, testLogger) + So(err, ShouldNotBeNil) + }) + Convey("GetReferrers returns error", func() { testLogger := log.NewLogger("debug", "") - mockedStore := mocks.MockedImageStore{ - GetReferrersFn: func(repo string, digest godigest.Digest, artifactType []string) (ispec.Index, error) { - return ispec.Index{}, ErrTestError + mockedStore := mocks.RepoDBMock{ + GetFilteredReferrersInfoFn: func(repo string, referredDigest godigest.Digest, artifactTypes []string, + ) ([]repodb.ReferrerInfo, error) { + return nil, ErrTestError }, } - _, err := getReferrers(mockedStore, "test", "", nil, testLogger) + _, err := getReferrers(mockedStore, "test", referredDigest, nil, testLogger) So(err, ShouldNotBeNil) }) @@ -1385,17 +1395,22 @@ func TestGetReferrers(t *testing.T) { "key": "value", }, } - mockedStore := mocks.MockedImageStore{ - GetReferrersFn: func(repo string, digest godigest.Digest, artifactTypes []string) (ispec.Index, error) { - return ispec.Index{ - Manifests: []ispec.Descriptor{ - referrerDescriptor, + mockedStore := mocks.RepoDBMock{ + GetFilteredReferrersInfoFn: func(repo string, referredDigest godigest.Digest, artifactTypes []string, + ) ([]repodb.ReferrerInfo, error) { + return []repodb.ReferrerInfo{ + { + Digest: referrerDescriptor.Digest.String(), + MediaType: referrerDescriptor.MediaType, + ArtifactType: referrerDescriptor.ArtifactType, + Size: int(referrerDescriptor.Size), + Annotations: referrerDescriptor.Annotations, }, }, nil }, } - referrers, err := getReferrers(mockedStore, "test", "", nil, testLogger) + referrers, err := getReferrers(mockedStore, "test", referredDigest, nil, testLogger) So(err, ShouldBeNil) So(*referrers[0].ArtifactType, ShouldEqual, referrerDescriptor.ArtifactType) So(*referrers[0].MediaType, ShouldEqual, referrerDescriptor.MediaType) diff --git a/pkg/extensions/search/schema.resolvers.go b/pkg/extensions/search/schema.resolvers.go index 5d7f7f89..3e8fb880 100644 --- a/pkg/extensions/search/schema.resolvers.go +++ b/pkg/extensions/search/schema.resolvers.go @@ -130,9 +130,7 @@ func (r *queryResolver) Image(ctx context.Context, image string) (*gql_generated // Referrers is the resolver for the Referrers field. func (r *queryResolver) Referrers(ctx context.Context, repo string, digest string, typeArg []string) ([]*gql_generated.Referrer, error) { - store := r.storeController.GetImageStore(repo) - - referrers, err := getReferrers(store, repo, digest, typeArg, r.log) + referrers, err := getReferrers(r.repoDB, repo, digest, typeArg, r.log) if err != nil { r.log.Error().Err(err).Msg("unable to get referrers from default store") diff --git a/pkg/meta/repodb/boltdb-wrapper/boltdb_wrapper.go b/pkg/meta/repodb/boltdb-wrapper/boltdb_wrapper.go index 304c200d..7be7763a 100644 --- a/pkg/meta/repodb/boltdb-wrapper/boltdb_wrapper.go +++ b/pkg/meta/repodb/boltdb-wrapper/boltdb_wrapper.go @@ -61,6 +61,11 @@ func NewBoltDBWrapper(params DBParameters) (*DBWrapper, error) { return err } + _, err = transaction.CreateBucketIfNotExists([]byte(repodb.ArtifactDataBucket)) + if err != nil { + return err + } + _, err = transaction.CreateBucketIfNotExists([]byte(repodb.RepoMetadataBucket)) if err != nil { return err @@ -133,6 +138,7 @@ func (bdw *DBWrapper) SetManifestMeta(repo string, manifestDigest godigest.Diges Tags: map[string]repodb.Descriptor{}, Statistics: map[string]repodb.DescriptorStatistics{}, Signatures: map[string]repodb.ManifestSignatures{}, + Referrers: map[string][]repodb.Descriptor{}, } repoMetaBlob := repoBuck.Get([]byte(repo)) @@ -259,6 +265,290 @@ func (bdw *DBWrapper) GetIndexData(indexDigest godigest.Digest) (repodb.IndexDat return indexMetadata, err } +func (bdw DBWrapper) SetArtifactData(artifactDigest godigest.Digest, artifactData repodb.ArtifactData) error { + err := bdw.DB.Update(func(tx *bolt.Tx) error { + buck := tx.Bucket([]byte(repodb.ArtifactDataBucket)) + + imBlob, err := json.Marshal(artifactData) + if err != nil { + return errors.Wrapf(err, "repodb: error while calculating blob for artifact with digest %s", artifactDigest) + } + + err = buck.Put([]byte(artifactDigest), imBlob) + if err != nil { + return errors.Wrapf(err, "repodb: error while setting artifact blob for digest %s", artifactDigest) + } + + return nil + }) + + return err +} + +func (bdw DBWrapper) GetArtifactData(artifactDigest godigest.Digest) (repodb.ArtifactData, error) { + var artifactData repodb.ArtifactData + + err := bdw.DB.View(func(tx *bolt.Tx) error { + buck := tx.Bucket([]byte(repodb.ArtifactDataBucket)) + + blob := buck.Get([]byte(artifactDigest)) + + if len(blob) == 0 { + return zerr.ErrArtifactDataNotFound + } + + err := json.Unmarshal(blob, &artifactData) + if err != nil { + return errors.Wrapf(err, "repodb: error while unmashaling artifact data for digest %s", artifactDigest) + } + + return nil + }) + + return artifactData, err +} + +func (bdw DBWrapper) SetReferrer(repo string, referredDigest godigest.Digest, referrer repodb.Descriptor) error { + err := bdw.DB.Update(func(tx *bolt.Tx) error { + buck := tx.Bucket([]byte(repodb.RepoMetadataBucket)) + + repoMetaBlob := buck.Get([]byte(repo)) + + // object not found + if len(repoMetaBlob) == 0 { + var err error + + // create a new object + repoMeta := repodb.RepoMetadata{ + Name: repo, + Tags: map[string]repodb.Descriptor{}, + Statistics: map[string]repodb.DescriptorStatistics{ + referredDigest.String(): {}, + }, + Signatures: map[string]repodb.ManifestSignatures{ + referredDigest.String(): {}, + }, + Referrers: map[string][]repodb.Descriptor{ + referredDigest.String(): { + { + Digest: referrer.Digest, + MediaType: referrer.MediaType, + }, + }, + }, + } + + repoMetaBlob, err = json.Marshal(repoMeta) + if err != nil { + return err + } + + return buck.Put([]byte(repo), repoMetaBlob) + } + var repoMeta repodb.RepoMetadata + + err := json.Unmarshal(repoMetaBlob, &repoMeta) + if err != nil { + return err + } + + refferers := repoMeta.Referrers[referredDigest.String()] + + for i := range refferers { + if refferers[i].Digest == referrer.Digest { + return nil + } + } + + refferers = append(refferers, repodb.Descriptor{ + Digest: referrer.Digest, + MediaType: referrer.MediaType, + }) + + repoMeta.Referrers[referredDigest.String()] = refferers + + repoMetaBlob, err = json.Marshal(repoMeta) + if err != nil { + return err + } + + return buck.Put([]byte(repo), repoMetaBlob) + }) + + return err +} + +func (bdw DBWrapper) DeleteReferrer(repo string, referredDigest godigest.Digest, + referrerDigest godigest.Digest, +) error { + return bdw.DB.Update(func(tx *bolt.Tx) error { + buck := tx.Bucket([]byte(repodb.RepoMetadataBucket)) + + repoMetaBlob := buck.Get([]byte(repo)) + + if len(repoMetaBlob) == 0 { + return zerr.ErrRepoMetaNotFound + } + + var repoMeta repodb.RepoMetadata + + err := json.Unmarshal(repoMetaBlob, &repoMeta) + if err != nil { + return err + } + + referrers := repoMeta.Referrers[referredDigest.String()] + + for i := range referrers { + if referrers[i].Digest == referrerDigest.String() { + referrers = append(referrers[:i], referrers[i+1:]...) + + break + } + } + + repoMeta.Referrers[referredDigest.String()] = referrers + + repoMetaBlob, err = json.Marshal(repoMeta) + if err != nil { + return err + } + + return buck.Put([]byte(repo), repoMetaBlob) + }) +} + +func (bdw DBWrapper) GetReferrers(repo string, referredDigest godigest.Digest) ([]repodb.Descriptor, error) { + var referrers []repodb.Descriptor + + err := bdw.DB.View(func(tx *bolt.Tx) error { + buck := tx.Bucket([]byte(repodb.RepoMetadataBucket)) + + repoMetaBlob := buck.Get([]byte(repo)) + if len(repoMetaBlob) == 0 { + return zerr.ErrRepoMetaNotFound + } + + var repoMeta repodb.RepoMetadata + + err := json.Unmarshal(repoMetaBlob, &repoMeta) + if err != nil { + return err + } + + referrers = repoMeta.Referrers[referredDigest.String()] + + return nil + }) + + return referrers, err +} + +func (bdw DBWrapper) GetFilteredReferrersInfo(repo string, referredDigest godigest.Digest, + artifactTypes []string, +) ([]repodb.ReferrerInfo, error) { + referrersDescriptors, err := bdw.GetReferrers(repo, referredDigest) + if err != nil { + bdw.Log.Error().Msgf("repodb: failed to get referrers for '%s@%s'", repo, referredDigest.String()) + + return nil, err + } + + referrersInfo := []repodb.ReferrerInfo{} + + err = bdw.DB.View(func(tx *bolt.Tx) error { + artifactBuck := tx.Bucket([]byte(repodb.ArtifactDataBucket)) + manifestBuck := tx.Bucket([]byte(repodb.ManifestDataBucket)) + + for _, descriptor := range referrersDescriptors { + referrerInfo := repodb.ReferrerInfo{} + + switch descriptor.MediaType { + case ispec.MediaTypeImageManifest: + manifestDataBlob := manifestBuck.Get([]byte(descriptor.Digest)) + + if len(manifestDataBlob) == 0 { + bdw.Log.Error().Msgf("repodb: manifest data not found for digest %s", descriptor.Digest) + + continue + } + + var manifestData repodb.ManifestData + + err = json.Unmarshal(manifestDataBlob, &manifestData) + if err != nil { + bdw.Log.Error().Err(err).Msgf("repodb: can't unmarhsal manifest data for digest %s", + descriptor.Digest) + + continue + } + + var manifestContent ispec.Manifest + + err := json.Unmarshal(manifestData.ManifestBlob, &manifestContent) + if err != nil { + bdw.Log.Error().Err(err).Msgf("repodb: can't unmarhsal manifest for digest %s", + descriptor.Digest) + + continue + } + + referrerInfo = repodb.ReferrerInfo{ + Digest: descriptor.Digest, + MediaType: ispec.MediaTypeImageManifest, + ArtifactType: manifestContent.Config.MediaType, + Size: len(manifestData.ManifestBlob), + Annotations: manifestContent.Annotations, + } + case ispec.MediaTypeArtifactManifest: + artifactDataBlob := artifactBuck.Get([]byte(descriptor.Digest)) + + if len(artifactDataBlob) == 0 { + bdw.Log.Error().Msgf("repodb: artifact data not found for digest %s", descriptor.Digest) + + continue + } + + var artifactData repodb.ArtifactData + + err = json.Unmarshal(artifactDataBlob, &artifactData) + if err != nil { + bdw.Log.Error().Err(err).Msgf("repodb: can't unmarhsal artifact data for digest %s", descriptor.Digest) + + continue + } + + manifestContent := ispec.Artifact{} + + err := json.Unmarshal(artifactData.ManifestBlob, &manifestContent) + if err != nil { + bdw.Log.Error().Err(err).Msgf("repodb: can't unmarhsal artifact manifest for digest %s", descriptor.Digest) + + continue + } + + referrerInfo = repodb.ReferrerInfo{ + Size: len(artifactData.ManifestBlob), + Digest: descriptor.Digest, + MediaType: manifestContent.MediaType, + Annotations: manifestContent.Annotations, + ArtifactType: manifestContent.ArtifactType, + } + } + + if !common.MatchesArtifactTypes(referrerInfo.ArtifactType, artifactTypes) { + continue + } + + referrersInfo = append(referrersInfo, referrerInfo) + } + + return nil + }) + + return referrersInfo, err +} + func (bdw *DBWrapper) SetRepoReference(repo string, reference string, manifestDigest godigest.Digest, mediaType string, ) error { @@ -274,13 +564,16 @@ func (bdw *DBWrapper) SetRepoReference(repo string, reference string, manifestDi // object not found if len(repoMetaBlob) == 0 { var err error - - repoMetaBlob, err = json.Marshal(repodb.RepoMetadata{ + // create a new object + repoMeta := repodb.RepoMetadata{ Name: repo, Tags: map[string]repodb.Descriptor{}, Statistics: map[string]repodb.DescriptorStatistics{}, Signatures: map[string]repodb.ManifestSignatures{}, - }) + Referrers: map[string][]repodb.Descriptor{}, + } + + repoMetaBlob, err = json.Marshal(repoMeta) if err != nil { return err } @@ -303,6 +596,7 @@ func (bdw *DBWrapper) SetRepoReference(repo string, reference string, manifestDi repoMeta.Statistics[manifestDigest.String()] = repodb.DescriptorStatistics{DownloadCount: 0} repoMeta.Signatures[manifestDigest.String()] = repodb.ManifestSignatures{} + repoMeta.Referrers[manifestDigest.String()] = []repodb.Descriptor{} repoMetaBlob, err = json.Marshal(repoMeta) if err != nil { diff --git a/pkg/meta/repodb/boltdb-wrapper/boltdb_wrapper_test.go b/pkg/meta/repodb/boltdb-wrapper/boltdb_wrapper_test.go index c2d80880..20904ab3 100644 --- a/pkg/meta/repodb/boltdb-wrapper/boltdb_wrapper_test.go +++ b/pkg/meta/repodb/boltdb-wrapper/boltdb_wrapper_test.go @@ -70,6 +70,142 @@ func TestWrapperErrors(t *testing.T) { So(err, ShouldNotBeNil) }) + Convey("GetArtifactData", func() { + err := boltdbWrapper.DB.Update(func(tx *bbolt.Tx) error { + artifactBuck := tx.Bucket([]byte(repodb.ArtifactDataBucket)) + + return artifactBuck.Put([]byte("artifactDigest"), []byte("wrong json")) + }) + So(err, ShouldBeNil) + + _, err = boltdbWrapper.GetArtifactData("artifactDigest") + So(err, ShouldNotBeNil) + }) + + Convey("SetReferrer", func() { + err := boltdbWrapper.DB.Update(func(tx *bbolt.Tx) error { + repoBuck := tx.Bucket([]byte(repodb.RepoMetadataBucket)) + + return repoBuck.Put([]byte("repo"), []byte("wrong json")) + }) + So(err, ShouldBeNil) + + err = boltdbWrapper.SetReferrer("repo", "ref", repodb.Descriptor{}) + So(err, ShouldNotBeNil) + }) + + Convey("DeleteReferrer", func() { + Convey("RepoMeta not found", func() { + err := boltdbWrapper.DeleteReferrer("r", "dig", "dig") + So(err, ShouldNotBeNil) + }) + + Convey("bad repo meta blob", func() { + err := boltdbWrapper.DB.Update(func(tx *bbolt.Tx) error { + repoBuck := tx.Bucket([]byte(repodb.RepoMetadataBucket)) + + return repoBuck.Put([]byte("repo"), []byte("wrong json")) + }) + So(err, ShouldBeNil) + + err = boltdbWrapper.DeleteReferrer("repo", "dig", "dig") + So(err, ShouldNotBeNil) + }) + }) + + Convey("GetReferrers", func() { + Convey("RepoMeta not found", func() { + _, err := boltdbWrapper.GetReferrers("repo", "dig") + So(err, ShouldNotBeNil) + }) + + Convey("bad repo meta blob", func() { + err := boltdbWrapper.DB.Update(func(tx *bbolt.Tx) error { + repoBuck := tx.Bucket([]byte(repodb.RepoMetadataBucket)) + + return repoBuck.Put([]byte("repo"), []byte("wrong json")) + }) + So(err, ShouldBeNil) + + _, err = boltdbWrapper.GetReferrers("repo", "dig") + So(err, ShouldNotBeNil) + }) + }) + + Convey("GetFilteredReferrersInfo", func() { + Convey("getReferrers fails", func() { + err := boltdbWrapper.DB.Update(func(tx *bbolt.Tx) error { + repoBuck := tx.Bucket([]byte(repodb.RepoMetadataBucket)) + + return repoBuck.Put([]byte("repo"), []byte("wrong json")) + }) + So(err, ShouldBeNil) + + _, err = boltdbWrapper.GetFilteredReferrersInfo("repo", "", nil) + So(err, ShouldNotBeNil) + }) + + Convey("unmarshal erorrs", func() { + err := boltdbWrapper.DB.Update(func(tx *bbolt.Tx) error { + manifestBuck := tx.Bucket([]byte(repodb.ManifestDataBucket)) + artifactBuck := tx.Bucket([]byte(repodb.ArtifactDataBucket)) + + err = manifestBuck.Put([]byte("manifestDataRef"), []byte("bad json")) + So(err, ShouldBeNil) + + err = artifactBuck.Put([]byte("artifactDataRef"), []byte("bad json")) + So(err, ShouldBeNil) + + badBlob, err := json.Marshal(repodb.ArtifactData{ + ManifestBlob: []byte("bad json"), + }) + So(err, ShouldBeNil) + + err = artifactBuck.Put([]byte("artifactManifestRef"), badBlob) + So(err, ShouldBeNil) + + badBlob, err = json.Marshal(repodb.ManifestData{ + ManifestBlob: []byte("bad json"), + }) + So(err, ShouldBeNil) + + err = manifestBuck.Put([]byte("badManifest"), badBlob) + So(err, ShouldBeNil) + + return nil + }) + So(err, ShouldBeNil) + + err = boltdbWrapper.SetReferrer("repo", "refDigest", repodb.Descriptor{ + Digest: "manifestDataRef", + MediaType: ispec.MediaTypeImageManifest, + }) + So(err, ShouldBeNil) + + err = boltdbWrapper.SetReferrer("repo", "refDigest", repodb.Descriptor{ + Digest: "artifactDataRef", + MediaType: ispec.MediaTypeArtifactManifest, + }) + So(err, ShouldBeNil) + + err = boltdbWrapper.SetReferrer("repo", "refDigest", repodb.Descriptor{ + Digest: "badManifest", + MediaType: ispec.MediaTypeImageManifest, + }) + So(err, ShouldBeNil) + + err = boltdbWrapper.SetReferrer("repo", "refDigest", repodb.Descriptor{ + Digest: "artifactManifestRef", + MediaType: ispec.MediaTypeArtifactManifest, + }) + So(err, ShouldBeNil) + + refInfo, err := boltdbWrapper.GetFilteredReferrersInfo("repo", "refDigest", nil) + So(err, ShouldBeNil) + So(len(refInfo), ShouldEqual, 0) + }) + }) + Convey("SetRepoReference", func() { err := boltdbWrapper.DB.Update(func(tx *bbolt.Tx) error { repoBuck := tx.Bucket([]byte(repodb.RepoMetadataBucket)) diff --git a/pkg/meta/repodb/common/common.go b/pkg/meta/repodb/common/common.go index 7a61cdc9..f0caa44a 100644 --- a/pkg/meta/repodb/common/common.go +++ b/pkg/meta/repodb/common/common.go @@ -1,6 +1,7 @@ package common import ( + "encoding/json" "strings" "time" @@ -197,3 +198,38 @@ func containsString(strSlice []string, str string) bool { return false } + +func GetReferredSubject(descriptorBlob []byte) (godigest.Digest, bool) { + var manifest ispec.Manifest + + err := json.Unmarshal(descriptorBlob, &manifest) + if err != nil { + return "", false + } + + if manifest.Subject == nil || manifest.Subject.Digest.String() == "" { + return "", false + } + + return manifest.Subject.Digest, true +} + +func MatchesArtifactTypes(descriptorMediaType string, artifactTypes []string) bool { + if len(artifactTypes) == 0 { + return true + } + + found := false + + for _, artifactType := range artifactTypes { + if artifactType != "" && descriptorMediaType != artifactType { + continue + } + + found = true + + break + } + + return found +} diff --git a/pkg/meta/repodb/common/common_test.go b/pkg/meta/repodb/common/common_test.go new file mode 100644 index 00000000..3e96a663 --- /dev/null +++ b/pkg/meta/repodb/common/common_test.go @@ -0,0 +1,24 @@ +package common_test + +import ( + "testing" + + . "github.com/smartystreets/goconvey/convey" + + "zotregistry.io/zot/pkg/meta/repodb/common" +) + +func TestUtils(t *testing.T) { + Convey("GetReferredSubject", t, func() { + _, err := common.GetReferredSubject([]byte("bad json")) + So(err, ShouldNotBeNil) + }) + + Convey("MatchesArtifactTypes", t, func() { + res := common.MatchesArtifactTypes("", nil) + So(res, ShouldBeTrue) + + res = common.MatchesArtifactTypes("type", []string{"someOtherType"}) + So(res, ShouldBeFalse) + }) +} diff --git a/pkg/meta/repodb/dynamodb-wrapper/dynamo_internal_test.go b/pkg/meta/repodb/dynamodb-wrapper/dynamo_internal_test.go index e3a933da..8898a4fa 100644 --- a/pkg/meta/repodb/dynamodb-wrapper/dynamo_internal_test.go +++ b/pkg/meta/repodb/dynamodb-wrapper/dynamo_internal_test.go @@ -29,6 +29,9 @@ func TestWrapperErrors(t *testing.T) { repoMetaTablename := "RepoMetadataTable" + uuid.String() manifestDataTablename := "ManifestDataTable" + uuid.String() + indexDataTablename := "IndexDataTable" + uuid.String() + artifactDataTablename := "ArtifactDataTable" + uuid.String() + versionTablename := "Version" + uuid.String() Convey("Create table errors", t, func() { @@ -52,6 +55,8 @@ func TestWrapperErrors(t *testing.T) { Client: dynamodb.NewFromConfig(cfg), RepoMetaTablename: repoMetaTablename, ManifestDataTablename: manifestDataTablename, + IndexDataTablename: indexDataTablename, + ArtifactDataTablename: artifactDataTablename, VersionTablename: versionTablename, Patches: version.GetDynamoDBPatches(), Log: log.Logger{Logger: zerolog.New(os.Stdout)}, @@ -91,6 +96,8 @@ func TestWrapperErrors(t *testing.T) { RepoMetaTablename: repoMetaTablename, ManifestDataTablename: manifestDataTablename, VersionTablename: versionTablename, + IndexDataTablename: indexDataTablename, + ArtifactDataTablename: artifactDataTablename, Patches: version.GetDynamoDBPatches(), Log: log.Logger{Logger: zerolog.New(os.Stdout)}, } diff --git a/pkg/meta/repodb/dynamodb-wrapper/dynamo_test.go b/pkg/meta/repodb/dynamodb-wrapper/dynamo_test.go index c8df81c0..861561ce 100644 --- a/pkg/meta/repodb/dynamodb-wrapper/dynamo_test.go +++ b/pkg/meta/repodb/dynamodb-wrapper/dynamo_test.go @@ -25,6 +25,8 @@ import ( "zotregistry.io/zot/pkg/test" ) +const badTablename = "bad tablename" + func TestIterator(t *testing.T) { const ( endpoint = "http://localhost:4566" @@ -40,6 +42,7 @@ func TestIterator(t *testing.T) { manifestDataTablename := "ManifestDataTable" + uuid.String() versionTablename := "Version" + uuid.String() indexDataTablename := "IndexDataTable" + uuid.String() + artifactDataTablename := "ArtifactDataTable" + uuid.String() Convey("TestIterator", t, func() { dynamoWrapper, err := dynamo.NewDynamoDBWrapper(dynamoParams.DBDriverParameters{ @@ -48,6 +51,7 @@ func TestIterator(t *testing.T) { RepoMetaTablename: repoMetaTablename, ManifestDataTablename: manifestDataTablename, IndexDataTablename: indexDataTablename, + ArtifactDataTablename: artifactDataTablename, VersionTablename: versionTablename, }) So(err, ShouldBeNil) @@ -133,6 +137,7 @@ func TestWrapperErrors(t *testing.T) { manifestDataTablename := "ManifestDataTable" + uuid.String() versionTablename := "Version" + uuid.String() indexDataTablename := "IndexDataTable" + uuid.String() + artifactDataTablename := "ArtifactData" + uuid.String() ctx := context.Background() @@ -143,6 +148,7 @@ func TestWrapperErrors(t *testing.T) { RepoMetaTablename: repoMetaTablename, ManifestDataTablename: manifestDataTablename, IndexDataTablename: indexDataTablename, + ArtifactDataTablename: artifactDataTablename, VersionTablename: versionTablename, }) So(err, ShouldBeNil) @@ -222,6 +228,114 @@ func TestWrapperErrors(t *testing.T) { So(err, ShouldNotBeNil) }) + Convey("GetArtifactData", func() { + dynamoWrapper.ArtifactDataTablename = badTablename + _, err = dynamoWrapper.GetArtifactData("dig") + So(err, ShouldNotBeNil) + }) + + Convey("GetArtifactData unmarhsal error", func() { + err = setBadArtifactData(dynamoWrapper.Client, artifactDataTablename, "dig") + So(err, ShouldBeNil) + + _, err = dynamoWrapper.GetArtifactData("dig") + So(err, ShouldNotBeNil) + }) + + Convey("SetReferrer client error", func() { + dynamoWrapper.RepoMetaTablename = badTablename + err := dynamoWrapper.SetReferrer("repo", "", repodb.Descriptor{}) + So(err, ShouldNotBeNil) + }) + + Convey("SetReferrer bad repoMeta", func() { + err := setBadRepoMeta(dynamoWrapper.Client, repoMetaTablename, "repo") + So(err, ShouldBeNil) + + err = dynamoWrapper.SetReferrer("repo", "", repodb.Descriptor{}) + So(err, ShouldNotBeNil) + }) + + Convey("GetReferrers client error", func() { + dynamoWrapper.RepoMetaTablename = badTablename + _, err := dynamoWrapper.GetReferrers("repo", "") + So(err, ShouldNotBeNil) + }) + + Convey("GetReferrers bad repoMeta", func() { + err := setBadRepoMeta(dynamoWrapper.Client, repoMetaTablename, "repo") + So(err, ShouldBeNil) + + _, err = dynamoWrapper.GetReferrers("repo", "") + So(err, ShouldNotBeNil) + }) + + Convey("DeleteReferrer client error", func() { + dynamoWrapper.RepoMetaTablename = badTablename + err := dynamoWrapper.DeleteReferrer("repo", "", "") + So(err, ShouldNotBeNil) + }) + + Convey("DeleteReferrer bad repoMeta", func() { + err := setBadRepoMeta(dynamoWrapper.Client, repoMetaTablename, "repo") + So(err, ShouldBeNil) + + err = dynamoWrapper.DeleteReferrer("repo", "", "") + So(err, ShouldNotBeNil) + }) + + Convey("GetFilteredReferrersInfo GetReferrers errors", func() { + dynamoWrapper.RepoMetaTablename = badTablename + _, err := dynamoWrapper.GetFilteredReferrersInfo("repo", "", nil) + So(err, ShouldNotBeNil) + }) + + Convey("GetFilteredReferrersInfo getData fails", func() { + dynamoWrapper.ManifestDataTablename = badTablename + dynamoWrapper.ArtifactDataTablename = badTablename + err = dynamoWrapper.SetReferrer("repo", "rf", repodb.Descriptor{ + Digest: "dig1", + MediaType: ispec.MediaTypeImageManifest, + }) + So(err, ShouldBeNil) + + err = dynamoWrapper.SetReferrer("repo", "rf", repodb.Descriptor{ + Digest: "dig2", + MediaType: ispec.MediaTypeArtifactManifest, + }) + So(err, ShouldBeNil) + + _, err := dynamoWrapper.GetFilteredReferrersInfo("repo", "rf", nil) + So(err, ShouldBeNil) + }) + + Convey("GetFilteredReferrersInfo bad descriptor blob", func() { + err = dynamoWrapper.SetArtifactData("dig2", repodb.ArtifactData{ + ManifestBlob: []byte("bad json"), + }) + So(err, ShouldBeNil) + + err = dynamoWrapper.SetManifestData("dig3", repodb.ManifestData{ + ManifestBlob: []byte("bad json"), + }) + So(err, ShouldBeNil) + + err = dynamoWrapper.SetReferrer("repo", "rf", repodb.Descriptor{ + Digest: "dig2", + MediaType: ispec.MediaTypeArtifactManifest, + }) + So(err, ShouldBeNil) + + err = dynamoWrapper.SetReferrer("repo", "rf", repodb.Descriptor{ + Digest: "dig3", + MediaType: ispec.MediaTypeImageManifest, + }) + So(err, ShouldBeNil) + + _, err := dynamoWrapper.GetFilteredReferrersInfo("repo", "rf", nil) + So(err, ShouldBeNil) + }) + Convey("IncrementRepoStars GetRepoMeta error", func() { err = dynamoWrapper.IncrementRepoStars("repo") So(err, ShouldNotBeNil) @@ -688,6 +802,7 @@ func TestWrapperErrors(t *testing.T) { RepoMetaTablename: "", ManifestDataTablename: manifestDataTablename, IndexDataTablename: indexDataTablename, + ArtifactDataTablename: artifactDataTablename, VersionTablename: versionTablename, }) So(err, ShouldNotBeNil) @@ -698,6 +813,7 @@ func TestWrapperErrors(t *testing.T) { RepoMetaTablename: repoMetaTablename, ManifestDataTablename: "", IndexDataTablename: indexDataTablename, + ArtifactDataTablename: artifactDataTablename, VersionTablename: versionTablename, }) So(err, ShouldNotBeNil) @@ -708,6 +824,7 @@ func TestWrapperErrors(t *testing.T) { RepoMetaTablename: repoMetaTablename, ManifestDataTablename: manifestDataTablename, IndexDataTablename: "", + ArtifactDataTablename: artifactDataTablename, VersionTablename: versionTablename, }) So(err, ShouldNotBeNil) @@ -718,6 +835,7 @@ func TestWrapperErrors(t *testing.T) { RepoMetaTablename: repoMetaTablename, ManifestDataTablename: manifestDataTablename, IndexDataTablename: indexDataTablename, + ArtifactDataTablename: artifactDataTablename, VersionTablename: "", }) So(err, ShouldNotBeNil) @@ -728,8 +846,20 @@ func TestWrapperErrors(t *testing.T) { RepoMetaTablename: repoMetaTablename, ManifestDataTablename: manifestDataTablename, IndexDataTablename: indexDataTablename, + ArtifactDataTablename: "", VersionTablename: versionTablename, }) + So(err, ShouldNotBeNil) + + _, err = dynamo.NewDynamoDBWrapper(dynamoParams.DBDriverParameters{ //nolint:contextcheck + Endpoint: endpoint, + Region: region, + RepoMetaTablename: repoMetaTablename, + ManifestDataTablename: manifestDataTablename, + IndexDataTablename: indexDataTablename, + VersionTablename: versionTablename, + ArtifactDataTablename: artifactDataTablename, + }) So(err, ShouldBeNil) }) } @@ -759,6 +889,31 @@ func setBadManifestData(client *dynamodb.Client, manifestDataTableName, digest s return err } +func setBadArtifactData(client *dynamodb.Client, artifactDataTablename, digest string) error { + mdAttributeValue, err := attributevalue.Marshal("string") + if err != nil { + return err + } + + _, err = client.UpdateItem(context.TODO(), &dynamodb.UpdateItemInput{ + ExpressionAttributeNames: map[string]string{ + "#AD": "ArtifactData", + }, + ExpressionAttributeValues: map[string]types.AttributeValue{ + ":ArtifactData": mdAttributeValue, + }, + Key: map[string]types.AttributeValue{ + "ArtifactDigest": &types.AttributeValueMemberS{ + Value: digest, + }, + }, + TableName: aws.String(artifactDataTablename), + UpdateExpression: aws.String("SET #AD = :ArtifactData"), + }) + + return err +} + func setBadIndexData(client *dynamodb.Client, indexDataTableName, digest string) error { mdAttributeValue, err := attributevalue.Marshal("string") if err != nil { diff --git a/pkg/meta/repodb/dynamodb-wrapper/dynamo_wrapper.go b/pkg/meta/repodb/dynamodb-wrapper/dynamo_wrapper.go index c73ea3ca..e91f90d8 100644 --- a/pkg/meta/repodb/dynamodb-wrapper/dynamo_wrapper.go +++ b/pkg/meta/repodb/dynamodb-wrapper/dynamo_wrapper.go @@ -32,6 +32,7 @@ type DBWrapper struct { RepoMetaTablename string IndexDataTablename string ManifestDataTablename string + ArtifactDataTablename string VersionTablename string Patches []func(client *dynamodb.Client, tableNames map[string]string) error Log log.Logger @@ -62,6 +63,7 @@ func NewDynamoDBWrapper(params dynamoParams.DBDriverParameters) (*DBWrapper, err RepoMetaTablename: params.RepoMetaTablename, ManifestDataTablename: params.ManifestDataTablename, IndexDataTablename: params.IndexDataTablename, + ArtifactDataTablename: params.ArtifactDataTablename, VersionTablename: params.VersionTablename, Patches: version.GetDynamoDBPatches(), Log: log.Logger{Logger: zerolog.New(os.Stdout)}, @@ -82,6 +84,11 @@ func NewDynamoDBWrapper(params dynamoParams.DBDriverParameters) (*DBWrapper, err return nil, err } + err = dynamoWrapper.createArtifactDataTable() + if err != nil { + return nil, err + } + err = dynamoWrapper.createIndexDataTable() if err != nil { return nil, err @@ -158,6 +165,7 @@ func (dwr *DBWrapper) SetManifestMeta(repo string, manifestDigest godigest.Diges Tags: map[string]repodb.Descriptor{}, Statistics: map[string]repodb.DescriptorStatistics{}, Signatures: map[string]repodb.ManifestSignatures{}, + Referrers: map[string][]repodb.Descriptor{}, } } @@ -307,6 +315,246 @@ func (dwr *DBWrapper) GetIndexData(indexDigest godigest.Digest) (repodb.IndexDat return indexData, nil } +func (dwr DBWrapper) SetArtifactData(artifactDigest godigest.Digest, artifactData repodb.ArtifactData) error { + artifactAttributeValue, err := attributevalue.Marshal(artifactData) + if err != nil { + return err + } + + _, err = dwr.Client.UpdateItem(context.TODO(), &dynamodb.UpdateItemInput{ + ExpressionAttributeNames: map[string]string{ + "#AD": "ArtifactData", + }, + ExpressionAttributeValues: map[string]types.AttributeValue{ + ":ArtifactData": artifactAttributeValue, + }, + Key: map[string]types.AttributeValue{ + "ArtifactDigest": &types.AttributeValueMemberS{ + Value: artifactDigest.String(), + }, + }, + TableName: aws.String(dwr.ArtifactDataTablename), + UpdateExpression: aws.String("SET #AD = :ArtifactData"), + }) + + return err +} + +func (dwr DBWrapper) GetArtifactData(artifactDigest godigest.Digest) (repodb.ArtifactData, error) { + resp, err := dwr.Client.GetItem(context.TODO(), &dynamodb.GetItemInput{ + TableName: aws.String(dwr.ArtifactDataTablename), + Key: map[string]types.AttributeValue{ + "ArtifactDigest": &types.AttributeValueMemberS{ + Value: artifactDigest.String(), + }, + }, + }) + if err != nil { + return repodb.ArtifactData{}, err + } + + if resp.Item == nil { + return repodb.ArtifactData{}, zerr.ErrRepoMetaNotFound + } + + var artifactData repodb.ArtifactData + + err = attributevalue.Unmarshal(resp.Item["ArtifactData"], &artifactData) + if err != nil { + return repodb.ArtifactData{}, err + } + + return artifactData, nil +} + +func (dwr DBWrapper) SetReferrer(repo string, referredDigest godigest.Digest, referrer repodb.Descriptor) error { + resp, err := dwr.Client.GetItem(context.TODO(), &dynamodb.GetItemInput{ + TableName: aws.String(dwr.RepoMetaTablename), + Key: map[string]types.AttributeValue{ + "RepoName": &types.AttributeValueMemberS{Value: repo}, + }, + }) + if err != nil { + return err + } + + repoMeta := repodb.RepoMetadata{ + Name: repo, + Tags: map[string]repodb.Descriptor{}, + Statistics: map[string]repodb.DescriptorStatistics{}, + Signatures: map[string]repodb.ManifestSignatures{}, + Referrers: map[string][]repodb.Descriptor{}, + } + + if resp.Item != nil { + err := attributevalue.Unmarshal(resp.Item["RepoMetadata"], &repoMeta) + if err != nil { + return err + } + } + + refferers := repoMeta.Referrers[referredDigest.String()] + + for i := range refferers { + if refferers[i].Digest == referrer.Digest { + return nil + } + } + + refferers = append(refferers, referrer) + + repoMeta.Referrers[referredDigest.String()] = refferers + + return dwr.setRepoMeta(repo, repoMeta) +} + +func (dwr DBWrapper) GetReferrers(repo string, referredDigest godigest.Digest) ([]repodb.Descriptor, error) { + resp, err := dwr.Client.GetItem(context.TODO(), &dynamodb.GetItemInput{ + TableName: aws.String(dwr.RepoMetaTablename), + Key: map[string]types.AttributeValue{ + "RepoName": &types.AttributeValueMemberS{Value: repo}, + }, + }) + if err != nil { + return []repodb.Descriptor{}, err + } + + repoMeta := repodb.RepoMetadata{ + Name: repo, + Tags: map[string]repodb.Descriptor{}, + Statistics: map[string]repodb.DescriptorStatistics{}, + Signatures: map[string]repodb.ManifestSignatures{}, + Referrers: map[string][]repodb.Descriptor{}, + } + + if resp.Item != nil { + err := attributevalue.Unmarshal(resp.Item["RepoMetadata"], &repoMeta) + if err != nil { + return []repodb.Descriptor{}, err + } + } + + return repoMeta.Referrers[referredDigest.String()], nil +} + +func (dwr DBWrapper) DeleteReferrer(repo string, referredDigest godigest.Digest, + referrerDigest godigest.Digest, +) error { + resp, err := dwr.Client.GetItem(context.TODO(), &dynamodb.GetItemInput{ + TableName: aws.String(dwr.RepoMetaTablename), + Key: map[string]types.AttributeValue{ + "RepoName": &types.AttributeValueMemberS{Value: repo}, + }, + }) + if err != nil { + return err + } + + repoMeta := repodb.RepoMetadata{ + Name: repo, + Tags: map[string]repodb.Descriptor{}, + Statistics: map[string]repodb.DescriptorStatistics{}, + Signatures: map[string]repodb.ManifestSignatures{}, + Referrers: map[string][]repodb.Descriptor{}, + } + + if resp.Item != nil { + err := attributevalue.Unmarshal(resp.Item["RepoMetadata"], &repoMeta) + if err != nil { + return err + } + } + + referrers := repoMeta.Referrers[referredDigest.String()] + + for i := range referrers { + if referrers[i].Digest == referrerDigest.String() { + referrers = append(referrers[:i], referrers[i+1:]...) + + break + } + } + + repoMeta.Referrers[referredDigest.String()] = referrers + + return dwr.setRepoMeta(repo, repoMeta) +} + +func (dwr DBWrapper) GetFilteredReferrersInfo(repo string, referredDigest godigest.Digest, + artifactTypes []string, +) ([]repodb.ReferrerInfo, error) { + referrersDescriptors, err := dwr.GetReferrers(repo, referredDigest) + if err != nil { + return nil, err + } + + referrersInfo := []repodb.ReferrerInfo{} + + for _, descriptor := range referrersDescriptors { + referrerInfo := repodb.ReferrerInfo{} + + switch descriptor.MediaType { + case ispec.MediaTypeImageManifest: + manifestData, err := dwr.GetManifestData(godigest.Digest(descriptor.Digest)) + if err != nil { + dwr.Log.Error().Msgf("repodb: manifest data not found for digest %s", descriptor.Digest) + + continue + } + + var manifestContent ispec.Manifest + + err = json.Unmarshal(manifestData.ManifestBlob, &manifestContent) + if err != nil { + dwr.Log.Error().Err(err).Msgf("repodb: can't unmarhsal manifest for digest %s", + descriptor.Digest) + + continue + } + + referrerInfo = repodb.ReferrerInfo{ + Digest: descriptor.Digest, + MediaType: ispec.MediaTypeImageManifest, + ArtifactType: manifestContent.Config.MediaType, + Size: len(manifestData.ManifestBlob), + Annotations: manifestContent.Annotations, + } + case ispec.MediaTypeArtifactManifest: + artifactData, err := dwr.GetArtifactData(godigest.Digest(descriptor.Digest)) + if err != nil { + dwr.Log.Error().Msgf("repodb: artifact data not found for digest %s", descriptor.Digest) + + continue + } + + manifestContent := ispec.Artifact{} + + err = json.Unmarshal(artifactData.ManifestBlob, &manifestContent) + if err != nil { + dwr.Log.Error().Err(err).Msgf("repodb: can't unmarhsal artifact manifest for digest %s", descriptor.Digest) + + continue + } + + referrerInfo = repodb.ReferrerInfo{ + Digest: descriptor.Digest, + MediaType: manifestContent.MediaType, + ArtifactType: manifestContent.ArtifactType, + Size: len(artifactData.ManifestBlob), + Annotations: manifestContent.Annotations, + } + } + + if !common.MatchesArtifactTypes(referrerInfo.ArtifactType, artifactTypes) { + continue + } + + referrersInfo = append(referrersInfo, referrerInfo) + } + + return referrersInfo, nil +} + func (dwr *DBWrapper) SetRepoReference(repo string, reference string, manifestDigest godigest.Digest, mediaType string, ) error { @@ -329,6 +577,7 @@ func (dwr *DBWrapper) SetRepoReference(repo string, reference string, manifestDi Tags: map[string]repodb.Descriptor{}, Statistics: map[string]repodb.DescriptorStatistics{}, Signatures: map[string]repodb.ManifestSignatures{}, + Referrers: map[string][]repodb.Descriptor{}, } if resp.Item != nil { @@ -347,6 +596,7 @@ func (dwr *DBWrapper) SetRepoReference(repo string, reference string, manifestDi repoMeta.Statistics[manifestDigest.String()] = repodb.DescriptorStatistics{DownloadCount: 0} repoMeta.Signatures[manifestDigest.String()] = repodb.ManifestSignatures{} + repoMeta.Referrers[manifestDigest.String()] = []repodb.Descriptor{} err = dwr.setRepoMeta(repo, repoMeta) @@ -1392,6 +1642,31 @@ func (dwr *DBWrapper) createIndexDataTable() error { return dwr.waitTableToBeCreated(dwr.IndexDataTablename) } +func (dwr DBWrapper) createArtifactDataTable() error { + _, err := dwr.Client.CreateTable(context.Background(), &dynamodb.CreateTableInput{ + TableName: aws.String(dwr.ArtifactDataTablename), + AttributeDefinitions: []types.AttributeDefinition{ + { + AttributeName: aws.String("ArtifactDigest"), + AttributeType: types.ScalarAttributeTypeS, + }, + }, + KeySchema: []types.KeySchemaElement{ + { + AttributeName: aws.String("ArtifactDigest"), + KeyType: types.KeyTypeHash, + }, + }, + BillingMode: types.BillingModePayPerRequest, + }) + + if err != nil && !strings.Contains(err.Error(), "Table already exists") { + return err + } + + return dwr.waitTableToBeCreated(dwr.ManifestDataTablename) +} + func (dwr *DBWrapper) createVersionTable() error { _, err := dwr.Client.CreateTable(context.Background(), &dynamodb.CreateTableInput{ TableName: aws.String(dwr.VersionTablename), diff --git a/pkg/meta/repodb/dynamodb-wrapper/params/parameters.go b/pkg/meta/repodb/dynamodb-wrapper/params/parameters.go index f5be16d5..4e2c7314 100644 --- a/pkg/meta/repodb/dynamodb-wrapper/params/parameters.go +++ b/pkg/meta/repodb/dynamodb-wrapper/params/parameters.go @@ -2,5 +2,5 @@ package params type DBDriverParameters struct { Endpoint, Region, RepoMetaTablename, ManifestDataTablename, IndexDataTablename, - VersionTablename string + ArtifactDataTablename, VersionTablename string } diff --git a/pkg/meta/repodb/sync_repodb.go b/pkg/meta/repodb/load_repodb.go similarity index 68% rename from pkg/meta/repodb/sync_repodb.go rename to pkg/meta/repodb/load_repodb.go index 0d291ae9..bf5b8cd9 100644 --- a/pkg/meta/repodb/sync_repodb.go +++ b/pkg/meta/repodb/load_repodb.go @@ -12,20 +12,21 @@ import ( "zotregistry.io/zot/pkg/storage" ) -// SyncRepoDB will sync all repos found in the rootdirectory of the oci layout that zot was deployed on. -func SyncRepoDB(repoDB RepoDB, storeController storage.StoreController, log log.Logger) error { +// ParseStorage will sync all repos found in the rootdirectory of the oci layout that zot was deployed on with the +// ParseStorage database. +func ParseStorage(repoDB RepoDB, storeController storage.StoreController, log log.Logger) error { allRepos, err := getAllRepos(storeController) if err != nil { rootDir := storeController.DefaultStore.RootDir() - log.Error().Err(err).Msgf("sync-repodb: failed to get all repo names present under %s", rootDir) + log.Error().Err(err).Msgf("load-local-layout: failed to get all repo names present under %s", rootDir) return err } for _, repo := range allRepos { - err := SyncRepo(repo, repoDB, storeController, log) + err := ParseRepo(repo, repoDB, storeController, log) if err != nil { - log.Error().Err(err).Msgf("sync-repodb: failed to sync repo %s", repo) + log.Error().Err(err).Msgf("load-local-layout: failed to sync repo %s", repo) return err } @@ -34,13 +35,13 @@ func SyncRepoDB(repoDB RepoDB, storeController storage.StoreController, log log. return nil } -// SyncRepo reads the contents of a repo and syncs all images signatures found. -func SyncRepo(repo string, repoDB RepoDB, storeController storage.StoreController, log log.Logger) error { +// ParseRepo reads the contents of a repo and syncs all images and signatures found. +func ParseRepo(repo string, repoDB RepoDB, storeController storage.StoreController, log log.Logger) error { imageStore := storeController.GetImageStore(repo) indexBlob, err := imageStore.GetIndexContent(repo) if err != nil { - log.Error().Err(err).Msgf("sync-repo: failed to read index.json for repo %s", repo) + log.Error().Err(err).Msgf("load-repo: failed to read index.json for repo %s", repo) return err } @@ -49,14 +50,14 @@ func SyncRepo(repo string, repoDB RepoDB, storeController storage.StoreControlle err = json.Unmarshal(indexBlob, &indexContent) if err != nil { - log.Error().Err(err).Msgf("sync-repo: failed to unmarshal index.json for repo %s", repo) + log.Error().Err(err).Msgf("load-repo: failed to unmarshal index.json for repo %s", repo) return err } err = resetRepoMetaTags(repo, repoDB, log) if err != nil && !errors.Is(err, zerr.ErrRepoMetaNotFound) { - log.Error().Err(err).Msgf("sync-repo: failed to reset tag field in RepoMetadata for repo %s", repo) + log.Error().Err(err).Msgf("load-repo: failed to reset tag field in RepoMetadata for repo %s", repo) return err } @@ -76,7 +77,7 @@ func SyncRepo(repo string, repoDB RepoDB, storeController storage.StoreControlle manifestMetaIsPresent, err := isManifestMetaPresent(repo, manifest, repoDB) if err != nil { - log.Error().Err(err).Msgf("sync-repo: error checking manifestMeta in RepoDB") + log.Error().Err(err).Msgf("load-repo: error checking manifestMeta in RepoDB") return err } @@ -84,7 +85,7 @@ func SyncRepo(repo string, repoDB RepoDB, storeController storage.StoreControlle if manifestMetaIsPresent && hasTag { err = repoDB.SetRepoReference(repo, tag, manifest.Digest, manifest.MediaType) if err != nil { - log.Error().Err(err).Msgf("sync-repo: failed to set repo tag for %s:%s", repo, tag) + log.Error().Err(err).Msgf("load-repo: failed to set repo tag for %s:%s", repo, tag) return err } @@ -94,7 +95,7 @@ func SyncRepo(repo string, repoDB RepoDB, storeController storage.StoreControlle manifestBlob, digest, _, err := imageStore.GetImageManifest(repo, manifest.Digest.String()) if err != nil { - log.Error().Err(err).Msgf("sync-repo: failed to set repo tag for %s:%s", repo, tag) + log.Error().Err(err).Msgf("load-repo: failed to set repo tag for %s:%s", repo, tag) return err } @@ -105,7 +106,7 @@ func SyncRepo(repo string, repoDB RepoDB, storeController storage.StoreControlle if errors.Is(err, zerr.ErrOrphanSignature) { continue } else { - log.Error().Err(err).Msgf("sync-repo: failed checking if image is signature for %s:%s", repo, tag) + log.Error().Err(err).Msgf("load-repo: failed checking if image is signature for %s:%s", repo, tag) return err } @@ -134,7 +135,7 @@ func SyncRepo(repo string, repoDB RepoDB, storeController storage.StoreControlle err = SetMetadataFromInput(repo, reference, manifest.MediaType, manifest.Digest, manifestBlob, imageStore, repoDB, log) if err != nil { - log.Error().Err(err).Msgf("sync-repo: failed to set metadata for %s:%s", repo, tag) + log.Error().Err(err).Msgf("load-repo: failed to set metadata for %s:%s", repo, tag) return err } @@ -142,12 +143,13 @@ func SyncRepo(repo string, repoDB RepoDB, storeController storage.StoreControlle // manage the signatures found for _, sigData := range signaturesFound { - err := repoDB.AddManifestSignature(repo, godigest.Digest(sigData.signedManifestDigest), SignatureMetadata{ - SignatureType: sigData.signatureType, - SignatureDigest: sigData.signatureDigest, - }) + err := repoDB.AddManifestSignature(repo, godigest.Digest(sigData.signedManifestDigest), + SignatureMetadata{ + SignatureType: sigData.signatureType, + SignatureDigest: sigData.signatureDigest, + }) if err != nil { - log.Error().Err(err).Msgf("sync-repo: failed set signature meta for signed image %s:%s manifest digest %s ", + log.Error().Err(err).Msgf("load-repo: failed set signature meta for signed image %s:%s manifest digest %s ", sigData.repo, sigData.tag, sigData.signedManifestDigest) return err @@ -161,13 +163,13 @@ func SyncRepo(repo string, repoDB RepoDB, storeController storage.StoreControlle func resetRepoMetaTags(repo string, repoDB RepoDB, log log.Logger) error { repoMeta, err := repoDB.GetRepoMeta(repo) if err != nil && !errors.Is(err, zerr.ErrRepoMetaNotFound) { - log.Error().Err(err).Msgf("sync-repo: failed to get RepoMeta for repo %s", repo) + log.Error().Err(err).Msgf("load-repo: failed to get RepoMeta for repo %s", repo) return err } if errors.Is(err, zerr.ErrRepoMetaNotFound) { - log.Info().Msgf("sync-repo: RepoMeta not found for repo %s, new RepoMeta will be created", repo) + log.Info().Msgf("load-repo: RepoMeta not found for repo %s, new RepoMeta will be created", repo) return nil } @@ -176,7 +178,7 @@ func resetRepoMetaTags(repo string, repoDB RepoDB, log log.Logger) error { // We should have a way to delete all tags at once err := repoDB.DeleteRepoTag(repo, tag) if err != nil { - log.Error().Err(err).Msgf("sync-repo: failed to delete tag %s from RepoMeta for repo %s", tag, repo) + log.Error().Err(err).Msgf("load-repo: failed to delete tag %s from RepoMeta for repo %s", tag, repo) return err } @@ -220,7 +222,7 @@ func isManifestMetaPresent(repo string, manifest ispec.Descriptor, repoDB RepoDB } // NewManifestMeta takes raw data about an image and createa a new ManifestMetadate object. -func NewManifestData(repoName string, manifestBlob []byte, imgStore storage.ImageStore, +func NewManifestData(repoName string, manifestBlob []byte, imageStore storage.ImageStore, ) (ManifestData, error) { var ( manifestContent ispec.Manifest @@ -233,7 +235,7 @@ func NewManifestData(repoName string, manifestBlob []byte, imgStore storage.Imag return ManifestData{}, err } - configBlob, err := imgStore.GetBlobContent(repoName, manifestContent.Config.Digest) + configBlob, err := imageStore.GetBlobContent(repoName, manifestContent.Config.Digest) if err != nil { return ManifestData{}, err } @@ -249,7 +251,7 @@ func NewManifestData(repoName string, manifestBlob []byte, imgStore storage.Imag return manifestData, nil } -func NewIndexData(repoName string, indexBlob []byte, +func NewIndexData(repoName string, indexBlob []byte, imageStore storage.ImageStore, ) IndexData { indexData := IndexData{} @@ -258,6 +260,13 @@ func NewIndexData(repoName string, indexBlob []byte, return indexData } +func NewArtifactData(repo string, descriptorBlob []byte, imageStore storage.ImageStore, +) ArtifactData { + return ArtifactData{ + ManifestBlob: descriptorBlob, + } +} + // SetMetadataFromInput tries to set manifest metadata and update repo metadata by adding the current tag // (in case the reference is a tag). The function expects image manifests and indexes (multi arch images). func SetMetadataFromInput(repo, reference, mediaType string, digest godigest.Digest, descriptorBlob []byte, @@ -277,12 +286,33 @@ func SetMetadataFromInput(repo, reference, mediaType string, digest godigest.Dig return err } case ispec.MediaTypeImageIndex: - indexData := NewIndexData(repo, descriptorBlob) + indexData := NewIndexData(repo, descriptorBlob, imageStore) err := repoDB.SetIndexData(digest, indexData) if err != nil { log.Error().Err(err).Msg("repodb: error while putting index data") + return err + } + case ispec.MediaTypeArtifactManifest: + artifactData := NewArtifactData(repo, descriptorBlob, imageStore) + + err := repoDB.SetArtifactData(digest, artifactData) + if err != nil { + log.Error().Err(err).Msg("repodb: error while putting artifact data") + + return err + } + } + + if refferredDigest, hasSubject := GetReferredSubject(descriptorBlob); hasSubject { + err := repoDB.SetReferrer(repo, refferredDigest, Descriptor{ + Digest: digest.String(), + MediaType: mediaType, + }) + if err != nil { + log.Error().Err(err).Msg("repodb: error while settingg referrer") + return err } } @@ -296,3 +326,18 @@ func SetMetadataFromInput(repo, reference, mediaType string, digest godigest.Dig return nil } + +func GetReferredSubject(descriptorBlob []byte) (godigest.Digest, bool) { + var manifest ispec.Manifest + + err := json.Unmarshal(descriptorBlob, &manifest) + if err != nil { + return "", false + } + + if manifest.Subject == nil || manifest.Subject.Digest.String() == "" { + return "", false + } + + return manifest.Subject.Digest, true +} diff --git a/pkg/meta/repodb/sync_repodb_test.go b/pkg/meta/repodb/load_repodb_test.go similarity index 90% rename from pkg/meta/repodb/sync_repodb_test.go rename to pkg/meta/repodb/load_repodb_test.go index eec1544a..c82376ec 100644 --- a/pkg/meta/repodb/sync_repodb_test.go +++ b/pkg/meta/repodb/load_repodb_test.go @@ -12,7 +12,6 @@ import ( godigest "github.com/opencontainers/go-digest" ispec "github.com/opencontainers/image-spec/specs-go/v1" - oras "github.com/oras-project/artifacts-spec/specs-go/v1" . "github.com/smartystreets/goconvey/convey" zerr "zotregistry.io/zot/errors" @@ -32,8 +31,8 @@ const repo = "repo" var ErrTestError = errors.New("test error") -func TestSyncRepoDBErrors(t *testing.T) { - Convey("SyncRepoDB", t, func() { +func TestLoadOCILayoutErrors(t *testing.T) { + Convey("LoadOCILayout", t, func() { imageStore := mocks.MockedImageStore{ GetIndexContentFn: func(repo string) ([]byte, error) { return nil, ErrTestError @@ -46,7 +45,7 @@ func TestSyncRepoDBErrors(t *testing.T) { repoDB := mocks.RepoDBMock{} // sync repo fail - err := repodb.SyncRepoDB(repoDB, storeController, log.NewLogger("debug", "")) + err := repodb.ParseStorage(repoDB, storeController, log.NewLogger("debug", "")) So(err, ShouldNotBeNil) Convey("getAllRepos errors", func() { @@ -67,12 +66,12 @@ func TestSyncRepoDBErrors(t *testing.T) { }, } - err := repodb.SyncRepoDB(repoDB, storeController, log.NewLogger("debug", "")) + err := repodb.ParseStorage(repoDB, storeController, log.NewLogger("debug", "")) So(err, ShouldNotBeNil) }) }) - Convey("SyncRepo", t, func() { + Convey("LoadRepo", t, func() { imageStore := mocks.MockedImageStore{} storeController := storage.StoreController{DefaultStore: &imageStore} repoDB := mocks.RepoDBMock{} @@ -83,7 +82,7 @@ func TestSyncRepoDBErrors(t *testing.T) { return nil, ErrTestError } - err := repodb.SyncRepo("repo", repoDB, storeController, log) + err := repodb.ParseRepo("repo", repoDB, storeController, log) So(err, ShouldNotBeNil) }) @@ -92,7 +91,7 @@ func TestSyncRepoDBErrors(t *testing.T) { return []byte("Invalid JSON"), nil } - err := repodb.SyncRepo("repo", repoDB, storeController, log) + err := repodb.ParseRepo("repo", repoDB, storeController, log) So(err, ShouldNotBeNil) }) @@ -106,7 +105,7 @@ func TestSyncRepoDBErrors(t *testing.T) { return repodb.RepoMetadata{}, ErrTestError } - err := repodb.SyncRepo("repo", repoDB, storeController, log) + err := repodb.ParseRepo("repo", repoDB, storeController, log) So(err, ShouldNotBeNil) }) @@ -125,7 +124,7 @@ func TestSyncRepoDBErrors(t *testing.T) { return ErrTestError } - err := repodb.SyncRepo("repo", repoDB, storeController, log) + err := repodb.ParseRepo("repo", repoDB, storeController, log) So(err, ShouldNotBeNil) }) }) @@ -154,7 +153,7 @@ func TestSyncRepoDBErrors(t *testing.T) { return repodb.ManifestMetadata{}, ErrTestError } - err = repodb.SyncRepo("repo", repoDB, storeController, log) + err = repodb.ParseRepo("repo", repoDB, storeController, log) So(err, ShouldNotBeNil) }) }) @@ -183,7 +182,7 @@ func TestSyncRepoDBErrors(t *testing.T) { return ErrTestError } - err = repodb.SyncRepo("repo", repoDB, storeController, log) + err = repodb.ParseRepo("repo", repoDB, storeController, log) So(err, ShouldNotBeNil) }) }) @@ -215,7 +214,7 @@ func TestSyncRepoDBErrors(t *testing.T) { imageStore.GetImageManifestFn = func(repo, reference string) ([]byte, godigest.Digest, string, error) { return nil, "", "", ErrTestError } - err = repodb.SyncRepo("repo", repoDB, storeController, log) + err = repodb.ParseRepo("repo", repoDB, storeController, log) So(err, ShouldNotBeNil) }) @@ -224,7 +223,7 @@ func TestSyncRepoDBErrors(t *testing.T) { imageStore.GetImageManifestFn = func(repo, reference string) ([]byte, godigest.Digest, string, error) { return []byte("Invalid JSON"), "", "", nil } - err = repodb.SyncRepo("repo", repoDB, storeController, log) + err = repodb.ParseRepo("repo", repoDB, storeController, log) So(err, ShouldNotBeNil) }) Convey("CheckIsImageSignature -> not signature", func() { @@ -241,7 +240,7 @@ func TestSyncRepoDBErrors(t *testing.T) { return nil, ErrTestError } - err = repodb.SyncRepo("repo", repoDB, storeController, log) + err = repodb.ParseRepo("repo", repoDB, storeController, log) So(err, ShouldNotBeNil) }) @@ -250,17 +249,19 @@ func TestSyncRepoDBErrors(t *testing.T) { return []byte("invalid JSON"), nil } - err = repodb.SyncRepo("repo", repoDB, storeController, log) + err = repodb.ParseRepo("repo", repoDB, storeController, log) So(err, ShouldNotBeNil) }) }) Convey("CheckIsImageSignature -> is signature", func() { - manifestContent := oras.Manifest{ - Subject: &oras.Descriptor{ + manifestContent := ispec.Artifact{ + Subject: &ispec.Descriptor{ Digest: "123", }, + ArtifactType: "application/vnd.cncf.notary.signature", } + manifestBlob, err := json.Marshal(manifestContent) So(err, ShouldBeNil) @@ -274,14 +275,14 @@ func TestSyncRepoDBErrors(t *testing.T) { return ErrTestError } - err = repodb.SyncRepo("repo", repoDB, storeController, log) + err = repodb.ParseRepo("repo", repoDB, storeController, log) So(err, ShouldNotBeNil) }) }) }) } -func TestSyncRepoDBWithStorage(t *testing.T) { +func TestLoadOCILayoutWithStorage(t *testing.T) { Convey("Boltdb", t, func() { rootDir := t.TempDir() @@ -359,7 +360,7 @@ func TestSyncRepoDBWithStorage(t *testing.T) { }) So(err, ShouldBeNil) - err = repodb.SyncRepoDB(repoDB, storeController, log.NewLogger("debug", "")) + err = repodb.ParseStorage(repoDB, storeController, log.NewLogger("debug", "")) So(err, ShouldBeNil) repos, err := repoDB.GetMultipleRepoMeta( @@ -435,7 +436,7 @@ func TestSyncRepoDBWithStorage(t *testing.T) { }) So(err, ShouldBeNil) - err = repodb.SyncRepoDB(repoDB, storeController, log.NewLogger("debug", "")) + err = repodb.ParseStorage(repoDB, storeController, log.NewLogger("debug", "")) So(err, ShouldBeNil) repos, err := repoDB.GetMultipleRepoMeta( @@ -451,7 +452,7 @@ func TestSyncRepoDBWithStorage(t *testing.T) { }) } -func TestSyncRepoDBDynamoWrapper(t *testing.T) { +func TestLoadOCILayoutDynamoWrapper(t *testing.T) { skipIt(t) Convey("Dynamodb", t, func() { @@ -532,6 +533,7 @@ func TestSyncRepoDBDynamoWrapper(t *testing.T) { RepoMetaTablename: "RepoMetadataTable", ManifestDataTablename: "ManifestDataTable", IndexDataTablename: "IndexDataTable", + ArtifactDataTablename: "ArtifactDataTable", VersionTablename: "Version", }) So(err, ShouldBeNil) @@ -542,7 +544,7 @@ func TestSyncRepoDBDynamoWrapper(t *testing.T) { err = dynamoWrapper.ResetRepoMetaTable() So(err, ShouldBeNil) - err = repodb.SyncRepoDB(dynamoWrapper, storeController, log.NewLogger("debug", "")) + err = repodb.ParseStorage(dynamoWrapper, storeController, log.NewLogger("debug", "")) So(err, ShouldBeNil) repos, err := dynamoWrapper.GetMultipleRepoMeta( @@ -618,12 +620,13 @@ func TestSyncRepoDBDynamoWrapper(t *testing.T) { Region: "us-east-2", RepoMetaTablename: "RepoMetadataTable", ManifestDataTablename: "ManifestDataTable", + ArtifactDataTablename: "ArtifactDataTable", IndexDataTablename: "IndexDataTable", VersionTablename: "Version", }) So(err, ShouldBeNil) - err = repodb.SyncRepoDB(repoDB, storeController, log.NewLogger("debug", "")) + err = repodb.ParseStorage(repoDB, storeController, log.NewLogger("debug", "")) So(err, ShouldBeNil) repos, err := repoDB.GetMultipleRepoMeta( @@ -640,6 +643,13 @@ func TestSyncRepoDBDynamoWrapper(t *testing.T) { }) } +func TestGetReferredSubject(t *testing.T) { + Convey("GetReferredSubject error", t, func() { + _, err := repodb.GetReferredSubject([]byte("bad json")) + So(err, ShouldNotBeNil) + }) +} + func skipIt(t *testing.T) { t.Helper() diff --git a/pkg/meta/repodb/repodb.go b/pkg/meta/repodb/repodb.go index e3a9700f..6acb014e 100644 --- a/pkg/meta/repodb/repodb.go +++ b/pkg/meta/repodb/repodb.go @@ -11,6 +11,7 @@ import ( const ( ManifestDataBucket = "ManifestData" IndexDataBucket = "IndexData" + ArtifactDataBucket = "ArtifactData" UserMetadataBucket = "UserMeta" RepoMetadataBucket = "RepoMetadata" VersionBucket = "Version" @@ -67,6 +68,26 @@ type RepoDB interface { //nolint:interfacebloat // GetIndexData returns indexData for a given Index from the database GetIndexData(indexDigest godigest.Digest) (IndexData, error) + // SetArtifactData sets artifactData for a given artifact in the database + SetArtifactData(artifactDigest godigest.Digest, artifactData ArtifactData) error + + // GetArtifactData returns artifactData for a given artifact from the database + GetArtifactData(artifactDigest godigest.Digest) (ArtifactData, error) + + // SetReferrer adds a referrer to the referrers list of a manifest inside a repo + SetReferrer(repo string, referredDigest godigest.Digest, referrer Descriptor) error + + // SetReferrer delets a referrer to the referrers list of a manifest inside a repo + DeleteReferrer(repo string, referredDigest godigest.Digest, referrerDigest godigest.Digest) error + + // GetReferrers returns the list of referrers for a referred manifest + GetReferrers(repo string, referredDigest godigest.Digest) ([]Descriptor, error) + + // GetFilteredReferrersInfo returnes a list of for all referrers of the given digest that match one of the + // artifact types. + GetFilteredReferrersInfo(repo string, referredDigest godigest.Digest, artifactTypes []string) ( + []ReferrerInfo, error) + // IncrementManifestDownloads adds 1 to the download count of a manifest IncrementImageDownloads(repo string, reference string) error @@ -107,6 +128,18 @@ type ManifestData struct { ConfigBlob []byte } +type ArtifactData struct { + ManifestBlob []byte +} + +type ReferrerInfo struct { + Digest string + MediaType string + ArtifactType string + Size int + Annotations map[string]string +} + // Descriptor represents an image. Multiple images might have the same digests but different tags. type Descriptor struct { Digest string @@ -125,7 +158,9 @@ type RepoMetadata struct { Statistics map[string]DescriptorStatistics Signatures map[string]ManifestSignatures - Stars int + Referrers map[string][]Descriptor + + Stars int } type LayerInfo struct { diff --git a/pkg/meta/repodb/repodb_test.go b/pkg/meta/repodb/repodb_test.go index aceb8291..53b8990b 100644 --- a/pkg/meta/repodb/repodb_test.go +++ b/pkg/meta/repodb/repodb_test.go @@ -76,6 +76,7 @@ func TestDynamoDBWrapper(t *testing.T) { manifestDataTablename := "ManifestDataTable" + uuid.String() versionTablename := "Version" + uuid.String() indexDataTablename := "IndexDataTable" + uuid.String() + artifactDataTablename := "ArtifactDataTable" + uuid.String() Convey("DynamoDB Wrapper", t, func() { dynamoDBDriverParams := dynamoParams.DBDriverParameters{ @@ -83,6 +84,7 @@ func TestDynamoDBWrapper(t *testing.T) { RepoMetaTablename: repoMetaTablename, ManifestDataTablename: manifestDataTablename, IndexDataTablename: indexDataTablename, + ArtifactDataTablename: artifactDataTablename, VersionTablename: versionTablename, Region: "us-east-2", } @@ -1757,6 +1759,202 @@ func RunRepoDBTests(repoDB repodb.RepoDB, preparationFuncs ...func() error) { _, err = repoDB.GetIndexData(godigest.FromString("inexistent")) So(err, ShouldNotBeNil) }) + + Convey("Test artifact logic", func() { + artifact, err := test.GetRandomArtifact(nil) + So(err, ShouldBeNil) + + artifactDigest, err := artifact.Digest() + So(err, ShouldBeNil) + + artifactData, err := artifact.ArtifactData() + So(err, ShouldBeNil) + + err = repoDB.SetArtifactData(artifactDigest, artifactData) + So(err, ShouldBeNil) + + result, err := repoDB.GetArtifactData(artifactDigest) + So(err, ShouldBeNil) + So(result, ShouldResemble, artifactData) + + _, err = repoDB.GetArtifactData(godigest.FromString("inexistent")) + So(err, ShouldNotBeNil) + }) + + Convey("Test Referrers", func() { + image, err := test.GetRandomImage("tag") + So(err, ShouldBeNil) + + referredDigest, err := image.Digest() + So(err, ShouldBeNil) + + manifestBlob, err := json.Marshal(image.Manifest) + So(err, ShouldBeNil) + + configBlob, err := json.Marshal(image.Config) + So(err, ShouldBeNil) + + manifestData := repodb.ManifestData{ + ManifestBlob: manifestBlob, + ConfigBlob: configBlob, + } + + err = repoDB.SetManifestData(referredDigest, manifestData) + So(err, ShouldBeNil) + + err = repoDB.SetRepoReference("repo", "tag", referredDigest, ispec.MediaTypeImageManifest) + So(err, ShouldBeNil) + + // ------- Add Artifact 1 + + artifact1, err := test.GetRandomArtifact(&ispec.Descriptor{ + Digest: referredDigest, + MediaType: ispec.MediaTypeImageManifest, + }) + So(err, ShouldBeNil) + + artifactDigest1, err := artifact1.Digest() + So(err, ShouldBeNil) + + err = repoDB.SetReferrer("repo", referredDigest, repodb.Descriptor{ + Digest: artifactDigest1.String(), + MediaType: ispec.MediaTypeImageManifest, + }) + So(err, ShouldBeNil) + + // ------- Add Artifact 2 + + artifact2, err := test.GetRandomArtifact(&ispec.Descriptor{ + Digest: referredDigest, + MediaType: ispec.MediaTypeImageManifest, + }) + So(err, ShouldBeNil) + + artifactDigest2, err := artifact2.Digest() + So(err, ShouldBeNil) + + err = repoDB.SetReferrer("repo", referredDigest, repodb.Descriptor{ + Digest: artifactDigest2.String(), + MediaType: ispec.MediaTypeArtifactManifest, + }) + So(err, ShouldBeNil) + + // ------ GetReferrers + + referrers, err := repoDB.GetReferrers("repo", referredDigest) + So(len(referrers), ShouldEqual, 2) + So(referrers, ShouldContain, repodb.Descriptor{ + Digest: artifactDigest1.String(), + MediaType: ispec.MediaTypeImageManifest, + }) + So(referrers, ShouldContain, repodb.Descriptor{ + Digest: artifactDigest2.String(), + MediaType: ispec.MediaTypeArtifactManifest, + }) + So(err, ShouldBeNil) + + // ------ DeleteReferrers + + err = repoDB.DeleteReferrer("repo", referredDigest, artifactDigest1) + So(err, ShouldBeNil) + + err = repoDB.DeleteReferrer("repo", referredDigest, artifactDigest2) + So(err, ShouldBeNil) + + referrers, err = repoDB.GetReferrers("repo", referredDigest) + So(err, ShouldBeNil) + So(len(referrers), ShouldEqual, 0) + }) + + Convey("Test Referrers on empty Repo", func() { + repoMeta, err := repoDB.GetRepoMeta("repo") + So(err, ShouldNotBeNil) + So(repoMeta, ShouldResemble, repodb.RepoMetadata{}) + + referredDigest := godigest.FromString("referredDigest") + referrerDigest := godigest.FromString("referrerDigest") + + err = repoDB.SetReferrer("repo", referredDigest, repodb.Descriptor{ + Digest: referrerDigest.String(), + MediaType: ispec.MediaTypeImageManifest, + }) + So(err, ShouldBeNil) + + repoMeta, err = repoDB.GetRepoMeta("repo") + So(err, ShouldBeNil) + So(repoMeta.Referrers[referredDigest.String()][0].Digest, ShouldResemble, referrerDigest.String()) + }) + + Convey("Test Referrers add same one twice", func() { + repoMeta, err := repoDB.GetRepoMeta("repo") + So(err, ShouldNotBeNil) + So(repoMeta, ShouldResemble, repodb.RepoMetadata{}) + + referredDigest := godigest.FromString("referredDigest") + referrerDigest := godigest.FromString("referrerDigest") + + err = repoDB.SetReferrer("repo", referredDigest, repodb.Descriptor{ + Digest: referrerDigest.String(), + MediaType: ispec.MediaTypeImageManifest, + }) + So(err, ShouldBeNil) + + err = repoDB.SetReferrer("repo", referredDigest, repodb.Descriptor{ + Digest: referrerDigest.String(), + MediaType: ispec.MediaTypeImageManifest, + }) + So(err, ShouldBeNil) + + repoMeta, err = repoDB.GetRepoMeta("repo") + So(err, ShouldBeNil) + So(len(repoMeta.Referrers[referredDigest.String()]), ShouldEqual, 1) + }) + + Convey("GetFilteredReferrersInfo", func() { + referredDigest := godigest.FromString("referredDigest") + + err := repoDB.SetReferrer("repo", referredDigest, repodb.Descriptor{ + Digest: "inexistendManifestDigest", + MediaType: ispec.MediaTypeImageManifest, + }) + So(err, ShouldBeNil) + + err = repoDB.SetReferrer("repo", referredDigest, repodb.Descriptor{ + Digest: "inexistendArtifactManifestDigest", + MediaType: ispec.MediaTypeArtifactManifest, + }) + So(err, ShouldBeNil) + + // ------- Set existent manifest and artifact manifest + err = repoDB.SetManifestData("goodManifest", repodb.ManifestData{ + ManifestBlob: []byte(`{"artifactType": "unwantedType"}`), + ConfigBlob: []byte("{}"), + }) + So(err, ShouldBeNil) + + err = repoDB.SetReferrer("repo", referredDigest, repodb.Descriptor{ + Digest: "goodManifest", + MediaType: ispec.MediaTypeImageManifest, + }) + So(err, ShouldBeNil) + + err = repoDB.SetArtifactData("goodArtifact", repodb.ArtifactData{ + ManifestBlob: []byte(`{"artifactType": "wantedType"}`), + }) + So(err, ShouldBeNil) + + err = repoDB.SetReferrer("repo", referredDigest, repodb.Descriptor{ + Digest: "goodArtifact", + MediaType: ispec.MediaTypeArtifactManifest, + }) + So(err, ShouldBeNil) + + referrerInfo, err := repoDB.GetFilteredReferrersInfo("repo", referredDigest, []string{"wantedType"}) + So(err, ShouldBeNil) + So(len(referrerInfo), ShouldEqual, 1) + So(referrerInfo[0].ArtifactType, ShouldResemble, "wantedType") + So(referrerInfo[0].Digest, ShouldResemble, "goodArtifact") + }) }) } diff --git a/pkg/meta/repodb/repodbfactory/repodb_factory_test.go b/pkg/meta/repodb/repodbfactory/repodb_factory_test.go index 5672f58c..8b968339 100644 --- a/pkg/meta/repodb/repodbfactory/repodb_factory_test.go +++ b/pkg/meta/repodb/repodbfactory/repodb_factory_test.go @@ -20,6 +20,7 @@ func TestCreateDynamo(t *testing.T) { RepoMetaTablename: "RepoMetadataTable", ManifestDataTablename: "ManifestDataTable", IndexDataTablename: "IndexDataTable", + ArtifactDataTablename: "ArtifactDataTable", VersionTablename: "Version", Region: "us-east-2", } diff --git a/pkg/meta/repodb/update/update.go b/pkg/meta/repodb/update/update.go index 0c57c3bd..d059aad2 100644 --- a/pkg/meta/repodb/update/update.go +++ b/pkg/meta/repodb/update/update.go @@ -7,19 +7,20 @@ import ( zerr "zotregistry.io/zot/errors" "zotregistry.io/zot/pkg/log" "zotregistry.io/zot/pkg/meta/repodb" + "zotregistry.io/zot/pkg/meta/repodb/common" "zotregistry.io/zot/pkg/storage" ) // OnUpdateManifest is called when a new manifest is added. It updates repodb according to the type // of image pushed(normal images, signatues, etc.). In care of any errors, it makes sure to keep // consistency between repodb and the image store. -func OnUpdateManifest(name, reference, mediaType string, digest godigest.Digest, body []byte, +func OnUpdateManifest(repo, reference, mediaType string, digest godigest.Digest, body []byte, storeController storage.StoreController, repoDB repodb.RepoDB, log log.Logger, ) error { - imgStore := storeController.GetImageStore(name) + imgStore := storeController.GetImageStore(repo) // check if image is a signature - isSignature, signatureType, signedManifestDigest, err := storage.CheckIsImageSignature(name, body, reference, + isSignature, signatureType, signedManifestDigest, err := storage.CheckIsImageSignature(repo, body, reference, storeController) if err != nil { if errors.Is(err, zerr.ErrOrphanSignature) { @@ -30,8 +31,8 @@ func OnUpdateManifest(name, reference, mediaType string, digest godigest.Digest, log.Error().Err(err).Msg("can't check if image is a signature or not") - if err := imgStore.DeleteImageManifest(name, reference, false); err != nil { - log.Error().Err(err).Msgf("couldn't remove image manifest %s in repo %s", reference, name) + if err := imgStore.DeleteImageManifest(repo, reference, false); err != nil { + log.Error().Err(err).Msgf("couldn't remove image manifest %s in repo %s", reference, repo) return err } @@ -42,7 +43,7 @@ func OnUpdateManifest(name, reference, mediaType string, digest godigest.Digest, metadataSuccessfullySet := true if isSignature { - err = repoDB.AddManifestSignature(name, signedManifestDigest, repodb.SignatureMetadata{ + err = repoDB.AddManifestSignature(repo, signedManifestDigest, repodb.SignatureMetadata{ SignatureType: signatureType, SignatureDigest: digest.String(), }) @@ -51,7 +52,7 @@ func OnUpdateManifest(name, reference, mediaType string, digest godigest.Digest, metadataSuccessfullySet = false } } else { - err := repodb.SetMetadataFromInput(name, reference, mediaType, digest, body, + err := repodb.SetMetadataFromInput(repo, reference, mediaType, digest, body, imgStore, repoDB, log) if err != nil { metadataSuccessfullySet = false @@ -59,10 +60,10 @@ func OnUpdateManifest(name, reference, mediaType string, digest godigest.Digest, } if !metadataSuccessfullySet { - log.Info().Msgf("uploding image meta was unsuccessful for tag %s in repo %s", reference, name) + log.Info().Msgf("uploding image meta was unsuccessful for tag %s in repo %s", reference, repo) - if err := imgStore.DeleteImageManifest(name, reference, false); err != nil { - log.Error().Err(err).Msgf("couldn't remove image manifest %s in repo %s", reference, name) + if err := imgStore.DeleteImageManifest(repo, reference, false); err != nil { + log.Error().Err(err).Msgf("couldn't remove image manifest %s in repo %s", reference, repo) return err } @@ -76,12 +77,12 @@ func OnUpdateManifest(name, reference, mediaType string, digest godigest.Digest, // OnDeleteManifest is called when a manifest is deleted. It updates repodb according to the type // of image pushed(normal images, signatues, etc.). In care of any errors, it makes sure to keep // consistency between repodb and the image store. -func OnDeleteManifest(name, reference, mediaType string, digest godigest.Digest, manifestBlob []byte, +func OnDeleteManifest(repo, reference, mediaType string, digest godigest.Digest, manifestBlob []byte, storeController storage.StoreController, repoDB repodb.RepoDB, log log.Logger, ) error { - imgStore := storeController.GetImageStore(name) + imgStore := storeController.GetImageStore(repo) - isSignature, signatureType, signedManifestDigest, err := storage.CheckIsImageSignature(name, manifestBlob, + isSignature, signatureType, signedManifestDigest, err := storage.CheckIsImageSignature(repo, manifestBlob, reference, storeController) if err != nil { if errors.Is(err, zerr.ErrOrphanSignature) { @@ -98,7 +99,7 @@ func OnDeleteManifest(name, reference, mediaType string, digest godigest.Digest, manageRepoMetaSuccessfully := true if isSignature { - err = repoDB.DeleteSignature(name, signedManifestDigest, repodb.SignatureMetadata{ + err = repoDB.DeleteSignature(repo, signedManifestDigest, repodb.SignatureMetadata{ SignatureDigest: digest.String(), SignatureType: signatureType, }) @@ -107,22 +108,31 @@ func OnDeleteManifest(name, reference, mediaType string, digest godigest.Digest, manageRepoMetaSuccessfully = false } } else { - err = repoDB.DeleteRepoTag(name, reference) + err = repoDB.DeleteRepoTag(repo, reference) if err != nil { log.Info().Msg("repodb: restoring image store") // restore image store - _, err := imgStore.PutImageManifest(name, reference, mediaType, manifestBlob) + _, err := imgStore.PutImageManifest(repo, reference, mediaType, manifestBlob) if err != nil { log.Error().Err(err).Msg("repodb: error while restoring image store, database is not consistent") } manageRepoMetaSuccessfully = false } + + if refferredDigest, hasSubject := common.GetReferredSubject(manifestBlob); hasSubject { + err := repoDB.DeleteReferrer(repo, refferredDigest, digest) + if err != nil { + log.Error().Err(err).Msg("repodb: error while deleting referrer") + + return err + } + } } if !manageRepoMetaSuccessfully { - log.Info().Msgf("repodb: deleting image meta was unsuccessful for tag %s in repo %s", reference, name) + log.Info().Msgf("repodb: deleting image meta was unsuccessful for tag %s in repo %s", reference, repo) return err } diff --git a/pkg/meta/repodb/update/update_test.go b/pkg/meta/repodb/update/update_test.go index 8c184746..769a5b11 100644 --- a/pkg/meta/repodb/update/update_test.go +++ b/pkg/meta/repodb/update/update_test.go @@ -6,9 +6,9 @@ import ( "testing" "time" + notreg "github.com/notaryproject/notation-go/registry" godigest "github.com/opencontainers/go-digest" ispec "github.com/opencontainers/image-spec/specs-go/v1" - oras "github.com/oras-project/artifacts-spec/specs-go/v1" . "github.com/smartystreets/goconvey/convey" zerr "zotregistry.io/zot/errors" @@ -95,10 +95,11 @@ func TestUpdateErrors(t *testing.T) { log := log.NewLogger("debug", "") Convey("zerr.ErrOrphanSignature", func() { - manifestContent := oras.Manifest{ - Subject: &oras.Descriptor{ + manifestContent := ispec.Artifact{ + Subject: &ispec.Descriptor{ Digest: "123", }, + ArtifactType: notreg.ArtifactTypeNotation, } manifestBlob, err := json.Marshal(manifestContent) So(err, ShouldBeNil) @@ -120,10 +121,11 @@ func TestUpdateErrors(t *testing.T) { log := log.NewLogger("debug", "") Convey("CheckIsImageSignature errors", func() { - manifestContent := oras.Manifest{ - Subject: &oras.Descriptor{ + manifestContent := ispec.Artifact{ + Subject: &ispec.Descriptor{ Digest: "123", }, + ArtifactType: notreg.ArtifactTypeNotation, } manifestBlob, err := json.Marshal(manifestContent) So(err, ShouldBeNil) @@ -143,6 +145,25 @@ func TestUpdateErrors(t *testing.T) { err = repoDBUpdate.OnDeleteManifest("repo", "tag1", "digest", "media", manifestBlob, storeController, repoDB, log) So(err, ShouldNotBeNil) + + imageStore.GetImageManifestFn = func(repo, reference string) ([]byte, godigest.Digest, string, error) { + return []byte{}, "", "", zerr.ErrManifestNotFound + } + + err = repoDBUpdate.OnDeleteManifest("repo", "tag1", "digest", "media", manifestBlob, + storeController, repoDB, log) + So(err, ShouldNotBeNil) + }) + + Convey("DeleteReferrers errors", func() { + repoDB.DeleteReferrerFn = func(repo string, referredDigest, referrerDigest godigest.Digest) error { + return ErrTestError + } + + err := repoDBUpdate.OnDeleteManifest("repo", "tag1", "digest", "media", + []byte(`{"subject": {"digest": "dig"}}`), + storeController, repoDB, log) + So(err, ShouldNotBeNil) }) }) @@ -153,10 +174,11 @@ func TestUpdateErrors(t *testing.T) { log := log.NewLogger("debug", "") Convey("CheckIsImageSignature errors", func() { - manifestContent := oras.Manifest{ - Subject: &oras.Descriptor{ + manifestContent := ispec.Artifact{ + Subject: &ispec.Descriptor{ Digest: "123", }, + ArtifactType: notreg.ArtifactTypeNotation, } manifestBlob, err := json.Marshal(manifestContent) So(err, ShouldBeNil) @@ -205,5 +227,56 @@ func TestUpdateErrors(t *testing.T) { manifestBlob, imageStore, repoDB, log) So(err, ShouldBeNil) }) + + Convey("SetMetadataFromInput SetData errors", func() { + imageStore := mocks.MockedImageStore{} + log := log.NewLogger("debug", "") + + repoDB := mocks.RepoDBMock{ + SetManifestDataFn: func(manifestDigest godigest.Digest, mm repodb.ManifestData) error { + return ErrTestError + }, + } + err := repodb.SetMetadataFromInput("repo", "ref", ispec.MediaTypeImageManifest, "digest", + []byte("{}"), imageStore, repoDB, log) + So(err, ShouldNotBeNil) + + repoDB = mocks.RepoDBMock{ + SetIndexDataFn: func(digest godigest.Digest, indexData repodb.IndexData) error { + return ErrTestError + }, + } + err = repodb.SetMetadataFromInput("repo", "ref", ispec.MediaTypeImageIndex, "digest", + []byte("{}"), imageStore, repoDB, log) + So(err, ShouldNotBeNil) + + repoDB = mocks.RepoDBMock{ + SetArtifactDataFn: func(digest godigest.Digest, artifactData repodb.ArtifactData) error { + return ErrTestError + }, + } + err = repodb.SetMetadataFromInput("repo", "ref", ispec.MediaTypeArtifactManifest, "digest", + []byte("{}"), imageStore, repoDB, log) + So(err, ShouldNotBeNil) + }) + + Convey("SetMetadataFromInput SetReferrer errors", func() { + imageStore := mocks.MockedImageStore{ + GetBlobContentFn: func(repo string, digest godigest.Digest) ([]byte, error) { + return []byte("{}"), nil + }, + } + log := log.NewLogger("debug", "") + + repoDB := mocks.RepoDBMock{ + SetReferrerFn: func(repo string, referredDigest godigest.Digest, referrer repodb.Descriptor) error { + return ErrTestError + }, + } + + err := repodb.SetMetadataFromInput("repo", "ref", ispec.MediaTypeImageManifest, "digest", + []byte(`{"subject": {"digest": "subjDigest"}}`), imageStore, repoDB, log) + So(err, ShouldNotBeNil) + }) }) } diff --git a/pkg/meta/repodb/version/version_test.go b/pkg/meta/repodb/version/version_test.go index ea401e08..09bb1f5b 100644 --- a/pkg/meta/repodb/version/version_test.go +++ b/pkg/meta/repodb/version/version_test.go @@ -119,6 +119,7 @@ func TestVersioningDynamoDB(t *testing.T) { Region: region, RepoMetaTablename: "RepoMetadataTable", ManifestDataTablename: "ManifestDataTable", + ArtifactDataTablename: "ArtifactDataTable", IndexDataTablename: "IndexDataTable", VersionTablename: "Version", }) diff --git a/pkg/storage/common.go b/pkg/storage/common.go index 2ce1d960..5953e95a 100644 --- a/pkg/storage/common.go +++ b/pkg/storage/common.go @@ -7,6 +7,7 @@ import ( "strings" "github.com/gobwas/glob" + notreg "github.com/notaryproject/notation-go/registry" godigest "github.com/opencontainers/go-digest" imeta "github.com/opencontainers/image-spec/specs-go" ispec "github.com/opencontainers/image-spec/specs-go/v1" @@ -18,6 +19,12 @@ import ( storageConstants "zotregistry.io/zot/pkg/storage/constants" ) +func SignatureMediaTypes() map[string]bool { + return map[string]bool{ + notreg.ArtifactTypeNotation: true, + } +} + func GetTagsByIndex(index ispec.Index) []string { tags := make([]string, 0) @@ -711,7 +718,7 @@ func IsSupportedMediaType(mediaType string) bool { mediaType == oras.MediaTypeArtifactManifest } -// imageIsSignature checks if the given image (repo:tag) represents a signature. The function +// CheckIsImageSignature checks if the given image (repo:tag) represents a signature. The function // returns: // // - bool: if the image is a signature or not @@ -726,7 +733,7 @@ func CheckIsImageSignature(repoName string, manifestBlob []byte, reference strin ) (bool, string, godigest.Digest, error) { const cosign = "cosign" - var manifestContent oras.Manifest + var manifestContent ispec.Artifact err := json.Unmarshal(manifestBlob, &manifestContent) if err != nil { @@ -734,7 +741,7 @@ func CheckIsImageSignature(repoName string, manifestBlob []byte, reference strin } // check notation signature - if manifestContent.Subject != nil { + if _, ok := SignatureMediaTypes()[manifestContent.ArtifactType]; ok && manifestContent.Subject != nil { imgStore := storeController.GetImageStore(repoName) _, signedImageManifestDigest, _, err := imgStore.GetImageManifest(repoName, diff --git a/pkg/test/common.go b/pkg/test/common.go index 835a74b3..26004561 100644 --- a/pkg/test/common.go +++ b/pkg/test/common.go @@ -105,6 +105,37 @@ func (img Image) Digest() (godigest.Digest, error) { return godigest.FromBytes(blob), nil } +type Artifact struct { + Manifest ispec.Artifact + Blobs []ArtifactBlobs + Reference string +} + +func (a Artifact) Digest() (godigest.Digest, error) { + blob, err := json.Marshal(a.Manifest) + if err != nil { + return "", err + } + + return godigest.FromBytes(blob), nil +} + +func (a Artifact) ArtifactData() (repodb.ArtifactData, error) { + blob, err := json.Marshal(a.Manifest) + if err != nil { + return repodb.ArtifactData{}, err + } + + return repodb.ArtifactData{ + ManifestBlob: blob, + }, nil +} + +type ArtifactBlobs struct { + Blob []byte + MediaType string +} + type MultiarchImage struct { Index ispec.Index Images []Image @@ -623,6 +654,22 @@ func GetRandomImageComponents(layerSize int) (ispec.Image, [][]byte, ispec.Manif return config, layers, manifest, nil } +func GetRandomImage(reference string) (Image, error) { + const layerSize = 20 + + config, layers, manifest, err := GetRandomImageComponents(layerSize) + if err != nil { + return Image{}, err + } + + return Image{ + Manifest: manifest, + Layers: layers, + Config: config, + Reference: reference, + }, nil +} + func GetImageComponentsWithConfig(conf ispec.Image) (ispec.Image, [][]byte, ispec.Manifest, error) { configBlob, err := json.Marshal(conf) if err = Error(err); err != nil { @@ -728,6 +775,49 @@ func GetImageWithComponents(config ispec.Image, layers [][]byte) (Image, error) }, nil } +func GetRandomArtifact(subject *ispec.Descriptor) (Artifact, error) { + var randBlob [10]byte + + _, err := rand.Read(randBlob[:]) + if err != nil { + return Artifact{}, err + } + + artifactBlobs := []ArtifactBlobs{ + { + Blob: randBlob[:], + MediaType: "application/octet-stream", + }, + } + + blobsDescriptors := make([]ispec.Descriptor, 0, len(artifactBlobs)) + + for _, artifactBlob := range artifactBlobs { + blobsDescriptors = append(blobsDescriptors, ispec.Descriptor{ + Digest: godigest.FromBytes(artifactBlob.Blob), + MediaType: artifactBlob.MediaType, + Size: int64(len(artifactBlob.Blob)), + }) + } + + artifactManifest := ispec.Artifact{ + MediaType: ispec.MediaTypeArtifactManifest, + Blobs: blobsDescriptors, + Subject: subject, + } + + artifactManifestBlob, err := json.Marshal(artifactManifest) + if err != nil { + return Artifact{}, err + } + + return Artifact{ + Manifest: artifactManifest, + Blobs: artifactBlobs, + Reference: godigest.FromBytes(artifactManifestBlob).String(), + }, nil +} + func GetCosignSignatureTagForManifest(manifest ispec.Manifest) (string, error) { manifestBlob, err := json.Marshal(manifest) if err != nil { @@ -743,6 +833,32 @@ func GetCosignSignatureTagForDigest(manifestDigest godigest.Digest) string { return manifestDigest.Algorithm().String() + "-" + manifestDigest.Encoded() + ".sig" } +func GetImageWithSubject(subjectDigest godigest.Digest, mediaType string) (Image, error) { + num := 100 + + conf, layers, manifest, err := GetRandomImageComponents(num) + if err != nil { + return Image{}, err + } + + manifest.Subject = &ispec.Descriptor{ + Digest: subjectDigest, + MediaType: mediaType, + } + + manifestBlob, err := json.Marshal(manifest) + if err != nil { + return Image{}, err + } + + return Image{ + Manifest: manifest, + Config: conf, + Layers: layers, + Reference: godigest.FromBytes(manifestBlob).String(), + }, nil +} + func UploadImage(img Image, baseURL, repo string) error { for _, blob := range img.Layers { resp, err := resty.R().Post(baseURL + "/v2/" + repo + "/blobs/uploads/") @@ -830,7 +946,19 @@ func UploadImage(img Image, baseURL, repo string) error { return err } -func UploadArtifact(baseURL, repo string, artifactManifest *ispec.Artifact) error { +func DeleteImage(repo, reference, baseURL string) (int, error) { + resp, err := resty.R().Delete( + fmt.Sprintf(baseURL+"/v2/%s/manifests/%s", repo, reference), + ) + if err != nil { + return -1, err + } + + return resp.StatusCode(), err +} + +// UploadArtifactManifest is used in tests where we don't need to upload the blobs of the artifact. +func UploadArtifactManifest(artifactManifest *ispec.Artifact, baseURL, repo string) error { // put manifest artifactManifestBlob, err := json.Marshal(artifactManifest) if err != nil { diff --git a/pkg/test/common_test.go b/pkg/test/common_test.go index 5df53ebf..6a577a70 100644 --- a/pkg/test/common_test.go +++ b/pkg/test/common_test.go @@ -8,6 +8,7 @@ import ( "encoding/json" "errors" "fmt" + "io" "os" "path" "testing" @@ -21,10 +22,12 @@ import ( "zotregistry.io/zot/pkg/api" "zotregistry.io/zot/pkg/api/config" + "zotregistry.io/zot/pkg/storage" "zotregistry.io/zot/pkg/test" + "zotregistry.io/zot/pkg/test/mocks" ) -var ErrTestError = errors.New("test error") +var ErrTestError = errors.New("ErrTestError") func TestCopyFiles(t *testing.T) { Convey("sourceDir does not exist", t, func() { @@ -238,7 +241,7 @@ func TestUploadArtifact(t *testing.T) { artifact := ispec.Artifact{} - err := test.UploadArtifact(baseURL, "test", &artifact) + err := test.UploadArtifactManifest(&artifact, baseURL, "test") So(err, ShouldNotBeNil) }) } @@ -1274,3 +1277,62 @@ func TestGenerateNotationCerts(t *testing.T) { So(err, ShouldNotBeNil) }) } + +func TestWriteImageToFileSystem(t *testing.T) { + Convey("WriteImageToFileSystem errors", t, func() { + err := test.WriteImageToFileSystem(test.Image{}, "repo", storage.StoreController{ + DefaultStore: mocks.MockedImageStore{ + InitRepoFn: func(name string) error { + return ErrTestError + }, + }, + }) + So(err, ShouldNotBeNil) + + err = test.WriteImageToFileSystem( + test.Image{Layers: [][]byte{[]byte("testLayer")}}, + "repo", + storage.StoreController{ + DefaultStore: mocks.MockedImageStore{ + FullBlobUploadFn: func(repo string, body io.Reader, digest godigest.Digest, + ) (string, int64, error) { + return "", 0, ErrTestError + }, + }, + }) + So(err, ShouldNotBeNil) + + count := 0 + err = test.WriteImageToFileSystem( + test.Image{Layers: [][]byte{[]byte("testLayer")}}, + "repo", + storage.StoreController{ + DefaultStore: mocks.MockedImageStore{ + FullBlobUploadFn: func(repo string, body io.Reader, digest godigest.Digest, + ) (string, int64, error) { + if count == 0 { + count++ + + return "", 0, nil + } + + return "", 0, ErrTestError + }, + }, + }) + So(err, ShouldNotBeNil) + + err = test.WriteImageToFileSystem( + test.Image{Layers: [][]byte{[]byte("testLayer")}}, + "repo", + storage.StoreController{ + DefaultStore: mocks.MockedImageStore{ + PutImageManifestFn: func(repo, reference, mediaType string, body []byte, + ) (godigest.Digest, error) { + return "", ErrTestError + }, + }, + }) + So(err, ShouldNotBeNil) + }) +} diff --git a/pkg/test/mocks/repo_db_mock.go b/pkg/test/mocks/repo_db_mock.go index 81526e8b..f741b51a 100644 --- a/pkg/test/mocks/repo_db_mock.go +++ b/pkg/test/mocks/repo_db_mock.go @@ -40,6 +40,19 @@ type RepoDBMock struct { GetIndexDataFn func(indexDigest godigest.Digest) (repodb.IndexData, error) + SetArtifactDataFn func(digest godigest.Digest, artifactData repodb.ArtifactData) error + + GetArtifactDataFn func(artifactDigest godigest.Digest) (repodb.ArtifactData, error) + + SetReferrerFn func(repo string, referredDigest godigest.Digest, referrer repodb.Descriptor) error + + DeleteReferrerFn func(repo string, referredDigest godigest.Digest, referrerDigest godigest.Digest) error + + GetReferrersFn func(repo string, referredDigest godigest.Digest) ([]repodb.Descriptor, error) + + GetFilteredReferrersInfoFn func(repo string, referredDigest godigest.Digest, artifactTypes []string) ( + []repodb.ReferrerInfo, error) + IncrementImageDownloadsFn func(repo string, reference string) error AddManifestSignatureFn func(repo string, signedManifestDigest godigest.Digest, sm repodb.SignatureMetadata) error @@ -103,14 +116,6 @@ func (sdm RepoDBMock) GetRepoStars(repo string) (int, error) { return 0, nil } -func (sdm RepoDBMock) SetRepoLogo(repo string, logoPath string) error { - if sdm.SetRepoLogoFn != nil { - return sdm.SetRepoLogoFn(repo, logoPath) - } - - return nil -} - func (sdm RepoDBMock) SetRepoReference(repo string, reference string, manifestDigest godigest.Digest, mediaType string, ) error { @@ -300,3 +305,55 @@ func (sdm RepoDBMock) PatchDB() error { return nil } + +func (sdm RepoDBMock) SetArtifactData(digest godigest.Digest, artifactData repodb.ArtifactData) error { + if sdm.SetArtifactDataFn != nil { + return sdm.SetArtifactDataFn(digest, artifactData) + } + + return nil +} + +func (sdm RepoDBMock) GetArtifactData(artifactDigest godigest.Digest) (repodb.ArtifactData, error) { + if sdm.GetArtifactDataFn != nil { + return sdm.GetArtifactDataFn(artifactDigest) + } + + return repodb.ArtifactData{}, nil +} + +func (sdm RepoDBMock) SetReferrer(repo string, referredDigest godigest.Digest, referrer repodb.Descriptor) error { + if sdm.SetReferrerFn != nil { + return sdm.SetReferrerFn(repo, referredDigest, referrer) + } + + return nil +} + +func (sdm RepoDBMock) DeleteReferrer(repo string, referredDigest godigest.Digest, + referrerDigest godigest.Digest, +) error { + if sdm.DeleteReferrerFn != nil { + return sdm.DeleteReferrerFn(repo, referredDigest, referrerDigest) + } + + return nil +} + +func (sdm RepoDBMock) GetReferrers(repo string, referredDigest godigest.Digest) ([]repodb.Descriptor, error) { + if sdm.GetReferrersFn != nil { + return sdm.GetReferrersFn(repo, referredDigest) + } + + return []repodb.Descriptor{}, nil +} + +func (sdm RepoDBMock) GetFilteredReferrersInfo(repo string, referredDigest godigest.Digest, + artifactTypes []string, +) ([]repodb.ReferrerInfo, error) { + if sdm.GetFilteredReferrersInfoFn != nil { + return sdm.GetFilteredReferrersInfoFn(repo, referredDigest, artifactTypes) + } + + return []repodb.ReferrerInfo{}, nil +} diff --git a/test/blackbox/cloud-only.bats b/test/blackbox/cloud-only.bats index e219db3d..4535f539 100644 --- a/test/blackbox/cloud-only.bats +++ b/test/blackbox/cloud-only.bats @@ -37,6 +37,7 @@ function setup() { "cacheTablename": "BlobTable", "repoMetaTablename": "RepoMetadataTable", "manifestDataTablename": "ManifestDataTable", + "artifactDataTablename": "ArtifactDataTable", "indexDataTablename": "IndexDataTable", "versionTablename": "Version" } diff --git a/test/blackbox/helpers_referrers.bash b/test/blackbox/helpers_referrers.bash new file mode 100644 index 00000000..5b6263d4 --- /dev/null +++ b/test/blackbox/helpers_referrers.bash @@ -0,0 +1,80 @@ +ROOT_DIR=$(git rev-parse --show-toplevel) +TEST_DATA_DIR=${ROOT_DIR}/test/data/ +OS="${OS:-linux}" +ARCH="${ARCH:-amd64}" +ZOT_PATH=${ROOT_DIR}/bin/zot-${OS}-${ARCH} + +mkdir -p ${TEST_DATA_DIR} + +function verify_prerequisites { + if [ ! -f ${BATS_RUN_TMPDIR}/.firstrun ]; then + env | grep proxy >&3 + touch ${BATS_RUN_TMPDIR}/.firstrun + fi + + if [ ! -f ${ZOT_PATH} ]; then + echo "you need to build ${ZOT_PATH} before running the tests" >&3 + return 1 + fi + + if [ ! -f ${ZOT_MINIMAL_PATH} ]; then + echo "you need to build ${ZOT_MINIMAL_PATH} before running tests" >&3 + return 1 + fi + + if [ ! command -v curl ] &>/dev/null; then + echo "you need to install curl as a prerequisite to running the tests" >&3 + return 1 + fi + + if [ ! command -v jq ] &>/dev/null; then + echo "you need to install jq as a prerequisite to running the tests" >&3 + return 1 + fi + + if [ ! command -v skopeo ] &>/dev/null; then + echo "you need to install skopeo as a prerequisite to running the tests" >&3 + return 1 + fi + + if [ ! command -v oras ] &>/dev/null; then + echo "you need to install oras as a prerequisite to running the tests" >&3 + return 1 + fi + return 0 +} + +function zot_serve() { + local zot_path=${1} + local config_file=${2} + local pid_dir=${3} + ${zot_path} serve ${config_file} & + echo $! >>${pid_dir}/zot.pid +} + +function zot_stop() { + local pid_dir=${1} + cat ${pid_dir}/zot.pid + kill $(cat ${pid_dir}/zot.pid) + rm ${pid_dir}/zot.pid +} + +function setup_zot_file_level() { + local config_file=${1} + zot_serve ${ZOT_PATH} ${config_file} ${BATS_FILE_TMPDIR} +} + +function teardown_zot_file_level() { + zot_stop ${BATS_FILE_TMPDIR} +} + +function wait_zot_reachable() { + zot_url=${1} + curl --connect-timeout 3 \ + --max-time 3 \ + --retry 10 \ + --retry-delay 0 \ + --retry-max-time 60 \ + --retry-connrefused \ + ${zot_url} +} diff --git a/test/blackbox/referrers.bats b/test/blackbox/referrers.bats new file mode 100644 index 00000000..6373f5f5 --- /dev/null +++ b/test/blackbox/referrers.bats @@ -0,0 +1,109 @@ +load helpers_referrers + +function setup() { + # Verify prerequisites are available + if ! verify_prerequisites; then + exit 1 + fi + + # Download test data to folder common for the entire suite, not just this file + skopeo --insecure-policy copy --format=oci docker://alpine:latest oci:${TEST_DATA_DIR}alpine:latest + + # Setup zot server + ZOT_ROOT_DIR=${BATS_FILE_TMPDIR}/zot + echo ${ZOT_ROOT_DIR} + ZOT_LOG_FILE=${ZOT_ROOT_DIR}/zot-log.json + ZOT_CONFIG_FILE=${BATS_FILE_TMPDIR}/zot_config.json + mkdir -p ${ZOT_ROOT_DIR} + touch ${ZOT_LOG_FILE} + cat >${ZOT_CONFIG_FILE} <${IMAGE_MANIFEST_REFERRER} <${ARTIFACT_MANIFEST_REFERRER} <