From 08983a845a0e6710e9aade539fed85ea0305fc18 Mon Sep 17 00:00:00 2001 From: Andrei Aaron Date: Mon, 23 Jan 2023 19:45:11 +0200 Subject: [PATCH] feat(repodb): implement pagination for ImageList and integrate it with RepoDB (#1129) * feat(repodb): implement pagination for ImageList and integrate it with RepoDB - it can now return all images from all repos, when provided repo parameter is "" Signed-off-by: Alex Stan (cherry picked from commit c003dcec9f805564946935e7eb091632f605035e) (cherry picked from commit 72feba979b9ddd452465a652bb31f439584a046c) Signed-off-by: Andrei Aaron * ci(timeouts): increase ci-cd workflow timeout for the build and test step Signed-off-by: Andrei Aaron Signed-off-by: Alex Stan Signed-off-by: Andrei Aaron Co-authored-by: Alex Stan --- .github/workflows/ci-cd.yml | 2 +- pkg/cli/image_cmd_test.go | 44 +-- pkg/extensions/search/common/common_test.go | 350 ++++++------------ pkg/extensions/search/convert/convert_test.go | 278 ++++++++++++++ .../search/gql_generated/generated.go | 21 +- pkg/extensions/search/resolver.go | 95 ++--- pkg/extensions/search/resolver_test.go | 143 +++++-- pkg/extensions/search/schema.graphql | 4 +- pkg/extensions/search/schema.resolvers.go | 29 +- 9 files changed, 569 insertions(+), 397 deletions(-) diff --git a/.github/workflows/ci-cd.yml b/.github/workflows/ci-cd.yml index 6fd4fb84..24390d1c 100644 --- a/.github/workflows/ci-cd.yml +++ b/.github/workflows/ci-cd.yml @@ -87,7 +87,7 @@ jobs: AWS_ACCESS_KEY_ID: fake AWS_SECRET_ACCESS_KEY: fake - name: Run build and test - timeout-minutes: 70 + timeout-minutes: 80 run: | echo "Building for $OS:$ARCH" cd $GITHUB_WORKSPACE diff --git a/pkg/cli/image_cmd_test.go b/pkg/cli/image_cmd_test.go index 7934686f..cc292647 100644 --- a/pkg/cli/image_cmd_test.go +++ b/pkg/cli/image_cmd_test.go @@ -368,7 +368,7 @@ func TestSignature(t *testing.T) { str := space.ReplaceAllString(buff.String(), " ") actual := strings.TrimSpace(str) So(actual, ShouldContainSubstring, "IMAGE NAME TAG DIGEST SIGNED SIZE") - So(actual, ShouldContainSubstring, "repo7 test:1.0 6742241d true 1B") + So(actual, ShouldContainSubstring, "repo7 test:1.0 6742241d true 447B") t.Log("Test getting all images using rest calls to get catalog and individual manifests") cmd = MockNewImageCommand(new(searchService)) @@ -445,7 +445,7 @@ func TestSignature(t *testing.T) { str := space.ReplaceAllString(buff.String(), " ") actual := strings.TrimSpace(str) So(actual, ShouldContainSubstring, "IMAGE NAME TAG DIGEST SIGNED SIZE") - So(actual, ShouldContainSubstring, "repo7 0.0.1 6742241d true 1B") + So(actual, ShouldContainSubstring, "repo7 0.0.1 6742241d true 447B") t.Log("Test getting all images using rest calls to get catalog and individual manifests") cmd = MockNewImageCommand(new(searchService)) @@ -913,8 +913,8 @@ func TestServerResponseGQL(t *testing.T) { str := space.ReplaceAllString(buff.String(), " ") actual := strings.TrimSpace(str) So(actual, ShouldContainSubstring, "IMAGE NAME TAG DIGEST SIGNED SIZE") - So(actual, ShouldContainSubstring, "repo7 test:2.0 883fc0c5 false 15B") - So(actual, ShouldContainSubstring, "repo7 test:1.0 883fc0c5 false 15B") + So(actual, ShouldContainSubstring, "repo7 test:2.0 883fc0c5 false 492B") + So(actual, ShouldContainSubstring, "repo7 test:1.0 883fc0c5 false 492B") Convey("Test all images invalid output format", func() { args := []string{"imagetest", "-o", "random"} configPath := makeConfigFile(fmt.Sprintf(`{"configs":[{"_name":"imagetest","url":"%s","showspinner":false}]}`, url)) @@ -945,14 +945,14 @@ func TestServerResponseGQL(t *testing.T) { str := space.ReplaceAllString(buff.String(), " ") actual := strings.TrimSpace(str) // Actual cli output should be something similar to (order of images may differ): - // IMAGE NAME TAG DIGEST CONFIG LAYERS SIZE - // repo7 test:2.0 a0ca253b b8781e88 15B - // b8781e88 15B - // repo7 test:1.0 a0ca253b b8781e88 15B - // b8781e88 15B + // IMAGE NAME TAG DIGEST CONFIG SIGNED LAYERS SIZE + // repo7 test:2.0 a0ca253b b8781e88 false 492B + // b8781e88 15B + // repo7 test:1.0 a0ca253b b8781e88 false 492B + // b8781e88 15B So(actual, ShouldContainSubstring, "IMAGE NAME TAG DIGEST CONFIG SIGNED LAYERS SIZE") - So(actual, ShouldContainSubstring, "repo7 test:2.0 883fc0c5 3a1d2d0c false 15B b8781e88 15B") - So(actual, ShouldContainSubstring, "repo7 test:1.0 883fc0c5 3a1d2d0c false 15B b8781e88 15B") + So(actual, ShouldContainSubstring, "repo7 test:2.0 883fc0c5 3a1d2d0c false 492B b8781e88 15B") + So(actual, ShouldContainSubstring, "repo7 test:1.0 883fc0c5 3a1d2d0c false 492B b8781e88 15B") }) Convey("Test all images with debug flag", func() { @@ -971,8 +971,8 @@ func TestServerResponseGQL(t *testing.T) { actual := strings.TrimSpace(str) So(actual, ShouldContainSubstring, "GET") So(actual, ShouldContainSubstring, "IMAGE NAME TAG DIGEST SIGNED SIZE") - So(actual, ShouldContainSubstring, "repo7 test:2.0 883fc0c5 false 15B") - So(actual, ShouldContainSubstring, "repo7 test:1.0 883fc0c5 false 15B") + So(actual, ShouldContainSubstring, "repo7 test:2.0 883fc0c5 false 492B") + So(actual, ShouldContainSubstring, "repo7 test:1.0 883fc0c5 false 492B") }) Convey("Test image by name config url", func() { @@ -990,8 +990,8 @@ func TestServerResponseGQL(t *testing.T) { str := space.ReplaceAllString(buff.String(), " ") actual := strings.TrimSpace(str) So(actual, ShouldContainSubstring, "IMAGE NAME TAG DIGEST SIGNED SIZE") - So(actual, ShouldContainSubstring, "repo7 test:2.0 883fc0c5 false 15B") - So(actual, ShouldContainSubstring, "repo7 test:1.0 883fc0c5 false 15B") + So(actual, ShouldContainSubstring, "repo7 test:2.0 883fc0c5 false 492B") + So(actual, ShouldContainSubstring, "repo7 test:1.0 883fc0c5 false 492B") Convey("with shorthand", func() { args := []string{"imagetest", "-n", "repo7"} @@ -1008,8 +1008,8 @@ func TestServerResponseGQL(t *testing.T) { str := space.ReplaceAllString(buff.String(), " ") actual := strings.TrimSpace(str) So(actual, ShouldContainSubstring, "IMAGE NAME TAG DIGEST SIGNED SIZE") - So(actual, ShouldContainSubstring, "repo7 test:2.0 883fc0c5 false 15B") - So(actual, ShouldContainSubstring, "repo7 test:1.0 883fc0c5 false 15B") + So(actual, ShouldContainSubstring, "repo7 test:2.0 883fc0c5 false 492B") + So(actual, ShouldContainSubstring, "repo7 test:1.0 883fc0c5 false 492B") }) Convey("invalid output format", func() { @@ -1193,11 +1193,11 @@ func TestServerResponse(t *testing.T) { str := space.ReplaceAllString(buff.String(), " ") actual := strings.TrimSpace(str) // Actual cli output should be something similar to (order of images may differ): - // IMAGE NAME TAG DIGEST CONFIG LAYERS SIZE - // repo7 test:2.0 a0ca253b b8781e88 492B - // b8781e88 15B - // repo7 test:1.0 a0ca253b b8781e88 492B - // b8781e88 15B + // IMAGE NAME TAG DIGEST CONFIG SIGNED LAYERS SIZE + // repo7 test:2.0 a0ca253b b8781e88 false 492B + // b8781e88 15B + // repo7 test:1.0 a0ca253b b8781e88 false 492B + // b8781e88 15B So(actual, ShouldContainSubstring, "IMAGE NAME TAG DIGEST CONFIG SIGNED LAYERS SIZE") So(actual, ShouldContainSubstring, "repo7 test:2.0 883fc0c5 3a1d2d0c false 492B b8781e88 15B") So(actual, ShouldContainSubstring, "repo7 test:1.0 883fc0c5 3a1d2d0c false 492B b8781e88 15B") diff --git a/pkg/extensions/search/common/common_test.go b/pkg/extensions/search/common/common_test.go index 6e737bf9..2d0f562c 100644 --- a/pkg/extensions/search/common/common_test.go +++ b/pkg/extensions/search/common/common_test.go @@ -34,7 +34,6 @@ import ( extconf "zotregistry.io/zot/pkg/extensions/config" "zotregistry.io/zot/pkg/extensions/monitoring" "zotregistry.io/zot/pkg/extensions/search/common" - "zotregistry.io/zot/pkg/extensions/search/convert" "zotregistry.io/zot/pkg/log" "zotregistry.io/zot/pkg/meta/repodb" "zotregistry.io/zot/pkg/storage" @@ -3169,146 +3168,69 @@ func TestImageList(t *testing.T) { err = json.Unmarshal(imageConfigBuf, &imageConfigInfo) So(err, ShouldBeNil) - query := fmt.Sprintf(`{ - ImageList(repo:"%s"){ - History{ - HistoryDescription{ - Author - Comment - Created - CreatedBy - EmptyLayer - }, - Layer{ - Digest - Size + Convey("without pagination, valid response", func() { + query := fmt.Sprintf(`{ + ImageList(repo:"%s"){ + History{ + HistoryDescription{ + Author + Comment + Created + CreatedBy + EmptyLayer + }, + Layer{ + Digest + Size + } } } - } - }`, repos[0]) + }`, repos[0]) - resp, err := resty.R().Get(baseURL + graphqlQueryPrefix + "?query=" + url.QueryEscape(query)) - So(err, ShouldBeNil) - So(resp.StatusCode(), ShouldEqual, 200) - So(resp, ShouldNotBeNil) + resp, err := resty.R().Get(baseURL + graphqlQueryPrefix + "?query=" + url.QueryEscape(query)) + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, 200) + So(resp, ShouldNotBeNil) - var responseStruct ImageListResponse - err = json.Unmarshal(resp.Body(), &responseStruct) - So(err, ShouldBeNil) + var responseStruct ImageListResponse + err = json.Unmarshal(resp.Body(), &responseStruct) + So(err, ShouldBeNil) - So(len(responseStruct.ImageList.SummaryList[0].History), ShouldEqual, len(imageConfigInfo.History)) - }) + So(len(responseStruct.ImageList.SummaryList), ShouldEqual, len(tags)) + So(len(responseStruct.ImageList.SummaryList[0].History), ShouldEqual, len(imageConfigInfo.History)) + }) - Convey("Test ImageSummary retuned by ImageList when getting tags timestamp info fails", t, func() { - invalid := "test" - tempDir := t.TempDir() - port := GetFreePort() - baseURL := GetBaseURL(port) - - conf := config.New() - conf.HTTP.Port = port - conf.Storage.RootDirectory = tempDir - defaultVal := true - conf.Extensions = &extconf.ExtensionConfig{ - Search: &extconf.SearchConfig{BaseConfig: extconf.BaseConfig{Enable: &defaultVal}}, - } - - conf.Extensions.Search.CVE = nil - - ctlr := api.NewController(conf) - - ctlrManager := NewControllerManager(ctlr) - ctlrManager.StartAndWait(port) - defer ctlrManager.StopServer() - - config := ispec.Image{ - Platform: ispec.Platform{ - Architecture: "amd64", - OS: "linux", - }, - RootFS: ispec.RootFS{ - Type: "layers", - DiffIDs: []godigest.Digest{}, - }, - Author: "ZotUser", - History: []ispec.History{}, - } - - configBlob, err := json.Marshal(config) - So(err, ShouldBeNil) - - configDigest := godigest.FromBytes(configBlob) - layerDigest := godigest.FromString(invalid) - layerblob := []byte(invalid) - schemaVersion := 2 - ispecManifest := ispec.Manifest{ - Versioned: specs.Versioned{ - SchemaVersion: schemaVersion, - }, - Config: ispec.Descriptor{ - MediaType: "application/vnd.oci.image.config.v1+json", - Digest: configDigest, - Size: int64(len(configBlob)), - }, - Layers: []ispec.Descriptor{ // just 1 layer in manifest - { - MediaType: "application/vnd.oci.image.layer.v1.tar", - Digest: layerDigest, - Size: int64(len(layerblob)), - }, - }, - Annotations: map[string]string{ - ispec.AnnotationRefName: "1.0", - }, - } - - err = UploadImage( - Image{ - Manifest: ispecManifest, - Config: config, - Layers: [][]byte{ - layerblob, - }, - Tag: "0.0.1", - }, - baseURL, - invalid, - ) - So(err, ShouldBeNil) - - configPath := path.Join(conf.Storage.RootDirectory, invalid, "blobs", - configDigest.Algorithm().String(), configDigest.Encoded()) - - err = os.Remove(configPath) - So(err, ShouldBeNil) - - query := fmt.Sprintf(`{ - ImageList(repo:"%s"){ - History{ - HistoryDescription{ - Author - Comment - Created - CreatedBy - EmptyLayer - }, - Layer{ - Digest - Size + Convey("Pagination with valid params", func() { + limit := 1 + query := fmt.Sprintf(`{ + ImageList(repo:"%s", requestedPage:{limit: %d, offset: 0, sortBy:RELEVANCE}){ + History{ + HistoryDescription{ + Author + Comment + Created + CreatedBy + EmptyLayer + }, + Layer{ + Digest + Size + } } } - } - }`, invalid) + }`, repos[0], limit) - resp, err := resty.R().Get(baseURL + graphqlQueryPrefix + "?query=" + url.QueryEscape(query)) - So(err, ShouldBeNil) - So(resp.StatusCode(), ShouldEqual, 200) - So(resp, ShouldNotBeNil) + resp, err := resty.R().Get(baseURL + graphqlQueryPrefix + "?query=" + url.QueryEscape(query)) + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, 200) + So(resp, ShouldNotBeNil) - var responseStruct ImageListResponse - err = json.Unmarshal(resp.Body(), &responseStruct) - So(err, ShouldBeNil) - So(len(responseStruct.ImageList.SummaryList), ShouldBeZeroValue) + var responseStruct ImageListResponse + err = json.Unmarshal(resp.Body(), &responseStruct) + So(err, ShouldBeNil) + + So(len(responseStruct.ImageList.SummaryList), ShouldEqual, limit) + }) }) } @@ -3504,114 +3426,6 @@ func TestGlobalSearchPagination(t *testing.T) { }) } -func TestBuildImageInfo(t *testing.T) { - Convey("Check image summary when layer count does not match history", t, func() { - invalid := "invalid" - - port := GetFreePort() - baseURL := GetBaseURL(port) - rootDir = t.TempDir() - conf := config.New() - conf.HTTP.Port = port - conf.Storage.RootDirectory = rootDir - defaultVal := true - conf.Extensions = &extconf.ExtensionConfig{ - Search: &extconf.SearchConfig{BaseConfig: extconf.BaseConfig{Enable: &defaultVal}}, - } - - conf.Extensions.Search.CVE = nil - - ctlr := api.NewController(conf) - - ctlrManager := NewControllerManager(ctlr) - ctlrManager.StartAndWait(port) - defer ctlrManager.StopServer() - - olu := &common.BaseOciLayoutUtils{ - StoreController: ctlr.StoreController, - Log: ctlr.Log, - } - - config := ispec.Image{ - Platform: ispec.Platform{ - OS: "linux", - Architecture: "amd64", - }, - RootFS: ispec.RootFS{ - Type: "layers", - DiffIDs: []godigest.Digest{}, - }, - Author: "ZotUser", - History: []ispec.History{ // should contain 3 elements, 2 of which corresponding to layers - { - EmptyLayer: false, - }, - { - EmptyLayer: false, - }, - { - EmptyLayer: true, - }, - }, - } - - configBlob, err := json.Marshal(config) - So(err, ShouldBeNil) - - configDigest := godigest.FromBytes(configBlob) - layerDigest := godigest.FromString(invalid) - layerblob := []byte(invalid) - schemaVersion := 2 - ispecManifest := ispec.Manifest{ - Versioned: specs.Versioned{ - SchemaVersion: schemaVersion, - }, - Config: ispec.Descriptor{ - MediaType: "application/vnd.oci.image.config.v1+json", - Digest: configDigest, - Size: int64(len(configBlob)), - }, - Layers: []ispec.Descriptor{ // just 1 layer in manifest - { - MediaType: "application/vnd.oci.image.layer.v1.tar", - Digest: layerDigest, - Size: int64(len(layerblob)), - }, - }, - } - manifestLayersSize := ispecManifest.Layers[0].Size - manifestBlob, err := json.Marshal(ispecManifest) - So(err, ShouldBeNil) - manifestDigest := godigest.FromBytes(manifestBlob) - err = UploadImage( - Image{ - Manifest: ispecManifest, - Config: config, - Layers: [][]byte{ - layerblob, - }, - Tag: "0.0.1", - }, - baseURL, - invalid, - ) - So(err, ShouldBeNil) - - imageConfig, err := olu.GetImageConfigInfo(invalid, manifestDigest) - So(err, ShouldBeNil) - - isSigned := false - - imageSummary := convert.BuildImageInfo(invalid, invalid, manifestDigest, ispecManifest, - imageConfig, isSigned) - - So(len(imageSummary.Layers), ShouldEqual, len(ispecManifest.Layers)) - imageSummaryLayerSize, err := strconv.Atoi(*imageSummary.Size) - So(err, ShouldBeNil) - So(imageSummaryLayerSize, ShouldEqual, manifestLayersSize) - }) -} - func TestRepoDBWhenSigningImages(t *testing.T) { Convey("SigningImages", t, func() { subpath := "/a" @@ -4498,6 +4312,50 @@ func TestBaseOciLayoutUtils(t *testing.T) { So(err, ShouldNotBeNil) }) + Convey("GetImageTagsWithTimestamp: GetImageInfo fails", t, func() { + index := ispec.Index{ + Manifests: []ispec.Descriptor{ + {Annotations: map[string]string{ispec.AnnotationRefName: "w"}}, {}, + }, + } + + indexBlob, err := json.Marshal(index) + So(err, ShouldBeNil) + + manifest := ispec.Manifest{ + Config: ispec.Descriptor{ + MediaType: "application/vnd.oci.image.config.v1+json", + Digest: "configDigest", + }, + Layers: []ispec.Descriptor{ + {}, + {}, + }, + } + + manifestBlob, err := json.Marshal(manifest) + So(err, ShouldBeNil) + + mockStoreController := mocks.MockedImageStore{ + GetBlobContentFn: func(repo string, digest godigest.Digest) ([]byte, error) { + if digest.String() == "configDigest" { + return nil, ErrTestError + } + + return manifestBlob, nil + }, + GetIndexContentFn: func(repo string) ([]byte, error) { + return indexBlob, nil + }, + } + + storeController := storage.StoreController{DefaultStore: mockStoreController} + olu := common.NewBaseOciLayoutUtils(storeController, log.NewLogger("debug", "")) + + _, err = olu.GetImageTagsWithTimestamp("repo") + So(err, ShouldNotBeNil) + }) + Convey("GetExpandedRepoInfo: fails", t, func() { index := ispec.Index{ Manifests: []ispec.Descriptor{ @@ -4570,6 +4428,20 @@ func TestBaseOciLayoutUtils(t *testing.T) { _, err = olu.GetExpandedRepoInfo("rep") So(err, ShouldBeNil) }) + + Convey("GetImageInfo fail", t, func() { + mockStoreController := mocks.MockedImageStore{ + GetBlobContentFn: func(repo string, digest godigest.Digest) ([]byte, error) { + return []byte{}, ErrTestError + }, + } + + storeController := storage.StoreController{DefaultStore: mockStoreController} + olu := common.NewBaseOciLayoutUtils(storeController, log.NewLogger("debug", "")) + + _, err := olu.GetImageInfo("", "") + So(err, ShouldNotBeNil) + }) } func TestSearchSize(t *testing.T) { diff --git a/pkg/extensions/search/convert/convert_test.go b/pkg/extensions/search/convert/convert_test.go index c9af323c..86bc2668 100644 --- a/pkg/extensions/search/convert/convert_test.go +++ b/pkg/extensions/search/convert/convert_test.go @@ -4,17 +4,24 @@ import ( "context" "encoding/json" "errors" + "strconv" "testing" "github.com/99designs/gqlgen/graphql" godigest "github.com/opencontainers/go-digest" + "github.com/opencontainers/image-spec/specs-go" ispec "github.com/opencontainers/image-spec/specs-go/v1" . "github.com/smartystreets/goconvey/convey" + "zotregistry.io/zot/pkg/api" + "zotregistry.io/zot/pkg/api/config" + extconf "zotregistry.io/zot/pkg/extensions/config" + "zotregistry.io/zot/pkg/extensions/search/common" "zotregistry.io/zot/pkg/extensions/search/convert" cveinfo "zotregistry.io/zot/pkg/extensions/search/cve" "zotregistry.io/zot/pkg/meta/repodb" bolt "zotregistry.io/zot/pkg/meta/repodb/boltdb-wrapper" + . "zotregistry.io/zot/pkg/test" "zotregistry.io/zot/pkg/test/mocks" ) @@ -74,3 +81,274 @@ func TestConvertErrors(t *testing.T) { So(graphql.GetErrors(ctx).Error(), ShouldContainSubstring, "unable to run vulnerability scan on tag") }) } + +func TestBuildImageInfo(t *testing.T) { + rootDir := t.TempDir() + + port := GetFreePort() + baseURL := GetBaseURL(port) + + conf := config.New() + conf.HTTP.Port = port + conf.Storage.RootDirectory = rootDir + defaultVal := true + conf.Extensions = &extconf.ExtensionConfig{ + Search: &extconf.SearchConfig{BaseConfig: extconf.BaseConfig{Enable: &defaultVal}}, + } + + conf.Extensions.Search.CVE = nil + + ctlr := api.NewController(conf) + ctlrManager := NewControllerManager(ctlr) + + ctlrManager.StartAndWait(port) + defer ctlrManager.StopServer() + + olu := &common.BaseOciLayoutUtils{ + StoreController: ctlr.StoreController, + Log: ctlr.Log, + } + + Convey("Check image summary when the image has no history", t, func() { + imageName := "nohistory" + + config := ispec.Image{ + Platform: ispec.Platform{ + OS: "linux", + Architecture: "amd64", + }, + RootFS: ispec.RootFS{ + Type: "layers", + DiffIDs: []godigest.Digest{}, + }, + Author: "ZotUser", + } + + configBlob, err := json.Marshal(config) + So(err, ShouldBeNil) + + configDigest := godigest.FromBytes(configBlob) + layerDigest := godigest.FromString(imageName) + layerblob := []byte(imageName) + schemaVersion := 2 + ispecManifest := ispec.Manifest{ + Versioned: specs.Versioned{ + SchemaVersion: schemaVersion, + }, + Config: ispec.Descriptor{ + MediaType: "application/vnd.oci.image.config.v1+json", + Digest: configDigest, + Size: int64(len(configBlob)), + }, + Layers: []ispec.Descriptor{ // just 1 layer in manifest + { + MediaType: "application/vnd.oci.image.layer.v1.tar", + Digest: layerDigest, + Size: int64(len(layerblob)), + }, + }, + } + manifestLayersSize := ispecManifest.Layers[0].Size + manifestBlob, err := json.Marshal(ispecManifest) + So(err, ShouldBeNil) + manifestDigest := godigest.FromBytes(manifestBlob) + err = UploadImage( + Image{ + Manifest: ispecManifest, + Config: config, + Layers: [][]byte{ + layerblob, + }, + Tag: "0.0.1", + }, + baseURL, + imageName, + ) + So(err, ShouldBeNil) + + imageConfig, err := olu.GetImageConfigInfo(imageName, manifestDigest) + So(err, ShouldBeNil) + + isSigned := false + + imageSummary := convert.BuildImageInfo(imageName, imageName, manifestDigest, ispecManifest, + imageConfig, isSigned) + + So(len(imageSummary.Layers), ShouldEqual, len(ispecManifest.Layers)) + imageSummaryLayerSize, err := strconv.Atoi(*imageSummary.Size) + So(err, ShouldBeNil) + So(imageSummaryLayerSize, ShouldEqual, manifestLayersSize) + }) + + Convey("Check image summary when layer count matche history entries", t, func() { + imageName := "valid" + + config := ispec.Image{ + Platform: ispec.Platform{ + OS: "linux", + Architecture: "amd64", + }, + RootFS: ispec.RootFS{ + Type: "layers", + DiffIDs: []godigest.Digest{}, + }, + Author: "ZotUser", + History: []ispec.History{ // should contain 3 elements, 2 of which corresponding to layers + { + EmptyLayer: false, + }, + { + EmptyLayer: false, + }, + { + EmptyLayer: true, + }, + }, + } + + configBlob, err := json.Marshal(config) + So(err, ShouldBeNil) + + configDigest := godigest.FromBytes(configBlob) + layerDigest := godigest.FromString("layer1") + layerblob := []byte("layer1") + layerDigest2 := godigest.FromString("layer2") + layerblob2 := []byte("layer2") + schemaVersion := 2 + ispecManifest := ispec.Manifest{ + Versioned: specs.Versioned{ + SchemaVersion: schemaVersion, + }, + Config: ispec.Descriptor{ + MediaType: "application/vnd.oci.image.config.v1+json", + Digest: configDigest, + Size: int64(len(configBlob)), + }, + Layers: []ispec.Descriptor{ // just 1 layer in manifest + { + MediaType: "application/vnd.oci.image.layer.v1.tar", + Digest: layerDigest, + Size: int64(len(layerblob)), + }, + { + MediaType: "application/vnd.oci.image.layer.v1.tar", + Digest: layerDigest2, + Size: int64(len(layerblob2)), + }, + }, + } + manifestLayersSize := ispecManifest.Layers[0].Size + ispecManifest.Layers[1].Size + manifestBlob, err := json.Marshal(ispecManifest) + So(err, ShouldBeNil) + manifestDigest := godigest.FromBytes(manifestBlob) + err = UploadImage( + Image{ + Manifest: ispecManifest, + Config: config, + Layers: [][]byte{ + layerblob, + layerblob2, + }, + Tag: "0.0.1", + }, + baseURL, + imageName, + ) + So(err, ShouldBeNil) + + imageConfig, err := olu.GetImageConfigInfo(imageName, manifestDigest) + So(err, ShouldBeNil) + + isSigned := false + + imageSummary := convert.BuildImageInfo(imageName, imageName, manifestDigest, ispecManifest, + imageConfig, isSigned) + + So(len(imageSummary.Layers), ShouldEqual, len(ispecManifest.Layers)) + imageSummaryLayerSize, err := strconv.Atoi(*imageSummary.Size) + So(err, ShouldBeNil) + So(imageSummaryLayerSize, ShouldEqual, manifestLayersSize) + }) + + Convey("Check image summary when layer count does not match history", t, func() { + imageName := "invalid" + + config := ispec.Image{ + Platform: ispec.Platform{ + OS: "linux", + Architecture: "amd64", + }, + RootFS: ispec.RootFS{ + Type: "layers", + DiffIDs: []godigest.Digest{}, + }, + Author: "ZotUser", + History: []ispec.History{ // should contain 3 elements, 2 of which corresponding to layers + { + EmptyLayer: false, + }, + { + EmptyLayer: false, + }, + { + EmptyLayer: true, + }, + }, + } + + configBlob, err := json.Marshal(config) + So(err, ShouldBeNil) + + configDigest := godigest.FromBytes(configBlob) + layerDigest := godigest.FromString(imageName) + layerblob := []byte(imageName) + schemaVersion := 2 + ispecManifest := ispec.Manifest{ + Versioned: specs.Versioned{ + SchemaVersion: schemaVersion, + }, + Config: ispec.Descriptor{ + MediaType: "application/vnd.oci.image.config.v1+json", + Digest: configDigest, + Size: int64(len(configBlob)), + }, + Layers: []ispec.Descriptor{ // just 1 layer in manifest + { + MediaType: "application/vnd.oci.image.layer.v1.tar", + Digest: layerDigest, + Size: int64(len(layerblob)), + }, + }, + } + manifestLayersSize := ispecManifest.Layers[0].Size + manifestBlob, err := json.Marshal(ispecManifest) + So(err, ShouldBeNil) + manifestDigest := godigest.FromBytes(manifestBlob) + err = UploadImage( + Image{ + Manifest: ispecManifest, + Config: config, + Layers: [][]byte{ + layerblob, + }, + Tag: "0.0.1", + }, + baseURL, + imageName, + ) + So(err, ShouldBeNil) + + imageConfig, err := olu.GetImageConfigInfo(imageName, manifestDigest) + So(err, ShouldBeNil) + + isSigned := false + + imageSummary := convert.BuildImageInfo(imageName, imageName, manifestDigest, ispecManifest, + imageConfig, isSigned) + + So(len(imageSummary.Layers), ShouldEqual, len(ispecManifest.Layers)) + imageSummaryLayerSize, err := strconv.Atoi(*imageSummary.Size) + So(err, ShouldBeNil) + So(imageSummaryLayerSize, ShouldEqual, manifestLayersSize) + }) +} diff --git a/pkg/extensions/search/gql_generated/generated.go b/pkg/extensions/search/gql_generated/generated.go index 0e3122d1..246d5f49 100644 --- a/pkg/extensions/search/gql_generated/generated.go +++ b/pkg/extensions/search/gql_generated/generated.go @@ -149,7 +149,7 @@ type ComplexityRoot struct { ExpandedRepoInfo func(childComplexity int, repo string) int GlobalSearch func(childComplexity int, query string, filter *Filter, requestedPage *PageInput) int Image func(childComplexity int, image string) int - ImageList func(childComplexity int, repo string) int + ImageList func(childComplexity int, repo string, requestedPage *PageInput) int ImageListForCve func(childComplexity int, id string, requestedPage *PageInput) int ImageListForDigest func(childComplexity int, id string, requestedPage *PageInput) int ImageListWithCVEFixed func(childComplexity int, id string, image string, requestedPage *PageInput) int @@ -191,7 +191,7 @@ type QueryResolver interface { ImageListWithCVEFixed(ctx context.Context, id string, image string, requestedPage *PageInput) ([]*ImageSummary, error) ImageListForDigest(ctx context.Context, id string, requestedPage *PageInput) ([]*ImageSummary, error) RepoListWithNewestImage(ctx context.Context, requestedPage *PageInput) (*PaginatedReposResult, error) - ImageList(ctx context.Context, repo string) ([]*ImageSummary, error) + ImageList(ctx context.Context, repo string, requestedPage *PageInput) ([]*ImageSummary, error) ExpandedRepoInfo(ctx context.Context, repo string) (*RepoInfo, error) GlobalSearch(ctx context.Context, query string, filter *Filter, requestedPage *PageInput) (*GlobalSearchResult, error) DerivedImageList(ctx context.Context, image string) ([]*ImageSummary, error) @@ -696,7 +696,7 @@ func (e *executableSchema) Complexity(typeName, field string, childComplexity in return 0, false } - return e.complexity.Query.ImageList(childComplexity, args["repo"].(string)), true + return e.complexity.Query.ImageList(childComplexity, args["repo"].(string), args["requestedPage"].(*PageInput)), true case "Query.ImageListForCVE": if e.complexity.Query.ImageListForCve == nil { @@ -1166,9 +1166,9 @@ type Query { RepoListWithNewestImage(requestedPage: PageInput): PaginatedReposResult! # Newest based on created timestamp """ - Returns all the images from the specified repo + Returns all the images from the specified repo | from all repos if specified repo is "" """ - ImageList(repo: String!): [ImageSummary!] + ImageList(repo: String!, requestedPage: PageInput): [ImageSummary!] """ Returns information about the specified repo @@ -1395,6 +1395,15 @@ func (ec *executionContext) field_Query_ImageList_args(ctx context.Context, rawA } } args["repo"] = arg0 + var arg1 *PageInput + if tmp, ok := rawArgs["requestedPage"]; ok { + ctx := graphql.WithPathContext(ctx, graphql.NewPathWithField("requestedPage")) + arg1, err = ec.unmarshalOPageInput2ᚖzotregistryᚗioᚋzotᚋpkgᚋextensionsᚋsearchᚋgql_generatedᚐPageInput(ctx, tmp) + if err != nil { + return nil, err + } + } + args["requestedPage"] = arg1 return args, nil } @@ -4515,7 +4524,7 @@ func (ec *executionContext) _Query_ImageList(ctx context.Context, field graphql. }() resTmp, err := ec.ResolverMiddleware(ctx, func(rctx context.Context) (interface{}, error) { ctx = rctx // use context from middleware stack in children - return ec.resolvers.Query().ImageList(rctx, fc.Args["repo"].(string)) + return ec.resolvers.Query().ImageList(rctx, fc.Args["repo"].(string), fc.Args["requestedPage"].(*PageInput)) }) if err != nil { ec.Error(ctx, err) diff --git a/pkg/extensions/search/resolver.go b/pkg/extensions/search/resolver.go index 0bbd3a90..a08098a1 100644 --- a/pkg/extensions/search/resolver.go +++ b/pkg/extensions/search/resolver.go @@ -701,74 +701,47 @@ func searchingForRepos(query string) bool { return !strings.Contains(query, ":") } -func (r *queryResolver) getImageList(store storage.ImageStore, imageName string) ( - []*gql_generated.ImageSummary, error, -) { - results := make([]*gql_generated.ImageSummary, 0) +func getImageList(ctx context.Context, repo string, repoDB repodb.RepoDB, cveInfo cveinfo.CveInfo, + requestedPage *gql_generated.PageInput, log log.Logger, //nolint:unparam +) ([]*gql_generated.ImageSummary, error) { + imageList := make([]*gql_generated.ImageSummary, 0) - repoList, err := store.GetRepositories() + if requestedPage == nil { + requestedPage = &gql_generated.PageInput{} + } + + skip := convert.SkipQGLField{ + Vulnerabilities: canSkipField(convert.GetPreloads(ctx), "Images.Vulnerabilities"), + } + + pageInput := repodb.PageInput{ + Limit: safeDerefferencing(requestedPage.Limit, 0), + Offset: safeDerefferencing(requestedPage.Offset, 0), + SortBy: repodb.SortCriteria( + safeDerefferencing(requestedPage.SortBy, gql_generated.SortCriteriaRelevance), + ), + } + + // reposMeta, manifestMetaMap, err := repoDB.SearchRepos(ctx, repo, repodb.Filter{}, pageInput) + reposMeta, manifestMetaMap, err := repoDB.FilterTags(ctx, + func(repoMeta repodb.RepoMetadata, manifestMeta repodb.ManifestMetadata) bool { + return true + }, + pageInput) if err != nil { - r.log.Error().Err(err).Msg("extension api: error extracting repositories list") - - return results, err + return []*gql_generated.ImageSummary{}, err } - layoutUtils := common.NewBaseOciLayoutUtils(r.storeController, r.log) - - for _, repo := range repoList { - if (imageName != "" && repo == imageName) || imageName == "" { - tagsInfo, err := layoutUtils.GetImageTagsWithTimestamp(repo) - if err != nil { - r.log.Error().Err(err).Msg("extension api: error getting tag timestamp info") - - return results, nil - } - - if len(tagsInfo) == 0 { - r.log.Info().Str("no tagsinfo found for repo", repo).Msg(" continuing traversing") - - continue - } - - for i := range tagsInfo { - // using a loop variable called tag would be reassigned after each iteration, using the same memory address - // directly access the value at the current index in the slice as ImageInfo requires pointers to tag fields - tag := tagsInfo[i] - digest := tag.Digest - - manifest, err := layoutUtils.GetImageBlobManifest(repo, digest) - if err != nil { - r.log.Error().Err(err).Msg("extension api: error reading manifest") - - return results, err - } - - imageConfig, err := layoutUtils.GetImageConfigInfo(repo, digest) - if err != nil { - return results, err - } - - isSigned := layoutUtils.CheckManifestSignature(repo, digest) - - tagPrefix := strings.HasPrefix(tag.Name, "sha256-") - tagSuffix := strings.HasSuffix(tag.Name, ".sig") - - imageInfo := convert.BuildImageInfo(repo, tag.Name, digest, manifest, - imageConfig, isSigned) - - // check if it's an image or a signature - if !tagPrefix && !tagSuffix { - results = append(results, imageInfo) - } - } + for _, repoMeta := range reposMeta { + if repoMeta.Name != repo && repo != "" { + continue } + imageSummaries := convert.RepoMeta2ImageSummaries(ctx, repoMeta, manifestMetaMap, skip, cveInfo) + + imageList = append(imageList, imageSummaries...) } - if len(results) == 0 { - r.log.Info().Msg("no repositories found") - } - - return results, nil + return imageList, nil } func getReferrers(store storage.ImageStore, repoName string, digest string, artifactType string, log log.Logger) ( diff --git a/pkg/extensions/search/resolver_test.go b/pkg/extensions/search/resolver_test.go index dff7f5c8..c0b298f2 100644 --- a/pkg/extensions/search/resolver_test.go +++ b/pkg/extensions/search/resolver_test.go @@ -569,7 +569,6 @@ func TestImageListForDigest(t *testing.T) { responseContext := graphql.WithResponseContext(context.Background(), graphql.DefaultErrorPresenter, graphql.DefaultRecover) - _, err := getImageListForDigest(responseContext, "invalid", mockSearchDB, mocks.CveInfoMock{}, nil) So(err, ShouldNotBeNil) }) @@ -595,7 +594,6 @@ func TestImageListForDigest(t *testing.T) { }, }) So(err, ShouldBeNil) - manifestBlob := []byte("invalid") manifestMetaDatas := map[string]repodb.ManifestMetadata{ @@ -648,7 +646,6 @@ func TestImageListForDigest(t *testing.T) { DownloadCount: 0, }, } - matchedTags := repos[0].Tags for tag, descriptor := range repos[0].Tags { if !filter(repos[0], manifestMetaDatas[descriptor.Digest]) { @@ -676,7 +673,6 @@ func TestImageListForDigest(t *testing.T) { responseContext := graphql.WithResponseContext(context.Background(), graphql.DefaultErrorPresenter, graphql.DefaultRecover) - imageSummaries, err := getImageListForDigest(responseContext, manifestDigest, mockSearchDB, mocks.CveInfoMock{}, &pageInput) So(err, ShouldBeNil) @@ -989,6 +985,102 @@ func TestImageListForDigest(t *testing.T) { }) } +func TestImageList(t *testing.T) { + Convey("getImageList", t, func() { + testLogger := log.NewLogger("debug", "") + Convey("no page requested, SearchRepoFn returns error", func() { + mockSearchDB := mocks.RepoDBMock{ + FilterTagsFn: func(ctx context.Context, filter repodb.FilterFunc, + requestedPage repodb.PageInput, + ) ([]repodb.RepoMetadata, map[string]repodb.ManifestMetadata, error) { + return []repodb.RepoMetadata{}, map[string]repodb.ManifestMetadata{}, ErrTestError + }, + } + + responseContext := graphql.WithResponseContext(context.Background(), graphql.DefaultErrorPresenter, + graphql.DefaultRecover) + + _, err := getImageList(responseContext, "test", mockSearchDB, mocks.CveInfoMock{}, nil, testLogger) + So(err, ShouldNotBeNil) + }) + + Convey("valid repoList returned", func() { + mockSearchDB := mocks.RepoDBMock{ + FilterTagsFn: func(ctx context.Context, filter repodb.FilterFunc, + requestedPage repodb.PageInput, + ) ([]repodb.RepoMetadata, map[string]repodb.ManifestMetadata, error) { + repos := []repodb.RepoMetadata{ + { + Name: "test", + Tags: map[string]repodb.Descriptor{ + "1.0.1": { + Digest: "digestTag1.0.1", + MediaType: ispec.MediaTypeImageManifest, + }, + }, + Signatures: map[string]repodb.ManifestSignatures{ + "digestTag1.0.1": { + "cosgin": []repodb.SignatureInfo{ + {SignatureManifestDigest: "digestSignature1"}, + }, + }, + }, + Stars: 100, + }, + } + + configBlob, err := json.Marshal(ispec.Image{ + Config: ispec.ImageConfig{ + Labels: map[string]string{}, + }, + }) + So(err, ShouldBeNil) + + manifestBlob, err := json.Marshal(ispec.Manifest{}) + So(err, ShouldBeNil) + + manifestMetaDatas := map[string]repodb.ManifestMetadata{ + "digestTag1.0.1": { + ManifestBlob: manifestBlob, + ConfigBlob: configBlob, + DownloadCount: 0, + Signatures: repodb.ManifestSignatures{ + "cosgin": []repodb.SignatureInfo{ + {SignatureManifestDigest: "digestSignature1"}, + }, + }, + }, + } + + return repos, manifestMetaDatas, nil + }, + } + + limit := 1 + ofset := 0 + sortCriteria := gql_generated.SortCriteriaAlphabeticAsc + pageInput := gql_generated.PageInput{ + Limit: &limit, + Offset: &ofset, + SortBy: &sortCriteria, + } + + responseContext := graphql.WithResponseContext(context.Background(), graphql.DefaultErrorPresenter, + graphql.DefaultRecover) + + imageSummaries, err := getImageList(responseContext, "test", mockSearchDB, + mocks.CveInfoMock{}, &pageInput, testLogger) + So(err, ShouldBeNil) + So(len(imageSummaries), ShouldEqual, 1) + + imageSummaries, err = getImageList(responseContext, "invalid", mockSearchDB, + mocks.CveInfoMock{}, &pageInput, testLogger) + So(err, ShouldBeNil) + So(len(imageSummaries), ShouldEqual, 0) + }) + }) +} + func TestGetReferrers(t *testing.T) { Convey("getReferrers", t, func() { Convey("GetReferrers returns error", func() { @@ -1302,14 +1394,14 @@ func TestQueryResolverErrors(t *testing.T) { Convey("ImageList getImageList() errors", func() { resolverConfig := NewResolver( log, - storage.StoreController{ - DefaultStore: mocks.MockedImageStore{ - GetRepositoriesFn: func() ([]string, error) { - return nil, ErrTestError - }, + storage.StoreController{}, + mocks.RepoDBMock{ + FilterTagsFn: func(ctx context.Context, filter repodb.FilterFunc, + requestedPage repodb.PageInput, + ) ([]repodb.RepoMetadata, map[string]repodb.ManifestMetadata, error) { + return []repodb.RepoMetadata{}, map[string]repodb.ManifestMetadata{}, ErrTestError }, }, - mocks.RepoDBMock{}, mocks.CveInfoMock{}, ) @@ -1317,36 +1409,7 @@ func TestQueryResolverErrors(t *testing.T) { resolverConfig, } - _, err := qr.ImageList(ctx, "repo") - So(err, ShouldNotBeNil) - }) - - Convey("ImageList subpaths getImageList() errors", func() { - resolverConfig := NewResolver( - log, - storage.StoreController{ - DefaultStore: mocks.MockedImageStore{ - GetRepositoriesFn: func() ([]string, error) { - return []string{"sub1/repo"}, nil - }, - }, - SubStore: map[string]storage.ImageStore{ - "/sub1": mocks.MockedImageStore{ - GetRepositoriesFn: func() ([]string, error) { - return nil, ErrTestError - }, - }, - }, - }, - mocks.RepoDBMock{}, - mocks.CveInfoMock{}, - ) - - qr := queryResolver{ - resolverConfig, - } - - _, err := qr.ImageList(ctx, "repo") + _, err := qr.ImageList(ctx, "repo", &gql_generated.PageInput{}) So(err, ShouldNotBeNil) }) diff --git a/pkg/extensions/search/schema.graphql b/pkg/extensions/search/schema.graphql index a2b8221c..ac8d2e7d 100644 --- a/pkg/extensions/search/schema.graphql +++ b/pkg/extensions/search/schema.graphql @@ -225,9 +225,9 @@ type Query { RepoListWithNewestImage(requestedPage: PageInput): PaginatedReposResult! # Newest based on created timestamp """ - Returns all the images from the specified repo + Returns all the images from the specified repo | from all repos if specified repo is "" """ - ImageList(repo: String!): [ImageSummary!] + ImageList(repo: String!, requestedPage: PageInput): [ImageSummary!] """ Returns information about the specified repo diff --git a/pkg/extensions/search/schema.resolvers.go b/pkg/extensions/search/schema.resolvers.go index b325dbbe..b0895e4e 100644 --- a/pkg/extensions/search/schema.resolvers.go +++ b/pkg/extensions/search/schema.resolvers.go @@ -64,39 +64,16 @@ func (r *queryResolver) RepoListWithNewestImage(ctx context.Context, requestedPa } // ImageList is the resolver for the ImageList field. -func (r *queryResolver) ImageList(ctx context.Context, repo string) ([]*gql_generated.ImageSummary, error) { +func (r *queryResolver) ImageList(ctx context.Context, repo string, requestedPage *gql_generated.PageInput) ([]*gql_generated.ImageSummary, error) { r.log.Info().Msg("extension api: getting a list of all images") - imageList := make([]*gql_generated.ImageSummary, 0) - - defaultStore := r.storeController.DefaultStore - - dsImageList, err := r.getImageList(defaultStore, repo) + imageList, err := getImageList(ctx, repo, r.repoDB, r.cveInfo, requestedPage, r.log) if err != nil { - r.log.Error().Err(err).Msg("extension api: error extracting default store image list") + r.log.Error().Err(err).Msgf("unable to retrieve image list for repo: %s", repo) return imageList, err } - if len(dsImageList) != 0 { - imageList = append(imageList, dsImageList...) - } - - subStore := r.storeController.SubStore - - for _, store := range subStore { - ssImageList, err := r.getImageList(store, repo) - if err != nil { - r.log.Error().Err(err).Msg("extension api: error extracting substore image list") - - return imageList, err - } - - if len(ssImageList) != 0 { - imageList = append(imageList, ssImageList...) - } - } - return imageList, nil }