From d63f715fe51c15fc874a1223de7a20c8e409e541 Mon Sep 17 00:00:00 2001 From: Shivam Mishra Date: Fri, 4 Sep 2020 13:16:15 -0700 Subject: [PATCH] search/cve: exclude unsupported images from fixed-tag list. If image vulnerability scan does not support any media type, considering those images as an infected image and now this images will not be shown in fixed images list. Fixes issue #130 --- pkg/cli/BUILD.bazel | 2 +- pkg/cli/cve_cmd_test.go | 2 +- pkg/extensions/search/BUILD.bazel | 1 - pkg/extensions/search/cve/BUILD.bazel | 1 + pkg/extensions/search/cve/cve.go | 189 ++++++++++++----------- pkg/extensions/search/cve/cve_test.go | 208 ++++++++++++++++++++++++-- pkg/extensions/search/resolver.go | 89 +++++------ 7 files changed, 340 insertions(+), 152 deletions(-) diff --git a/pkg/cli/BUILD.bazel b/pkg/cli/BUILD.bazel index 1afa8052..b1642a5e 100644 --- a/pkg/cli/BUILD.bazel +++ b/pkg/cli/BUILD.bazel @@ -39,7 +39,7 @@ go_test( "image_cmd_test.go", "root_test.go", ], - data = [ + data = [ "//:exported_testdata", ], embed = [":go_default_library"], diff --git a/pkg/cli/cve_cmd_test.go b/pkg/cli/cve_cmd_test.go index 2965bf9b..824aeb21 100644 --- a/pkg/cli/cve_cmd_test.go +++ b/pkg/cli/cve_cmd_test.go @@ -425,7 +425,7 @@ func TestServerCVEResponse(t *testing.T) { str := space.ReplaceAllString(buff.String(), " ") str = strings.TrimSpace(str) So(err, ShouldBeNil) - So(str, ShouldEqual, "") + So(strings.TrimSpace(str), ShouldContainSubstring, "IMAGE NAME TAG DIGEST SIZE") }) Convey("invalid image", func() { diff --git a/pkg/extensions/search/BUILD.bazel b/pkg/extensions/search/BUILD.bazel index d30eab15..b3df86e3 100644 --- a/pkg/extensions/search/BUILD.bazel +++ b/pkg/extensions/search/BUILD.bazel @@ -10,7 +10,6 @@ go_library( importpath = "github.com/anuvu/zot/pkg/extensions/search", visibility = ["//visibility:public"], deps = [ - "//errors:go_default_library", "//pkg/extensions/search/cve:go_default_library", "//pkg/log:go_default_library", "//pkg/storage:go_default_library", diff --git a/pkg/extensions/search/cve/BUILD.bazel b/pkg/extensions/search/cve/BUILD.bazel index 4568b77f..6eb88827 100644 --- a/pkg/extensions/search/cve/BUILD.bazel +++ b/pkg/extensions/search/cve/BUILD.bazel @@ -31,6 +31,7 @@ go_test( deps = [ "//pkg/api:go_default_library", "//pkg/log:go_default_library", + "@com_github_opencontainers_image_spec//specs-go/v1:go_default_library", "@com_github_smartystreets_goconvey//convey:go_default_library", "@in_gopkg_resty_v1//:go_default_library", ], diff --git a/pkg/extensions/search/cve/cve.go b/pkg/extensions/search/cve/cve.go index fba1cae3..e401e1d2 100644 --- a/pkg/extensions/search/cve/cve.go +++ b/pkg/extensions/search/cve/cve.go @@ -45,7 +45,7 @@ func ScanImage(config *config.Config) (report.Results, error) { } func (cveinfo CveInfo) IsValidImageFormat(imagePath string) (bool, error) { - imageDir := getImageDir(imagePath) + imageDir, inputTag := getImageDirAndTag(imagePath) if !dirExists(imageDir) { cveinfo.Log.Error().Msg("Image Directory not exists") @@ -53,45 +53,21 @@ func (cveinfo CveInfo) IsValidImageFormat(imagePath string) (bool, error) { return false, errors.ErrRepoNotFound } - buf, err := ioutil.ReadFile(path.Join(imageDir, "index.json")) + manifests, err := cveinfo.getImageManifests(imageDir) if err != nil { - if os.IsNotExist(err) { - cveinfo.Log.Error().Err(err).Msg("Index.json does not exist") - - return false, errors.ErrRepoNotFound - } - - cveinfo.Log.Error().Err(err).Msg("Unable to open index.json") - - return false, errors.ErrRepoNotFound - } - - var index ispec.Index - - var blobManifest v1.Manifest - - var digest godigest.Digest - - if err := json.Unmarshal(buf, &index); err != nil { - cveinfo.Log.Error().Err(err).Msg("Unable to marshal index.json file") - return false, err } - for _, m := range index.Manifests { - digest = m.Digest + for _, m := range manifests { + tag, ok := m.Annotations[ispec.AnnotationRefName] - blobBuf, err := ioutil.ReadFile(path.Join(imageDir, "blobs", digest.Algorithm().String(), digest.Encoded())) - if err != nil { - cveinfo.Log.Error().Err(err).Msg("Failed to read manifest file") - - return false, err + if ok && inputTag != "" && tag != inputTag { + continue } - if err := json.Unmarshal(blobBuf, &blobManifest); err != nil { - cveinfo.Log.Error().Err(err).Msg("Invalid manifest json") - + blobManifest, err := cveinfo.getImageBlobManifest(imageDir, m.Digest) + if err != nil { return false, err } @@ -104,7 +80,7 @@ func (cveinfo CveInfo) IsValidImageFormat(imagePath string) (bool, error) { default: cveinfo.Log.Debug().Msg("Image media type not supported for scanning") - return false, nil + return false, errors.ErrScanNotSupported } } } @@ -121,79 +97,61 @@ func dirExists(d string) bool { return fi.IsDir() } -func getImageDir(imageName string) string { +func getImageDirAndTag(imageName string) (string, string) { var imageDir string + + var imageTag string + if strings.Contains(imageName, ":") { - imageDir = strings.Split(imageName, ":")[0] + splitImageName := strings.Split(imageName, ":") + imageDir = splitImageName[0] + imageTag = splitImageName[1] } else { imageDir = imageName } - return imageDir + return imageDir, imageTag } // GetImageTagsWithTimestamp returns a list of image tags with timestamp available in the specified repository. func (cveinfo CveInfo) GetImageTagsWithTimestamp(rootDir string, repo string) ([]TagInfo, error) { + tagsInfo := make([]TagInfo, 0) + dir := path.Join(rootDir, repo) if !dirExists(dir) { return nil, errors.ErrRepoNotFound } - var digest godigest.Digest + manifests, err := cveinfo.getImageManifests(dir) - buf, err := ioutil.ReadFile(path.Join(dir, "index.json")) if err != nil { - cveinfo.Log.Error().Err(err).Str("dir", dir).Msg("failed to read index.json") - return nil, errors.ErrRepoNotFound + cveinfo.Log.Error().Err(err).Msg("Unable to read image manifests") + + return tagsInfo, err } - var index ispec.Index - if err := json.Unmarshal(buf, &index); err != nil { - cveinfo.Log.Error().Err(err).Str("dir", dir).Msg("invalid JSON") - return nil, errors.ErrRepoNotFound - } + for _, manifest := range manifests { + digest := manifest.Digest - tagsInfo := make([]TagInfo, 0) - - var blobIndex ispec.Manifest - - var layerIndex ispec.Image - - for _, manifest := range index.Manifests { - digest = manifest.Digest v, ok := manifest.Annotations[ispec.AnnotationRefName] - - blobBuf, err := ioutil.ReadFile(path.Join(dir, "blobs", digest.Algorithm().String(), digest.Encoded())) - if err != nil { - cveinfo.Log.Error().Err(err).Msg("Unable to open Image Metadata file") - - return nil, err - } - - if err := json.Unmarshal(blobBuf, &blobIndex); err != nil { - cveinfo.Log.Error().Err(err).Msg("Unable to marshal blob index") - - return nil, err - } - - digest = blobIndex.Config.Digest - - blobBuf, err = ioutil.ReadFile(path.Join(dir, "blobs", digest.Algorithm().String(), digest.Encoded())) - if err != nil { - cveinfo.Log.Error().Err(err).Msg("Unable to open Image Layers file") - - return nil, err - } - - if err := json.Unmarshal(blobBuf, &layerIndex); err != nil { - cveinfo.Log.Error().Err(err).Msg("Unable to marshal blob index") - - return nil, err - } - - timeStamp := *layerIndex.History[0].Created - if ok { + imageBlobManifest, err := cveinfo.getImageBlobManifest(dir, digest) + + if err != nil { + cveinfo.Log.Error().Err(err).Msg("Unable to read image blob manifest") + + return tagsInfo, err + } + + imageInfo, err := cveinfo.getImageInfo(dir, imageBlobManifest.Config.Digest) + if err != nil { + cveinfo.Log.Error().Err(err).Msg("Unable to read image info") + + return tagsInfo, err + } + + timeStamp := *imageInfo.History[0].Created + tagsInfo = append(tagsInfo, TagInfo{Name: v, Timestamp: timeStamp}) } } @@ -224,3 +182,66 @@ func GetFixedTags(allTags []TagInfo, infectedTags []TagInfo) []TagInfo { return fixedTags } + +func (cveinfo CveInfo) getImageManifests(imagePath string) ([]ispec.Descriptor, error) { + buf, err := ioutil.ReadFile(path.Join(imagePath, "index.json")) + + if err != nil { + if os.IsNotExist(err) { + cveinfo.Log.Error().Err(err).Msg("Index.json does not exist") + + return nil, errors.ErrRepoNotFound + } + + cveinfo.Log.Error().Err(err).Msg("Unable to open index.json") + + return nil, errors.ErrRepoNotFound + } + + var index ispec.Index + + if err := json.Unmarshal(buf, &index); err != nil { + cveinfo.Log.Error().Err(err).Str("dir", imagePath).Msg("invalid JSON") + return nil, errors.ErrRepoNotFound + } + + return index.Manifests, nil +} + +func (cveinfo CveInfo) getImageBlobManifest(imageDir string, digest godigest.Digest) (v1.Manifest, error) { + var blobIndex v1.Manifest + + blobBuf, err := ioutil.ReadFile(path.Join(imageDir, "blobs", digest.Algorithm().String(), digest.Encoded())) + if err != nil { + cveinfo.Log.Error().Err(err).Msg("Unable to open image Metadata file") + + return blobIndex, err + } + + if err := json.Unmarshal(blobBuf, &blobIndex); err != nil { + cveinfo.Log.Error().Err(err).Msg("Unable to marshal blob index") + + return blobIndex, err + } + + return blobIndex, nil +} + +func (cveinfo CveInfo) getImageInfo(imageDir string, hash v1.Hash) (ispec.Image, error) { + var imageInfo ispec.Image + + blobBuf, err := ioutil.ReadFile(path.Join(imageDir, "blobs", hash.Algorithm, hash.Hex)) + if err != nil { + cveinfo.Log.Error().Err(err).Msg("Unable to open image Layers file") + + return imageInfo, err + } + + if err := json.Unmarshal(blobBuf, &imageInfo); err != nil { + cveinfo.Log.Error().Err(err).Msg("Unable to marshal blob index") + + return imageInfo, err + } + + return imageInfo, err +} diff --git a/pkg/extensions/search/cve/cve_test.go b/pkg/extensions/search/cve/cve_test.go index 7f81144e..a8c3d372 100644 --- a/pkg/extensions/search/cve/cve_test.go +++ b/pkg/extensions/search/cve/cve_test.go @@ -15,6 +15,7 @@ import ( "github.com/anuvu/zot/pkg/api" cveinfo "github.com/anuvu/zot/pkg/extensions/search/cve" "github.com/anuvu/zot/pkg/log" + ispec "github.com/opencontainers/image-spec/specs-go/v1" . "github.com/smartystreets/goconvey/convey" "gopkg.in/resty.v1" ) @@ -78,12 +79,12 @@ func testSetup() error { dbDir = dir - err = copyFiles("../../../../test/data/zot-test", path.Join(dbDir, "zot-test")) + err = generateTestData() if err != nil { return err } - err = generateTestData() + err = copyFiles("../../../../test/data", dbDir) if err != nil { return err } @@ -91,13 +92,30 @@ func testSetup() error { return nil } -func generateTestData() error { +func generateTestData() error { // nolint: gocyclo // Image dir with no files err := os.Mkdir(path.Join(dbDir, "zot-noindex-test"), 0755) if err != nil { return err } + err = os.Mkdir(path.Join(dbDir, "zot-nonreadable-test"), 0755) + if err != nil { + return err + } + + index := ispec.Index{} + index.SchemaVersion = 2 + buf, err := json.Marshal(index) + + if err != nil { + return err + } + + if err = ioutil.WriteFile(path.Join(dbDir, "zot-nonreadable-test", "index.json"), buf, 0111); err != nil { + return err + } + // Image dir with invalid index.json err = os.Mkdir(path.Join(dbDir, "zot-squashfs-invalid-index"), 0755) if err != nil { @@ -147,24 +165,130 @@ func generateTestData() error { return err } - // Image dir to test squashfs media type + // Create a squashfs image + err = os.MkdirAll(path.Join(dbDir, "zot-squashfs-test", "blobs/sha256"), 0755) if err != nil { return err } - content = fmt.Sprintf(`{"schemaVersion":2,"manifests":[{"mediaType":"application/vnd.oci.image.manifest.v1+json","digest":"sha256:2a9b097b4e4c613dd8185eba55163201a221909f3d430f8df87cd3639afc5929","size":1240,"annotations":{"org.opencontainers.image.ref.name":"commit-aaa7c6e7-squashfs"},"platform":{"architecture":"amd64","os":"linux"}}]} - `) + il := ispec.ImageLayout{Version: ispec.ImageLayoutVersion} + buf, err = json.Marshal(il) - err = makeTestFile(path.Join(dbDir, "zot-squashfs-invalid-blob", "index.json"), content) if err != nil { return err } - content = fmt.Sprintf(`{"schemaVersion":2,"config"{"mediaType":"application/vnd.oci.image.config.v1+json","digest":"sha256:4b37d4133908ac9a3032ba996020f2ad41354a616b071ca7e726a1df18a0f354","size":1691},"layers":[{"mediaType":"application/vnd.oci.image.layer.squashfs","digest":"sha256:a01a66356aace53222e92fb6fd990b23eb44ab0e58dff6f853fa9f771ecf3ac5","size":54996992},{"mediaType":"application/vnd.oci.image.layer.squashfs","digest":"sha256:91c26d6934ef2b5c5c4d8458af9bfc4ca46cf90c22380193154964abc8298a7a","size":52330496},{"mediaType":"application/vnd.oci.image.layer.squashfs","digest":"sha256:f281a550ca49746cfc6b8f1ac52f8086b3d5845db2ca18fde980dab62ae3bf7d","size":23343104},{"mediaType":"application/vnd.oci.image.layer.squashfs","digest":"sha256:7ee02568717acdda336c9d56d4dc6ea7f3b1c553e43bb0c0ecc6fd3bbd059d1a","size":5910528},{"mediaType":"application/vnd.oci.image.layer.squashfs","digest":"sha256:8fb33b130588b239235dedd560cdf49d29bbf6f2db5419ac68e4592a85c1f416","size":123269120},{"mediaType":"application/vnd.oci.image.layer.squashfs","digest":"sha256:1b49f0b33d4a696bb94d84c9acab3623e2c195bfb446d446a583a2f9f27b04c3","size":113901568}],"annotations":{"com.cisco.stacker.git_version":"7-dev19-63-gaaa7c6e7","ws.tycho.stacker.git_version":"0.3.26"}} - `) + if err = ioutil.WriteFile(path.Join(dbDir, "zot-squashfs-test", "oci-layout"), buf, 0644); err != nil { //nolint: gosec + return err + } - err = makeTestFile(path.Join(dbDir, "zot-squashfs-invalid-blob", "blobs/sha256", "2a9b097b4e4c613dd8185eba55163201a221909f3d430f8df87cd3639afc5929"), content) + err = os.Mkdir(path.Join(dbDir, "zot-squashfs-test", ".uploads"), 0755) + if err != nil { + return err + } + + content = fmt.Sprintf(`{"schemaVersion":2,"manifests":[{"mediaType":"application/vnd.oci.image.manifest.v1+json","digest":"sha256:eca04f027f414362596f2632746d8a178362170b9ac9af772011fedcc3877ebb","size":886,"annotations":{"org.opencontainers.image.ref.name":"0.3.25"},"platform":{"architecture":"amd64","os":"linux"}},{"mediaType":"application/vnd.oci.image.manifest.v1+json","digest":"sha256:45df53588e59759a12bd3eca553cdc9063939baac9a28d7ebb6101e4ec230b76","size":873,"annotations":{"org.opencontainers.image.ref.name":"0.3.22-squashfs"},"platform":{"architecture":"amd64","os":"linux"}},{"mediaType":"application/vnd.oci.image.manifest.v1+json","digest":"sha256:71448405a4b89539fcfa581afb4dc7d257f98857686b8138b08a1c539f313099","size":886,"annotations":{"org.opencontainers.image.ref.name":"0.3.19"},"platform":{"architecture":"amd64","os":"linux"}}]}`) + + err = makeTestFile(path.Join(dbDir, "zot-squashfs-test", "index.json"), content) + if err != nil { + return err + } + + content = fmt.Sprintf(`{"schemaVersion":2,"config":{"mediaType":"application/vnd.oci.image.config.v1+json","digest":"sha256:c5c2fd2b07ad4cb025cf20936d6bce6085584b8377780599be4da8a91739f0e8","size":1738},"layers":[{"mediaType":"application/vnd.oci.image.layer.v1.tar+gzip","digest":"sha256:3414b5ef0ad2f0390daaf55b63c422eeedef6191d47036a69d8ee396fabdce72","size":58993484},{"mediaType":"application/vnd.oci.image.layer.v1.tar+gzip","digest":"sha256:a3b04fff744c13dfa4883e01fa35e01af8daa7f72d9e9b6b7fad1f28843846b6","size":55631733},{"mediaType":"application/vnd.oci.image.layer.v1.tar+gzip","digest":"sha256:754f517f58f302190aa94e025c25890c18e1e811127aed1ef25c189278ec4ab0","size":113612795},{"mediaType":"application/vnd.oci.image.layer.v1.tar+gzip","digest":"sha256:ec004cd43488b803d6e232599e83a3164394d44fcd9f44755fed7b5791087ede","size":108889651}],"annotations":{"ws.tycho.stacker.git_version":"0.3.19"}}`) + + err = makeTestFile(path.Join(dbDir, "zot-squashfs-test", "blobs/sha256", "71448405a4b89539fcfa581afb4dc7d257f98857686b8138b08a1c539f313099"), content) + if err != nil { + return err + } + + content = fmt.Sprintf(`{"created": "2020-04-08T05:32:49.805795564Z","author": "","architecture": "amd64","os": "linux","config": {"Env": ["PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"]},"rootfs": {"type": "layers","diff_ids": []},"history": [{"created": "2020-04-08T05:08:43.590117872Z","created_by": "stacker umoci repack"}, {"created": "2020-04-08T05:08:53.213437118Z","created_by": "stacker build","author": "","empty_layer": true}, {"created": "2020-04-08T05:12:15.999154739Z","created_by": "stacker umoci repack","author": ""}, {"created": "2020-04-08T05:12:31.0513552Z","created_by": "stacker build","author": "","empty_layer": true}, {"created": "2020-04-08T05:20:38.068800557Z","created_by": "stacker umoci repack","author": ""}, {"created": "2020-04-08T05:21:01.956154957Z","created_by": "stacker build","author": "","empty_layer": true}, {"created": "2020-04-08T05:32:24.582132274Z","created_by": "stacker umoci repack","author": ""}, {"created": "2020-04-08T05:32:49.805795564Z","created_by": "stacker build","author": "","empty_layer": true}]}`) + + err = makeTestFile(path.Join(dbDir, "zot-squashfs-test", "blobs/sha256", "c5c2fd2b07ad4cb025cf20936d6bce6085584b8377780599be4da8a91739f0e8"), content) + if err != nil { + return err + } + + content = fmt.Sprintf(`{"schemaVersion":2,"config":{"mediaType":"application/vnd.oci.image.config.v1+json","digest":"sha256:5f00b5570a5561a6f9b7e66e4f26e2e30c4d09b43a8d3f993f3c1c99be6137a6","size":1740},"layers":[{"mediaType":"application/vnd.oci.image.layer.v1.tar+gzip","digest":"sha256:f8b7e41ce10d9a0f614f068326c43431c2777e6fc346f729c2a643bfab24af83","size":59451113},{"mediaType":"application/vnd.oci.image.layer.v1.tar+gzip","digest":"sha256:9ca9274f196b56a708a7a672d3de88184c0158a30744d355dd0411f3a6850fa6","size":55685756},{"mediaType":"application/vnd.oci.image.layer.v1.tar+gzip","digest":"sha256:6c1ca50788f93937e9ce9341b564f86cbbcd28e367ed6a57cfc776aee4a9d050","size":113726186},{"mediaType":"application/vnd.oci.image.layer.v1.tar+gzip","digest":"sha256:d1a92139df86bdf00c818db75bf1ecc860857d142b426e9971a62f5f90e2aa57","size":108755643}],"annotations":{"ws.tycho.stacker.git_version":"0.3.25"}}`) + + err = makeTestFile(path.Join(dbDir, "zot-squashfs-test", "blobs/sha256", "eca04f027f414362596f2632746d8a178362170b9ac9af772011fedcc3877ebb"), content) + if err != nil { + return err + } + + content = fmt.Sprintf(`{"created": "2020-04-08T05:32:49.805795564Z","author": "","architecture": "amd64","os": "linux","config": {"Env": ["PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"]},"rootfs": {"type": "layers","diff_ids": []},"history": [{"created": "2020-05-11T18:17:24.516727354Z","created_by": "stacker umoci repack"}, {"created": "2020-04-08T05:08:53.213437118Z","created_by": "stacker build","author": "","empty_layer": true}, {"created": "2020-04-08T05:12:15.999154739Z","created_by": "stacker umoci repack","author": ""}, {"created": "2020-04-08T05:12:31.0513552Z","created_by": "stacker build","author": "","empty_layer": true}, {"created": "2020-04-08T05:20:38.068800557Z","created_by": "stacker umoci repack","author": ""}, {"created": "2020-04-08T05:21:01.956154957Z","created_by": "stacker build","author": "","empty_layer": true}, {"created": "2020-04-08T05:32:24.582132274Z","created_by": "stacker umoci repack","author": ""}, {"created": "2020-04-08T05:32:49.805795564Z","created_by": "stacker build","author": "","empty_layer": true}]}`) + + err = makeTestFile(path.Join(dbDir, "zot-squashfs-test", "blobs/sha256", "5f00b5570a5561a6f9b7e66e4f26e2e30c4d09b43a8d3f993f3c1c99be6137a6"), content) + if err != nil { + return err + } + + content = fmt.Sprintf(`{"schemaVersion":2,"config":{"mediaType":"application/vnd.oci.image.config.v1+json","digest":"sha256:1fc1d045b241b04fea54333d76d4f57eb1961f9a314413f02a956b76e77a99f0","size":1218},"layers":[{"mediaType":"application/vnd.oci.image.layer.squashfs","digest":"sha256:c40d72b1556293c00a3e4b6c64c46ef4c7ae4d876dc18bad942b7d1903e8e5b7","size":54745420},{"mediaType":"application/vnd.oci.image.layer.squashfs","digest":"sha256:4115890e3e2563e545e03f264bfecb0097e24e02306ae3e7668dea52e00c6cc2","size":52213357},{"mediaType":"application/vnd.oci.image.layer.squashfs","digest":"sha256:91859e13e0cf704d5405199d73a2d1a0718391dbb183a77c4cb85d99e923ff57","size":109479329},{"mediaType":"application/vnd.oci.image.layer.squashfs","digest":"sha256:20aef84d8098d47a0643a2f99ce05f0deed957b3a259fb708c538f23ed97cc82","size":103996238}],"annotations":{"ws.tycho.stacker.git_version":"0.3.25"}}`) + + err = makeTestFile(path.Join(dbDir, "zot-squashfs-test", "blobs/sha256", "45df53588e59759a12bd3eca553cdc9063939baac9a28d7ebb6101e4ec230b76"), content) + if err != nil { + return err + } + + content = fmt.Sprintf(`{"created": "2020-04-08T05:32:49.805795564Z","author": "","architecture": "amd64","os": "linux","config": {"Env": ["PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"]},"rootfs": {"type": "layers","diff_ids": []},"history": [{"created": "2020-05-11T18:17:24.516727354Z","created_by": "stacker umoci repack"}, {"created": "2020-04-08T05:08:53.213437118Z","created_by": "stacker build","author": "","empty_layer": true}, {"created": "2020-04-08T05:12:15.999154739Z","created_by": "stacker umoci repack","author": ""}, {"created": "2020-05-11T19:30:02.467956112Z","created_by": "stacker build","author": "","empty_layer": true}, {"created": "2020-04-08T05:20:38.068800557Z","created_by": "stacker umoci repack","author": ""}, {"created": "2020-04-08T05:21:01.956154957Z","created_by": "stacker build","author": "","empty_layer": true}, {"created": "2020-04-08T05:32:24.582132274Z","created_by": "stacker umoci repack","author": ""}, {"created": "2020-04-08T05:32:49.805795564Z","created_by": "stacker build","author": "","empty_layer": true}]}`) + + err = makeTestFile(path.Join(dbDir, "zot-squashfs-test", "blobs/sha256", "1fc1d045b241b04fea54333d76d4f57eb1961f9a314413f02a956b76e77a99f0"), content) + if err != nil { + return err + } + + // Create a image with invalid layer blob + + err = os.MkdirAll(path.Join(dbDir, "zot-invalid-layer", "blobs/sha256"), 0755) + if err != nil { + return err + } + + content = fmt.Sprintf(`{"schemaVersion":2,"manifests":[{"mediaType":"application/vnd.oci.image.manifest.v1+json","digest":"sha256:eca04f027f414362596f2632746d8a178362170b9ac9af772011fedcc3877ebb","size":886,"annotations":{"org.opencontainers.image.ref.name":"0.3.25"},"platform":{"architecture":"amd64","os":"linux"}},{"mediaType":"application/vnd.oci.image.manifest.v1+json","digest":"sha256:45df53588e59759a12bd3eca553cdc9063939baac9a28d7ebb6101e4ec230b76","size":873,"annotations":{"org.opencontainers.image.ref.name":"0.3.22-squashfs"},"platform":{"architecture":"amd64","os":"linux"}},{"mediaType":"application/vnd.oci.image.manifest.v1+json","digest":"sha256:71448405a4b89539fcfa581afb4dc7d257f98857686b8138b08a1c539f313099","size":886,"annotations":{"org.opencontainers.image.ref.name":"0.3.19"},"platform":{"architecture":"amd64","os":"linux"}}]}`) + + err = makeTestFile(path.Join(dbDir, "zot-invalid-layer", "index.json"), content) + if err != nil { + return err + } + + content = fmt.Sprintf(`{"schemaVersion":2,"config":{"mediaType":"application/vnd.oci.image.config.v1+json","digest":"sha256:5f00b5570a5561a6f9b7e66e4f26e2e30c4d09b43a8d3f993f3c1c99be6137a6","size":1740},"layers":[{"mediaType":"application/vnd.oci.image.layer.v1.tar+gzip","digest":"sha256:f8b7e41ce10d9a0f614f068326c43431c2777e6fc346f729c2a643bfab24af83","size":59451113},{"mediaType":"application/vnd.oci.image.layer.v1.tar+gzip","digest":"sha256:9ca9274f196b56a708a7a672d3de88184c0158a30744d355dd0411f3a6850fa6","size":55685756},{"mediaType":"application/vnd.oci.image.layer.v1.tar+gzip","digest":"sha256:6c1ca50788f93937e9ce9341b564f86cbbcd28e367ed6a57cfc776aee4a9d050","size":113726186},{"mediaType":"application/vnd.oci.image.layer.v1.tar+gzip","digest":"sha256:d1a92139df86bdf00c818db75bf1ecc860857d142b426e9971a62f5f90e2aa57","size":108755643}],"annotations":{"ws.tycho.stacker.git_version":"0.3.25"}}`) + + err = makeTestFile(path.Join(dbDir, "zot-invalid-layer", "blobs/sha256", "eca04f027f414362596f2632746d8a178362170b9ac9af772011fedcc3877ebb"), content) + if err != nil { + return err + } + + content = fmt.Sprintf(`{"created":"2020-05-11T19:12:23.239785708Z","author":"root@jenkinsProduction-Atom-Full-Build-c3-master-159CI","architecture":"amd64","os":"linux","config":{"Env":["PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"]},"rootfs":{"type":"layers","diff_ids":["sha256:8817d297aa60796f41f559ba688d29b31830854014091233575d474f3a6e808e","sha256:dd5a09481ae1f5caf8d1dbc87bc7f86a01af030796467ba25851ad69964d226d","sha256:a8bce2aaf5ce6f1a5459b72de64927a1e507a911453789bf60df06752222cacd","sha256:dc0b750a934e8f376af23de6dcab1af282967498844a6510aed2c61277f20c11"]},"history":[{"created":"2020-05-11T18:17:24.516727354Z","created_by":"stacker umoci repack"},{"created":"2020-05-11T18:17:33.111086359Z","created_by":"stacker build","author":"root@jenkinsProduction-Atom-Full-Build-c3-master-159CI","empty_layer":true},{"created":"2020-05-11T18:18:43.147035914Z","created_by":"stacker umoci repack","author":"root@jenkinsProduction-Atom-Full-Build-c3-master-159CI"},{"created":"2020-05-11T18:19:03.346279546Z","created_by":"stacker build","author":"root@jenkinsProduction-Atom-Full-Build-c3-master-159CI","empty_layer":true},{"created":"2020-05-11T18:27:01.623678656Z","created_by":"stacker umoci repack","author":"root@jenkinsProduction-Atom-Full-Build-c3-master-159CI"},{"created":"2020-05-11T18:27:23.420280147Z","created_by":"stacker build","author":"root@jenkinsProduction-Atom-Full-Build-c3-master-159CI","empty_layer":true},{"created":"2020-05-11T19:11:54.886053615Z","created_by":"stacker umoci repack","author":"root@jenkinsProduction-Atom-Full-Build-c3-master-159CI"},{"created":"2020-05-11T19:12:23.239785708Z","created_by":"stacker build","author":"root@jenkinsProduction-Atom-Full-Build-c3-master-159CI","empty_layer":true}]`) + + err = makeTestFile(path.Join(dbDir, "zot-invalid-layer", "blobs/sha256", "5f00b5570a5561a6f9b7e66e4f26e2e30c4d09b43a8d3f993f3c1c99be6137a6"), content) + if err != nil { + return err + } + + // Create a image with no layer blob + + err = os.MkdirAll(path.Join(dbDir, "zot-no-layer", "blobs/sha256"), 0755) + if err != nil { + return err + } + + content = fmt.Sprintf(`{"schemaVersion":2,"manifests":[{"mediaType":"application/vnd.oci.image.manifest.v1+json","digest":"sha256:eca04f027f414362596f2632746d8a178362170b9ac9af772011fedcc3877ebb","size":886,"annotations":{"org.opencontainers.image.ref.name":"0.3.25"},"platform":{"architecture":"amd64","os":"linux"}},{"mediaType":"application/vnd.oci.image.manifest.v1+json","digest":"sha256:45df53588e59759a12bd3eca553cdc9063939baac9a28d7ebb6101e4ec230b76","size":873,"annotations":{"org.opencontainers.image.ref.name":"0.3.22-squashfs"},"platform":{"architecture":"amd64","os":"linux"}},{"mediaType":"application/vnd.oci.image.manifest.v1+json","digest":"sha256:71448405a4b89539fcfa581afb4dc7d257f98857686b8138b08a1c539f313099","size":886,"annotations":{"org.opencontainers.image.ref.name":"0.3.19"},"platform":{"architecture":"amd64","os":"linux"}}]}`) + + err = makeTestFile(path.Join(dbDir, "zot-no-layer", "index.json"), content) + if err != nil { + return err + } + + content = fmt.Sprintf(`{"schemaVersion":2,"config":{"mediaType":"application/vnd.oci.image.config.v1+json","digest":"sha256:5f00b5570a5561a6f9b7e66e4f26e2e30c4d09b43a8d3f993f3c1c99be6137a6","size":1740},"layers":[{"mediaType":"application/vnd.oci.image.layer.v1.tar+gzip","digest":"sha256:f8b7e41ce10d9a0f614f068326c43431c2777e6fc346f729c2a643bfab24af83","size":59451113},{"mediaType":"application/vnd.oci.image.layer.v1.tar+gzip","digest":"sha256:9ca9274f196b56a708a7a672d3de88184c0158a30744d355dd0411f3a6850fa6","size":55685756},{"mediaType":"application/vnd.oci.image.layer.v1.tar+gzip","digest":"sha256:6c1ca50788f93937e9ce9341b564f86cbbcd28e367ed6a57cfc776aee4a9d050","size":113726186},{"mediaType":"application/vnd.oci.image.layer.v1.tar+gzip","digest":"sha256:d1a92139df86bdf00c818db75bf1ecc860857d142b426e9971a62f5f90e2aa57","size":108755643}],"annotations":{"ws.tycho.stacker.git_version":"0.3.25"}}`) + + err = makeTestFile(path.Join(dbDir, "zot-no-layer", "blobs/sha256", "eca04f027f414362596f2632746d8a178362170b9ac9af772011fedcc3877ebb"), content) + if err != nil { + return err + } + + content = fmt.Sprintf(`{"created":"2020-05-11T19:12:23.239785708Z","author":"root@jenkinsProduction-Atom-Full-Build-c3-master-159CI","architecture":"amd64","os":"linux","config":{"Env":["PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"]},"rootfs":{"type":"layers","diff_ids":["sha256:8817d297aa60796f41f559ba688d29b31830854014091233575d474f3a6e808e","sha256:dd5a09481ae1f5caf8d1dbc87bc7f86a01af030796467ba25851ad69964d226d","sha256:a8bce2aaf5ce6f1a5459b72de64927a1e507a911453789bf60df06752222cacd","sha256:dc0b750a934e8f376af23de6dcab1af282967498844a6510aed2c61277f20c11"]},"history":[{"created":"2020-05-11T18:17:24.516727354Z","created_by":"stacker umoci repack"},{"created":"2020-05-11T18:17:33.111086359Z","created_by":"stacker build","author":"root@jenkinsProduction-Atom-Full-Build-c3-master-159CI","empty_layer":true},{"created":"2020-05-11T18:18:43.147035914Z","created_by":"stacker umoci repack","author":"root@jenkinsProduction-Atom-Full-Build-c3-master-159CI"},{"created":"2020-05-11T18:19:03.346279546Z","created_by":"stacker build","author":"root@jenkinsProduction-Atom-Full-Build-c3-master-159CI","empty_layer":true},{"created":"2020-05-11T18:27:01.623678656Z","created_by":"stacker umoci repack","author":"root@jenkinsProduction-Atom-Full-Build-c3-master-159CI"},{"created":"2020-05-11T18:27:23.420280147Z","created_by":"stacker build","author":"root@jenkinsProduction-Atom-Full-Build-c3-master-159CI","empty_layer":true},{"created":"2020-05-11T19:11:54.886053615Z","created_by":"stacker umoci repack","author":"root@jenkinsProduction-Atom-Full-Build-c3-master-159CI"},{"created":"2020-05-11T19:12:23.239785708Z","created_by":"stacker build","author":"root@jenkinsProduction-Atom-Full-Build-c3-master-159CI","empty_layer":true}]`) + + err = makeTestFile(path.Join(dbDir, "zot-no-layer", "blobs/sha256", "5f00b5570a5561a6f9b7e66e4f26e2e30c4d09b43a8d3f993f3c1c99be6137a"), content) if err != nil { return err } @@ -258,9 +382,45 @@ func TestImageFormat(t *testing.T) { So(err, ShouldBeNil) So(isValidImage, ShouldEqual, true) + isValidImage, err = cve.IsValidImageFormat(path.Join(dbDir, "zot-test:0.0.1")) + So(err, ShouldBeNil) + So(isValidImage, ShouldEqual, true) + + isValidImage, err = cve.IsValidImageFormat(path.Join(dbDir, "zot-test:0.0.")) + So(err, ShouldBeNil) + So(isValidImage, ShouldEqual, false) + isValidImage, err = cve.IsValidImageFormat(path.Join(dbDir, "zot-noindex-test")) So(err, ShouldNotBeNil) So(isValidImage, ShouldEqual, false) + + isValidImage, err = cve.IsValidImageFormat(path.Join(dbDir, "zot--tet")) + So(err, ShouldNotBeNil) + So(isValidImage, ShouldEqual, false) + + isValidImage, err = cve.IsValidImageFormat(path.Join(dbDir, "zot-noindex-test")) + So(err, ShouldNotBeNil) + So(isValidImage, ShouldEqual, false) + + isValidImage, err = cve.IsValidImageFormat(path.Join(dbDir, "zot-squashfs-noblobs")) + So(err, ShouldNotBeNil) + So(isValidImage, ShouldEqual, false) + + isValidImage, err = cve.IsValidImageFormat(path.Join(dbDir, "zot-squashfs-invalid-index")) + So(err, ShouldNotBeNil) + So(isValidImage, ShouldEqual, false) + + isValidImage, err = cve.IsValidImageFormat(path.Join(dbDir, "zot-squashfs-invalid-blob")) + So(err, ShouldNotBeNil) + So(isValidImage, ShouldEqual, false) + + isValidImage, err = cve.IsValidImageFormat(path.Join(dbDir, "zot-squashfs-test:0.3.22-squashfs")) + So(err, ShouldNotBeNil) + So(isValidImage, ShouldEqual, false) + + isValidImage, err = cve.IsValidImageFormat(path.Join(dbDir, "zot-nonreadable-test")) + So(err, ShouldNotBeNil) + So(isValidImage, ShouldEqual, false) }) } @@ -289,6 +449,14 @@ func TestImageTag(t *testing.T) { imageTags, err = cve.GetImageTagsWithTimestamp(dbDir, "zot-squashfs-invalid-blob") So(err, ShouldNotBeNil) So(len(imageTags), ShouldEqual, 0) + + imageTags, err = cve.GetImageTagsWithTimestamp(dbDir, "zot-invalid-layer") + So(err, ShouldNotBeNil) + So(len(imageTags), ShouldEqual, 0) + + imageTags, err = cve.GetImageTagsWithTimestamp(dbDir, "zot-no-layer") + So(err, ShouldNotBeNil) + So(len(imageTags), ShouldEqual, 0) }) } @@ -377,10 +545,6 @@ func TestCVESearch(t *testing.T) { id := cveResult.ImgList.CVEResultForImage.CVEList[0].ID - resp, _ = resty.R().SetBasicAuth(username, passphrase).Get(BaseURL1 + "/query?query={ImageListForCVE(id:\"" + id + "\"){Name%20Tags}}") - So(resp, ShouldNotBeNil) - So(resp.StatusCode(), ShouldEqual, 200) - resp, _ = resty.R().SetBasicAuth(username, passphrase).Get(BaseURL1 + "/query?query={ImageListWithCVEFixed(id:\"" + id + "\",image:\"zot-test\"){Tags{Name%20Timestamp}}}") So(resp, ShouldNotBeNil) So(resp.StatusCode(), ShouldEqual, 200) @@ -390,6 +554,14 @@ func TestCVESearch(t *testing.T) { So(err, ShouldBeNil) So(len(imgFixedCVEResult.ImgResults.ImgResultForFixedCVE.Tags), ShouldEqual, 0) + resp, _ = resty.R().SetBasicAuth(username, passphrase).Get(BaseURL1 + "/query?query={ImageListWithCVEFixed(id:\"" + id + "\",image:\"zot-cve-test\"){Tags{Name%20Timestamp}}}") + So(resp, ShouldNotBeNil) + So(resp.StatusCode(), ShouldEqual, 200) + + err = json.Unmarshal(resp.Body(), &imgFixedCVEResult) + So(err, ShouldBeNil) + So(len(imgFixedCVEResult.ImgResults.ImgResultForFixedCVE.Tags), ShouldEqual, 0) + resp, _ = resty.R().SetBasicAuth(username, passphrase).Get(BaseURL1 + "/query?query={ImageListWithCVEFixed(id:\"" + id + "\",image:\"zot-test\"){Tags{Name%20Timestamp}}}") So(resp, ShouldNotBeNil) So(resp.StatusCode(), ShouldEqual, 200) @@ -427,6 +599,10 @@ func TestCVESearch(t *testing.T) { So(resp, ShouldNotBeNil) So(resp.StatusCode(), ShouldEqual, 200) + resp, _ = resty.R().SetBasicAuth(username, passphrase).Get(BaseURL1 + "/query?query={ImageListWithCVEFixed(id:\"" + id + "\",image:\"zot-squashfs-test\"){Tags{Name%20Timestamp}}}") + So(resp, ShouldNotBeNil) + So(resp.StatusCode(), ShouldEqual, 200) + resp, _ = resty.R().SetBasicAuth(username, passphrase).Get(BaseURL1 + "/query?query={CVEListForImage(image:\"zot-squashfs-invalid-blob:commit-aaa7c6e7-squashfs\"){Tag%20CVEList{Id%20Description%20Severity%20PackageList{Name%20InstalledVersion%20FixedVersion}}}}") So(resp, ShouldNotBeNil) So(resp.StatusCode(), ShouldEqual, 200) @@ -499,6 +675,10 @@ func TestCVESearch(t *testing.T) { resp, _ = resty.R().SetBasicAuth(username, passphrase).Get(BaseURL1 + "/query?query={CVEListForImage(reo:\"zot-test:1.0.0\"){Tag%20CVEList{Id%20Description%20Severity}}}") So(resp, ShouldNotBeNil) So(resp.StatusCode(), ShouldEqual, 422) + + resp, _ = resty.R().SetBasicAuth(username, passphrase).Get(BaseURL1 + "/query?query={ImageListForCVE(id:\"" + id + "\"){Name%20Tags}}") + So(resp, ShouldNotBeNil) + So(resp.StatusCode(), ShouldEqual, 200) }) } diff --git a/pkg/extensions/search/resolver.go b/pkg/extensions/search/resolver.go index cdd9d13b..52bcd13e 100644 --- a/pkg/extensions/search/resolver.go +++ b/pkg/extensions/search/resolver.go @@ -7,7 +7,6 @@ import ( "path" "strings" - "github.com/anuvu/zot/errors" "github.com/anuvu/zot/pkg/log" cveinfo "github.com/anuvu/zot/pkg/extensions/search/cve" @@ -59,12 +58,6 @@ func (r *queryResolver) CVEListForImage(ctx context.Context, image string) (*CVE if !isValidImage { r.cveInfo.Log.Debug().Msg("Image media type not supported for scanning") - return &CVEResultForImage{}, errors.ErrScanNotSupported - } - - if err != nil { - r.cveInfo.Log.Error().Err(err).Msg("Error scanning image repository") - return &CVEResultForImage{}, err } @@ -157,25 +150,6 @@ func (r *queryResolver) ImageListForCve(ctx context.Context, id string) ([]*ImgR for _, repo := range repoList { r.cveInfo.Log.Info().Str("Extracting list of tags available in image", repo).Msg("") - isValidImage, err := r.cveInfo.IsValidImageFormat(path.Join(r.dir, repo)) - if !isValidImage { - r.cveInfo.Log.Debug().Str("Image media type not supported for scanning", repo) - - continue - } - - if err != nil { - r.cveInfo.Log.Error().Err(err).Str("Error reading image media type", repo) - - return cveResult, err - } - - if err != nil { - r.cveInfo.Log.Error().Err(err).Str("Error reading image media type", repo) - - return cveResult, err - } - tagList, err := r.imgStore.GetImageTags(repo) if err != nil { r.cveInfo.Log.Error().Err(err).Msg("Not able to get list of Image Tag") @@ -188,6 +162,13 @@ func (r *queryResolver) ImageListForCve(ctx context.Context, id string) ([]*ImgR for _, tag := range tagList { r.cveInfo.CveTrivyConfig.TrivyConfig.Input = path.Join(r.dir, repo+":"+tag) + isValidImage, _ := r.cveInfo.IsValidImageFormat(r.cveInfo.CveTrivyConfig.TrivyConfig.Input) + if !isValidImage { + r.cveInfo.Log.Debug().Str("Image media type not supported for scanning", repo) + + continue + } + r.cveInfo.Log.Info().Str("Scanning Image", path.Join(r.dir, repo+":"+tag)).Msg("") results, err := cveinfo.ScanImage(r.cveInfo.CveTrivyConfig) @@ -224,20 +205,9 @@ func (r *queryResolver) ImageListWithCVEFixed(ctx context.Context, id string, im r.cveInfo.Log.Info().Str("Extracting list of tags available in image", image).Msg("") - isValidImage, err := r.cveInfo.IsValidImageFormat(path.Join(r.dir, image)) - if !isValidImage { - r.cveInfo.Log.Debug().Msg("Image media type not supported for scanning") - - return imgResultForFixedCVE, errors.ErrScanNotSupported - } - - if err != nil { - return imgResultForFixedCVE, err - } - tagsInfo, err := r.cveInfo.GetImageTagsWithTimestamp(r.dir, image) if err != nil { - r.cveInfo.Log.Error().Err(err).Msg("Error while readling image media type") + r.cveInfo.Log.Error().Err(err).Msg("Error while readling image tags") return imgResultForFixedCVE, err } @@ -249,6 +219,15 @@ func (r *queryResolver) ImageListWithCVEFixed(ctx context.Context, id string, im for _, tag := range tagsInfo { r.cveInfo.CveTrivyConfig.TrivyConfig.Input = path.Join(r.dir, image+":"+tag.Name) + isValidImage, _ := r.cveInfo.IsValidImageFormat(r.cveInfo.CveTrivyConfig.TrivyConfig.Input) + if !isValidImage { + r.cveInfo.Log.Debug().Msg("Image media type not supported for scanning, adding as a infected image") + + infectedTags = append(infectedTags, cveinfo.TagInfo{Name: tag.Name, Timestamp: tag.Timestamp}) + + continue + } + r.cveInfo.Log.Info().Str("Scanning image", path.Join(r.dir, image+":"+tag.Name)).Msg("") results, err := cveinfo.ScanImage(r.cveInfo.CveTrivyConfig) @@ -275,27 +254,35 @@ func (r *queryResolver) ImageListWithCVEFixed(ctx context.Context, id string, im } } + var finalTagList []*TagInfo + if len(infectedTags) != 0 { r.cveInfo.Log.Info().Msg("Comparing fixed tags timestamp") fixedTags := cveinfo.GetFixedTags(tagsInfo, infectedTags) - finalTagList := make([]*TagInfo, 0) + finalTagList = getGraphqlCompatibleTags(fixedTags) + } else { + r.cveInfo.Log.Info().Msg("Input image does not contain any tag that have given cve") - for _, tag := range fixedTags { - copyTag := tag.Name - - copyTimeStamp := tag.Timestamp - - finalTagList = append(finalTagList, &TagInfo{Name: ©Tag, Timestamp: ©TimeStamp}) - } - - imgResultForFixedCVE = &ImgResultForFixedCve{Tags: finalTagList} - - return imgResultForFixedCVE, nil + finalTagList = getGraphqlCompatibleTags(tagsInfo) } - r.cveInfo.Log.Info().Msg("Input image does not contain any tag that does not have given cve") + imgResultForFixedCVE = &ImgResultForFixedCve{Tags: finalTagList} return imgResultForFixedCVE, nil } + +func getGraphqlCompatibleTags(fixedTags []cveinfo.TagInfo) []*TagInfo { + finalTagList := make([]*TagInfo, 0) + + for _, tag := range fixedTags { + copyTag := tag.Name + + copyTimeStamp := tag.Timestamp + + finalTagList = append(finalTagList, &TagInfo{Name: ©Tag, Timestamp: ©TimeStamp}) + } + + return finalTagList +}