From 5c01c4eab490adda197f9842e2e1f00281f238fa Mon Sep 17 00:00:00 2001 From: Ramkumar Chinchani <45800463+rchincha@users.noreply.github.com> Date: Sat, 20 Aug 2022 01:18:48 -0700 Subject: [PATCH] support OCI image index at manifest endpoint (#638) Signed-off-by: Ramkumar Chinchani Signed-off-by: Ramkumar Chinchani --- .github/workflows/ecosystem-tools.yaml | 13 +- pkg/api/controller_test.go | 571 +++++++++++++++++++++++++ pkg/storage/local.go | 178 +++++++- pkg/storage/s3/s3.go | 168 +++++++- pkg/storage/s3/s3_test.go | 451 +++++++++++++++++++ test/blackbox/pushpull.bats | 37 ++ 6 files changed, 1400 insertions(+), 18 deletions(-) diff --git a/.github/workflows/ecosystem-tools.yaml b/.github/workflows/ecosystem-tools.yaml index f265a139..d6ccd376 100644 --- a/.github/workflows/ecosystem-tools.yaml +++ b/.github/workflows/ecosystem-tools.yaml @@ -26,14 +26,13 @@ jobs: go get -u github.com/swaggo/swag/cmd/swag go mod download sudo apt-get update - sudo apt-get -y install rpm uidmap + sudo apt-get install libgpgme-dev libassuan-dev libbtrfs-dev libdevmapper-dev pkg-config rpm uidmap # install skopeo - . /etc/os-release - echo "deb https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/xUbuntu_${VERSION_ID}/ /" | sudo tee /etc/apt/sources.list.d/devel:kubic:libcontainers:stable.list - curl -L https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/xUbuntu_${VERSION_ID}/Release.key | sudo apt-key add - - sudo apt-get update - sudo apt-get -y upgrade - sudo apt-get -y install skopeo + git clone -b v1.9.0 https://github.com/containers/skopeo.git + cd skopeo + make bin/skopeo + sudo cp bin/skopeo /usr/bin + skopeo -v - name: Run push-pull tests run: | make push-pull diff --git a/pkg/api/controller_test.go b/pkg/api/controller_test.go index 359958f3..735699ff 100644 --- a/pkg/api/controller_test.go +++ b/pkg/api/controller_test.go @@ -4641,6 +4641,577 @@ func TestStorageCommit(t *testing.T) { }) } +func TestManifestImageIndex(t *testing.T) { + Convey("Make a new controller", t, func() { + port := test.GetFreePort() + baseURL := test.GetBaseURL(port) + conf := config.New() + conf.HTTP.Port = port + + ctlr := api.NewController(conf) + dir := t.TempDir() + ctlr.Config.Storage.RootDirectory = dir + + go startServer(ctlr) + defer stopServer(ctlr) + test.WaitTillServerReady(baseURL) + + rthdlr := api.NewRouteHandler(ctlr) + + // create a blob/layer + resp, err := resty.R().Post(baseURL + "/v2/index/blobs/uploads/") + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusAccepted) + loc := test.Location(baseURL, resp) + So(loc, ShouldNotBeEmpty) + + // since we are not specifying any prefix i.e provided in config while starting server, + // so it should store index1 to global root dir + _, err = os.Stat(path.Join(dir, "index")) + So(err, ShouldBeNil) + + resp, err = resty.R().Get(loc) + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusNoContent) + content := []byte("this is a blob1") + digest := godigest.FromBytes(content) + So(digest, ShouldNotBeNil) + bdgst1 := digest + // monolithic blob upload: success + resp, err = resty.R().SetQueryParam("digest", digest.String()). + SetHeader("Content-Type", "application/octet-stream").SetBody(content).Put(loc) + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusCreated) + blobLoc := resp.Header().Get("Location") + So(blobLoc, ShouldNotBeEmpty) + So(resp.Header().Get("Content-Length"), ShouldEqual, "0") + So(resp.Header().Get(constants.DistContentDigestKey), ShouldNotBeEmpty) + + // check a non-existent manifest + resp, err = resty.R().SetHeader("Content-Type", ispec.MediaTypeImageManifest). + SetBody(content).Head(baseURL + "/v2/unknown/manifests/test:1.0") + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusNotFound) + + // upload image config blob + resp, err = resty.R().Post(baseURL + "/v2/index/blobs/uploads/") + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusAccepted) + loc = test.Location(baseURL, resp) + cblob, cdigest := test.GetRandomImageConfig() + + resp, err = resty.R(). + SetContentLength(true). + SetHeader("Content-Length", fmt.Sprintf("%d", len(cblob))). + SetHeader("Content-Type", "application/octet-stream"). + SetQueryParam("digest", cdigest.String()). + SetBody(cblob). + Put(loc) + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusCreated) + + // create a manifest + manifest := ispec.Manifest{ + Config: ispec.Descriptor{ + MediaType: ispec.MediaTypeImageConfig, + Digest: cdigest, + Size: int64(len(cblob)), + }, + Layers: []ispec.Descriptor{ + { + MediaType: ispec.MediaTypeImageLayer, + Digest: digest, + Size: int64(len(content)), + }, + }, + } + manifest.SchemaVersion = 2 + content, err = json.Marshal(manifest) + So(err, ShouldBeNil) + digest = godigest.FromBytes(content) + So(digest, ShouldNotBeNil) + m1content := content + resp, err = resty.R().SetHeader("Content-Type", ispec.MediaTypeImageManifest). + SetBody(content).Put(baseURL + "/v2/index/manifests/test:1.0") + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusCreated) + digestHdr := resp.Header().Get(constants.DistContentDigestKey) + So(digestHdr, ShouldNotBeEmpty) + So(digestHdr, ShouldEqual, digest.String()) + + // create another manifest but upload using its sha256 reference + + // upload image config blob + resp, err = resty.R().Post(baseURL + "/v2/index/blobs/uploads/") + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusAccepted) + loc = test.Location(baseURL, resp) + cblob, cdigest = test.GetRandomImageConfig() + + resp, err = resty.R(). + SetContentLength(true). + SetHeader("Content-Length", fmt.Sprintf("%d", len(cblob))). + SetHeader("Content-Type", "application/octet-stream"). + SetQueryParam("digest", cdigest.String()). + SetBody(cblob). + Put(loc) + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusCreated) + + // create a manifest + manifest = ispec.Manifest{ + Config: ispec.Descriptor{ + MediaType: ispec.MediaTypeImageConfig, + Digest: cdigest, + Size: int64(len(cblob)), + }, + Layers: []ispec.Descriptor{ + { + MediaType: ispec.MediaTypeImageLayer, + Digest: bdgst1, + Size: int64(len(content)), + }, + }, + } + manifest.SchemaVersion = 2 + content, err = json.Marshal(manifest) + So(err, ShouldBeNil) + digest = godigest.FromBytes(content) + So(digest, ShouldNotBeNil) + m2dgst := digest + m2size := len(content) + resp, err = resty.R().SetHeader("Content-Type", ispec.MediaTypeImageManifest). + SetBody(content).Put(baseURL + fmt.Sprintf("/v2/index/manifests/%s", digest)) + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusCreated) + digestHdr = resp.Header().Get(constants.DistContentDigestKey) + So(digestHdr, ShouldNotBeEmpty) + So(digestHdr, ShouldEqual, digest.String()) + + Convey("Image index", func() { + // upload image config blob + resp, err = resty.R().Post(baseURL + "/v2/index/blobs/uploads/") + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusAccepted) + loc = test.Location(baseURL, resp) + cblob, cdigest := test.GetRandomImageConfig() + + resp, err = resty.R(). + SetContentLength(true). + SetHeader("Content-Length", fmt.Sprintf("%d", len(cblob))). + SetHeader("Content-Type", "application/octet-stream"). + SetQueryParam("digest", cdigest.String()). + SetBody(cblob). + Put(loc) + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusCreated) + + // create a manifest + manifest := ispec.Manifest{ + Config: ispec.Descriptor{ + MediaType: ispec.MediaTypeImageConfig, + Digest: cdigest, + Size: int64(len(cblob)), + }, + Layers: []ispec.Descriptor{ + { + MediaType: ispec.MediaTypeImageLayer, + Digest: bdgst1, + Size: int64(len(content)), + }, + }, + } + manifest.SchemaVersion = 2 + content, err = json.Marshal(manifest) + So(err, ShouldBeNil) + digest = godigest.FromBytes(content) + So(digest, ShouldNotBeNil) + resp, err = resty.R().SetHeader("Content-Type", ispec.MediaTypeImageManifest). + SetBody(content).Put(baseURL + fmt.Sprintf("/v2/index/manifests/%s", digest.String())) + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusCreated) + digestHdr := resp.Header().Get(constants.DistContentDigestKey) + So(digestHdr, ShouldNotBeEmpty) + So(digestHdr, ShouldEqual, digest.String()) + + var index ispec.Index + index.SchemaVersion = 2 + index.Manifests = []ispec.Descriptor{ + { + MediaType: ispec.MediaTypeImageIndex, + Digest: digest, + Size: int64(len(content)), + }, + { + MediaType: ispec.MediaTypeImageIndex, + Digest: m2dgst, + Size: int64(m2size), + }, + } + + content, err = json.Marshal(index) + So(err, ShouldBeNil) + digest = godigest.FromBytes(content) + So(digest, ShouldNotBeNil) + index1dgst := digest + resp, err = resty.R().SetHeader("Content-Type", ispec.MediaTypeImageIndex). + SetBody(content).Put(baseURL + "/v2/index/manifests/test:index1") + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusCreated) + digestHdr = resp.Header().Get(constants.DistContentDigestKey) + So(digestHdr, ShouldNotBeEmpty) + So(digestHdr, ShouldEqual, digest.String()) + resp, err = resty.R().SetHeader("Content-Type", ispec.MediaTypeImageIndex). + Get(baseURL + "/v2/index/manifests/test:index1") + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusOK) + So(resp.Body(), ShouldNotBeEmpty) + So(resp.Header().Get("Content-Type"), ShouldNotBeEmpty) + + // upload another image config blob + resp, err = resty.R().Post(baseURL + "/v2/index/blobs/uploads/") + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusAccepted) + loc = test.Location(baseURL, resp) + cblob, cdigest = test.GetRandomImageConfig() + + resp, err = resty.R(). + SetContentLength(true). + SetHeader("Content-Length", fmt.Sprintf("%d", len(cblob))). + SetHeader("Content-Type", "application/octet-stream"). + SetQueryParam("digest", cdigest.String()). + SetBody(cblob). + Put(loc) + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusCreated) + + // create another manifest + manifest = ispec.Manifest{ + Config: ispec.Descriptor{ + MediaType: ispec.MediaTypeImageConfig, + Digest: cdigest, + Size: int64(len(cblob)), + }, + Layers: []ispec.Descriptor{ + { + MediaType: ispec.MediaTypeImageLayer, + Digest: bdgst1, + Size: int64(len(content)), + }, + }, + } + manifest.SchemaVersion = 2 + content, err = json.Marshal(manifest) + So(err, ShouldBeNil) + digest = godigest.FromBytes(content) + So(digest, ShouldNotBeNil) + m4dgst := digest + m4size := len(content) + resp, err = resty.R().SetHeader("Content-Type", ispec.MediaTypeImageManifest). + SetBody(content).Put(baseURL + fmt.Sprintf("/v2/index/manifests/%s", digest)) + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusCreated) + digestHdr = resp.Header().Get(constants.DistContentDigestKey) + So(digestHdr, ShouldNotBeEmpty) + So(digestHdr, ShouldEqual, digest.String()) + + index.SchemaVersion = 2 + index.Manifests = []ispec.Descriptor{ + { + MediaType: ispec.MediaTypeImageIndex, + Digest: digest, + Size: int64(len(content)), + }, + { + MediaType: ispec.MediaTypeImageIndex, + Digest: m2dgst, + Size: int64(m2size), + }, + } + + content, err = json.Marshal(index) + So(err, ShouldBeNil) + digest = godigest.FromBytes(content) + So(digest, ShouldNotBeNil) + resp, err = resty.R().SetHeader("Content-Type", ispec.MediaTypeImageIndex). + SetBody(content).Put(baseURL + "/v2/index/manifests/test:index2") + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusCreated) + digestHdr = resp.Header().Get(constants.DistContentDigestKey) + So(digestHdr, ShouldNotBeEmpty) + So(digestHdr, ShouldEqual, digest.String()) + resp, err = resty.R().SetHeader("Content-Type", ispec.MediaTypeImageIndex). + Get(baseURL + "/v2/index/manifests/test:index2") + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusOK) + So(resp.Body(), ShouldNotBeEmpty) + So(resp.Header().Get("Content-Type"), ShouldNotBeEmpty) + + Convey("List tags", func() { + request, _ := http.NewRequestWithContext(context.TODO(), "GET", baseURL, nil) + request = mux.SetURLVars(request, map[string]string{"name": "index"}) + response := httptest.NewRecorder() + + rthdlr.ListTags(response, request) + + resp := response.Result() + defer resp.Body.Close() + So(resp, ShouldNotBeNil) + So(resp.StatusCode, ShouldEqual, http.StatusOK) + + var tags api.ImageTags + err = json.NewDecoder(resp.Body).Decode(&tags) + So(err, ShouldBeNil) + So(len(tags.Tags), ShouldEqual, 3) + So(tags.Tags, ShouldContain, "test:1.0") + So(tags.Tags, ShouldContain, "test:index1") + So(tags.Tags, ShouldContain, "test:index2") + }) + + Convey("Another index with same manifest", func() { + var index ispec.Index + index.SchemaVersion = 2 + index.Manifests = []ispec.Descriptor{ + { + MediaType: ispec.MediaTypeImageIndex, + Digest: m4dgst, + Size: int64(m4size), + }, + } + + content, err = json.Marshal(index) + So(err, ShouldBeNil) + digest = godigest.FromBytes(content) + So(digest, ShouldNotBeNil) + resp, err = resty.R().SetHeader("Content-Type", ispec.MediaTypeImageIndex). + SetBody(content).Put(baseURL + "/v2/index/manifests/test:index3") + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusCreated) + digestHdr = resp.Header().Get(constants.DistContentDigestKey) + So(digestHdr, ShouldNotBeEmpty) + So(digestHdr, ShouldEqual, digest.String()) + }) + + Convey("Another index using digest with same manifest", func() { + var index ispec.Index + index.SchemaVersion = 2 + index.Manifests = []ispec.Descriptor{ + { + MediaType: ispec.MediaTypeImageIndex, + Digest: m4dgst, + Size: int64(m4size), + }, + } + + content, err = json.Marshal(index) + So(err, ShouldBeNil) + digest = godigest.FromBytes(content) + So(digest, ShouldNotBeNil) + resp, err = resty.R().SetHeader("Content-Type", ispec.MediaTypeImageIndex). + SetBody(content).Put(baseURL + fmt.Sprintf("/v2/index/manifests/%s", digest)) + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusCreated) + digestHdr = resp.Header().Get(constants.DistContentDigestKey) + So(digestHdr, ShouldNotBeEmpty) + So(digestHdr, ShouldEqual, digest.String()) + }) + + Convey("Deleting an image index", func() { + // delete manifest by tag should pass + resp, err = resty.R().Delete(baseURL + "/v2/index/manifests/test:index3") + So(err, ShouldBeNil) + resp, err = resty.R().SetHeader("Content-Type", ispec.MediaTypeImageIndex). + Get(baseURL + "/v2/index/manifests/test:index3") + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusNotFound) + So(resp.Body(), ShouldNotBeEmpty) + resp, err = resty.R().Delete(baseURL + "/v2/index/manifests/test:index1") + So(err, ShouldBeNil) + resp, err = resty.R().SetHeader("Content-Type", ispec.MediaTypeImageIndex). + Get(baseURL + "/v2/index/manifests/test:index1") + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusNotFound) + So(resp.Body(), ShouldNotBeEmpty) + resp, err = resty.R().SetHeader("Content-Type", ispec.MediaTypeImageIndex). + Get(baseURL + "/v2/index/manifests/test:index2") + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusOK) + So(resp.Body(), ShouldNotBeEmpty) + So(resp.Header().Get("Content-Type"), ShouldNotBeEmpty) + }) + + Convey("Deleting an image index by digest", func() { + // delete manifest by tag should pass + resp, err = resty.R().Delete(baseURL + "/v2/index/manifests/test:index3") + So(err, ShouldBeNil) + resp, err = resty.R().SetHeader("Content-Type", ispec.MediaTypeImageIndex). + Get(baseURL + "/v2/index/manifests/test:index3") + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusNotFound) + So(resp.Body(), ShouldNotBeEmpty) + resp, err = resty.R().Delete(baseURL + fmt.Sprintf("/v2/index/manifests/%s", index1dgst)) + So(err, ShouldBeNil) + resp, err = resty.R().SetHeader("Content-Type", ispec.MediaTypeImageIndex). + Get(baseURL + "/v2/index/manifests/test:index1") + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusNotFound) + So(resp.Body(), ShouldNotBeEmpty) + resp, err = resty.R().SetHeader("Content-Type", ispec.MediaTypeImageIndex). + Get(baseURL + "/v2/index/manifests/test:index2") + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusOK) + So(resp.Body(), ShouldNotBeEmpty) + So(resp.Header().Get("Content-Type"), ShouldNotBeEmpty) + }) + + Convey("Update an index tag with different manifest", func() { + // create a blob/layer + resp, err := resty.R().Post(baseURL + "/v2/index/blobs/uploads/") + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusAccepted) + loc := test.Location(baseURL, resp) + So(loc, ShouldNotBeEmpty) + + resp, err = resty.R().Get(loc) + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusNoContent) + content := []byte("this is another blob") + digest := godigest.FromBytes(content) + So(digest, ShouldNotBeNil) + // monolithic blob upload: success + resp, err = resty.R().SetQueryParam("digest", digest.String()). + SetHeader("Content-Type", "application/octet-stream").SetBody(content).Put(loc) + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusCreated) + blobLoc := resp.Header().Get("Location") + So(blobLoc, ShouldNotBeEmpty) + So(resp.Header().Get("Content-Length"), ShouldEqual, "0") + So(resp.Header().Get(constants.DistContentDigestKey), ShouldNotBeEmpty) + + // create a manifest with same blob but a different tag + manifest = ispec.Manifest{ + Config: ispec.Descriptor{ + MediaType: ispec.MediaTypeImageConfig, + Digest: cdigest, + Size: int64(len(cblob)), + }, + Layers: []ispec.Descriptor{ + { + MediaType: ispec.MediaTypeImageLayer, + Digest: digest, + Size: int64(len(content)), + }, + }, + } + manifest.SchemaVersion = 2 + content, err = json.Marshal(manifest) + So(err, ShouldBeNil) + digest = godigest.FromBytes(content) + So(digest, ShouldNotBeNil) + resp, err = resty.R().SetHeader("Content-Type", ispec.MediaTypeImageManifest). + SetBody(content).Put(baseURL + fmt.Sprintf("/v2/index/manifests/%s", digest)) + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusCreated) + digestHdr = resp.Header().Get(constants.DistContentDigestKey) + So(digestHdr, ShouldNotBeEmpty) + So(digestHdr, ShouldEqual, digest.String()) + + index.SchemaVersion = 2 + index.Manifests = []ispec.Descriptor{ + { + MediaType: ispec.MediaTypeImageIndex, + Digest: digest, + Size: int64(len(content)), + }, + } + + content, err = json.Marshal(index) + So(err, ShouldBeNil) + digest = godigest.FromBytes(content) + So(digest, ShouldNotBeNil) + resp, err = resty.R().SetHeader("Content-Type", ispec.MediaTypeImageIndex). + SetBody(content).Put(baseURL + "/v2/index/manifests/test:index1") + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusCreated) + digestHdr = resp.Header().Get(constants.DistContentDigestKey) + So(digestHdr, ShouldNotBeEmpty) + So(digestHdr, ShouldEqual, digest.String()) + resp, err = resty.R().SetHeader("Content-Type", ispec.MediaTypeImageIndex). + Get(baseURL + "/v2/index/manifests/test:index1") + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusOK) + So(resp.Body(), ShouldNotBeEmpty) + So(resp.Header().Get("Content-Type"), ShouldNotBeEmpty) + + // delete manifest by tag should pass + resp, err = resty.R().Delete(baseURL + "/v2/index/manifests/test:index1") + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusAccepted) + resp, err = resty.R().SetHeader("Content-Type", ispec.MediaTypeImageIndex). + Get(baseURL + "/v2/index/manifests/test:index1") + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusNotFound) + So(resp.Body(), ShouldNotBeEmpty) + }) + + Convey("Negative test cases", func() { + Convey("Delete index", func() { + err = os.Remove(path.Join(dir, "index", "blobs", index1dgst.Algorithm().String(), index1dgst.Encoded())) + So(err, ShouldBeNil) + resp, err = resty.R().Delete(baseURL + fmt.Sprintf("/v2/index/manifests/%s", index1dgst)) + So(err, ShouldBeNil) + resp, err = resty.R().SetHeader("Content-Type", ispec.MediaTypeImageIndex). + Get(baseURL + "/v2/index/manifests/test:index1") + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusNotFound) + So(resp.Body(), ShouldNotBeEmpty) + }) + + Convey("Corrupt index", func() { + err = ioutil.WriteFile(path.Join(dir, "index", "blobs", index1dgst.Algorithm().String(), index1dgst.Encoded()), + []byte("deadbeef"), storage.DefaultFilePerms) + So(err, ShouldBeNil) + resp, err = resty.R().Delete(baseURL + fmt.Sprintf("/v2/index/manifests/%s", index1dgst)) + So(err, ShouldBeNil) + resp, err = resty.R().SetHeader("Content-Type", ispec.MediaTypeImageIndex). + Get(baseURL + "/v2/index/manifests/test:index1") + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusInternalServerError) + So(resp.Body(), ShouldBeEmpty) + }) + + Convey("Change media-type", func() { + // previously a manifest, try writing an image index + var index ispec.Index + index.SchemaVersion = 2 + index.Manifests = []ispec.Descriptor{ + { + MediaType: ispec.MediaTypeImageIndex, + Digest: m4dgst, + Size: int64(m4size), + }, + } + + content, err = json.Marshal(index) + So(err, ShouldBeNil) + digest = godigest.FromBytes(content) + So(digest, ShouldNotBeNil) + resp, err = resty.R().SetHeader("Content-Type", ispec.MediaTypeImageIndex). + SetBody(content).Put(baseURL + "/v2/index/manifests/test:1.0") + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusBadRequest) + + // previously an image index, try writing a manifest + resp, err = resty.R().SetHeader("Content-Type", ispec.MediaTypeImageManifest). + SetBody(m1content).Put(baseURL + "/v2/index/manifests/test:index1") + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusBadRequest) + }) + }) + }) + }) +} + func TestInjectInterruptedImageManifest(t *testing.T) { Convey("Make a new controller", t, func() { port := test.GetFreePort() diff --git a/pkg/storage/local.go b/pkg/storage/local.go index 0ffddf76..2d7fc957 100644 --- a/pkg/storage/local.go +++ b/pkg/storage/local.go @@ -538,8 +538,102 @@ func (is *ImageStoreLocal) validateOCIManifest(repo, reference string, manifest return "", nil } +/** +before an image index manifest is pushed to a repo, its constituent manifests +are pushed first, so when updating/removing this image index manifest, we also +need to determine if there are other image index manifests which refer to the +same constitutent manifests so that they can be garbage-collected correctly + +pruneImageManifestsFromIndex is a helper routine to achieve this. +*/ +func pruneImageManifestsFromIndex(dir string, digest godigest.Digest, // nolint: gocyclo + outIndex ispec.Index, otherImgIndexes []ispec.Descriptor, log zerolog.Logger, +) ([]ispec.Descriptor, error) { + indexPath := path.Join(dir, "blobs", digest.Algorithm().String(), digest.Encoded()) + + buf, err := ioutil.ReadFile(indexPath) + if err != nil { + log.Error().Err(err).Str("dir", dir).Msg("failed to read index.json") + + return nil, err + } + + var imgIndex ispec.Index + if err := json.Unmarshal(buf, &imgIndex); err != nil { + log.Error().Err(err).Str("path", indexPath).Msg("invalid JSON") + + return nil, err + } + + inUse := map[string]uint{} + + for _, manifest := range imgIndex.Manifests { + inUse[manifest.Digest.Encoded()]++ + } + + for _, otherIndex := range otherImgIndexes { + indexPath := path.Join(dir, "blobs", otherIndex.Digest.Algorithm().String(), otherIndex.Digest.Encoded()) + + buf, err := ioutil.ReadFile(indexPath) + if err != nil { + log.Error().Err(err).Str("dir", dir).Msg("failed to read index.json") + + return nil, err + } + + var oindex ispec.Index + if err := json.Unmarshal(buf, &oindex); err != nil { + log.Error().Err(err).Str("path", indexPath).Msg("invalid JSON") + + return nil, err + } + + for _, omanifest := range oindex.Manifests { + _, ok := inUse[omanifest.Digest.Encoded()] + if ok { + inUse[omanifest.Digest.Encoded()]++ + } + } + } + + prunedManifests := []ispec.Descriptor{} + + // for all manifests in the index, skip those that either have a tag or + // are used in other imgIndexes + for _, outManifest := range outIndex.Manifests { + if outManifest.MediaType != ispec.MediaTypeImageManifest { + prunedManifests = append(prunedManifests, outManifest) + + continue + } + + _, ok := outManifest.Annotations[ispec.AnnotationRefName] + if ok { + prunedManifests = append(prunedManifests, outManifest) + + continue + } + + count, ok := inUse[outManifest.Digest.Encoded()] + if !ok { + prunedManifests = append(prunedManifests, outManifest) + + continue + } + + if count != 1 { + // this manifest is in use in other image indexes + prunedManifests = append(prunedManifests, outManifest) + + continue + } + } + + return prunedManifests, nil +} + // PutImageManifest adds an image manifest to the repository. -func (is *ImageStoreLocal) PutImageManifest(repo, reference, mediaType string, +func (is *ImageStoreLocal) PutImageManifest(repo, reference, mediaType string, // nolint: gocyclo body []byte, ) (string, error) { if err := is.InitRepo(repo); err != nil { @@ -551,7 +645,7 @@ func (is *ImageStoreLocal) PutImageManifest(repo, reference, mediaType string, // validate the manifest if !IsSupportedMediaType(mediaType) { is.log.Debug().Interface("actual", mediaType). - Interface("expected", ispec.MediaTypeImageManifest).Msg("bad manifest media type") + Msg("bad manifest media type") return "", zerr.ErrBadManifest } @@ -607,12 +701,13 @@ func (is *ImageStoreLocal) PutImageManifest(repo, reference, mediaType string, // create a new descriptor desc := ispec.Descriptor{ MediaType: mediaType, Size: int64(len(body)), Digest: mDigest, - Platform: &ispec.Platform{Architecture: "amd64", OS: "linux"}, } if !refIsDigest { desc.Annotations = map[string]string{ispec.AnnotationRefName: reference} } + var oldDgst godigest.Digest + for midx, manifest := range index.Manifests { if reference == manifest.Digest.String() { // nothing changed, so don't update @@ -631,6 +726,7 @@ func (is *ImageStoreLocal) PutImageManifest(repo, reference, mediaType string, break } + // manifest contents have changed for the same tag, // so update index.json descriptor is.log.Info(). @@ -638,9 +734,22 @@ func (is *ImageStoreLocal) PutImageManifest(repo, reference, mediaType string, Int64("new size", int64(len(body))). Str("old digest", desc.Digest.String()). Str("new digest", mDigest.String()). + Str("old mediaType", manifest.MediaType). + Str("new mediaType", mediaType). Msg("updating existing tag with new manifest contents") + // changing media-type is disallowed! + if manifest.MediaType != mediaType { + err = zerr.ErrBadManifest + is.log.Error().Err(err). + Str("old mediaType", manifest.MediaType). + Str("new mediaType", mediaType).Msg("cannot change media-type") + + return "", err + } + desc = manifest + oldDgst = manifest.Digest desc.Size = int64(len(body)) desc.Digest = mDigest @@ -666,6 +775,30 @@ func (is *ImageStoreLocal) PutImageManifest(repo, reference, mediaType string, return "", err } + /* additionally, unmarshal an image index and for all manifests in that + index, ensure that they do not have a name or they are not in other + manifest indexes else GC can never clean them */ + if (mediaType == ispec.MediaTypeImageIndex) && (oldDgst != "") { + otherImgIndexes := []ispec.Descriptor{} + + for _, manifest := range index.Manifests { + if manifest.MediaType == ispec.MediaTypeImageIndex { + otherImgIndexes = append(otherImgIndexes, manifest) + } + } + + otherImgIndexes = append(otherImgIndexes, desc) + + dir := path.Join(is.rootDir, repo) + + prunedManifests, err := pruneImageManifestsFromIndex(dir, oldDgst, index, otherImgIndexes, is.log) + if err != nil { + return "", err + } + + index.Manifests = prunedManifests + } + // now update "index.json" index.Manifests = append(index.Manifests, desc) dir = path.Join(is.rootDir, repo) @@ -731,7 +864,7 @@ func (is *ImageStoreLocal) DeleteImageManifest(repo, reference string) error { isTag := false // as per spec "reference" can be a digest and a tag - digest, err := godigest.Parse(reference) + dgst, err := godigest.Parse(reference) if err != nil { is.log.Debug().Str("invalid digest: ", reference).Msg("storage: assuming tag") @@ -757,38 +890,66 @@ func (is *ImageStoreLocal) DeleteImageManifest(repo, reference string) error { found := false + isImageIndex := false + var manifest ispec.Descriptor // we are deleting, so keep only those manifests that don't match outIndex := index outIndex.Manifests = []ispec.Descriptor{} + otherImgIndexes := []ispec.Descriptor{} + for _, manifest = range index.Manifests { if isTag { tag, ok := manifest.Annotations[ispec.AnnotationRefName] if ok && tag == reference { is.log.Debug().Str("deleting tag", tag).Msg("") - digest = manifest.Digest + dgst = manifest.Digest found = true + if manifest.MediaType == ispec.MediaTypeImageIndex { + isImageIndex = true + } + continue } } else if reference == manifest.Digest.String() { is.log.Debug().Str("deleting reference", reference).Msg("") found = true + if manifest.MediaType == ispec.MediaTypeImageIndex { + isImageIndex = true + } + continue } outIndex.Manifests = append(outIndex.Manifests, manifest) + + if manifest.MediaType == ispec.MediaTypeImageIndex { + otherImgIndexes = append(otherImgIndexes, manifest) + } } if !found { return zerr.ErrManifestNotFound } + /* additionally, unmarshal an image index and for all manifests in that + index, ensure that they do not have a name or they are not in other + manifest indexes else GC can never clean them */ + if isImageIndex { + prunedManifests, err := pruneImageManifestsFromIndex(dir, dgst, outIndex, otherImgIndexes, is.log) + if err != nil { + return err + } + + outIndex.Manifests = prunedManifests + } + // now update "index.json" dir = path.Join(is.rootDir, repo) file := path.Join(dir, "index.json") @@ -813,7 +974,7 @@ func (is *ImageStoreLocal) DeleteImageManifest(repo, reference string) error { toDelete := true for _, manifest = range outIndex.Manifests { - if digest.String() == manifest.Digest.String() { + if dgst.String() == manifest.Digest.String() { toDelete = false break @@ -821,7 +982,7 @@ func (is *ImageStoreLocal) DeleteImageManifest(repo, reference string) error { } if toDelete { - p := path.Join(dir, "blobs", digest.Algorithm().String(), digest.Encoded()) + p := path.Join(dir, "blobs", dgst.Algorithm().String(), dgst.Encoded()) _ = os.Remove(p) } @@ -1580,7 +1741,8 @@ func (is *ImageStoreLocal) writeFile(filename string, data []byte) error { } func IsSupportedMediaType(mediaType string) bool { - return mediaType == ispec.MediaTypeImageManifest || + return mediaType == ispec.MediaTypeImageIndex || + mediaType == ispec.MediaTypeImageManifest || mediaType == artifactspec.MediaTypeArtifactManifest } diff --git a/pkg/storage/s3/s3.go b/pkg/storage/s3/s3.go index fa1b0c53..44f290b1 100644 --- a/pkg/storage/s3/s3.go +++ b/pkg/storage/s3/s3.go @@ -397,8 +397,102 @@ func (is *ObjectStorage) GetImageManifest(repo, reference string) ([]byte, strin return buf, digest.String(), mediaType, nil } +/** +before an image index manifest is pushed to a repo, its constituent manifests +are pushed first, so when updating/removing this image index manifest, we also +need to determine if there are other image index manifests which refer to the +same constitutent manifests so that they can be garbage-collected correctly + +pruneImageManifestsFromIndex is a helper routine to achieve this. +*/ +func (is *ObjectStorage) pruneImageManifestsFromIndex(dir string, digest godigest.Digest, // nolint: gocyclo + outIndex ispec.Index, otherImgIndexes []ispec.Descriptor, log zerolog.Logger, +) ([]ispec.Descriptor, error) { + indexPath := path.Join(dir, "blobs", digest.Algorithm().String(), digest.Encoded()) + + buf, err := is.store.GetContent(context.Background(), indexPath) + if err != nil { + log.Error().Err(err).Str("dir", dir).Msg("failed to read index.json") + + return nil, err + } + + var imgIndex ispec.Index + if err := json.Unmarshal(buf, &imgIndex); err != nil { + log.Error().Err(err).Str("path", indexPath).Msg("invalid JSON") + + return nil, err + } + + inUse := map[string]uint{} + + for _, manifest := range imgIndex.Manifests { + inUse[manifest.Digest.Encoded()]++ + } + + for _, otherIndex := range otherImgIndexes { + indexPath := path.Join(dir, "blobs", otherIndex.Digest.Algorithm().String(), otherIndex.Digest.Encoded()) + + buf, err := is.store.GetContent(context.Background(), indexPath) + if err != nil { + log.Error().Err(err).Str("dir", dir).Msg("failed to read index.json") + + return nil, err + } + + var oindex ispec.Index + if err := json.Unmarshal(buf, &oindex); err != nil { + log.Error().Err(err).Str("path", indexPath).Msg("invalid JSON") + + return nil, err + } + + for _, omanifest := range oindex.Manifests { + _, ok := inUse[omanifest.Digest.Encoded()] + if ok { + inUse[omanifest.Digest.Encoded()]++ + } + } + } + + prunedManifests := []ispec.Descriptor{} + + // for all manifests in the index, skip those that either have a tag or + // are used in other imgIndexes + for _, outManifest := range outIndex.Manifests { + if outManifest.MediaType != ispec.MediaTypeImageManifest { + prunedManifests = append(prunedManifests, outManifest) + + continue + } + + _, ok := outManifest.Annotations[ispec.AnnotationRefName] + if ok { + prunedManifests = append(prunedManifests, outManifest) + + continue + } + + count, ok := inUse[outManifest.Digest.Encoded()] + if !ok { + prunedManifests = append(prunedManifests, outManifest) + + continue + } + + if count != 1 { + // this manifest is in use in other image indexes + prunedManifests = append(prunedManifests, outManifest) + + continue + } + } + + return prunedManifests, nil +} + // PutImageManifest adds an image manifest to the repository. -func (is *ObjectStorage) PutImageManifest(repo, reference, mediaType string, +func (is *ObjectStorage) PutImageManifest(repo, reference, mediaType string, //nolint: gocyclo body []byte) (string, error, ) { if err := is.InitRepo(repo); err != nil { @@ -407,9 +501,10 @@ func (is *ObjectStorage) PutImageManifest(repo, reference, mediaType string, return "", err } - if mediaType != ispec.MediaTypeImageManifest { + // validate the manifest + if !storage.IsSupportedMediaType(mediaType) { is.log.Debug().Interface("actual", mediaType). - Interface("expected", ispec.MediaTypeImageManifest).Msg("bad manifest media type") + Msg("bad manifest media type") return "", zerr.ErrBadManifest } @@ -489,6 +584,8 @@ func (is *ObjectStorage) PutImageManifest(repo, reference, mediaType string, desc.Annotations = map[string]string{ispec.AnnotationRefName: reference} } + var oldDgst godigest.Digest + for midx, manifest := range index.Manifests { if reference == manifest.Digest.String() { // nothing changed, so don't update @@ -515,9 +612,22 @@ func (is *ObjectStorage) PutImageManifest(repo, reference, mediaType string, Int64("new size", int64(len(body))). Str("old digest", desc.Digest.String()). Str("new digest", mDigest.String()). + Str("old digest", desc.Digest.String()). + Str("new digest", mDigest.String()). Msg("updating existing tag with new manifest contents") + // changing media-type is disallowed! + if manifest.MediaType != mediaType { + err = zerr.ErrBadManifest + is.log.Error().Err(err). + Str("old mediaType", manifest.MediaType). + Str("new mediaType", mediaType).Msg("cannot change media-type") + + return "", err + } + desc = manifest + oldDgst = manifest.Digest desc.Size = int64(len(body)) desc.Digest = mDigest @@ -541,6 +651,30 @@ func (is *ObjectStorage) PutImageManifest(repo, reference, mediaType string, return "", err } + /* additionally, unmarshal an image index and for all manifests in that + index, ensure that they do not have a name or they are not in other + manifest indexes else GC can never clean them */ + if (mediaType == ispec.MediaTypeImageIndex) && (oldDgst != "") { + otherImgIndexes := []ispec.Descriptor{} + + for _, manifest := range index.Manifests { + if manifest.MediaType == ispec.MediaTypeImageIndex { + otherImgIndexes = append(otherImgIndexes, manifest) + } + } + + otherImgIndexes = append(otherImgIndexes, desc) + + dir := path.Join(is.rootDir, repo) + + prunedManifests, err := is.pruneImageManifestsFromIndex(dir, oldDgst, index, otherImgIndexes, is.log) + if err != nil { + return "", err + } + + index.Manifests = prunedManifests + } + // now update "index.json" index.Manifests = append(index.Manifests, desc) dir = path.Join(is.rootDir, repo) @@ -618,12 +752,16 @@ func (is *ObjectStorage) DeleteImageManifest(repo, reference string) error { found := false + isImageIndex := false + var manifest ispec.Descriptor // we are deleting, so keep only those manifests that don't match outIndex := index outIndex.Manifests = []ispec.Descriptor{} + otherImgIndexes := []ispec.Descriptor{} + for _, manifest = range index.Manifests { if isTag { tag, ok := manifest.Annotations[ispec.AnnotationRefName] @@ -634,16 +772,28 @@ func (is *ObjectStorage) DeleteImageManifest(repo, reference string) error { found = true + if manifest.MediaType == ispec.MediaTypeImageIndex { + isImageIndex = true + } + continue } } else if reference == manifest.Digest.String() { is.log.Debug().Str("deleting reference", reference).Msg("") found = true + if manifest.MediaType == ispec.MediaTypeImageIndex { + isImageIndex = true + } + continue } outIndex.Manifests = append(outIndex.Manifests, manifest) + + if manifest.MediaType == ispec.MediaTypeImageIndex { + otherImgIndexes = append(otherImgIndexes, manifest) + } } if !found { @@ -653,6 +803,18 @@ func (is *ObjectStorage) DeleteImageManifest(repo, reference string) error { is.Lock(&lockLatency) defer is.Unlock(&lockLatency) + /* additionally, unmarshal an image index and for all manifests in that + index, ensure that they do not have a name or they are not in other + manifest indexes else GC can never clean them */ + if isImageIndex { + prunedManifests, err := is.pruneImageManifestsFromIndex(dir, dgst, outIndex, otherImgIndexes, is.log) + if err != nil { + return err + } + + outIndex.Manifests = prunedManifests + } + // now update "index.json" dir = path.Join(is.rootDir, repo) file := path.Join(dir, "index.json") diff --git a/pkg/storage/s3/s3_test.go b/pkg/storage/s3/s3_test.go index 1065c367..ac01afe2 100644 --- a/pkg/storage/s3/s3_test.go +++ b/pkg/storage/s3/s3_test.go @@ -1115,6 +1115,457 @@ func TestS3Dedupe(t *testing.T) { }) } +func TestS3ManifestImageIndex(t *testing.T) { + skipIt(t) + + Convey("Test against s3 image store", t, func() { + uuid, err := guuid.NewV4() + if err != nil { + panic(err) + } + + testDir := path.Join("/oci-repo-test", uuid.String()) + + storeDriver, imgStore, _ := createObjectsStore(testDir, t.TempDir(), true) + defer cleanupStorage(storeDriver, testDir) + + // create a blob/layer + upload, err := imgStore.NewBlobUpload("index") + So(err, ShouldBeNil) + So(upload, ShouldNotBeEmpty) + + content := []byte("this is a blob1") + buf := bytes.NewBuffer(content) + buflen := buf.Len() + digest := godigest.FromBytes(content) + So(digest, ShouldNotBeNil) + blob, err := imgStore.PutBlobChunkStreamed("index", upload, buf) + So(err, ShouldBeNil) + So(blob, ShouldEqual, buflen) + bdgst1 := digest + bsize1 := len(content) + + err = imgStore.FinishBlobUpload("index", upload, buf, digest.String()) + So(err, ShouldBeNil) + So(blob, ShouldEqual, buflen) + + // upload image config blob + upload, err = imgStore.NewBlobUpload("index") + So(err, ShouldBeNil) + So(upload, ShouldNotBeEmpty) + + cblob, cdigest := test.GetRandomImageConfig() + buf = bytes.NewBuffer(cblob) + buflen = buf.Len() + blob, err = imgStore.PutBlobChunkStreamed("index", upload, buf) + So(err, ShouldBeNil) + So(blob, ShouldEqual, buflen) + + err = imgStore.FinishBlobUpload("index", upload, buf, cdigest.String()) + So(err, ShouldBeNil) + So(blob, ShouldEqual, buflen) + + // create a manifest + manifest := ispec.Manifest{ + Config: ispec.Descriptor{ + MediaType: ispec.MediaTypeImageConfig, + Digest: cdigest, + Size: int64(len(cblob)), + }, + Layers: []ispec.Descriptor{ + { + MediaType: ispec.MediaTypeImageLayer, + Digest: bdgst1, + Size: int64(bsize1), + }, + }, + } + manifest.SchemaVersion = 2 + content, err = json.Marshal(manifest) + So(err, ShouldBeNil) + digest = godigest.FromBytes(content) + So(digest, ShouldNotBeNil) + m1content := content + _, err = imgStore.PutImageManifest("index", "test:1.0", ispec.MediaTypeImageManifest, content) + So(err, ShouldBeNil) + + // create another manifest but upload using its sha256 reference + + // upload image config blob + upload, err = imgStore.NewBlobUpload("index") + So(err, ShouldBeNil) + So(upload, ShouldNotBeEmpty) + + cblob, cdigest = test.GetRandomImageConfig() + buf = bytes.NewBuffer(cblob) + buflen = buf.Len() + blob, err = imgStore.PutBlobChunkStreamed("index", upload, buf) + So(err, ShouldBeNil) + So(blob, ShouldEqual, buflen) + + err = imgStore.FinishBlobUpload("index", upload, buf, cdigest.String()) + So(err, ShouldBeNil) + So(blob, ShouldEqual, buflen) + + // create a manifest + manifest = ispec.Manifest{ + Config: ispec.Descriptor{ + MediaType: ispec.MediaTypeImageConfig, + Digest: cdigest, + Size: int64(len(cblob)), + }, + Layers: []ispec.Descriptor{ + { + MediaType: ispec.MediaTypeImageLayer, + Digest: bdgst1, + Size: int64(bsize1), + }, + }, + } + manifest.SchemaVersion = 2 + content, err = json.Marshal(manifest) + So(err, ShouldBeNil) + digest = godigest.FromBytes(content) + So(digest, ShouldNotBeNil) + m2dgst := digest + m2size := len(content) + _, err = imgStore.PutImageManifest("index", digest.String(), ispec.MediaTypeImageManifest, content) + So(err, ShouldBeNil) + + Convey("Image index", func() { + // upload image config blob + upload, err = imgStore.NewBlobUpload("index") + So(err, ShouldBeNil) + So(upload, ShouldNotBeEmpty) + + cblob, cdigest = test.GetRandomImageConfig() + buf = bytes.NewBuffer(cblob) + buflen = buf.Len() + blob, err = imgStore.PutBlobChunkStreamed("index", upload, buf) + So(err, ShouldBeNil) + So(blob, ShouldEqual, buflen) + + err = imgStore.FinishBlobUpload("index", upload, buf, cdigest.String()) + So(err, ShouldBeNil) + So(blob, ShouldEqual, buflen) + + // create a manifest + manifest := ispec.Manifest{ + Config: ispec.Descriptor{ + MediaType: ispec.MediaTypeImageConfig, + Digest: cdigest, + Size: int64(len(cblob)), + }, + Layers: []ispec.Descriptor{ + { + MediaType: ispec.MediaTypeImageLayer, + Digest: bdgst1, + Size: int64(bsize1), + }, + }, + } + manifest.SchemaVersion = 2 + content, err = json.Marshal(manifest) + So(err, ShouldBeNil) + digest = godigest.FromBytes(content) + So(digest, ShouldNotBeNil) + _, err = imgStore.PutImageManifest("index", digest.String(), ispec.MediaTypeImageManifest, content) + So(err, ShouldBeNil) + + var index ispec.Index + index.SchemaVersion = 2 + index.Manifests = []ispec.Descriptor{ + { + MediaType: ispec.MediaTypeImageIndex, + Digest: digest, + Size: int64(len(content)), + }, + { + MediaType: ispec.MediaTypeImageIndex, + Digest: m2dgst, + Size: int64(m2size), + }, + } + + content, err = json.Marshal(index) + So(err, ShouldBeNil) + digest = godigest.FromBytes(content) + So(digest, ShouldNotBeNil) + index1dgst := digest + _, err = imgStore.PutImageManifest("index", "test:index1", ispec.MediaTypeImageIndex, content) + So(err, ShouldBeNil) + _, _, _, err = imgStore.GetImageManifest("index", "test:index1") + So(err, ShouldBeNil) + + // upload another image config blob + upload, err = imgStore.NewBlobUpload("index") + So(err, ShouldBeNil) + So(upload, ShouldNotBeEmpty) + + cblob, cdigest = test.GetRandomImageConfig() + buf = bytes.NewBuffer(cblob) + buflen = buf.Len() + blob, err = imgStore.PutBlobChunkStreamed("index", upload, buf) + So(err, ShouldBeNil) + So(blob, ShouldEqual, buflen) + + err = imgStore.FinishBlobUpload("index", upload, buf, cdigest.String()) + So(err, ShouldBeNil) + So(blob, ShouldEqual, buflen) + + // create another manifest + manifest = ispec.Manifest{ + Config: ispec.Descriptor{ + MediaType: ispec.MediaTypeImageConfig, + Digest: cdigest, + Size: int64(len(cblob)), + }, + Layers: []ispec.Descriptor{ + { + MediaType: ispec.MediaTypeImageLayer, + Digest: bdgst1, + Size: int64(bsize1), + }, + }, + } + manifest.SchemaVersion = 2 + content, err = json.Marshal(manifest) + So(err, ShouldBeNil) + digest = godigest.FromBytes(content) + So(digest, ShouldNotBeNil) + m4dgst := digest + m4size := len(content) + _, err = imgStore.PutImageManifest("index", digest.String(), ispec.MediaTypeImageManifest, content) + So(err, ShouldBeNil) + + index.SchemaVersion = 2 + index.Manifests = []ispec.Descriptor{ + { + MediaType: ispec.MediaTypeImageIndex, + Digest: digest, + Size: int64(len(content)), + }, + { + MediaType: ispec.MediaTypeImageIndex, + Digest: m2dgst, + Size: int64(m2size), + }, + } + + content, err = json.Marshal(index) + So(err, ShouldBeNil) + digest = godigest.FromBytes(content) + So(digest, ShouldNotBeNil) + _, err = imgStore.PutImageManifest("index", "test:index2", ispec.MediaTypeImageIndex, content) + So(err, ShouldBeNil) + _, _, _, err = imgStore.GetImageManifest("index", "test:index2") + So(err, ShouldBeNil) + + Convey("List tags", func() { + tags, err := imgStore.GetImageTags("index") + So(err, ShouldBeNil) + So(len(tags), ShouldEqual, 3) + So(tags, ShouldContain, "test:1.0") + So(tags, ShouldContain, "test:index1") + So(tags, ShouldContain, "test:index2") + }) + + Convey("Another index with same manifest", func() { + var index ispec.Index + index.SchemaVersion = 2 + index.Manifests = []ispec.Descriptor{ + { + MediaType: ispec.MediaTypeImageIndex, + Digest: m4dgst, + Size: int64(m4size), + }, + } + + content, err = json.Marshal(index) + So(err, ShouldBeNil) + digest = godigest.FromBytes(content) + So(digest, ShouldNotBeNil) + _, err = imgStore.PutImageManifest("index", "test:index3", ispec.MediaTypeImageIndex, content) + So(err, ShouldBeNil) + _, _, _, err = imgStore.GetImageManifest("index", "test:index3") + So(err, ShouldBeNil) + }) + + Convey("Another index using digest with same manifest", func() { + var index ispec.Index + index.SchemaVersion = 2 + index.Manifests = []ispec.Descriptor{ + { + MediaType: ispec.MediaTypeImageIndex, + Digest: m4dgst, + Size: int64(m4size), + }, + } + + content, err = json.Marshal(index) + So(err, ShouldBeNil) + digest = godigest.FromBytes(content) + So(digest, ShouldNotBeNil) + _, err = imgStore.PutImageManifest("index", digest.String(), ispec.MediaTypeImageIndex, content) + So(err, ShouldBeNil) + _, _, _, err = imgStore.GetImageManifest("index", digest.String()) + So(err, ShouldBeNil) + }) + + Convey("Deleting an image index", func() { + // delete manifest by tag should pass + err := imgStore.DeleteImageManifest("index", "test:index3") + So(err, ShouldNotBeNil) + _, _, _, err = imgStore.GetImageManifest("index", "test:index3") + So(err, ShouldNotBeNil) + + err = imgStore.DeleteImageManifest("index", "test:index1") + So(err, ShouldBeNil) + _, _, _, err = imgStore.GetImageManifest("index", "test:index1") + So(err, ShouldNotBeNil) + + _, _, _, err = imgStore.GetImageManifest("index", "test:index2") + So(err, ShouldBeNil) + }) + + Convey("Deleting an image index by digest", func() { + // delete manifest by tag should pass + err := imgStore.DeleteImageManifest("index", "test:index3") + So(err, ShouldNotBeNil) + _, _, _, err = imgStore.GetImageManifest("index", "test:index3") + So(err, ShouldNotBeNil) + + err = imgStore.DeleteImageManifest("index", index1dgst.String()) + So(err, ShouldBeNil) + _, _, _, err = imgStore.GetImageManifest("index", "test:index1") + So(err, ShouldNotBeNil) + + _, _, _, err = imgStore.GetImageManifest("index", "test:index2") + So(err, ShouldBeNil) + }) + + Convey("Update an index tag with different manifest", func() { + // create a blob/layer + upload, err := imgStore.NewBlobUpload("index") + So(err, ShouldBeNil) + So(upload, ShouldNotBeEmpty) + + content := []byte("this is another blob") + buf := bytes.NewBuffer(content) + buflen := buf.Len() + digest := godigest.FromBytes(content) + So(digest, ShouldNotBeNil) + blob, err := imgStore.PutBlobChunkStreamed("index", upload, buf) + So(err, ShouldBeNil) + So(blob, ShouldEqual, buflen) + + err = imgStore.FinishBlobUpload("index", upload, buf, digest.String()) + So(err, ShouldBeNil) + So(blob, ShouldEqual, buflen) + + // create a manifest with same blob but a different tag + manifest = ispec.Manifest{ + Config: ispec.Descriptor{ + MediaType: ispec.MediaTypeImageConfig, + Digest: cdigest, + Size: int64(len(cblob)), + }, + Layers: []ispec.Descriptor{ + { + MediaType: ispec.MediaTypeImageLayer, + Digest: digest, + Size: int64(len(content)), + }, + }, + } + manifest.SchemaVersion = 2 + content, err = json.Marshal(manifest) + So(err, ShouldBeNil) + digest = godigest.FromBytes(content) + So(digest, ShouldNotBeNil) + _, err = imgStore.PutImageManifest("index", digest.String(), ispec.MediaTypeImageManifest, content) + So(err, ShouldBeNil) + _, _, _, err = imgStore.GetImageManifest("index", digest.String()) + So(err, ShouldBeNil) + + index.SchemaVersion = 2 + index.Manifests = []ispec.Descriptor{ + { + MediaType: ispec.MediaTypeImageIndex, + Digest: digest, + Size: int64(len(content)), + }, + } + + content, err = json.Marshal(index) + So(err, ShouldBeNil) + digest = godigest.FromBytes(content) + So(digest, ShouldNotBeNil) + _, err = imgStore.PutImageManifest("index", "test:index1", ispec.MediaTypeImageIndex, content) + So(err, ShouldBeNil) + _, _, _, err = imgStore.GetImageManifest("index", "test:index1") + So(err, ShouldBeNil) + + err = imgStore.DeleteImageManifest("index", "test:index1") + So(err, ShouldBeNil) + _, _, _, err = imgStore.GetImageManifest("index", "test:index1") + So(err, ShouldNotBeNil) + }) + + Convey("Negative test cases", func() { + Convey("Delete index", func() { + cleanupStorage(storeDriver, path.Join(testDir, "index", "blobs", + index1dgst.Algorithm().String(), index1dgst.Encoded())) + + err = imgStore.DeleteImageManifest("index", index1dgst.String()) + So(err, ShouldNotBeNil) + _, _, _, err = imgStore.GetImageManifest("index", "test:index1") + So(err, ShouldNotBeNil) + }) + + Convey("Corrupt index", func() { + wrtr, err := storeDriver.Writer(context.Background(), + path.Join(testDir, "index", "blobs", + index1dgst.Algorithm().String(), index1dgst.Encoded()), + false) + So(err, ShouldBeNil) + _, err = wrtr.Write([]byte("deadbeef")) + So(err, ShouldBeNil) + wrtr.Close() + err = imgStore.DeleteImageManifest("index", index1dgst.String()) + So(err, ShouldBeNil) + _, _, _, err = imgStore.GetImageManifest("index", "test:index1") + So(err, ShouldNotBeNil) + }) + + Convey("Change media-type", func() { + // previously a manifest, try writing an image index + var index ispec.Index + index.SchemaVersion = 2 + index.Manifests = []ispec.Descriptor{ + { + MediaType: ispec.MediaTypeImageIndex, + Digest: m4dgst, + Size: int64(m4size), + }, + } + + content, err = json.Marshal(index) + So(err, ShouldBeNil) + digest = godigest.FromBytes(content) + So(digest, ShouldNotBeNil) + _, err = imgStore.PutImageManifest("index", "test:1.0", ispec.MediaTypeImageIndex, content) + So(err, ShouldNotBeNil) + + // previously an image index, try writing a manifest + _, err = imgStore.PutImageManifest("index", "test:index1", ispec.MediaTypeImageManifest, m1content) + So(err, ShouldNotBeNil) + }) + }) + }) + }) +} + func TestS3DedupeErr(t *testing.T) { skipIt(t) diff --git a/test/blackbox/pushpull.bats b/test/blackbox/pushpull.bats index 95f1d638..e14a3d3c 100644 --- a/test/blackbox/pushpull.bats +++ b/test/blackbox/pushpull.bats @@ -63,6 +63,43 @@ function teardown_file() { run cat ${BATS_FILE_TMPDIR}/oci/golang/index.json [ "$status" -eq 0 ] [ $(echo "${lines[-1]}" | jq '.manifests[].annotations."org.opencontainers.image.ref.name"') = '"1.18"' ] + run curl -X DELETE http://127.0.0.1:8080/v2/golang/manifests/1.18 + [ "$status" -eq 0 ] +} + +@test "push image index" { + # --multi-arch below pushes an image index (containing many images) instead + # of an image manifest (single image) + run skopeo --insecure-policy copy --format=oci --dest-tls-verify=false --multi-arch=all \ + docker://public.ecr.aws/docker/library/busybox:latest \ + docker://127.0.0.1:8080/busybox:latest + [ "$status" -eq 0 ] + run curl http://127.0.0.1:8080/v2/_catalog + [ "$status" -eq 0 ] + [ $(echo "${lines[-1]}" | jq '.repositories[0]') = '"busybox"' ] + run curl http://127.0.0.1:8080/v2/busybox/tags/list + [ "$status" -eq 0 ] + [ $(echo "${lines[-1]}" | jq '.tags[]') = '"latest"' ] +} + +@test "pull image index" { + local oci_data_dir=${BATS_FILE_TMPDIR}/oci + run skopeo --insecure-policy copy --src-tls-verify=false --multi-arch=all \ + docker://127.0.0.1:8080/busybox:latest \ + oci:${oci_data_dir}/busybox:latest + [ "$status" -eq 0 ] + run cat ${BATS_FILE_TMPDIR}/oci/busybox/index.json + [ "$status" -eq 0 ] + [ $(echo "${lines[-1]}" | jq '.manifests[].annotations."org.opencontainers.image.ref.name"') = '"latest"' ] + run skopeo --insecure-policy --override-arch=arm64 --override-os=linux copy --src-tls-verify=false --multi-arch=all \ + docker://127.0.0.1:8080/busybox:latest \ + oci:${oci_data_dir}/busybox:latest + [ "$status" -eq 0 ] + run cat ${BATS_FILE_TMPDIR}/oci/busybox/index.json + [ "$status" -eq 0 ] + [ $(echo "${lines[-1]}" | jq '.manifests[].annotations."org.opencontainers.image.ref.name"') = '"latest"' ] + run curl -X DELETE http://127.0.0.1:8080/v2/busybox/manifests/latest + [ "$status" -eq 0 ] } @test "push oras artifact" {