diff --git a/pkg/extensions/sync/remote.go b/pkg/extensions/sync/remote.go index 0bb4a5bc..0694e22b 100644 --- a/pkg/extensions/sync/remote.go +++ b/pkg/extensions/sync/remote.go @@ -9,8 +9,10 @@ import ( "github.com/containers/image/v5/docker" dockerReference "github.com/containers/image/v5/docker/reference" + "github.com/containers/image/v5/manifest" "github.com/containers/image/v5/types" "github.com/opencontainers/go-digest" + ispec "github.com/opencontainers/image-spec/specs-go/v1" "zotregistry.io/zot/pkg/api/constants" "zotregistry.io/zot/pkg/common" @@ -109,7 +111,21 @@ func (registry *RemoteRegistry) GetManifestContent(imageReference types.ImageRef return []byte{}, "", "", err } - return manifestBuf, mediaType, digest.FromBytes(manifestBuf), nil + // if mediatype is docker then convert to OCI + switch mediaType { + case manifest.DockerV2Schema2MediaType: + manifestBuf, err = convertDockerManifestToOCI(imageSource, manifestBuf) + if err != nil { + return []byte{}, "", "", err + } + case manifest.DockerV2ListMediaType: + manifestBuf, err = convertDockerIndexToOCI(imageSource, manifestBuf) + if err != nil { + return []byte{}, "", "", err + } + } + + return manifestBuf, ispec.MediaTypeImageManifest, digest.FromBytes(manifestBuf), nil } func (registry *RemoteRegistry) GetRepoTags(repo string) ([]string, error) { diff --git a/pkg/extensions/sync/sync_internal_test.go b/pkg/extensions/sync/sync_internal_test.go index cca2c527..8ef0fafd 100644 --- a/pkg/extensions/sync/sync_internal_test.go +++ b/pkg/extensions/sync/sync_internal_test.go @@ -9,8 +9,10 @@ import ( "encoding/json" "fmt" "os" + "path" "testing" + dockerManifest "github.com/containers/image/v5/manifest" "github.com/containers/image/v5/oci/layout" "github.com/containers/image/v5/types" godigest "github.com/opencontainers/go-digest" @@ -426,3 +428,197 @@ func TestLocalRegistry(t *testing.T) { }) }) } + +func TestConvertDockerToOCI(t *testing.T) { + Convey("test converting docker to oci functions", t, func() { + dir := t.TempDir() + + test.CopyTestFiles("../../../test/data/zot-test", path.Join(dir, "zot-test")) + + imageRef, err := layout.NewReference(path.Join(dir, "zot-test"), "0.0.1") + So(err, ShouldBeNil) + + imageSource, err := imageRef.NewImageSource(context.Background(), &types.SystemContext{}) + So(err, ShouldBeNil) + + defer imageSource.Close() + + Convey("trigger Unmarshal manifest error", func() { + _, err = convertDockerManifestToOCI(imageSource, []byte{}) + So(err, ShouldNotBeNil) + }) + + Convey("trigger getImageConfigContent() error", func() { + manifestBuf, _, err := imageSource.GetManifest(context.Background(), nil) + So(err, ShouldBeNil) + + var manifest ispec.Manifest + + err = json.Unmarshal(manifestBuf, &manifest) + So(err, ShouldBeNil) + + err = os.Chmod(path.Join(dir, "zot-test", "blobs/sha256", manifest.Config.Digest.Encoded()), 0o000) + So(err, ShouldBeNil) + + _, err = convertDockerManifestToOCI(imageSource, manifestBuf) + So(err, ShouldNotBeNil) + }) + + Convey("trigger Unmarshal config error", func() { + manifestBuf, _, err := imageSource.GetManifest(context.Background(), nil) + So(err, ShouldBeNil) + + var manifest ispec.Manifest + + err = json.Unmarshal(manifestBuf, &manifest) + So(err, ShouldBeNil) + + err = os.WriteFile(path.Join(dir, "zot-test", "blobs/sha256", manifest.Config.Digest.Encoded()), + []byte{}, storageConstants.DefaultFilePerms) + So(err, ShouldBeNil) + + _, err = convertDockerManifestToOCI(imageSource, manifestBuf) + So(err, ShouldNotBeNil) + }) + + Convey("trigger convertDockerLayersToOCI error", func() { + manifestBuf, _, err := imageSource.GetManifest(context.Background(), nil) + So(err, ShouldBeNil) + + var manifest ispec.Manifest + + err = json.Unmarshal(manifestBuf, &manifest) + So(err, ShouldBeNil) + + manifestDigest := godigest.FromBytes(manifestBuf) + + manifest.Layers[0].MediaType = "unknown" + + newManifest, err := json.Marshal(manifest) + So(err, ShouldBeNil) + + err = os.WriteFile(path.Join(dir, "zot-test", "blobs/sha256", manifestDigest.Encoded()), + newManifest, storageConstants.DefaultFilePerms) + So(err, ShouldBeNil) + + _, err = convertDockerManifestToOCI(imageSource, manifestBuf) + So(err, ShouldNotBeNil) + }) + + Convey("trigger convertDockerIndexToOCI error", func() { + manifestBuf, _, err := imageSource.GetManifest(context.Background(), nil) + So(err, ShouldBeNil) + + _, err = convertDockerIndexToOCI(imageSource, manifestBuf) + So(err, ShouldNotBeNil) + + // make zot-test image an index image + + var manifest ispec.Manifest + + err = json.Unmarshal(manifestBuf, &manifest) + So(err, ShouldBeNil) + + dockerNewManifest := ispec.Manifest{ + MediaType: dockerManifest.DockerV2Schema2MediaType, + Config: manifest.Config, + Layers: manifest.Layers, + } + + dockerNewManifestBuf, err := json.Marshal(dockerNewManifest) + So(err, ShouldBeNil) + + dockerManifestDigest := godigest.FromBytes(manifestBuf) + + err = os.WriteFile(path.Join(dir, "zot-test", "blobs/sha256", dockerManifestDigest.Encoded()), + dockerNewManifestBuf, storageConstants.DefaultFilePerms) + So(err, ShouldBeNil) + + var index ispec.Index + + index.Manifests = append(index.Manifests, ispec.Descriptor{ + Digest: dockerManifestDigest, + Size: int64(len(dockerNewManifestBuf)), + MediaType: dockerManifest.DockerV2Schema2MediaType, + }) + + index.MediaType = dockerManifest.DockerV2ListMediaType + + dockerIndexBuf, err := json.Marshal(index) + So(err, ShouldBeNil) + + dockerIndexDigest := godigest.FromBytes(dockerIndexBuf) + + err = os.WriteFile(path.Join(dir, "zot-test", "blobs/sha256", dockerIndexDigest.Encoded()), + dockerIndexBuf, storageConstants.DefaultFilePerms) + So(err, ShouldBeNil) + + // write index.json + + var indexJSON ispec.Index + + indexJSONBuf, err := os.ReadFile(path.Join(dir, "zot-test", "index.json")) + So(err, ShouldBeNil) + + err = json.Unmarshal(indexJSONBuf, &indexJSON) + So(err, ShouldBeNil) + + indexJSON.Manifests = append(indexJSON.Manifests, ispec.Descriptor{ + Digest: dockerIndexDigest, + Size: int64(len(dockerIndexBuf)), + MediaType: ispec.MediaTypeImageIndex, + Annotations: map[string]string{ + ispec.AnnotationRefName: "0.0.2", + }, + }) + + indexJSONBuf, err = json.Marshal(indexJSON) + So(err, ShouldBeNil) + + err = os.WriteFile(path.Join(dir, "zot-test", "index.json"), indexJSONBuf, storageConstants.DefaultFilePerms) + So(err, ShouldBeNil) + + imageRef, err := layout.NewReference(path.Join(dir, "zot-test"), "0.0.2") + So(err, ShouldBeNil) + + imageSource, err := imageRef.NewImageSource(context.Background(), &types.SystemContext{}) + So(err, ShouldBeNil) + + _, err = convertDockerIndexToOCI(imageSource, dockerIndexBuf) + So(err, ShouldNotBeNil) + + err = os.Chmod(path.Join(dir, "zot-test", "blobs/sha256", dockerManifestDigest.Encoded()), 0o000) + So(err, ShouldBeNil) + + _, err = convertDockerIndexToOCI(imageSource, dockerIndexBuf) + So(err, ShouldNotBeNil) + }) + }) +} + +func TestConvertDockerLayersToOCI(t *testing.T) { + Convey("test converting docker to oci functions", t, func() { + dockerLayers := []ispec.Descriptor{ + { + MediaType: dockerManifest.DockerV2Schema2ForeignLayerMediaType, + }, + { + MediaType: dockerManifest.DockerV2Schema2ForeignLayerMediaTypeGzip, + }, + { + MediaType: dockerManifest.DockerV2SchemaLayerMediaTypeUncompressed, + }, + { + MediaType: dockerManifest.DockerV2Schema2LayerMediaType, + }, + } + + err := convertDockerLayersToOCI(dockerLayers) + So(err, ShouldBeNil) + + So(dockerLayers[0].MediaType, ShouldEqual, ispec.MediaTypeImageLayerNonDistributable) //nolint: staticcheck + So(dockerLayers[1].MediaType, ShouldEqual, ispec.MediaTypeImageLayerNonDistributableGzip) //nolint: staticcheck + So(dockerLayers[2].MediaType, ShouldEqual, ispec.MediaTypeImageLayer) + So(dockerLayers[3].MediaType, ShouldEqual, ispec.MediaTypeImageLayerGzip) + }) +} diff --git a/pkg/extensions/sync/sync_test.go b/pkg/extensions/sync/sync_test.go index 1e8455d3..9c6b1c2f 100644 --- a/pkg/extensions/sync/sync_test.go +++ b/pkg/extensions/sync/sync_test.go @@ -20,6 +20,7 @@ import ( "testing" "time" + dockerManifest "github.com/containers/image/v5/manifest" notreg "github.com/notaryproject/notation-go/registry" godigest "github.com/opencontainers/go-digest" ispec "github.com/opencontainers/image-spec/specs-go/v1" @@ -1103,6 +1104,333 @@ func TestSyncWithNonDistributableBlob(t *testing.T) { }) } +func TestDockerImagesAreSkipped(t *testing.T) { + Convey("Verify docker images are skipped when they are already synced", t, func() { + updateDuration, _ := time.ParseDuration("30m") + + sctlr, srcBaseURL, srcDir, _, _ := makeUpstreamServer(t, false, false) + + scm := test.NewControllerManager(sctlr) + scm.StartAndWait(sctlr.Config.HTTP.Port) + defer scm.StopServer() + + var tlsVerify bool + + maxRetries := 1 + delay := 1 * time.Second + + indexRepoName := "index" + + syncRegistryConfig := syncconf.RegistryConfig{ + Content: []syncconf.Content{ + { + Prefix: testImage, + }, + { + Prefix: indexRepoName, + }, + }, + URLs: []string{srcBaseURL}, + PollInterval: updateDuration, + TLSVerify: &tlsVerify, + CertDir: "", + MaxRetries: &maxRetries, + OnDemand: true, + RetryDelay: &delay, + } + + defaultVal := true + syncConfig := &syncconf.Config{ + Enable: &defaultVal, + Registries: []syncconf.RegistryConfig{syncRegistryConfig}, + } + + dctlr, destBaseURL, destDir, _ := makeDownstreamServer(t, false, syncConfig) + + Convey("skipping already synced docker image", func() { + // because we can not store images in docker format, modify the test image so that it has docker mediatype + indexContent, err := os.ReadFile(path.Join(srcDir, testImage, "index.json")) + So(err, ShouldBeNil) + So(indexContent, ShouldNotBeNil) + + var index ispec.Index + err = json.Unmarshal(indexContent, &index) + So(err, ShouldBeNil) + + var configBlobDigest godigest.Digest + + for idx, manifestDesc := range index.Manifests { + manifestContent, err := os.ReadFile(path.Join(srcDir, testImage, "blobs/sha256", manifestDesc.Digest.Encoded())) + So(err, ShouldBeNil) + + var manifest ispec.Manifest + + err = json.Unmarshal(manifestContent, &manifest) + So(err, ShouldBeNil) + + configBlobDigest = manifest.Config.Digest + + manifest.MediaType = dockerManifest.DockerV2Schema2MediaType + manifest.Config.MediaType = dockerManifest.DockerV2Schema2ConfigMediaType + index.Manifests[idx].MediaType = dockerManifest.DockerV2Schema2MediaType + + for idx := range manifest.Layers { + manifest.Layers[idx].MediaType = dockerManifest.DockerV2Schema2LayerMediaType + } + + manifestBuf, err := json.Marshal(manifest) + So(err, ShouldBeNil) + + manifestDigest := godigest.FromBytes(manifestBuf) + index.Manifests[idx].Digest = manifestDigest + + // write modified manifest, remove old one + err = os.WriteFile(path.Join(srcDir, testImage, "blobs/sha256", manifestDigest.Encoded()), + manifestBuf, storageConstants.DefaultFilePerms) + So(err, ShouldBeNil) + + err = os.Remove(path.Join(srcDir, testImage, "blobs/sha256", manifestDesc.Digest.Encoded())) + So(err, ShouldBeNil) + } + + indexBuf, err := json.Marshal(index) + So(err, ShouldBeNil) + + err = os.WriteFile(path.Join(srcDir, testImage, "index.json"), indexBuf, storageConstants.DefaultFilePerms) + So(err, ShouldBeNil) + + dcm := test.NewControllerManager(dctlr) + dcm.StartAndWait(dctlr.Config.HTTP.Port) + defer dcm.StopServer() + + resp, err := resty.R().Get(destBaseURL + "/v2/" + testImage + "/manifests/" + testImageTag) + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusOK) + + // now it should be skipped + resp, err = resty.R().Get(destBaseURL + "/v2/" + testImage + "/manifests/" + testImageTag) + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusOK) + + found, err := test.ReadLogFileAndSearchString(dctlr.Config.Log.Output, + "skipping image because it's already synced", 20*time.Second) + if err != nil { + panic(err) + } + + if !found { + data, err := os.ReadFile(dctlr.Config.Log.Output) + So(err, ShouldBeNil) + + t.Logf("downstream log: %s", string(data)) + } + + So(found, ShouldBeTrue) + + Convey("trigger config blob upstream error", func() { + // remove synced image + err := os.RemoveAll(path.Join(destDir, testImage)) + So(err, ShouldBeNil) + + err = os.Chmod(path.Join(srcDir, testImage, "blobs/sha256", configBlobDigest.Encoded()), 0o000) + So(err, ShouldBeNil) + + resp, err = resty.R().Get(destBaseURL + "/v2/" + testImage + "/manifests/" + testImageTag) + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusNotFound) + }) + }) + + Convey("skipping already synced multiarch docker image", func() { + // create an image index on upstream + var index ispec.Index + index.SchemaVersion = 2 + index.MediaType = ispec.MediaTypeImageIndex + + // upload multiple manifests + for i := 0; i < 4; i++ { + config, layers, manifest, err := test.GetImageComponents(1000 + i) + So(err, ShouldBeNil) + + manifestContent, err := json.Marshal(manifest) + So(err, ShouldBeNil) + + manifestDigest := godigest.FromBytes(manifestContent) + + err = test.UploadImage( + test.Image{ + Manifest: manifest, + Config: config, + Layers: layers, + Reference: manifestDigest.String(), + }, + srcBaseURL, + "index") + So(err, ShouldBeNil) + + index.Manifests = append(index.Manifests, ispec.Descriptor{ + Digest: manifestDigest, + MediaType: ispec.MediaTypeImageManifest, + Size: int64(len(manifestContent)), + }) + } + + content, err := json.Marshal(index) + So(err, ShouldBeNil) + + digest := godigest.FromBytes(content) + So(digest, ShouldNotBeNil) + + resp, err := resty.R().SetHeader("Content-Type", ispec.MediaTypeImageIndex). + SetBody(content).Put(srcBaseURL + "/v2/index/manifests/latest") + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusCreated) + + resp, err = resty.R().SetHeader("Content-Type", ispec.MediaTypeImageIndex). + Get(srcBaseURL + "/v2/index/manifests/latest") + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusOK) + So(resp.Body(), ShouldNotBeEmpty) + So(resp.Header().Get("Content-Type"), ShouldNotBeEmpty) + + // 'convert' oci multi arch image to docker multi arch + + indexContent, err := os.ReadFile(path.Join(srcDir, indexRepoName, "index.json")) + So(err, ShouldBeNil) + So(indexContent, ShouldNotBeNil) + + var newIndex ispec.Index + err = json.Unmarshal(indexContent, &newIndex) + So(err, ShouldBeNil) + + /* first find multiarch manifest in index.json + so that we can update both multiarch manifest and index.json at the same time*/ + var indexManifest ispec.Index + indexManifest.Manifests = make([]ispec.Descriptor, 4) + + var indexManifestIdx int + for idx, manifestDesc := range newIndex.Manifests { + if manifestDesc.MediaType == ispec.MediaTypeImageIndex { + indexManifestContent, err := os.ReadFile(path.Join(srcDir, indexRepoName, "blobs/sha256", + manifestDesc.Digest.Encoded())) + So(err, ShouldBeNil) + + err = json.Unmarshal(indexManifestContent, &indexManifest) + So(err, ShouldBeNil) + indexManifestIdx = idx + } + } + + var configBlobDigest godigest.Digest + var indexManifestContent []byte + for idx, manifestDesc := range newIndex.Manifests { + if manifestDesc.MediaType == ispec.MediaTypeImageManifest { + manifestContent, err := os.ReadFile(path.Join(srcDir, indexRepoName, "blobs/sha256", + manifestDesc.Digest.Encoded())) + So(err, ShouldBeNil) + + var manifest ispec.Manifest + + err = json.Unmarshal(manifestContent, &manifest) + So(err, ShouldBeNil) + + configBlobDigest = manifest.Config.Digest + + manifest.MediaType = dockerManifest.DockerV2Schema2MediaType + manifest.Config.MediaType = dockerManifest.DockerV2Schema2ConfigMediaType + newIndex.Manifests[idx].MediaType = dockerManifest.DockerV2Schema2MediaType + indexManifest.Manifests[idx].MediaType = dockerManifest.DockerV2Schema2MediaType + + for idx := range manifest.Layers { + manifest.Layers[idx].MediaType = dockerManifest.DockerV2Schema2LayerMediaType + } + + manifestBuf, err := json.Marshal(manifest) + So(err, ShouldBeNil) + + manifestDigest := godigest.FromBytes(manifestBuf) + newIndex.Manifests[idx].Digest = manifestDigest + indexManifest.Manifests[idx].Digest = manifestDigest + + // write modified manifest, remove old one + err = os.WriteFile(path.Join(srcDir, indexRepoName, "blobs/sha256", manifestDigest.Encoded()), + manifestBuf, storageConstants.DefaultFilePerms) + So(err, ShouldBeNil) + + err = os.Remove(path.Join(srcDir, indexRepoName, "blobs/sha256", manifestDesc.Digest.Encoded())) + So(err, ShouldBeNil) + } + + indexManifest.MediaType = dockerManifest.DockerV2ListMediaType + // write converted multi arch manifest + indexManifestContent, err = json.Marshal(indexManifest) + So(err, ShouldBeNil) + + err = os.WriteFile(path.Join(srcDir, indexRepoName, "blobs/sha256", + godigest.FromBytes(indexManifestContent).Encoded()), indexManifestContent, storageConstants.DefaultFilePerms) + So(err, ShouldBeNil) + } + + newIndex.Manifests[indexManifestIdx].MediaType = dockerManifest.DockerV2ListMediaType + newIndex.Manifests[indexManifestIdx].Digest = godigest.FromBytes(indexManifestContent) + + indexBuf, err := json.Marshal(newIndex) + So(err, ShouldBeNil) + + err = os.WriteFile(path.Join(srcDir, indexRepoName, "index.json"), indexBuf, storageConstants.DefaultFilePerms) + So(err, ShouldBeNil) + + dcm := test.NewControllerManager(dctlr) + dcm.StartAndWait(dctlr.Config.HTTP.Port) + defer dcm.StopServer() + + // sync + resp, err = resty.R().SetHeader("Content-Type", ispec.MediaTypeImageIndex). + Get(destBaseURL + "/v2/" + indexRepoName + "/manifests/" + "latest") + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusOK) + So(resp.Body(), ShouldNotBeEmpty) + So(resp.Header().Get("Content-Type"), ShouldNotBeEmpty) + + // sync again, should skip + resp, err = resty.R().SetHeader("Content-Type", ispec.MediaTypeImageIndex). + Get(destBaseURL + "/v2/" + indexRepoName + "/manifests/" + "latest") + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusOK) + So(resp.Body(), ShouldNotBeEmpty) + So(resp.Header().Get("Content-Type"), ShouldNotBeEmpty) + + found, err := test.ReadLogFileAndSearchString(dctlr.Config.Log.Output, + "skipping image because it's already synced", 20*time.Second) + if err != nil { + panic(err) + } + + if !found { + data, err := os.ReadFile(dctlr.Config.Log.Output) + So(err, ShouldBeNil) + + t.Logf("downstream log: %s", string(data)) + } + + So(found, ShouldBeTrue) + + Convey("trigger config blob upstream error", func() { + // remove synced image + err := os.RemoveAll(path.Join(destDir, indexRepoName)) + So(err, ShouldBeNil) + + err = os.Chmod(path.Join(srcDir, indexRepoName, "blobs/sha256", configBlobDigest.Encoded()), 0o000) + So(err, ShouldBeNil) + + resp, err = resty.R().Get(destBaseURL + "/v2/" + indexRepoName + "/manifests/" + "latest") + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusNotFound) + }) + }) + }) +} + func TestPeriodically(t *testing.T) { Convey("Verify sync feature", t, func() { updateDuration, _ := time.ParseDuration("30m") diff --git a/pkg/extensions/sync/utils.go b/pkg/extensions/sync/utils.go index 04e69d4c..5eb21006 100644 --- a/pkg/extensions/sync/utils.go +++ b/pkg/extensions/sync/utils.go @@ -4,6 +4,7 @@ package sync import ( + "bytes" "context" "encoding/json" "fmt" @@ -14,6 +15,7 @@ import ( "github.com/containers/image/v5/copy" "github.com/containers/image/v5/docker" "github.com/containers/image/v5/manifest" + "github.com/containers/image/v5/pkg/blobinfocache/none" "github.com/containers/image/v5/signature" "github.com/containers/image/v5/types" "github.com/docker/distribution/reference" @@ -170,3 +172,131 @@ func isSupportedMediaType(mediaType string) bool { return false } + +// given an imageSource and a docker manifest, convert it to OCI. +func convertDockerManifestToOCI(imageSource types.ImageSource, dockerManifestBuf []byte) ([]byte, error) { + var ociManifest ispec.Manifest + + // unmarshal docker manifest into OCI manifest + err := json.Unmarshal(dockerManifestBuf, &ociManifest) + if err != nil { + return []byte{}, err + } + + configContent, err := getImageConfigContent(imageSource, ociManifest.Config.Digest) + if err != nil { + return []byte{}, err + } + + // marshal config blob into OCI config, will remove keys specific to docker + var ociConfig ispec.Image + + err = json.Unmarshal(configContent, &ociConfig) + if err != nil { + return []byte{}, err + } + + ociConfigContent, err := json.Marshal(ociConfig) + if err != nil { + return []byte{}, err + } + + // convert layers + err = convertDockerLayersToOCI(ociManifest.Layers) + if err != nil { + return []byte{}, err + } + + // convert config and manifest mediatype + ociManifest.Config.Size = int64(len(ociConfigContent)) + ociManifest.Config.Digest = digest.FromBytes(ociConfigContent) + ociManifest.Config.MediaType = ispec.MediaTypeImageConfig + ociManifest.MediaType = ispec.MediaTypeImageManifest + + return json.Marshal(ociManifest) +} + +// convert docker layers mediatypes to OCI mediatypes. +func convertDockerLayersToOCI(dockerLayers []ispec.Descriptor) error { + for idx, layer := range dockerLayers { + switch layer.MediaType { + case manifest.DockerV2Schema2ForeignLayerMediaType: + dockerLayers[idx].MediaType = ispec.MediaTypeImageLayerNonDistributable //nolint: staticcheck + case manifest.DockerV2Schema2ForeignLayerMediaTypeGzip: + dockerLayers[idx].MediaType = ispec.MediaTypeImageLayerNonDistributableGzip //nolint: staticcheck + case manifest.DockerV2SchemaLayerMediaTypeUncompressed: + dockerLayers[idx].MediaType = ispec.MediaTypeImageLayer + case manifest.DockerV2Schema2LayerMediaType: + dockerLayers[idx].MediaType = ispec.MediaTypeImageLayerGzip + default: + return zerr.ErrMediaTypeNotSupported + } + } + + return nil +} + +// given an imageSource and a docker index manifest, convert it to OCI. +func convertDockerIndexToOCI(imageSource types.ImageSource, dockerManifestBuf []byte) ([]byte, error) { + // get docker index + originalIndex, err := manifest.ListFromBlob(dockerManifestBuf, manifest.DockerV2ListMediaType) + if err != nil { + return []byte{}, err + } + + // get manifests digests + manifestsDigests := originalIndex.Instances() + + manifestsUpdates := make([]manifest.ListUpdate, 0, len(manifestsDigests)) + + // convert each manifests in index from docker to OCI + for _, manifestDigest := range manifestsDigests { + digestCopy := manifestDigest + + indexManifestBuf, _, err := imageSource.GetManifest(context.Background(), &digestCopy) + if err != nil { + return []byte{}, err + } + + convertedIndexManifest, err := convertDockerManifestToOCI(imageSource, indexManifestBuf) + if err != nil { + return []byte{}, err + } + + manifestsUpdates = append(manifestsUpdates, manifest.ListUpdate{ + Digest: digest.FromBytes(convertedIndexManifest), + Size: int64(len(convertedIndexManifest)), + MediaType: ispec.MediaTypeImageManifest, + }) + } + + // update all manifests in index + if err := originalIndex.UpdateInstances(manifestsUpdates); err != nil { + return []byte{}, err + } + + // convert index to OCI + convertedList, err := originalIndex.ConvertToMIMEType(ispec.MediaTypeImageIndex) + if err != nil { + return []byte{}, err + } + + return convertedList.Serialize() +} + +// given an image source and a config blob digest, get blob config content. +func getImageConfigContent(imageSource types.ImageSource, configDigest digest.Digest, +) ([]byte, error) { + configBlob, _, err := imageSource.GetBlob(context.Background(), types.BlobInfo{ + Digest: configDigest, + }, none.NoCache) + if err != nil { + return nil, err + } + + configBuf := new(bytes.Buffer) + + _, err = configBuf.ReadFrom(configBlob) + + return configBuf.Bytes(), err +} diff --git a/test/blackbox/sync_docker.bats b/test/blackbox/sync_docker.bats index 47bda01e..c14327c3 100644 --- a/test/blackbox/sync_docker.bats +++ b/test/blackbox/sync_docker.bats @@ -129,6 +129,15 @@ function teardown_file() { run curl http://127.0.0.1:8090/v2/registry/tags/list [ "$status" -eq 0 ] [ $(echo "${lines[-1]}" | jq '.tags[]') = '"latest"' ] + + # make sure image is skipped when synced again + run skopeo --insecure-policy copy --multi-arch=all --src-tls-verify=false \ + docker://127.0.0.1:8090/registry \ + oci:${TEST_DATA_DIR} + [ "$status" -eq 0 ] + + run $("cat /tmp/blackbox.log | grep -q registry:latest.*.skipping image because it's already synced") + [ "$status" -eq 0 ] } @test "sync docker image on demand" { @@ -143,6 +152,15 @@ function teardown_file() { run curl http://127.0.0.1:8090/v2/archlinux/tags/list [ "$status" -eq 0 ] [ $(echo "${lines[-1]}" | jq '.tags[]') = '"latest"' ] + + # make sure image is skipped when synced again + run skopeo --insecure-policy copy --src-tls-verify=false \ + docker://127.0.0.1:8090/archlinux \ + oci:${TEST_DATA_DIR} + [ "$status" -eq 0 ] + + run $("cat /tmp/blackbox.log | grep -q archlinux:latest.*.skipping image because it's already synced") + [ "$status" -eq 0 ] } @test "sync k8s image list on demand" {