mirror of
https://github.com/project-zot/zot.git
synced 2026-06-16 04:17:55 +08:00
fix(sync): fixed skipping docker images when they already synced (#1521)
before syncing an image we first check if it's already present in our storage to do that we get the manifest from remote and compare it with the local one but in the case of syncing docker images, because the conversion to OCI format is done while syncing, we get a docker manifest before conversion, so sync detects that local manifest and remote one are different, so it starts syncing again. to overcome this, convert remote docker manifests to OCI manifests and then compare. Signed-off-by: Petu Eusebiu <peusebiu@cisco.com>
This commit is contained in:
peusebiu
@@ -9,8 +9,10 @@ import (
|
||||
|
||||
"github.com/containers/image/v5/docker"
|
||||
dockerReference "github.com/containers/image/v5/docker/reference"
|
||||
"github.com/containers/image/v5/manifest"
|
||||
"github.com/containers/image/v5/types"
|
||||
"github.com/opencontainers/go-digest"
|
||||
ispec "github.com/opencontainers/image-spec/specs-go/v1"
|
||||
|
||||
"zotregistry.io/zot/pkg/api/constants"
|
||||
"zotregistry.io/zot/pkg/common"
|
||||
@@ -109,7 +111,21 @@ func (registry *RemoteRegistry) GetManifestContent(imageReference types.ImageRef
|
||||
return []byte{}, "", "", err
|
||||
}
|
||||
|
||||
return manifestBuf, mediaType, digest.FromBytes(manifestBuf), nil
|
||||
// if mediatype is docker then convert to OCI
|
||||
switch mediaType {
|
||||
case manifest.DockerV2Schema2MediaType:
|
||||
manifestBuf, err = convertDockerManifestToOCI(imageSource, manifestBuf)
|
||||
if err != nil {
|
||||
return []byte{}, "", "", err
|
||||
}
|
||||
case manifest.DockerV2ListMediaType:
|
||||
manifestBuf, err = convertDockerIndexToOCI(imageSource, manifestBuf)
|
||||
if err != nil {
|
||||
return []byte{}, "", "", err
|
||||
}
|
||||
}
|
||||
|
||||
return manifestBuf, ispec.MediaTypeImageManifest, digest.FromBytes(manifestBuf), nil
|
||||
}
|
||||
|
||||
func (registry *RemoteRegistry) GetRepoTags(repo string) ([]string, error) {
|
||||
|
||||
@@ -9,8 +9,10 @@ import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"path"
|
||||
"testing"
|
||||
|
||||
dockerManifest "github.com/containers/image/v5/manifest"
|
||||
"github.com/containers/image/v5/oci/layout"
|
||||
"github.com/containers/image/v5/types"
|
||||
godigest "github.com/opencontainers/go-digest"
|
||||
@@ -426,3 +428,197 @@ func TestLocalRegistry(t *testing.T) {
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
func TestConvertDockerToOCI(t *testing.T) {
|
||||
Convey("test converting docker to oci functions", t, func() {
|
||||
dir := t.TempDir()
|
||||
|
||||
test.CopyTestFiles("../../../test/data/zot-test", path.Join(dir, "zot-test"))
|
||||
|
||||
imageRef, err := layout.NewReference(path.Join(dir, "zot-test"), "0.0.1")
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
imageSource, err := imageRef.NewImageSource(context.Background(), &types.SystemContext{})
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
defer imageSource.Close()
|
||||
|
||||
Convey("trigger Unmarshal manifest error", func() {
|
||||
_, err = convertDockerManifestToOCI(imageSource, []byte{})
|
||||
So(err, ShouldNotBeNil)
|
||||
})
|
||||
|
||||
Convey("trigger getImageConfigContent() error", func() {
|
||||
manifestBuf, _, err := imageSource.GetManifest(context.Background(), nil)
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
var manifest ispec.Manifest
|
||||
|
||||
err = json.Unmarshal(manifestBuf, &manifest)
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
err = os.Chmod(path.Join(dir, "zot-test", "blobs/sha256", manifest.Config.Digest.Encoded()), 0o000)
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
_, err = convertDockerManifestToOCI(imageSource, manifestBuf)
|
||||
So(err, ShouldNotBeNil)
|
||||
})
|
||||
|
||||
Convey("trigger Unmarshal config error", func() {
|
||||
manifestBuf, _, err := imageSource.GetManifest(context.Background(), nil)
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
var manifest ispec.Manifest
|
||||
|
||||
err = json.Unmarshal(manifestBuf, &manifest)
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
err = os.WriteFile(path.Join(dir, "zot-test", "blobs/sha256", manifest.Config.Digest.Encoded()),
|
||||
[]byte{}, storageConstants.DefaultFilePerms)
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
_, err = convertDockerManifestToOCI(imageSource, manifestBuf)
|
||||
So(err, ShouldNotBeNil)
|
||||
})
|
||||
|
||||
Convey("trigger convertDockerLayersToOCI error", func() {
|
||||
manifestBuf, _, err := imageSource.GetManifest(context.Background(), nil)
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
var manifest ispec.Manifest
|
||||
|
||||
err = json.Unmarshal(manifestBuf, &manifest)
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
manifestDigest := godigest.FromBytes(manifestBuf)
|
||||
|
||||
manifest.Layers[0].MediaType = "unknown"
|
||||
|
||||
newManifest, err := json.Marshal(manifest)
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
err = os.WriteFile(path.Join(dir, "zot-test", "blobs/sha256", manifestDigest.Encoded()),
|
||||
newManifest, storageConstants.DefaultFilePerms)
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
_, err = convertDockerManifestToOCI(imageSource, manifestBuf)
|
||||
So(err, ShouldNotBeNil)
|
||||
})
|
||||
|
||||
Convey("trigger convertDockerIndexToOCI error", func() {
|
||||
manifestBuf, _, err := imageSource.GetManifest(context.Background(), nil)
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
_, err = convertDockerIndexToOCI(imageSource, manifestBuf)
|
||||
So(err, ShouldNotBeNil)
|
||||
|
||||
// make zot-test image an index image
|
||||
|
||||
var manifest ispec.Manifest
|
||||
|
||||
err = json.Unmarshal(manifestBuf, &manifest)
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
dockerNewManifest := ispec.Manifest{
|
||||
MediaType: dockerManifest.DockerV2Schema2MediaType,
|
||||
Config: manifest.Config,
|
||||
Layers: manifest.Layers,
|
||||
}
|
||||
|
||||
dockerNewManifestBuf, err := json.Marshal(dockerNewManifest)
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
dockerManifestDigest := godigest.FromBytes(manifestBuf)
|
||||
|
||||
err = os.WriteFile(path.Join(dir, "zot-test", "blobs/sha256", dockerManifestDigest.Encoded()),
|
||||
dockerNewManifestBuf, storageConstants.DefaultFilePerms)
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
var index ispec.Index
|
||||
|
||||
index.Manifests = append(index.Manifests, ispec.Descriptor{
|
||||
Digest: dockerManifestDigest,
|
||||
Size: int64(len(dockerNewManifestBuf)),
|
||||
MediaType: dockerManifest.DockerV2Schema2MediaType,
|
||||
})
|
||||
|
||||
index.MediaType = dockerManifest.DockerV2ListMediaType
|
||||
|
||||
dockerIndexBuf, err := json.Marshal(index)
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
dockerIndexDigest := godigest.FromBytes(dockerIndexBuf)
|
||||
|
||||
err = os.WriteFile(path.Join(dir, "zot-test", "blobs/sha256", dockerIndexDigest.Encoded()),
|
||||
dockerIndexBuf, storageConstants.DefaultFilePerms)
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
// write index.json
|
||||
|
||||
var indexJSON ispec.Index
|
||||
|
||||
indexJSONBuf, err := os.ReadFile(path.Join(dir, "zot-test", "index.json"))
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
err = json.Unmarshal(indexJSONBuf, &indexJSON)
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
indexJSON.Manifests = append(indexJSON.Manifests, ispec.Descriptor{
|
||||
Digest: dockerIndexDigest,
|
||||
Size: int64(len(dockerIndexBuf)),
|
||||
MediaType: ispec.MediaTypeImageIndex,
|
||||
Annotations: map[string]string{
|
||||
ispec.AnnotationRefName: "0.0.2",
|
||||
},
|
||||
})
|
||||
|
||||
indexJSONBuf, err = json.Marshal(indexJSON)
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
err = os.WriteFile(path.Join(dir, "zot-test", "index.json"), indexJSONBuf, storageConstants.DefaultFilePerms)
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
imageRef, err := layout.NewReference(path.Join(dir, "zot-test"), "0.0.2")
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
imageSource, err := imageRef.NewImageSource(context.Background(), &types.SystemContext{})
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
_, err = convertDockerIndexToOCI(imageSource, dockerIndexBuf)
|
||||
So(err, ShouldNotBeNil)
|
||||
|
||||
err = os.Chmod(path.Join(dir, "zot-test", "blobs/sha256", dockerManifestDigest.Encoded()), 0o000)
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
_, err = convertDockerIndexToOCI(imageSource, dockerIndexBuf)
|
||||
So(err, ShouldNotBeNil)
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
func TestConvertDockerLayersToOCI(t *testing.T) {
|
||||
Convey("test converting docker to oci functions", t, func() {
|
||||
dockerLayers := []ispec.Descriptor{
|
||||
{
|
||||
MediaType: dockerManifest.DockerV2Schema2ForeignLayerMediaType,
|
||||
},
|
||||
{
|
||||
MediaType: dockerManifest.DockerV2Schema2ForeignLayerMediaTypeGzip,
|
||||
},
|
||||
{
|
||||
MediaType: dockerManifest.DockerV2SchemaLayerMediaTypeUncompressed,
|
||||
},
|
||||
{
|
||||
MediaType: dockerManifest.DockerV2Schema2LayerMediaType,
|
||||
},
|
||||
}
|
||||
|
||||
err := convertDockerLayersToOCI(dockerLayers)
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
So(dockerLayers[0].MediaType, ShouldEqual, ispec.MediaTypeImageLayerNonDistributable) //nolint: staticcheck
|
||||
So(dockerLayers[1].MediaType, ShouldEqual, ispec.MediaTypeImageLayerNonDistributableGzip) //nolint: staticcheck
|
||||
So(dockerLayers[2].MediaType, ShouldEqual, ispec.MediaTypeImageLayer)
|
||||
So(dockerLayers[3].MediaType, ShouldEqual, ispec.MediaTypeImageLayerGzip)
|
||||
})
|
||||
}
|
||||
|
||||
@@ -20,6 +20,7 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
dockerManifest "github.com/containers/image/v5/manifest"
|
||||
notreg "github.com/notaryproject/notation-go/registry"
|
||||
godigest "github.com/opencontainers/go-digest"
|
||||
ispec "github.com/opencontainers/image-spec/specs-go/v1"
|
||||
@@ -1103,6 +1104,333 @@ func TestSyncWithNonDistributableBlob(t *testing.T) {
|
||||
})
|
||||
}
|
||||
|
||||
func TestDockerImagesAreSkipped(t *testing.T) {
|
||||
Convey("Verify docker images are skipped when they are already synced", t, func() {
|
||||
updateDuration, _ := time.ParseDuration("30m")
|
||||
|
||||
sctlr, srcBaseURL, srcDir, _, _ := makeUpstreamServer(t, false, false)
|
||||
|
||||
scm := test.NewControllerManager(sctlr)
|
||||
scm.StartAndWait(sctlr.Config.HTTP.Port)
|
||||
defer scm.StopServer()
|
||||
|
||||
var tlsVerify bool
|
||||
|
||||
maxRetries := 1
|
||||
delay := 1 * time.Second
|
||||
|
||||
indexRepoName := "index"
|
||||
|
||||
syncRegistryConfig := syncconf.RegistryConfig{
|
||||
Content: []syncconf.Content{
|
||||
{
|
||||
Prefix: testImage,
|
||||
},
|
||||
{
|
||||
Prefix: indexRepoName,
|
||||
},
|
||||
},
|
||||
URLs: []string{srcBaseURL},
|
||||
PollInterval: updateDuration,
|
||||
TLSVerify: &tlsVerify,
|
||||
CertDir: "",
|
||||
MaxRetries: &maxRetries,
|
||||
OnDemand: true,
|
||||
RetryDelay: &delay,
|
||||
}
|
||||
|
||||
defaultVal := true
|
||||
syncConfig := &syncconf.Config{
|
||||
Enable: &defaultVal,
|
||||
Registries: []syncconf.RegistryConfig{syncRegistryConfig},
|
||||
}
|
||||
|
||||
dctlr, destBaseURL, destDir, _ := makeDownstreamServer(t, false, syncConfig)
|
||||
|
||||
Convey("skipping already synced docker image", func() {
|
||||
// because we can not store images in docker format, modify the test image so that it has docker mediatype
|
||||
indexContent, err := os.ReadFile(path.Join(srcDir, testImage, "index.json"))
|
||||
So(err, ShouldBeNil)
|
||||
So(indexContent, ShouldNotBeNil)
|
||||
|
||||
var index ispec.Index
|
||||
err = json.Unmarshal(indexContent, &index)
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
var configBlobDigest godigest.Digest
|
||||
|
||||
for idx, manifestDesc := range index.Manifests {
|
||||
manifestContent, err := os.ReadFile(path.Join(srcDir, testImage, "blobs/sha256", manifestDesc.Digest.Encoded()))
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
var manifest ispec.Manifest
|
||||
|
||||
err = json.Unmarshal(manifestContent, &manifest)
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
configBlobDigest = manifest.Config.Digest
|
||||
|
||||
manifest.MediaType = dockerManifest.DockerV2Schema2MediaType
|
||||
manifest.Config.MediaType = dockerManifest.DockerV2Schema2ConfigMediaType
|
||||
index.Manifests[idx].MediaType = dockerManifest.DockerV2Schema2MediaType
|
||||
|
||||
for idx := range manifest.Layers {
|
||||
manifest.Layers[idx].MediaType = dockerManifest.DockerV2Schema2LayerMediaType
|
||||
}
|
||||
|
||||
manifestBuf, err := json.Marshal(manifest)
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
manifestDigest := godigest.FromBytes(manifestBuf)
|
||||
index.Manifests[idx].Digest = manifestDigest
|
||||
|
||||
// write modified manifest, remove old one
|
||||
err = os.WriteFile(path.Join(srcDir, testImage, "blobs/sha256", manifestDigest.Encoded()),
|
||||
manifestBuf, storageConstants.DefaultFilePerms)
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
err = os.Remove(path.Join(srcDir, testImage, "blobs/sha256", manifestDesc.Digest.Encoded()))
|
||||
So(err, ShouldBeNil)
|
||||
}
|
||||
|
||||
indexBuf, err := json.Marshal(index)
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
err = os.WriteFile(path.Join(srcDir, testImage, "index.json"), indexBuf, storageConstants.DefaultFilePerms)
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
dcm := test.NewControllerManager(dctlr)
|
||||
dcm.StartAndWait(dctlr.Config.HTTP.Port)
|
||||
defer dcm.StopServer()
|
||||
|
||||
resp, err := resty.R().Get(destBaseURL + "/v2/" + testImage + "/manifests/" + testImageTag)
|
||||
So(err, ShouldBeNil)
|
||||
So(resp.StatusCode(), ShouldEqual, http.StatusOK)
|
||||
|
||||
// now it should be skipped
|
||||
resp, err = resty.R().Get(destBaseURL + "/v2/" + testImage + "/manifests/" + testImageTag)
|
||||
So(err, ShouldBeNil)
|
||||
So(resp.StatusCode(), ShouldEqual, http.StatusOK)
|
||||
|
||||
found, err := test.ReadLogFileAndSearchString(dctlr.Config.Log.Output,
|
||||
"skipping image because it's already synced", 20*time.Second)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
if !found {
|
||||
data, err := os.ReadFile(dctlr.Config.Log.Output)
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
t.Logf("downstream log: %s", string(data))
|
||||
}
|
||||
|
||||
So(found, ShouldBeTrue)
|
||||
|
||||
Convey("trigger config blob upstream error", func() {
|
||||
// remove synced image
|
||||
err := os.RemoveAll(path.Join(destDir, testImage))
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
err = os.Chmod(path.Join(srcDir, testImage, "blobs/sha256", configBlobDigest.Encoded()), 0o000)
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
resp, err = resty.R().Get(destBaseURL + "/v2/" + testImage + "/manifests/" + testImageTag)
|
||||
So(err, ShouldBeNil)
|
||||
So(resp.StatusCode(), ShouldEqual, http.StatusNotFound)
|
||||
})
|
||||
})
|
||||
|
||||
Convey("skipping already synced multiarch docker image", func() {
|
||||
// create an image index on upstream
|
||||
var index ispec.Index
|
||||
index.SchemaVersion = 2
|
||||
index.MediaType = ispec.MediaTypeImageIndex
|
||||
|
||||
// upload multiple manifests
|
||||
for i := 0; i < 4; i++ {
|
||||
config, layers, manifest, err := test.GetImageComponents(1000 + i)
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
manifestContent, err := json.Marshal(manifest)
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
manifestDigest := godigest.FromBytes(manifestContent)
|
||||
|
||||
err = test.UploadImage(
|
||||
test.Image{
|
||||
Manifest: manifest,
|
||||
Config: config,
|
||||
Layers: layers,
|
||||
Reference: manifestDigest.String(),
|
||||
},
|
||||
srcBaseURL,
|
||||
"index")
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
index.Manifests = append(index.Manifests, ispec.Descriptor{
|
||||
Digest: manifestDigest,
|
||||
MediaType: ispec.MediaTypeImageManifest,
|
||||
Size: int64(len(manifestContent)),
|
||||
})
|
||||
}
|
||||
|
||||
content, err := json.Marshal(index)
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
digest := godigest.FromBytes(content)
|
||||
So(digest, ShouldNotBeNil)
|
||||
|
||||
resp, err := resty.R().SetHeader("Content-Type", ispec.MediaTypeImageIndex).
|
||||
SetBody(content).Put(srcBaseURL + "/v2/index/manifests/latest")
|
||||
So(err, ShouldBeNil)
|
||||
So(resp.StatusCode(), ShouldEqual, http.StatusCreated)
|
||||
|
||||
resp, err = resty.R().SetHeader("Content-Type", ispec.MediaTypeImageIndex).
|
||||
Get(srcBaseURL + "/v2/index/manifests/latest")
|
||||
So(err, ShouldBeNil)
|
||||
So(resp.StatusCode(), ShouldEqual, http.StatusOK)
|
||||
So(resp.Body(), ShouldNotBeEmpty)
|
||||
So(resp.Header().Get("Content-Type"), ShouldNotBeEmpty)
|
||||
|
||||
// 'convert' oci multi arch image to docker multi arch
|
||||
|
||||
indexContent, err := os.ReadFile(path.Join(srcDir, indexRepoName, "index.json"))
|
||||
So(err, ShouldBeNil)
|
||||
So(indexContent, ShouldNotBeNil)
|
||||
|
||||
var newIndex ispec.Index
|
||||
err = json.Unmarshal(indexContent, &newIndex)
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
/* first find multiarch manifest in index.json
|
||||
so that we can update both multiarch manifest and index.json at the same time*/
|
||||
var indexManifest ispec.Index
|
||||
indexManifest.Manifests = make([]ispec.Descriptor, 4)
|
||||
|
||||
var indexManifestIdx int
|
||||
for idx, manifestDesc := range newIndex.Manifests {
|
||||
if manifestDesc.MediaType == ispec.MediaTypeImageIndex {
|
||||
indexManifestContent, err := os.ReadFile(path.Join(srcDir, indexRepoName, "blobs/sha256",
|
||||
manifestDesc.Digest.Encoded()))
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
err = json.Unmarshal(indexManifestContent, &indexManifest)
|
||||
So(err, ShouldBeNil)
|
||||
indexManifestIdx = idx
|
||||
}
|
||||
}
|
||||
|
||||
var configBlobDigest godigest.Digest
|
||||
var indexManifestContent []byte
|
||||
for idx, manifestDesc := range newIndex.Manifests {
|
||||
if manifestDesc.MediaType == ispec.MediaTypeImageManifest {
|
||||
manifestContent, err := os.ReadFile(path.Join(srcDir, indexRepoName, "blobs/sha256",
|
||||
manifestDesc.Digest.Encoded()))
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
var manifest ispec.Manifest
|
||||
|
||||
err = json.Unmarshal(manifestContent, &manifest)
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
configBlobDigest = manifest.Config.Digest
|
||||
|
||||
manifest.MediaType = dockerManifest.DockerV2Schema2MediaType
|
||||
manifest.Config.MediaType = dockerManifest.DockerV2Schema2ConfigMediaType
|
||||
newIndex.Manifests[idx].MediaType = dockerManifest.DockerV2Schema2MediaType
|
||||
indexManifest.Manifests[idx].MediaType = dockerManifest.DockerV2Schema2MediaType
|
||||
|
||||
for idx := range manifest.Layers {
|
||||
manifest.Layers[idx].MediaType = dockerManifest.DockerV2Schema2LayerMediaType
|
||||
}
|
||||
|
||||
manifestBuf, err := json.Marshal(manifest)
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
manifestDigest := godigest.FromBytes(manifestBuf)
|
||||
newIndex.Manifests[idx].Digest = manifestDigest
|
||||
indexManifest.Manifests[idx].Digest = manifestDigest
|
||||
|
||||
// write modified manifest, remove old one
|
||||
err = os.WriteFile(path.Join(srcDir, indexRepoName, "blobs/sha256", manifestDigest.Encoded()),
|
||||
manifestBuf, storageConstants.DefaultFilePerms)
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
err = os.Remove(path.Join(srcDir, indexRepoName, "blobs/sha256", manifestDesc.Digest.Encoded()))
|
||||
So(err, ShouldBeNil)
|
||||
}
|
||||
|
||||
indexManifest.MediaType = dockerManifest.DockerV2ListMediaType
|
||||
// write converted multi arch manifest
|
||||
indexManifestContent, err = json.Marshal(indexManifest)
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
err = os.WriteFile(path.Join(srcDir, indexRepoName, "blobs/sha256",
|
||||
godigest.FromBytes(indexManifestContent).Encoded()), indexManifestContent, storageConstants.DefaultFilePerms)
|
||||
So(err, ShouldBeNil)
|
||||
}
|
||||
|
||||
newIndex.Manifests[indexManifestIdx].MediaType = dockerManifest.DockerV2ListMediaType
|
||||
newIndex.Manifests[indexManifestIdx].Digest = godigest.FromBytes(indexManifestContent)
|
||||
|
||||
indexBuf, err := json.Marshal(newIndex)
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
err = os.WriteFile(path.Join(srcDir, indexRepoName, "index.json"), indexBuf, storageConstants.DefaultFilePerms)
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
dcm := test.NewControllerManager(dctlr)
|
||||
dcm.StartAndWait(dctlr.Config.HTTP.Port)
|
||||
defer dcm.StopServer()
|
||||
|
||||
// sync
|
||||
resp, err = resty.R().SetHeader("Content-Type", ispec.MediaTypeImageIndex).
|
||||
Get(destBaseURL + "/v2/" + indexRepoName + "/manifests/" + "latest")
|
||||
So(err, ShouldBeNil)
|
||||
So(resp.StatusCode(), ShouldEqual, http.StatusOK)
|
||||
So(resp.Body(), ShouldNotBeEmpty)
|
||||
So(resp.Header().Get("Content-Type"), ShouldNotBeEmpty)
|
||||
|
||||
// sync again, should skip
|
||||
resp, err = resty.R().SetHeader("Content-Type", ispec.MediaTypeImageIndex).
|
||||
Get(destBaseURL + "/v2/" + indexRepoName + "/manifests/" + "latest")
|
||||
So(err, ShouldBeNil)
|
||||
So(resp.StatusCode(), ShouldEqual, http.StatusOK)
|
||||
So(resp.Body(), ShouldNotBeEmpty)
|
||||
So(resp.Header().Get("Content-Type"), ShouldNotBeEmpty)
|
||||
|
||||
found, err := test.ReadLogFileAndSearchString(dctlr.Config.Log.Output,
|
||||
"skipping image because it's already synced", 20*time.Second)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
if !found {
|
||||
data, err := os.ReadFile(dctlr.Config.Log.Output)
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
t.Logf("downstream log: %s", string(data))
|
||||
}
|
||||
|
||||
So(found, ShouldBeTrue)
|
||||
|
||||
Convey("trigger config blob upstream error", func() {
|
||||
// remove synced image
|
||||
err := os.RemoveAll(path.Join(destDir, indexRepoName))
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
err = os.Chmod(path.Join(srcDir, indexRepoName, "blobs/sha256", configBlobDigest.Encoded()), 0o000)
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
resp, err = resty.R().Get(destBaseURL + "/v2/" + indexRepoName + "/manifests/" + "latest")
|
||||
So(err, ShouldBeNil)
|
||||
So(resp.StatusCode(), ShouldEqual, http.StatusNotFound)
|
||||
})
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
func TestPeriodically(t *testing.T) {
|
||||
Convey("Verify sync feature", t, func() {
|
||||
updateDuration, _ := time.ParseDuration("30m")
|
||||
|
||||
@@ -4,6 +4,7 @@
|
||||
package sync
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
@@ -14,6 +15,7 @@ import (
|
||||
"github.com/containers/image/v5/copy"
|
||||
"github.com/containers/image/v5/docker"
|
||||
"github.com/containers/image/v5/manifest"
|
||||
"github.com/containers/image/v5/pkg/blobinfocache/none"
|
||||
"github.com/containers/image/v5/signature"
|
||||
"github.com/containers/image/v5/types"
|
||||
"github.com/docker/distribution/reference"
|
||||
@@ -170,3 +172,131 @@ func isSupportedMediaType(mediaType string) bool {
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
// given an imageSource and a docker manifest, convert it to OCI.
|
||||
func convertDockerManifestToOCI(imageSource types.ImageSource, dockerManifestBuf []byte) ([]byte, error) {
|
||||
var ociManifest ispec.Manifest
|
||||
|
||||
// unmarshal docker manifest into OCI manifest
|
||||
err := json.Unmarshal(dockerManifestBuf, &ociManifest)
|
||||
if err != nil {
|
||||
return []byte{}, err
|
||||
}
|
||||
|
||||
configContent, err := getImageConfigContent(imageSource, ociManifest.Config.Digest)
|
||||
if err != nil {
|
||||
return []byte{}, err
|
||||
}
|
||||
|
||||
// marshal config blob into OCI config, will remove keys specific to docker
|
||||
var ociConfig ispec.Image
|
||||
|
||||
err = json.Unmarshal(configContent, &ociConfig)
|
||||
if err != nil {
|
||||
return []byte{}, err
|
||||
}
|
||||
|
||||
ociConfigContent, err := json.Marshal(ociConfig)
|
||||
if err != nil {
|
||||
return []byte{}, err
|
||||
}
|
||||
|
||||
// convert layers
|
||||
err = convertDockerLayersToOCI(ociManifest.Layers)
|
||||
if err != nil {
|
||||
return []byte{}, err
|
||||
}
|
||||
|
||||
// convert config and manifest mediatype
|
||||
ociManifest.Config.Size = int64(len(ociConfigContent))
|
||||
ociManifest.Config.Digest = digest.FromBytes(ociConfigContent)
|
||||
ociManifest.Config.MediaType = ispec.MediaTypeImageConfig
|
||||
ociManifest.MediaType = ispec.MediaTypeImageManifest
|
||||
|
||||
return json.Marshal(ociManifest)
|
||||
}
|
||||
|
||||
// convert docker layers mediatypes to OCI mediatypes.
|
||||
func convertDockerLayersToOCI(dockerLayers []ispec.Descriptor) error {
|
||||
for idx, layer := range dockerLayers {
|
||||
switch layer.MediaType {
|
||||
case manifest.DockerV2Schema2ForeignLayerMediaType:
|
||||
dockerLayers[idx].MediaType = ispec.MediaTypeImageLayerNonDistributable //nolint: staticcheck
|
||||
case manifest.DockerV2Schema2ForeignLayerMediaTypeGzip:
|
||||
dockerLayers[idx].MediaType = ispec.MediaTypeImageLayerNonDistributableGzip //nolint: staticcheck
|
||||
case manifest.DockerV2SchemaLayerMediaTypeUncompressed:
|
||||
dockerLayers[idx].MediaType = ispec.MediaTypeImageLayer
|
||||
case manifest.DockerV2Schema2LayerMediaType:
|
||||
dockerLayers[idx].MediaType = ispec.MediaTypeImageLayerGzip
|
||||
default:
|
||||
return zerr.ErrMediaTypeNotSupported
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// given an imageSource and a docker index manifest, convert it to OCI.
|
||||
func convertDockerIndexToOCI(imageSource types.ImageSource, dockerManifestBuf []byte) ([]byte, error) {
|
||||
// get docker index
|
||||
originalIndex, err := manifest.ListFromBlob(dockerManifestBuf, manifest.DockerV2ListMediaType)
|
||||
if err != nil {
|
||||
return []byte{}, err
|
||||
}
|
||||
|
||||
// get manifests digests
|
||||
manifestsDigests := originalIndex.Instances()
|
||||
|
||||
manifestsUpdates := make([]manifest.ListUpdate, 0, len(manifestsDigests))
|
||||
|
||||
// convert each manifests in index from docker to OCI
|
||||
for _, manifestDigest := range manifestsDigests {
|
||||
digestCopy := manifestDigest
|
||||
|
||||
indexManifestBuf, _, err := imageSource.GetManifest(context.Background(), &digestCopy)
|
||||
if err != nil {
|
||||
return []byte{}, err
|
||||
}
|
||||
|
||||
convertedIndexManifest, err := convertDockerManifestToOCI(imageSource, indexManifestBuf)
|
||||
if err != nil {
|
||||
return []byte{}, err
|
||||
}
|
||||
|
||||
manifestsUpdates = append(manifestsUpdates, manifest.ListUpdate{
|
||||
Digest: digest.FromBytes(convertedIndexManifest),
|
||||
Size: int64(len(convertedIndexManifest)),
|
||||
MediaType: ispec.MediaTypeImageManifest,
|
||||
})
|
||||
}
|
||||
|
||||
// update all manifests in index
|
||||
if err := originalIndex.UpdateInstances(manifestsUpdates); err != nil {
|
||||
return []byte{}, err
|
||||
}
|
||||
|
||||
// convert index to OCI
|
||||
convertedList, err := originalIndex.ConvertToMIMEType(ispec.MediaTypeImageIndex)
|
||||
if err != nil {
|
||||
return []byte{}, err
|
||||
}
|
||||
|
||||
return convertedList.Serialize()
|
||||
}
|
||||
|
||||
// given an image source and a config blob digest, get blob config content.
|
||||
func getImageConfigContent(imageSource types.ImageSource, configDigest digest.Digest,
|
||||
) ([]byte, error) {
|
||||
configBlob, _, err := imageSource.GetBlob(context.Background(), types.BlobInfo{
|
||||
Digest: configDigest,
|
||||
}, none.NoCache)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
configBuf := new(bytes.Buffer)
|
||||
|
||||
_, err = configBuf.ReadFrom(configBlob)
|
||||
|
||||
return configBuf.Bytes(), err
|
||||
}
|
||||
|
||||
@@ -129,6 +129,15 @@ function teardown_file() {
|
||||
run curl http://127.0.0.1:8090/v2/registry/tags/list
|
||||
[ "$status" -eq 0 ]
|
||||
[ $(echo "${lines[-1]}" | jq '.tags[]') = '"latest"' ]
|
||||
|
||||
# make sure image is skipped when synced again
|
||||
run skopeo --insecure-policy copy --multi-arch=all --src-tls-verify=false \
|
||||
docker://127.0.0.1:8090/registry \
|
||||
oci:${TEST_DATA_DIR}
|
||||
[ "$status" -eq 0 ]
|
||||
|
||||
run $("cat /tmp/blackbox.log | grep -q registry:latest.*.skipping image because it's already synced")
|
||||
[ "$status" -eq 0 ]
|
||||
}
|
||||
|
||||
@test "sync docker image on demand" {
|
||||
@@ -143,6 +152,15 @@ function teardown_file() {
|
||||
run curl http://127.0.0.1:8090/v2/archlinux/tags/list
|
||||
[ "$status" -eq 0 ]
|
||||
[ $(echo "${lines[-1]}" | jq '.tags[]') = '"latest"' ]
|
||||
|
||||
# make sure image is skipped when synced again
|
||||
run skopeo --insecure-policy copy --src-tls-verify=false \
|
||||
docker://127.0.0.1:8090/archlinux \
|
||||
oci:${TEST_DATA_DIR}
|
||||
[ "$status" -eq 0 ]
|
||||
|
||||
run $("cat /tmp/blackbox.log | grep -q archlinux:latest.*.skipping image because it's already synced")
|
||||
[ "$status" -eq 0 ]
|
||||
}
|
||||
|
||||
@test "sync k8s image list on demand" {
|
||||
|
||||
Reference in New Issue
Block a user