image level lint: enforce manifest mandatory annotations

closes #536

Signed-off-by: Lisca Ana-Roberta <ana.kagome@yahoo.com>
This commit is contained in:
Lisca Ana-Roberta
2022-06-24 16:08:47 +03:00
committed by Andrei Aaron
parent 3d72dad507
commit 87fc941b3c
34 changed files with 1391 additions and 110 deletions
+9
View File
@@ -0,0 +1,9 @@
package storage
import (
godigest "github.com/opencontainers/go-digest"
)
type Lint interface {
Lint(repo string, manifestDigest godigest.Digest, imageStore ImageStore) (bool, error)
}
+61 -24
View File
@@ -25,6 +25,7 @@ import (
"github.com/opencontainers/umoci/oci/casext"
artifactspec "github.com/oras-project/artifacts-spec/specs-go/v1"
"github.com/rs/zerolog"
"github.com/sigstore/cosign/pkg/oci/remote"
zerr "zotregistry.io/zot/errors"
"zotregistry.io/zot/pkg/extensions/monitoring"
zlog "zotregistry.io/zot/pkg/log"
@@ -64,6 +65,7 @@ type ImageStoreLocal struct {
gcDelay time.Duration
log zerolog.Logger
metrics monitoring.MetricServer
linter Lint
}
func (is *ImageStoreLocal) RootDir() string {
@@ -105,7 +107,7 @@ func (sc StoreController) GetImageStore(name string) ImageStore {
// NewImageStore returns a new image store backed by a file storage.
func NewImageStore(rootDir string, gc bool, gcDelay time.Duration, dedupe, commit bool,
log zlog.Logger, metrics monitoring.MetricServer,
log zlog.Logger, metrics monitoring.MetricServer, linter Lint,
) ImageStore {
if _, err := os.Stat(rootDir); os.IsNotExist(err) {
if err := os.MkdirAll(rootDir, DefaultDirPerms); err != nil {
@@ -125,6 +127,7 @@ func NewImageStore(rootDir string, gc bool, gcDelay time.Duration, dedupe, commi
commit: commit,
log: log.With().Caller().Logger(),
metrics: metrics,
linter: linter,
}
if dedupe {
@@ -549,29 +552,9 @@ func (is *ImageStoreLocal) PutImageManifest(repo, reference, mediaType string,
return "", zerr.ErrBadManifest
}
if mediaType == ispec.MediaTypeImageManifest {
var manifest ispec.Manifest
if err := json.Unmarshal(body, &manifest); err != nil {
is.log.Error().Err(err).Msg("unable to unmarshal JSON")
return "", zerr.ErrBadManifest
}
if manifest.Config.MediaType == ispec.MediaTypeImageConfig {
digest, err := is.validateOCIManifest(repo, reference, &manifest)
if err != nil {
is.log.Error().Err(err).Msg("invalid oci image manifest")
return digest, err
}
}
} else if mediaType == artifactspec.MediaTypeArtifactManifest {
var m notation.Descriptor
if err := json.Unmarshal(body, &m); err != nil {
is.log.Error().Err(err).Msg("unable to unmarshal JSON")
return "", zerr.ErrBadManifest
}
dig, err := validateManifest(is, repo, reference, mediaType, body)
if err != nil {
return dig, err
}
mDigest := godigest.FromBytes(body)
@@ -666,6 +649,7 @@ func (is *ImageStoreLocal) PutImageManifest(repo, reference, mediaType string,
_ = ensureDir(dir, is.log)
file := path.Join(dir, mDigest.Encoded())
// in case the linter will not pass, it will be garbage collected
if err := is.writeFile(file, body); err != nil {
is.log.Error().Err(err).Str("file", file).Msg("unable to write")
@@ -684,6 +668,28 @@ func (is *ImageStoreLocal) PutImageManifest(repo, reference, mediaType string,
return "", err
}
// apply linter only on images, not signatures
if is.linter != nil {
if mediaType == ispec.MediaTypeImageManifest &&
// check that image manifest is not cosign signature
!strings.HasPrefix(reference, "sha256-") &&
!strings.HasSuffix(reference, remote.SignatureTagSuffix) {
// lint new index with new manifest before writing to disk
is.Unlock(&lockLatency)
pass, err := is.linter.Lint(repo, mDigest, is)
is.Lock(&lockLatency)
if err != nil {
is.log.Error().Err(err).Msg("linter error")
return "", err
}
if !pass {
return "", zerr.ErrImageLintAnnotations
}
}
}
err = is.writeFile(file, buf)
if err := test.Error(err); err != nil {
is.log.Error().Err(err).Str("file", file).Msg("unable to write")
@@ -1616,6 +1622,37 @@ func (is *ImageStoreLocal) garbageCollect(dir string, repo string) error {
return nil
}
func validateManifest(imgStore *ImageStoreLocal, repo, reference,
mediaType string, body []byte,
) (string, error) {
if mediaType == ispec.MediaTypeImageManifest {
var manifest ispec.Manifest
if err := json.Unmarshal(body, &manifest); err != nil {
imgStore.log.Error().Err(err).Msg("unable to unmarshal JSON")
return "", zerr.ErrBadManifest
}
if manifest.Config.MediaType == ispec.MediaTypeImageConfig {
digest, err := imgStore.validateOCIManifest(repo, reference, &manifest)
if err != nil {
imgStore.log.Error().Err(err).Msg("invalid oci image manifest")
return digest, err
}
}
} else if mediaType == artifactspec.MediaTypeArtifactManifest {
var m notation.Descriptor
if err := json.Unmarshal(body, &m); err != nil {
imgStore.log.Error().Err(err).Msg("unable to unmarshal JSON")
return "", zerr.ErrBadManifest
}
}
return "", nil
}
func ifOlderThan(imgStore *ImageStoreLocal, repo string, delay time.Duration) casext.GCPolicy {
return func(ctx context.Context, digest godigest.Digest) (bool, error) {
blobPath := imgStore.BlobPath(repo, digest)
+1 -1
View File
@@ -27,7 +27,7 @@ func TestElevatedPrivilegesInvalidDedupe(t *testing.T) {
log := log.Logger{Logger: zerolog.New(os.Stdout)}
metrics := monitoring.NewMetricsServer(false, log)
imgStore := storage.NewImageStore(dir, true, storage.DefaultGCDelay, true, true, log, metrics)
imgStore := storage.NewImageStore(dir, true, storage.DefaultGCDelay, true, true, log, metrics, nil)
upload, err := imgStore.NewBlobUpload("dedupe1")
So(err, ShouldBeNil)
+41 -23
View File
@@ -36,7 +36,8 @@ func TestStorageFSAPIs(t *testing.T) {
log := log.Logger{Logger: zerolog.New(os.Stdout)}
metrics := monitoring.NewMetricsServer(false, log)
imgStore := storage.NewImageStore(dir, true, storage.DefaultGCDelay, true, true, log, metrics)
imgStore := storage.NewImageStore(dir, true, storage.DefaultGCDelay, true,
true, log, metrics, nil)
Convey("Repo layout", t, func(c C) {
repoName := "test"
@@ -169,7 +170,7 @@ func TestGetReferrers(t *testing.T) {
log := log.Logger{Logger: zerolog.New(os.Stdout)}
metrics := monitoring.NewMetricsServer(false, log)
imgStore := storage.NewImageStore(dir, true, storage.DefaultGCDelay, true, true, log, metrics)
imgStore := storage.NewImageStore(dir, true, storage.DefaultGCDelay, true, true, log, metrics, nil)
Convey("Get referrers", t, func(c C) {
err := test.CopyFiles("../../test/data/zot-test", path.Join(dir, "zot-test"))
@@ -218,7 +219,8 @@ func TestDedupeLinks(t *testing.T) {
log := log.Logger{Logger: zerolog.New(os.Stdout)}
metrics := monitoring.NewMetricsServer(false, log)
imgStore := storage.NewImageStore(dir, true, storage.DefaultGCDelay, true, true, log, metrics)
imgStore := storage.NewImageStore(dir, true, storage.DefaultGCDelay,
true, true, log, metrics, nil)
Convey("Dedupe", t, func(c C) {
// manifest1
@@ -272,7 +274,8 @@ func TestDedupeLinks(t *testing.T) {
manifestBuf, err := json.Marshal(manifest)
So(err, ShouldBeNil)
digest = godigest.FromBytes(manifestBuf)
_, err = imgStore.PutImageManifest("dedupe1", digest.String(), ispec.MediaTypeImageManifest, manifestBuf)
_, err = imgStore.PutImageManifest("dedupe1", digest.String(),
ispec.MediaTypeImageManifest, manifestBuf)
So(err, ShouldBeNil)
_, _, _, err = imgStore.GetImageManifest("dedupe1", digest.String())
@@ -356,7 +359,7 @@ func TestDedupe(t *testing.T) {
log := log.Logger{Logger: zerolog.New(os.Stdout)}
metrics := monitoring.NewMetricsServer(false, log)
il := storage.NewImageStore(dir, true, storage.DefaultGCDelay, true, true, log, metrics)
il := storage.NewImageStore(dir, true, storage.DefaultGCDelay, true, true, log, metrics, nil)
So(il.DedupeBlob("", "", ""), ShouldNotBeNil)
})
@@ -371,9 +374,11 @@ func TestNegativeCases(t *testing.T) {
log := log.Logger{Logger: zerolog.New(os.Stdout)}
metrics := monitoring.NewMetricsServer(false, log)
So(storage.NewImageStore(dir, true, storage.DefaultGCDelay, true, true, log, metrics), ShouldNotBeNil)
So(storage.NewImageStore(dir, true, storage.DefaultGCDelay, true,
true, log, metrics, nil), ShouldNotBeNil)
if os.Geteuid() != 0 {
So(storage.NewImageStore("/deadBEEF", true, storage.DefaultGCDelay, true, true, log, metrics), ShouldBeNil)
So(storage.NewImageStore("/deadBEEF", true, storage.DefaultGCDelay,
true, true, log, metrics, nil), ShouldBeNil)
}
})
@@ -382,7 +387,8 @@ func TestNegativeCases(t *testing.T) {
log := log.Logger{Logger: zerolog.New(os.Stdout)}
metrics := monitoring.NewMetricsServer(false, log)
imgStore := storage.NewImageStore(dir, true, storage.DefaultGCDelay, true, true, log, metrics)
imgStore := storage.NewImageStore(dir, true, storage.DefaultGCDelay,
true, true, log, metrics, nil)
err := os.Chmod(dir, 0o000) // remove all perms
if err != nil {
@@ -417,7 +423,8 @@ func TestNegativeCases(t *testing.T) {
log := log.Logger{Logger: zerolog.New(os.Stdout)}
metrics := monitoring.NewMetricsServer(false, log)
imgStore := storage.NewImageStore(dir, true, storage.DefaultGCDelay, true, true, log, metrics)
imgStore := storage.NewImageStore(dir, true, storage.DefaultGCDelay, true,
true, log, metrics, nil)
So(imgStore, ShouldNotBeNil)
So(imgStore.InitRepo("test"), ShouldBeNil)
@@ -531,7 +538,8 @@ func TestNegativeCases(t *testing.T) {
log := log.Logger{Logger: zerolog.New(os.Stdout)}
metrics := monitoring.NewMetricsServer(false, log)
imgStore := storage.NewImageStore(dir, true, storage.DefaultGCDelay, true, true, log, metrics)
imgStore := storage.NewImageStore(dir, true, storage.DefaultGCDelay,
true, true, log, metrics, nil)
So(imgStore, ShouldNotBeNil)
So(imgStore.InitRepo("test"), ShouldBeNil)
@@ -554,7 +562,8 @@ func TestNegativeCases(t *testing.T) {
log := log.Logger{Logger: zerolog.New(os.Stdout)}
metrics := monitoring.NewMetricsServer(false, log)
imgStore := storage.NewImageStore(dir, true, storage.DefaultGCDelay, true, true, log, metrics)
imgStore := storage.NewImageStore(dir, true, storage.DefaultGCDelay, true,
true, log, metrics, nil)
So(imgStore, ShouldNotBeNil)
So(imgStore.InitRepo("test"), ShouldBeNil)
@@ -595,7 +604,8 @@ func TestNegativeCases(t *testing.T) {
log := log.Logger{Logger: zerolog.New(os.Stdout)}
metrics := monitoring.NewMetricsServer(false, log)
imgStore := storage.NewImageStore(dir, true, storage.DefaultGCDelay, true, true, log, metrics)
imgStore := storage.NewImageStore(dir, true, storage.DefaultGCDelay,
true, true, log, metrics, nil)
So(imgStore, ShouldNotBeNil)
So(imgStore.InitRepo("test"), ShouldBeNil)
@@ -739,7 +749,8 @@ func TestInjectWriteFile(t *testing.T) {
log := log.Logger{Logger: zerolog.New(os.Stdout)}
metrics := monitoring.NewMetricsServer(false, log)
imgStore := storage.NewImageStore(dir, true, storage.DefaultGCDelay, true, true, log, metrics)
imgStore := storage.NewImageStore(dir, true, storage.DefaultGCDelay,
true, true, log, metrics, nil)
Convey("Failure path1", func() {
injected := test.InjectFailure(0)
@@ -769,7 +780,8 @@ func TestInjectWriteFile(t *testing.T) {
log := log.Logger{Logger: zerolog.New(os.Stdout)}
metrics := monitoring.NewMetricsServer(false, log)
imgStore := storage.NewImageStore(dir, true, storage.DefaultGCDelay, true, false, log, metrics)
imgStore := storage.NewImageStore(dir, true, storage.DefaultGCDelay,
true, false, log, metrics, nil)
Convey("Failure path not reached", func() {
err := imgStore.InitRepo("repo1")
@@ -786,7 +798,8 @@ func TestGarbageCollect(t *testing.T) {
metrics := monitoring.NewMetricsServer(false, log)
Convey("Garbage collect with default/long delay", func() {
imgStore := storage.NewImageStore(dir, true, storage.DefaultGCDelay, true, true, log, metrics)
imgStore := storage.NewImageStore(dir, true, storage.DefaultGCDelay,
true, true, log, metrics, nil)
repoName := "gc-long"
upload, err := imgStore.NewBlobUpload(repoName)
@@ -853,7 +866,7 @@ func TestGarbageCollect(t *testing.T) {
})
Convey("Garbage collect with short delay", func() {
imgStore := storage.NewImageStore(dir, true, 1*time.Second, true, true, log, metrics)
imgStore := storage.NewImageStore(dir, true, 1*time.Second, true, true, log, metrics, nil)
repoName := "gc-short"
// upload orphan blob
@@ -949,7 +962,7 @@ func TestGarbageCollect(t *testing.T) {
Convey("Garbage collect with dedupe", func() {
// garbage-collect is repo-local and dedupe is global and they can interact in strange ways
imgStore := storage.NewImageStore(dir, true, 5*time.Second, true, true, log, metrics)
imgStore := storage.NewImageStore(dir, true, 5*time.Second, true, true, log, metrics, nil)
// first upload an image to the first repo and wait for GC timeout
@@ -1150,7 +1163,7 @@ func TestGarbageCollectForImageStore(t *testing.T) {
log := log.NewLogger("debug", logFile.Name())
metrics := monitoring.NewMetricsServer(false, log)
imgStore := storage.NewImageStore(dir, true, 1*time.Second, true, true, log, metrics)
imgStore := storage.NewImageStore(dir, true, 1*time.Second, true, true, log, metrics, nil)
repoName := "gc-all-repos-short"
err := test.CopyFiles("../../test/data/zot-test", path.Join(dir, repoName))
@@ -1182,7 +1195,7 @@ func TestGarbageCollectForImageStore(t *testing.T) {
log := log.NewLogger("debug", logFile.Name())
metrics := monitoring.NewMetricsServer(false, log)
imgStore := storage.NewImageStore(dir, true, 1*time.Second, true, true, log, metrics)
imgStore := storage.NewImageStore(dir, true, 1*time.Second, true, true, log, metrics, nil)
repoName := "gc-all-repos-short"
err := test.CopyFiles("../../test/data/zot-test", path.Join(dir, repoName))
@@ -1227,7 +1240,8 @@ func TestInitRepo(t *testing.T) {
log := log.Logger{Logger: zerolog.New(os.Stdout)}
metrics := monitoring.NewMetricsServer(false, log)
imgStore := storage.NewImageStore(dir, true, storage.DefaultGCDelay, true, true, log, metrics)
imgStore := storage.NewImageStore(dir, true, storage.DefaultGCDelay,
true, true, log, metrics, nil)
err := os.Mkdir(path.Join(dir, "test-dir"), 0o000)
So(err, ShouldBeNil)
@@ -1243,7 +1257,8 @@ func TestValidateRepo(t *testing.T) {
log := log.Logger{Logger: zerolog.New(os.Stdout)}
metrics := monitoring.NewMetricsServer(false, log)
imgStore := storage.NewImageStore(dir, true, storage.DefaultGCDelay, true, true, log, metrics)
imgStore := storage.NewImageStore(dir, true, storage.DefaultGCDelay,
true, true, log, metrics, nil)
err := os.Mkdir(path.Join(dir, "test-dir"), 0o000)
So(err, ShouldBeNil)
@@ -1259,7 +1274,9 @@ func TestGetRepositoriesError(t *testing.T) {
log := log.Logger{Logger: zerolog.New(os.Stdout)}
metrics := monitoring.NewMetricsServer(false, log)
imgStore := storage.NewImageStore(dir, true, storage.DefaultGCDelay, true, true, log, metrics)
imgStore := storage.NewImageStore(dir, true, storage.DefaultGCDelay,
true, true, log, metrics, nil,
)
// create valid directory with permissions
err := os.Mkdir(path.Join(dir, "test-dir"), 0o755)
@@ -1279,7 +1296,8 @@ func TestPutBlobChunkStreamed(t *testing.T) {
log := log.Logger{Logger: zerolog.New(os.Stdout)}
metrics := monitoring.NewMetricsServer(false, log)
imgStore := storage.NewImageStore(dir, true, storage.DefaultGCDelay, true, true, log, metrics)
imgStore := storage.NewImageStore(dir, true, storage.DefaultGCDelay,
true, true, log, metrics, nil)
uuid, err := imgStore.NewBlobUpload("test")
So(err, ShouldBeNil)
+27 -3
View File
@@ -11,6 +11,7 @@ import (
"os"
"path"
"path/filepath"
"strings"
"sync"
"time"
@@ -23,6 +24,7 @@ import (
ispec "github.com/opencontainers/image-spec/specs-go/v1"
artifactspec "github.com/oras-project/artifacts-spec/specs-go/v1"
"github.com/rs/zerolog"
"github.com/sigstore/cosign/pkg/oci/remote"
zerr "zotregistry.io/zot/errors"
"zotregistry.io/zot/pkg/extensions/monitoring"
zlog "zotregistry.io/zot/pkg/log"
@@ -50,6 +52,7 @@ type ObjectStorage struct {
metrics monitoring.MetricServer
cache *storage.Cache
dedupe bool
linter storage.Lint
}
func (is *ObjectStorage) RootDir() string {
@@ -67,7 +70,7 @@ func (is *ObjectStorage) DirExists(d string) bool {
// NewObjectStorage returns a new image store backed by cloud storages.
// see https://github.com/docker/docker.github.io/tree/master/registry/storage-drivers
func NewImageStore(rootDir string, cacheDir string, gc bool, gcDelay time.Duration, dedupe, commit bool,
log zlog.Logger, metrics monitoring.MetricServer,
log zlog.Logger, metrics monitoring.MetricServer, linter storage.Lint,
store driver.StorageDriver,
) storage.ImageStore {
imgStore := &ObjectStorage{
@@ -79,6 +82,7 @@ func NewImageStore(rootDir string, cacheDir string, gc bool, gcDelay time.Durati
multiPartUploads: sync.Map{},
metrics: metrics,
dedupe: dedupe,
linter: linter,
}
cachePath := path.Join(cacheDir, CacheDBName+storage.DBExtensionName)
@@ -395,8 +399,8 @@ func (is *ObjectStorage) GetImageManifest(repo, reference string) ([]byte, strin
// PutImageManifest adds an image manifest to the repository.
func (is *ObjectStorage) PutImageManifest(repo, reference, mediaType string,
body []byte,
) (string, error) {
body []byte) (string, error,
) {
if err := is.InitRepo(repo); err != nil {
is.log.Debug().Err(err).Msg("init repo")
@@ -549,6 +553,26 @@ func (is *ObjectStorage) PutImageManifest(repo, reference, mediaType string,
return "", err
}
// apply linter only on images, not signatures
if is.linter != nil {
if mediaType == ispec.MediaTypeImageManifest &&
// check that image manifest is not cosign signature
!strings.HasPrefix(reference, "sha256-") &&
!strings.HasSuffix(reference, remote.SignatureTagSuffix) {
// lint new index with new manifest before writing to disk
pass, err := is.linter.Lint(repo, mDigest, is)
if err != nil {
is.log.Error().Err(err).Msg("linter error")
return "", err
}
if !pass {
return "", zerr.ErrImageLintAnnotations
}
}
}
if err = is.store.PutContent(context.Background(), indexPath, buf); err != nil {
is.log.Error().Err(err).Str("file", manifestPath).Msg("unable to write")
+11 -5
View File
@@ -56,7 +56,9 @@ func skipIt(t *testing.T) {
func createMockStorage(rootDir string, cacheDir string, dedupe bool, store driver.StorageDriver) storage.ImageStore {
log := log.Logger{Logger: zerolog.New(os.Stdout)}
metrics := monitoring.NewMetricsServer(false, log)
il := s3.NewImageStore(rootDir, cacheDir, false, storage.DefaultGCDelay, dedupe, false, log, metrics, store)
il := s3.NewImageStore(rootDir, cacheDir, false, storage.DefaultGCDelay,
dedupe, false, log, metrics, nil, store,
)
return il
}
@@ -95,7 +97,8 @@ func createObjectsStore(rootDir string, cacheDir string, dedupe bool) (
log := log.Logger{Logger: zerolog.New(os.Stdout)}
metrics := monitoring.NewMetricsServer(false, log)
il := s3.NewImageStore(rootDir, cacheDir, false, storage.DefaultGCDelay, dedupe, false, log, metrics, store)
il := s3.NewImageStore(rootDir, cacheDir, false, storage.DefaultGCDelay,
dedupe, false, log, metrics, nil, store)
return store, il, err
}
@@ -894,7 +897,8 @@ func TestS3Dedupe(t *testing.T) {
manifestBuf, err := json.Marshal(manifest)
So(err, ShouldBeNil)
digest = godigest.FromBytes(manifestBuf)
_, err = imgStore.PutImageManifest("dedupe1", digest.String(), ispec.MediaTypeImageManifest, manifestBuf)
_, err = imgStore.PutImageManifest("dedupe1", digest.String(),
ispec.MediaTypeImageManifest, manifestBuf)
So(err, ShouldBeNil)
_, _, _, err = imgStore.GetImageManifest("dedupe1", digest.String())
@@ -956,7 +960,8 @@ func TestS3Dedupe(t *testing.T) {
manifestBuf, err = json.Marshal(manifest)
So(err, ShouldBeNil)
digest = godigest.FromBytes(manifestBuf)
_, err = imgStore.PutImageManifest("dedupe2", "1.0", ispec.MediaTypeImageManifest, manifestBuf)
_, err = imgStore.PutImageManifest("dedupe2", "1.0", ispec.MediaTypeImageManifest,
manifestBuf)
So(err, ShouldBeNil)
_, _, _, err = imgStore.GetImageManifest("dedupe2", digest.String())
@@ -1078,7 +1083,8 @@ func TestS3Dedupe(t *testing.T) {
manifestBuf, err = json.Marshal(manifest)
So(err, ShouldBeNil)
digest = godigest.FromBytes(manifestBuf)
_, err = imgStore.PutImageManifest("dedupe3", "1.0", ispec.MediaTypeImageManifest, manifestBuf)
_, err = imgStore.PutImageManifest("dedupe3", "1.0", ispec.MediaTypeImageManifest,
manifestBuf)
So(err, ShouldBeNil)
_, _, _, err = imgStore.GetImageManifest("dedupe3", digest.String())
+5 -3
View File
@@ -31,7 +31,8 @@ func TestCheckAllBlobsIntegrity(t *testing.T) {
metrics := monitoring.NewMetricsServer(false, log)
imgStore := storage.NewImageStore(dir, true, storage.DefaultGCDelay, true, true, log, metrics)
imgStore := storage.NewImageStore(dir, true, storage.DefaultGCDelay,
true, true, log, metrics, nil)
Convey("Scrub only one repo", t, func(c C) {
// initialize repo
@@ -117,10 +118,11 @@ func TestCheckAllBlobsIntegrity(t *testing.T) {
}
mnfst.SchemaVersion = 2
mb, err := json.Marshal(mnfst)
mbytes, err := json.Marshal(mnfst)
So(err, ShouldBeNil)
manifest, err = imgStore.PutImageManifest(repoName, tag, ispec.MediaTypeImageManifest, mb)
manifest, err = imgStore.PutImageManifest(repoName, tag, ispec.MediaTypeImageManifest,
mbytes)
So(err, ShouldBeNil)
Convey("Blobs integrity not affected", func() {
+132 -9
View File
@@ -5,6 +5,7 @@ import (
"context"
_ "crypto/sha256"
"encoding/json"
"errors"
"fmt"
"os"
"path"
@@ -28,6 +29,7 @@ import (
"zotregistry.io/zot/pkg/storage"
"zotregistry.io/zot/pkg/storage/s3"
"zotregistry.io/zot/pkg/test"
"zotregistry.io/zot/pkg/test/mocks"
)
func cleanupStorage(store driver.StorageDriver, name string) {
@@ -73,7 +75,9 @@ func createObjectsStore(rootDir string, cacheDir string) (driver.StorageDriver,
log := log.Logger{Logger: zerolog.New(os.Stdout)}
metrics := monitoring.NewMetricsServer(false, log)
il := s3.NewImageStore(rootDir, cacheDir, false, storage.DefaultGCDelay, true, false, log, metrics, store)
il := s3.NewImageStore(rootDir, cacheDir, false, storage.DefaultGCDelay,
true, false, log, metrics, nil, store,
)
return store, il, err
}
@@ -117,7 +121,8 @@ func TestStorageAPIs(t *testing.T) {
log := log.Logger{Logger: zerolog.New(os.Stdout)}
metrics := monitoring.NewMetricsServer(false, log)
imgStore = storage.NewImageStore(dir, true, storage.DefaultGCDelay, true, true, log, metrics)
imgStore = storage.NewImageStore(dir, true, storage.DefaultGCDelay, true,
true, log, metrics, nil)
}
Convey("Repo layout", t, func(c C) {
@@ -446,10 +451,12 @@ func TestStorageAPIs(t *testing.T) {
})
Convey("Bad image manifest", func() {
_, err = imgStore.PutImageManifest("test", digest.String(), ispec.MediaTypeImageManifest, manifestBuf)
_, err = imgStore.PutImageManifest("test", digest.String(),
ispec.MediaTypeImageManifest, manifestBuf)
So(err, ShouldNotBeNil)
_, err = imgStore.PutImageManifest("test", digest.String(), ispec.MediaTypeImageManifest, []byte("bad json"))
_, err = imgStore.PutImageManifest("test", digest.String(),
ispec.MediaTypeImageManifest, []byte("bad json"))
So(err, ShouldNotBeNil)
_, _, _, err = imgStore.GetImageManifest("test", digest.String())
@@ -483,11 +490,13 @@ func TestStorageAPIs(t *testing.T) {
manifestBuf, err = json.Marshal(manifest)
So(err, ShouldBeNil)
digest := godigest.FromBytes(manifestBuf)
_, err = imgStore.PutImageManifest("test", digest.String(), ispec.MediaTypeImageManifest, manifestBuf)
_, err = imgStore.PutImageManifest("test", digest.String(),
ispec.MediaTypeImageManifest, manifestBuf)
So(err, ShouldBeNil)
// same manifest for coverage
_, err = imgStore.PutImageManifest("test", digest.String(), ispec.MediaTypeImageManifest, manifestBuf)
_, err = imgStore.PutImageManifest("test", digest.String(),
ispec.MediaTypeImageManifest, manifestBuf)
So(err, ShouldBeNil)
_, _, _, err = imgStore.GetImageManifest("test", digest.String())
@@ -661,6 +670,117 @@ func TestStorageAPIs(t *testing.T) {
}
}
func TestMandatoryAnnotations(t *testing.T) {
for _, testcase := range testCases {
testcase := testcase
t.Run(testcase.testCaseName, func(t *testing.T) {
var imgStore storage.ImageStore
var testDir, tdir string
var store driver.StorageDriver
log := log.Logger{Logger: zerolog.New(os.Stdout)}
metrics := monitoring.NewMetricsServer(false, log)
if testcase.storageType == "s3" {
skipIt(t)
uuid, err := guuid.NewV4()
if err != nil {
panic(err)
}
testDir = path.Join("/oci-repo-test", uuid.String())
tdir = t.TempDir()
store, _, _ = createObjectsStore(testDir, tdir)
imgStore = s3.NewImageStore(testDir, tdir, false, 1, false, false, log, metrics,
&mocks.MockedLint{
LintFn: func(repo string, manifestDigest godigest.Digest, imageStore storage.ImageStore) (bool, error) {
return false, nil
},
}, store)
defer cleanupStorage(store, testDir)
} else {
tdir = t.TempDir()
imgStore = storage.NewImageStore(tdir, true, storage.DefaultGCDelay, true,
true, log, metrics, &mocks.MockedLint{
LintFn: func(repo string, manifestDigest godigest.Digest, imageStore storage.ImageStore) (bool, error) {
return false, nil
},
})
}
Convey("Setup manifest", t, func() {
content := []byte("test-data1")
buf := bytes.NewBuffer(content)
buflen := buf.Len()
digest := godigest.FromBytes(content)
_, _, err := imgStore.FullBlobUpload("test", bytes.NewReader(buf.Bytes()), digest.String())
So(err, ShouldBeNil)
cblob, cdigest := test.GetRandomImageConfig()
_, clen, err := imgStore.FullBlobUpload("test", bytes.NewReader(cblob), cdigest.String())
So(err, ShouldBeNil)
So(clen, ShouldEqual, len(cblob))
annotationsMap := make(map[string]string)
annotationsMap[ispec.AnnotationRefName] = "1.0"
manifest := ispec.Manifest{
Config: ispec.Descriptor{
MediaType: "application/vnd.oci.image.config.v1+json",
Digest: cdigest,
Size: int64(len(cblob)),
},
Layers: []ispec.Descriptor{
{
MediaType: "application/vnd.oci.image.layer.v1.tar",
Digest: digest,
Size: int64(buflen),
},
},
Annotations: annotationsMap,
}
manifest.SchemaVersion = 2
manifestBuf, err := json.Marshal(manifest)
So(err, ShouldBeNil)
Convey("Missing mandatory annotations", func() {
_, err = imgStore.PutImageManifest("test", "1.0.0", ispec.MediaTypeImageManifest, manifestBuf)
So(err, ShouldNotBeNil)
})
Convey("Error on mandatory annotations", func() {
if testcase.storageType == "s3" {
imgStore = s3.NewImageStore(testDir, tdir, false, 1, false, false, log, metrics,
&mocks.MockedLint{
LintFn: func(repo string, manifestDigest godigest.Digest, imageStore storage.ImageStore) (bool, error) {
// nolint: goerr113
return false, errors.New("linter error")
},
}, store)
} else {
imgStore = storage.NewImageStore(tdir, true, storage.DefaultGCDelay, true,
true, log, metrics, &mocks.MockedLint{
LintFn: func(repo string, manifestDigest godigest.Digest, imageStore storage.ImageStore) (bool, error) {
// nolint: goerr113
return false, errors.New("linter error")
},
})
}
_, err = imgStore.PutImageManifest("test", "1.0.0", ispec.MediaTypeImageManifest, manifestBuf)
So(err, ShouldNotBeNil)
})
})
})
}
}
func TestStorageHandler(t *testing.T) {
for _, testcase := range testCases {
testcase := testcase
@@ -700,11 +820,14 @@ func TestStorageHandler(t *testing.T) {
metrics := monitoring.NewMetricsServer(false, log)
// Create ImageStore
firstStore = storage.NewImageStore(firstRootDir, false, storage.DefaultGCDelay, false, false, log, metrics)
firstStore = storage.NewImageStore(firstRootDir, false, storage.DefaultGCDelay,
false, false, log, metrics, nil)
secondStore = storage.NewImageStore(secondRootDir, false, storage.DefaultGCDelay, false, false, log, metrics)
secondStore = storage.NewImageStore(secondRootDir, false,
storage.DefaultGCDelay, false, false, log, metrics, nil)
thirdStore = storage.NewImageStore(thirdRootDir, false, storage.DefaultGCDelay, false, false, log, metrics)
thirdStore = storage.NewImageStore(thirdRootDir, false, storage.DefaultGCDelay,
false, false, log, metrics, nil)
}
Convey("Test storage handler", t, func() {