github/zot
1
0
Fork 0
mirror of https://github.com/project-zot/zot.git synced 2026-06-16 04:17:55 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(retention): added image retention policies (#1866)

Browse Source 
feat(metaDB): add more image statistics info

Signed-off-by: Petu Eusebiu <peusebiu@cisco.com>
This commit is contained in:
peusebiu
2023-11-01 18:16:18 +02:00
committed by GitHub
parent a79d79a03a
commit 9074f8483b
71 changed files with 3454 additions and 745 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/extensions/search/convert/convert_internal_test.go
+1 -1
View File
@@ -39,7 +39,7 @@ func TestCVEConvert(t *testing.T) {
Blob: ispec.DescriptorEmptyJSON.Data,
}}).DefaultConfig().Build()
err = metaDB.SetRepoReference("repo1", "0.1.0", image.AsImageMeta())
err = metaDB.SetRepoReference(context.Background(), "repo1", "0.1.0", image.AsImageMeta())
So(err, ShouldBeNil)
repoMetaList, err := metaDB.SearchRepos(context.Background(), "")
pkg/extensions/search/cve/cve_test.go
+14 -14
View File
@@ -770,32 +770,32 @@ func TestCVEStruct(t *testing.T) { //nolint:gocyclo
image11 := CreateImageWith().DefaultLayers().
ImageConfig(ispec.Image{Created: DateRef(2008, 1, 1, 12, 0, 0, 0, time.UTC)}).Build()
err = metaDB.SetRepoReference(repo1, "0.1.0", image11.AsImageMeta())
err = metaDB.SetRepoReference(context.Background(), repo1, "0.1.0", image11.AsImageMeta())
So(err, ShouldBeNil)
image12 := CreateImageWith().DefaultLayers().
ImageConfig(ispec.Image{Created: DateRef(2009, 1, 1, 12, 0, 0, 0, time.UTC)}).Build()
err = metaDB.SetRepoReference(repo1, "1.0.0", image12.AsImageMeta())
err = metaDB.SetRepoReference(context.Background(), repo1, "1.0.0", image12.AsImageMeta())
So(err, ShouldBeNil)
image13 := CreateImageWith().DefaultLayers().
ImageConfig(ispec.Image{Created: DateRef(2010, 1, 1, 12, 0, 0, 0, time.UTC)}).Build()
err = metaDB.SetRepoReference(repo1, "1.1.0", image13.AsImageMeta())
err = metaDB.SetRepoReference(context.Background(), repo1, "1.1.0", image13.AsImageMeta())
So(err, ShouldBeNil)
image14 := CreateImageWith().DefaultLayers().
ImageConfig(ispec.Image{Created: DateRef(2011, 1, 1, 12, 0, 0, 0, time.UTC)}).Build()
err = metaDB.SetRepoReference(repo1, "1.0.1", image14.AsImageMeta())
err = metaDB.SetRepoReference(context.Background(), repo1, "1.0.1", image14.AsImageMeta())
So(err, ShouldBeNil)
// Create metadb data for scannable image with no vulnerabilities
image61 := CreateImageWith().DefaultLayers().
ImageConfig(ispec.Image{Created: DateRef(2016, 1, 1, 12, 0, 0, 0, time.UTC)}).Build()
err = metaDB.SetRepoReference(repo6, "1.0.0", image61.AsImageMeta())
err = metaDB.SetRepoReference(context.Background(), repo6, "1.0.0", image61.AsImageMeta())
So(err, ShouldBeNil)
// Create metadb data for image not supporting scanning
@@ -805,50 +805,50 @@ func TestCVEStruct(t *testing.T) { //nolint:gocyclo
Digest: godigest.FromBytes([]byte{10, 10, 10}),
}}).ImageConfig(ispec.Image{Created: DateRef(2009, 1, 1, 12, 0, 0, 0, time.UTC)}).Build()
err = metaDB.SetRepoReference(repo2, "1.0.0", image21.AsImageMeta())
err = metaDB.SetRepoReference(context.Background(), repo2, "1.0.0", image21.AsImageMeta())
So(err, ShouldBeNil)
// Create metadb data for invalid images/negative tests
image := CreateRandomImage()
err = metaDB.SetRepoReference(repo3, "invalid-manifest", image.AsImageMeta())
err = metaDB.SetRepoReference(context.Background(), repo3, "invalid-manifest", image.AsImageMeta())
So(err, ShouldBeNil)
image41 := CreateImageWith().DefaultLayers().
CustomConfigBlob([]byte("invalid config blob"), ispec.MediaTypeImageConfig).Build()
err = metaDB.SetRepoReference(repo4, "invalid-config", image41.AsImageMeta())
err = metaDB.SetRepoReference(context.Background(), repo4, "invalid-config", image41.AsImageMeta())
So(err, ShouldBeNil)
digest51 := godigest.FromString("abc8")
randomImgData := CreateRandomImage().AsImageMeta()
randomImgData.Digest = digest51
randomImgData.Manifests[0].Digest = digest51
err = metaDB.SetRepoReference(repo5, "nonexitent-manifest", randomImgData)
err = metaDB.SetRepoReference(context.Background(), repo5, "nonexitent-manifest", randomImgData)
So(err, ShouldBeNil)
// Create metadb data for scannable image which errors during scan
image71 := CreateImageWith().DefaultLayers().
ImageConfig(ispec.Image{Created: DateRef(2000, 1, 1, 12, 0, 0, 0, time.UTC)}).Build()
err = metaDB.SetRepoReference(repo7, "1.0.0", image71.AsImageMeta())
err = metaDB.SetRepoReference(context.Background(), repo7, "1.0.0", image71.AsImageMeta())
So(err, ShouldBeNil)
// create multiarch image with vulnerabilities
multiarchImage := CreateRandomMultiarch()
err = metaDB.SetRepoReference(repoMultiarch, multiarchImage.Images[0].DigestStr(),
err = metaDB.SetRepoReference(context.Background(), repoMultiarch, multiarchImage.Images[0].DigestStr(),
multiarchImage.Images[0].AsImageMeta())
So(err, ShouldBeNil)
err = metaDB.SetRepoReference(repoMultiarch, multiarchImage.Images[1].DigestStr(),
err = metaDB.SetRepoReference(context.Background(), repoMultiarch, multiarchImage.Images[1].DigestStr(),
multiarchImage.Images[1].AsImageMeta())
So(err, ShouldBeNil)
err = metaDB.SetRepoReference(repoMultiarch, multiarchImage.Images[2].DigestStr(),
err = metaDB.SetRepoReference(context.Background(), repoMultiarch, multiarchImage.Images[2].DigestStr(),
multiarchImage.Images[2].AsImageMeta())
So(err, ShouldBeNil)
err = metaDB.SetRepoReference(repoMultiarch, "tagIndex", multiarchImage.AsImageMeta())
err = metaDB.SetRepoReference(context.Background(), repoMultiarch, "tagIndex", multiarchImage.AsImageMeta())
So(err, ShouldBeNil)
err = metaDB.SetRepoMeta("repo-with-bad-tag-digest", mTypes.RepoMeta{
pkg/extensions/search/cve/pagination_test.go
+3 -2
View File
@@ -4,6 +4,7 @@
package cveinfo_test
import (
"context"
"fmt"
"sort"
"testing"
@@ -41,7 +42,7 @@ func TestCVEPagination(t *testing.T) {
Blob: ispec.DescriptorEmptyJSON.Data,
}}).ImageConfig(ispec.Image{Created: &timeStamp11}).Build()
err = metaDB.SetRepoReference("repo1", "0.1.0", image.AsImageMeta())
err = metaDB.SetRepoReference(context.Background(), "repo1", "0.1.0", image.AsImageMeta())
So(err, ShouldBeNil)
timeStamp12 := time.Date(2009, 1, 1, 12, 0, 0, 0, time.UTC)
@@ -53,7 +54,7 @@ func TestCVEPagination(t *testing.T) {
Blob: ispec.DescriptorEmptyJSON.Data,
}}).ImageConfig(ispec.Image{Created: &timeStamp12}).Build()
err = metaDB.SetRepoReference("repo1", "1.0.0", image2.AsImageMeta())
err = metaDB.SetRepoReference(context.Background(), "repo1", "1.0.0", image2.AsImageMeta())
So(err, ShouldBeNil)
// MetaDB loaded with initial data, mock the scanner
pkg/extensions/search/cve/scan_test.go
+15 -14
View File
@@ -74,32 +74,32 @@ func TestScanGeneratorWithMockedData(t *testing.T) { //nolint: gocyclo
image11 := CreateImageWith().DefaultLayers().
ImageConfig(ispec.Image{Created: DateRef(2008, 1, 1, 12, 0, 0, 0, time.UTC)}).Build()
err = metaDB.SetRepoReference("repo1", "0.1.0", image11.AsImageMeta())
err = metaDB.SetRepoReference(context.Background(), "repo1", "0.1.0", image11.AsImageMeta())
So(err, ShouldBeNil)
image12 := CreateImageWith().DefaultLayers().
ImageConfig(ispec.Image{Created: DateRef(2009, 1, 1, 12, 0, 0, 0, time.UTC)}).Build()
err = metaDB.SetRepoReference("repo1", "1.0.0", image12.AsImageMeta())
err = metaDB.SetRepoReference(context.Background(), "repo1", "1.0.0", image12.AsImageMeta())
So(err, ShouldBeNil)
image13 := CreateImageWith().DefaultLayers().
ImageConfig(ispec.Image{Created: DateRef(2010, 1, 1, 12, 0, 0, 0, time.UTC)}).Build()
err = metaDB.SetRepoReference("repo1", "1.1.0", image13.AsImageMeta())
err = metaDB.SetRepoReference(context.Background(), "repo1", "1.1.0", image13.AsImageMeta())
So(err, ShouldBeNil)
image14 := CreateImageWith().DefaultLayers().
ImageConfig(ispec.Image{Created: DateRef(2011, 1, 1, 12, 0, 0, 0, time.UTC)}).Build()
err = metaDB.SetRepoReference("repo1", "1.0.1", image14.AsImageMeta())
err = metaDB.SetRepoReference(context.Background(), "repo1", "1.0.1", image14.AsImageMeta())
So(err, ShouldBeNil)
// Create metadb data for scannable image with no vulnerabilities
image61 := CreateImageWith().DefaultLayers().
ImageConfig(ispec.Image{Created: DateRef(2016, 1, 1, 12, 0, 0, 0, time.UTC)}).Build()
err = metaDB.SetRepoReference("repo6", "1.0.0", image61.AsImageMeta())
err = metaDB.SetRepoReference(context.Background(), "repo6", "1.0.0", image61.AsImageMeta())
So(err, ShouldBeNil)
// Create metadb data for image not supporting scanning
@@ -109,49 +109,50 @@ func TestScanGeneratorWithMockedData(t *testing.T) { //nolint: gocyclo
Digest: godigest.FromBytes([]byte{10, 10, 10}),
}}).ImageConfig(ispec.Image{Created: DateRef(2009, 1, 1, 12, 0, 0, 0, time.UTC)}).Build()
err = metaDB.SetRepoReference("repo2", "1.0.0", image21.AsImageMeta())
err = metaDB.SetRepoReference(context.Background(), "repo2", "1.0.0", image21.AsImageMeta())
So(err, ShouldBeNil)
// Create metadb data for invalid images/negative tests
img := CreateRandomImage()
digest31 := img.Digest()
err = metaDB.SetRepoReference("repo3", "invalid-manifest", img.AsImageMeta())
err = metaDB.SetRepoReference(context.Background(), "repo3", "invalid-manifest", img.AsImageMeta())
So(err, ShouldBeNil)
image41 := CreateImageWith().DefaultLayers().
CustomConfigBlob([]byte("invalid config blob"), ispec.MediaTypeImageConfig).Build()
err = metaDB.SetRepoReference("repo4", "invalid-config", image41.AsImageMeta())
err = metaDB.SetRepoReference(context.Background(), "repo4", "invalid-config", image41.AsImageMeta())
So(err, ShouldBeNil)
image15 := CreateRandomMultiarch()
digest51 := image15.Digest()
err = metaDB.SetRepoReference("repo5", "nonexitent-manifests-for-multiarch", image15.AsImageMeta())
err = metaDB.SetRepoReference(context.Background(), "repo5", "nonexitent-manifests-for-multiarch",
image15.AsImageMeta())
So(err, ShouldBeNil)
// Create metadb data for scannable image which errors during scan
image71 := CreateImageWith().DefaultLayers().
ImageConfig(ispec.Image{Created: DateRef(2000, 1, 1, 12, 0, 0, 0, time.UTC)}).Build()
err = metaDB.SetRepoReference("repo7", "1.0.0", image71.AsImageMeta())
err = metaDB.SetRepoReference(context.Background(), "repo7", "1.0.0", image71.AsImageMeta())
So(err, ShouldBeNil)
// Create multiarch image with vulnerabilities
multiarchImage := CreateRandomMultiarch()
err = metaDB.SetRepoReference(repoIndex, multiarchImage.Images[0].DigestStr(),
err = metaDB.SetRepoReference(context.Background(), repoIndex, multiarchImage.Images[0].DigestStr(),
multiarchImage.Images[0].AsImageMeta())
So(err, ShouldBeNil)
err = metaDB.SetRepoReference(repoIndex, multiarchImage.Images[1].DigestStr(),
err = metaDB.SetRepoReference(context.Background(), repoIndex, multiarchImage.Images[1].DigestStr(),
multiarchImage.Images[1].AsImageMeta())
So(err, ShouldBeNil)
err = metaDB.SetRepoReference(repoIndex, multiarchImage.Images[2].DigestStr(),
err = metaDB.SetRepoReference(context.Background(), repoIndex, multiarchImage.Images[2].DigestStr(),
multiarchImage.Images[2].AsImageMeta())
So(err, ShouldBeNil)
err = metaDB.SetRepoReference(repoIndex, "tagIndex", multiarchImage.AsImageMeta())
err = metaDB.SetRepoReference(context.Background(), repoIndex, "tagIndex", multiarchImage.AsImageMeta())
So(err, ShouldBeNil)
err = metaDB.SetRepoMeta("repo-with-bad-tag-digest", mTypes.RepoMeta{
pkg/extensions/search/cve/trivy/scanner_internal_test.go
+4 -2
View File
@@ -5,6 +5,7 @@ package trivy
import (
"bytes"
"context"
"encoding/json"
"os"
"path"
@@ -299,7 +300,7 @@ func TestImageScannable(t *testing.T) {
Blob: ispec.DescriptorEmptyJSON.Data,
}}).ImageConfig(validConfig).Build()
err = metaDB.SetRepoReference("repo1", "valid", validImage.AsImageMeta())
err = metaDB.SetRepoReference(context.Background(), "repo1", "valid", validImage.AsImageMeta())
if err != nil {
panic(err)
}
@@ -312,7 +313,8 @@ func TestImageScannable(t *testing.T) {
Blob: ispec.DescriptorEmptyJSON.Data,
}}).ImageConfig(validConfig).Build()
err = metaDB.SetRepoReference("repo1", "unscannable-layer", imageWithUnscannableLayer.AsImageMeta())
err = metaDB.SetRepoReference(context.Background(), "repo1",
"unscannable-layer", imageWithUnscannableLayer.AsImageMeta())
if err != nil {
panic(err)
}
pkg/extensions/search/search_test.go
+2 -2
View File
@@ -4484,7 +4484,7 @@ func TestMetaDBWhenPushingImages(t *testing.T) {
Convey("SetManifestMeta succeeds but SetRepoReference fails", func() {
ctlr.MetaDB = mocks.MetaDBMock{
SetRepoReferenceFn: func(repo, reference string, imageMeta mTypes.ImageMeta) error {
SetRepoReferenceFn: func(ctx context.Context, repo, reference string, imageMeta mTypes.ImageMeta) error {
return ErrTestError
},
}
@@ -5196,7 +5196,7 @@ func TestMetaDBWhenReadingImages(t *testing.T) {
Convey("Error when incrementing", func() {
ctlr.MetaDB = mocks.MetaDBMock{
IncrementImageDownloadsFn: func(repo string, tag string) error {
UpdateStatsOnDownloadFn: func(repo string, tag string) error {
return ErrTestError
},
}
pkg/extensions/sync/local.go
+3 -2
View File
@@ -4,6 +4,7 @@
package sync
import (
"context"
"encoding/json"
"errors"
"fmt"
@@ -164,7 +165,7 @@ func (registry *LocalRegistry) CommitImage(imageReference types.ImageReference,
}
if registry.metaDB != nil {
err = meta.SetImageMetaFromInput(repo, reference, mediaType,
err = meta.SetImageMetaFromInput(context.Background(), repo, reference, mediaType,
manifestDigest, manifestBlob, imageStore, registry.metaDB, registry.log)
if err != nil {
return fmt.Errorf("metaDB: failed to set metadata for image '%s %s': %w", repo, reference, err)
@@ -222,7 +223,7 @@ func (registry *LocalRegistry) copyManifest(repo string, manifestContent []byte,
}
if registry.metaDB != nil {
err = meta.SetImageMetaFromInput(repo, reference, ispec.MediaTypeImageManifest,
err = meta.SetImageMetaFromInput(context.Background(), repo, reference, ispec.MediaTypeImageManifest,
digest, manifestContent, imageStore, registry.metaDB, registry.log)
if err != nil {
registry.log.Error().Str("errorType", common.TypeOf(err)).
pkg/extensions/sync/references/cosign.go
+1 -1
View File
@@ -153,7 +153,7 @@ func (ref CosignReference) SyncReferences(ctx context.Context, localRepo, remote
ref.log.Debug().Str("repository", localRepo).Str("subject", subjectDigestStr).
Msg("metaDB: trying to sync cosign reference for image")
err = meta.SetImageMetaFromInput(localRepo, cosignTag, ispec.MediaTypeImageManifest,
err = meta.SetImageMetaFromInput(ctx, localRepo, cosignTag, ispec.MediaTypeImageManifest,
referenceDigest, manifestBuf, ref.storeController.GetImageStore(localRepo),
ref.metaDB, ref.log)
pkg/extensions/sync/references/oci.go
+1 -1
View File
@@ -137,7 +137,7 @@ func (ref OciReferences) SyncReferences(ctx context.Context, localRepo, remoteRe
ref.log.Debug().Str("repository", localRepo).Str("subject", subjectDigestStr).
Msg("metaDB: trying to add oci references for image")
err = meta.SetImageMetaFromInput(localRepo, referenceDigest.String(), referrer.MediaType,
err = meta.SetImageMetaFromInput(ctx, localRepo, referenceDigest.String(), referrer.MediaType,
referenceDigest, referenceBuf, ref.storeController.GetImageStore(localRepo),
ref.metaDB, ref.log)
if err != nil {
pkg/extensions/sync/references/oras.go
+2 -1
View File
@@ -154,7 +154,8 @@ func (ref ORASReferences) SyncReferences(ctx context.Context, localRepo, remoteR
ref.log.Debug().Str("repository", localRepo).Str("subject", subjectDigestStr).
Msg("metaDB: trying to sync oras artifact for image")
err := meta.SetImageMetaFromInput(localRepo, referenceDigest.String(), referrer.MediaType,
err := meta.SetImageMetaFromInput(context.Background(), localRepo, //nolint:contextcheck
referenceDigest.String(), referrer.MediaType,
referenceDigest, orasBuf, ref.storeController.GetImageStore(localRepo),
ref.metaDB, ref.log)
if err != nil {
pkg/extensions/sync/references/referrers_tag.go
+1 -1
View File
@@ -113,7 +113,7 @@ func (ref TagReferences) SyncReferences(ctx context.Context, localRepo, remoteRe
ref.log.Debug().Str("repository", localRepo).Str("subject", subjectDigestStr).
Msg("metaDB: trying to add oci references for image")
err = meta.SetImageMetaFromInput(localRepo, referenceDigest.String(), referrer.MediaType,
err = meta.SetImageMetaFromInput(ctx, localRepo, referenceDigest.String(), referrer.MediaType,
referenceDigest, referenceBuf, ref.storeController.GetImageStore(localRepo),
ref.metaDB, ref.log)
if err != nil {
pkg/extensions/sync/sync_internal_test.go
+2 -2
View File
@@ -337,7 +337,7 @@ func TestLocalRegistry(t *testing.T) {
Convey("trigger metaDB error on index manifest in CommitImage()", func() {
registry := NewLocalRegistry(storage.StoreController{DefaultStore: syncImgStore}, mocks.MetaDBMock{
SetRepoReferenceFn: func(repo string, reference string, imageMeta mTypes.ImageMeta) error {
SetRepoReferenceFn: func(ctx context.Context, repo string, reference string, imageMeta mTypes.ImageMeta) error {
if reference == "1.0" {
return zerr.ErrRepoMetaNotFound
}
@@ -352,7 +352,7 @@ func TestLocalRegistry(t *testing.T) {
Convey("trigger metaDB error on image manifest in CommitImage()", func() {
registry := NewLocalRegistry(storage.StoreController{DefaultStore: syncImgStore}, mocks.MetaDBMock{
SetRepoReferenceFn: func(repo, reference string, imageMeta mTypes.ImageMeta) error {
SetRepoReferenceFn: func(ctx context.Context, repo, reference string, imageMeta mTypes.ImageMeta) error {
return zerr.ErrRepoMetaNotFound
},
}, log)
pkg/extensions/sync/sync_test.go
+2 -2
View File
@@ -877,7 +877,7 @@ func TestOnDemand(t *testing.T) {
return nil
},
SetRepoReferenceFn: func(repo, reference string, imageMeta mTypes.ImageMeta) error {
SetRepoReferenceFn: func(ctx context.Context, repo, reference string, imageMeta mTypes.ImageMeta) error {
if strings.HasPrefix(reference, "sha256-") &&
(strings.HasSuffix(reference, remote.SignatureTagSuffix) ||
strings.HasSuffix(reference, remote.SBOMTagSuffix)) ||
@@ -1017,7 +1017,7 @@ func TestOnDemand(t *testing.T) {
// metadb fails for syncReferrersTag"
dctlr.MetaDB = mocks.MetaDBMock{
SetRepoReferenceFn: func(repo, reference string, imageMeta mTypes.ImageMeta) error {
SetRepoReferenceFn: func(ctx context.Context, repo, reference string, imageMeta mTypes.ImageMeta) error {
if imageMeta.Digest.String() == ociRefImage.ManifestDescriptor.Digest.String() {
return sync.ErrTestError
}