github/zot
1
0
Fork 0
mirror of https://github.com/project-zot/zot.git synced 2026-06-17 12:58:02 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(retention): added image retention policies (#1866)

Browse Source 
feat(metaDB): add more image statistics info

Signed-off-by: Petu Eusebiu <peusebiu@cisco.com>
This commit is contained in:
peusebiu
2023-11-01 18:16:18 +02:00
committed by GitHub
parent a79d79a03a
commit 9074f8483b
71 changed files with 3454 additions and 745 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/extensions/search/convert/convert_internal_test.go
+1 -1
View File
@@ -39,7 +39,7 @@ func TestCVEConvert(t *testing.T) {
Blob: ispec.DescriptorEmptyJSON.Data,
}}).DefaultConfig().Build()
err = metaDB.SetRepoReference("repo1", "0.1.0", image.AsImageMeta())
err = metaDB.SetRepoReference(context.Background(), "repo1", "0.1.0", image.AsImageMeta())
So(err, ShouldBeNil)
repoMetaList, err := metaDB.SearchRepos(context.Background(), "")
pkg/extensions/search/cve/cve_test.go
+14 -14
View File
@@ -770,32 +770,32 @@ func TestCVEStruct(t *testing.T) { //nolint:gocyclo
image11 := CreateImageWith().DefaultLayers().
ImageConfig(ispec.Image{Created: DateRef(2008, 1, 1, 12, 0, 0, 0, time.UTC)}).Build()
err = metaDB.SetRepoReference(repo1, "0.1.0", image11.AsImageMeta())
err = metaDB.SetRepoReference(context.Background(), repo1, "0.1.0", image11.AsImageMeta())
So(err, ShouldBeNil)
image12 := CreateImageWith().DefaultLayers().
ImageConfig(ispec.Image{Created: DateRef(2009, 1, 1, 12, 0, 0, 0, time.UTC)}).Build()
err = metaDB.SetRepoReference(repo1, "1.0.0", image12.AsImageMeta())
err = metaDB.SetRepoReference(context.Background(), repo1, "1.0.0", image12.AsImageMeta())
So(err, ShouldBeNil)
image13 := CreateImageWith().DefaultLayers().
ImageConfig(ispec.Image{Created: DateRef(2010, 1, 1, 12, 0, 0, 0, time.UTC)}).Build()
err = metaDB.SetRepoReference(repo1, "1.1.0", image13.AsImageMeta())
err = metaDB.SetRepoReference(context.Background(), repo1, "1.1.0", image13.AsImageMeta())
So(err, ShouldBeNil)
image14 := CreateImageWith().DefaultLayers().
ImageConfig(ispec.Image{Created: DateRef(2011, 1, 1, 12, 0, 0, 0, time.UTC)}).Build()
err = metaDB.SetRepoReference(repo1, "1.0.1", image14.AsImageMeta())
err = metaDB.SetRepoReference(context.Background(), repo1, "1.0.1", image14.AsImageMeta())
So(err, ShouldBeNil)
// Create metadb data for scannable image with no vulnerabilities
image61 := CreateImageWith().DefaultLayers().
ImageConfig(ispec.Image{Created: DateRef(2016, 1, 1, 12, 0, 0, 0, time.UTC)}).Build()
err = metaDB.SetRepoReference(repo6, "1.0.0", image61.AsImageMeta())
err = metaDB.SetRepoReference(context.Background(), repo6, "1.0.0", image61.AsImageMeta())
So(err, ShouldBeNil)
// Create metadb data for image not supporting scanning
@@ -805,50 +805,50 @@ func TestCVEStruct(t *testing.T) { //nolint:gocyclo
Digest: godigest.FromBytes([]byte{10, 10, 10}),
}}).ImageConfig(ispec.Image{Created: DateRef(2009, 1, 1, 12, 0, 0, 0, time.UTC)}).Build()
err = metaDB.SetRepoReference(repo2, "1.0.0", image21.AsImageMeta())
err = metaDB.SetRepoReference(context.Background(), repo2, "1.0.0", image21.AsImageMeta())
So(err, ShouldBeNil)
// Create metadb data for invalid images/negative tests
image := CreateRandomImage()
err = metaDB.SetRepoReference(repo3, "invalid-manifest", image.AsImageMeta())
err = metaDB.SetRepoReference(context.Background(), repo3, "invalid-manifest", image.AsImageMeta())
So(err, ShouldBeNil)
image41 := CreateImageWith().DefaultLayers().
CustomConfigBlob([]byte("invalid config blob"), ispec.MediaTypeImageConfig).Build()
err = metaDB.SetRepoReference(repo4, "invalid-config", image41.AsImageMeta())
err = metaDB.SetRepoReference(context.Background(), repo4, "invalid-config", image41.AsImageMeta())
So(err, ShouldBeNil)
digest51 := godigest.FromString("abc8")
randomImgData := CreateRandomImage().AsImageMeta()
randomImgData.Digest = digest51
randomImgData.Manifests[0].Digest = digest51
err = metaDB.SetRepoReference(repo5, "nonexitent-manifest", randomImgData)
err = metaDB.SetRepoReference(context.Background(), repo5, "nonexitent-manifest", randomImgData)
So(err, ShouldBeNil)
// Create metadb data for scannable image which errors during scan
image71 := CreateImageWith().DefaultLayers().
ImageConfig(ispec.Image{Created: DateRef(2000, 1, 1, 12, 0, 0, 0, time.UTC)}).Build()
err = metaDB.SetRepoReference(repo7, "1.0.0", image71.AsImageMeta())
err = metaDB.SetRepoReference(context.Background(), repo7, "1.0.0", image71.AsImageMeta())
So(err, ShouldBeNil)
// create multiarch image with vulnerabilities
multiarchImage := CreateRandomMultiarch()
err = metaDB.SetRepoReference(repoMultiarch, multiarchImage.Images[0].DigestStr(),
err = metaDB.SetRepoReference(context.Background(), repoMultiarch, multiarchImage.Images[0].DigestStr(),
multiarchImage.Images[0].AsImageMeta())
So(err, ShouldBeNil)
err = metaDB.SetRepoReference(repoMultiarch, multiarchImage.Images[1].DigestStr(),
err = metaDB.SetRepoReference(context.Background(), repoMultiarch, multiarchImage.Images[1].DigestStr(),
multiarchImage.Images[1].AsImageMeta())
So(err, ShouldBeNil)
err = metaDB.SetRepoReference(repoMultiarch, multiarchImage.Images[2].DigestStr(),
err = metaDB.SetRepoReference(context.Background(), repoMultiarch, multiarchImage.Images[2].DigestStr(),
multiarchImage.Images[2].AsImageMeta())
So(err, ShouldBeNil)
err = metaDB.SetRepoReference(repoMultiarch, "tagIndex", multiarchImage.AsImageMeta())
err = metaDB.SetRepoReference(context.Background(), repoMultiarch, "tagIndex", multiarchImage.AsImageMeta())
So(err, ShouldBeNil)
err = metaDB.SetRepoMeta("repo-with-bad-tag-digest", mTypes.RepoMeta{
pkg/extensions/search/cve/pagination_test.go
+3 -2
View File
@@ -4,6 +4,7 @@
package cveinfo_test
import (
"context"
"fmt"
"sort"
"testing"
@@ -41,7 +42,7 @@ func TestCVEPagination(t *testing.T) {
Blob: ispec.DescriptorEmptyJSON.Data,
}}).ImageConfig(ispec.Image{Created: &timeStamp11}).Build()
err = metaDB.SetRepoReference("repo1", "0.1.0", image.AsImageMeta())
err = metaDB.SetRepoReference(context.Background(), "repo1", "0.1.0", image.AsImageMeta())
So(err, ShouldBeNil)
timeStamp12 := time.Date(2009, 1, 1, 12, 0, 0, 0, time.UTC)
@@ -53,7 +54,7 @@ func TestCVEPagination(t *testing.T) {
Blob: ispec.DescriptorEmptyJSON.Data,
}}).ImageConfig(ispec.Image{Created: &timeStamp12}).Build()
err = metaDB.SetRepoReference("repo1", "1.0.0", image2.AsImageMeta())
err = metaDB.SetRepoReference(context.Background(), "repo1", "1.0.0", image2.AsImageMeta())
So(err, ShouldBeNil)
// MetaDB loaded with initial data, mock the scanner
pkg/extensions/search/cve/scan_test.go
+15 -14
View File
@@ -74,32 +74,32 @@ func TestScanGeneratorWithMockedData(t *testing.T) { //nolint: gocyclo
image11 := CreateImageWith().DefaultLayers().
ImageConfig(ispec.Image{Created: DateRef(2008, 1, 1, 12, 0, 0, 0, time.UTC)}).Build()
err = metaDB.SetRepoReference("repo1", "0.1.0", image11.AsImageMeta())
err = metaDB.SetRepoReference(context.Background(), "repo1", "0.1.0", image11.AsImageMeta())
So(err, ShouldBeNil)
image12 := CreateImageWith().DefaultLayers().
ImageConfig(ispec.Image{Created: DateRef(2009, 1, 1, 12, 0, 0, 0, time.UTC)}).Build()
err = metaDB.SetRepoReference("repo1", "1.0.0", image12.AsImageMeta())
err = metaDB.SetRepoReference(context.Background(), "repo1", "1.0.0", image12.AsImageMeta())
So(err, ShouldBeNil)
image13 := CreateImageWith().DefaultLayers().
ImageConfig(ispec.Image{Created: DateRef(2010, 1, 1, 12, 0, 0, 0, time.UTC)}).Build()
err = metaDB.SetRepoReference("repo1", "1.1.0", image13.AsImageMeta())
err = metaDB.SetRepoReference(context.Background(), "repo1", "1.1.0", image13.AsImageMeta())
So(err, ShouldBeNil)
image14 := CreateImageWith().DefaultLayers().
ImageConfig(ispec.Image{Created: DateRef(2011, 1, 1, 12, 0, 0, 0, time.UTC)}).Build()
err = metaDB.SetRepoReference("repo1", "1.0.1", image14.AsImageMeta())
err = metaDB.SetRepoReference(context.Background(), "repo1", "1.0.1", image14.AsImageMeta())
So(err, ShouldBeNil)
// Create metadb data for scannable image with no vulnerabilities
image61 := CreateImageWith().DefaultLayers().
ImageConfig(ispec.Image{Created: DateRef(2016, 1, 1, 12, 0, 0, 0, time.UTC)}).Build()
err = metaDB.SetRepoReference("repo6", "1.0.0", image61.AsImageMeta())
err = metaDB.SetRepoReference(context.Background(), "repo6", "1.0.0", image61.AsImageMeta())
So(err, ShouldBeNil)
// Create metadb data for image not supporting scanning
@@ -109,49 +109,50 @@ func TestScanGeneratorWithMockedData(t *testing.T) { //nolint: gocyclo
Digest: godigest.FromBytes([]byte{10, 10, 10}),
}}).ImageConfig(ispec.Image{Created: DateRef(2009, 1, 1, 12, 0, 0, 0, time.UTC)}).Build()
err = metaDB.SetRepoReference("repo2", "1.0.0", image21.AsImageMeta())
err = metaDB.SetRepoReference(context.Background(), "repo2", "1.0.0", image21.AsImageMeta())
So(err, ShouldBeNil)
// Create metadb data for invalid images/negative tests
img := CreateRandomImage()
digest31 := img.Digest()
err = metaDB.SetRepoReference("repo3", "invalid-manifest", img.AsImageMeta())
err = metaDB.SetRepoReference(context.Background(), "repo3", "invalid-manifest", img.AsImageMeta())
So(err, ShouldBeNil)
image41 := CreateImageWith().DefaultLayers().
CustomConfigBlob([]byte("invalid config blob"), ispec.MediaTypeImageConfig).Build()
err = metaDB.SetRepoReference("repo4", "invalid-config", image41.AsImageMeta())
err = metaDB.SetRepoReference(context.Background(), "repo4", "invalid-config", image41.AsImageMeta())
So(err, ShouldBeNil)
image15 := CreateRandomMultiarch()
digest51 := image15.Digest()
err = metaDB.SetRepoReference("repo5", "nonexitent-manifests-for-multiarch", image15.AsImageMeta())
err = metaDB.SetRepoReference(context.Background(), "repo5", "nonexitent-manifests-for-multiarch",
image15.AsImageMeta())
So(err, ShouldBeNil)
// Create metadb data for scannable image which errors during scan
image71 := CreateImageWith().DefaultLayers().
ImageConfig(ispec.Image{Created: DateRef(2000, 1, 1, 12, 0, 0, 0, time.UTC)}).Build()
err = metaDB.SetRepoReference("repo7", "1.0.0", image71.AsImageMeta())
err = metaDB.SetRepoReference(context.Background(), "repo7", "1.0.0", image71.AsImageMeta())
So(err, ShouldBeNil)
// Create multiarch image with vulnerabilities
multiarchImage := CreateRandomMultiarch()
err = metaDB.SetRepoReference(repoIndex, multiarchImage.Images[0].DigestStr(),
err = metaDB.SetRepoReference(context.Background(), repoIndex, multiarchImage.Images[0].DigestStr(),
multiarchImage.Images[0].AsImageMeta())
So(err, ShouldBeNil)
err = metaDB.SetRepoReference(repoIndex, multiarchImage.Images[1].DigestStr(),
err = metaDB.SetRepoReference(context.Background(), repoIndex, multiarchImage.Images[1].DigestStr(),
multiarchImage.Images[1].AsImageMeta())
So(err, ShouldBeNil)
err = metaDB.SetRepoReference(repoIndex, multiarchImage.Images[2].DigestStr(),
err = metaDB.SetRepoReference(context.Background(), repoIndex, multiarchImage.Images[2].DigestStr(),
multiarchImage.Images[2].AsImageMeta())
So(err, ShouldBeNil)
err = metaDB.SetRepoReference(repoIndex, "tagIndex", multiarchImage.AsImageMeta())
err = metaDB.SetRepoReference(context.Background(), repoIndex, "tagIndex", multiarchImage.AsImageMeta())
So(err, ShouldBeNil)
err = metaDB.SetRepoMeta("repo-with-bad-tag-digest", mTypes.RepoMeta{
pkg/extensions/search/cve/trivy/scanner_internal_test.go
+4 -2
View File
@@ -5,6 +5,7 @@ package trivy
import (
"bytes"
"context"
"encoding/json"
"os"
"path"
@@ -299,7 +300,7 @@ func TestImageScannable(t *testing.T) {
Blob: ispec.DescriptorEmptyJSON.Data,
}}).ImageConfig(validConfig).Build()
err = metaDB.SetRepoReference("repo1", "valid", validImage.AsImageMeta())
err = metaDB.SetRepoReference(context.Background(), "repo1", "valid", validImage.AsImageMeta())
if err != nil {
panic(err)
}
@@ -312,7 +313,8 @@ func TestImageScannable(t *testing.T) {
Blob: ispec.DescriptorEmptyJSON.Data,
}}).ImageConfig(validConfig).Build()
err = metaDB.SetRepoReference("repo1", "unscannable-layer", imageWithUnscannableLayer.AsImageMeta())
err = metaDB.SetRepoReference(context.Background(), "repo1",
"unscannable-layer", imageWithUnscannableLayer.AsImageMeta())
if err != nil {
panic(err)
}
pkg/extensions/search/search_test.go
+2 -2
View File
@@ -4484,7 +4484,7 @@ func TestMetaDBWhenPushingImages(t *testing.T) {
Convey("SetManifestMeta succeeds but SetRepoReference fails", func() {
ctlr.MetaDB = mocks.MetaDBMock{
SetRepoReferenceFn: func(repo, reference string, imageMeta mTypes.ImageMeta) error {
SetRepoReferenceFn: func(ctx context.Context, repo, reference string, imageMeta mTypes.ImageMeta) error {
return ErrTestError
},
}
@@ -5196,7 +5196,7 @@ func TestMetaDBWhenReadingImages(t *testing.T) {
Convey("Error when incrementing", func() {
ctlr.MetaDB = mocks.MetaDBMock{
IncrementImageDownloadsFn: func(repo string, tag string) error {
UpdateStatsOnDownloadFn: func(repo string, tag string) error {
return ErrTestError
},
}