github/zot
1
0
Fork 0
mirror of https://github.com/project-zot/zot.git synced 2026-06-16 04:17:55 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(retention): added image retention policies (#1866)

Browse Source 
feat(metaDB): add more image statistics info

Signed-off-by: Petu Eusebiu <peusebiu@cisco.com>
This commit is contained in:
peusebiu
2023-11-01 18:16:18 +02:00
committed by GitHub
parent a79d79a03a
commit 9074f8483b
71 changed files with 3454 additions and 745 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/api/config/config.go
+72 -14
View File
@@ -23,17 +23,37 @@ var (
)
type StorageConfig struct {
RootDirectory string
Dedupe bool
RemoteCache bool
GC bool
Commit bool
GCDelay time.Duration
GCInterval time.Duration
GCReferrers bool
UntaggedImageRetentionDelay time.Duration
StorageDriver map[string]interface{} `mapstructure:",omitempty"`
CacheDriver map[string]interface{} `mapstructure:",omitempty"`
RootDirectory string
Dedupe bool
RemoteCache bool
GC bool
Commit bool
GCDelay time.Duration // applied for blobs
GCInterval time.Duration
Retention ImageRetention
StorageDriver map[string]interface{} `mapstructure:",omitempty"`
CacheDriver map[string]interface{} `mapstructure:",omitempty"`
}
type ImageRetention struct {
DryRun bool
Delay time.Duration // applied for referrers and untagged
Policies []RetentionPolicy
}
type RetentionPolicy struct {
Repositories []string
DeleteReferrers bool
DeleteUntagged *bool
KeepTags []KeepTagsPolicy
}
type KeepTagsPolicy struct {
Patterns []string
PulledWithin *time.Duration
PushedWithin *time.Duration
MostRecentlyPushedCount int
MostRecentlyPulledCount int
}
type TLSConfig struct {
@@ -195,9 +215,11 @@ func New() *Config {
BinaryType: BinaryType,
Storage: GlobalStorageConfig{
StorageConfig: StorageConfig{
GC: true, GCReferrers: true, GCDelay: storageConstants.DefaultGCDelay,
UntaggedImageRetentionDelay: storageConstants.DefaultUntaggedImgeRetentionDelay,
GCInterval: storageConstants.DefaultGCInterval, Dedupe: true,
Dedupe: true,
GC: true,
GCDelay: storageConstants.DefaultGCDelay,
GCInterval: storageConstants.DefaultGCInterval,
Retention: ImageRetention{},
},
},
HTTP: HTTPConfig{Address: "127.0.0.1", Port: "8080", Auth: &AuthConfig{FailDelay: 0}},
@@ -373,6 +395,42 @@ func (c *Config) IsImageTrustEnabled() bool {
return c.Extensions != nil && c.Extensions.Trust != nil && *c.Extensions.Trust.Enable
}
// check if tags retention is enabled.
func (c *Config) IsRetentionEnabled() bool {
var needsMetaDB bool
for _, retentionPolicy := range c.Storage.Retention.Policies {
for _, tagRetentionPolicy := range retentionPolicy.KeepTags {
if c.isTagsRetentionEnabled(tagRetentionPolicy) {
needsMetaDB = true
}
}
}
for _, subpath := range c.Storage.SubPaths {
for _, retentionPolicy := range subpath.Retention.Policies {
for _, tagRetentionPolicy := range retentionPolicy.KeepTags {
if c.isTagsRetentionEnabled(tagRetentionPolicy) {
needsMetaDB = true
}
}
}
}
return needsMetaDB
}
func (c *Config) isTagsRetentionEnabled(tagRetentionPolicy KeepTagsPolicy) bool {
if tagRetentionPolicy.MostRecentlyPulledCount != 0 ||
tagRetentionPolicy.MostRecentlyPushedCount != 0 ||
tagRetentionPolicy.PulledWithin != nil ||
tagRetentionPolicy.PushedWithin != nil {
return true
}
return false
}
func (c *Config) IsCosignEnabled() bool {
return c.IsImageTrustEnabled() && c.Extensions.Trust.Cosign
}
pkg/api/config/config_test.go
+45
View File
@@ -65,6 +65,7 @@ func TestConfig(t *testing.T) {
So(err, ShouldBeNil)
So(isSame, ShouldBeTrue)
})
Convey("Test DeepCopy() & Sanitize()", t, func() {
conf := config.New()
So(conf, ShouldNotBeNil)
@@ -81,4 +82,48 @@ func TestConfig(t *testing.T) {
err = config.DeepCopy(obj, conf)
So(err, ShouldNotBeNil)
})
Convey("Test IsRetentionEnabled()", t, func() {
conf := config.New()
So(conf.IsRetentionEnabled(), ShouldBeFalse)
conf.Storage.Retention.Policies = []config.RetentionPolicy{
{
Repositories: []string{"repo"},
},
}
So(conf.IsRetentionEnabled(), ShouldBeFalse)
policies := []config.RetentionPolicy{
{
Repositories: []string{"repo"},
KeepTags: []config.KeepTagsPolicy{
{
Patterns: []string{"tag"},
MostRecentlyPulledCount: 2,
},
},
},
}
conf.Storage.Retention = config.ImageRetention{
Policies: policies,
}
So(conf.IsRetentionEnabled(), ShouldBeTrue)
subPaths := make(map[string]config.StorageConfig)
subPaths["/a"] = config.StorageConfig{
GC: true,
Retention: config.ImageRetention{
Policies: policies,
},
}
conf.Storage.SubPaths = subPaths
So(conf.IsRetentionEnabled(), ShouldBeTrue)
})
}
pkg/api/controller.go
+29 -10
View File
@@ -277,7 +277,8 @@ func (c *Controller) initCookieStore() error {
func (c *Controller) InitMetaDB(reloadCtx context.Context) error {
// init metaDB if search is enabled or we need to store user profiles, api keys or signatures
if c.Config.IsSearchEnabled() || c.Config.IsBasicAuthnEnabled() || c.Config.IsImageTrustEnabled() {
if c.Config.IsSearchEnabled() || c.Config.IsBasicAuthnEnabled() || c.Config.IsImageTrustEnabled() ||
c.Config.IsRetentionEnabled() {
driver, err := meta.New(c.Config.Storage.StorageConfig, c.Log) //nolint:contextcheck
if err != nil {
return err
@@ -293,7 +294,7 @@ func (c *Controller) InitMetaDB(reloadCtx context.Context) error {
return err
}
err = meta.ParseStorage(driver, c.StoreController, c.Log)
err = meta.ParseStorage(driver, c.StoreController, c.Log) //nolint: contextcheck
if err != nil {
return err
}
@@ -309,10 +310,30 @@ func (c *Controller) LoadNewConfig(reloadCtx context.Context, newConfig *config.
c.Config.HTTP.AccessControl = newConfig.HTTP.AccessControl
// reload periodical gc config
c.Config.Storage.GCInterval = newConfig.Storage.GCInterval
c.Config.Storage.GC = newConfig.Storage.GC
c.Config.Storage.Dedupe = newConfig.Storage.Dedupe
c.Config.Storage.GCDelay = newConfig.Storage.GCDelay
c.Config.Storage.GCReferrers = newConfig.Storage.GCReferrers
c.Config.Storage.GCInterval = newConfig.Storage.GCInterval
// only if we have a metaDB already in place
if c.Config.IsRetentionEnabled() {
c.Config.Storage.Retention = newConfig.Storage.Retention
}
for subPath, storageConfig := range newConfig.Storage.SubPaths {
subPathConfig, ok := c.Config.Storage.SubPaths[subPath]
if ok {
subPathConfig.GC = storageConfig.GC
subPathConfig.Dedupe = storageConfig.Dedupe
subPathConfig.GCDelay = storageConfig.GCDelay
subPathConfig.GCInterval = storageConfig.GCInterval
// only if we have a metaDB already in place
if c.Config.IsRetentionEnabled() {
subPathConfig.Retention = storageConfig.Retention
}
c.Config.Storage.SubPaths[subPath] = subPathConfig
}
}
// reload background tasks
if newConfig.Extensions != nil {
@@ -356,10 +377,9 @@ func (c *Controller) StartBackgroundTasks(reloadCtx context.Context) {
// Enable running garbage-collect periodically for DefaultStore
if c.Config.Storage.GC {
gc := gc.NewGarbageCollect(c.StoreController.DefaultStore, c.MetaDB, gc.Options{
Referrers: c.Config.Storage.GCReferrers,
Delay: c.Config.Storage.GCDelay,
RetentionDelay: c.Config.Storage.UntaggedImageRetentionDelay,
}, c.Log)
ImageRetention: c.Config.Storage.Retention,
}, c.Audit, c.Log)
gc.CleanImageStorePeriodically(c.Config.Storage.GCInterval, taskScheduler)
}
@@ -383,10 +403,9 @@ func (c *Controller) StartBackgroundTasks(reloadCtx context.Context) {
if storageConfig.GC {
gc := gc.NewGarbageCollect(c.StoreController.SubStore[route], c.MetaDB,
gc.Options{
Referrers: storageConfig.GCReferrers,
Delay: storageConfig.GCDelay,
RetentionDelay: storageConfig.UntaggedImageRetentionDelay,
}, c.Log)
ImageRetention: storageConfig.Retention,
}, c.Audit, c.Log)
gc.CleanImageStorePeriodically(storageConfig.GCInterval, taskScheduler)
}
pkg/api/controller_test.go
+46 -14
View File
@@ -5122,6 +5122,7 @@ func TestHardLink(t *testing.T) {
port := test.GetFreePort()
conf := config.New()
conf.HTTP.Port = port
conf.Storage.GC = false
dir := t.TempDir()
@@ -7781,6 +7782,8 @@ func TestGCSignaturesAndUntaggedManifestsWithMetaDB(t *testing.T) {
ctx := context.Background()
Convey("Make controller", t, func() {
trueVal := true
Convey("Garbage collect signatures without subject and manifests without tags", func(c C) {
repoName := "testrepo" //nolint:goconst
tag := "0.0.1"
@@ -7790,6 +7793,11 @@ func TestGCSignaturesAndUntaggedManifestsWithMetaDB(t *testing.T) {
conf := config.New()
conf.HTTP.Port = port
logFile, err := os.CreateTemp("", "zot-log*.txt")
So(err, ShouldBeNil)
conf.Log.Audit = logFile.Name()
value := true
searchConfig := &extconf.SearchConfig{
BaseConfig: extconf.BaseConfig{Enable: &value},
@@ -7806,7 +7814,21 @@ func TestGCSignaturesAndUntaggedManifestsWithMetaDB(t *testing.T) {
ctlr.Config.Storage.RootDirectory = dir
ctlr.Config.Storage.GC = true
ctlr.Config.Storage.GCDelay = 1 * time.Millisecond
ctlr.Config.Storage.UntaggedImageRetentionDelay = 1 * time.Millisecond
ctlr.Config.Storage.Retention = config.ImageRetention{
Delay: 1 * time.Millisecond,
Policies: []config.RetentionPolicy{
{
Repositories: []string{"**"},
DeleteReferrers: true,
DeleteUntagged: &trueVal,
KeepTags: []config.KeepTagsPolicy{
{
Patterns: []string{".*"}, // just for coverage
},
},
},
},
}
ctlr.Config.Storage.Dedupe = false
@@ -7817,16 +7839,14 @@ func TestGCSignaturesAndUntaggedManifestsWithMetaDB(t *testing.T) {
img := CreateDefaultImage()
err := UploadImage(img, baseURL, repoName, tag)
err = UploadImage(img, baseURL, repoName, tag)
So(err, ShouldBeNil)
gc := gc.NewGarbageCollect(ctlr.StoreController.DefaultStore, ctlr.MetaDB,
gc.Options{
Referrers: ctlr.Config.Storage.GCReferrers,
Delay: ctlr.Config.Storage.GCDelay,
RetentionDelay: ctlr.Config.Storage.UntaggedImageRetentionDelay,
},
ctlr.Log)
ImageRetention: ctlr.Config.Storage.Retention,
}, ctlr.Audit, ctlr.Log)
resp, err := resty.R().Get(baseURL + fmt.Sprintf("/v2/%s/manifests/%s", repoName, tag))
So(err, ShouldBeNil)
@@ -7897,7 +7917,7 @@ func TestGCSignaturesAndUntaggedManifestsWithMetaDB(t *testing.T) {
So(len(index.Manifests), ShouldEqual, 1)
// shouldn't do anything
err = gc.CleanRepo(repoName)
err = gc.CleanRepo(repoName) //nolint: contextcheck
So(err, ShouldBeNil)
// make sure both signatures are stored in repodb
@@ -7988,7 +8008,7 @@ func TestGCSignaturesAndUntaggedManifestsWithMetaDB(t *testing.T) {
So(err, ShouldBeNil)
newManifestDigest := godigest.FromBytes(manifestBuf)
err = gc.CleanRepo(repoName)
err = gc.CleanRepo(repoName) //nolint: contextcheck
So(err, ShouldBeNil)
// make sure both signatures are removed from metaDB and repo reference for untagged is removed
@@ -8051,7 +8071,16 @@ func TestGCSignaturesAndUntaggedManifestsWithMetaDB(t *testing.T) {
ctlr.Config.Storage.RootDirectory = dir
ctlr.Config.Storage.GC = true
ctlr.Config.Storage.GCDelay = 1 * time.Second
ctlr.Config.Storage.UntaggedImageRetentionDelay = 1 * time.Second
ctlr.Config.Storage.Retention = config.ImageRetention{
Delay: 1 * time.Second,
Policies: []config.RetentionPolicy{
{
Repositories: []string{"**"},
DeleteReferrers: true,
DeleteUntagged: &trueVal,
},
},
}
err := WriteImageToFileSystem(CreateDefaultImage(), repoName, tag,
ociutils.GetDefaultStoreController(dir, ctlr.Log))
@@ -8063,10 +8092,9 @@ func TestGCSignaturesAndUntaggedManifestsWithMetaDB(t *testing.T) {
gc := gc.NewGarbageCollect(ctlr.StoreController.DefaultStore, ctlr.MetaDB,
gc.Options{
Referrers: ctlr.Config.Storage.GCReferrers,
Delay: ctlr.Config.Storage.GCDelay,
RetentionDelay: ctlr.Config.Storage.UntaggedImageRetentionDelay,
}, ctlr.Log)
ImageRetention: ctlr.Config.Storage.Retention,
}, ctlr.Audit, ctlr.Log)
resp, err := resty.R().Get(baseURL + fmt.Sprintf("/v2/%s/manifests/%s", repoName, tag))
So(err, ShouldBeNil)
@@ -8196,8 +8224,12 @@ func TestPeriodicGC(t *testing.T) {
subPaths := make(map[string]config.StorageConfig)
subPaths["/a"] = config.StorageConfig{
RootDirectory: subDir, GC: true, GCDelay: 1 * time.Second,
UntaggedImageRetentionDelay: 1 * time.Second, GCInterval: 24 * time.Hour, RemoteCache: false, Dedupe: false,
RootDirectory: subDir,
GC: true,
GCDelay: 1 * time.Second,
GCInterval: 24 * time.Hour,
RemoteCache: false,
Dedupe: false,
} //nolint:lll // gofumpt conflicts with lll
ctlr.Config.Storage.Dedupe = false
ctlr.Config.Storage.SubPaths = subPaths
pkg/api/routes.go
+2 -2
View File
@@ -721,8 +721,8 @@ func (rh *RouteHandler) UpdateManifest(response http.ResponseWriter, request *ht
}
if rh.c.MetaDB != nil {
err := meta.OnUpdateManifest(name, reference, mediaType, digest, body, rh.c.StoreController, rh.c.MetaDB,
rh.c.Log)
err := meta.OnUpdateManifest(request.Context(), name, reference, mediaType,
digest, body, rh.c.StoreController, rh.c.MetaDB, rh.c.Log)
if err != nil {
response.WriteHeader(http.StatusInternalServerError)