mirror of
https://github.com/project-zot/zot.git
synced 2026-06-15 20:07:55 +08:00
fix periodic background tasks - gc and scrub
Signed-off-by: Andreea-Lupu <andreealupu1470@yahoo.com>
This commit is contained in:
Andreea-Lupu
committed by
Ramkumar Chinchani
Ramkumar Chinchani
parent
d0b52612a2
commit
081ba0b2f2
+128
-12
@@ -20,6 +20,7 @@ import (
|
||||
"zotregistry.io/zot/errors"
|
||||
"zotregistry.io/zot/pkg/api/config"
|
||||
ext "zotregistry.io/zot/pkg/extensions"
|
||||
extconf "zotregistry.io/zot/pkg/extensions/config"
|
||||
"zotregistry.io/zot/pkg/extensions/monitoring"
|
||||
"zotregistry.io/zot/pkg/log"
|
||||
"zotregistry.io/zot/pkg/storage"
|
||||
@@ -350,23 +351,13 @@ func (c *Controller) Shutdown() {
|
||||
}
|
||||
|
||||
func (c *Controller) StartBackgroundTasks(reloadCtx context.Context) {
|
||||
// Enable running garbage-collect periodically for DefaultStore
|
||||
if c.Config.Storage.GC && c.Config.Storage.GCInterval != 0 {
|
||||
c.StoreController.DefaultStore.RunGCPeriodically(c.Config.Storage.GCInterval)
|
||||
}
|
||||
|
||||
// Enable extensions if extension config is provided for DefaultStore
|
||||
if c.Config != nil && c.Config.Extensions != nil {
|
||||
ext.EnableExtensions(c.Config, c.Log, c.Config.Storage.RootDirectory)
|
||||
}
|
||||
|
||||
if c.Config.Storage.SubPaths != nil {
|
||||
for route, storageConfig := range c.Config.Storage.SubPaths {
|
||||
// Enable running garbage-collect periodically for subImageStore
|
||||
if storageConfig.GC && storageConfig.GCInterval != 0 {
|
||||
c.StoreController.SubStore[route].RunGCPeriodically(storageConfig.GCInterval)
|
||||
}
|
||||
|
||||
for _, storageConfig := range c.Config.Storage.SubPaths {
|
||||
// Enable extensions if extension config is provided for subImageStore
|
||||
if c.Config != nil && c.Config.Extensions != nil {
|
||||
ext.EnableExtensions(c.Config, c.Log, storageConfig.RootDirectory)
|
||||
@@ -382,6 +373,131 @@ func (c *Controller) StartBackgroundTasks(reloadCtx context.Context) {
|
||||
}
|
||||
|
||||
if c.Config.Extensions != nil {
|
||||
ext.EnableScrubExtension(c.Config, c.StoreController, c.Log)
|
||||
ext.EnableScrubExtension(c.Config, c.Log, false, nil, "")
|
||||
}
|
||||
|
||||
go StartPeriodicTasks(c.StoreController.DefaultStore, c.StoreController.SubStore, c.Config.Storage.SubPaths,
|
||||
c.Config.Storage.GC, c.Config.Storage.GCInterval, c.Config.Extensions, c.Log)
|
||||
}
|
||||
|
||||
func StartPeriodicTasks(defaultStore storage.ImageStore, subStore map[string]storage.ImageStore,
|
||||
subPaths map[string]config.StorageConfig, gcEnabled bool, gcInterval time.Duration,
|
||||
extensions *extconf.ExtensionConfig, log log.Logger,
|
||||
) {
|
||||
// start periodic gc and/or scrub for DefaultStore
|
||||
StartPeriodicTasksForImageStore(defaultStore, gcEnabled, gcInterval, extensions, log)
|
||||
|
||||
for route, storageConfig := range subPaths {
|
||||
// Enable running garbage-collect or/and scrub periodically for subImageStore
|
||||
StartPeriodicTasksForImageStore(subStore[route], storageConfig.GC, storageConfig.GCInterval, extensions, log)
|
||||
}
|
||||
}
|
||||
|
||||
func StartPeriodicTasksForImageStore(imageStore storage.ImageStore, configGC bool, configGCInterval time.Duration,
|
||||
extensions *extconf.ExtensionConfig, log log.Logger,
|
||||
) {
|
||||
scrubInterval := time.Duration(0)
|
||||
gcInterval := time.Duration(0)
|
||||
|
||||
gc := false
|
||||
scrub := false
|
||||
|
||||
if configGC && configGCInterval != 0 {
|
||||
gcInterval = configGCInterval
|
||||
gc = true
|
||||
}
|
||||
|
||||
if extensions != nil && extensions.Scrub != nil && extensions.Scrub.Interval != 0 {
|
||||
scrubInterval = extensions.Scrub.Interval
|
||||
scrub = true
|
||||
}
|
||||
|
||||
interval := minPeriodicInterval(scrub, gc, scrubInterval, gcInterval)
|
||||
if interval == time.Duration(0) {
|
||||
return
|
||||
}
|
||||
|
||||
log.Info().Msg(fmt.Sprintf("Periodic interval for %s set to %s", imageStore.RootDir(), interval))
|
||||
|
||||
var lastGC, lastScrub time.Time
|
||||
|
||||
for {
|
||||
log.Info().Msg(fmt.Sprintf("Starting periodic background tasks for %s", imageStore.RootDir()))
|
||||
|
||||
// Enable running garbage-collect or/and scrub periodically for imageStore
|
||||
RunBackgroundTasks(imageStore, gc, scrub, log)
|
||||
|
||||
log.Info().Msg(fmt.Sprintf("Finishing periodic background tasks for %s", imageStore.RootDir()))
|
||||
|
||||
if gc {
|
||||
lastGC = time.Now()
|
||||
}
|
||||
|
||||
if scrub {
|
||||
lastScrub = time.Now()
|
||||
}
|
||||
|
||||
time.Sleep(interval)
|
||||
|
||||
if !lastGC.IsZero() && time.Since(lastGC) >= gcInterval {
|
||||
gc = true
|
||||
}
|
||||
|
||||
if !lastScrub.IsZero() && time.Since(lastScrub) >= scrubInterval {
|
||||
scrub = true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func RunBackgroundTasks(imgStore storage.ImageStore, gc, scrub bool, log log.Logger) {
|
||||
repos, err := imgStore.GetRepositories()
|
||||
if err != nil {
|
||||
log.Error().Err(err).Msg(fmt.Sprintf("error while running background task for %s", imgStore.RootDir()))
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
for _, repo := range repos {
|
||||
if gc {
|
||||
start := time.Now()
|
||||
|
||||
// run gc for this repo
|
||||
imgStore.RunGCRepo(repo)
|
||||
|
||||
elapsed := time.Since(start)
|
||||
log.Info().Msg(fmt.Sprintf("gc for %s executed in %s", repo, elapsed))
|
||||
time.Sleep(1 * time.Minute)
|
||||
}
|
||||
|
||||
if scrub {
|
||||
start := time.Now()
|
||||
|
||||
// run scrub for this repo
|
||||
ext.EnableScrubExtension(nil, log, true, imgStore, repo)
|
||||
|
||||
elapsed := time.Since(start)
|
||||
log.Info().Msg(fmt.Sprintf("scrub for %s executed in %s", repo, elapsed))
|
||||
time.Sleep(1 * time.Minute)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func minPeriodicInterval(scrub, gc bool, scrubInterval, gcInterval time.Duration) time.Duration {
|
||||
if scrub && gc {
|
||||
if scrubInterval <= gcInterval {
|
||||
return scrubInterval
|
||||
}
|
||||
|
||||
return gcInterval
|
||||
}
|
||||
|
||||
if scrub {
|
||||
return scrubInterval
|
||||
}
|
||||
|
||||
if gc {
|
||||
return gcInterval
|
||||
}
|
||||
|
||||
return time.Duration(0)
|
||||
}
|
||||
|
||||
@@ -4741,6 +4741,8 @@ func TestInjectTooManyOpenFiles(t *testing.T) {
|
||||
|
||||
func TestPeriodicGC(t *testing.T) {
|
||||
Convey("Periodic gc enabled for default store", t, func() {
|
||||
repoName := "test"
|
||||
|
||||
port := test.GetFreePort()
|
||||
baseURL := test.GetBaseURL(port)
|
||||
conf := config.New()
|
||||
@@ -4748,7 +4750,6 @@ func TestPeriodicGC(t *testing.T) {
|
||||
|
||||
logFile, err := ioutil.TempFile("", "zot-log*.txt")
|
||||
So(err, ShouldBeNil)
|
||||
conf.Log.Level = "debug"
|
||||
conf.Log.Output = logFile.Name()
|
||||
defer os.Remove(logFile.Name()) // clean up
|
||||
|
||||
@@ -4759,20 +4760,29 @@ func TestPeriodicGC(t *testing.T) {
|
||||
ctlr.Config.Storage.GCInterval = 1 * time.Hour
|
||||
ctlr.Config.Storage.GCDelay = 1 * time.Second
|
||||
|
||||
err = test.CopyFiles("../../test/data/zot-test", path.Join(dir, repoName))
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
go startServer(ctlr)
|
||||
defer stopServer(ctlr)
|
||||
test.WaitTillServerReady(baseURL)
|
||||
|
||||
time.Sleep(500 * time.Millisecond)
|
||||
|
||||
data, err := os.ReadFile(logFile.Name())
|
||||
So(err, ShouldBeNil)
|
||||
So(string(data), ShouldContainSubstring,
|
||||
"\"GC\":true,\"Commit\":false,\"GCDelay\":1000000000,\"GCInterval\":3600000000000")
|
||||
So(string(data), ShouldContainSubstring,
|
||||
fmt.Sprintf("executing GC of orphaned blobs for %s", ctlr.StoreController.DefaultStore.RootDir()))
|
||||
fmt.Sprintf("Starting periodic background tasks for %s", ctlr.StoreController.DefaultStore.RootDir())) //nolint:lll
|
||||
So(string(data), ShouldNotContainSubstring,
|
||||
fmt.Sprintf("error while running GC for %s", ctlr.StoreController.DefaultStore.RootDir()))
|
||||
fmt.Sprintf("error while running background task for %s", ctlr.StoreController.DefaultStore.RootDir()))
|
||||
So(string(data), ShouldContainSubstring,
|
||||
fmt.Sprintf("GC completed for %s, next GC scheduled after", ctlr.StoreController.DefaultStore.RootDir()))
|
||||
fmt.Sprintf("executing GC of orphaned blobs for %s", path.Join(ctlr.StoreController.DefaultStore.RootDir(), repoName))) //nolint:lll
|
||||
So(string(data), ShouldContainSubstring,
|
||||
fmt.Sprintf("GC completed for %s", path.Join(ctlr.StoreController.DefaultStore.RootDir(), repoName))) //nolint:lll
|
||||
})
|
||||
|
||||
Convey("Periodic GC enabled for substore", t, func() {
|
||||
@@ -4783,7 +4793,6 @@ func TestPeriodicGC(t *testing.T) {
|
||||
|
||||
logFile, err := ioutil.TempFile("", "zot-log*.txt")
|
||||
So(err, ShouldBeNil)
|
||||
conf.Log.Level = "debug"
|
||||
conf.Log.Output = logFile.Name()
|
||||
defer os.Remove(logFile.Name()) // clean up
|
||||
|
||||
@@ -4811,7 +4820,81 @@ func TestPeriodicGC(t *testing.T) {
|
||||
So(string(data), ShouldContainSubstring,
|
||||
fmt.Sprintf("\"SubPaths\":{\"/a\":{\"RootDirectory\":\"%s\",\"GC\":true,\"Dedupe\":false,\"Commit\":false,\"GCDelay\":1000000000,\"GCInterval\":86400000000000", subDir)) //nolint:lll // gofumpt conflicts with lll
|
||||
So(string(data), ShouldContainSubstring,
|
||||
fmt.Sprintf("executing GC of orphaned blobs for %s", ctlr.StoreController.SubStore["/a"].RootDir()))
|
||||
fmt.Sprintf("Starting periodic background tasks for %s", ctlr.StoreController.SubStore["/a"].RootDir())) //nolint:lll
|
||||
})
|
||||
}
|
||||
|
||||
func TestPeriodicTasks(t *testing.T) {
|
||||
Convey("Both periodic gc and periodic scrub enabled for default store with scrubInterval < gcInterval", t, func() {
|
||||
port := test.GetFreePort()
|
||||
baseURL := test.GetBaseURL(port)
|
||||
conf := config.New()
|
||||
conf.HTTP.Port = port
|
||||
|
||||
logFile, err := ioutil.TempFile("", "zot-log*.txt")
|
||||
So(err, ShouldBeNil)
|
||||
conf.Log.Output = logFile.Name()
|
||||
defer os.Remove(logFile.Name()) // clean up
|
||||
|
||||
ctlr := api.NewController(conf)
|
||||
dir := t.TempDir()
|
||||
ctlr.Config.Storage.RootDirectory = dir
|
||||
ctlr.Config.Storage.GC = true
|
||||
ctlr.Config.Storage.GCInterval = 12 * time.Hour
|
||||
ctlr.Config.Storage.GCDelay = 1 * time.Second
|
||||
ctlr.Config.Extensions = &extconf.ExtensionConfig{Scrub: &extconf.ScrubConfig{Interval: 8 * time.Hour}}
|
||||
|
||||
go startServer(ctlr)
|
||||
defer stopServer(ctlr)
|
||||
test.WaitTillServerReady(baseURL)
|
||||
|
||||
data, err := os.ReadFile(logFile.Name())
|
||||
So(err, ShouldBeNil)
|
||||
So(string(data), ShouldContainSubstring,
|
||||
fmt.Sprintf("Starting periodic background tasks for %s", ctlr.StoreController.DefaultStore.RootDir())) //nolint:lll
|
||||
So(string(data), ShouldNotContainSubstring,
|
||||
fmt.Sprintf("error while running background task for %s", ctlr.StoreController.DefaultStore.RootDir()))
|
||||
So(string(data), ShouldContainSubstring,
|
||||
fmt.Sprintf("Finishing periodic background tasks for %s", ctlr.StoreController.DefaultStore.RootDir())) //nolint:lll
|
||||
So(string(data), ShouldContainSubstring,
|
||||
fmt.Sprintf("Periodic interval for %s set to %s",
|
||||
ctlr.StoreController.DefaultStore.RootDir(), ctlr.Config.Extensions.Scrub.Interval))
|
||||
})
|
||||
|
||||
Convey("Both periodic gc and periodic scrub enabled for default store with gcInterval < scrubInterval", t, func() {
|
||||
port := test.GetFreePort()
|
||||
baseURL := test.GetBaseURL(port)
|
||||
conf := config.New()
|
||||
conf.HTTP.Port = port
|
||||
|
||||
logFile, err := ioutil.TempFile("", "zot-log*.txt")
|
||||
So(err, ShouldBeNil)
|
||||
conf.Log.Output = logFile.Name()
|
||||
defer os.Remove(logFile.Name()) // clean up
|
||||
|
||||
ctlr := api.NewController(conf)
|
||||
dir := t.TempDir()
|
||||
ctlr.Config.Storage.RootDirectory = dir
|
||||
ctlr.Config.Storage.GC = true
|
||||
ctlr.Config.Storage.GCInterval = 8 * time.Hour
|
||||
ctlr.Config.Storage.GCDelay = 1 * time.Second
|
||||
ctlr.Config.Extensions = &extconf.ExtensionConfig{Scrub: &extconf.ScrubConfig{Interval: 12 * time.Hour}}
|
||||
|
||||
go startServer(ctlr)
|
||||
defer stopServer(ctlr)
|
||||
test.WaitTillServerReady(baseURL)
|
||||
|
||||
data, err := os.ReadFile(logFile.Name())
|
||||
So(err, ShouldBeNil)
|
||||
So(string(data), ShouldContainSubstring,
|
||||
fmt.Sprintf("Starting periodic background tasks for %s", ctlr.StoreController.DefaultStore.RootDir())) //nolint:lll
|
||||
So(string(data), ShouldNotContainSubstring,
|
||||
fmt.Sprintf("error while running background task for %s", ctlr.StoreController.DefaultStore.RootDir()))
|
||||
So(string(data), ShouldContainSubstring,
|
||||
fmt.Sprintf("Finishing periodic background tasks for %s", ctlr.StoreController.DefaultStore.RootDir())) //nolint:lll
|
||||
So(string(data), ShouldContainSubstring,
|
||||
fmt.Sprintf("Periodic interval for %s set to %s",
|
||||
ctlr.StoreController.DefaultStore.RootDir(), ctlr.Config.Storage.GCInterval))
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
@@ -56,7 +56,7 @@ type MockedImageStore struct {
|
||||
getBlobContentFn func(repo, digest string) ([]byte, error)
|
||||
getReferrersFn func(repo, digest string, mediaType string) ([]artifactspec.Descriptor, error)
|
||||
urlForPathFn func(path string) (string, error)
|
||||
runGCPeriodicallyFn func(gcInterval time.Duration)
|
||||
runGCRepoFn func(repo string)
|
||||
}
|
||||
|
||||
func (is *MockedImageStore) Lock(t *time.Time) {
|
||||
@@ -302,9 +302,9 @@ func (is *MockedImageStore) URLForPath(path string) (string, error) {
|
||||
return "", nil
|
||||
}
|
||||
|
||||
func (is *MockedImageStore) RunGCPeriodically(gcInterval time.Duration) {
|
||||
if is != nil && is.runGCPeriodicallyFn != nil {
|
||||
is.runGCPeriodicallyFn(gcInterval)
|
||||
func (is *MockedImageStore) RunGCRepo(repo string) {
|
||||
if is != nil && is.runGCRepoFn != nil {
|
||||
is.runGCRepoFn(repo)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user