mirror of
https://github.com/project-zot/zot.git
synced 2026-06-16 12:28:01 +08:00
initial design for task scheduler (#700)
Signed-off-by: Andreea-Lupu <andreealupu1470@yahoo.com>
This commit is contained in:
Andreea Lupu
+16
-128
@@ -21,10 +21,10 @@ import (
|
||||
"zotregistry.io/zot/errors"
|
||||
"zotregistry.io/zot/pkg/api/config"
|
||||
ext "zotregistry.io/zot/pkg/extensions"
|
||||
extconf "zotregistry.io/zot/pkg/extensions/config"
|
||||
"zotregistry.io/zot/pkg/extensions/lint"
|
||||
"zotregistry.io/zot/pkg/extensions/monitoring"
|
||||
"zotregistry.io/zot/pkg/log"
|
||||
"zotregistry.io/zot/pkg/scheduler"
|
||||
"zotregistry.io/zot/pkg/storage"
|
||||
"zotregistry.io/zot/pkg/storage/s3"
|
||||
)
|
||||
@@ -444,6 +444,14 @@ func (c *Controller) Shutdown() {
|
||||
}
|
||||
|
||||
func (c *Controller) StartBackgroundTasks(reloadCtx context.Context) {
|
||||
taskScheduler := scheduler.NewScheduler(c.Log)
|
||||
taskScheduler.RunScheduler(reloadCtx)
|
||||
|
||||
// Enable running garbage-collect periodically for DefaultStore
|
||||
if c.Config.Storage.GC && c.Config.Storage.GCInterval != 0 {
|
||||
c.StoreController.DefaultStore.RunGCPeriodically(c.Config.Storage.GCInterval, taskScheduler)
|
||||
}
|
||||
|
||||
// Enable extensions if extension config is provided for DefaultStore
|
||||
if c.Config != nil && c.Config.Extensions != nil {
|
||||
ext.EnableMetricsExtension(c.Config, c.Log, c.Config.Storage.RootDirectory)
|
||||
@@ -451,7 +459,12 @@ func (c *Controller) StartBackgroundTasks(reloadCtx context.Context) {
|
||||
}
|
||||
|
||||
if c.Config.Storage.SubPaths != nil {
|
||||
for _, storageConfig := range c.Config.Storage.SubPaths {
|
||||
for route, storageConfig := range c.Config.Storage.SubPaths {
|
||||
// Enable running garbage-collect periodically for subImageStore
|
||||
if storageConfig.GC && storageConfig.GCInterval != 0 {
|
||||
c.StoreController.SubStore[route].RunGCPeriodically(storageConfig.GCInterval, taskScheduler)
|
||||
}
|
||||
|
||||
// Enable extensions if extension config is provided for subImageStore
|
||||
if c.Config != nil && c.Config.Extensions != nil {
|
||||
ext.EnableMetricsExtension(c.Config, c.Log, storageConfig.RootDirectory)
|
||||
@@ -468,131 +481,6 @@ func (c *Controller) StartBackgroundTasks(reloadCtx context.Context) {
|
||||
}
|
||||
|
||||
if c.Config.Extensions != nil {
|
||||
ext.EnableScrubExtension(c.Config, c.Log, false, nil, "")
|
||||
}
|
||||
|
||||
go StartPeriodicTasks(c.StoreController.DefaultStore, c.StoreController.SubStore, c.Config.Storage.SubPaths,
|
||||
c.Config.Storage.GC, c.Config.Storage.GCInterval, c.Config.Extensions, c.Log)
|
||||
}
|
||||
|
||||
func StartPeriodicTasks(defaultStore storage.ImageStore, subStore map[string]storage.ImageStore,
|
||||
subPaths map[string]config.StorageConfig, gcEnabled bool, gcInterval time.Duration,
|
||||
extensions *extconf.ExtensionConfig, log log.Logger,
|
||||
) {
|
||||
// start periodic gc and/or scrub for DefaultStore
|
||||
StartPeriodicTasksForImageStore(defaultStore, gcEnabled, gcInterval, extensions, log)
|
||||
|
||||
for route, storageConfig := range subPaths {
|
||||
// Enable running garbage-collect or/and scrub periodically for subImageStore
|
||||
StartPeriodicTasksForImageStore(subStore[route], storageConfig.GC, storageConfig.GCInterval, extensions, log)
|
||||
ext.EnableScrubExtension(c.Config, c.Log, c.StoreController, taskScheduler)
|
||||
}
|
||||
}
|
||||
|
||||
func StartPeriodicTasksForImageStore(imageStore storage.ImageStore, configGC bool, configGCInterval time.Duration,
|
||||
extensions *extconf.ExtensionConfig, log log.Logger,
|
||||
) {
|
||||
scrubInterval := time.Duration(0)
|
||||
gcInterval := time.Duration(0)
|
||||
|
||||
gc := false
|
||||
scrub := false
|
||||
|
||||
if configGC && configGCInterval != 0 {
|
||||
gcInterval = configGCInterval
|
||||
gc = true
|
||||
}
|
||||
|
||||
if extensions != nil && extensions.Scrub != nil && extensions.Scrub.Interval != 0 {
|
||||
scrubInterval = extensions.Scrub.Interval
|
||||
scrub = true
|
||||
}
|
||||
|
||||
interval := minPeriodicInterval(scrub, gc, scrubInterval, gcInterval)
|
||||
if interval == time.Duration(0) {
|
||||
return
|
||||
}
|
||||
|
||||
log.Info().Msg(fmt.Sprintf("Periodic interval for %s set to %s", imageStore.RootDir(), interval))
|
||||
|
||||
var lastGC, lastScrub time.Time
|
||||
|
||||
for {
|
||||
log.Info().Msg(fmt.Sprintf("Starting periodic background tasks for %s", imageStore.RootDir()))
|
||||
|
||||
// Enable running garbage-collect or/and scrub periodically for imageStore
|
||||
RunBackgroundTasks(imageStore, gc, scrub, log)
|
||||
|
||||
log.Info().Msg(fmt.Sprintf("Finishing periodic background tasks for %s", imageStore.RootDir()))
|
||||
|
||||
if gc {
|
||||
lastGC = time.Now()
|
||||
}
|
||||
|
||||
if scrub {
|
||||
lastScrub = time.Now()
|
||||
}
|
||||
|
||||
time.Sleep(interval)
|
||||
|
||||
if !lastGC.IsZero() && time.Since(lastGC) >= gcInterval {
|
||||
gc = true
|
||||
}
|
||||
|
||||
if !lastScrub.IsZero() && time.Since(lastScrub) >= scrubInterval {
|
||||
scrub = true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func RunBackgroundTasks(imgStore storage.ImageStore, gc, scrub bool, log log.Logger) {
|
||||
repos, err := imgStore.GetRepositories()
|
||||
if err != nil {
|
||||
log.Error().Err(err).Msg(fmt.Sprintf("error while running background task for %s", imgStore.RootDir()))
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
for _, repo := range repos {
|
||||
if gc {
|
||||
start := time.Now()
|
||||
|
||||
// run gc for this repo
|
||||
imgStore.RunGCRepo(repo)
|
||||
|
||||
elapsed := time.Since(start)
|
||||
log.Info().Msg(fmt.Sprintf("gc for %s executed in %s", repo, elapsed))
|
||||
time.Sleep(1 * time.Minute)
|
||||
}
|
||||
|
||||
if scrub {
|
||||
start := time.Now()
|
||||
|
||||
// run scrub for this repo
|
||||
ext.EnableScrubExtension(nil, log, true, imgStore, repo)
|
||||
|
||||
elapsed := time.Since(start)
|
||||
log.Info().Msg(fmt.Sprintf("scrub for %s executed in %s", repo, elapsed))
|
||||
time.Sleep(1 * time.Minute)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func minPeriodicInterval(scrub, gc bool, scrubInterval, gcInterval time.Duration) time.Duration {
|
||||
if scrub && gc {
|
||||
if scrubInterval <= gcInterval {
|
||||
return scrubInterval
|
||||
}
|
||||
|
||||
return gcInterval
|
||||
}
|
||||
|
||||
if scrub {
|
||||
return scrubInterval
|
||||
}
|
||||
|
||||
if gc {
|
||||
return gcInterval
|
||||
}
|
||||
|
||||
return time.Duration(0)
|
||||
}
|
||||
|
||||
@@ -6007,14 +6007,10 @@ func TestPeriodicGC(t *testing.T) {
|
||||
So(err, ShouldBeNil)
|
||||
So(string(data), ShouldContainSubstring,
|
||||
"\"GC\":true,\"Commit\":false,\"GCDelay\":1000000000,\"GCInterval\":3600000000000")
|
||||
So(string(data), ShouldContainSubstring,
|
||||
fmt.Sprintf("Starting periodic background tasks for %s", ctlr.StoreController.DefaultStore.RootDir())) //nolint:lll
|
||||
So(string(data), ShouldNotContainSubstring,
|
||||
fmt.Sprintf("error while running background task for %s", ctlr.StoreController.DefaultStore.RootDir()))
|
||||
So(string(data), ShouldContainSubstring,
|
||||
fmt.Sprintf("executing GC of orphaned blobs for %s", path.Join(ctlr.StoreController.DefaultStore.RootDir(), repoName))) //nolint:lll
|
||||
So(string(data), ShouldContainSubstring,
|
||||
fmt.Sprintf("GC completed for %s", path.Join(ctlr.StoreController.DefaultStore.RootDir(), repoName))) //nolint:lll
|
||||
fmt.Sprintf("GC successfully completed for %s", path.Join(ctlr.StoreController.DefaultStore.RootDir(), repoName))) //nolint:lll
|
||||
})
|
||||
|
||||
Convey("Periodic GC enabled for substore", t, func() {
|
||||
@@ -6051,82 +6047,6 @@ func TestPeriodicGC(t *testing.T) {
|
||||
// periodic GC is enabled for sub store
|
||||
So(string(data), ShouldContainSubstring,
|
||||
fmt.Sprintf("\"SubPaths\":{\"/a\":{\"RootDirectory\":\"%s\",\"GC\":true,\"Dedupe\":false,\"Commit\":false,\"GCDelay\":1000000000,\"GCInterval\":86400000000000", subDir)) //nolint:lll // gofumpt conflicts with lll
|
||||
So(string(data), ShouldContainSubstring,
|
||||
fmt.Sprintf("Starting periodic background tasks for %s", ctlr.StoreController.SubStore["/a"].RootDir())) //nolint:lll
|
||||
})
|
||||
}
|
||||
|
||||
func TestPeriodicTasks(t *testing.T) {
|
||||
Convey("Both periodic gc and periodic scrub enabled for default store with scrubInterval < gcInterval", t, func() {
|
||||
port := test.GetFreePort()
|
||||
baseURL := test.GetBaseURL(port)
|
||||
conf := config.New()
|
||||
conf.HTTP.Port = port
|
||||
|
||||
logFile, err := os.CreateTemp("", "zot-log*.txt")
|
||||
So(err, ShouldBeNil)
|
||||
conf.Log.Output = logFile.Name()
|
||||
defer os.Remove(logFile.Name()) // clean up
|
||||
|
||||
ctlr := api.NewController(conf)
|
||||
dir := t.TempDir()
|
||||
ctlr.Config.Storage.RootDirectory = dir
|
||||
ctlr.Config.Storage.GC = true
|
||||
ctlr.Config.Storage.GCInterval = 12 * time.Hour
|
||||
ctlr.Config.Storage.GCDelay = 1 * time.Second
|
||||
ctlr.Config.Extensions = &extconf.ExtensionConfig{Scrub: &extconf.ScrubConfig{Interval: 8 * time.Hour}}
|
||||
|
||||
go startServer(ctlr)
|
||||
defer stopServer(ctlr)
|
||||
test.WaitTillServerReady(baseURL)
|
||||
|
||||
data, err := os.ReadFile(logFile.Name())
|
||||
So(err, ShouldBeNil)
|
||||
So(string(data), ShouldContainSubstring,
|
||||
fmt.Sprintf("Starting periodic background tasks for %s", ctlr.StoreController.DefaultStore.RootDir())) //nolint:lll
|
||||
So(string(data), ShouldNotContainSubstring,
|
||||
fmt.Sprintf("error while running background task for %s", ctlr.StoreController.DefaultStore.RootDir()))
|
||||
So(string(data), ShouldContainSubstring,
|
||||
fmt.Sprintf("Finishing periodic background tasks for %s", ctlr.StoreController.DefaultStore.RootDir())) //nolint:lll
|
||||
So(string(data), ShouldContainSubstring,
|
||||
fmt.Sprintf("Periodic interval for %s set to %s",
|
||||
ctlr.StoreController.DefaultStore.RootDir(), ctlr.Config.Extensions.Scrub.Interval))
|
||||
})
|
||||
|
||||
Convey("Both periodic gc and periodic scrub enabled for default store with gcInterval < scrubInterval", t, func() {
|
||||
port := test.GetFreePort()
|
||||
baseURL := test.GetBaseURL(port)
|
||||
conf := config.New()
|
||||
conf.HTTP.Port = port
|
||||
|
||||
logFile, err := os.CreateTemp("", "zot-log*.txt")
|
||||
So(err, ShouldBeNil)
|
||||
conf.Log.Output = logFile.Name()
|
||||
defer os.Remove(logFile.Name()) // clean up
|
||||
|
||||
ctlr := api.NewController(conf)
|
||||
dir := t.TempDir()
|
||||
ctlr.Config.Storage.RootDirectory = dir
|
||||
ctlr.Config.Storage.GC = true
|
||||
ctlr.Config.Storage.GCInterval = 8 * time.Hour
|
||||
ctlr.Config.Storage.GCDelay = 1 * time.Second
|
||||
ctlr.Config.Extensions = &extconf.ExtensionConfig{Scrub: &extconf.ScrubConfig{Interval: 12 * time.Hour}}
|
||||
|
||||
go startServer(ctlr)
|
||||
defer stopServer(ctlr)
|
||||
test.WaitTillServerReady(baseURL)
|
||||
|
||||
data, err := os.ReadFile(logFile.Name())
|
||||
So(err, ShouldBeNil)
|
||||
So(string(data), ShouldContainSubstring,
|
||||
fmt.Sprintf("Starting periodic background tasks for %s", ctlr.StoreController.DefaultStore.RootDir())) //nolint:lll
|
||||
So(string(data), ShouldNotContainSubstring,
|
||||
fmt.Sprintf("error while running background task for %s", ctlr.StoreController.DefaultStore.RootDir()))
|
||||
So(string(data), ShouldContainSubstring,
|
||||
fmt.Sprintf("Finishing periodic background tasks for %s", ctlr.StoreController.DefaultStore.RootDir())) //nolint:lll
|
||||
So(string(data), ShouldContainSubstring,
|
||||
fmt.Sprintf("Periodic interval for %s set to %s",
|
||||
ctlr.StoreController.DefaultStore.RootDir(), ctlr.Config.Storage.GCInterval))
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user