github/zot
1
0
Fork 0
mirror of https://github.com/project-zot/zot.git synced 2026-06-17 04:48:26 +08:00
Code Issues Packages Projects Releases Wiki Activity

initial design for task scheduler (#700)

Browse Source 
Signed-off-by: Andreea-Lupu <andreealupu1470@yahoo.com>
This commit is contained in:
Andreea Lupu
2022-09-23 08:27:56 +03:00
committed by GitHub
parent 7517f2a5bb
commit f686ab6bf6
16 changed files with 859 additions and 254 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/extensions/extension_scrub.go
+62 -11
View File
@@ -4,30 +4,81 @@
package extensions
import (
"errors"
"io"
"time"
"zotregistry.io/zot/pkg/api/config"
"zotregistry.io/zot/pkg/extensions/scrub"
"zotregistry.io/zot/pkg/log"
"zotregistry.io/zot/pkg/scheduler"
"zotregistry.io/zot/pkg/storage"
)
// EnableScrubExtension enables scrub extension.
func EnableScrubExtension(config *config.Config, log log.Logger, run bool, imgStore storage.ImageStore, repo string) {
if !run {
if config.Extensions.Scrub != nil &&
config.Extensions.Scrub.Interval != 0 {
minScrubInterval, _ := time.ParseDuration("2h")
func EnableScrubExtension(config *config.Config, log log.Logger, storeController storage.StoreController,
sch *scheduler.Scheduler,
) {
if config.Extensions.Scrub != nil &&
config.Extensions.Scrub.Interval != 0 {
minScrubInterval, _ := time.ParseDuration("2h")
if config.Extensions.Scrub.Interval < minScrubInterval {
config.Extensions.Scrub.Interval = minScrubInterval
if config.Extensions.Scrub.Interval < minScrubInterval {
config.Extensions.Scrub.Interval = minScrubInterval
log.Warn().Msg("Scrub interval set to too-short interval < 2h, changing scrub duration to 2 hours and continuing.") //nolint:lll // gofumpt conflicts with lll
log.Warn().Msg("Scrub interval set to too-short interval < 2h, changing scrub duration to 2 hours and continuing.") //nolint:lll // gofumpt conflicts with lll
}
generator := &taskGenerator{
imgStore: storeController.DefaultStore,
log: log,
}
sch.SubmitGenerator(generator, config.Extensions.Scrub.Interval, scheduler.LowPriority)
if config.Storage.SubPaths != nil {
for route := range config.Storage.SubPaths {
generator := &taskGenerator{
imgStore: storeController.SubStore[route],
log: log,
}
sch.SubmitGenerator(generator, config.Extensions.Scrub.Interval, scheduler.LowPriority)
}
} else {
log.Info().Msg("Scrub config not provided, skipping scrub")
}
} else {
scrub.RunScrubRepo(imgStore, repo, log)
log.Info().Msg("Scrub config not provided, skipping scrub")
}
}
type taskGenerator struct {
imgStore storage.ImageStore
log log.Logger
lastRepo string
done bool
}
func (gen *taskGenerator) GenerateTask() (scheduler.Task, error) {
repo, err := gen.imgStore.GetNextRepository(gen.lastRepo)
if err != nil && !errors.Is(err, io.EOF) {
return nil, err
}
if repo == "" {
gen.done = true
return nil, nil
}
gen.lastRepo = repo
return scrub.NewTask(gen.imgStore, repo, gen.log), nil
}
func (gen *taskGenerator) IsDone() bool {
return gen.done
}
func (gen *taskGenerator) Reset() {
gen.lastRepo = ""
gen.done = false
}
pkg/extensions/extension_scrub_disabled.go
+3 -3
View File
@@ -6,13 +6,13 @@ package extensions
import (
"zotregistry.io/zot/pkg/api/config"
"zotregistry.io/zot/pkg/log"
"zotregistry.io/zot/pkg/scheduler"
"zotregistry.io/zot/pkg/storage"
)
// EnableScrubExtension ...
func EnableScrubExtension(config *config.Config,
log log.Logger, run bool,
imgStore storage.ImageStore, repo string,
func EnableScrubExtension(config *config.Config, log log.Logger, storeController storage.StoreController,
sch *scheduler.Scheduler,
) {
log.Warn().Msg("skipping enabling scrub extension because given zot binary doesn't include this feature," +
"please build a binary that does so")
pkg/extensions/scrub/scrub.go
+21 -2
View File
@@ -12,7 +12,7 @@ import (
)
// Scrub Extension for repo...
func RunScrubRepo(imgStore storage.ImageStore, repo string, log log.Logger) {
func RunScrubRepo(imgStore storage.ImageStore, repo string, log log.Logger) error {
execMsg := fmt.Sprintf("executing scrub to check manifest/blob integrity for %s", path.Join(imgStore.RootDir(), repo))
log.Info().Msg(execMsg)
@@ -20,6 +20,9 @@ func RunScrubRepo(imgStore storage.ImageStore, repo string, log log.Logger) {
if err != nil {
errMessage := fmt.Sprintf("error while running scrub for %s", path.Join(imgStore.RootDir(), repo))
log.Error().Err(err).Msg(errMessage)
log.Info().Msg(fmt.Sprintf("scrub unsuccessfully completed for %s", path.Join(imgStore.RootDir(), repo)))
return err
}
for _, result := range results {
@@ -39,5 +42,21 @@ func RunScrubRepo(imgStore storage.ImageStore, repo string, log log.Logger) {
}
}
log.Info().Msg(fmt.Sprintf("scrub completed for %s", path.Join(imgStore.RootDir(), repo)))
log.Info().Msg(fmt.Sprintf("scrub successfully completed for %s", path.Join(imgStore.RootDir(), repo)))
return nil
}
type Task struct {
imgStore storage.ImageStore
repo string
log log.Logger
}
func NewTask(imgStore storage.ImageStore, repo string, log log.Logger) *Task {
return &Task{imgStore, repo, log}
}
func (scrubT *Task) DoWork() error {
return RunScrubRepo(scrubT.imgStore, scrubT.repo, scrubT.log)
}
pkg/extensions/scrub/scrub_test.go
+14 -9
View File
@@ -42,8 +42,11 @@ func TestScrubExtension(t *testing.T) {
conf.HTTP.Port = port
dir := t.TempDir()
subdir := t.TempDir()
conf.Storage.RootDirectory = dir
substore := config.StorageConfig{RootDirectory: subdir}
conf.Storage.SubPaths = map[string]config.StorageConfig{"/a": substore}
conf.Log.Output = logFile.Name()
scrubConfig := &extconf.ScrubConfig{
Interval: 2,
@@ -75,7 +78,7 @@ func TestScrubExtension(t *testing.T) {
time.Sleep(100 * time.Millisecond)
}
time.Sleep(1 * time.Second)
time.Sleep(6 * time.Second)
defer func(controller *api.Controller) {
ctx := context.Background()
@@ -140,7 +143,7 @@ func TestScrubExtension(t *testing.T) {
time.Sleep(100 * time.Millisecond)
}
time.Sleep(500 * time.Millisecond)
time.Sleep(6 * time.Second)
defer func(controller *api.Controller) {
ctx := context.Background()
@@ -152,7 +155,7 @@ func TestScrubExtension(t *testing.T) {
So(string(data), ShouldContainSubstring, "scrub: blobs/manifest affected")
})
Convey("RunBackgroundTasks error - not enough permissions to access root directory", t, func(c C) {
Convey("Generator error - not enough permissions to access root directory", t, func(c C) {
port := test.GetFreePort()
url := test.GetBaseURL(port)
@@ -200,7 +203,7 @@ func TestScrubExtension(t *testing.T) {
time.Sleep(100 * time.Millisecond)
}
time.Sleep(500 * time.Millisecond)
time.Sleep(6 * time.Second)
defer func(controller *api.Controller) {
ctx := context.Background()
@@ -209,8 +212,7 @@ func TestScrubExtension(t *testing.T) {
data, err := os.ReadFile(logFile.Name())
So(err, ShouldBeNil)
So(string(data), ShouldContainSubstring,
fmt.Sprintf("error while running background task for %s", ctlr.StoreController.DefaultStore.RootDir()))
So(string(data), ShouldContainSubstring, "error while executing generator")
So(os.Chmod(path.Join(dir, repoName), 0o755), ShouldBeNil)
})
@@ -238,7 +240,8 @@ func TestRunScrubRepo(t *testing.T) {
panic(err)
}
scrub.RunScrubRepo(imgStore, repoName, log)
err = scrub.RunScrubRepo(imgStore, repoName, log)
So(err, ShouldBeNil)
data, err := os.ReadFile(logFile.Name())
So(err, ShouldBeNil)
@@ -274,7 +277,8 @@ func TestRunScrubRepo(t *testing.T) {
panic(err)
}
scrub.RunScrubRepo(imgStore, repoName, log)
err = scrub.RunScrubRepo(imgStore, repoName, log)
So(err, ShouldBeNil)
data, err := os.ReadFile(logFile.Name())
So(err, ShouldBeNil)
@@ -304,7 +308,8 @@ func TestRunScrubRepo(t *testing.T) {
So(os.Chmod(path.Join(dir, repoName), 0o000), ShouldBeNil)
scrub.RunScrubRepo(imgStore, repoName, log)
err = scrub.RunScrubRepo(imgStore, repoName, log)
So(err, ShouldNotBeNil)
data, err := os.ReadFile(logFile.Name())
So(err, ShouldBeNil)