github/zot
1
0
Fork 0
mirror of https://github.com/project-zot/zot.git synced 2026-06-17 04:48:26 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix periodic background tasks - gc and scrub

Browse Source 
Signed-off-by: Andreea-Lupu <andreealupu1470@yahoo.com>
This commit is contained in:
Andreea-Lupu
2022-05-10 01:30:11 +03:00
committed by Ramkumar Chinchani
parent d0b52612a2
commit 081ba0b2f2
13 changed files with 384 additions and 118 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/extensions/extensions.go
+12 -17
View File
@@ -88,27 +88,22 @@ func EnableSyncExtension(ctx context.Context, config *config.Config, wg *goSync.
}
// EnableScrubExtension enables scrub extension.
func EnableScrubExtension(config *config.Config, storeController storage.StoreController,
log log.Logger,
) {
if config.Extensions.Scrub != nil &&
config.Extensions.Scrub.Interval != 0 {
minScrubInterval, _ := time.ParseDuration("2h")
func EnableScrubExtension(config *config.Config, log log.Logger, run bool, imgStore storage.ImageStore, repo string) {
if !run {
if config.Extensions.Scrub != nil &&
config.Extensions.Scrub.Interval != 0 {
minScrubInterval, _ := time.ParseDuration("2h")
if config.Extensions.Scrub.Interval < minScrubInterval {
config.Extensions.Scrub.Interval = minScrubInterval
if config.Extensions.Scrub.Interval < minScrubInterval {
config.Extensions.Scrub.Interval = minScrubInterval
log.Warn().Msg("Scrub interval set to too-short interval < 2h, changing scrub duration to 2 hours and continuing.") //nolint:lll // gofumpt conflicts with lll
}
go func() {
err := scrub.Run(log, config.Extensions.Scrub.Interval, storeController)
if err != nil {
log.Error().Err(err).Msg("error while trying to scrub")
log.Warn().Msg("Scrub interval set to too-short interval < 2h, changing scrub duration to 2 hours and continuing.") //nolint:lll // gofumpt conflicts with lll
}
}()
} else {
log.Info().Msg("Scrub config not provided, skipping scrub")
}
} else {
log.Info().Msg("Scrub config not provided, skipping scrub")
scrub.RunScrubRepo(imgStore, repo, log)
}
}
pkg/extensions/minimal.go
+1 -3
View File
@@ -40,9 +40,7 @@ func EnableSyncExtension(ctx context.Context, config *config.Config, wg *goSync.
}
// EnableScrubExtension ...
func EnableScrubExtension(config *config.Config, storeController storage.StoreController,
log log.Logger,
) {
func EnableScrubExtension(config *config.Config, log log.Logger, run bool, imgStore storage.ImageStore, repo string) {
log.Warn().Msg("skipping enabling scrub extension because given zot binary doesn't support any extensions," +
"please build zot full binary for this feature")
}
pkg/extensions/scrub/scrub.go
+29 -30
View File
@@ -4,41 +4,40 @@
package scrub
import (
"time"
"fmt"
"path"
"zotregistry.io/zot/pkg/log"
"zotregistry.io/zot/pkg/storage"
)
// Scrub Extension...
func Run(log log.Logger, scrubInterval time.Duration, storeController storage.StoreController) error {
for {
log.Info().Msg("executing scrub to check manifest/blob integrity")
// Scrub Extension for repo...
func RunScrubRepo(imgStore storage.ImageStore, repo string, log log.Logger) {
execMsg := fmt.Sprintf("executing scrub to check manifest/blob integrity for %s", path.Join(imgStore.RootDir(), repo))
log.Info().Msg(execMsg)
results, err := storeController.CheckAllBlobsIntegrity()
if err != nil {
return err
}
for _, result := range results.ScrubResults {
if result.Status == "ok" {
log.Info().
Str("image", result.ImageName).
Str("tag", result.Tag).
Str("status", result.Status).
Msg("scrub: blobs/manifest ok")
} else {
log.Warn().
Str("image", result.ImageName).
Str("tag", result.Tag).
Str("status", result.Status).
Str("error", result.Error).
Msg("scrub: blobs/manifest affected")
}
}
log.Info().Str("Scrub completed, next scrub scheduled after", scrubInterval.String()).Msg("")
time.Sleep(scrubInterval)
results, err := storage.CheckRepo(repo, imgStore)
if err != nil {
errMessage := fmt.Sprintf("error while running scrub for %s", path.Join(imgStore.RootDir(), repo))
log.Error().Err(err).Msg(errMessage)
}
for _, result := range results {
if result.Status == "ok" {
log.Info().
Str("image", result.ImageName).
Str("tag", result.Tag).
Str("status", result.Status).
Msg("scrub: blobs/manifest ok")
} else {
log.Warn().
Str("image", result.ImageName).
Str("tag", result.Tag).
Str("status", result.Status).
Str("error", result.Error).
Msg("scrub: blobs/manifest affected")
}
}
log.Info().Msg(fmt.Sprintf("scrub completed for %s", path.Join(imgStore.RootDir(), repo)))
}
pkg/extensions/scrub/scrub_test.go
+90 -2
View File
@@ -5,6 +5,7 @@ package scrub_test
import (
"context"
"fmt"
"io/ioutil"
"os"
"path"
@@ -17,6 +18,10 @@ import (
"zotregistry.io/zot/pkg/api"
"zotregistry.io/zot/pkg/api/config"
extconf "zotregistry.io/zot/pkg/extensions/config"
"zotregistry.io/zot/pkg/extensions/monitoring"
"zotregistry.io/zot/pkg/extensions/scrub"
"zotregistry.io/zot/pkg/log"
"zotregistry.io/zot/pkg/storage"
"zotregistry.io/zot/pkg/test"
)
@@ -148,7 +153,7 @@ func TestScrubExtension(t *testing.T) {
So(string(data), ShouldContainSubstring, "scrub: blobs/manifest affected")
})
Convey("CheckAllBlobsIntegrity error - not enough permissions to access root directory", t, func(c C) {
Convey("RunBackgroundTasks error - not enough permissions to access root directory", t, func(c C) {
port := test.GetFreePort()
url := test.GetBaseURL(port)
@@ -205,8 +210,91 @@ func TestScrubExtension(t *testing.T) {
data, err := os.ReadFile(logFile.Name())
So(err, ShouldBeNil)
So(string(data), ShouldContainSubstring, "error while trying to scrub")
So(string(data), ShouldContainSubstring,
fmt.Sprintf("error while running background task for %s", ctlr.StoreController.DefaultStore.RootDir()))
So(os.Chmod(path.Join(dir, repoName), 0o755), ShouldBeNil)
})
}
func TestRunScrubRepo(t *testing.T) {
Convey("Blobs integrity not affected", t, func(c C) {
logFile, err := ioutil.TempFile("", "zot-log*.txt")
So(err, ShouldBeNil)
defer os.Remove(logFile.Name()) // clean up
dir := t.TempDir()
log := log.NewLogger("debug", logFile.Name())
metrics := monitoring.NewMetricsServer(false, log)
imgStore := storage.NewImageStore(dir, true, 1*time.Second, true, true, log, metrics)
err = test.CopyFiles("../../../test/data/zot-test", path.Join(dir, repoName))
if err != nil {
panic(err)
}
scrub.RunScrubRepo(imgStore, repoName, log)
data, err := os.ReadFile(logFile.Name())
So(err, ShouldBeNil)
So(string(data), ShouldContainSubstring, "scrub: blobs/manifest ok")
})
Convey("Blobs integrity affected", t, func(c C) {
logFile, err := ioutil.TempFile("", "zot-log*.txt")
So(err, ShouldBeNil)
defer os.Remove(logFile.Name()) // clean up
dir := t.TempDir()
log := log.NewLogger("debug", logFile.Name())
metrics := monitoring.NewMetricsServer(false, log)
imgStore := storage.NewImageStore(dir, true, 1*time.Second, true, true, log, metrics)
err = test.CopyFiles("../../../test/data/zot-test", path.Join(dir, repoName))
if err != nil {
panic(err)
}
var manifestDigest digest.Digest
manifestDigest, _, _ = test.GetOciLayoutDigests("../../../test/data/zot-test")
err = os.Remove(path.Join(dir, repoName, "blobs/sha256", manifestDigest.Encoded()))
if err != nil {
panic(err)
}
scrub.RunScrubRepo(imgStore, repoName, log)
data, err := os.ReadFile(logFile.Name())
So(err, ShouldBeNil)
So(string(data), ShouldContainSubstring, "scrub: blobs/manifest affected")
})
Convey("CheckRepo error - not enough permissions to access root directory", t, func(c C) {
logFile, err := ioutil.TempFile("", "zot-log*.txt")
So(err, ShouldBeNil)
defer os.Remove(logFile.Name()) // clean up
dir := t.TempDir()
log := log.NewLogger("debug", logFile.Name())
metrics := monitoring.NewMetricsServer(false, log)
imgStore := storage.NewImageStore(dir, true, 1*time.Second, true, true, log, metrics)
err = test.CopyFiles("../../../test/data/zot-test", path.Join(dir, repoName))
if err != nil {
panic(err)
}
So(os.Chmod(path.Join(dir, repoName), 0o000), ShouldBeNil)
scrub.RunScrubRepo(imgStore, repoName, log)
data, err := os.ReadFile(logFile.Name())
So(err, ShouldBeNil)
So(string(data), ShouldContainSubstring,
fmt.Sprintf("error while running scrub for %s", imgStore.RootDir()))
So(os.Chmod(path.Join(dir, repoName), 0o755), ShouldBeNil)
})
}