mirror of
https://github.com/project-zot/zot.git
synced 2026-06-16 04:17:55 +08:00
fix: remove inline GC and schedule a background task instead (#1610)
* fix: remove inline GC and set a default value of gc interval - remove inline GC - add a default value of GC interval - run the GC periodically by default with the default value if no interval provided - generate GC tasks with a random delay(0-30s) between - add IsReady() method to scheduler.TaskGenerator interface Signed-off-by: Andreea-Lupu <andreealupu1470@yahoo.com> * ci: add test for gc with short interval Signed-off-by: Andreea-Lupu <andreealupu1470@yahoo.com> --------- Signed-off-by: Andreea-Lupu <andreealupu1470@yahoo.com>
This commit is contained in:
Andreea Lupu
@@ -185,7 +185,10 @@ func New() *Config {
|
||||
ReleaseTag: ReleaseTag,
|
||||
BinaryType: BinaryType,
|
||||
Storage: GlobalStorageConfig{
|
||||
StorageConfig: StorageConfig{GC: true, GCDelay: storageConstants.DefaultGCDelay, Dedupe: true},
|
||||
StorageConfig: StorageConfig{
|
||||
GC: true, GCDelay: storageConstants.DefaultGCDelay,
|
||||
GCInterval: storageConstants.DefaultGCInterval, Dedupe: true,
|
||||
},
|
||||
},
|
||||
HTTP: HTTPConfig{Address: "127.0.0.1", Port: "8080", Auth: &AuthConfig{FailDelay: 0}},
|
||||
Log: &LogConfig{Level: "debug"},
|
||||
|
||||
@@ -323,7 +323,7 @@ func (c *Controller) StartBackgroundTasks(reloadCtx context.Context) {
|
||||
taskScheduler.RunScheduler(reloadCtx)
|
||||
|
||||
// Enable running garbage-collect periodically for DefaultStore
|
||||
if c.Config.Storage.GC && c.Config.Storage.GCInterval != 0 {
|
||||
if c.Config.Storage.GC {
|
||||
c.StoreController.DefaultStore.RunGCPeriodically(c.Config.Storage.GCInterval, taskScheduler)
|
||||
}
|
||||
|
||||
@@ -339,7 +339,7 @@ func (c *Controller) StartBackgroundTasks(reloadCtx context.Context) {
|
||||
if c.Config.Storage.SubPaths != nil {
|
||||
for route, storageConfig := range c.Config.Storage.SubPaths {
|
||||
// Enable running garbage-collect periodically for subImageStore
|
||||
if storageConfig.GC && storageConfig.GCInterval != 0 {
|
||||
if storageConfig.GC {
|
||||
c.StoreController.SubStore[route].RunGCPeriodically(storageConfig.GCInterval, taskScheduler)
|
||||
}
|
||||
|
||||
|
||||
+17
-44
@@ -4514,6 +4514,7 @@ func TestCrossRepoMount(t *testing.T) {
|
||||
cm.StopServer()
|
||||
|
||||
ctlr.Config.Storage.Dedupe = true
|
||||
ctlr.Config.Storage.GC = false
|
||||
ctlr.Config.Storage.RootDirectory = newDir
|
||||
cm = test.NewControllerManager(ctlr) //nolint: varnamelen
|
||||
cm.StartAndWait(port)
|
||||
@@ -7363,48 +7364,6 @@ func TestInjectTooManyOpenFiles(t *testing.T) {
|
||||
So(resp.StatusCode, ShouldEqual, http.StatusCreated)
|
||||
}
|
||||
})
|
||||
Convey("code coverage: error inside PutImageManifest method of img store (umoci.OpenLayout error)", func() {
|
||||
injected := inject.InjectFailure(3)
|
||||
|
||||
request, _ := http.NewRequestWithContext(context.TODO(), http.MethodPut, baseURL, bytes.NewReader(content))
|
||||
request = mux.SetURLVars(request, map[string]string{"name": "repotest", "reference": "1.0"})
|
||||
request.Header.Set("Content-Type", "application/vnd.oci.image.manifest.v1+json")
|
||||
response := httptest.NewRecorder()
|
||||
|
||||
rthdlr.UpdateManifest(response, request)
|
||||
|
||||
resp := response.Result()
|
||||
defer resp.Body.Close()
|
||||
|
||||
So(resp, ShouldNotBeNil)
|
||||
|
||||
if injected {
|
||||
So(resp.StatusCode, ShouldEqual, http.StatusInternalServerError)
|
||||
} else {
|
||||
So(resp.StatusCode, ShouldEqual, http.StatusCreated)
|
||||
}
|
||||
})
|
||||
Convey("code coverage: error inside PutImageManifest method of img store (oci.GC)", func() {
|
||||
injected := inject.InjectFailure(4)
|
||||
|
||||
request, _ := http.NewRequestWithContext(context.TODO(), http.MethodPut, baseURL, bytes.NewReader(content))
|
||||
request = mux.SetURLVars(request, map[string]string{"name": "repotest", "reference": "1.0"})
|
||||
request.Header.Set("Content-Type", "application/vnd.oci.image.manifest.v1+json")
|
||||
response := httptest.NewRecorder()
|
||||
|
||||
rthdlr.UpdateManifest(response, request)
|
||||
|
||||
resp := response.Result()
|
||||
defer resp.Body.Close()
|
||||
|
||||
So(resp, ShouldNotBeNil)
|
||||
|
||||
if injected {
|
||||
So(resp.StatusCode, ShouldEqual, http.StatusInternalServerError)
|
||||
} else {
|
||||
So(resp.StatusCode, ShouldEqual, http.StatusCreated)
|
||||
}
|
||||
})
|
||||
Convey("when index.json is not in json format", func() {
|
||||
resp, err = resty.R().SetHeader("Content-Type", "application/vnd.oci.image.manifest.v1+json").
|
||||
SetBody(content).Put(baseURL + "/v2/repotest/manifests/v1.0")
|
||||
@@ -7447,6 +7406,8 @@ func TestGCSignaturesAndUntaggedManifests(t *testing.T) {
|
||||
ctlr.Config.Storage.GC = true
|
||||
ctlr.Config.Storage.GCDelay = 1 * time.Millisecond
|
||||
|
||||
ctlr.Config.Storage.Dedupe = false
|
||||
|
||||
test.CopyTestFiles("../../test/data/zot-test", path.Join(dir, repoName))
|
||||
|
||||
cm := test.NewControllerManager(ctlr)
|
||||
@@ -7530,6 +7491,9 @@ func TestGCSignaturesAndUntaggedManifests(t *testing.T) {
|
||||
img := test.CreateRandomImage()
|
||||
|
||||
err = test.UploadImage(img, baseURL, repoName, img.DigestStr())
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
err = ctlr.StoreController.DefaultStore.RunGCRepo(repoName)
|
||||
So(err, ShouldNotBeNil)
|
||||
|
||||
err = os.Chmod(path.Join(dir, repoName, "blobs", "sha256", refs.Manifests[0].Digest.Encoded()), 0o755)
|
||||
@@ -7541,6 +7505,9 @@ func TestGCSignaturesAndUntaggedManifests(t *testing.T) {
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
err = test.UploadImage(img, baseURL, repoName, tag)
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
err = ctlr.StoreController.DefaultStore.RunGCRepo(repoName)
|
||||
So(err, ShouldNotBeNil)
|
||||
|
||||
err = os.WriteFile(path.Join(dir, repoName, "blobs", "sha256", refs.Manifests[0].Digest.Encoded()), content, 0o600)
|
||||
@@ -7579,6 +7546,9 @@ func TestGCSignaturesAndUntaggedManifests(t *testing.T) {
|
||||
So(err, ShouldBeNil)
|
||||
newManifestDigest := godigest.FromBytes(manifestBuf)
|
||||
|
||||
err = ctlr.StoreController.DefaultStore.RunGCRepo(repoName)
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
// both signatures should be gc'ed
|
||||
resp, err = resty.R().Get(baseURL + fmt.Sprintf("/v2/%s/manifests/%s", repoName, cosignTag))
|
||||
So(err, ShouldBeNil)
|
||||
@@ -7669,6 +7639,9 @@ func TestGCSignaturesAndUntaggedManifests(t *testing.T) {
|
||||
So(err, ShouldBeNil)
|
||||
So(resp.StatusCode(), ShouldEqual, http.StatusCreated)
|
||||
|
||||
err = ctlr.StoreController.DefaultStore.RunGCRepo(repoName)
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
resp, err = resty.R().SetHeader("Content-Type", ispec.MediaTypeImageIndex).
|
||||
Get(baseURL + fmt.Sprintf("/v2/%s/manifests/latest", repoName))
|
||||
So(err, ShouldBeNil)
|
||||
@@ -7752,9 +7725,9 @@ func TestPeriodicGC(t *testing.T) {
|
||||
|
||||
data, err := os.ReadFile(logFile.Name())
|
||||
So(err, ShouldBeNil)
|
||||
// periodic GC is not enabled for default store
|
||||
// periodic GC is enabled by default for default store with a default interval
|
||||
So(string(data), ShouldContainSubstring,
|
||||
"\"GCDelay\":3600000000000,\"GCInterval\":0,\"")
|
||||
"\"GCDelay\":3600000000000,\"GCInterval\":3600000000000,\"")
|
||||
// periodic GC is enabled for sub store
|
||||
So(string(data), ShouldContainSubstring,
|
||||
fmt.Sprintf("\"SubPaths\":{\"/a\":{\"RootDirectory\":\"%s\",\"Dedupe\":false,\"RemoteCache\":false,\"GC\":true,\"Commit\":false,\"GCDelay\":1000000000,\"GCInterval\":86400000000000", subDir)) //nolint:lll // gofumpt conflicts with lll
|
||||
|
||||
Reference in New Issue
Block a user