github/zot
1
0
Fork 0
mirror of https://github.com/project-zot/zot.git synced 2026-06-16 04:17:55 +08:00
Code Issues Packages Projects Releases Wiki Activity

make gc periodic

Browse Source 
Signed-off-by: Andreea-Lupu <andreealupu1470@yahoo.com>
This commit is contained in:
Andreea-Lupu
2022-03-21 20:40:37 +02:00
committed by Ramkumar Chinchani
parent 89c5f4f604
commit 5e35dfa28f
10 changed files with 327 additions and 36 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/api/config/config.go
+2
View File
@@ -25,6 +25,7 @@ type StorageConfig struct {
Dedupe bool
Commit bool
GCDelay time.Duration
GCInterval time.Duration
StorageDriver map[string]interface{} `mapstructure:",omitempty"`
}
@@ -99,6 +100,7 @@ type GlobalStorageConfig struct {
GC bool
Commit bool
GCDelay time.Duration
GCInterval time.Duration
RootDirectory string
StorageDriver map[string]interface{} `mapstructure:",omitempty"`
SubPaths map[string]StorageConfig
pkg/api/controller.go
+38 -18
View File
@@ -259,11 +259,6 @@ func (c *Controller) InitImageStore(reloadCtx context.Context) error {
}
c.StoreController.DefaultStore = defaultStore
// Enable extensions if extension config is provided
if c.Config != nil && c.Config.Extensions != nil {
ext.EnableExtensions(c.Config, c.Log, c.Config.Storage.RootDirectory)
}
} else {
// we can't proceed without global storage
c.Log.Error().Err(errors.ErrImgStoreNotFound).Msg("controller: no storage config provided")
@@ -309,25 +304,13 @@ func (c *Controller) InitImageStore(reloadCtx context.Context) error {
subImageStore[route] = s3.NewImageStore(storageConfig.RootDirectory,
storageConfig.GC, storageConfig.GCDelay, storageConfig.Dedupe, storageConfig.Commit, c.Log, c.Metrics, store)
}
// Enable extensions if extension config is provided
if c.Config != nil && c.Config.Extensions != nil {
ext.EnableExtensions(c.Config, c.Log, storageConfig.RootDirectory)
}
}
c.StoreController.SubStore = subImageStore
}
}
// Enable extensions if extension config is provided
if c.Config.Extensions != nil && c.Config.Extensions.Sync != nil && *c.Config.Extensions.Sync.Enable {
ext.EnableSyncExtension(reloadCtx, c.Config, c.wgShutDown, c.StoreController, c.Log)
}
if c.Config.Extensions != nil {
ext.EnableScrubExtension(c.Config, c.StoreController, c.Log)
}
c.StartBackgroundTasks(reloadCtx)
return nil
}
@@ -356,3 +339,40 @@ func (c *Controller) Shutdown() {
ctx := context.Background()
_ = c.Server.Shutdown(ctx)
}
func (c *Controller) StartBackgroundTasks(reloadCtx context.Context) {
// Enable running garbage-collect periodically for DefaultStore
if c.Config.Storage.GC && c.Config.Storage.GCInterval != 0 {
c.StoreController.DefaultStore.RunGCPeriodically(c.Config.Storage.GCInterval)
}
// Enable extensions if extension config is provided for DefaultStore
if c.Config != nil && c.Config.Extensions != nil {
ext.EnableExtensions(c.Config, c.Log, c.Config.Storage.RootDirectory)
}
if c.Config.Storage.SubPaths != nil {
for route, storageConfig := range c.Config.Storage.SubPaths {
// Enable running garbage-collect periodically for subImageStore
if storageConfig.GC && storageConfig.GCInterval != 0 {
c.StoreController.SubStore[route].RunGCPeriodically(storageConfig.GCInterval)
}
// Enable extensions if extension config is provided for subImageStore
if c.Config != nil && c.Config.Extensions != nil {
ext.EnableExtensions(c.Config, c.Log, storageConfig.RootDirectory)
}
}
}
// Enable extensions if extension config is provided for storeController
if c.Config.Extensions != nil {
if c.Config.Extensions.Sync != nil && *c.Config.Extensions.Sync.Enable {
ext.EnableSyncExtension(reloadCtx, c.Config, c.wgShutDown, c.StoreController, c.Log)
}
}
if c.Config.Extensions != nil {
ext.EnableScrubExtension(c.Config, c.StoreController, c.Log)
}
}
pkg/api/controller_test.go
+76
View File
@@ -4472,6 +4472,82 @@ func TestInjectTooManyOpenFiles(t *testing.T) {
})
}
func TestPeriodicGC(t *testing.T) {
Convey("Periodic gc enabled for default store", t, func() {
port := test.GetFreePort()
baseURL := test.GetBaseURL(port)
conf := config.New()
conf.HTTP.Port = port
logFile, err := ioutil.TempFile("", "zot-log*.txt")
So(err, ShouldBeNil)
conf.Log.Level = "debug"
conf.Log.Output = logFile.Name()
defer os.Remove(logFile.Name()) // clean up
ctlr := api.NewController(conf)
dir := t.TempDir()
ctlr.Config.Storage.RootDirectory = dir
ctlr.Config.Storage.GC = true
ctlr.Config.Storage.GCInterval = 1 * time.Hour
ctlr.Config.Storage.GCDelay = 1 * time.Second
go startServer(ctlr)
defer stopServer(ctlr)
test.WaitTillServerReady(baseURL)
data, err := os.ReadFile(logFile.Name())
So(err, ShouldBeNil)
So(string(data), ShouldContainSubstring,
"\"GC\":true,\"Commit\":false,\"GCDelay\":1000000000,\"GCInterval\":3600000000000")
So(string(data), ShouldContainSubstring,
fmt.Sprintf("executing GC of orphaned blobs for %s", ctlr.StoreController.DefaultStore.RootDir()))
So(string(data), ShouldNotContainSubstring,
fmt.Sprintf("error while running GC for %s", ctlr.StoreController.DefaultStore.RootDir()))
So(string(data), ShouldContainSubstring,
fmt.Sprintf("GC completed for %s, next GC scheduled after", ctlr.StoreController.DefaultStore.RootDir()))
})
Convey("Periodic GC enabled for substore", t, func() {
port := test.GetFreePort()
baseURL := test.GetBaseURL(port)
conf := config.New()
conf.HTTP.Port = port
logFile, err := ioutil.TempFile("", "zot-log*.txt")
So(err, ShouldBeNil)
conf.Log.Level = "debug"
conf.Log.Output = logFile.Name()
defer os.Remove(logFile.Name()) // clean up
ctlr := api.NewController(conf)
dir := t.TempDir()
subDir := t.TempDir()
subPaths := make(map[string]config.StorageConfig)
subPaths["/a"] = config.StorageConfig{RootDirectory: subDir, GC: true, GCDelay: 1 * time.Second, GCInterval: 24 * time.Hour} // nolint:lll
ctlr.Config.Storage.SubPaths = subPaths
ctlr.Config.Storage.RootDirectory = dir
go startServer(ctlr)
defer stopServer(ctlr)
test.WaitTillServerReady(baseURL)
data, err := os.ReadFile(logFile.Name())
So(err, ShouldBeNil)
// periodic GC is not enabled for default store
So(string(data), ShouldContainSubstring,
"\"GCDelay\":3600000000000,\"GCInterval\":0,\"RootDirectory\":\""+dir+"\"")
// periodic GC is enabled for sub store
So(string(data), ShouldContainSubstring,
fmt.Sprintf("\"SubPaths\":{\"/a\":{\"RootDirectory\":\"%s\",\"GC\":true,\"Dedupe\":false,\"Commit\":false,\"GCDelay\":1000000000,\"GCInterval\":86400000000000", subDir)) // nolint:lll
So(string(data), ShouldContainSubstring,
fmt.Sprintf("executing GC of orphaned blobs for %s", ctlr.StoreController.SubStore["/a"].RootDir()))
})
}
func getAllBlobs(imagePath string) []string {
blobList := make([]string, 0)