github/zot
1
0
Fork 0
mirror of https://github.com/project-zot/zot.git synced 2026-06-16 04:17:55 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(storage): rebuild storage(s3/local) dedupe index when switching dedupe status (#1062)

Browse Source 
Signed-off-by: Petu Eusebiu <peusebiu@cisco.com>
This commit is contained in:
peusebiu
2023-04-07 19:49:24 +03:00
committed by GitHub
parent 96232bb11c
commit f35ff53146
23 changed files with 2520 additions and 203 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/api/controller.go
+15 -1
View File
@@ -451,11 +451,16 @@ func getUseRelPaths(storageConfig *config.StorageConfig) bool {
}
func CreateCacheDatabaseDriver(storageConfig config.StorageConfig, log log.Logger) cache.Cache {
if storageConfig.Dedupe {
if storageConfig.Dedupe || storageConfig.StorageDriver != nil {
if !storageConfig.RemoteCache {
params := cache.BoltDBDriverParameters{}
params.RootDir = storageConfig.RootDirectory
params.Name = constants.BoltdbName
if storageConfig.StorageDriver != nil {
params.Name = s3.CacheDBName
}
params.UseRelPaths = getUseRelPaths(&storageConfig)
driver, _ := storage.Create("boltdb", params, log)
@@ -548,6 +553,9 @@ func (c *Controller) StartBackgroundTasks(reloadCtx context.Context) {
c.StoreController.DefaultStore.RunGCPeriodically(c.Config.Storage.GCInterval, taskScheduler)
}
// Enable running dedupe blobs both ways (dedupe or restore deduped blobs)
c.StoreController.DefaultStore.RunDedupeBlobs(time.Duration(0), taskScheduler)
// Enable extensions if extension config is provided for DefaultStore
if c.Config != nil && c.Config.Extensions != nil {
ext.EnableMetricsExtension(c.Config, c.Log, c.Config.Storage.RootDirectory)
@@ -565,6 +573,12 @@ func (c *Controller) StartBackgroundTasks(reloadCtx context.Context) {
if c.Config != nil && c.Config.Extensions != nil {
ext.EnableMetricsExtension(c.Config, c.Log, storageConfig.RootDirectory)
}
// Enable running dedupe blobs both ways (dedupe or restore deduped blobs) for subpaths
substore := c.StoreController.SubStore[route]
if substore != nil {
substore.RunDedupeBlobs(time.Duration(0), taskScheduler)
}
}
}
pkg/api/controller_test.go
+14 -6
View File
@@ -3345,10 +3345,10 @@ func TestCrossRepoMount(t *testing.T) {
dir := t.TempDir()
ctlr := makeController(conf, dir, "../../test/data")
ctlr.Config.Storage.RemoteCache = false
ctlr.Config.Storage.Dedupe = false
cm := test.NewControllerManager(ctlr)
cm := test.NewControllerManager(ctlr) //nolint: varnamelen
cm.StartAndWait(port)
defer cm.StopServer()
params := make(map[string]string)
@@ -3449,6 +3449,14 @@ func TestCrossRepoMount(t *testing.T) {
// in cache, now try mount blob request status and it should be 201 because now blob is present in cache
// and it should do hard link.
// restart server with dedupe enabled
cm.StopServer()
ctlr.Config.Storage.Dedupe = true
cm.StartAndWait(port)
// wait for dedupe task to run
time.Sleep(15 * time.Second)
params["mount"] = string(manifestDigest)
postResponse, err = client.R().
SetBasicAuth(username, passphrase).SetQueryParams(params).
@@ -6479,7 +6487,7 @@ func TestPeriodicGC(t *testing.T) {
ctlr := api.NewController(conf)
dir := t.TempDir()
ctlr.Config.Storage.RootDirectory = dir
ctlr.Config.Storage.Dedupe = false
ctlr.Config.Storage.GC = true
ctlr.Config.Storage.GCInterval = 1 * time.Hour
ctlr.Config.Storage.GCDelay = 1 * time.Second
@@ -6518,8 +6526,8 @@ func TestPeriodicGC(t *testing.T) {
subPaths := make(map[string]config.StorageConfig)
subPaths["/a"] = config.StorageConfig{RootDirectory: subDir, GC: true, GCDelay: 1 * time.Second, GCInterval: 24 * time.Hour, RemoteCache: false} //nolint:lll // gofumpt conflicts with lll
subPaths["/a"] = config.StorageConfig{RootDirectory: subDir, GC: true, GCDelay: 1 * time.Second, GCInterval: 24 * time.Hour, RemoteCache: false, Dedupe: false} //nolint:lll // gofumpt conflicts with lll
ctlr.Config.Storage.Dedupe = false
ctlr.Config.Storage.SubPaths = subPaths
cm := test.NewControllerManager(ctlr)
@@ -6552,8 +6560,8 @@ func TestPeriodicGC(t *testing.T) {
ctlr := api.NewController(conf)
dir := t.TempDir()
ctlr.Config.Storage.RootDirectory = dir
ctlr.Config.Storage.Dedupe = false
ctlr.Config.Storage.RootDirectory = dir
ctlr.Config.Storage.GC = true
ctlr.Config.Storage.GCInterval = 1 * time.Hour
ctlr.Config.Storage.GCDelay = 1 * time.Second