github/zot
1
0
Fork 0
mirror of https://github.com/project-zot/zot.git synced 2026-06-17 04:48:26 +08:00
Code Issues Packages Projects Releases Wiki Activity

refactor(cache): rewrote/refactored cachedb functionality to use interface (#667)

Browse Source 
Moved boltdb to a driver implementation for such interface
Added CreateCacheDatabaseDriver in controller
Fixed default directory creation (boltDB will only create the file, not the dir
Added coverage tests
Added example config for boltdb
Re-added caching on subpaths, rewrote CreateCacheDatabaseDriver
Fix tests
Made cacheDriver argument mandatory for NewImageStore, added more validation, added defaults
Moved cache interface to own file, removed useRelPaths from config
Got rid of cache config, refactored
Moved cache to own package and folder
Renamed + removed cache factory to backend, replaced CloudCache to RemoteCache
Moved storage constants back to storage package
moved cache interface and factory to storage package, changed remoteCache defaulting

Signed-off-by: Catalin Hofnar <catalin.hofnar@gmail.com>
This commit is contained in:
Catalin-George Hofnar
2022-11-03 00:53:08 +02:00
committed by GitHub
parent e6539290d4
commit 4170d2adbc
31 changed files with 1204 additions and 468 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/extensions/lint/lint_test.go
+7 -7
View File
@@ -495,7 +495,7 @@ func TestVerifyMandatoryAnnotationsFunction(t *testing.T) {
linter := lint.NewLinter(lintConfig, log.NewLogger("debug", ""))
imgStore := local.NewImageStore(dir, false, 0, false, false,
log.NewLogger("debug", ""), monitoring.NewMetricsServer(false, log.NewLogger("debug", "")), linter)
log.NewLogger("debug", ""), monitoring.NewMetricsServer(false, log.NewLogger("debug", "")), linter, nil)
indexContent, err := imgStore.GetIndexContent("zot-test")
So(err, ShouldBeNil)
@@ -528,7 +528,7 @@ func TestVerifyMandatoryAnnotationsFunction(t *testing.T) {
linter := lint.NewLinter(lintConfig, log.NewLogger("debug", ""))
imgStore := local.NewImageStore(dir, false, 0, false, false,
log.NewLogger("debug", ""), monitoring.NewMetricsServer(false, log.NewLogger("debug", "")), linter)
log.NewLogger("debug", ""), monitoring.NewMetricsServer(false, log.NewLogger("debug", "")), linter, nil)
indexContent, err := imgStore.GetIndexContent("zot-test")
So(err, ShouldBeNil)
@@ -599,7 +599,7 @@ func TestVerifyMandatoryAnnotationsFunction(t *testing.T) {
linter := lint.NewLinter(lintConfig, log.NewLogger("debug", ""))
imgStore := local.NewImageStore(dir, false, 0, false, false,
log.NewLogger("debug", ""), monitoring.NewMetricsServer(false, log.NewLogger("debug", "")), linter)
log.NewLogger("debug", ""), monitoring.NewMetricsServer(false, log.NewLogger("debug", "")), linter, nil)
pass, err := linter.CheckMandatoryAnnotations("zot-test", digest, imgStore)
So(err, ShouldBeNil)
@@ -662,7 +662,7 @@ func TestVerifyMandatoryAnnotationsFunction(t *testing.T) {
linter := lint.NewLinter(lintConfig, log.NewLogger("debug", ""))
imgStore := local.NewImageStore(dir, false, 0, false, false,
log.NewLogger("debug", ""), monitoring.NewMetricsServer(false, log.NewLogger("debug", "")), linter)
log.NewLogger("debug", ""), monitoring.NewMetricsServer(false, log.NewLogger("debug", "")), linter, nil)
pass, err := linter.CheckMandatoryAnnotations("zot-test", digest, imgStore)
So(err, ShouldBeNil)
@@ -727,7 +727,7 @@ func TestVerifyMandatoryAnnotationsFunction(t *testing.T) {
linter := lint.NewLinter(lintConfig, log.NewLogger("debug", ""))
imgStore := local.NewImageStore(dir, false, 0, false, false,
log.NewLogger("debug", ""), monitoring.NewMetricsServer(false, log.NewLogger("debug", "")), linter)
log.NewLogger("debug", ""), monitoring.NewMetricsServer(false, log.NewLogger("debug", "")), linter, nil)
pass, err := linter.CheckMandatoryAnnotations("zot-test", digest, imgStore)
So(err, ShouldBeNil)
@@ -791,7 +791,7 @@ func TestVerifyMandatoryAnnotationsFunction(t *testing.T) {
linter := lint.NewLinter(lintConfig, log.NewLogger("debug", ""))
imgStore := local.NewImageStore(dir, false, 0, false, false,
log.NewLogger("debug", ""), monitoring.NewMetricsServer(false, log.NewLogger("debug", "")), linter)
log.NewLogger("debug", ""), monitoring.NewMetricsServer(false, log.NewLogger("debug", "")), linter, nil)
err = os.Chmod(path.Join(dir, "zot-test", "blobs"), 0o000)
if err != nil {
@@ -890,7 +890,7 @@ func TestVerifyMandatoryAnnotationsFunction(t *testing.T) {
linter := lint.NewLinter(lintConfig, log.NewLogger("debug", ""))
imgStore := local.NewImageStore(dir, false, 0, false, false,
log.NewLogger("debug", ""), monitoring.NewMetricsServer(false, log.NewLogger("debug", "")), linter)
log.NewLogger("debug", ""), monitoring.NewMetricsServer(false, log.NewLogger("debug", "")), linter, nil)
err = os.Chmod(path.Join(dir, "zot-test", "blobs", "sha256", manifest.Config.Digest.Encoded()), 0o000)
if err != nil {
pkg/extensions/scrub/scrub_test.go
+21 -3
View File
@@ -21,6 +21,8 @@ import (
"zotregistry.io/zot/pkg/extensions/monitoring"
"zotregistry.io/zot/pkg/extensions/scrub"
"zotregistry.io/zot/pkg/log"
"zotregistry.io/zot/pkg/storage"
"zotregistry.io/zot/pkg/storage/cache"
"zotregistry.io/zot/pkg/storage/local"
"zotregistry.io/zot/pkg/test"
)
@@ -239,8 +241,13 @@ func TestRunScrubRepo(t *testing.T) {
dir := t.TempDir()
log := log.NewLogger("debug", logFile.Name())
metrics := monitoring.NewMetricsServer(false, log)
cacheDriver, _ := storage.Create("boltdb", cache.BoltDBDriverParameters{
RootDir: dir,
Name: "cache",
UseRelPaths: true,
}, log)
imgStore := local.NewImageStore(dir, true, 1*time.Second, true,
true, log, metrics, nil)
true, log, metrics, nil, cacheDriver)
err = test.CopyFiles("../../../test/data/zot-test", path.Join(dir, repoName))
if err != nil {
@@ -269,8 +276,13 @@ func TestRunScrubRepo(t *testing.T) {
dir := t.TempDir()
log := log.NewLogger("debug", logFile.Name())
metrics := monitoring.NewMetricsServer(false, log)
cacheDriver, _ := storage.Create("boltdb", cache.BoltDBDriverParameters{
RootDir: dir,
Name: "cache",
UseRelPaths: true,
}, log)
imgStore := local.NewImageStore(dir, true, 1*time.Second, true,
true, log, metrics, nil)
true, log, metrics, nil, cacheDriver)
err = test.CopyFiles("../../../test/data/zot-test", path.Join(dir, repoName))
if err != nil {
@@ -305,8 +317,14 @@ func TestRunScrubRepo(t *testing.T) {
dir := t.TempDir()
log := log.NewLogger("debug", logFile.Name())
metrics := monitoring.NewMetricsServer(false, log)
cacheDriver, _ := storage.Create("boltdb", cache.BoltDBDriverParameters{
RootDir: dir,
Name: "cache",
UseRelPaths: true,
}, log)
imgStore := local.NewImageStore(dir, true, 1*time.Second,
true, true, log, metrics, nil)
true, true, log, metrics, nil, cacheDriver,
)
err = test.CopyFiles("../../../test/data/zot-test", path.Join(dir, repoName))
if err != nil {
pkg/extensions/search/common/common_test.go
+2 -2
View File
@@ -1029,10 +1029,10 @@ func TestUtilsMethod(t *testing.T) {
metrics := monitoring.NewMetricsServer(false, log)
defaultStore := local.NewImageStore(rootDir, false,
storage.DefaultGCDelay, false, false, log, metrics, nil)
storage.DefaultGCDelay, false, false, log, metrics, nil, nil)
subStore := local.NewImageStore(subRootDir, false,
storage.DefaultGCDelay, false, false, log, metrics, nil)
storage.DefaultGCDelay, false, false, log, metrics, nil, nil)
subStoreMap := make(map[string]storage.ImageStore)
pkg/extensions/search/cve/cve_test.go
+2 -2
View File
@@ -88,7 +88,7 @@ func testSetup() error {
conf.Extensions = &extconf.ExtensionConfig{}
conf.Extensions.Lint = &extconf.LintConfig{}
storeController := storage.StoreController{DefaultStore: local.NewImageStore(dir, false, storage.DefaultGCDelay, false, false, log, metrics, nil)}
storeController := storage.StoreController{DefaultStore: local.NewImageStore(dir, false, storage.DefaultGCDelay, false, false, log, metrics, nil, nil)}
layoutUtils := common.NewBaseOciLayoutUtils(storeController, log)
scanner := trivy.NewScanner(storeController, layoutUtils, log)
@@ -332,7 +332,7 @@ func TestImageFormat(t *testing.T) {
metrics := monitoring.NewMetricsServer(false, log)
defaultStore := local.NewImageStore(dbDir, false, storage.DefaultGCDelay,
false, false, log, metrics, nil)
false, false, log, metrics, nil, nil)
storeController := storage.StoreController{DefaultStore: defaultStore}
cveInfo := cveinfo.NewCVEInfo(storeController, log)
pkg/extensions/search/cve/trivy/scanner_internal_test.go
+3 -3
View File
@@ -66,11 +66,11 @@ func TestMultipleStoragePath(t *testing.T) {
conf.Extensions.Lint = &extconf.LintConfig{}
// Create ImageStore
firstStore := local.NewImageStore(firstRootDir, false, storage.DefaultGCDelay, false, false, log, metrics, nil)
firstStore := local.NewImageStore(firstRootDir, false, storage.DefaultGCDelay, false, false, log, metrics, nil, nil)
secondStore := local.NewImageStore(secondRootDir, false, storage.DefaultGCDelay, false, false, log, metrics, nil)
secondStore := local.NewImageStore(secondRootDir, false, storage.DefaultGCDelay, false, false, log, metrics, nil, nil)
thirdStore := local.NewImageStore(thirdRootDir, false, storage.DefaultGCDelay, false, false, log, metrics, nil)
thirdStore := local.NewImageStore(thirdRootDir, false, storage.DefaultGCDelay, false, false, log, metrics, nil, nil)
storeController := storage.StoreController{}
pkg/extensions/search/digest/digest_test.go
+1 -1
View File
@@ -85,7 +85,7 @@ func testSetup(t *testing.T) (string, string, *digestinfo.DigestInfo) {
log := log.NewLogger("debug", "")
metrics := monitoring.NewMetricsServer(false, log)
storeController := storage.StoreController{
DefaultStore: local.NewImageStore(rootDir, false, storage.DefaultGCDelay, false, false, log, metrics, nil),
DefaultStore: local.NewImageStore(rootDir, false, storage.DefaultGCDelay, false, false, log, metrics, nil, nil),
}
digestInfo := digestinfo.NewDigestInfo(storeController, log)
pkg/extensions/search/resolver_test.go
+1 -1
View File
@@ -266,7 +266,7 @@ func TestUserAvailableRepos(t *testing.T) {
log := log.Logger{Logger: zerolog.New(os.Stdout)}
dir := t.TempDir()
metrics := monitoring.NewMetricsServer(false, log)
defaultStore := local.NewImageStore(dir, false, 0, false, false, log, metrics, nil)
defaultStore := local.NewImageStore(dir, false, 0, false, false, log, metrics, nil, nil)
repoList, err := defaultStore.GetRepositories()
So(err, ShouldBeNil)
pkg/extensions/sync/sync_internal_test.go
+6 -6
View File
@@ -65,7 +65,7 @@ func TestInjectSyncUtils(t *testing.T) {
log := log.Logger{Logger: zerolog.New(os.Stdout)}
metrics := monitoring.NewMetricsServer(false, log)
imageStore := local.NewImageStore(t.TempDir(), false, storage.DefaultGCDelay,
false, false, log, metrics, nil,
false, false, log, metrics, nil, nil,
)
injected = test.InjectFailure(0)
@@ -164,7 +164,7 @@ func TestSyncInternal(t *testing.T) {
metrics := monitoring.NewMetricsServer(false, log)
imageStore := local.NewImageStore(t.TempDir(), false, storage.DefaultGCDelay,
false, false, log, metrics, nil)
false, false, log, metrics, nil, nil)
err := os.Chmod(imageStore.RootDir(), 0o000)
So(err, ShouldBeNil)
@@ -346,7 +346,7 @@ func TestSyncInternal(t *testing.T) {
metrics := monitoring.NewMetricsServer(false, log)
imageStore := local.NewImageStore(storageDir, false, storage.DefaultGCDelay,
false, false, log, metrics, nil)
false, false, log, metrics, nil, nil)
refs := ReferenceList{[]artifactspec.Descriptor{
{
@@ -439,7 +439,7 @@ func TestSyncInternal(t *testing.T) {
metrics := monitoring.NewMetricsServer(false, log)
imageStore := local.NewImageStore(storageDir, false, storage.DefaultGCDelay,
false, false, log, metrics, nil)
false, false, log, metrics, nil, nil)
storeController := storage.StoreController{}
storeController.DefaultStore = imageStore
@@ -461,7 +461,7 @@ func TestSyncInternal(t *testing.T) {
}
testImageStore := local.NewImageStore(testRootDir, false,
storage.DefaultGCDelay, false, false, log, metrics, nil)
storage.DefaultGCDelay, false, false, log, metrics, nil, nil)
manifestContent, _, _, err := testImageStore.GetImageManifest(testImage, testImageTag)
So(err, ShouldBeNil)
@@ -538,7 +538,7 @@ func TestSyncInternal(t *testing.T) {
LintFn: func(repo string, manifestDigest godigest.Digest, imageStore storage.ImageStore) (bool, error) {
return false, nil
},
},
}, nil,
)
err = pushSyncedLocalImage(repo, "latest", testRootDir, imageStoreWithLinter, log)
pkg/extensions/sync/utils.go
+1 -1
View File
@@ -336,7 +336,7 @@ func pushSyncedLocalImage(localRepo, reference, localCachePath string,
metrics := monitoring.NewMetricsServer(false, log)
cacheImageStore := local.NewImageStore(localCachePath, false,
storage.DefaultGCDelay, false, false, log, metrics, nil)
storage.DefaultGCDelay, false, false, log, metrics, nil, nil)
manifestContent, _, mediaType, err := cacheImageStore.GetImageManifest(localRepo, reference)
if err != nil {