github/zot
1
0
Fork 0
mirror of https://github.com/project-zot/zot.git synced 2026-06-16 20:38:08 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix(gc): sync repodb when gc'ing manifests (#1819)

Browse Source 
fix(gc): fix cleaning deduped blobs because they have the modTime of
the original blobs, fixed by updating the modTime when hard linking
the blobs.
fix(gc): failing to parse rootDir at zot startup when using s3 storage
because there are no files under rootDir and we can not create empty dirs
on s3, fixed by creating an empty file under rootDir.

Signed-off-by: Petu Eusebiu <peusebiu@cisco.com>
This commit is contained in:
peusebiu
2023-09-22 21:51:20 +03:00
committed by GitHub
parent 7c78f80a96
commit 1df743f173
39 changed files with 2050 additions and 1266 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/extensions/extension_image_trust_test.go
+5 -5
View File
@@ -195,7 +195,7 @@ func RunSignatureUploadAndVerificationTests(t *testing.T, cacheDriverParams map[
writers := io.MultiWriter(os.Stdout, logFile)
logger.Logger = logger.Output(writers)
imageStore := local.NewImageStore(globalDir, false, false, 0, 0, false, false,
imageStore := local.NewImageStore(globalDir, false, false,
logger, monitoring.NewMetricsServer(false, logger), nil, nil)
storeController := storage.StoreController{
@@ -315,7 +315,7 @@ func RunSignatureUploadAndVerificationTests(t *testing.T, cacheDriverParams map[
writers := io.MultiWriter(os.Stdout, logFile)
logger.Logger = logger.Output(writers)
imageStore := local.NewImageStore(globalDir, false, false, 0, 0, false, false,
imageStore := local.NewImageStore(globalDir, false, false,
logger, monitoring.NewMetricsServer(false, logger), nil, nil)
storeController := storage.StoreController{
@@ -422,7 +422,7 @@ func RunSignatureUploadAndVerificationTests(t *testing.T, cacheDriverParams map[
writers := io.MultiWriter(os.Stdout, logFile)
logger.Logger = logger.Output(writers)
imageStore := local.NewImageStore(globalDir, false, false, 0, 0, false, false,
imageStore := local.NewImageStore(globalDir, false, false,
logger, monitoring.NewMetricsServer(false, logger), nil, nil)
storeController := storage.StoreController{
@@ -584,7 +584,7 @@ func RunSignatureUploadAndVerificationTests(t *testing.T, cacheDriverParams map[
writers := io.MultiWriter(os.Stdout, logFile)
logger.Logger = logger.Output(writers)
imageStore := local.NewImageStore(globalDir, false, false, 0, 0, false, false,
imageStore := local.NewImageStore(globalDir, false, false,
logger, monitoring.NewMetricsServer(false, logger), nil, nil)
storeController := storage.StoreController{
@@ -845,7 +845,7 @@ func RunSignatureUploadAndVerificationTests(t *testing.T, cacheDriverParams map[
writers := io.MultiWriter(os.Stdout, logFile)
logger.Logger = logger.Output(writers)
imageStore := local.NewImageStore(globalDir, false, false, 0, 0, false, false,
imageStore := local.NewImageStore(globalDir, false, false,
logger, monitoring.NewMetricsServer(false, logger), nil, nil)
storeController := storage.StoreController{
pkg/extensions/lint/lint_test.go
+7 -7
View File
@@ -492,7 +492,7 @@ func TestVerifyMandatoryAnnotationsFunction(t *testing.T) {
var index ispec.Index
linter := lint.NewLinter(lintConfig, log.NewLogger("debug", ""))
imgStore := local.NewImageStore(dir, false, false, 0, 0, false, false,
imgStore := local.NewImageStore(dir, false, false,
log.NewLogger("debug", ""), monitoring.NewMetricsServer(false, log.NewLogger("debug", "")), linter, nil)
indexContent, err := imgStore.GetIndexContent("zot-test")
@@ -524,7 +524,7 @@ func TestVerifyMandatoryAnnotationsFunction(t *testing.T) {
var index ispec.Index
linter := lint.NewLinter(lintConfig, log.NewLogger("debug", ""))
imgStore := local.NewImageStore(dir, false, false, 0, 0, false, false,
imgStore := local.NewImageStore(dir, false, false,
log.NewLogger("debug", ""), monitoring.NewMetricsServer(false, log.NewLogger("debug", "")), linter, nil)
indexContent, err := imgStore.GetIndexContent("zot-test")
@@ -594,7 +594,7 @@ func TestVerifyMandatoryAnnotationsFunction(t *testing.T) {
index.Manifests = append(index.Manifests, manifestDesc)
linter := lint.NewLinter(lintConfig, log.NewLogger("debug", ""))
imgStore := local.NewImageStore(dir, false, false, 0, 0, false, false,
imgStore := local.NewImageStore(dir, false, false,
log.NewLogger("debug", ""), monitoring.NewMetricsServer(false, log.NewLogger("debug", "")), linter, nil)
pass, err := linter.CheckMandatoryAnnotations("zot-test", digest, imgStore)
@@ -656,7 +656,7 @@ func TestVerifyMandatoryAnnotationsFunction(t *testing.T) {
index.Manifests = append(index.Manifests, manifestDesc)
linter := lint.NewLinter(lintConfig, log.NewLogger("debug", ""))
imgStore := local.NewImageStore(dir, false, false, 0, 0, false, false,
imgStore := local.NewImageStore(dir, false, false,
log.NewLogger("debug", ""), monitoring.NewMetricsServer(false, log.NewLogger("debug", "")), linter, nil)
pass, err := linter.CheckMandatoryAnnotations("zot-test", digest, imgStore)
@@ -720,7 +720,7 @@ func TestVerifyMandatoryAnnotationsFunction(t *testing.T) {
index.Manifests = append(index.Manifests, manifestDesc)
linter := lint.NewLinter(lintConfig, log.NewLogger("debug", ""))
imgStore := local.NewImageStore(dir, false, false, 0, 0, false, false,
imgStore := local.NewImageStore(dir, false, false,
log.NewLogger("debug", ""), monitoring.NewMetricsServer(false, log.NewLogger("debug", "")), linter, nil)
pass, err := linter.CheckMandatoryAnnotations("zot-test", digest, imgStore)
@@ -783,7 +783,7 @@ func TestVerifyMandatoryAnnotationsFunction(t *testing.T) {
index.Manifests = append(index.Manifests, manifestDesc)
linter := lint.NewLinter(lintConfig, log.NewLogger("debug", ""))
imgStore := local.NewImageStore(dir, false, false, 0, 0, false, false,
imgStore := local.NewImageStore(dir, false, false,
log.NewLogger("debug", ""), monitoring.NewMetricsServer(false, log.NewLogger("debug", "")), linter, nil)
err = os.Chmod(path.Join(dir, "zot-test", "blobs"), 0o000)
@@ -881,7 +881,7 @@ func TestVerifyMandatoryAnnotationsFunction(t *testing.T) {
index.Manifests = append(index.Manifests, manifestDesc)
linter := lint.NewLinter(lintConfig, log.NewLogger("debug", ""))
imgStore := local.NewImageStore(dir, false, false, 0, 0, false, false,
imgStore := local.NewImageStore(dir, false, false,
log.NewLogger("debug", ""), monitoring.NewMetricsServer(false, log.NewLogger("debug", "")), linter, nil)
err = os.Chmod(path.Join(dir, "zot-test", "blobs", "sha256", manifest.Config.Digest.Encoded()), 0o000)
pkg/extensions/scrub/scrub_test.go
+3 -5
View File
@@ -200,7 +200,7 @@ func TestRunScrubRepo(t *testing.T) {
Name: "cache",
UseRelPaths: true,
}, log)
imgStore := local.NewImageStore(dir, true, true, 1*time.Second, 1*time.Second, true,
imgStore := local.NewImageStore(dir, true,
true, log, metrics, nil, cacheDriver)
srcStorageCtlr := test.GetDefaultStoreController(dir, log)
@@ -236,7 +236,7 @@ func TestRunScrubRepo(t *testing.T) {
Name: "cache",
UseRelPaths: true,
}, log)
imgStore := local.NewImageStore(dir, true, true, 1*time.Second, 1*time.Second, true,
imgStore := local.NewImageStore(dir, true,
true, log, metrics, nil, cacheDriver)
srcStorageCtlr := test.GetDefaultStoreController(dir, log)
@@ -278,9 +278,7 @@ func TestRunScrubRepo(t *testing.T) {
Name: "cache",
UseRelPaths: true,
}, log)
imgStore := local.NewImageStore(dir, true, true, 1*time.Second,
1*time.Second, true, true, log, metrics, nil, cacheDriver,
)
imgStore := local.NewImageStore(dir, true, true, log, metrics, nil, cacheDriver)
srcStorageCtlr := test.GetDefaultStoreController(dir, log)
image := CreateDefaultVulnerableImage()
pkg/extensions/search/cve/cve_test.go
+1 -3
View File
@@ -37,7 +37,6 @@ import (
"zotregistry.io/zot/pkg/meta/boltdb"
mTypes "zotregistry.io/zot/pkg/meta/types"
"zotregistry.io/zot/pkg/storage"
storageConstants "zotregistry.io/zot/pkg/storage/constants"
"zotregistry.io/zot/pkg/storage/local"
. "zotregistry.io/zot/pkg/test"
. "zotregistry.io/zot/pkg/test/image-utils"
@@ -323,8 +322,7 @@ func TestImageFormat(t *testing.T) {
dbDir := t.TempDir()
metrics := monitoring.NewMetricsServer(false, log)
defaultStore := local.NewImageStore(imgDir, false, false, storageConstants.DefaultGCDelay,
storageConstants.DefaultUntaggedImgeRetentionDelay, false, false, log, metrics, nil, nil)
defaultStore := local.NewImageStore(imgDir, false, false, log, metrics, nil, nil)
storeController := storage.StoreController{DefaultStore: defaultStore}
params := boltdb.DBParameters{
pkg/extensions/search/cve/trivy/scanner_internal_test.go
+6 -13
View File
@@ -24,7 +24,6 @@ import (
"zotregistry.io/zot/pkg/meta/boltdb"
mTypes "zotregistry.io/zot/pkg/meta/types"
"zotregistry.io/zot/pkg/storage"
storageConstants "zotregistry.io/zot/pkg/storage/constants"
"zotregistry.io/zot/pkg/storage/imagestore"
"zotregistry.io/zot/pkg/storage/local"
storageTypes "zotregistry.io/zot/pkg/storage/types"
@@ -75,14 +74,11 @@ func TestMultipleStoragePath(t *testing.T) {
// Create ImageStore
firstStore := local.NewImageStore(firstRootDir, false, false, storageConstants.DefaultGCDelay,
storageConstants.DefaultUntaggedImgeRetentionDelay, false, false, log, metrics, nil, nil)
firstStore := local.NewImageStore(firstRootDir, false, false, log, metrics, nil, nil)
secondStore := local.NewImageStore(secondRootDir, false, false, storageConstants.DefaultGCDelay,
storageConstants.DefaultUntaggedImgeRetentionDelay, false, false, log, metrics, nil, nil)
secondStore := local.NewImageStore(secondRootDir, false, false, log, metrics, nil, nil)
thirdStore := local.NewImageStore(thirdRootDir, false, false, storageConstants.DefaultGCDelay,
storageConstants.DefaultUntaggedImgeRetentionDelay, false, false, log, metrics, nil, nil)
thirdStore := local.NewImageStore(thirdRootDir, false, false, log, metrics, nil, nil)
storeController := storage.StoreController{}
@@ -190,8 +186,7 @@ func TestTrivyLibraryErrors(t *testing.T) {
metrics := monitoring.NewMetricsServer(false, log)
// Create ImageStore
store := local.NewImageStore(rootDir, false, false, storageConstants.DefaultGCDelay,
storageConstants.DefaultUntaggedImgeRetentionDelay, false, false, log, metrics, nil, nil)
store := local.NewImageStore(rootDir, false, false, log, metrics, nil, nil)
storeController := storage.StoreController{}
storeController.DefaultStore = store
@@ -408,8 +403,7 @@ func TestImageScannable(t *testing.T) {
// Continue with initializing the objects the scanner depends on
metrics := monitoring.NewMetricsServer(false, log)
store := local.NewImageStore(rootDir, false, false, storageConstants.DefaultGCDelay,
storageConstants.DefaultUntaggedImgeRetentionDelay, false, false, log, metrics, nil, nil)
store := local.NewImageStore(rootDir, false, false, log, metrics, nil, nil)
storeController := storage.StoreController{}
storeController.DefaultStore = store
@@ -475,8 +469,7 @@ func TestDefaultTrivyDBUrl(t *testing.T) {
metrics := monitoring.NewMetricsServer(false, log)
// Create ImageStore
store := local.NewImageStore(rootDir, false, false, storageConstants.DefaultGCDelay,
storageConstants.DefaultUntaggedImgeRetentionDelay, false, false, log, metrics, nil, nil)
store := local.NewImageStore(rootDir, false, false, log, metrics, nil, nil)
storeController := storage.StoreController{}
storeController.DefaultStore = store
pkg/extensions/search/cve/trivy/scanner_test.go
+1 -1
View File
@@ -186,7 +186,7 @@ func TestVulnerableLayer(t *testing.T) {
tempDir := t.TempDir()
log := log.NewLogger("debug", "")
imageStore := local.NewImageStore(tempDir, false, false, 0, 0, false, false,
imageStore := local.NewImageStore(tempDir, false, false,
log, monitoring.NewMetricsServer(false, log), nil, nil)
storeController := storage.StoreController{
pkg/extensions/search/search_test.go
+6 -10
View File
@@ -40,7 +40,6 @@ import (
"zotregistry.io/zot/pkg/log"
mTypes "zotregistry.io/zot/pkg/meta/types"
"zotregistry.io/zot/pkg/storage"
storageConstants "zotregistry.io/zot/pkg/storage/constants"
"zotregistry.io/zot/pkg/storage/local"
storageTypes "zotregistry.io/zot/pkg/storage/types"
. "zotregistry.io/zot/pkg/test"
@@ -1162,7 +1161,7 @@ func TestExpandedRepoInfo(t *testing.T) {
ctlr := api.NewController(conf)
imageStore := local.NewImageStore(tempDir, false, false, 0, 0, false, false,
imageStore := local.NewImageStore(tempDir, false, false,
log.NewLogger("debug", ""), monitoring.NewMetricsServer(false, log.NewLogger("debug", "")), nil, nil)
storeController := storage.StoreController{
@@ -1290,8 +1289,7 @@ func TestExpandedRepoInfo(t *testing.T) {
log := log.NewLogger("debug", "")
metrics := monitoring.NewMetricsServer(false, log)
testStorage := local.NewImageStore(rootDir, false, false, storageConstants.DefaultGCDelay,
storageConstants.DefaultUntaggedImgeRetentionDelay, false, false, log, metrics, nil, nil)
testStorage := local.NewImageStore(rootDir, false, false, log, metrics, nil, nil)
resp, err := resty.R().Get(baseURL + "/v2/")
So(resp, ShouldNotBeNil)
@@ -1636,7 +1634,7 @@ func TestExpandedRepoInfo(t *testing.T) {
conf.Extensions.Search.CVE = nil
ctlr := api.NewController(conf)
imageStore := local.NewImageStore(conf.Storage.RootDirectory, false, false, 0, 0, false, false,
imageStore := local.NewImageStore(conf.Storage.RootDirectory, false, false,
log.NewLogger("debug", ""), monitoring.NewMetricsServer(false, log.NewLogger("debug", "")), nil, nil)
storeController := storage.StoreController{
@@ -5387,8 +5385,7 @@ func TestMetaDBWhenDeletingImages(t *testing.T) {
// get signatur digest
log := log.NewLogger("debug", "")
metrics := monitoring.NewMetricsServer(false, log)
storage := local.NewImageStore(dir, false, false, storageConstants.DefaultGCDelay,
storageConstants.DefaultUntaggedImgeRetentionDelay, false, false, log, metrics, nil, nil)
storage := local.NewImageStore(dir, false, false, log, metrics, nil, nil)
indexBlob, err := storage.GetIndexContent(repo)
So(err, ShouldBeNil)
@@ -5464,8 +5461,7 @@ func TestMetaDBWhenDeletingImages(t *testing.T) {
// get signatur digest
log := log.NewLogger("debug", "")
metrics := monitoring.NewMetricsServer(false, log)
storage := local.NewImageStore(dir, false, false, storageConstants.DefaultGCDelay,
storageConstants.DefaultUntaggedImgeRetentionDelay, false, false, log, metrics, nil, nil)
storage := local.NewImageStore(dir, false, false, log, metrics, nil, nil)
indexBlob, err := storage.GetIndexContent(repo)
So(err, ShouldBeNil)
@@ -5679,7 +5675,7 @@ func TestMetaDBWhenDeletingImages(t *testing.T) {
}
ctlr.MetaDB = mocks.MetaDBMock{
DeleteRepoTagFn: func(repo, tag string) error { return ErrTestError },
RemoveRepoReferenceFn: func(repo, reference string, manifestDigest godigest.Digest) error { return ErrTestError },
}
resp, err = resty.R().Delete(baseURL + "/v2/" + "repo1" + "/manifests/" +
pkg/extensions/search/userprefs_test.go
+1 -1
View File
@@ -544,7 +544,7 @@ func TestChangingRepoState(t *testing.T) {
}
// ------ Create the test repos
defaultStore := local.NewImageStore(conf.Storage.RootDirectory, false, false, 0, 0, false, false,
defaultStore := local.NewImageStore(conf.Storage.RootDirectory, false, false,
log.NewLogger("debug", ""), monitoring.NewMetricsServer(false, log.NewLogger("debug", "")), nil, nil)
err = WriteImageToFileSystem(img, accesibleRepo, "tag", storage.StoreController{
pkg/extensions/sync/local.go
+1 -4
View File
@@ -24,7 +24,6 @@ import (
mTypes "zotregistry.io/zot/pkg/meta/types"
"zotregistry.io/zot/pkg/storage"
storageCommon "zotregistry.io/zot/pkg/storage/common"
storageConstants "zotregistry.io/zot/pkg/storage/constants"
"zotregistry.io/zot/pkg/storage/local"
storageTypes "zotregistry.io/zot/pkg/storage/types"
)
@@ -281,9 +280,7 @@ func getImageStoreFromImageReference(imageReference types.ImageReference, repo,
metrics := monitoring.NewMetricsServer(false, log.Logger{})
tempImageStore := local.NewImageStore(tempRootDir, false, false,
storageConstants.DefaultGCDelay, storageConstants.DefaultUntaggedImgeRetentionDelay,
false, false, log.Logger{}, metrics, nil, nil)
tempImageStore := local.NewImageStore(tempRootDir, false, false, log.Logger{}, metrics, nil, nil)
return tempImageStore
}
pkg/extensions/sync/sync_internal_test.go
+3 -7
View File
@@ -69,9 +69,7 @@ func TestInjectSyncUtils(t *testing.T) {
log := log.Logger{Logger: zerolog.New(os.Stdout)}
metrics := monitoring.NewMetricsServer(false, log)
imageStore := local.NewImageStore(t.TempDir(), false, false, storageConstants.DefaultGCDelay,
storageConstants.DefaultUntaggedImgeRetentionDelay, false, false, log, metrics, nil, nil,
)
imageStore := local.NewImageStore(t.TempDir(), false, false, log, metrics, nil, nil)
injected = inject.InjectFailure(0)
ols := NewOciLayoutStorage(storage.StoreController{DefaultStore: imageStore})
@@ -183,8 +181,7 @@ func TestLocalRegistry(t *testing.T) {
UseRelPaths: true,
}, log)
syncImgStore := local.NewImageStore(dir, true, true, storageConstants.DefaultGCDelay,
storageConstants.DefaultUntaggedImgeRetentionDelay, true, true, log, metrics, nil, cacheDriver)
syncImgStore := local.NewImageStore(dir, true, true, log, metrics, nil, cacheDriver)
repoName := "repo"
registry := NewLocalRegistry(storage.StoreController{DefaultStore: syncImgStore}, nil, log)
@@ -301,8 +298,7 @@ func TestLocalRegistry(t *testing.T) {
MandatoryAnnotations: []string{"annot1"},
}, log)
syncImgStore := local.NewImageStore(dir, true, true, storageConstants.DefaultGCDelay,
storageConstants.DefaultUntaggedImgeRetentionDelay, true, true, log, metrics, linter, cacheDriver)
syncImgStore := local.NewImageStore(dir, true, true, log, metrics, linter, cacheDriver)
repoName := "repo"
registry := NewLocalRegistry(storage.StoreController{DefaultStore: syncImgStore}, nil, log)