github/zot
1
0
Fork 0
mirror of https://github.com/project-zot/zot.git synced 2026-06-16 20:38:08 +08:00
Code Issues Packages Projects Releases Wiki Activity

refactor(storage): refactoring storage (#1459)

Browse Source 
Signed-off-by: Laurentiu Niculae <niculae.laurentiu1@gmail.com>
This commit is contained in:
LaurentiuNiculae
2023-05-26 21:08:19 +03:00
committed by GitHub
parent 9acd19f7ea
commit a3f355c278
45 changed files with 850 additions and 751 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/extensions/sync/signatures.go
+2 -1
View File
@@ -24,6 +24,7 @@ import (
"zotregistry.io/zot/pkg/meta/repodb"
"zotregistry.io/zot/pkg/meta/signatures"
"zotregistry.io/zot/pkg/storage"
storageTypes "zotregistry.io/zot/pkg/storage/types"
)
type signaturesCopier struct {
@@ -520,7 +521,7 @@ func (sig *signaturesCopier) canSkipOCIRefs(localRepo, digestStr string, index i
return true, nil
}
func syncBlob(sig *signaturesCopier, imageStore storage.ImageStore, localRepo, remoteRepo string,
func syncBlob(sig *signaturesCopier, imageStore storageTypes.ImageStore, localRepo, remoteRepo string,
digest godigest.Digest,
) error {
getBlobURL := sig.upstreamURL
pkg/extensions/sync/sync.go
+2 -2
View File
@@ -24,7 +24,7 @@ import (
"zotregistry.io/zot/pkg/log"
"zotregistry.io/zot/pkg/meta/repodb"
"zotregistry.io/zot/pkg/storage"
"zotregistry.io/zot/pkg/test"
"zotregistry.io/zot/pkg/test/inject"
)
const (
@@ -315,7 +315,7 @@ func getLocalContexts(log log.Logger) (*types.SystemContext, *signature.PolicyCo
policy = &signature.Policy{Default: []signature.PolicyRequirement{signature.NewPRInsecureAcceptAnything()}}
policyContext, err := signature.NewPolicyContext(policy)
if err := test.Error(err); err != nil {
if err := inject.Error(err); err != nil {
log.Error().Str("errorType", common.TypeOf(err)).
Err(err).Msg("couldn't create policy context")
pkg/extensions/sync/sync_internal_test.go
+18 -15
View File
@@ -27,8 +27,11 @@ import (
"zotregistry.io/zot/pkg/extensions/monitoring"
"zotregistry.io/zot/pkg/log"
"zotregistry.io/zot/pkg/storage"
storageConstants "zotregistry.io/zot/pkg/storage/constants"
"zotregistry.io/zot/pkg/storage/local"
storageTypes "zotregistry.io/zot/pkg/storage/types"
"zotregistry.io/zot/pkg/test"
"zotregistry.io/zot/pkg/test/inject"
"zotregistry.io/zot/pkg/test/mocks"
)
@@ -51,13 +54,13 @@ func TestInjectSyncUtils(t *testing.T) {
taggedRef, err := reference.WithTag(ref, "tag")
So(err, ShouldBeNil)
injected := test.InjectFailure(0)
injected := inject.InjectFailure(0)
if injected {
_, err = getImageTags(context.Background(), &types.SystemContext{}, taggedRef)
So(err, ShouldNotBeNil)
}
injected = test.InjectFailure(0)
injected = inject.InjectFailure(0)
_, _, err = getLocalContexts(log.NewLogger("debug", ""))
if injected {
So(err, ShouldNotBeNil)
@@ -67,10 +70,10 @@ func TestInjectSyncUtils(t *testing.T) {
log := log.Logger{Logger: zerolog.New(os.Stdout)}
metrics := monitoring.NewMetricsServer(false, log)
imageStore := local.NewImageStore(t.TempDir(), false, storage.DefaultGCDelay,
imageStore := local.NewImageStore(t.TempDir(), false, storageConstants.DefaultGCDelay,
false, false, log, metrics, nil, nil,
)
injected = test.InjectFailure(0)
injected = inject.InjectFailure(0)
_, err = getLocalCachePath(imageStore, testImage)
if injected {
@@ -183,7 +186,7 @@ func TestSyncInternal(t *testing.T) {
log := log.NewLogger("debug", "")
metrics := monitoring.NewMetricsServer(false, log)
imageStore := local.NewImageStore(t.TempDir(), false, storage.DefaultGCDelay,
imageStore := local.NewImageStore(t.TempDir(), false, storageConstants.DefaultGCDelay,
false, false, log, metrics, nil, nil,
)
@@ -200,7 +203,7 @@ func TestSyncInternal(t *testing.T) {
log := log.Logger{Logger: zerolog.New(os.Stdout)}
metrics := monitoring.NewMetricsServer(false, log)
imageStore := local.NewImageStore(t.TempDir(), false, storage.DefaultGCDelay,
imageStore := local.NewImageStore(t.TempDir(), false, storageConstants.DefaultGCDelay,
false, false, log, metrics, nil, nil)
err := os.Chmod(imageStore.RootDir(), 0o000)
@@ -375,7 +378,7 @@ func TestSyncInternal(t *testing.T) {
}
metrics := monitoring.NewMetricsServer(false, log)
imageStore := local.NewImageStore(t.TempDir(), false, storage.DefaultGCDelay,
imageStore := local.NewImageStore(t.TempDir(), false, storageConstants.DefaultGCDelay,
false, false, log, metrics, nil, nil,
)
mockRepoDB := mocks.RepoDBMock{}
@@ -401,7 +404,7 @@ func TestSyncInternal(t *testing.T) {
log := log.Logger{Logger: zerolog.New(os.Stdout)}
metrics := monitoring.NewMetricsServer(false, log)
imageStore := local.NewImageStore(storageDir, false, storage.DefaultGCDelay,
imageStore := local.NewImageStore(storageDir, false, storageConstants.DefaultGCDelay,
false, false, log, metrics, nil, nil)
refs := ispec.Index{Manifests: []ispec.Descriptor{
@@ -642,7 +645,7 @@ func TestSyncInternal(t *testing.T) {
log := log.Logger{Logger: zerolog.New(os.Stdout)}
metrics := monitoring.NewMetricsServer(false, log)
imageStore := local.NewImageStore(storageDir, false, storage.DefaultGCDelay,
imageStore := local.NewImageStore(storageDir, false, storageConstants.DefaultGCDelay,
false, false, log, metrics, nil, nil)
storeController := storage.StoreController{}
@@ -662,7 +665,7 @@ func TestSyncInternal(t *testing.T) {
test.CopyTestFiles("../../../test/data", testRootDir)
testImageStore := local.NewImageStore(testRootDir, false,
storage.DefaultGCDelay, false, false, log, metrics, nil, nil)
storageConstants.DefaultGCDelay, false, false, log, metrics, nil, nil)
manifestContent, _, _, err := testImageStore.GetImageManifest(testImage, testImageTag)
So(err, ShouldBeNil)
@@ -729,14 +732,14 @@ func TestSyncInternal(t *testing.T) {
err = pushSyncedLocalImage(repo, "latest", testRootDir, nil, imageStore, log)
So(err, ShouldNotBeNil)
err = os.Chmod(path.Join(testRootDir, repo, "blobs",
index.Manifests[0].Digest.Algorithm().String(), index.Manifests[0].Digest.Encoded()), local.DefaultDirPerms)
err = os.Chmod(path.Join(testRootDir, repo, "blobs", index.Manifests[0].Digest.Algorithm().String(),
index.Manifests[0].Digest.Encoded()), storageConstants.DefaultDirPerms)
So(err, ShouldBeNil)
// trigger linter error on manifest push
imageStoreWithLinter := local.NewImageStore(t.TempDir(), false, storage.DefaultGCDelay,
imageStoreWithLinter := local.NewImageStore(t.TempDir(), false, storageConstants.DefaultGCDelay,
false, false, log, metrics, &mocks.MockedLint{
LintFn: func(repo string, manifestDigest godigest.Digest, imageStore storage.ImageStore) (bool, error) {
LintFn: func(repo string, manifestDigest godigest.Digest, imageStore storageTypes.ImageStore) (bool, error) {
return false, nil
},
}, nil,
@@ -769,7 +772,7 @@ func TestSyncInternal(t *testing.T) {
err = pushSyncedLocalImage(repo, "latest", testRootDir, nil, imageStore, log)
So(err, ShouldNotBeNil)
err = os.Chmod(configBlobPath, local.DefaultDirPerms)
err = os.Chmod(configBlobPath, storageConstants.DefaultDirPerms)
So(err, ShouldBeNil)
err = os.RemoveAll(path.Join(imageStore.RootDir(), repo, "index.json"))
pkg/extensions/sync/sync_test.go
+4 -5
View File
@@ -42,8 +42,7 @@ import (
"zotregistry.io/zot/pkg/extensions/sync"
logger "zotregistry.io/zot/pkg/log"
"zotregistry.io/zot/pkg/meta/signatures"
"zotregistry.io/zot/pkg/storage"
"zotregistry.io/zot/pkg/storage/local"
storageConstants "zotregistry.io/zot/pkg/storage/constants"
"zotregistry.io/zot/pkg/test"
"zotregistry.io/zot/pkg/test/mocks"
)
@@ -1660,7 +1659,7 @@ func TestBasicAuth(t *testing.T) {
"a": {
RootDirectory: destDir,
GC: true,
GCDelay: storage.DefaultGCDelay,
GCDelay: storageConstants.DefaultGCDelay,
Dedupe: true,
},
}
@@ -2506,7 +2505,7 @@ func TestSubPaths(t *testing.T) {
subpath: {
RootDirectory: subPathDestDir,
GC: true,
GCDelay: storage.DefaultGCDelay,
GCDelay: storageConstants.DefaultGCDelay,
Dedupe: true,
},
}
@@ -5203,7 +5202,7 @@ func TestSyncWithDestination(t *testing.T) {
test.CopyTestFiles("../../../test/data", srcDir)
err := os.MkdirAll(path.Join(sctlr.Config.Storage.RootDirectory, "/zot-fold"), local.DefaultDirPerms)
err := os.MkdirAll(path.Join(sctlr.Config.Storage.RootDirectory, "/zot-fold"), storageConstants.DefaultDirPerms)
So(err, ShouldBeNil)
// move upstream images under /zot-fold
pkg/extensions/sync/utils.go
+15 -13
View File
@@ -32,9 +32,11 @@ import (
"zotregistry.io/zot/pkg/extensions/monitoring"
"zotregistry.io/zot/pkg/log"
"zotregistry.io/zot/pkg/meta/repodb"
"zotregistry.io/zot/pkg/storage"
storageCommon "zotregistry.io/zot/pkg/storage/common"
storageConstants "zotregistry.io/zot/pkg/storage/constants"
"zotregistry.io/zot/pkg/storage/local"
"zotregistry.io/zot/pkg/test"
storageTypes "zotregistry.io/zot/pkg/storage/types"
"zotregistry.io/zot/pkg/test/inject"
)
type ReferenceList struct {
@@ -57,7 +59,7 @@ func getTagFromRef(ref types.ImageReference, log log.Logger) reference.Tagged {
func getImageTags(ctx context.Context, sysCtx *types.SystemContext, repoRef reference.Named) ([]string, error) {
dockerRef, err := docker.NewReference(reference.TagNameOnly(repoRef))
// hard to reach test case, injected error, see pkg/test/dev.go
if err = test.Error(err); err != nil {
if err = inject.Error(err); err != nil {
return nil, err // Should never happen for a reference with tag and no digest
}
@@ -266,7 +268,7 @@ func getFileCredentials(filepath string) (syncconf.CredentialsFile, error) {
}
func pushSyncedLocalImage(localRepo, reference, localCachePath string,
repoDB repodb.RepoDB, imageStore storage.ImageStore, log log.Logger,
repoDB repodb.RepoDB, imageStore storageTypes.ImageStore, log log.Logger,
) error {
log.Info().Str("image", localCachePath+"/"+localRepo+":"+reference).Msg("pushing synced local image to local registry")
@@ -275,7 +277,7 @@ func pushSyncedLocalImage(localRepo, reference, localCachePath string,
metrics := monitoring.NewMetricsServer(false, log)
cacheImageStore := local.NewImageStore(localCachePath, false,
storage.DefaultGCDelay, false, false, log, metrics, nil, nil)
storageConstants.DefaultGCDelay, false, false, log, metrics, nil, nil)
manifestBlob, manifestDigest, mediaType, err := cacheImageStore.GetImageManifest(localRepo, reference)
if err != nil {
@@ -361,7 +363,7 @@ func pushSyncedLocalImage(localRepo, reference, localCachePath string,
}
func copyManifest(localRepo string, manifestContent []byte, reference string, repoDB repodb.RepoDB,
cacheImageStore, imageStore storage.ImageStore, log log.Logger,
cacheImageStore, imageStore storageTypes.ImageStore, log log.Logger,
) error {
var manifest ispec.Manifest
@@ -376,7 +378,7 @@ func copyManifest(localRepo string, manifestContent []byte, reference string, re
}
for _, blob := range manifest.Layers {
if storage.IsNonDistributable(blob.MediaType) {
if storageCommon.IsNonDistributable(blob.MediaType) {
continue
}
@@ -421,7 +423,7 @@ func copyManifest(localRepo string, manifestContent []byte, reference string, re
// Copy a blob from one image store to another image store.
func copyBlob(localRepo string, blobDigest godigest.Digest, blobMediaType string,
souceImageStore, destinationImageStore storage.ImageStore, log log.Logger,
souceImageStore, destinationImageStore storageTypes.ImageStore, log log.Logger,
) error {
if found, _, _ := destinationImageStore.CheckBlob(localRepo, blobDigest); found {
// Blob is already at destination, nothing to do
@@ -508,12 +510,12 @@ func getLocalImageRef(localCachePath, repo, reference string) (types.ImageRefere
}
// Returns the localCachePath with an UUID at the end. Only to be called once per repo.
func getLocalCachePath(imageStore storage.ImageStore, repo string) (string, error) {
func getLocalCachePath(imageStore storageTypes.ImageStore, repo string) (string, error) {
localRepoPath := path.Join(imageStore.RootDir(), repo, SyncBlobUploadDir)
// check if SyncBlobUploadDir exists, create if not
var err error
if _, err = os.ReadDir(localRepoPath); os.IsNotExist(err) {
if err = os.MkdirAll(localRepoPath, local.DefaultDirPerms); err != nil {
if err = os.MkdirAll(localRepoPath, storageConstants.DefaultDirPerms); err != nil {
return "", err
}
}
@@ -525,14 +527,14 @@ func getLocalCachePath(imageStore storage.ImageStore, repo string) (string, erro
// create uuid folder
uuid, err := guuid.NewV4()
// hard to reach test case, injected error, see pkg/test/dev.go
if err := test.Error(err); err != nil {
if err := inject.Error(err); err != nil {
return "", err
}
localCachePath := path.Join(localRepoPath, uuid.String())
cachedRepoPath := path.Join(localCachePath, repo)
if err = os.MkdirAll(cachedRepoPath, local.DefaultDirPerms); err != nil {
if err = os.MkdirAll(cachedRepoPath, storageConstants.DefaultDirPerms); err != nil {
return "", err
}
@@ -540,7 +542,7 @@ func getLocalCachePath(imageStore storage.ImageStore, repo string) (string, erro
}
// canSkipImage returns whether or not we already synced this image.
func canSkipImage(repo, tag string, digest godigest.Digest, imageStore storage.ImageStore, log log.Logger,
func canSkipImage(repo, tag string, digest godigest.Digest, imageStore storageTypes.ImageStore, log log.Logger,
) (bool, error) {
// check image already synced
_, localImageManifestDigest, _, err := imageStore.GetImageManifest(repo, tag)