github/zot
1
0
Fork 0
mirror of https://github.com/project-zot/zot.git synced 2026-06-16 04:17:55 +08:00
Code Issues Packages Projects Releases Wiki Activity

refactor(metadb): improve UX by speeding up metadb serialize/deserialize (#1842)

Browse Source 
Use protocol buffers and update the metadb interface to better suit our search needs

Signed-off-by: Ramkumar Chinchani <rchincha@cisco.com>
Signed-off-by: Laurentiu Niculae <niculae.laurentiu1@gmail.com>
Co-authored-by: Ramkumar Chinchani <rchincha@cisco.com>
This commit is contained in:
LaurentiuNiculae
2023-10-30 22:06:04 +02:00
committed by GitHub
parent f34af3cb3f
commit 56ad9e6707
82 changed files with 10455 additions and 11676 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/extensions/sync/references/cosign.go
+3 -15
View File
@@ -153,21 +153,9 @@ func (ref CosignReference) SyncReferences(ctx context.Context, localRepo, remote
ref.log.Debug().Str("repository", localRepo).Str("subject", subjectDigestStr).
Msg("metaDB: trying to sync cosign reference for image")
isSig, sigType, signedManifestDig, err := storage.CheckIsImageSignature(localRepo, manifestBuf,
cosignTag)
if err != nil {
return refsDigests, fmt.Errorf("failed to check if cosign reference '%s@%s' is a signature: %w", localRepo,
cosignTag, err)
}
if isSig {
err = addSigToMeta(ref.metaDB, localRepo, sigType, cosignTag, signedManifestDig, referenceDigest,
manifestBuf, imageStore, ref.log)
} else {
err = meta.SetImageMetaFromInput(localRepo, cosignTag, ispec.MediaTypeImageManifest,
referenceDigest, manifestBuf, ref.storeController.GetImageStore(localRepo),
ref.metaDB, ref.log)
}
err = meta.SetImageMetaFromInput(localRepo, cosignTag, ispec.MediaTypeImageManifest,
referenceDigest, manifestBuf, ref.storeController.GetImageStore(localRepo),
ref.metaDB, ref.log)
if err != nil {
return refsDigests, fmt.Errorf("failed to set metadata for cosign reference in '%s@%s': %w",
pkg/extensions/sync/references/oci.go
+3 -16
View File
@@ -137,22 +137,9 @@ func (ref OciReferences) SyncReferences(ctx context.Context, localRepo, remoteRe
ref.log.Debug().Str("repository", localRepo).Str("subject", subjectDigestStr).
Msg("metaDB: trying to add oci references for image")
isSig, sigType, signedManifestDig, err := storage.CheckIsImageSignature(localRepo, referenceBuf,
referrer.Digest.String())
if err != nil {
return refsDigests, fmt.Errorf("failed to check if oci reference '%s@%s' is a signature: %w", localRepo,
referrer.Digest.String(), err)
}
if isSig {
err = addSigToMeta(ref.metaDB, localRepo, sigType, referrer.Digest.String(), signedManifestDig, referenceDigest,
referenceBuf, imageStore, ref.log)
} else {
err = meta.SetImageMetaFromInput(localRepo, referenceDigest.String(), referrer.MediaType,
referenceDigest, referenceBuf, ref.storeController.GetImageStore(localRepo),
ref.metaDB, ref.log)
}
err = meta.SetImageMetaFromInput(localRepo, referenceDigest.String(), referrer.MediaType,
referenceDigest, referenceBuf, ref.storeController.GetImageStore(localRepo),
ref.metaDB, ref.log)
if err != nil {
return refsDigests, fmt.Errorf("failed to set metadata for oci reference in '%s@%s': %w",
localRepo, subjectDigestStr, err)
pkg/extensions/sync/references/referrers_tag.go
+3 -16
View File
@@ -113,22 +113,9 @@ func (ref TagReferences) SyncReferences(ctx context.Context, localRepo, remoteRe
ref.log.Debug().Str("repository", localRepo).Str("subject", subjectDigestStr).
Msg("metaDB: trying to add oci references for image")
isSig, sigType, signedManifestDig, err := storage.CheckIsImageSignature(localRepo, referenceBuf,
referrer.Digest.String())
if err != nil {
return refsDigests, fmt.Errorf("failed to check if oci reference '%s@%s' is a signature: %w", localRepo,
referrer.Digest.String(), err)
}
if isSig {
err = addSigToMeta(ref.metaDB, localRepo, sigType, referrer.Digest.String(), signedManifestDig, referenceDigest,
referenceBuf, imageStore, ref.log)
} else {
err = meta.SetImageMetaFromInput(localRepo, referenceDigest.String(), referrer.MediaType,
referenceDigest, referenceBuf, ref.storeController.GetImageStore(localRepo),
ref.metaDB, ref.log)
}
err = meta.SetImageMetaFromInput(localRepo, referenceDigest.String(), referrer.MediaType,
referenceDigest, referenceBuf, ref.storeController.GetImageStore(localRepo),
ref.metaDB, ref.log)
if err != nil {
return refsDigests, fmt.Errorf("failed to set metadata for oci reference in '%s@%s': %w",
localRepo, subjectDigestStr, err)
pkg/extensions/sync/sync_internal_test.go
+4 -3
View File
@@ -27,6 +27,7 @@ import (
"zotregistry.io/zot/pkg/extensions/monitoring"
client "zotregistry.io/zot/pkg/extensions/sync/httpclient"
"zotregistry.io/zot/pkg/log"
mTypes "zotregistry.io/zot/pkg/meta/types"
"zotregistry.io/zot/pkg/storage"
"zotregistry.io/zot/pkg/storage/cache"
storageConstants "zotregistry.io/zot/pkg/storage/constants"
@@ -336,8 +337,8 @@ func TestLocalRegistry(t *testing.T) {
Convey("trigger metaDB error on index manifest in CommitImage()", func() {
registry := NewLocalRegistry(storage.StoreController{DefaultStore: syncImgStore}, mocks.MetaDBMock{
SetRepoReferenceFn: func(repo, Reference string, manifestDigest godigest.Digest, mediaType string) error {
if Reference == "1.0" {
SetRepoReferenceFn: func(repo string, reference string, imageMeta mTypes.ImageMeta) error {
if reference == "1.0" {
return zerr.ErrRepoMetaNotFound
}
@@ -351,7 +352,7 @@ func TestLocalRegistry(t *testing.T) {
Convey("trigger metaDB error on image manifest in CommitImage()", func() {
registry := NewLocalRegistry(storage.StoreController{DefaultStore: syncImgStore}, mocks.MetaDBMock{
SetRepoReferenceFn: func(repo, Reference string, manifestDigest godigest.Digest, mediaType string) error {
SetRepoReferenceFn: func(repo, reference string, imageMeta mTypes.ImageMeta) error {
return zerr.ErrRepoMetaNotFound
},
}, log)
pkg/extensions/sync/sync_test.go
+4 -6
View File
@@ -877,9 +877,7 @@ func TestOnDemand(t *testing.T) {
return nil
},
SetRepoReferenceFn: func(repo, reference string, manifestDigest godigest.Digest,
mediaType string,
) error {
SetRepoReferenceFn: func(repo, reference string, imageMeta mTypes.ImageMeta) error {
if strings.HasPrefix(reference, "sha256-") &&
(strings.HasSuffix(reference, remote.SignatureTagSuffix) ||
strings.HasSuffix(reference, remote.SBOMTagSuffix)) ||
@@ -1019,8 +1017,8 @@ func TestOnDemand(t *testing.T) {
// metadb fails for syncReferrersTag"
dctlr.MetaDB = mocks.MetaDBMock{
SetManifestDataFn: func(manifestDigest godigest.Digest, mm mTypes.ManifestData) error {
if manifestDigest.String() == ociRefImage.ManifestDescriptor.Digest.String() {
SetRepoReferenceFn: func(repo, reference string, imageMeta mTypes.ImageMeta) error {
if imageMeta.Digest.String() == ociRefImage.ManifestDescriptor.Digest.String() {
return sync.ErrTestError
}
@@ -4670,7 +4668,7 @@ func TestSyncedSignaturesMetaDB(t *testing.T) {
So(err, ShouldBeNil)
So(resp.StatusCode(), ShouldEqual, http.StatusOK)
repoMeta, err := dctlr.MetaDB.GetRepoMeta(repoName)
repoMeta, err := dctlr.MetaDB.GetRepoMeta(context.Background(), repoName)
So(err, ShouldBeNil)
So(repoMeta.Tags, ShouldContainKey, tag)
So(len(repoMeta.Tags), ShouldEqual, 1)