mirror of
https://github.com/project-zot/zot.git
synced 2026-06-17 21:17:58 +08:00
273b15364b
* chore(metadb): add writer version to interface Signed-off-by: Jacob McSwain <jacob@mcswain.dev> * chore(metadb): add writer version to db mock Signed-off-by: Jacob McSwain <jacob@mcswain.dev> * chore(metadb): implement writer version for bolt, redis, and dynamodb Signed-off-by: Jacob McSwain <jacob@mcswain.dev> * feat(metadb): add optional fast restart path that skips storage walk when binary identity matches metaDB stamp binary identity is determined by the current release tag/commit and stored in metaDB after a successful storage parse. When fast restart is enabled, the next startup will skip the parse if the stored identity matches the current binary Signed-off-by: Jacob McSwain <jacob@mcswain.dev> * chore(cli): serve: add a way to force reparse storage Signed-off-by: Jacob McSwain <jacob@mcswain.dev> * refactor(meta): version: split to avoid global state mutation in tests Signed-off-by: Jacob McSwain <jacob@mcswain.dev> * fix(meta): version: include commit in writerVersion to distinguish retags Signed-off-by: Jacob McSwain <jacob@mcswain.dev> * chore(config): add IsFastRestartEnabled() test Signed-off-by: Jacob McSwain <jacob@mcswain.dev> * fix(meta): skip writer-version stamp when storage parse is incomplete ParseStorage returns nil even when individual repos fail to parse or are only partially parsed (a missing manifest blob), so MaybeParseStorage would stamp a partially-populated metaDB as good. On the next restart fastRestart trusts the stamp, skips the storage walk, and never recovers. Track per-repo outcomes via parseStats and stamp only when the walk fully populated the metaDB, otherwise log and continue so the next restart reparses Signed-off-by: Jacob McSwain <jacob@mcswain.dev> * fix(docs): readme: remove trailing comma from JSON config Signed-off-by: Jacob McSwain <jacob@mcswain.dev> * fix(meta): dynamodb: use context.Background instead of context.TODO Signed-off-by: Jacob McSwain <jacob@mcswain.dev> * fix(meta): invalidate fast restart on storage config changes Signed-off-by: Jacob McSwain <jacob@mcswain.dev> * chore(meta): dynamodb: use context.Background() instead of context.TODO() Signed-off-by: Jacob McSwain <jacob@mcswain.dev> * docs(meta): dynamodb: add comment about nil AttributeValue handling in GetWriterVersion Signed-off-by: Jacob McSwain <jacob@mcswain.dev> * chore: rename writer-version stamp to fast-restart stamp also replaces the version/commit tracking to use BinaryVersion instead of WriterVersion This should make things more clear Signed-off-by: Jacob McSwain <jacob@mcswain.dev> * fix(config): ensure FastRestart is on GlobalStorageConfig This is not a per-subpath setting Signed-off-by: Jacob McSwain <jacob@mcswain.dev> * fix(metadb): redis: tests: ensure clients are closed Signed-off-by: Jacob McSwain <jacob@mcswain.dev> --------- Signed-off-by: Jacob McSwain <jacob@mcswain.dev>
142 lines
4.3 KiB
Go
142 lines
4.3 KiB
Go
package meta
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"testing"
|
|
"time"
|
|
|
|
godigest "github.com/opencontainers/go-digest"
|
|
ispec "github.com/opencontainers/image-spec/specs-go/v1"
|
|
. "github.com/smartystreets/goconvey/convey"
|
|
|
|
zerr "zotregistry.dev/zot/v2/errors"
|
|
"zotregistry.dev/zot/v2/pkg/log"
|
|
mTypes "zotregistry.dev/zot/v2/pkg/meta/types"
|
|
"zotregistry.dev/zot/v2/pkg/storage"
|
|
testimage "zotregistry.dev/zot/v2/pkg/test/image-utils"
|
|
"zotregistry.dev/zot/v2/pkg/test/mocks"
|
|
)
|
|
|
|
var errParseInternal = errors.New("parse internal test error")
|
|
|
|
func indexBlobFor(digest godigest.Digest, tag string) []byte {
|
|
blob, err := json.Marshal(ispec.Index{
|
|
MediaType: ispec.MediaTypeImageIndex,
|
|
Manifests: []ispec.Descriptor{{
|
|
MediaType: ispec.MediaTypeImageManifest,
|
|
Digest: digest,
|
|
Annotations: map[string]string{ispec.AnnotationRefName: tag},
|
|
}},
|
|
})
|
|
if err != nil {
|
|
panic("image index should always be marshable")
|
|
}
|
|
|
|
return blob
|
|
}
|
|
|
|
func TestParseStatsComplete(t *testing.T) {
|
|
Convey("parseStats.complete is true only with no failed or partial repos", t, func() {
|
|
So(parseStats{}.complete(), ShouldBeTrue)
|
|
So(parseStats{failedRepos: 1}.complete(), ShouldBeFalse)
|
|
So(parseStats{partialRepos: 1}.complete(), ShouldBeFalse)
|
|
So(parseStats{failedRepos: 2, partialRepos: 3}.complete(), ShouldBeFalse)
|
|
})
|
|
}
|
|
|
|
func TestFastRestartStamp(t *testing.T) {
|
|
Convey("FastRestartStamp joins the binary version and storage fingerprint", t, func() {
|
|
So(FastRestartStamp("v2.3.4+abc123", "deadbeef"), ShouldEqual, "v2.3.4+abc123|deadbeef")
|
|
})
|
|
|
|
Convey("FastRestartStamp returns empty when the binary version is empty", t, func() {
|
|
So(FastRestartStamp("", "deadbeef"), ShouldEqual, "")
|
|
})
|
|
|
|
Convey("FastRestartStamp returns empty when the storage fingerprint is empty", t, func() {
|
|
So(FastRestartStamp("v2.3.4+abc123", ""), ShouldEqual, "")
|
|
})
|
|
}
|
|
|
|
func TestParseStorageStats(t *testing.T) {
|
|
logger := log.NewTestLogger()
|
|
|
|
// A valid image whose manifest + config blobs parse cleanly through ParseRepo.
|
|
validImage := testimage.CreateRandomImage()
|
|
|
|
manifestBlob, err := json.Marshal(validImage.Manifest)
|
|
if err != nil {
|
|
t.Fatalf("marshal manifest: %v", err)
|
|
}
|
|
|
|
configBlob, err := json.Marshal(validImage.Config)
|
|
if err != nil {
|
|
t.Fatalf("marshal config: %v", err)
|
|
}
|
|
|
|
goodDigest := godigest.FromString("good-manifest")
|
|
missingDigest := godigest.FromString("missing-manifest")
|
|
|
|
// blobFor maps the descriptor digests our mocked index references back to the
|
|
// valid image blobs, anything else is reported as a missing blob.
|
|
blobFor := func(_ string, digest godigest.Digest) ([]byte, error) {
|
|
switch digest {
|
|
case goodDigest:
|
|
return manifestBlob, nil
|
|
case validImage.ConfigDescriptor.Digest:
|
|
return configBlob, nil
|
|
default:
|
|
return nil, zerr.ErrBlobNotFound
|
|
}
|
|
}
|
|
|
|
metaDB := mocks.MetaDBMock{
|
|
SetRepoReferenceFn: func(context.Context, string, string, mTypes.ImageMeta) error { return nil },
|
|
}
|
|
|
|
Convey("a fully-parsed repo yields a complete parseStats", t, func() {
|
|
store := storage.StoreController{DefaultStore: mocks.MockedImageStore{
|
|
GetRepositoriesFn: func() ([]string, error) { return []string{"goodrepo"}, nil },
|
|
GetIndexContentFn: func(string) ([]byte, error) { return indexBlobFor(goodDigest, "gtag"), nil },
|
|
GetBlobContentFn: blobFor,
|
|
}}
|
|
|
|
stats, err := parseStorage(metaDB, store, logger)
|
|
So(err, ShouldBeNil)
|
|
So(stats.failedRepos, ShouldEqual, 0)
|
|
So(stats.partialRepos, ShouldEqual, 0)
|
|
So(stats.complete(), ShouldBeTrue)
|
|
})
|
|
|
|
Convey("failed and partial repos are counted independently", t, func() {
|
|
store := storage.StoreController{DefaultStore: mocks.MockedImageStore{
|
|
GetRepositoriesFn: func() ([]string, error) {
|
|
return []string{"failrepo", "partialrepo", "goodrepo"}, nil
|
|
},
|
|
StatIndexFn: func(repo string) (bool, int64, time.Time, error) {
|
|
if repo == "failrepo" {
|
|
return false, 0, time.Time{}, errParseInternal
|
|
}
|
|
|
|
return true, 0, time.Time{}, nil
|
|
},
|
|
GetIndexContentFn: func(repo string) ([]byte, error) {
|
|
if repo == "partialrepo" {
|
|
return indexBlobFor(missingDigest, "ptag"), nil
|
|
}
|
|
|
|
return indexBlobFor(goodDigest, "gtag"), nil
|
|
},
|
|
GetBlobContentFn: blobFor,
|
|
}}
|
|
|
|
stats, err := parseStorage(metaDB, store, logger)
|
|
So(err, ShouldBeNil)
|
|
So(stats.failedRepos, ShouldEqual, 1)
|
|
So(stats.partialRepos, ShouldEqual, 1)
|
|
So(stats.complete(), ShouldBeFalse)
|
|
})
|
|
}
|