metadb: add optional fast restart path that skips storage walk when (version + commit + storage config) matches metaDB stamp (#4026)

* chore(metadb): add writer version to interface Signed-off-by: Jacob McSwain <jacob@mcswain.dev> * chore(metadb): add writer version to db mock Signed-off-by: Jacob McSwain <jacob@mcswain.dev> * chore(metadb): implement writer version for bolt, redis, and dynamodb Signed-off-by: Jacob McSwain <jacob@mcswain.dev> * feat(metadb): add optional fast restart path that skips storage walk when binary identity matches metaDB stamp binary identity is determined by the current release tag/commit and stored in metaDB after a successful storage parse. When fast restart is enabled, the next startup will skip the parse if the stored identity matches the current binary Signed-off-by: Jacob McSwain <jacob@mcswain.dev> * chore(cli): serve: add a way to force reparse storage Signed-off-by: Jacob McSwain <jacob@mcswain.dev> * refactor(meta): version: split to avoid global state mutation in tests Signed-off-by: Jacob McSwain <jacob@mcswain.dev> * fix(meta): version: include commit in writerVersion to distinguish retags Signed-off-by: Jacob McSwain <jacob@mcswain.dev> * chore(config): add IsFastRestartEnabled() test Signed-off-by: Jacob McSwain <jacob@mcswain.dev> * fix(meta): skip writer-version stamp when storage parse is incomplete ParseStorage returns nil even when individual repos fail to parse or are only partially parsed (a missing manifest blob), so MaybeParseStorage would stamp a partially-populated metaDB as good. On the next restart fastRestart trusts the stamp, skips the storage walk, and never recovers. Track per-repo outcomes via parseStats and stamp only when the walk fully populated the metaDB, otherwise log and continue so the next restart reparses Signed-off-by: Jacob McSwain <jacob@mcswain.dev> * fix(docs): readme: remove trailing comma from JSON config Signed-off-by: Jacob McSwain <jacob@mcswain.dev> * fix(meta): dynamodb: use context.Background instead of context.TODO Signed-off-by: Jacob McSwain <jacob@mcswain.dev> * fix(meta): invalidate fast restart on storage config changes Signed-off-by: Jacob McSwain <jacob@mcswain.dev> * chore(meta): dynamodb: use context.Background() instead of context.TODO() Signed-off-by: Jacob McSwain <jacob@mcswain.dev> * docs(meta): dynamodb: add comment about nil AttributeValue handling in GetWriterVersion Signed-off-by: Jacob McSwain <jacob@mcswain.dev> * chore: rename writer-version stamp to fast-restart stamp also replaces the version/commit tracking to use BinaryVersion instead of WriterVersion This should make things more clear Signed-off-by: Jacob McSwain <jacob@mcswain.dev> * fix(config): ensure FastRestart is on GlobalStorageConfig This is not a per-subpath setting Signed-off-by: Jacob McSwain <jacob@mcswain.dev> * fix(metadb): redis: tests: ensure clients are closed Signed-off-by: Jacob McSwain <jacob@mcswain.dev> --------- Signed-off-by: Jacob McSwain <jacob@mcswain.dev>
2026-06-18 05:28:07 +08:00 · 2026-06-09 12:47:20 -05:00
parent d480380ef7
commit 273b15364b
19 changed files with 1103 additions and 44 deletions
@@ -42,20 +42,21 @@ const (
 )

 type RedisDB struct {
-	Client             redis.UniversalClient
-	imgTrustStore      mTypes.ImageTrustStore
-	Patches            []func(client redis.UniversalClient) error
-	Version            string
-	Log                log.Logger
-	RS                 *redsync.Redsync
-	ImageMetaKey       string
-	RepoMetaKey        string
-	RepoBlobsKey       string
-	RepoLastUpdatedKey string
-	UserDataKey        string
-	VersionKey         string
-	UserAPIKeysKey     string
-	LocksKey           string
+	Client              redis.UniversalClient
+	imgTrustStore       mTypes.ImageTrustStore
+	Patches             []func(client redis.UniversalClient) error
+	Version             string
+	Log                 log.Logger
+	RS                  *redsync.Redsync
+	ImageMetaKey        string
+	RepoMetaKey         string
+	RepoBlobsKey        string
+	RepoLastUpdatedKey  string
+	UserDataKey         string
+	VersionKey          string
+	FastRestartStampKey string
+	UserAPIKeysKey      string
+	LocksKey            string
 }

 type DBDriverParameters struct {
@@ -64,19 +65,20 @@ type DBDriverParameters struct {

 func New(client redis.UniversalClient, params DBDriverParameters, log log.Logger) (*RedisDB, error) {
 	redisWrapper := RedisDB{
-		Client:             client,
-		Log:                log,
-		Patches:            version.GetRedisDBPatches(),
-		Version:            version.CurrentVersion,
-		imgTrustStore:      nil,
-		ImageMetaKey:       join(params.KeyPrefix, ImageMetaBucket),
-		RepoMetaKey:        join(params.KeyPrefix, RepoMetaBucket),
-		RepoBlobsKey:       join(params.KeyPrefix, RepoBlobsBucket),
-		RepoLastUpdatedKey: join(params.KeyPrefix, RepoLastUpdatedBucket),
-		UserDataKey:        join(params.KeyPrefix, UserDataBucket),
-		VersionKey:         join(params.KeyPrefix, VersionBucket),
-		UserAPIKeysKey:     join(params.KeyPrefix, UserAPIKeysBucket),
-		LocksKey:           join(params.KeyPrefix, LocksBucket),
+		Client:              client,
+		Log:                 log,
+		Patches:             version.GetRedisDBPatches(),
+		Version:             version.CurrentVersion,
+		imgTrustStore:       nil,
+		ImageMetaKey:        join(params.KeyPrefix, ImageMetaBucket),
+		RepoMetaKey:         join(params.KeyPrefix, RepoMetaBucket),
+		RepoBlobsKey:        join(params.KeyPrefix, RepoBlobsBucket),
+		RepoLastUpdatedKey:  join(params.KeyPrefix, RepoLastUpdatedBucket),
+		UserDataKey:         join(params.KeyPrefix, UserDataBucket),
+		VersionKey:          join(params.KeyPrefix, VersionBucket),
+		FastRestartStampKey: join(params.KeyPrefix, mTypes.FastRestartStampKey),
+		UserAPIKeysKey:      join(params.KeyPrefix, UserAPIKeysBucket),
+		LocksKey:            join(params.KeyPrefix, LocksBucket),
 	}

 	if err := client.Ping(context.Background()).Err(); err != nil {
@@ -2165,6 +2167,12 @@ func (rc *RedisDB) ResetDB() error {
 			return fmt.Errorf("failed to delete version bucket: %w", err)
 		}

+		if err := txrp.Del(ctx, rc.FastRestartStampKey).Err(); err != nil {
+			rc.Log.Error().Err(err).Str("del", rc.FastRestartStampKey).Msg("failed to delete fast-restart stamp key")
+
+			return fmt.Errorf("failed to delete fast-restart stamp key: %w", err)
+		}
+
 		return nil
 	})

@@ -2218,6 +2226,35 @@ func (rc *RedisDB) PatchDB() error {
 	return err
 }

+func (rc *RedisDB) GetFastRestartStamp() (string, error) {
+	ctx := context.Background()
+
+	stamp, err := rc.Client.Get(ctx, rc.FastRestartStampKey).Result()
+	if err != nil {
+		if errors.Is(err, redis.Nil) {
+			return "", nil
+		}
+
+		rc.Log.Error().Err(err).Str("get", rc.FastRestartStampKey).Msg("failed to get fast-restart stamp")
+
+		return "", err
+	}
+
+	return stamp, nil
+}
+
+func (rc *RedisDB) SetFastRestartStamp(stamp string) error {
+	ctx := context.Background()
+
+	if err := rc.Client.Set(ctx, rc.FastRestartStampKey, stamp, 0).Err(); err != nil {
+		rc.Log.Error().Err(err).Str("set", rc.FastRestartStampKey).Msg("failed to set fast-restart stamp")
+
+		return err
+	}
+
+	return nil
+}
+
 func (rc *RedisDB) ImageTrustStore() mTypes.ImageTrustStore {
 	return rc.imgTrustStore
 }
@@ -33,6 +33,7 @@ func Test(t *testing.T) {
 		So(metaDB.UserDataKey, ShouldEqual, "zot:UserData")
 		So(metaDB.UserAPIKeysKey, ShouldEqual, "zot:UserAPIKeys")
 		So(metaDB.VersionKey, ShouldEqual, "zot:Version")
+		So(metaDB.FastRestartStampKey, ShouldEqual, "zot:FastRestartStamp")
 		So(metaDB.LocksKey, ShouldEqual, "zot:Locks")

 		So(metaDB.getUserLockKey("user1"), ShouldEqual, "zot:Locks:User:user1")
@@ -51,6 +52,7 @@ func Test(t *testing.T) {
 		So(metaDB.UserDataKey, ShouldEqual, "someprefix:UserData")
 		So(metaDB.UserAPIKeysKey, ShouldEqual, "someprefix:UserAPIKeys")
 		So(metaDB.VersionKey, ShouldEqual, "someprefix:Version")
+		So(metaDB.FastRestartStampKey, ShouldEqual, "someprefix:FastRestartStamp")
 		So(metaDB.LocksKey, ShouldEqual, "someprefix:Locks")

 		So(metaDB.getUserLockKey("user1"), ShouldEqual, "someprefix:Locks:User:user1")
@@ -45,6 +45,7 @@ func TestRedisMocked(t *testing.T) {
 		So(log, ShouldNotBeNil)

 		client, mock := redismock.NewClientMock()
+		defer client.Close()
 		defer DumpKeys(t, client) // Troubleshoot test failures

 		mock.ExpectPing().SetVal("PONG")
@@ -236,6 +237,7 @@ func TestRedisRepoMeta(t *testing.T) {
 		So(err, ShouldBeNil)

 		client := goredis.NewClient(opts)
+		defer client.Close()
 		defer DumpKeys(t, client) // Troubleshoot test failures

 		params := redis.DBDriverParameters{KeyPrefix: "zot"}
@@ -412,6 +414,7 @@ func TestRedisUnreachable(t *testing.T) {
 		connOpts, err := goredis.ParseURL("redis://" + miniRedis.Addr())
 		So(err, ShouldBeNil)
 		workingClient := goredis.NewClient(connOpts)
+		defer workingClient.Close()

 		params := redis.DBDriverParameters{KeyPrefix: "zot"}

@@ -422,6 +425,7 @@ func TestRedisUnreachable(t *testing.T) {
 		connOpts, err = goredis.ParseURL("redis://127.0.0.1:" + test.GetFreePort())
 		So(err, ShouldBeNil)
 		brokenClient := goredis.NewClient(connOpts)
+		defer brokenClient.Close()

 		// Replace connection with the unreachable server
 		metaDB.Client = brokenClient
@@ -586,6 +590,7 @@ func TestWrapperErrors(t *testing.T) {
 		So(err, ShouldBeNil)

 		client := goredis.NewClient(opts)
+		defer client.Close()
 		params := redis.DBDriverParameters{KeyPrefix: keyPrefix}

 		metaDB, err := redis.New(client, params, log)
@@ -1773,3 +1778,57 @@ func DumpKeys(t *testing.T, client goredis.UniversalClient) {
 		}
 	}
 }
+
+func TestRedisFastRestartStamp(t *testing.T) {
+	miniRedis := miniredis.RunT(t)
+
+	Convey("FastRestartStamp", t, func() {
+		log := log.NewTestLogger()
+		So(log, ShouldNotBeNil)
+
+		opts, err := goredis.ParseURL("redis://" + miniRedis.Addr())
+		So(err, ShouldBeNil)
+
+		client := goredis.NewClient(opts)
+		defer client.Close()
+		defer DumpKeys(t, client) // Troubleshoot test failures
+
+		params := redis.DBDriverParameters{KeyPrefix: keyPrefix}
+
+		metaDB, err := redis.New(client, params, log)
+		So(err, ShouldBeNil)
+		So(metaDB, ShouldNotBeNil)
+
+		Convey("returns empty before set", func() {
+			v, err := metaDB.GetFastRestartStamp()
+			So(err, ShouldBeNil)
+			So(v, ShouldEqual, "")
+		})
+
+		Convey("round-trips a value", func() {
+			So(metaDB.SetFastRestartStamp("v2.3.4"), ShouldBeNil)
+
+			v, err := metaDB.GetFastRestartStamp()
+			So(err, ShouldBeNil)
+			So(v, ShouldEqual, "v2.3.4")
+		})
+
+		Convey("overwrites a previous value", func() {
+			So(metaDB.SetFastRestartStamp("v1"), ShouldBeNil)
+			So(metaDB.SetFastRestartStamp("v2"), ShouldBeNil)
+
+			v, err := metaDB.GetFastRestartStamp()
+			So(err, ShouldBeNil)
+			So(v, ShouldEqual, "v2")
+		})
+
+		Convey("ResetDB clears the stamp", func() {
+			So(metaDB.SetFastRestartStamp("v1"), ShouldBeNil)
+			So(metaDB.ResetDB(), ShouldBeNil)
+
+			v, err := metaDB.GetFastRestartStamp()
+			So(err, ShouldBeNil)
+			So(v, ShouldEqual, "")
+		})
+	})
+}