metadb: add optional fast restart path that skips storage walk when (version + commit + storage config) matches metaDB stamp (#4026)

* chore(metadb): add writer version to interface Signed-off-by: Jacob McSwain <jacob@mcswain.dev> * chore(metadb): add writer version to db mock Signed-off-by: Jacob McSwain <jacob@mcswain.dev> * chore(metadb): implement writer version for bolt, redis, and dynamodb Signed-off-by: Jacob McSwain <jacob@mcswain.dev> * feat(metadb): add optional fast restart path that skips storage walk when binary identity matches metaDB stamp binary identity is determined by the current release tag/commit and stored in metaDB after a successful storage parse. When fast restart is enabled, the next startup will skip the parse if the stored identity matches the current binary Signed-off-by: Jacob McSwain <jacob@mcswain.dev> * chore(cli): serve: add a way to force reparse storage Signed-off-by: Jacob McSwain <jacob@mcswain.dev> * refactor(meta): version: split to avoid global state mutation in tests Signed-off-by: Jacob McSwain <jacob@mcswain.dev> * fix(meta): version: include commit in writerVersion to distinguish retags Signed-off-by: Jacob McSwain <jacob@mcswain.dev> * chore(config): add IsFastRestartEnabled() test Signed-off-by: Jacob McSwain <jacob@mcswain.dev> * fix(meta): skip writer-version stamp when storage parse is incomplete ParseStorage returns nil even when individual repos fail to parse or are only partially parsed (a missing manifest blob), so MaybeParseStorage would stamp a partially-populated metaDB as good. On the next restart fastRestart trusts the stamp, skips the storage walk, and never recovers. Track per-repo outcomes via parseStats and stamp only when the walk fully populated the metaDB, otherwise log and continue so the next restart reparses Signed-off-by: Jacob McSwain <jacob@mcswain.dev> * fix(docs): readme: remove trailing comma from JSON config Signed-off-by: Jacob McSwain <jacob@mcswain.dev> * fix(meta): dynamodb: use context.Background instead of context.TODO Signed-off-by: Jacob McSwain <jacob@mcswain.dev> * fix(meta): invalidate fast restart on storage config changes Signed-off-by: Jacob McSwain <jacob@mcswain.dev> * chore(meta): dynamodb: use context.Background() instead of context.TODO() Signed-off-by: Jacob McSwain <jacob@mcswain.dev> * docs(meta): dynamodb: add comment about nil AttributeValue handling in GetWriterVersion Signed-off-by: Jacob McSwain <jacob@mcswain.dev> * chore: rename writer-version stamp to fast-restart stamp also replaces the version/commit tracking to use BinaryVersion instead of WriterVersion This should make things more clear Signed-off-by: Jacob McSwain <jacob@mcswain.dev> * fix(config): ensure FastRestart is on GlobalStorageConfig This is not a per-subpath setting Signed-off-by: Jacob McSwain <jacob@mcswain.dev> * fix(metadb): redis: tests: ensure clients are closed Signed-off-by: Jacob McSwain <jacob@mcswain.dev> --------- Signed-off-by: Jacob McSwain <jacob@mcswain.dev>
2026-06-18 05:28:07 +08:00 · 2026-06-09 12:47:20 -05:00
parent d480380ef7
commit 273b15364b
19 changed files with 1103 additions and 44 deletions
@@ -1698,6 +1698,34 @@ func (bdw *BoltDB) PatchDB() error {
 	return nil
 }

+func (bdw *BoltDB) GetFastRestartStamp() (string, error) {
+	var stamp string
+
+	err := bdw.DB.View(func(tx *bbolt.Tx) error {
+		buck := tx.Bucket([]byte(VersionBucket))
+		if buck == nil {
+			return nil
+		}
+
+		stamp = string(buck.Get([]byte(mTypes.FastRestartStampKey)))
+
+		return nil
+	})
+
+	return stamp, err
+}
+
+func (bdw *BoltDB) SetFastRestartStamp(stamp string) error {
+	return bdw.DB.Update(func(tx *bbolt.Tx) error {
+		buck, err := tx.CreateBucketIfNotExists([]byte(VersionBucket))
+		if err != nil {
+			return err
+		}
+
+		return buck.Put([]byte(mTypes.FastRestartStampKey), []byte(stamp))
+	})
+}
+
 func getUserStars(ctx context.Context, transaction *bbolt.Tx) []string {
 	userAc, err := reqCtx.UserAcFromContext(ctx)
 	if err != nil {
@@ -2155,6 +2183,12 @@ func (bdw *BoltDB) ResetDB() error {
 			return err
 		}

+		if versionBuck := transaction.Bucket([]byte(VersionBucket)); versionBuck != nil {
+			if err := versionBuck.Delete([]byte(mTypes.FastRestartStampKey)); err != nil {
+				return err
+			}
+		}
+
 		return nil
 	})

@@ -1235,3 +1235,48 @@ func TestBoltDBCountRepos(t *testing.T) {
 		})
 	})
 }
+
+func TestBoltDBFastRestartStamp(t *testing.T) {
+	Convey("FastRestartStamp", t, func() {
+		tmpDir := t.TempDir()
+		boltDBParams := boltdb.DBParameters{RootDir: tmpDir}
+		boltDriver, err := boltdb.GetBoltDriver(boltDBParams)
+		So(err, ShouldBeNil)
+
+		boltdbWrapper, err := boltdb.New(boltDriver, log.NewTestLogger())
+		So(err, ShouldBeNil)
+		So(boltdbWrapper, ShouldNotBeNil)
+
+		Convey("returns empty before set", func() {
+			v, err := boltdbWrapper.GetFastRestartStamp()
+			So(err, ShouldBeNil)
+			So(v, ShouldEqual, "")
+		})
+
+		Convey("round-trips a value", func() {
+			So(boltdbWrapper.SetFastRestartStamp("v2.3.4"), ShouldBeNil)
+
+			v, err := boltdbWrapper.GetFastRestartStamp()
+			So(err, ShouldBeNil)
+			So(v, ShouldEqual, "v2.3.4")
+		})
+
+		Convey("overwrites a previous value", func() {
+			So(boltdbWrapper.SetFastRestartStamp("v1"), ShouldBeNil)
+			So(boltdbWrapper.SetFastRestartStamp("v2"), ShouldBeNil)
+
+			v, err := boltdbWrapper.GetFastRestartStamp()
+			So(err, ShouldBeNil)
+			So(v, ShouldEqual, "v2")
+		})
+
+		Convey("ResetDB clears the stamp", func() {
+			So(boltdbWrapper.SetFastRestartStamp("v1"), ShouldBeNil)
+			So(boltdbWrapper.ResetDB(), ShouldBeNil)
+
+			v, err := boltdbWrapper.GetFastRestartStamp()
+			So(err, ShouldBeNil)
+			So(v, ShouldEqual, "")
+		})
+	})
+}