metadb: add optional fast restart path that skips storage walk when (version + commit + storage config) matches metaDB stamp (#4026)

* chore(metadb): add writer version to interface Signed-off-by: Jacob McSwain <jacob@mcswain.dev> * chore(metadb): add writer version to db mock Signed-off-by: Jacob McSwain <jacob@mcswain.dev> * chore(metadb): implement writer version for bolt, redis, and dynamodb Signed-off-by: Jacob McSwain <jacob@mcswain.dev> * feat(metadb): add optional fast restart path that skips storage walk when binary identity matches metaDB stamp binary identity is determined by the current release tag/commit and stored in metaDB after a successful storage parse. When fast restart is enabled, the next startup will skip the parse if the stored identity matches the current binary Signed-off-by: Jacob McSwain <jacob@mcswain.dev> * chore(cli): serve: add a way to force reparse storage Signed-off-by: Jacob McSwain <jacob@mcswain.dev> * refactor(meta): version: split to avoid global state mutation in tests Signed-off-by: Jacob McSwain <jacob@mcswain.dev> * fix(meta): version: include commit in writerVersion to distinguish retags Signed-off-by: Jacob McSwain <jacob@mcswain.dev> * chore(config): add IsFastRestartEnabled() test Signed-off-by: Jacob McSwain <jacob@mcswain.dev> * fix(meta): skip writer-version stamp when storage parse is incomplete ParseStorage returns nil even when individual repos fail to parse or are only partially parsed (a missing manifest blob), so MaybeParseStorage would stamp a partially-populated metaDB as good. On the next restart fastRestart trusts the stamp, skips the storage walk, and never recovers. Track per-repo outcomes via parseStats and stamp only when the walk fully populated the metaDB, otherwise log and continue so the next restart reparses Signed-off-by: Jacob McSwain <jacob@mcswain.dev> * fix(docs): readme: remove trailing comma from JSON config Signed-off-by: Jacob McSwain <jacob@mcswain.dev> * fix(meta): dynamodb: use context.Background instead of context.TODO Signed-off-by: Jacob McSwain <jacob@mcswain.dev> * fix(meta): invalidate fast restart on storage config changes Signed-off-by: Jacob McSwain <jacob@mcswain.dev> * chore(meta): dynamodb: use context.Background() instead of context.TODO() Signed-off-by: Jacob McSwain <jacob@mcswain.dev> * docs(meta): dynamodb: add comment about nil AttributeValue handling in GetWriterVersion Signed-off-by: Jacob McSwain <jacob@mcswain.dev> * chore: rename writer-version stamp to fast-restart stamp also replaces the version/commit tracking to use BinaryVersion instead of WriterVersion This should make things more clear Signed-off-by: Jacob McSwain <jacob@mcswain.dev> * fix(config): ensure FastRestart is on GlobalStorageConfig This is not a per-subpath setting Signed-off-by: Jacob McSwain <jacob@mcswain.dev> * fix(metadb): redis: tests: ensure clients are closed Signed-off-by: Jacob McSwain <jacob@mcswain.dev> --------- Signed-off-by: Jacob McSwain <jacob@mcswain.dev>
2026-06-19 22:27:58 +08:00 · 2026-06-09 12:47:20 -05:00
parent d480380ef7
commit 273b15364b
19 changed files with 1103 additions and 44 deletions
@@ -331,7 +331,7 @@ func (dwr *DynamoDB) setProtoRepoMeta(repo string, repoMeta *proto_go.RepoMeta)
 		return err
 	}

-	_, err = dwr.Client.UpdateItem(context.TODO(), &dynamodb.UpdateItemInput{
+	_, err = dwr.Client.UpdateItem(context.Background(), &dynamodb.UpdateItemInput{
 		ExpressionAttributeNames: map[string]string{
 			"#RM": "RepoMeta",
 		},
@@ -625,7 +625,7 @@ func (dwr *DynamoDB) setRepoBlobsInfo(repo string, repoBlobs *proto_go.RepoBlobs
 		return err
 	}

-	_, err = dwr.Client.UpdateItem(context.TODO(), &dynamodb.UpdateItemInput{
+	_, err = dwr.Client.UpdateItem(context.Background(), &dynamodb.UpdateItemInput{
 		ExpressionAttributeNames: map[string]string{
 			"#RBI": "RepoBlobsInfo",
 			"#RLU": "RepoLastUpdated",
@@ -1518,7 +1518,7 @@ func (dwr *DynamoDB) RemoveRepoReference(repo, reference string, manifestDigest
 ) error {
 	ctx := context.Background()

-	protoRepoMeta, err := dwr.getProtoRepoMeta(context.Background(), repo)
+	protoRepoMeta, err := dwr.getProtoRepoMeta(ctx, repo)
 	if err != nil {
 		if errors.Is(err, zerr.ErrRepoMetaNotFound) {
 			return nil
@@ -1527,7 +1527,7 @@ func (dwr *DynamoDB) RemoveRepoReference(repo, reference string, manifestDigest
 		return err
 	}

-	protoImageMeta, err := dwr.GetProtoImageMeta(context.TODO(), manifestDigest)
+	protoImageMeta, err := dwr.GetProtoImageMeta(ctx, manifestDigest)
 	if err != nil {
 		if errors.Is(err, zerr.ErrImageMetaNotFound) {
 			return nil
@@ -2228,6 +2228,56 @@ func (dwr *DynamoDB) PatchDB() error {
 	return nil
 }

+func (dwr *DynamoDB) GetFastRestartStamp() (string, error) {
+	resp, err := dwr.Client.GetItem(context.Background(), &dynamodb.GetItemInput{
+		TableName: aws.String(dwr.VersionTablename),
+		Key: map[string]types.AttributeValue{
+			"TableKey": &types.AttributeValueMemberS{Value: mTypes.FastRestartStampKey},
+		},
+	})
+	if err != nil {
+		return "", err
+	}
+
+	if resp.Item == nil {
+		return "", nil
+	}
+
+	var stamp string
+
+	// In aws-sdk-go-v2, a missing attribute arrives as a nil AttributeValue,
+	// which Unmarshal treats as null, setting the attribute to its zero
+	// value ("") and returning nil rather than an error
+	if err := attributevalue.Unmarshal(resp.Item["Version"], &stamp); err != nil {
+		return "", err
+	}
+
+	return stamp, nil
+}
+
+func (dwr *DynamoDB) SetFastRestartStamp(stamp string) error {
+	mdAttributeValue, err := attributevalue.Marshal(stamp)
+	if err != nil {
+		return err
+	}
+
+	_, err = dwr.Client.UpdateItem(context.Background(), &dynamodb.UpdateItemInput{
+		ExpressionAttributeNames: map[string]string{
+			"#V": "Version",
+		},
+		ExpressionAttributeValues: map[string]types.AttributeValue{
+			":Version": mdAttributeValue,
+		},
+		Key: map[string]types.AttributeValue{
+			"TableKey": &types.AttributeValueMemberS{Value: mTypes.FastRestartStampKey},
+		},
+		TableName:        aws.String(dwr.VersionTablename),
+		UpdateExpression: aws.String("SET #V = :Version"),
+	})
+
+	return err
+}
+
 func (dwr *DynamoDB) ResetDB() error {
 	err := dwr.ResetTable(dwr.APIKeyTablename)
 	if err != nil {
@@ -2254,6 +2304,16 @@ func (dwr *DynamoDB) ResetDB() error {
 		return err
 	}

+	_, err = dwr.Client.DeleteItem(context.Background(), &dynamodb.DeleteItemInput{
+		TableName: aws.String(dwr.VersionTablename),
+		Key: map[string]types.AttributeValue{
+			"TableKey": &types.AttributeValueMemberS{Value: mTypes.FastRestartStampKey},
+		},
+	})
+	if err != nil {
+		return err
+	}
+
 	return nil
 }

@@ -2402,7 +2462,7 @@ func (dwr *DynamoDB) createVersionTable() error {
 		return err
 	}

-	_, err = dwr.Client.UpdateItem(context.TODO(), &dynamodb.UpdateItemInput{
+	_, err = dwr.Client.UpdateItem(context.Background(), &dynamodb.UpdateItemInput{
 		ExpressionAttributeNames: map[string]string{
 			"#V": "Version",
 		},
@@ -2432,7 +2492,7 @@ func (dwr *DynamoDB) createVersionTable() error {
 }

 func (dwr *DynamoDB) getDBVersion() (string, error) {
-	resp, err := dwr.Client.GetItem(context.TODO(), &dynamodb.GetItemInput{
+	resp, err := dwr.Client.GetItem(context.Background(), &dynamodb.GetItemInput{
 		TableName: aws.String(dwr.VersionTablename),
 		Key: map[string]types.AttributeValue{
 			"TableKey": &types.AttributeValueMemberS{Value: version.DBVersionKey},