feat: add TaggedTimestamp to ImageSummary returned by graphql API (#3731)

feat(meta): add TaggedTimestamp field and preserve during re-parsing

Add TaggedTimestamp field to track when image tags were created, exposed
through GraphQL API. Previously, when zot restarted and re-parsed storage,
ResetRepoReferences would clear all tags, causing timestamp information to
be lost and reset to the service restart time for existing images.

This change adds TaggedTimestamp support and modifies ResetRepoReferences to
selectively preserve tags that still exist in storage, maintaining their
TaggedTimestamp values. Tags that no longer exist in storage are removed as
before.

Changes:
- Add TaggedTimestamp field to GraphQL ImageSummary schema
- Update GraphQL conversion functions to populate TaggedTimestamp with
  fallback to PushTimestamp when unavailable
- Updated ResetRepoReferences interface to accept tagsToKeep parameter
- Modified ParseRepo to collect tags from storage before resetting
- Updated all backend implementations (Redis, DynamoDB, BoltDB) to preserve
  tags in tagsToKeep instead of clearing all tags
- Updated tests and mocks to match new signature

This ensures TaggedTimestamp accurately reflects when tags were originally
created, and exposes this information through the GraphQL API.

Signed-off-by: Andrei Aaron <andreifdaaron@gmail.com>

pkg/extensions/search/convert/convert_test.go

@@ -272,6 +272,188 @@ func ref[T any](val T) *T {
 	return &ref
 }
 
+func TestTaggedTimestamp(t *testing.T) {
+	ctx := context.Background()
+
+	Convey("Test TaggedTimestamp in ImageSummary", t, func() {
+		Convey("TaggedTimestamp is populated from tag descriptor", func() {
+			taggedTime := time.Date(2024, time.January, 15, 10, 30, 0, 0, time.UTC)
+			pushTime := time.Date(2024, time.January, 10, 8, 0, 0, 0, time.UTC)
+
+			repoMeta := mTypes.RepoMeta{
+				Name: "repo",
+				Tags: map[string]mTypes.Descriptor{
+					"tag1": {
+						Digest:          "sha256:abc123",
+						MediaType:       ispec.MediaTypeImageManifest,
+						TaggedTimestamp: taggedTime,
+					},
+				},
+				Statistics: map[string]mTypes.DescriptorStatistics{
+					"sha256:abc123": {
+						PushTimestamp: pushTime,
+					},
+				},
+			}
+
+			imageMeta := mTypes.ImageMeta{
+				Digest:    godigest.FromString("sha256:abc123"),
+				MediaType: ispec.MediaTypeImageManifest,
+				Manifests: []mTypes.ManifestMeta{
+					{
+						Digest: godigest.FromString("sha256:abc123"),
+						Manifest: ispec.Manifest{
+							Config: ispec.Descriptor{
+								Digest: godigest.FromString("sha256:config123"),
+							},
+						},
+						Config: ispec.Image{},
+					},
+				},
+			}
+
+			fullImageMeta := convert.GetFullImageMeta("tag1", repoMeta, imageMeta)
+			So(fullImageMeta.TaggedTimestamp, ShouldEqual, taggedTime)
+
+			imageSummary, _, err := convert.ImageManifest2ImageSummary(ctx, fullImageMeta)
+			So(err, ShouldBeNil)
+			So(imageSummary.TaggedTimestamp, ShouldNotBeNil)
+			So(*imageSummary.TaggedTimestamp, ShouldEqual, taggedTime)
+		})
+
+		Convey("TaggedTimestamp falls back to PushTimestamp when zero", func() {
+			pushTime := time.Date(2024, time.January, 10, 8, 0, 0, 0, time.UTC)
+
+			// Use a proper digest that will match when converted to string
+			imageDigest := godigest.FromString("sha256:abc123")
+			digestStr := imageDigest.String()
+
+			repoMeta := mTypes.RepoMeta{
+				Name: "repo",
+				Tags: map[string]mTypes.Descriptor{
+					"tag1": {
+						Digest:          digestStr,
+						MediaType:       ispec.MediaTypeImageManifest,
+						TaggedTimestamp: time.Time{}, // Zero time
+					},
+				},
+				Statistics: map[string]mTypes.DescriptorStatistics{
+					digestStr: {
+						PushTimestamp: pushTime,
+					},
+				},
+			}
+
+			imageMeta := mTypes.ImageMeta{
+				Digest:    imageDigest,
+				MediaType: ispec.MediaTypeImageManifest,
+				Manifests: []mTypes.ManifestMeta{
+					{
+						Digest: imageDigest,
+						Manifest: ispec.Manifest{
+							Config: ispec.Descriptor{
+								Digest: godigest.FromString("sha256:config123"),
+							},
+						},
+						Config: ispec.Image{},
+					},
+				},
+			}
+
+			fullImageMeta := convert.GetFullImageMeta("tag1", repoMeta, imageMeta)
+			So(fullImageMeta.TaggedTimestamp.IsZero(), ShouldBeTrue)
+
+			imageSummary, _, err := convert.ImageManifest2ImageSummary(ctx, fullImageMeta)
+			So(err, ShouldBeNil)
+			So(imageSummary.TaggedTimestamp, ShouldNotBeNil)
+			So(*imageSummary.TaggedTimestamp, ShouldEqual, pushTime)
+		})
+
+		Convey("TaggedTimestamp is propagated to nested manifests in ImageIndex", func() {
+			taggedTime := time.Date(2024, time.January, 15, 10, 30, 0, 0, time.UTC)
+			pushTime := time.Date(2024, time.January, 10, 8, 0, 0, 0, time.UTC)
+
+			// Create a multiarch image
+			multiarchImage := CreateMultiarchWith().Images([]Image{
+				CreateRandomImage(),
+				CreateRandomImage(),
+			}).Build()
+
+			indexDigestStr := multiarchImage.DigestStr()
+
+			repoMeta := mTypes.RepoMeta{
+				Name: "repo",
+				Tags: map[string]mTypes.Descriptor{
+					"tag1": {
+						Digest:          indexDigestStr,
+						MediaType:       ispec.MediaTypeImageIndex,
+						TaggedTimestamp: taggedTime,
+					},
+				},
+				Statistics: map[string]mTypes.DescriptorStatistics{
+					indexDigestStr: {
+						PushTimestamp: pushTime,
+					},
+				},
+			}
+
+			imageMeta := multiarchImage.AsImageMeta()
+			fullImageMeta := convert.GetFullImageMeta("tag1", repoMeta, imageMeta)
+			So(fullImageMeta.TaggedTimestamp, ShouldEqual, taggedTime)
+
+			imageSummary, _, err := convert.ImageIndex2ImageSummary(ctx, fullImageMeta)
+			So(err, ShouldBeNil)
+			So(imageSummary.TaggedTimestamp, ShouldNotBeNil)
+			So(*imageSummary.TaggedTimestamp, ShouldEqual, taggedTime)
+
+			// Verify that nested manifests also have the correct TaggedTimestamp
+			So(len(imageSummary.Manifests), ShouldBeGreaterThan, 0)
+
+			for _, manifestSummary := range imageSummary.Manifests {
+				// Each manifest summary is part of the index, so they should inherit the index's TaggedTimestamp
+				// Note: ManifestSummary doesn't have TaggedTimestamp field, but the parent ImageSummary does
+				So(manifestSummary, ShouldNotBeNil)
+			}
+		})
+
+		Convey("TaggedTimestamp falls back to PushTimestamp for ImageIndex when zero", func() {
+			pushTime := time.Date(2024, time.January, 10, 8, 0, 0, 0, time.UTC)
+
+			// Create a multiarch image
+			multiarchImage := CreateMultiarchWith().Images([]Image{
+				CreateRandomImage(),
+			}).Build()
+
+			indexDigestStr := multiarchImage.DigestStr()
+
+			repoMeta := mTypes.RepoMeta{
+				Name: "repo",
+				Tags: map[string]mTypes.Descriptor{
+					"tag1": {
+						Digest:          indexDigestStr,
+						MediaType:       ispec.MediaTypeImageIndex,
+						TaggedTimestamp: time.Time{}, // Zero time
+					},
+				},
+				Statistics: map[string]mTypes.DescriptorStatistics{
+					indexDigestStr: {
+						PushTimestamp: pushTime,
+					},
+				},
+			}
+
+			imageMeta := multiarchImage.AsImageMeta()
+			fullImageMeta := convert.GetFullImageMeta("tag1", repoMeta, imageMeta)
+			So(fullImageMeta.TaggedTimestamp.IsZero(), ShouldBeTrue)
+
+			imageSummary, _, err := convert.ImageIndex2ImageSummary(ctx, fullImageMeta)
+			So(err, ShouldBeNil)
+			So(imageSummary.TaggedTimestamp, ShouldNotBeNil)
+			So(*imageSummary.TaggedTimestamp, ShouldEqual, pushTime)
+		})
+	})
+}
+
 func TestPaginatedConvert(t *testing.T) {
 	ctx := context.Background()
 

pkg/extensions/search/convert/metadb.go

@@ -162,17 +162,23 @@ func RepoMeta2ExpandedRepoInfo(ctx context.Context, repoMeta mTypes.RepoMeta,
 
 func GetFullImageMeta(tag string, repoMeta mTypes.RepoMeta, imageMeta mTypes.ImageMeta,
 ) mTypes.FullImageMeta {
+	taggedTimestamp := time.Time{}
+	if descriptor, ok := repoMeta.Tags[tag]; ok {
+		taggedTimestamp = descriptor.TaggedTimestamp
+	}
+
 	return mTypes.FullImageMeta{
-		Repo:       repoMeta.Name,
-		Tag:        tag,
-		MediaType:  imageMeta.MediaType,
-		Digest:     imageMeta.Digest,
-		Size:       imageMeta.Size,
-		Index:      imageMeta.Index,
-		Manifests:  GetFullManifestMeta(repoMeta, imageMeta.Manifests),
-		Referrers:  repoMeta.Referrers[imageMeta.Digest.String()],
-		Statistics: repoMeta.Statistics[imageMeta.Digest.String()],
-		Signatures: repoMeta.Signatures[imageMeta.Digest.String()],
+		Repo:            repoMeta.Name,
+		Tag:             tag,
+		MediaType:       imageMeta.MediaType,
+		Digest:          imageMeta.Digest,
+		Size:            imageMeta.Size,
+		Index:           imageMeta.Index,
+		Manifests:       GetFullManifestMeta(repoMeta, imageMeta.Manifests),
+		Referrers:       repoMeta.Referrers[imageMeta.Digest.String()],
+		Statistics:      repoMeta.Statistics[imageMeta.Digest.String()],
+		Signatures:      repoMeta.Signatures[imageMeta.Digest.String()],
+		TaggedTimestamp: taggedTimestamp,
 	}
 }
 
@@ -407,21 +413,29 @@ func ImageIndex2ImageSummary(ctx context.Context, fullImageMeta mTypes.FullImage
 		manifestSummaries   = make([]*gql_generated.ManifestSummary, 0, len(fullImageMeta.Manifests))
 		indexBlobs          = map[string]int64{}
 
-		indexDigestStr = fullImageMeta.Digest.String()
-		indexMediaType = ispec.MediaTypeImageIndex
+		indexDigestStr  = fullImageMeta.Digest.String()
+		indexMediaType  = ispec.MediaTypeImageIndex
+		pushTimestamp   = fullImageMeta.Statistics.PushTimestamp
+		taggedTimestamp = fullImageMeta.TaggedTimestamp
 	)
 
+	// Fallback to PushTimestamp if TaggedTimestamp is not available
+	if taggedTimestamp.IsZero() {
+		taggedTimestamp = pushTimestamp
+	}
+
 	for _, imageManifest := range fullImageMeta.Manifests {
 		imageManifestSummary, manifestBlobs, err := ImageManifest2ImageSummary(ctx, mTypes.FullImageMeta{
-			Repo:       fullImageMeta.Repo,
-			Tag:        fullImageMeta.Tag,
-			MediaType:  ispec.MediaTypeImageManifest,
-			Digest:     imageManifest.Digest,
-			Size:       imageManifest.Size,
-			Manifests:  []mTypes.FullManifestMeta{imageManifest},
-			Referrers:  imageManifest.Referrers,
-			Statistics: imageManifest.Statistics,
-			Signatures: imageManifest.Signatures,
+			Repo:            fullImageMeta.Repo,
+			Tag:             fullImageMeta.Tag,
+			MediaType:       ispec.MediaTypeImageManifest,
+			Digest:          imageManifest.Digest,
+			Size:            imageManifest.Size,
+			Manifests:       []mTypes.FullManifestMeta{imageManifest},
+			Referrers:       imageManifest.Referrers,
+			Statistics:      imageManifest.Statistics,
+			Signatures:      imageManifest.Signatures,
+			TaggedTimestamp: fullImageMeta.TaggedTimestamp,
 		})
 		if err != nil {
 			return &gql_generated.ImageSummary{}, map[string]int64{}, err
@@ -462,25 +476,27 @@ func ImageIndex2ImageSummary(ctx context.Context, fullImageMeta mTypes.FullImage
 	}
 
 	indexSummary := gql_generated.ImageSummary{
-		RepoName:      &repo,
-		Tag:           &tag,
-		Digest:        &indexDigestStr,
-		MediaType:     &indexMediaType,
-		Manifests:     manifestSummaries,
-		LastUpdated:   imageLastUpdated,
-		IsSigned:      &isSigned,
-		SignatureInfo: signaturesInfo,
-		Size:          ref(strconv.FormatInt(indexSize, 10)),
-		DownloadCount: ref(fullImageMeta.Statistics.DownloadCount),
-		Description:   &annotations.Description,
-		Title:         &annotations.Title,
-		Documentation: &annotations.Documentation,
-		Licenses:      &annotations.Licenses,
-		Labels:        &annotations.Labels,
-		Source:        &annotations.Source,
-		Vendor:        &annotations.Vendor,
-		Authors:       &annotations.Authors,
-		Referrers:     getReferrers(fullImageMeta.Referrers),
+		RepoName:        &repo,
+		Tag:             &tag,
+		Digest:          &indexDigestStr,
+		MediaType:       &indexMediaType,
+		Manifests:       manifestSummaries,
+		LastUpdated:     imageLastUpdated,
+		IsSigned:        &isSigned,
+		SignatureInfo:   signaturesInfo,
+		Size:            ref(strconv.FormatInt(indexSize, 10)),
+		DownloadCount:   ref(fullImageMeta.Statistics.DownloadCount),
+		PushTimestamp:   &pushTimestamp,
+		TaggedTimestamp: &taggedTimestamp,
+		Description:     &annotations.Description,
+		Title:           &annotations.Title,
+		Documentation:   &annotations.Documentation,
+		Licenses:        &annotations.Licenses,
+		Labels:          &annotations.Labels,
+		Source:          &annotations.Source,
+		Vendor:          &annotations.Vendor,
+		Authors:         &annotations.Authors,
+		Referrers:       getReferrers(fullImageMeta.Referrers),
 	}
 
 	return &indexSummary, indexBlobs, nil
@@ -504,8 +520,14 @@ func ImageManifest2ImageSummary(ctx context.Context, fullImageMeta mTypes.FullIm
 		isSigned          = isImageSigned(fullImageMeta.Signatures)
 		lastPullTimestamp = fullImageMeta.Statistics.LastPullTimestamp
 		pushTimestamp     = fullImageMeta.Statistics.PushTimestamp
+		taggedTimestamp   = fullImageMeta.TaggedTimestamp
 	)
 
+	// Fallback to PushTimestamp if TaggedTimestamp is not available
+	if taggedTimestamp.IsZero() {
+		taggedTimestamp = pushTimestamp
+	}
+
 	imageSize, imageBlobsMap := getImageBlobsInfo(manifestDigest, manifestSize, configDigest, configSize,
 		manifest.Manifest.Layers)
 	imageSizeStr := strconv.FormatInt(imageSize, 10)
@@ -558,6 +580,7 @@ func ImageManifest2ImageSummary(ctx context.Context, fullImageMeta mTypes.FullIm
 		DownloadCount:     &downloadCount,
 		LastPullTimestamp: &lastPullTimestamp,
 		PushTimestamp:     &pushTimestamp,
+		TaggedTimestamp:   &taggedTimestamp,
 		Description:       &annotations.Description,
 		Title:             &annotations.Title,
 		Documentation:     &annotations.Documentation,

pkg/extensions/search/gql_generated/generated.go

@@ -117,6 +117,7 @@ type ComplexityRoot struct {
 		Size              func(childComplexity int) int
 		Source            func(childComplexity int) int
 		Tag               func(childComplexity int) int
+		TaggedTimestamp   func(childComplexity int) int
 		Title             func(childComplexity int) int
 		Vendor            func(childComplexity int) int
 		Vulnerabilities   func(childComplexity int) int
@@ -581,6 +582,12 @@ func (e *executableSchema) Complexity(ctx context.Context, typeName, field strin
 		}
 
 		return e.complexity.ImageSummary.Tag(childComplexity), true
+	case "ImageSummary.TaggedTimestamp":
+		if e.complexity.ImageSummary.TaggedTimestamp == nil {
+			break
+		}
+
+		return e.complexity.ImageSummary.TaggedTimestamp(childComplexity), true
 	case "ImageSummary.Title":
 		if e.complexity.ImageSummary.Title == nil {
 			break
@@ -1426,10 +1433,14 @@ type ImageSummary {
     """
     LastPullTimestamp: Time
     """
-	Timestamp when the image was pushed to the registry
+    Timestamp when the image was pushed to the registry
     """
     PushTimestamp: Time
     """
+    Timestamp when the image manifest was tagged (if the data is unavailable it falls back to when the image was pushed to the registry)
+    """
+    TaggedTimestamp: Time
+    """
     Timestamp of the last modification done to the image (from config or the last updated layer)
     """
     LastUpdated: Time
@@ -3132,6 +3143,8 @@ func (ec *executionContext) fieldContext_GlobalSearchResult_Images(_ context.Con
 				return ec.fieldContext_ImageSummary_LastPullTimestamp(ctx, field)
 			case "PushTimestamp":
 				return ec.fieldContext_ImageSummary_PushTimestamp(ctx, field)
+			case "TaggedTimestamp":
+				return ec.fieldContext_ImageSummary_TaggedTimestamp(ctx, field)
 			case "LastUpdated":
 				return ec.fieldContext_ImageSummary_LastUpdated(ctx, field)
 			case "Description":
@@ -3811,6 +3824,35 @@ func (ec *executionContext) fieldContext_ImageSummary_PushTimestamp(_ context.Co
 	return fc, nil
 }
 
+func (ec *executionContext) _ImageSummary_TaggedTimestamp(ctx context.Context, field graphql.CollectedField, obj *ImageSummary) (ret graphql.Marshaler) {
+	return graphql.ResolveField(
+		ctx,
+		ec.OperationContext,
+		field,
+		ec.fieldContext_ImageSummary_TaggedTimestamp,
+		func(ctx context.Context) (any, error) {
+			return obj.TaggedTimestamp, nil
+		},
+		nil,
+		ec.marshalOTime2ᚖtimeᚐTime,
+		true,
+		false,
+	)
+}
+
+func (ec *executionContext) fieldContext_ImageSummary_TaggedTimestamp(_ context.Context, field graphql.CollectedField) (fc *graphql.FieldContext, err error) {
+	fc = &graphql.FieldContext{
+		Object:     "ImageSummary",
+		Field:      field,
+		IsMethod:   false,
+		IsResolver: false,
+		Child: func(ctx context.Context, field graphql.CollectedField) (*graphql.FieldContext, error) {
+			return nil, errors.New("field of type Time does not have child fields")
+		},
+	}
+	return fc, nil
+}
+
 func (ec *executionContext) _ImageSummary_LastUpdated(ctx context.Context, field graphql.CollectedField, obj *ImageSummary) (ret graphql.Marshaler) {
 	return graphql.ResolveField(
 		ctx,
@@ -5272,6 +5314,8 @@ func (ec *executionContext) fieldContext_PaginatedImagesResult_Results(_ context
 				return ec.fieldContext_ImageSummary_LastPullTimestamp(ctx, field)
 			case "PushTimestamp":
 				return ec.fieldContext_ImageSummary_PushTimestamp(ctx, field)
+			case "TaggedTimestamp":
+				return ec.fieldContext_ImageSummary_TaggedTimestamp(ctx, field)
 			case "LastUpdated":
 				return ec.fieldContext_ImageSummary_LastUpdated(ctx, field)
 			case "Description":
@@ -6027,6 +6071,8 @@ func (ec *executionContext) fieldContext_Query_Image(ctx context.Context, field
 				return ec.fieldContext_ImageSummary_LastPullTimestamp(ctx, field)
 			case "PushTimestamp":
 				return ec.fieldContext_ImageSummary_PushTimestamp(ctx, field)
+			case "TaggedTimestamp":
+				return ec.fieldContext_ImageSummary_TaggedTimestamp(ctx, field)
 			case "LastUpdated":
 				return ec.fieldContext_ImageSummary_LastUpdated(ctx, field)
 			case "Description":
@@ -6521,6 +6567,8 @@ func (ec *executionContext) fieldContext_RepoInfo_Images(_ context.Context, fiel
 				return ec.fieldContext_ImageSummary_LastPullTimestamp(ctx, field)
 			case "PushTimestamp":
 				return ec.fieldContext_ImageSummary_PushTimestamp(ctx, field)
+			case "TaggedTimestamp":
+				return ec.fieldContext_ImageSummary_TaggedTimestamp(ctx, field)
 			case "LastUpdated":
 				return ec.fieldContext_ImageSummary_LastUpdated(ctx, field)
 			case "Description":
@@ -6802,6 +6850,8 @@ func (ec *executionContext) fieldContext_RepoSummary_NewestImage(_ context.Conte
 				return ec.fieldContext_ImageSummary_LastPullTimestamp(ctx, field)
 			case "PushTimestamp":
 				return ec.fieldContext_ImageSummary_PushTimestamp(ctx, field)
+			case "TaggedTimestamp":
+				return ec.fieldContext_ImageSummary_TaggedTimestamp(ctx, field)
 			case "LastUpdated":
 				return ec.fieldContext_ImageSummary_LastUpdated(ctx, field)
 			case "Description":
@@ -9040,6 +9090,8 @@ func (ec *executionContext) _ImageSummary(ctx context.Context, sel ast.Selection
 			out.Values[i] = ec._ImageSummary_LastPullTimestamp(ctx, field, obj)
 		case "PushTimestamp":
 			out.Values[i] = ec._ImageSummary_PushTimestamp(ctx, field, obj)
+		case "TaggedTimestamp":
+			out.Values[i] = ec._ImageSummary_TaggedTimestamp(ctx, field, obj)
 		case "LastUpdated":
 			out.Values[i] = ec._ImageSummary_LastUpdated(ctx, field, obj)
 		case "Description":

pkg/extensions/search/gql_generated/models_gen.go

@@ -152,6 +152,8 @@ type ImageSummary struct {
 	LastPullTimestamp *time.Time `json:"LastPullTimestamp,omitempty"`
 	// Timestamp when the image was pushed to the registry
 	PushTimestamp *time.Time `json:"PushTimestamp,omitempty"`
+	// Timestamp when the image manifest was tagged (if the data is unavailable it falls back to when the image was pushed to the registry)
+	TaggedTimestamp *time.Time `json:"TaggedTimestamp,omitempty"`
 	// Timestamp of the last modification done to the image (from config or the last updated layer)
 	LastUpdated *time.Time `json:"LastUpdated,omitempty"`
 	// Human-readable description of the software packaged in the image

pkg/extensions/search/schema.graphql

@@ -209,10 +209,14 @@ type ImageSummary {
     """
     LastPullTimestamp: Time
     """
-	Timestamp when the image was pushed to the registry
+    Timestamp when the image was pushed to the registry
     """
     PushTimestamp: Time
     """
+    Timestamp when the image manifest was tagged (if the data is unavailable it falls back to when the image was pushed to the registry)
+    """
+    TaggedTimestamp: Time
+    """
     Timestamp of the last modification done to the image (from config or the last updated layer)
     """
     LastUpdated: Time