fix(meta): fixes for LastUpdated and TaggedTimestamp (#3754)

1. Parse repos without metadata in ParseStorage

The timestamp check in ParseStorage was skipping repos that exist in
storage but don't have metadata. When GetRepoLastUpdated returns zero
time (no metadata), we should always parse the repo to create its
metadata. Check if metaLastUpdated is zero before comparing timestamps.
If zero, always parse regardless of storageLastUpdated.

2. Change the logic of how LastUpdated is computed in RepoSummary

It is not the latest tagged timestamp from the available images or
the last updated image created timestamp, based on whichever is the
latest.

Signed-off-by: Andrei Aaron <andreifdaaron@gmail.com>

pkg/extensions/search/convert/convert_test.go

@@ -1020,6 +1020,160 @@ func TestIndexAnnotations(t *testing.T) {
 	})
 }
 
+func TestRepoMeta2RepoSummary(t *testing.T) {
+	ctx := context.Background()
+
+	Convey("Test RepoMeta2RepoSummary LastUpdated with TaggedTimestamp", t, func() {
+		now := time.Now()
+		olderTime := now.Add(-2 * time.Hour)
+		newerTime := now.Add(-1 * time.Hour)
+		newestTime := now
+		futureTime := now.Add(1 * time.Hour)
+
+		// Create a repo with multiple tags having different TaggedTimestamp values
+		repoMeta := mTypes.RepoMeta{
+			Name: "test-repo",
+			Tags: map[mTypes.Tag]mTypes.Descriptor{
+				"tag1": {
+					Digest:          "sha256:digest1",
+					MediaType:       "application/vnd.oci.image.manifest.v1+json",
+					TaggedTimestamp: olderTime,
+				},
+				"tag2": {
+					Digest:          "sha256:digest2",
+					MediaType:       "application/vnd.oci.image.manifest.v1+json",
+					TaggedTimestamp: newerTime,
+				},
+				"tag3": {
+					Digest:          "sha256:digest3",
+					MediaType:       "application/vnd.oci.image.manifest.v1+json",
+					TaggedTimestamp: futureTime, // This is the newest TaggedTimestamp
+				},
+			},
+			LastUpdatedImage: &mTypes.LastUpdatedImage{
+				Descriptor: mTypes.Descriptor{
+					Digest:    "sha256:digest2",
+					MediaType: "application/vnd.oci.image.manifest.v1+json",
+				},
+				Tag:         "tag2",
+				LastUpdated: &newestTime, // This is newer than olderTime and newerTime, but older than futureTime
+			},
+		}
+
+		imageMetaMap := map[string]mTypes.ImageMeta{
+			"sha256:digest2": {
+				Digest:    godigest.FromString("sha256:digest2"),
+				MediaType: "application/vnd.oci.image.manifest.v1+json",
+				Manifests: []mTypes.ManifestMeta{
+					{
+						Digest: godigest.FromString("sha256:digest2"),
+						Config: ispec.Image{
+							Created: &newestTime,
+						},
+					},
+				},
+			},
+		}
+
+		repoSummary := convert.RepoMeta2RepoSummary(ctx, repoMeta, imageMetaMap)
+
+		// LastUpdated should be futureTime (the maximum of all TaggedTimestamp values and LastUpdatedImage.LastUpdated)
+		So(repoSummary, ShouldNotBeNil)
+		So(repoSummary.LastUpdated, ShouldNotBeNil)
+		So(*repoSummary.LastUpdated, ShouldEqual, futureTime)
+	})
+
+	Convey("Test RepoMeta2RepoSummary LastUpdated when TaggedTimestamp is older than LastUpdated", t, func() {
+		now := time.Now()
+		olderTime := now.Add(-2 * time.Hour)
+		newestTime := now
+
+		repoMeta := mTypes.RepoMeta{
+			Name: "test-repo",
+			Tags: map[mTypes.Tag]mTypes.Descriptor{
+				"tag1": {
+					Digest:          "sha256:digest1",
+					MediaType:       "application/vnd.oci.image.manifest.v1+json",
+					TaggedTimestamp: olderTime, // Older than LastUpdated
+				},
+			},
+			LastUpdatedImage: &mTypes.LastUpdatedImage{
+				Descriptor: mTypes.Descriptor{
+					Digest:    "sha256:digest1",
+					MediaType: "application/vnd.oci.image.manifest.v1+json",
+				},
+				Tag:         "tag1",
+				LastUpdated: &newestTime,
+			},
+		}
+
+		imageMetaMap := map[string]mTypes.ImageMeta{
+			"sha256:digest1": {
+				Digest:    godigest.FromString("sha256:digest1"),
+				MediaType: "application/vnd.oci.image.manifest.v1+json",
+				Manifests: []mTypes.ManifestMeta{
+					{
+						Digest: godigest.FromString("sha256:digest1"),
+						Config: ispec.Image{
+							Created: &newestTime,
+						},
+					},
+				},
+			},
+		}
+
+		repoSummary := convert.RepoMeta2RepoSummary(ctx, repoMeta, imageMetaMap)
+
+		// LastUpdated should be newestTime (the LastUpdatedImage.LastUpdated, which is newer than TaggedTimestamp)
+		So(repoSummary, ShouldNotBeNil)
+		So(repoSummary.LastUpdated, ShouldNotBeNil)
+		So(*repoSummary.LastUpdated, ShouldEqual, newestTime)
+	})
+
+	Convey("Test RepoMeta2RepoSummary LastUpdated with zero timestamps", t, func() {
+		zeroTime := time.Time{}
+
+		repoMeta := mTypes.RepoMeta{
+			Name: "test-repo",
+			Tags: map[mTypes.Tag]mTypes.Descriptor{
+				"tag1": {
+					Digest:          "sha256:digest1",
+					MediaType:       "application/vnd.oci.image.manifest.v1+json",
+					TaggedTimestamp: zeroTime,
+				},
+			},
+			LastUpdatedImage: &mTypes.LastUpdatedImage{
+				Descriptor: mTypes.Descriptor{
+					Digest:    "sha256:digest1",
+					MediaType: "application/vnd.oci.image.manifest.v1+json",
+				},
+				Tag:         "tag1",
+				LastUpdated: nil,
+			},
+		}
+
+		imageMetaMap := map[string]mTypes.ImageMeta{
+			"sha256:digest1": {
+				Digest:    godigest.FromString("sha256:digest1"),
+				MediaType: "application/vnd.oci.image.manifest.v1+json",
+				Manifests: []mTypes.ManifestMeta{
+					{
+						Digest: godigest.FromString("sha256:digest1"),
+						Config: ispec.Image{},
+					},
+				},
+			},
+		}
+
+		repoSummary := convert.RepoMeta2RepoSummary(ctx, repoMeta, imageMetaMap)
+
+		// LastUpdated should be zero time when all timestamps are zero
+		So(repoSummary, ShouldNotBeNil)
+		So(repoSummary.LastUpdated, ShouldNotBeNil)
+		So(*repoSummary.LastUpdated, ShouldEqual, zeroTime)
+	})
+}
+
 func TestConvertErrors(t *testing.T) {
 	ctx := context.Background()
 	log := log.NewTestLogger()

pkg/extensions/search/convert/metadb.go

@@ -324,24 +324,38 @@ func RepoMeta2RepoSummary(ctx context.Context, repoMeta mTypes.RepoMeta,
 	imageMetaMap map[string]mTypes.ImageMeta,
 ) *gql_generated.RepoSummary {
 	var (
-		repoName                 = repoMeta.Name
-		lastUpdatedImage         = deref(repoMeta.LastUpdatedImage, mTypes.LastUpdatedImage{})
-		lastUpdatedImageMeta     = imageMetaMap[lastUpdatedImage.Digest]
-		lastUpdatedTag           = lastUpdatedImage.Tag
-		repoLastUpdatedTimestamp = lastUpdatedImage.LastUpdated
-		repoPlatforms            = repoMeta.Platforms
-		repoVendors              = repoMeta.Vendors
-		repoDownloadCount        = repoMeta.DownloadCount
-		repoStarCount            = repoMeta.StarCount
-		repoIsUserStarred        = repoMeta.IsStarred    // value specific to the current user
-		repoIsUserBookMarked     = repoMeta.IsBookmarked // value specific to the current user
-		repoSize                 = repoMeta.Size
+		repoName             = repoMeta.Name
+		lastUpdatedImage     = deref(repoMeta.LastUpdatedImage, mTypes.LastUpdatedImage{})
+		lastUpdatedImageMeta = imageMetaMap[lastUpdatedImage.Digest]
+		lastUpdatedTag       = lastUpdatedImage.Tag
+		repoPlatforms        = repoMeta.Platforms
+		repoVendors          = repoMeta.Vendors
+		repoDownloadCount    = repoMeta.DownloadCount
+		repoStarCount        = repoMeta.StarCount
+		repoIsUserStarred    = repoMeta.IsStarred    // value specific to the current user
+		repoIsUserBookMarked = repoMeta.IsBookmarked // value specific to the current user
+		repoSize             = repoMeta.Size
 	)
 
-	if repoLastUpdatedTimestamp == nil {
-		repoLastUpdatedTimestamp = &time.Time{}
+	// Compute LastUpdated as the latest of:
+	// 1. The LastUpdated timestamp of the last updated image
+	// 2. All TaggedTimestamp values of all tags in the repository
+	var maxTimestamp time.Time
+
+	// Start with the LastUpdated from the last updated image
+	if lastUpdatedImage.LastUpdated != nil {
+		maxTimestamp = *lastUpdatedImage.LastUpdated
 	}
 
+	// Check all TaggedTimestamp values from all tags
+	for _, descriptor := range repoMeta.Tags {
+		if !descriptor.TaggedTimestamp.IsZero() && descriptor.TaggedTimestamp.After(maxTimestamp) {
+			maxTimestamp = descriptor.TaggedTimestamp
+		}
+	}
+
+	repoLastUpdatedTimestamp := &maxTimestamp
+
 	imageSummary, _, err := FullImageMeta2ImageSummary(ctx, GetFullImageMeta(lastUpdatedTag, repoMeta,
 		lastUpdatedImageMeta))
 	_ = err

pkg/extensions/search/cve/update_test.go

@@ -66,7 +66,7 @@ func TestCVEDBGenerator(t *testing.T) {
 
 		// Wait for trivy db to download
 		found, err := test.ReadLogFileAndCountStringOccurence(logPath,
-			"cve-db update completed, next update scheduled after interval", 140*time.Second, 2)
+			"cve-db update completed, next update scheduled after interval", 240*time.Second, 2)
 		So(err, ShouldBeNil)
 		So(found, ShouldBeTrue)
 	})

pkg/extensions/search/search_test.go

@@ -3403,6 +3403,7 @@ func TestGlobalSearch(t *testing.T) { //nolint: gocyclo
 
 		allExpectedRepoInfoMap := make(map[string]zcommon.RepoInfo)
 		allExpectedImageSummaryMap := make(map[string]zcommon.ImageSummary)
+		expectedLastUpdatedMap := make(map[string]time.Time)
 
 		for _, repo := range repos {
 			repoInfo, err := olu.GetExpandedRepoInfo(repo)
@@ -3413,6 +3414,23 @@ func TestGlobalSearch(t *testing.T) { //nolint: gocyclo
 				imageName := fmt.Sprintf("%s:%s", repo, image.Tag)
 				allExpectedImageSummaryMap[imageName] = image
 			}
+
+			// Compute expected LastUpdated as the maximum of:
+			// 1. NewestImage.LastUpdated (from the last updated image)
+			// 2. All TaggedTimestamp values from all tags in the repository
+			repoMeta, err := ctlr.MetaDB.GetRepoMeta(context.Background(), repo)
+			So(err, ShouldBeNil)
+
+			expectedLastUpdated := repoInfo.Summary.NewestImage.LastUpdated
+
+			// Check all TaggedTimestamp values from all tags
+			for _, descriptor := range repoMeta.Tags {
+				if !descriptor.TaggedTimestamp.IsZero() && descriptor.TaggedTimestamp.After(expectedLastUpdated) {
+					expectedLastUpdated = descriptor.TaggedTimestamp
+				}
+			}
+
+			expectedLastUpdatedMap[repo] = expectedLastUpdated
 		}
 
 		query := `
@@ -3493,10 +3511,16 @@ func TestGlobalSearch(t *testing.T) { //nolint: gocyclo
 			// Check if data in NewestImage is consistent with the data in RepoSummary
 			So(repoName, ShouldEqual, repoSummary.NewestImage.RepoName)
 			So(repoSummary.Name, ShouldEqual, repoSummary.NewestImage.RepoName)
-			So(repoSummary.LastUpdated, ShouldEqual, repoSummary.NewestImage.LastUpdated)
+
+			// Verify the actual LastUpdated matches the computed expected value
+			expectedLastUpdated, exists := expectedLastUpdatedMap[repoName]
+			So(exists, ShouldBeTrue)
+			So(repoSummary.LastUpdated, ShouldEqual, expectedLastUpdated)
 
 			// The data in the RepoSummary returned from the request matches the data returned from the disk
 			repoInfo := allExpectedRepoInfoMap[repoName]
+			// Update the expected LastUpdated to account for TaggedTimestamp (which is not available from disk)
+			repoInfo.Summary.LastUpdated = expectedLastUpdated
 
 			t.Logf("Validate repo summary returned by global search with vulnerability scanning disabled")
 			verifyRepoSummaryFields(t, &repoSummary, &repoInfo.Summary)
@@ -3747,6 +3771,7 @@ func TestGlobalSearch(t *testing.T) { //nolint: gocyclo
 
 		allExpectedRepoInfoMap := make(map[string]zcommon.RepoInfo)
 		allExpectedImageSummaryMap := make(map[string]zcommon.ImageSummary)
+		expectedLastUpdatedMap := make(map[string]time.Time)
 
 		for _, repo := range repos {
 			repoInfo, err := olu.GetExpandedRepoInfo(repo)
@@ -3757,6 +3782,23 @@ func TestGlobalSearch(t *testing.T) { //nolint: gocyclo
 				imageName := fmt.Sprintf("%s:%s", repo, image.Tag)
 				allExpectedImageSummaryMap[imageName] = image
 			}
+
+			// Compute expected LastUpdated as the maximum of:
+			// 1. NewestImage.LastUpdated (from the last updated image)
+			// 2. All TaggedTimestamp values from all tags in the repository
+			repoMeta, err := ctlr.MetaDB.GetRepoMeta(context.Background(), repo)
+			So(err, ShouldBeNil)
+
+			expectedLastUpdated := repoInfo.Summary.NewestImage.LastUpdated
+
+			// Check all TaggedTimestamp values from all tags
+			for _, descriptor := range repoMeta.Tags {
+				if !descriptor.TaggedTimestamp.IsZero() && descriptor.TaggedTimestamp.After(expectedLastUpdated) {
+					expectedLastUpdated = descriptor.TaggedTimestamp
+				}
+			}
+
+			expectedLastUpdatedMap[repo] = expectedLastUpdated
 		}
 
 		query := `
@@ -3833,10 +3875,16 @@ func TestGlobalSearch(t *testing.T) { //nolint: gocyclo
 			// Check if data in NewestImage is consistent with the data in RepoSummary
 			So(repoName, ShouldEqual, repoSummary.NewestImage.RepoName)
 			So(repoSummary.Name, ShouldEqual, repoSummary.NewestImage.RepoName)
-			So(repoSummary.LastUpdated, ShouldEqual, repoSummary.NewestImage.LastUpdated)
+
+			// Verify the actual LastUpdated matches the computed expected value
+			expectedLastUpdated, exists := expectedLastUpdatedMap[repoName]
+			So(exists, ShouldBeTrue)
+			So(repoSummary.LastUpdated, ShouldEqual, expectedLastUpdated)
 
 			// The data in the RepoSummary returned from the request matches the data returned from the disk
 			repoInfo := allExpectedRepoInfoMap[repoName]
+			// Update the expected LastUpdated to account for TaggedTimestamp (which is not available from disk)
+			repoInfo.Summary.LastUpdated = expectedLastUpdated
 
 			t.Logf("Validate repo summary returned by global search with vulnerability scanning enabled")
 			verifyRepoSummaryFields(t, &repoSummary, &repoInfo.Summary)