Files
zot/pkg/extensions/search/convert/metadb.go
T
John Wagner 8f0bf18d7e fix(search): expose LastPullTimestamp and PushedBy on index ImageSummary (#3865)
ImageIndex2ImageSummary was missing LastPullTimestamp assignment, causing
multi-arch image queries to always return null for this field. Also adds
the PushedBy field (already stored in MetaDB) to the GraphQL schema and
both conversion paths (manifest and index).

Signed-off-by: cainydev <wajo432@gmail.com>
2026-03-10 16:40:54 -07:00

685 lines
21 KiB
Go

package convert
import (
"context"
"sort"
"strconv"
"time"
"github.com/99designs/gqlgen/graphql"
ispec "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/vektah/gqlparser/v2/gqlerror"
zerr "zotregistry.dev/zot/v2/errors"
zcommon "zotregistry.dev/zot/v2/pkg/common"
cveinfo "zotregistry.dev/zot/v2/pkg/extensions/search/cve"
"zotregistry.dev/zot/v2/pkg/extensions/search/gql_generated"
"zotregistry.dev/zot/v2/pkg/extensions/search/pagination"
"zotregistry.dev/zot/v2/pkg/log"
mTypes "zotregistry.dev/zot/v2/pkg/meta/types"
reqCtx "zotregistry.dev/zot/v2/pkg/requestcontext"
)
type SkipQGLField struct {
Vulnerabilities bool
}
func UpdateLastUpdatedTimestamp(repoLastUpdatedTimestamp *time.Time,
lastUpdatedImageSummary *gql_generated.ImageSummary, imageSummary *gql_generated.ImageSummary,
) *gql_generated.ImageSummary {
newLastUpdatedImageSummary := lastUpdatedImageSummary
if repoLastUpdatedTimestamp.Equal(time.Time{}) {
// initialize with first time value
*repoLastUpdatedTimestamp = *imageSummary.LastUpdated
newLastUpdatedImageSummary = imageSummary
} else if repoLastUpdatedTimestamp.Before(*imageSummary.LastUpdated) {
*repoLastUpdatedTimestamp = *imageSummary.LastUpdated
newLastUpdatedImageSummary = imageSummary
}
return newLastUpdatedImageSummary
}
func getReferrers(referrersInfo []mTypes.ReferrerInfo) []*gql_generated.Referrer {
referrers := make([]*gql_generated.Referrer, 0, len(referrersInfo))
for _, referrerInfo := range referrersInfo {
referrers = append(referrers, &gql_generated.Referrer{
MediaType: &referrerInfo.MediaType,
ArtifactType: &referrerInfo.ArtifactType,
Size: &referrerInfo.Size,
Digest: &referrerInfo.Digest,
Annotations: getAnnotationsFromMap(referrerInfo.Annotations),
})
}
return referrers
}
func getAnnotationsFromMap(annotationsMap map[string]string) []*gql_generated.Annotation {
annotations := make([]*gql_generated.Annotation, 0, len(annotationsMap))
for key, value := range annotationsMap {
annotations = append(annotations, &gql_generated.Annotation{
Key: &key,
Value: &value,
})
}
return annotations
}
func getImageBlobsInfo(manifestDigest string, manifestSize int64, configDigest string, configSize int64,
layers []ispec.Descriptor,
) (int64, map[string]int64) {
// Pre-allocate map with known size: config + manifest + layers
imageBlobsMap := make(map[string]int64, 2+len(layers))
imageSize := int64(0)
// add config size
imageSize += configSize
imageBlobsMap[configDigest] = configSize
// add manifest size
imageSize += manifestSize
imageBlobsMap[manifestDigest] = manifestSize
// add layers size
for _, layer := range layers {
imageBlobsMap[layer.Digest.String()] = layer.Size
imageSize += layer.Size
}
return imageSize, imageBlobsMap
}
func RepoMeta2ImageSummaries(ctx context.Context, repoMeta mTypes.RepoMeta,
imageMeta map[string]mTypes.ImageMeta, skip SkipQGLField, cveInfo cveinfo.CveInfo,
) []*gql_generated.ImageSummary {
imageSummaries := make([]*gql_generated.ImageSummary, 0, len(repoMeta.Tags))
// Make sure the tags are sorted
// We need to implement a proper fix for this taking into account
// the sorting criteria used in the requested page
tags := make([]string, 0, len(repoMeta.Tags))
for tag := range repoMeta.Tags {
tags = append(tags, tag)
}
// Sorting ascending by tag name should do for now
sort.Strings(tags)
for _, tag := range tags {
descriptor := repoMeta.Tags[tag]
imageSummary, _, err := FullImageMeta2ImageSummary(ctx, GetFullImageMeta(tag, repoMeta, imageMeta[descriptor.Digest]))
if err != nil {
continue
}
// CVE scanning is expensive, only scan for final slice of results
updateImageSummaryVulnerabilities(ctx, imageSummary, skip, cveInfo)
imageSummaries = append(imageSummaries, imageSummary)
}
return imageSummaries
}
func RepoMeta2ExpandedRepoInfo(ctx context.Context, repoMeta mTypes.RepoMeta,
imageMetaMap map[string]mTypes.ImageMeta, skip SkipQGLField, cveInfo cveinfo.CveInfo, log log.Logger,
) (*gql_generated.RepoSummary, []*gql_generated.ImageSummary) {
repoName := repoMeta.Name
imageSummaries := make([]*gql_generated.ImageSummary, 0, len(repoMeta.Tags))
userCanDeleteTag, _ := reqCtx.CanDelete(ctx, repoName)
for tag, descriptor := range repoMeta.Tags {
imageMeta := imageMetaMap[descriptor.Digest]
imageSummary, _, err := FullImageMeta2ImageSummary(ctx, GetFullImageMeta(tag, repoMeta, imageMeta))
if err != nil {
log.Error().Str("repository", repoName).Str("reference", tag).Str("component", "metadb").
Msg("error while converting descriptor for image")
continue
}
imageSummary.IsDeletable = &userCanDeleteTag
updateImageSummaryVulnerabilities(ctx, imageSummary, skip, cveInfo)
imageSummaries = append(imageSummaries, imageSummary)
}
repoSummary := RepoMeta2RepoSummary(ctx, repoMeta, imageMetaMap)
updateRepoSummaryVulnerabilities(ctx, repoSummary, skip, cveInfo)
return repoSummary, imageSummaries
}
func GetFullImageMeta(tag string, repoMeta mTypes.RepoMeta, imageMeta mTypes.ImageMeta,
) mTypes.FullImageMeta {
taggedTimestamp := time.Time{}
if descriptor, ok := repoMeta.Tags[tag]; ok {
taggedTimestamp = descriptor.TaggedTimestamp
}
return mTypes.FullImageMeta{
Repo: repoMeta.Name,
Tag: tag,
MediaType: imageMeta.MediaType,
Digest: imageMeta.Digest,
Size: imageMeta.Size,
Index: imageMeta.Index,
Manifests: GetFullManifestMeta(repoMeta, imageMeta.Manifests),
Referrers: repoMeta.Referrers[imageMeta.Digest.String()],
Statistics: repoMeta.Statistics[imageMeta.Digest.String()],
Signatures: repoMeta.Signatures[imageMeta.Digest.String()],
TaggedTimestamp: taggedTimestamp,
}
}
func GetFullManifestMeta(repoMeta mTypes.RepoMeta, manifests []mTypes.ManifestMeta) []mTypes.FullManifestMeta {
results := make([]mTypes.FullManifestMeta, 0, len(manifests))
for i := range manifests {
results = append(results, mTypes.FullManifestMeta{
ManifestMeta: manifests[i],
Referrers: repoMeta.Referrers[manifests[i].Digest.String()],
Statistics: repoMeta.Statistics[manifests[i].Digest.String()],
Signatures: repoMeta.Signatures[manifests[i].Digest.String()],
})
}
return results
}
func StringMap2Annotations(strMap map[string]string) []*gql_generated.Annotation {
annotations := make([]*gql_generated.Annotation, 0, len(strMap))
for key, value := range strMap {
annotations = append(annotations, &gql_generated.Annotation{
Key: &key,
Value: &value,
})
}
return annotations
}
func GetPreloads(ctx context.Context) map[string]bool {
if !graphql.HasOperationContext(ctx) {
return map[string]bool{}
}
nestedPreloads := GetNestedPreloads(
graphql.GetOperationContext(ctx),
graphql.CollectFieldsCtx(ctx, nil),
"",
)
preloads := map[string]bool{}
for _, str := range nestedPreloads {
preloads[str] = true
}
return preloads
}
func GetNestedPreloads(ctx *graphql.OperationContext, fields []graphql.CollectedField, prefix string,
) []string {
preloads := []string{}
for _, column := range fields {
prefixColumn := GetPreloadString(prefix, column.Name)
preloads = append(preloads, prefixColumn)
preloads = append(preloads,
GetNestedPreloads(ctx, graphql.CollectFields(ctx, column.Selections, nil), prefixColumn)...,
)
}
return preloads
}
func GetPreloadString(prefix, name string) string {
if len(prefix) > 0 {
return prefix + "." + name
}
return name
}
func GetSignaturesInfo(isSigned bool, signatures mTypes.ManifestSignatures) []*gql_generated.SignatureSummary {
signaturesInfo := []*gql_generated.SignatureSummary{}
if !isSigned {
return signaturesInfo
}
for sigType, signatures := range signatures {
for _, sig := range signatures {
for _, layer := range sig.LayersInfo {
var (
isTrusted bool
author string
tool string
)
if layer.Signer != "" {
author = layer.Signer
if !layer.Date.IsZero() && time.Now().After(layer.Date) {
isTrusted = false
} else {
isTrusted = true
}
} else {
isTrusted = false
author = ""
}
tool = sigType
signaturesInfo = append(signaturesInfo,
&gql_generated.SignatureSummary{Tool: &tool, IsTrusted: &isTrusted, Author: &author})
}
}
}
return signaturesInfo
}
func PaginatedRepoMeta2RepoSummaries(ctx context.Context, repoMetaList []mTypes.RepoMeta,
imageMetaMap map[string]mTypes.ImageMeta, filter mTypes.Filter, pageInput pagination.PageInput,
cveInfo cveinfo.CveInfo, skip SkipQGLField,
) ([]*gql_generated.RepoSummary, zcommon.PageInfo, error) {
reposPageFinder, err := pagination.NewRepoSumPageFinder(pageInput.Limit, pageInput.Offset, pageInput.SortBy)
if err != nil {
return []*gql_generated.RepoSummary{}, zcommon.PageInfo{}, err
}
for _, repoMeta := range repoMetaList {
repoSummary := RepoMeta2RepoSummary(ctx, repoMeta, imageMetaMap)
if RepoSumAcceptedByFilter(repoSummary, filter) {
reposPageFinder.Add(repoSummary)
}
}
page, pageInfo := reposPageFinder.Page()
// CVE scanning is expensive, only scan for the current page
for _, repoSummary := range page {
updateRepoSummaryVulnerabilities(ctx, repoSummary, skip, cveInfo)
}
return page, pageInfo, nil
}
func RepoMeta2RepoSummary(ctx context.Context, repoMeta mTypes.RepoMeta,
imageMetaMap map[string]mTypes.ImageMeta,
) *gql_generated.RepoSummary {
var (
repoName = repoMeta.Name
lastUpdatedImage = deref(repoMeta.LastUpdatedImage, mTypes.LastUpdatedImage{})
lastUpdatedImageMeta = imageMetaMap[lastUpdatedImage.Digest]
lastUpdatedTag = lastUpdatedImage.Tag
repoPlatforms = repoMeta.Platforms
repoVendors = repoMeta.Vendors
repoDownloadCount = repoMeta.DownloadCount
repoStarCount = repoMeta.StarCount
repoIsUserStarred = repoMeta.IsStarred // value specific to the current user
repoIsUserBookMarked = repoMeta.IsBookmarked // value specific to the current user
repoSize = repoMeta.Size
)
// Compute LastUpdated as the latest of:
// 1. The LastUpdated timestamp of the last updated image
// 2. All TaggedTimestamp values of all tags in the repository
var maxTimestamp time.Time
// Start with the LastUpdated from the last updated image
if lastUpdatedImage.LastUpdated != nil {
maxTimestamp = *lastUpdatedImage.LastUpdated
}
// Check all TaggedTimestamp values from all tags
for _, descriptor := range repoMeta.Tags {
if !descriptor.TaggedTimestamp.IsZero() && descriptor.TaggedTimestamp.After(maxTimestamp) {
maxTimestamp = descriptor.TaggedTimestamp
}
}
repoLastUpdatedTimestamp := &maxTimestamp
imageSummary, _, err := FullImageMeta2ImageSummary(ctx, GetFullImageMeta(lastUpdatedTag, repoMeta,
lastUpdatedImageMeta))
_ = err
return &gql_generated.RepoSummary{
Name: &repoName,
LastUpdated: repoLastUpdatedTimestamp,
Size: ref(strconv.FormatInt(repoSize, 10)),
Platforms: getGqlPlatforms(repoPlatforms),
Vendors: getGqlVendors(repoVendors),
NewestImage: imageSummary,
DownloadCount: &repoDownloadCount,
StarCount: &repoStarCount,
IsBookmarked: &repoIsUserBookMarked,
IsStarred: &repoIsUserStarred,
Rank: ref(repoMeta.Rank),
}
}
func getGqlVendors(repoVendors []string) []*string {
result := make([]*string, 0, len(repoVendors))
for i := range repoVendors {
result = append(result, &repoVendors[i])
}
return result
}
func getGqlPlatforms(repoPlatforms []ispec.Platform) []*gql_generated.Platform {
result := make([]*gql_generated.Platform, 0, len(repoPlatforms))
for i := range repoPlatforms {
result = append(result, &gql_generated.Platform{
Os: ref(repoPlatforms[i].OS),
Arch: ref(getArch(repoPlatforms[i].Architecture, repoPlatforms[i].Variant)),
})
}
return result
}
type (
ManifestDigest = string
BlobDigest = string
)
func FullImageMeta2ImageSummary(ctx context.Context, fullImageMeta mTypes.FullImageMeta,
) (*gql_generated.ImageSummary, map[BlobDigest]int64, error) {
switch fullImageMeta.MediaType {
case ispec.MediaTypeImageManifest:
return ImageManifest2ImageSummary(ctx, fullImageMeta)
case ispec.MediaTypeImageIndex:
return ImageIndex2ImageSummary(ctx, fullImageMeta)
default:
return nil, nil, zerr.ErrMediaTypeNotSupported
}
}
func ImageIndex2ImageSummary(ctx context.Context, fullImageMeta mTypes.FullImageMeta,
) (*gql_generated.ImageSummary, map[BlobDigest]int64, error) {
var (
repo = fullImageMeta.Repo
tag = fullImageMeta.Tag
indexLastUpdated time.Time
isSigned = isImageSigned(fullImageMeta.Signatures)
indexSize = int64(0)
manifestAnnotations *ImageAnnotations
manifestSummaries = make([]*gql_generated.ManifestSummary, 0, len(fullImageMeta.Manifests))
indexBlobs = map[string]int64{}
indexDigestStr = fullImageMeta.Digest.String()
indexMediaType = ispec.MediaTypeImageIndex
lastPullTimestamp = fullImageMeta.Statistics.LastPullTimestamp
pushTimestamp = fullImageMeta.Statistics.PushTimestamp
pushedBy = fullImageMeta.Statistics.PushedBy
taggedTimestamp = fullImageMeta.TaggedTimestamp
)
// Fallback to PushTimestamp if TaggedTimestamp is not available
if taggedTimestamp.IsZero() {
taggedTimestamp = pushTimestamp
}
for _, imageManifest := range fullImageMeta.Manifests {
imageManifestSummary, manifestBlobs, err := ImageManifest2ImageSummary(ctx, mTypes.FullImageMeta{
Repo: fullImageMeta.Repo,
Tag: fullImageMeta.Tag,
MediaType: ispec.MediaTypeImageManifest,
Digest: imageManifest.Digest,
Size: imageManifest.Size,
Manifests: []mTypes.FullManifestMeta{imageManifest},
Referrers: imageManifest.Referrers,
Statistics: imageManifest.Statistics,
Signatures: imageManifest.Signatures,
TaggedTimestamp: fullImageMeta.TaggedTimestamp,
})
if err != nil {
return &gql_generated.ImageSummary{}, map[string]int64{}, err
}
manifestSize := int64(0)
for digest, size := range manifestBlobs {
indexBlobs[digest] = size
manifestSize += size
}
if indexLastUpdated.Before(*imageManifestSummary.LastUpdated) {
indexLastUpdated = *imageManifestSummary.LastUpdated
}
annotations := GetAnnotations(imageManifest.Manifest.Annotations, imageManifest.Config.Config.Labels)
if manifestAnnotations == nil {
manifestAnnotations = &annotations
}
indexSize += manifestSize
manifestSummaries = append(manifestSummaries, imageManifestSummary.Manifests[0])
}
signaturesInfo := GetSignaturesInfo(isSigned, fullImageMeta.Signatures)
if manifestAnnotations == nil {
manifestAnnotations = &ImageAnnotations{}
}
annotations := GetIndexAnnotations(fullImageMeta.Index.Annotations, manifestAnnotations)
imageLastUpdated := annotations.Created
if imageLastUpdated == nil {
imageLastUpdated = &indexLastUpdated
}
indexSummary := gql_generated.ImageSummary{
RepoName: &repo,
Tag: &tag,
Digest: &indexDigestStr,
MediaType: &indexMediaType,
Manifests: manifestSummaries,
LastUpdated: imageLastUpdated,
IsSigned: &isSigned,
SignatureInfo: signaturesInfo,
Size: ref(strconv.FormatInt(indexSize, 10)),
DownloadCount: ref(fullImageMeta.Statistics.DownloadCount),
LastPullTimestamp: &lastPullTimestamp,
PushTimestamp: &pushTimestamp,
PushedBy: &pushedBy,
TaggedTimestamp: &taggedTimestamp,
Description: &annotations.Description,
Title: &annotations.Title,
Documentation: &annotations.Documentation,
Licenses: &annotations.Licenses,
Labels: &annotations.Labels,
Source: &annotations.Source,
Vendor: &annotations.Vendor,
Authors: &annotations.Authors,
Referrers: getReferrers(fullImageMeta.Referrers),
}
return &indexSummary, indexBlobs, nil
}
func ImageManifest2ImageSummary(ctx context.Context, fullImageMeta mTypes.FullImageMeta,
) (*gql_generated.ImageSummary, map[BlobDigest]int64, error) {
manifest := fullImageMeta.Manifests[0]
var (
repoName = fullImageMeta.Repo
tag = fullImageMeta.Tag
configDigest = manifest.Manifest.Config.Digest.String()
configSize = manifest.Manifest.Config.Size
manifestDigest = manifest.Digest.String()
manifestSize = manifest.Size
mediaType = manifest.Manifest.MediaType
artifactType = zcommon.GetManifestArtifactType(fullImageMeta.Manifests[0].Manifest)
platform = getPlatform(manifest.Config.Platform)
downloadCount = fullImageMeta.Statistics.DownloadCount
isSigned = isImageSigned(fullImageMeta.Signatures)
lastPullTimestamp = fullImageMeta.Statistics.LastPullTimestamp
pushTimestamp = fullImageMeta.Statistics.PushTimestamp
pushedBy = fullImageMeta.Statistics.PushedBy
taggedTimestamp = fullImageMeta.TaggedTimestamp
)
// Fallback to PushTimestamp if TaggedTimestamp is not available
if taggedTimestamp.IsZero() {
taggedTimestamp = pushTimestamp
}
imageSize, imageBlobsMap := getImageBlobsInfo(manifestDigest, manifestSize, configDigest, configSize,
manifest.Manifest.Layers)
imageSizeStr := strconv.FormatInt(imageSize, 10)
annotations := GetAnnotations(manifest.Manifest.Annotations, manifest.Config.Config.Labels)
authors := annotations.Authors
if authors == "" {
authors = manifest.Config.Author
}
imageLastUpdated := annotations.Created
if imageLastUpdated == nil {
configCreated := zcommon.GetImageLastUpdated(manifest.Config)
imageLastUpdated = &configCreated
}
historyEntries, err := getAllHistory(manifest.Manifest, manifest.Config)
if err != nil {
graphql.AddError(ctx, gqlerror.Errorf("error generating history on tag %s in repo %s: "+
"manifest digest: %s, error: %s", tag, repoName, manifest.Digest, err.Error()))
}
signaturesInfo := GetSignaturesInfo(isSigned, fullImageMeta.Signatures)
manifestSummary := gql_generated.ManifestSummary{
Digest: &manifestDigest,
ConfigDigest: &configDigest,
LastUpdated: imageLastUpdated,
Size: &imageSizeStr,
IsSigned: &isSigned,
SignatureInfo: signaturesInfo,
Platform: &platform,
DownloadCount: &downloadCount,
Layers: getLayersSummaries(manifest.Manifest),
History: historyEntries,
Referrers: getReferrers(fullImageMeta.Referrers),
ArtifactType: &artifactType,
}
imageSummary := gql_generated.ImageSummary{
RepoName: &repoName,
Tag: &tag,
Digest: &manifestDigest,
MediaType: &mediaType,
Manifests: []*gql_generated.ManifestSummary{&manifestSummary},
LastUpdated: imageLastUpdated,
IsSigned: &isSigned,
SignatureInfo: signaturesInfo,
Size: &imageSizeStr,
DownloadCount: &downloadCount,
LastPullTimestamp: &lastPullTimestamp,
PushTimestamp: &pushTimestamp,
PushedBy: &pushedBy,
TaggedTimestamp: &taggedTimestamp,
Description: &annotations.Description,
Title: &annotations.Title,
Documentation: &annotations.Documentation,
Licenses: &annotations.Licenses,
Labels: &annotations.Labels,
Source: &annotations.Source,
Vendor: &annotations.Vendor,
Authors: &authors,
Referrers: manifestSummary.Referrers,
}
return &imageSummary, imageBlobsMap, nil
}
func isImageSigned(manifestSignatures mTypes.ManifestSignatures) bool {
for _, signatures := range manifestSignatures {
if len(signatures) > 0 {
return true
}
}
return false
}
func getPlatform(platform ispec.Platform) gql_generated.Platform {
return gql_generated.Platform{
Os: ref(platform.OS),
Arch: ref(getArch(platform.Architecture, platform.Variant)),
}
}
func getArch(arch string, variant string) string {
if variant != "" {
arch = arch + "/" + variant
}
return arch
}
func ref[T any](val T) *T {
ref := val
return &ref
}
func deref[T any](pointer *T, defaultVal T) T {
if pointer != nil {
return *pointer
}
return defaultVal
}
func PaginatedFullImageMeta2ImageSummaries(ctx context.Context, imageMetaList []mTypes.FullImageMeta, skip SkipQGLField,
cveInfo cveinfo.CveInfo, filter mTypes.Filter, pageInput pagination.PageInput,
) ([]*gql_generated.ImageSummary, zcommon.PageInfo, error) {
imagePageFinder, err := pagination.NewImgSumPageFinder(pageInput.Limit, pageInput.Offset, pageInput.SortBy)
if err != nil {
return []*gql_generated.ImageSummary{}, zcommon.PageInfo{}, err
}
for _, imageMeta := range imageMetaList {
imageSummary, _, err := FullImageMeta2ImageSummary(ctx, imageMeta)
if err != nil {
continue
}
if ImgSumAcceptedByFilter(imageSummary, filter) {
imagePageFinder.Add(imageSummary)
}
}
page, pageInfo := imagePageFinder.Page()
for _, imageSummary := range page {
// CVE scanning is expensive, only scan for this page
updateImageSummaryVulnerabilities(ctx, imageSummary, skip, cveInfo)
}
return page, pageInfo, nil
}