github/zot
1
0
Fork 0
mirror of https://github.com/project-zot/zot.git synced 2026-06-17 12:58:02 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(pagination): move pagination and sorting image summary results after conversion (#1637)

Browse Source 
fix(config): check for config media type when pushing to repodb

Signed-off-by: Laurentiu Niculae <niculae.laurentiu1@gmail.com>
This commit is contained in:
LaurentiuNiculae
2023-07-31 22:16:09 +03:00
committed by GitHub
parent 20391a21c0
commit 9e38ca51e3
39 changed files with 2361 additions and 3182 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/meta/boltdb/boltdb.go
+119 -410
View File
@@ -16,7 +16,6 @@ import (
zcommon "zotregistry.io/zot/pkg/common"
"zotregistry.io/zot/pkg/log"
"zotregistry.io/zot/pkg/meta/common"
"zotregistry.io/zot/pkg/meta/pagination"
"zotregistry.io/zot/pkg/meta/signatures"
mTypes "zotregistry.io/zot/pkg/meta/types"
"zotregistry.io/zot/pkg/meta/version"
@@ -297,17 +296,17 @@ func (bdw BoltDB) SetReferrer(repo string, referredDigest godigest.Digest, refer
return err
}
refferers := repoMeta.Referrers[referredDigest.String()]
referrers := repoMeta.Referrers[referredDigest.String()]
for i := range refferers {
if refferers[i].Digest == referrer.Digest {
for i := range referrers {
if referrers[i].Digest == referrer.Digest {
return nil
}
}
refferers = append(refferers, referrer)
referrers = append(referrers, referrer)
repoMeta.Referrers[referredDigest.String()] = refferers
repoMeta.Referrers[referredDigest.String()] = referrers
repoMetaBlob, err = json.Marshal(repoMeta)
if err != nil {
@@ -645,19 +644,10 @@ func (bdw *BoltDB) GetRepoStars(repo string) (int, error) {
}
func (bdw *BoltDB) GetMultipleRepoMeta(ctx context.Context, filter func(repoMeta mTypes.RepoMetadata) bool,
requestedPage mTypes.PageInput,
) ([]mTypes.RepoMetadata, error) {
var (
foundRepos = make([]mTypes.RepoMetadata, 0)
pageFinder pagination.PageFinder
)
foundRepos := []mTypes.RepoMetadata{}
pageFinder, err := pagination.NewBaseRepoPageFinder(requestedPage.Limit, requestedPage.Offset, requestedPage.SortBy)
if err != nil {
return nil, err
}
err = bdw.DB.View(func(tx *bbolt.Tx) error {
err := bdw.DB.View(func(tx *bbolt.Tx) error {
buck := tx.Bucket([]byte(RepoMetadataBucket))
cursor := buck.Cursor()
@@ -675,14 +665,10 @@ func (bdw *BoltDB) GetMultipleRepoMeta(ctx context.Context, filter func(repoMeta
}
if filter(repoMeta) {
pageFinder.Add(mTypes.DetailedRepoMeta{
RepoMetadata: repoMeta,
})
foundRepos = append(foundRepos, repoMeta)
}
}
foundRepos, _ = pageFinder.Page()
return nil
})
@@ -965,34 +951,21 @@ func (bdw *BoltDB) DeleteSignature(repo string, signedManifestDigest godigest.Di
return err
}
func (bdw *BoltDB) SearchRepos(ctx context.Context, searchText string, filter mTypes.Filter,
requestedPage mTypes.PageInput,
) ([]mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, map[string]mTypes.IndexData, zcommon.PageInfo,
error,
) {
func (bdw *BoltDB) SearchRepos(ctx context.Context, searchText string,
) ([]mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, map[string]mTypes.IndexData, error) {
var (
foundRepos = make([]mTypes.RepoMetadata, 0)
foundManifestMetadataMap = make(map[string]mTypes.ManifestMetadata)
foundindexDataMap = make(map[string]mTypes.IndexData)
pageFinder pagination.PageFinder
pageInfo zcommon.PageInfo
foundRepos = make([]mTypes.RepoMetadata, 0)
manifestMetadataMap = make(map[string]mTypes.ManifestMetadata)
indexDataMap = make(map[string]mTypes.IndexData)
)
pageFinder, err := pagination.NewBaseRepoPageFinder(requestedPage.Limit, requestedPage.Offset, requestedPage.SortBy)
if err != nil {
return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, map[string]mTypes.IndexData{},
zcommon.PageInfo{}, err
}
err = bdw.DB.View(func(transaction *bbolt.Tx) error {
err := bdw.DB.View(func(transaction *bbolt.Tx) error {
var (
manifestMetadataMap = make(map[string]mTypes.ManifestMetadata)
indexDataMap = make(map[string]mTypes.IndexData)
repoBuck = transaction.Bucket([]byte(RepoMetadataBucket))
indexBuck = transaction.Bucket([]byte(IndexDataBucket))
manifestBuck = transaction.Bucket([]byte(ManifestDataBucket))
userBookmarks = getUserBookmarks(ctx, transaction)
userStars = getUserStars(ctx, transaction)
repoBuck = transaction.Bucket([]byte(RepoMetadataBucket))
indexBuck = transaction.Bucket([]byte(IndexDataBucket))
manifestBuck = transaction.Bucket([]byte(ManifestDataBucket))
userBookmarks = getUserBookmarks(ctx, transaction)
userStars = getUserStars(ctx, transaction)
)
cursor := repoBuck.Cursor()
@@ -1009,22 +982,14 @@ func (bdw *BoltDB) SearchRepos(ctx context.Context, searchText string, filter mT
return err
}
repoMeta.IsBookmarked = zcommon.Contains(userBookmarks, repoMeta.Name)
repoMeta.IsStarred = zcommon.Contains(userStars, repoMeta.Name)
rank := common.RankRepoName(searchText, repoMeta.Name)
if rank == -1 {
continue
}
var (
repoDownloads = 0
repoLastUpdated = time.Time{}
osSet = map[string]bool{}
archSet = map[string]bool{}
noImageChecked = true
isSigned = false
)
repoMeta.IsBookmarked = zcommon.Contains(userBookmarks, repoMeta.Name)
repoMeta.IsStarred = zcommon.Contains(userStars, repoMeta.Name)
repoMeta.Rank = rank
for tag, descriptor := range repoMeta.Tags {
switch descriptor.MediaType {
@@ -1038,24 +1003,6 @@ func (bdw *BoltDB) SearchRepos(ctx context.Context, searchText string, filter mT
manifestDigest, err)
}
manifestFilterData, err := collectImageManifestFilterData(manifestDigest, repoMeta, manifestMeta)
if err != nil {
return fmt.Errorf("metadb: error collecting filter data for manifest with digest %s %w",
manifestDigest, err)
}
repoDownloads += manifestFilterData.DownloadCount
for _, os := range manifestFilterData.OsList {
osSet[os] = true
}
for _, arch := range manifestFilterData.ArchList {
archSet[arch] = true
}
repoLastUpdated, noImageChecked, isSigned = common.CheckImageLastUpdated(repoLastUpdated, isSigned,
noImageChecked, manifestFilterData)
manifestMetadataMap[descriptor.Digest] = manifestMeta
case ispec.MediaTypeImageIndex:
indexDigest := descriptor.Digest
@@ -1074,27 +1021,18 @@ func (bdw *BoltDB) SearchRepos(ctx context.Context, searchText string, filter mT
repoName, tag, err)
}
// this also updates manifestMetadataMap
indexFilterData, err := collectImageIndexFilterInfo(indexDigest, repoMeta, indexData, manifestMetadataMap,
manifestBuck)
if err != nil {
return fmt.Errorf("metadb: error collecting filter data for index with digest %s %w",
indexDigest, err)
for _, manifest := range indexContent.Manifests {
manifestDigest := manifest.Digest
manifestMeta, err := fetchManifestMetaWithCheck(repoMeta, manifestDigest.String(),
manifestMetadataMap, manifestBuck)
if err != nil {
return err
}
manifestMetadataMap[manifest.Digest.String()] = manifestMeta
}
for _, arch := range indexFilterData.ArchList {
archSet[arch] = true
}
for _, os := range indexFilterData.OsList {
osSet[os] = true
}
repoDownloads += indexFilterData.DownloadCount
repoLastUpdated, noImageChecked, isSigned = common.CheckImageLastUpdated(repoLastUpdated, isSigned,
noImageChecked, indexFilterData)
indexDataMap[indexDigest] = indexData
default:
bdw.Log.Error().Str("mediaType", descriptor.MediaType).Msg("Unsupported media type")
@@ -1103,37 +1041,13 @@ func (bdw *BoltDB) SearchRepos(ctx context.Context, searchText string, filter mT
}
}
repoFilterData := mTypes.FilterData{
OsList: common.GetMapKeys(osSet),
ArchList: common.GetMapKeys(archSet),
LastUpdated: repoLastUpdated,
DownloadCount: repoDownloads,
IsSigned: isSigned,
IsBookmarked: repoMeta.IsBookmarked,
IsStarred: repoMeta.IsStarred,
}
if !common.AcceptedByFilter(filter, repoFilterData) {
continue
}
pageFinder.Add(mTypes.DetailedRepoMeta{
RepoMetadata: repoMeta,
Rank: rank,
Downloads: repoDownloads,
UpdateTime: repoLastUpdated,
})
foundRepos = append(foundRepos, repoMeta)
}
foundRepos, pageInfo = pageFinder.Page()
foundManifestMetadataMap, foundindexDataMap, err = common.FilterDataByRepo(foundRepos, manifestMetadataMap,
indexDataMap)
return err
return nil
})
return foundRepos, foundManifestMetadataMap, foundindexDataMap, pageInfo, err
return foundRepos, manifestMetadataMap, indexDataMap, err
}
func fetchManifestMetaWithCheck(repoMeta mTypes.RepoMetadata, manifestDigest string,
@@ -1187,95 +1101,6 @@ func fetchIndexDataWithCheck(indexDigest string, indexDataMap map[string]mTypes.
return indexData, err
}
func collectImageManifestFilterData(digest string, repoMeta mTypes.RepoMetadata,
manifestMeta mTypes.ManifestMetadata,
) (mTypes.FilterData, error) {
// get fields related to filtering
var (
configContent ispec.Image
osList []string
archList []string
)
err := json.Unmarshal(manifestMeta.ConfigBlob, &configContent)
if err != nil {
return mTypes.FilterData{},
fmt.Errorf("metadb: error while unmarshaling config content %w", err)
}
if configContent.OS != "" {
osList = append(osList, configContent.OS)
}
if configContent.Architecture != "" {
archList = append(archList, configContent.Architecture)
}
return mTypes.FilterData{
DownloadCount: repoMeta.Statistics[digest].DownloadCount,
OsList: osList,
ArchList: archList,
LastUpdated: common.GetImageLastUpdatedTimestamp(configContent),
IsSigned: common.CheckIsSigned(repoMeta.Signatures[digest]),
}, nil
}
func collectImageIndexFilterInfo(indexDigest string, repoMeta mTypes.RepoMetadata,
indexData mTypes.IndexData, manifestMetadataMap map[string]mTypes.ManifestMetadata,
manifestBuck *bbolt.Bucket,
) (mTypes.FilterData, error) {
var indexContent ispec.Index
err := json.Unmarshal(indexData.IndexBlob, &indexContent)
if err != nil {
return mTypes.FilterData{},
fmt.Errorf("metadb: error while unmarshaling index content for digest %s %w", indexDigest, err)
}
var (
indexLastUpdated time.Time
firstManifestChecked = false
indexOsList = []string{}
indexArchList = []string{}
)
for _, manifest := range indexContent.Manifests {
manifestDigest := manifest.Digest
manifestMeta, err := fetchManifestMetaWithCheck(repoMeta, manifestDigest.String(),
manifestMetadataMap, manifestBuck)
if err != nil {
return mTypes.FilterData{},
fmt.Errorf("%w", err)
}
manifestFilterData, err := collectImageManifestFilterData(manifestDigest.String(), repoMeta,
manifestMeta)
if err != nil {
return mTypes.FilterData{},
fmt.Errorf("%w", err)
}
indexOsList = append(indexOsList, manifestFilterData.OsList...)
indexArchList = append(indexArchList, manifestFilterData.ArchList...)
if !firstManifestChecked || indexLastUpdated.Before(manifestFilterData.LastUpdated) {
indexLastUpdated = manifestFilterData.LastUpdated
firstManifestChecked = true
}
manifestMetadataMap[manifest.Digest.String()] = manifestMeta
}
return mTypes.FilterData{
DownloadCount: repoMeta.Statistics[indexDigest].DownloadCount,
LastUpdated: indexLastUpdated,
OsList: indexOsList,
ArchList: indexArchList,
IsSigned: common.CheckIsSigned(repoMeta.Signatures[indexDigest]),
}, nil
}
func NewManifestMetadata(manifestDigest string, repoMeta mTypes.RepoMetadata,
manifestData mTypes.ManifestData,
) mTypes.ManifestMetadata {
@@ -1294,28 +1119,16 @@ func NewManifestMetadata(manifestDigest string, repoMeta mTypes.RepoMetadata,
return manifestMeta
}
func (bdw *BoltDB) FilterTags(ctx context.Context, filterFunc mTypes.FilterFunc, filter mTypes.Filter,
requestedPage mTypes.PageInput,
) ([]mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, map[string]mTypes.IndexData,
zcommon.PageInfo, error,
func (bdw *BoltDB) FilterTags(ctx context.Context, filterFunc mTypes.FilterFunc,
) ([]mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, map[string]mTypes.IndexData, error,
) {
var (
foundRepos = make([]mTypes.RepoMetadata, 0)
manifestMetadataMap = make(map[string]mTypes.ManifestMetadata)
indexDataMap = make(map[string]mTypes.IndexData)
foundManifestMetadataMap = make(map[string]mTypes.ManifestMetadata)
foundindexDataMap = make(map[string]mTypes.IndexData)
pageFinder pagination.PageFinder
pageInfo zcommon.PageInfo
foundRepos = make([]mTypes.RepoMetadata, 0)
manifestMetadataMap = make(map[string]mTypes.ManifestMetadata)
indexDataMap = make(map[string]mTypes.IndexData)
)
pageFinder, err := pagination.NewBaseImagePageFinder(requestedPage.Limit, requestedPage.Offset, requestedPage.SortBy)
if err != nil {
return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, map[string]mTypes.IndexData{},
zcommon.PageInfo{}, err
}
err = bdw.DB.View(func(transaction *bbolt.Tx) error {
err := bdw.DB.View(func(transaction *bbolt.Tx) error {
var (
repoBuck = transaction.Bucket([]byte(RepoMetadataBucket))
indexBuck = transaction.Bucket([]byte(IndexDataBucket))
@@ -1354,16 +1167,6 @@ func (bdw *BoltDB) FilterTags(ctx context.Context, filterFunc mTypes.FilterFunc,
return fmt.Errorf("metadb: error while unmashaling manifest metadata for digest %s %w", manifestDigest, err)
}
imageFilterData, err := collectImageManifestFilterData(manifestDigest, repoMeta, manifestMeta)
if err != nil {
return fmt.Errorf("metadb: error collecting filter data for manifest with digest %s %w",
manifestDigest, err)
}
if !common.AcceptedByFilter(filter, imageFilterData) {
continue
}
if filterFunc(repoMeta, manifestMeta) {
matchedTags[tag] = descriptor
manifestMetadataMap[manifestDigest] = manifestMeta
@@ -1393,16 +1196,6 @@ func (bdw *BoltDB) FilterTags(ctx context.Context, filterFunc mTypes.FilterFunc,
return fmt.Errorf("metadb: error while getting manifest data for digest %s %w", manifestDigest, err)
}
manifestFilterData, err := collectImageManifestFilterData(manifestDigest, repoMeta, manifestMeta)
if err != nil {
return fmt.Errorf("metadb: error collecting filter data for manifest with digest %s %w",
manifestDigest, err)
}
if !common.AcceptedByFilter(filter, manifestFilterData) {
continue
}
if filterFunc(repoMeta, manifestMeta) {
matchedManifests = append(matchedManifests, manifest)
manifestMetadataMap[manifestDigest] = manifestMeta
@@ -1435,44 +1228,21 @@ func (bdw *BoltDB) FilterTags(ctx context.Context, filterFunc mTypes.FilterFunc,
repoMeta.Tags = matchedTags
pageFinder.Add(mTypes.DetailedRepoMeta{
RepoMetadata: repoMeta,
})
foundRepos = append(foundRepos, repoMeta)
}
foundRepos, pageInfo = pageFinder.Page()
foundManifestMetadataMap, foundindexDataMap, err = common.FilterDataByRepo(foundRepos, manifestMetadataMap,
indexDataMap)
return err
return nil
})
return foundRepos, foundManifestMetadataMap, foundindexDataMap, pageInfo, err
return foundRepos, manifestMetadataMap, indexDataMap, err
}
func (bdw *BoltDB) FilterRepos(ctx context.Context,
filter mTypes.FilterRepoFunc,
requestedPage mTypes.PageInput,
) (
[]mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, map[string]mTypes.IndexData, zcommon.PageInfo, error,
func (bdw *BoltDB) FilterRepos(ctx context.Context, filter mTypes.FilterRepoFunc) (
[]mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, map[string]mTypes.IndexData, error,
) {
var (
foundRepos = make([]mTypes.RepoMetadata, 0)
pageFinder pagination.PageFinder
pageInfo zcommon.PageInfo
)
foundRepos := make([]mTypes.RepoMetadata, 0)
pageFinder, err := pagination.NewBaseRepoPageFinder(
requestedPage.Limit,
requestedPage.Offset,
requestedPage.SortBy,
)
if err != nil {
return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, map[string]mTypes.IndexData{}, pageInfo, err
}
err = bdw.DB.View(func(tx *bbolt.Tx) error {
err := bdw.DB.View(func(tx *bbolt.Tx) error {
var (
buck = tx.Bucket([]byte(RepoMetadataBucket))
cursor = buck.Cursor()
@@ -1496,49 +1266,32 @@ func (bdw *BoltDB) FilterRepos(ctx context.Context,
repoMeta.IsStarred = zcommon.Contains(userStars, repoMeta.Name)
if filter(repoMeta) {
pageFinder.Add(mTypes.DetailedRepoMeta{
RepoMetadata: repoMeta,
})
foundRepos = append(foundRepos, repoMeta)
}
}
foundRepos, pageInfo = pageFinder.Page()
return nil
})
if err != nil {
return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, map[string]mTypes.IndexData{}, pageInfo, err
return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, map[string]mTypes.IndexData{}, err
}
foundManifestMetadataMap, foundIndexDataMap, err := common.FetchDataForRepos(bdw, foundRepos)
return foundRepos, foundManifestMetadataMap, foundIndexDataMap, pageInfo, err
return foundRepos, foundManifestMetadataMap, foundIndexDataMap, err
}
func (bdw *BoltDB) SearchTags(ctx context.Context, searchText string, filter mTypes.Filter,
requestedPage mTypes.PageInput,
) ([]mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, map[string]mTypes.IndexData, zcommon.PageInfo, error) {
func (bdw *BoltDB) SearchTags(ctx context.Context, searchText string,
) ([]mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, map[string]mTypes.IndexData, error) {
var (
foundRepos = make([]mTypes.RepoMetadata, 0)
manifestMetadataMap = make(map[string]mTypes.ManifestMetadata)
indexDataMap = make(map[string]mTypes.IndexData)
foundManifestMetadataMap = make(map[string]mTypes.ManifestMetadata)
foundindexDataMap = make(map[string]mTypes.IndexData)
pageInfo zcommon.PageInfo
pageFinder pagination.PageFinder
foundRepos = make([]mTypes.RepoMetadata, 0)
manifestMetadataMap = make(map[string]mTypes.ManifestMetadata)
indexDataMap = make(map[string]mTypes.IndexData)
)
pageFinder, err := pagination.NewBaseImagePageFinder(requestedPage.Limit, requestedPage.Offset, requestedPage.SortBy)
if err != nil {
return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, map[string]mTypes.IndexData{},
zcommon.PageInfo{}, err
}
searchedRepo, searchedTag, err := common.GetRepoTag(searchText)
if err != nil {
return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, map[string]mTypes.IndexData{},
zcommon.PageInfo{},
fmt.Errorf("metadb: error while parsing search text, invalid format %w", err)
}
@@ -1547,143 +1300,99 @@ func (bdw *BoltDB) SearchTags(ctx context.Context, searchText string, filter mTy
repoBuck = transaction.Bucket([]byte(RepoMetadataBucket))
indexBuck = transaction.Bucket([]byte(IndexDataBucket))
manifestBuck = transaction.Bucket([]byte(ManifestDataBucket))
cursor = repoBuck.Cursor()
userBookmarks = getUserBookmarks(ctx, transaction)
userStars = getUserStars(ctx, transaction)
)
repoName, repoMetaBlob := cursor.Seek([]byte(searchedRepo))
repoName, repoMetaBlob := repoBuck.Cursor().Seek([]byte(searchedRepo))
for ; repoName != nil; repoName, repoMetaBlob = cursor.Next() {
if ok, err := localCtx.RepoIsUserAvailable(ctx, string(repoName)); !ok || err != nil {
if string(repoName) != searchedRepo {
return nil
}
if ok, err := localCtx.RepoIsUserAvailable(ctx, string(repoName)); !ok || err != nil {
return err
}
repoMeta := mTypes.RepoMetadata{}
err := json.Unmarshal(repoMetaBlob, &repoMeta)
if err != nil {
return err
}
repoMeta.IsBookmarked = zcommon.Contains(userBookmarks, repoMeta.Name)
repoMeta.IsStarred = zcommon.Contains(userStars, repoMeta.Name)
matchedTags := make(map[string]mTypes.Descriptor)
for tag, descriptor := range repoMeta.Tags {
if !strings.HasPrefix(tag, searchedTag) {
continue
}
repoMeta := mTypes.RepoMetadata{}
matchedTags[tag] = descriptor
err := json.Unmarshal(repoMetaBlob, &repoMeta)
if err != nil {
return err
}
switch descriptor.MediaType {
case ispec.MediaTypeImageManifest:
manifestDigest := descriptor.Digest
repoMeta.IsBookmarked = zcommon.Contains(userBookmarks, repoMeta.Name)
repoMeta.IsStarred = zcommon.Contains(userStars, repoMeta.Name)
if string(repoName) != searchedRepo {
continue
}
matchedTags := make(map[string]mTypes.Descriptor)
for tag, descriptor := range repoMeta.Tags {
if !strings.HasPrefix(tag, searchedTag) {
continue
manifestMeta, err := fetchManifestMetaWithCheck(repoMeta, manifestDigest, manifestMetadataMap, manifestBuck)
if err != nil {
return fmt.Errorf("metadb: error fetching manifest meta for manifest with digest %s %w",
manifestDigest, err)
}
matchedTags[tag] = descriptor
manifestMetadataMap[descriptor.Digest] = manifestMeta
case ispec.MediaTypeImageIndex:
indexDigest := descriptor.Digest
switch descriptor.MediaType {
case ispec.MediaTypeImageManifest:
manifestDigest := descriptor.Digest
indexData, err := fetchIndexDataWithCheck(indexDigest, indexDataMap, indexBuck)
if err != nil {
return fmt.Errorf("metadb: error fetching index data for index with digest %s %w",
indexDigest, err)
}
var indexContent ispec.Index
err = json.Unmarshal(indexData.IndexBlob, &indexContent)
if err != nil {
return fmt.Errorf("metadb: error collecting filter data for index with digest %s %w",
indexDigest, err)
}
for _, manifest := range indexContent.Manifests {
manifestDigest := manifest.Digest.String()
manifestMeta, err := fetchManifestMetaWithCheck(repoMeta, manifestDigest, manifestMetadataMap, manifestBuck)
if err != nil {
return fmt.Errorf("metadb: error fetching manifest meta for manifest with digest %s %w",
return fmt.Errorf("metadb: error fetching from db manifest meta for manifest with digest %s %w",
manifestDigest, err)
}
imageFilterData, err := collectImageManifestFilterData(manifestDigest, repoMeta, manifestMeta)
if err != nil {
return fmt.Errorf("metadb: error collecting filter data for manifest with digest %s %w",
manifestDigest, err)
}
if !common.AcceptedByFilter(filter, imageFilterData) {
delete(matchedTags, tag)
continue
}
manifestMetadataMap[descriptor.Digest] = manifestMeta
case ispec.MediaTypeImageIndex:
indexDigest := descriptor.Digest
indexData, err := fetchIndexDataWithCheck(indexDigest, indexDataMap, indexBuck)
if err != nil {
return fmt.Errorf("metadb: error fetching index data for index with digest %s %w",
indexDigest, err)
}
var indexContent ispec.Index
err = json.Unmarshal(indexData.IndexBlob, &indexContent)
if err != nil {
return fmt.Errorf("metadb: error collecting filter data for index with digest %s %w",
indexDigest, err)
}
manifestHasBeenMatched := false
for _, manifest := range indexContent.Manifests {
manifestDigest := manifest.Digest.String()
manifestMeta, err := fetchManifestMetaWithCheck(repoMeta, manifestDigest, manifestMetadataMap, manifestBuck)
if err != nil {
return fmt.Errorf("metadb: error fetching from db manifest meta for manifest with digest %s %w",
manifestDigest, err)
}
manifestFilterData, err := collectImageManifestFilterData(manifestDigest, repoMeta, manifestMeta)
if err != nil {
return fmt.Errorf("metadb: error collecting filter data for manifest with digest %s %w",
manifestDigest, err)
}
manifestMetadataMap[manifestDigest] = manifestMeta
if common.AcceptedByFilter(filter, manifestFilterData) {
manifestHasBeenMatched = true
}
}
if !manifestHasBeenMatched {
delete(matchedTags, tag)
for _, manifest := range indexContent.Manifests {
delete(manifestMetadataMap, manifest.Digest.String())
}
continue
}
indexDataMap[indexDigest] = indexData
default:
bdw.Log.Error().Str("mediaType", descriptor.MediaType).Msg("Unsupported media type")
continue
manifestMetadataMap[manifestDigest] = manifestMeta
}
}
if len(matchedTags) == 0 {
indexDataMap[indexDigest] = indexData
default:
bdw.Log.Error().Str("mediaType", descriptor.MediaType).Msg("Unsupported media type")
continue
}
repoMeta.Tags = matchedTags
pageFinder.Add(mTypes.DetailedRepoMeta{
RepoMetadata: repoMeta,
})
}
foundRepos, pageInfo = pageFinder.Page()
if len(matchedTags) == 0 {
return nil
}
foundManifestMetadataMap, foundindexDataMap, err = common.FilterDataByRepo(foundRepos, manifestMetadataMap,
indexDataMap)
repoMeta.Tags = matchedTags
foundRepos = append(foundRepos, repoMeta)
return nil
})
return foundRepos, foundManifestMetadataMap, foundindexDataMap, pageInfo, err
return foundRepos, manifestMetadataMap, indexDataMap, err
}
func (bdw *BoltDB) ToggleStarRepo(ctx context.Context, repo string) (mTypes.ToggleState, error) {
pkg/meta/boltdb/boltdb_test.go
+24 -180
View File
@@ -314,8 +314,8 @@ func TestWrapperErrors(t *testing.T) {
})
So(err, ShouldBeNil)
_, _, _, _, err = boltdbWrapper.FilterRepos(context.Background(),
func(repoMeta mTypes.RepoMetadata) bool { return true }, mTypes.PageInput{})
_, _, _, err = boltdbWrapper.FilterRepos(context.Background(),
func(repoMeta mTypes.RepoMetadata) bool { return true })
So(err, ShouldNotBeNil)
})
@@ -453,7 +453,7 @@ func TestWrapperErrors(t *testing.T) {
_, err = boltdbWrapper.GetMultipleRepoMeta(context.TODO(), func(repoMeta mTypes.RepoMetadata) bool {
return true
}, mTypes.PageInput{})
})
So(err, ShouldNotBeNil)
})
@@ -619,7 +619,7 @@ func TestWrapperErrors(t *testing.T) {
})
So(err, ShouldBeNil)
_, _, _, _, err = boltdbWrapper.SearchRepos(context.Background(), "", mTypes.Filter{}, mTypes.PageInput{})
_, _, _, err = boltdbWrapper.SearchRepos(context.Background(), "")
So(err, ShouldNotBeNil)
err = boltdbWrapper.DB.Update(func(tx *bbolt.Tx) error {
@@ -660,47 +660,10 @@ func TestWrapperErrors(t *testing.T) {
})
So(err, ShouldBeNil)
_, _, _, _, err = boltdbWrapper.SearchRepos(context.Background(), "repo1", mTypes.Filter{}, mTypes.PageInput{})
_, _, _, err = boltdbWrapper.SearchRepos(context.Background(), "repo1")
So(err, ShouldNotBeNil)
_, _, _, _, err = boltdbWrapper.SearchRepos(context.Background(), "repo2", mTypes.Filter{}, mTypes.PageInput{})
So(err, ShouldNotBeNil)
err = boltdbWrapper.DB.Update(func(tx *bbolt.Tx) error {
repoBuck := tx.Bucket([]byte(boltdb.RepoMetadataBucket))
dataBuck := tx.Bucket([]byte(boltdb.ManifestDataBucket))
manifestMeta := mTypes.ManifestMetadata{
ManifestBlob: []byte("{}"),
ConfigBlob: []byte("wrong json"),
Signatures: mTypes.ManifestSignatures{},
}
manifestMetaBlob, err := json.Marshal(manifestMeta)
if err != nil {
return err
}
err = dataBuck.Put([]byte("dig1"), manifestMetaBlob)
if err != nil {
return err
}
repoMeta = mTypes.RepoMetadata{
Name: "repo1",
Tags: map[string]mTypes.Descriptor{
"tag1": {Digest: "dig1", MediaType: ispec.MediaTypeImageManifest},
},
Signatures: map[string]mTypes.ManifestSignatures{},
}
repoMetaBlob, err = json.Marshal(repoMeta)
So(err, ShouldBeNil)
return repoBuck.Put([]byte("repo1"), repoMetaBlob)
})
So(err, ShouldBeNil)
_, _, _, _, err = boltdbWrapper.SearchRepos(context.Background(), "repo1", mTypes.Filter{}, mTypes.PageInput{})
_, _, _, err = boltdbWrapper.SearchRepos(context.Background(), "repo2")
So(err, ShouldNotBeNil)
})
@@ -714,10 +677,10 @@ func TestWrapperErrors(t *testing.T) {
err = setBadIndexData(boltdbWrapper.DB, indexDigest.String())
So(err, ShouldBeNil)
_, _, _, _, err = boltdbWrapper.SearchRepos(ctx, "", mTypes.Filter{}, mTypes.PageInput{})
_, _, _, err = boltdbWrapper.SearchRepos(ctx, "")
So(err, ShouldNotBeNil)
_, _, _, _, err = boltdbWrapper.SearchTags(ctx, "repo:", mTypes.Filter{}, mTypes.PageInput{})
_, _, _, err = boltdbWrapper.SearchTags(ctx, "repo:")
So(err, ShouldNotBeNil)
})
@@ -732,49 +695,10 @@ func TestWrapperErrors(t *testing.T) {
})
So(err, ShouldBeNil)
_, _, _, _, err = boltdbWrapper.SearchRepos(ctx, "", mTypes.Filter{}, mTypes.PageInput{})
_, _, _, err = boltdbWrapper.SearchRepos(ctx, "")
So(err, ShouldNotBeNil)
_, _, _, _, err = boltdbWrapper.SearchTags(ctx, "repo:", mTypes.Filter{}, mTypes.PageInput{})
So(err, ShouldNotBeNil)
})
Convey("Good index data, bad manifest inside index", func() {
var (
indexDigest = digest.FromString("indexDigest")
manifestDigestFromIndex1 = digest.FromString("manifestDigestFromIndex1")
manifestDigestFromIndex2 = digest.FromString("manifestDigestFromIndex2")
)
err := boltdbWrapper.SetRepoReference("repo", "tag1", indexDigest, ispec.MediaTypeImageIndex) //nolint:contextcheck
So(err, ShouldBeNil)
indexBlob, err := test.GetIndexBlobWithManifests([]digest.Digest{
manifestDigestFromIndex1, manifestDigestFromIndex2,
})
So(err, ShouldBeNil)
err = boltdbWrapper.SetIndexData(indexDigest, mTypes.IndexData{
IndexBlob: indexBlob,
})
So(err, ShouldBeNil)
err = boltdbWrapper.SetManifestData(manifestDigestFromIndex1, mTypes.ManifestData{
ManifestBlob: []byte("Bad Manifest"),
ConfigBlob: []byte("Bad Manifest"),
})
So(err, ShouldBeNil)
err = boltdbWrapper.SetManifestData(manifestDigestFromIndex2, mTypes.ManifestData{
ManifestBlob: []byte("Bad Manifest"),
ConfigBlob: []byte("Bad Manifest"),
})
So(err, ShouldBeNil)
_, _, _, _, err = boltdbWrapper.SearchRepos(ctx, "", mTypes.Filter{}, mTypes.PageInput{})
So(err, ShouldNotBeNil)
_, _, _, _, err = boltdbWrapper.SearchTags(ctx, "repo:", mTypes.Filter{}, mTypes.PageInput{})
_, _, _, err = boltdbWrapper.SearchTags(ctx, "repo:")
So(err, ShouldNotBeNil)
})
})
@@ -789,10 +713,10 @@ func TestWrapperErrors(t *testing.T) {
})
So(err, ShouldBeNil)
_, _, _, _, err = boltdbWrapper.SearchTags(ctx, "", mTypes.Filter{}, mTypes.PageInput{})
_, _, _, err = boltdbWrapper.SearchTags(ctx, "")
So(err, ShouldNotBeNil)
_, _, _, _, err = boltdbWrapper.SearchTags(ctx, "repo1:", mTypes.Filter{}, mTypes.PageInput{})
_, _, _, err = boltdbWrapper.SearchTags(ctx, "repo1:")
So(err, ShouldNotBeNil)
err = boltdbWrapper.DB.Update(func(tx *bbolt.Tx) error {
@@ -866,13 +790,10 @@ func TestWrapperErrors(t *testing.T) {
})
So(err, ShouldBeNil)
_, _, _, _, err = boltdbWrapper.SearchTags(ctx, "repo1:", mTypes.Filter{}, mTypes.PageInput{})
_, _, _, err = boltdbWrapper.SearchTags(ctx, "repo1:")
So(err, ShouldNotBeNil)
_, _, _, _, err = boltdbWrapper.SearchTags(ctx, "repo2:", mTypes.Filter{}, mTypes.PageInput{})
So(err, ShouldNotBeNil)
_, _, _, _, err = boltdbWrapper.SearchTags(ctx, "repo3:", mTypes.Filter{}, mTypes.PageInput{})
_, _, _, err = boltdbWrapper.SearchTags(ctx, "repo2:")
So(err, ShouldNotBeNil)
})
@@ -886,11 +807,8 @@ func TestWrapperErrors(t *testing.T) {
err = setBadIndexData(boltdbWrapper.DB, indexDigest.String())
So(err, ShouldBeNil)
_, _, _, _, err = boltdbWrapper.FilterTags(ctx,
func(repoMeta mTypes.RepoMetadata, manifestMeta mTypes.ManifestMetadata) bool { return true },
mTypes.Filter{},
mTypes.PageInput{},
)
_, _, _, err = boltdbWrapper.FilterTags(ctx,
func(repoMeta mTypes.RepoMetadata, manifestMeta mTypes.ManifestMetadata) bool { return true })
So(err, ShouldNotBeNil)
})
@@ -905,11 +823,8 @@ func TestWrapperErrors(t *testing.T) {
})
So(err, ShouldBeNil)
_, _, _, _, err = boltdbWrapper.FilterTags(ctx,
func(repoMeta mTypes.RepoMetadata, manifestMeta mTypes.ManifestMetadata) bool { return true },
mTypes.Filter{},
mTypes.PageInput{},
)
_, _, _, err = boltdbWrapper.FilterTags(ctx,
func(repoMeta mTypes.RepoMetadata, manifestMeta mTypes.ManifestMetadata) bool { return true })
So(err, ShouldNotBeNil)
})
@@ -945,77 +860,10 @@ func TestWrapperErrors(t *testing.T) {
})
So(err, ShouldBeNil)
_, _, _, _, err = boltdbWrapper.FilterTags(ctx,
func(repoMeta mTypes.RepoMetadata, manifestMeta mTypes.ManifestMetadata) bool { return false },
mTypes.Filter{},
mTypes.PageInput{},
)
_, _, _, err = boltdbWrapper.FilterTags(ctx,
func(repoMeta mTypes.RepoMetadata, manifestMeta mTypes.ManifestMetadata) bool { return false })
So(err, ShouldBeNil)
})
Convey("FilterTags bad config blob in image with a single manifest", func() {
manifestDigest := digest.FromString("manifestDigestBadConfig")
err := boltdbWrapper.SetRepoReference("repo", "tag1", //nolint:contextcheck
manifestDigest, ispec.MediaTypeImageManifest,
)
So(err, ShouldBeNil)
err = boltdbWrapper.SetManifestData(manifestDigest, mTypes.ManifestData{
ManifestBlob: []byte("{}"),
ConfigBlob: []byte("bad blob"),
})
So(err, ShouldBeNil)
_, _, _, _, err = boltdbWrapper.FilterTags(ctx,
func(repoMeta mTypes.RepoMetadata, manifestMeta mTypes.ManifestMetadata) bool { return true },
mTypes.Filter{},
mTypes.PageInput{},
)
So(err, ShouldNotBeNil)
})
Convey("FilterTags bad config blob in index", func() {
var (
indexDigest = digest.FromString("indexDigest")
manifestDigestFromIndex1 = digest.FromString("manifestDigestFromIndexGoodConfig")
manifestDigestFromIndex2 = digest.FromString("manifestDigestFromIndexBadConfig")
)
err := boltdbWrapper.SetRepoReference("repo", "tag1", //nolint:contextcheck
indexDigest, ispec.MediaTypeImageIndex,
)
So(err, ShouldBeNil)
indexBlob, err := test.GetIndexBlobWithManifests([]digest.Digest{
manifestDigestFromIndex1, manifestDigestFromIndex2,
})
So(err, ShouldBeNil)
err = boltdbWrapper.SetIndexData(indexDigest, mTypes.IndexData{
IndexBlob: indexBlob,
})
So(err, ShouldBeNil)
err = boltdbWrapper.SetManifestData(manifestDigestFromIndex1, mTypes.ManifestData{
ManifestBlob: []byte("{}"),
ConfigBlob: []byte("{}"),
})
So(err, ShouldBeNil)
err = boltdbWrapper.SetManifestData(manifestDigestFromIndex2, mTypes.ManifestData{
ManifestBlob: []byte("{}"),
ConfigBlob: []byte("bad blob"),
})
So(err, ShouldBeNil)
_, _, _, _, err = boltdbWrapper.FilterTags(ctx,
func(repoMeta mTypes.RepoMetadata, manifestMeta mTypes.ManifestMetadata) bool { return true },
mTypes.Filter{},
mTypes.PageInput{},
)
So(err, ShouldNotBeNil)
})
})
Convey("ToggleStarRepo bad context errors", func() {
@@ -1161,18 +1009,14 @@ func TestWrapperErrors(t *testing.T) {
err := boltdbWrapper.SetRepoReference("repo", "tag1", digest, "invalid type") //nolint:contextcheck
So(err, ShouldBeNil)
_, _, _, _, err = boltdbWrapper.SearchRepos(ctx, "", mTypes.Filter{}, mTypes.PageInput{})
_, _, _, err = boltdbWrapper.SearchRepos(ctx, "")
So(err, ShouldBeNil)
_, _, _, _, err = boltdbWrapper.SearchTags(ctx, "repo:", mTypes.Filter{}, mTypes.PageInput{})
_, _, _, err = boltdbWrapper.SearchTags(ctx, "repo:")
So(err, ShouldBeNil)
_, _, _, _, err = boltdbWrapper.FilterTags(
ctx,
func(repoMeta mTypes.RepoMetadata, manifestMeta mTypes.ManifestMetadata) bool { return true },
mTypes.Filter{},
mTypes.PageInput{},
)
_, _, _, err = boltdbWrapper.FilterTags(ctx,
func(repoMeta mTypes.RepoMetadata, manifestMeta mTypes.ManifestMetadata) bool { return true })
So(err, ShouldBeNil)
})
pkg/meta/common/common.go
+11 -88
View File
@@ -142,34 +142,6 @@ func RankRepoName(searchText string, repoName string) int {
return -1
}
func GetImageLastUpdatedTimestamp(configContent ispec.Image) time.Time {
var timeStamp *time.Time
if configContent.Created != nil && !configContent.Created.IsZero() {
return *configContent.Created
}
if len(configContent.History) != 0 {
timeStamp = configContent.History[len(configContent.History)-1].Created
}
if timeStamp == nil {
timeStamp = &time.Time{}
}
return *timeStamp
}
func CheckIsSigned(signatures mTypes.ManifestSignatures) bool {
for _, signatures := range signatures {
if len(signatures) > 0 {
return true
}
}
return false
}
func GetRepoTag(searchText string) (string, string, error) {
const repoTagCount = 2
@@ -185,66 +157,6 @@ func GetRepoTag(searchText string) (string, string, error) {
return repo, tag, nil
}
func GetMapKeys[K comparable, V any](genericMap map[K]V) []K {
keys := make([]K, 0, len(genericMap))
for k := range genericMap {
keys = append(keys, k)
}
return keys
}
// acceptedByFilter checks that data contains at least 1 element of each filter
// criteria(os, arch) present in filter.
func AcceptedByFilter(filter mTypes.Filter, data mTypes.FilterData) bool {
if filter.Arch != nil {
foundArch := false
for _, arch := range filter.Arch {
foundArch = foundArch || containsString(data.ArchList, *arch)
}
if !foundArch {
return false
}
}
if filter.Os != nil {
foundOs := false
for _, os := range filter.Os {
foundOs = foundOs || containsString(data.OsList, *os)
}
if !foundOs {
return false
}
}
if filter.HasToBeSigned != nil && *filter.HasToBeSigned != data.IsSigned {
return false
}
if filter.IsBookmarked != nil && *filter.IsBookmarked != data.IsBookmarked {
return false
}
if filter.IsStarred != nil && *filter.IsStarred != data.IsStarred {
return false
}
return true
}
func containsString(strSlice []string, str string) bool {
for _, val := range strSlice {
if strings.EqualFold(val, str) {
return true
}
}
return false
}
func GetReferredSubject(descriptorBlob []byte) (godigest.Digest, bool) {
var manifest ispec.Manifest
@@ -421,3 +333,14 @@ func GetImageDescriptor(metaDB mTypes.MetaDB, repo, tag string) (mTypes.Descript
return imageDescriptor, nil
}
func InitializeImageConfig(blob []byte) ispec.Image {
var configContent ispec.Image
err := json.Unmarshal(blob, &configContent)
if err != nil {
return ispec.Image{}
}
return configContent
}
pkg/meta/common/common_test.go
+33
View File
@@ -118,6 +118,39 @@ func TestUtils(t *testing.T) {
})
Convey("FilterDataByRepo", t, func() {
Convey("Functionality", func() {
_, _, err := common.FilterDataByRepo(
[]mTypes.RepoMetadata{{
Tags: map[string]mTypes.Descriptor{
"manifest": {
Digest: "manifestDigest",
MediaType: ispec.MediaTypeImageManifest,
},
"index": {
Digest: "indexDigest",
MediaType: ispec.MediaTypeImageIndex,
},
"rand": {
Digest: "randDigest",
MediaType: "rand",
},
},
}},
map[string]mTypes.ManifestMetadata{},
map[string]mTypes.IndexData{
"indexDigest": {
IndexBlob: []byte(`{
"manifests": [
{
"digest": "manifestDigest"
}
]
}`),
},
},
)
So(err, ShouldBeNil)
})
Convey("Errors", func() {
// Unmarshal index data error
_, _, err := common.FilterDataByRepo(
pkg/meta/dynamodb/dynamo.go → pkg/meta/dynamodb/dynamodb.go
+118 -401
View File
@@ -19,7 +19,6 @@ import (
zcommon "zotregistry.io/zot/pkg/common"
"zotregistry.io/zot/pkg/log"
"zotregistry.io/zot/pkg/meta/common"
"zotregistry.io/zot/pkg/meta/pagination"
"zotregistry.io/zot/pkg/meta/signatures"
mTypes "zotregistry.io/zot/pkg/meta/types"
"zotregistry.io/zot/pkg/meta/version"
@@ -328,17 +327,17 @@ func (dwr DynamoDB) SetReferrer(repo string, referredDigest godigest.Digest, ref
}
}
refferers := repoMeta.Referrers[referredDigest.String()]
referrers := repoMeta.Referrers[referredDigest.String()]
for i := range refferers {
if refferers[i].Digest == referrer.Digest {
for i := range referrers {
if referrers[i].Digest == referrer.Digest {
return nil
}
}
refferers = append(refferers, referrer)
referrers = append(referrers, referrer)
repoMeta.Referrers[referredDigest.String()] = refferers
repoMeta.Referrers[referredDigest.String()] = referrers
return dwr.SetRepoMeta(repo, repoMeta)
}
@@ -787,27 +786,21 @@ func (dwr *DynamoDB) DeleteSignature(repo string, signedManifestDigest godigest.
}
func (dwr *DynamoDB) GetMultipleRepoMeta(ctx context.Context,
filter func(repoMeta mTypes.RepoMetadata) bool, requestedPage mTypes.PageInput,
filter func(repoMeta mTypes.RepoMetadata) bool,
) ([]mTypes.RepoMetadata, error) {
var (
foundRepos = []mTypes.RepoMetadata{}
repoMetaAttributeIterator AttributesIterator
pageFinder pagination.PageFinder
)
repoMetaAttributeIterator = NewBaseDynamoAttributesIterator(
dwr.Client, dwr.RepoMetaTablename, "RepoMetadata", 0, dwr.Log,
)
pageFinder, err := pagination.NewBaseRepoPageFinder(requestedPage.Limit, requestedPage.Offset, requestedPage.SortBy)
if err != nil {
return nil, err
}
repoMetaAttribute, err := repoMetaAttributeIterator.First(ctx)
for ; repoMetaAttribute != nil; repoMetaAttribute, err = repoMetaAttributeIterator.Next(ctx) {
if err != nil {
// log
return []mTypes.RepoMetadata{}, err
}
@@ -823,26 +816,20 @@ func (dwr *DynamoDB) GetMultipleRepoMeta(ctx context.Context,
}
if filter(repoMeta) {
pageFinder.Add(mTypes.DetailedRepoMeta{
RepoMetadata: repoMeta,
})
foundRepos = append(foundRepos, repoMeta)
}
}
foundRepos, _ := pageFinder.Page()
return foundRepos, err
}
func (dwr *DynamoDB) SearchRepos(ctx context.Context, searchText string, filter mTypes.Filter,
requestedPage mTypes.PageInput,
) ([]mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, map[string]mTypes.IndexData, zcommon.PageInfo, error) {
func (dwr *DynamoDB) SearchRepos(ctx context.Context, searchText string,
) ([]mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, map[string]mTypes.IndexData, error) {
var (
repos = []mTypes.RepoMetadata{}
manifestMetadataMap = make(map[string]mTypes.ManifestMetadata)
indexDataMap = make(map[string]mTypes.IndexData)
repoMetaAttributeIterator AttributesIterator
pageFinder pagination.PageFinder
pageInfo zcommon.PageInfo
userBookmarks = getUserBookmarks(ctx, dwr)
userStars = getUserStars(ctx, dwr)
@@ -852,18 +839,12 @@ func (dwr *DynamoDB) SearchRepos(ctx context.Context, searchText string, filter
dwr.Client, dwr.RepoMetaTablename, "RepoMetadata", 0, dwr.Log,
)
pageFinder, err := pagination.NewBaseRepoPageFinder(requestedPage.Limit, requestedPage.Offset, requestedPage.SortBy)
if err != nil {
return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, map[string]mTypes.IndexData{},
pageInfo, err
}
repoMetaAttribute, err := repoMetaAttributeIterator.First(ctx)
for ; repoMetaAttribute != nil; repoMetaAttribute, err = repoMetaAttributeIterator.Next(ctx) {
if err != nil {
return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, map[string]mTypes.IndexData{},
pageInfo, err
err
}
var repoMeta mTypes.RepoMetadata
@@ -871,7 +852,7 @@ func (dwr *DynamoDB) SearchRepos(ctx context.Context, searchText string, filter
err := attributevalue.Unmarshal(repoMetaAttribute, &repoMeta)
if err != nil {
return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, map[string]mTypes.IndexData{},
pageInfo, err
err
}
if ok, err := localCtx.RepoIsUserAvailable(ctx, repoMeta.Name); !ok || err != nil {
@@ -885,15 +866,7 @@ func (dwr *DynamoDB) SearchRepos(ctx context.Context, searchText string, filter
repoMeta.IsBookmarked = zcommon.Contains(userBookmarks, repoMeta.Name)
repoMeta.IsStarred = zcommon.Contains(userStars, repoMeta.Name)
var (
repoDownloads = 0
repoLastUpdated = time.Time{}
osSet = map[string]bool{}
archSet = map[string]bool{}
noImageChecked = true
isSigned = false
)
repoMeta.Rank = rank
for _, descriptor := range repoMeta.Tags {
switch descriptor.MediaType {
@@ -904,64 +877,37 @@ func (dwr *DynamoDB) SearchRepos(ctx context.Context, searchText string, filter
manifestMetadataMap)
if err != nil {
return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, map[string]mTypes.IndexData{},
pageInfo,
fmt.Errorf("%w", err)
err
}
manifestFilterData, err := collectImageManifestFilterData(manifestDigest, repoMeta, manifestMeta)
if err != nil {
return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, map[string]mTypes.IndexData{},
pageInfo,
fmt.Errorf("%w", err)
}
repoDownloads += manifestFilterData.DownloadCount
for _, os := range manifestFilterData.OsList {
osSet[os] = true
}
for _, arch := range manifestFilterData.ArchList {
archSet[arch] = true
}
repoLastUpdated, noImageChecked, isSigned = common.CheckImageLastUpdated(repoLastUpdated, isSigned,
noImageChecked, manifestFilterData)
manifestMetadataMap[descriptor.Digest] = manifestMeta
case ispec.MediaTypeImageIndex:
indexDigest := descriptor.Digest
indexData, err := dwr.fetchIndexDataWithCheck(indexDigest, indexDataMap) //nolint:contextcheck
indexData, err := dwr.fetchIndexDataWithCheck(descriptor.Digest, indexDataMap) //nolint:contextcheck
if err != nil {
return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, map[string]mTypes.IndexData{},
pageInfo,
fmt.Errorf("%w", err)
err
}
// this also updates manifestMetadataMap
indexFilterData, err := dwr.collectImageIndexFilterInfo(indexDigest, repoMeta, indexData, //nolint:contextcheck
manifestMetadataMap)
var indexContent ispec.Index
err = json.Unmarshal(indexData.IndexBlob, &indexContent)
if err != nil {
return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, map[string]mTypes.IndexData{},
pageInfo,
fmt.Errorf("%w", err)
fmt.Errorf("metadb: error while unmarshaling index content for digest %s %w", descriptor.Digest, err)
}
for _, arch := range indexFilterData.ArchList {
archSet[arch] = true
for _, manifest := range indexContent.Manifests {
manifestMeta, err := dwr.fetchManifestMetaWithCheck(repoMeta.Name, manifest.Digest.String(), //nolint: contextcheck
manifestMetadataMap)
if err != nil {
return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, map[string]mTypes.IndexData{},
err
}
manifestMetadataMap[manifest.Digest.String()] = manifestMeta
}
for _, os := range indexFilterData.OsList {
osSet[os] = true
}
repoDownloads += indexFilterData.DownloadCount
repoLastUpdated, noImageChecked, isSigned = common.CheckImageLastUpdated(repoLastUpdated, isSigned,
noImageChecked, indexFilterData)
indexDataMap[indexDigest] = indexData
indexDataMap[descriptor.Digest] = indexData
default:
dwr.Log.Error().Str("mediaType", descriptor.MediaType).Msg("Unsupported media type")
@@ -969,32 +915,10 @@ func (dwr *DynamoDB) SearchRepos(ctx context.Context, searchText string, filter
}
}
repoFilterData := mTypes.FilterData{
OsList: common.GetMapKeys(osSet),
ArchList: common.GetMapKeys(archSet),
LastUpdated: repoLastUpdated,
DownloadCount: repoDownloads,
IsSigned: isSigned,
}
if !common.AcceptedByFilter(filter, repoFilterData) {
continue
}
pageFinder.Add(mTypes.DetailedRepoMeta{
RepoMetadata: repoMeta,
Rank: rank,
Downloads: repoDownloads,
UpdateTime: repoLastUpdated,
})
repos = append(repos, repoMeta)
}
foundRepos, pageInfo := pageFinder.Page()
foundManifestMetadataMap, foundindexDataMap, err := common.FilterDataByRepo(foundRepos, manifestMetadataMap,
indexDataMap)
return foundRepos, foundManifestMetadataMap, foundindexDataMap, pageInfo, err
return repos, manifestMetadataMap, indexDataMap, nil
}
func getUserStars(ctx context.Context, dwr *DynamoDB) []string {
@@ -1035,38 +959,6 @@ func (dwr *DynamoDB) fetchManifestMetaWithCheck(repoName string, manifestDigest
return manifestMeta, nil
}
func collectImageManifestFilterData(digest string, repoMeta mTypes.RepoMetadata,
manifestMeta mTypes.ManifestMetadata,
) (mTypes.FilterData, error) {
// get fields related to filtering
var (
configContent ispec.Image
osList []string
archList []string
)
err := json.Unmarshal(manifestMeta.ConfigBlob, &configContent)
if err != nil {
return mTypes.FilterData{}, fmt.Errorf("metadb: error while unmarshaling config content %w", err)
}
if configContent.OS != "" {
osList = append(osList, configContent.OS)
}
if configContent.Architecture != "" {
archList = append(archList, configContent.Architecture)
}
return mTypes.FilterData{
DownloadCount: repoMeta.Statistics[digest].DownloadCount,
OsList: osList,
ArchList: archList,
LastUpdated: common.GetImageLastUpdatedTimestamp(configContent),
IsSigned: common.CheckIsSigned(repoMeta.Signatures[digest]),
}, nil
}
func (dwr *DynamoDB) fetchIndexDataWithCheck(indexDigest string, indexDataMap map[string]mTypes.IndexData,
) (mTypes.IndexData, error) {
var (
@@ -1087,72 +979,14 @@ func (dwr *DynamoDB) fetchIndexDataWithCheck(indexDigest string, indexDataMap ma
return indexData, err
}
func (dwr *DynamoDB) collectImageIndexFilterInfo(indexDigest string, repoMeta mTypes.RepoMetadata,
indexData mTypes.IndexData, manifestMetadataMap map[string]mTypes.ManifestMetadata,
) (mTypes.FilterData, error) {
var indexContent ispec.Index
err := json.Unmarshal(indexData.IndexBlob, &indexContent)
if err != nil {
return mTypes.FilterData{},
fmt.Errorf("metadb: error while unmarshaling index content for digest %s %w", indexDigest, err)
}
var (
indexLastUpdated time.Time
firstManifestChecked = false
indexOsList = []string{}
indexArchList = []string{}
)
for _, manifest := range indexContent.Manifests {
manifestDigest := manifest.Digest
manifestMeta, err := dwr.fetchManifestMetaWithCheck(repoMeta.Name, manifestDigest.String(),
manifestMetadataMap)
if err != nil {
return mTypes.FilterData{},
fmt.Errorf("%w", err)
}
manifestFilterData, err := collectImageManifestFilterData(manifestDigest.String(), repoMeta,
manifestMeta)
if err != nil {
return mTypes.FilterData{},
fmt.Errorf("%w", err)
}
indexOsList = append(indexOsList, manifestFilterData.OsList...)
indexArchList = append(indexArchList, manifestFilterData.ArchList...)
if !firstManifestChecked || indexLastUpdated.Before(manifestFilterData.LastUpdated) {
indexLastUpdated = manifestFilterData.LastUpdated
firstManifestChecked = true
}
manifestMetadataMap[manifest.Digest.String()] = manifestMeta
}
return mTypes.FilterData{
DownloadCount: repoMeta.Statistics[indexDigest].DownloadCount,
LastUpdated: indexLastUpdated,
OsList: indexOsList,
ArchList: indexArchList,
IsSigned: common.CheckIsSigned(repoMeta.Signatures[indexDigest]),
}, nil
}
func (dwr *DynamoDB) FilterTags(ctx context.Context, filterFunc mTypes.FilterFunc, filter mTypes.Filter,
requestedPage mTypes.PageInput,
) ([]mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, map[string]mTypes.IndexData,
zcommon.PageInfo, error,
func (dwr *DynamoDB) FilterTags(ctx context.Context, filterFunc mTypes.FilterFunc,
) ([]mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, map[string]mTypes.IndexData, error,
) {
var (
foundRepos = make([]mTypes.RepoMetadata, 0)
manifestMetadataMap = make(map[string]mTypes.ManifestMetadata)
indexDataMap = make(map[string]mTypes.IndexData)
repoMetaAttributeIterator AttributesIterator
pageFinder pagination.PageFinder
pageInfo zcommon.PageInfo
userBookmarks = getUserBookmarks(ctx, dwr)
userStars = getUserStars(ctx, dwr)
)
@@ -1161,18 +995,12 @@ func (dwr *DynamoDB) FilterTags(ctx context.Context, filterFunc mTypes.FilterFun
dwr.Client, dwr.RepoMetaTablename, "RepoMetadata", 0, dwr.Log,
)
pageFinder, err := pagination.NewBaseImagePageFinder(requestedPage.Limit, requestedPage.Offset, requestedPage.SortBy)
if err != nil {
return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, map[string]mTypes.IndexData{},
pageInfo, err
}
repoMetaAttribute, err := repoMetaAttributeIterator.First(ctx)
for ; repoMetaAttribute != nil; repoMetaAttribute, err = repoMetaAttributeIterator.Next(ctx) {
if err != nil {
return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, map[string]mTypes.IndexData{},
pageInfo, err
err
}
var repoMeta mTypes.RepoMetadata
@@ -1180,7 +1008,7 @@ func (dwr *DynamoDB) FilterTags(ctx context.Context, filterFunc mTypes.FilterFun
err := attributevalue.Unmarshal(repoMetaAttribute, &repoMeta)
if err != nil {
return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, map[string]mTypes.IndexData{},
pageInfo, err
err
}
if ok, err := localCtx.RepoIsUserAvailable(ctx, repoMeta.Name); !ok || err != nil {
@@ -1201,23 +1029,9 @@ func (dwr *DynamoDB) FilterTags(ctx context.Context, filterFunc mTypes.FilterFun
manifestMetadataMap)
if err != nil {
return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, map[string]mTypes.IndexData{},
pageInfo,
fmt.Errorf("metadb: error while unmashaling manifest metadata for digest %s \n%w", manifestDigest, err)
}
imageFilterData, err := collectImageManifestFilterData(manifestDigest, repoMeta, manifestMeta)
if err != nil {
return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, map[string]mTypes.IndexData{},
pageInfo,
fmt.Errorf("metadb: error collecting filter data for manifest with digest %s %w", manifestDigest, err)
}
if !common.AcceptedByFilter(filter, imageFilterData) {
delete(matchedTags, tag)
continue
}
if filterFunc(repoMeta, manifestMeta) {
matchedTags[tag] = descriptor
manifestMetadataMap[manifestDigest] = manifestMeta
@@ -1228,7 +1042,6 @@ func (dwr *DynamoDB) FilterTags(ctx context.Context, filterFunc mTypes.FilterFun
indexData, err := dwr.fetchIndexDataWithCheck(indexDigest, indexDataMap) //nolint:contextcheck
if err != nil {
return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, map[string]mTypes.IndexData{},
pageInfo,
fmt.Errorf("metadb: error while getting index data for digest %s %w", indexDigest, err)
}
@@ -1237,7 +1050,6 @@ func (dwr *DynamoDB) FilterTags(ctx context.Context, filterFunc mTypes.FilterFun
err = json.Unmarshal(indexData.IndexBlob, &indexContent)
if err != nil {
return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, map[string]mTypes.IndexData{},
pageInfo,
fmt.Errorf("metadb: error while unmashaling index content for digest %s %w", indexDigest, err)
}
@@ -1250,21 +1062,9 @@ func (dwr *DynamoDB) FilterTags(ctx context.Context, filterFunc mTypes.FilterFun
manifestMetadataMap)
if err != nil {
return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, map[string]mTypes.IndexData{},
pageInfo,
fmt.Errorf("%w metadb: error while getting manifest data for digest %s", err, manifestDigest)
}
manifestFilterData, err := collectImageManifestFilterData(manifestDigest, repoMeta, manifestMeta)
if err != nil {
return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, map[string]mTypes.IndexData{},
pageInfo,
fmt.Errorf("metadb: error collecting filter data for manifest with digest %s %w", manifestDigest, err)
}
if !common.AcceptedByFilter(filter, manifestFilterData) {
continue
}
if filterFunc(repoMeta, manifestMeta) {
matchedManifests = append(matchedManifests, manifest)
manifestMetadataMap[manifestDigest] = manifestMeta
@@ -1277,7 +1077,7 @@ func (dwr *DynamoDB) FilterTags(ctx context.Context, filterFunc mTypes.FilterFun
indexBlob, err := json.Marshal(indexContent)
if err != nil {
return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, map[string]mTypes.IndexData{},
pageInfo, err
err
}
indexData.IndexBlob = indexBlob
@@ -1298,29 +1098,17 @@ func (dwr *DynamoDB) FilterTags(ctx context.Context, filterFunc mTypes.FilterFun
repoMeta.Tags = matchedTags
pageFinder.Add(mTypes.DetailedRepoMeta{
RepoMetadata: repoMeta,
})
foundRepos = append(foundRepos, repoMeta)
}
foundRepos, pageInfo := pageFinder.Page()
foundManifestMetadataMap, foundindexDataMap, err := common.FilterDataByRepo(foundRepos, manifestMetadataMap,
indexDataMap)
return foundRepos, foundManifestMetadataMap, foundindexDataMap, pageInfo, err
return foundRepos, manifestMetadataMap, indexDataMap, err
}
func (dwr *DynamoDB) FilterRepos(ctx context.Context,
filter mTypes.FilterRepoFunc,
requestedPage mTypes.PageInput,
) (
[]mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, map[string]mTypes.IndexData,
zcommon.PageInfo, error,
) {
func (dwr *DynamoDB) FilterRepos(ctx context.Context, filter mTypes.FilterRepoFunc,
) ([]mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, map[string]mTypes.IndexData, error) {
var (
foundRepos = []mTypes.RepoMetadata{}
repoMetaAttributeIterator AttributesIterator
pageInfo zcommon.PageInfo
userBookmarks = getUserBookmarks(ctx, dwr)
userStars = getUserStars(ctx, dwr)
)
@@ -1329,22 +1117,12 @@ func (dwr *DynamoDB) FilterRepos(ctx context.Context,
dwr.Client, dwr.RepoMetaTablename, "RepoMetadata", 0, dwr.Log,
)
pageFinder, err := pagination.NewBaseRepoPageFinder(requestedPage.Limit, requestedPage.Offset, requestedPage.SortBy)
if err != nil {
return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, map[string]mTypes.IndexData{},
pageInfo, err
}
repoMetaAttribute, err := repoMetaAttributeIterator.First(ctx)
if err != nil {
return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, map[string]mTypes.IndexData{},
pageInfo, err
}
for ; repoMetaAttribute != nil; repoMetaAttribute, err = repoMetaAttributeIterator.Next(ctx) {
if err != nil {
return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, map[string]mTypes.IndexData{},
pageInfo, err
err
}
var repoMeta mTypes.RepoMetadata
@@ -1352,7 +1130,7 @@ func (dwr *DynamoDB) FilterRepos(ctx context.Context,
err := attributevalue.Unmarshal(repoMetaAttribute, &repoMeta)
if err != nil {
return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, map[string]mTypes.IndexData{},
pageInfo, err
err
}
if ok, err := localCtx.RepoIsUserAvailable(ctx, repoMeta.Name); !ok || err != nil {
@@ -1363,40 +1141,28 @@ func (dwr *DynamoDB) FilterRepos(ctx context.Context,
repoMeta.IsStarred = zcommon.Contains(userStars, repoMeta.Name)
if filter(repoMeta) {
pageFinder.Add(mTypes.DetailedRepoMeta{
RepoMetadata: repoMeta,
})
foundRepos = append(foundRepos, repoMeta)
}
}
foundRepos, pageInfo := pageFinder.Page()
foundManifestMetadataMap, foundIndexDataMap, err := common.FetchDataForRepos(dwr, foundRepos)
return foundRepos, foundManifestMetadataMap, foundIndexDataMap, pageInfo, err
return foundRepos, foundManifestMetadataMap, foundIndexDataMap, err
}
func (dwr *DynamoDB) SearchTags(ctx context.Context, searchText string, filter mTypes.Filter,
requestedPage mTypes.PageInput,
func (dwr *DynamoDB) SearchTags(ctx context.Context, searchText string,
) ([]mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, map[string]mTypes.IndexData,
zcommon.PageInfo, error,
error,
) {
var (
foundRepos = make([]mTypes.RepoMetadata, 0, 1)
manifestMetadataMap = make(map[string]mTypes.ManifestMetadata)
indexDataMap = make(map[string]mTypes.IndexData)
repoMetaAttributeIterator AttributesIterator
pageFinder pagination.PageFinder
pageInfo zcommon.PageInfo
userBookmarks = getUserBookmarks(ctx, dwr)
userStars = getUserStars(ctx, dwr)
)
pageFinder, err := pagination.NewBaseImagePageFinder(requestedPage.Limit, requestedPage.Offset, requestedPage.SortBy)
if err != nil {
return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, map[string]mTypes.IndexData{},
pageInfo, err
}
repoMetaAttributeIterator = NewBaseDynamoAttributesIterator(
dwr.Client, dwr.RepoMetaTablename, "RepoMetadata", 0, dwr.Log,
)
@@ -1404,154 +1170,105 @@ func (dwr *DynamoDB) SearchTags(ctx context.Context, searchText string, filter m
searchedRepo, searchedTag, err := common.GetRepoTag(searchText)
if err != nil {
return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, map[string]mTypes.IndexData{},
pageInfo,
fmt.Errorf("metadb: error while parsing search text, invalid format %w", err)
}
repoMetaAttribute, err := repoMetaAttributeIterator.First(ctx)
if err != nil {
return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, map[string]mTypes.IndexData{},
err
}
for ; repoMetaAttribute != nil; repoMetaAttribute, err = repoMetaAttributeIterator.Next(ctx) {
if err != nil {
// log
return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, map[string]mTypes.IndexData{},
pageInfo, err
}
var repoMeta mTypes.RepoMetadata
var repoMeta mTypes.RepoMetadata
err = attributevalue.Unmarshal(repoMetaAttribute, &repoMeta)
if err != nil {
return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, map[string]mTypes.IndexData{},
err
}
err := attributevalue.Unmarshal(repoMetaAttribute, &repoMeta)
if err != nil {
return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, map[string]mTypes.IndexData{},
pageInfo, err
}
if ok, err := localCtx.RepoIsUserAvailable(ctx, repoMeta.Name); !ok || err != nil {
return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, map[string]mTypes.IndexData{},
err
}
if ok, err := localCtx.RepoIsUserAvailable(ctx, repoMeta.Name); !ok || err != nil {
if repoMeta.Name != searchedRepo {
return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, map[string]mTypes.IndexData{},
err
}
repoMeta.IsBookmarked = zcommon.Contains(userBookmarks, repoMeta.Name)
repoMeta.IsStarred = zcommon.Contains(userStars, repoMeta.Name)
matchedTags := make(map[string]mTypes.Descriptor)
for tag, descriptor := range repoMeta.Tags {
if !strings.HasPrefix(tag, searchedTag) {
continue
}
if repoMeta.Name != searchedRepo {
continue
}
matchedTags[tag] = descriptor
repoMeta.IsBookmarked = zcommon.Contains(userBookmarks, repoMeta.Name)
repoMeta.IsStarred = zcommon.Contains(userStars, repoMeta.Name)
switch descriptor.MediaType {
case ispec.MediaTypeImageManifest:
manifestDigest := descriptor.Digest
matchedTags := make(map[string]mTypes.Descriptor)
for tag, descriptor := range repoMeta.Tags {
if !strings.HasPrefix(tag, searchedTag) {
continue
manifestMeta, err := dwr.fetchManifestMetaWithCheck(repoMeta.Name, manifestDigest, //nolint:contextcheck
manifestMetadataMap)
if err != nil {
return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, map[string]mTypes.IndexData{},
fmt.Errorf("metadb: error while unmashaling manifest metadata for digest %s %w", descriptor.Digest, err)
}
matchedTags[tag] = descriptor
manifestMetadataMap[descriptor.Digest] = manifestMeta
case ispec.MediaTypeImageIndex:
indexDigest := descriptor.Digest
switch descriptor.MediaType {
case ispec.MediaTypeImageManifest:
manifestDigest := descriptor.Digest
indexData, err := dwr.fetchIndexDataWithCheck(indexDigest, indexDataMap) //nolint:contextcheck
if err != nil {
return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, map[string]mTypes.IndexData{},
fmt.Errorf("%w", err)
}
var indexContent ispec.Index
err = json.Unmarshal(indexData.IndexBlob, &indexContent)
if err != nil {
return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, map[string]mTypes.IndexData{},
fmt.Errorf("metadb: error while unmashaling index content for digest %s %w", indexDigest, err)
}
for _, manifest := range indexContent.Manifests {
manifestDigest := manifest.Digest.String()
manifestMeta, err := dwr.fetchManifestMetaWithCheck(repoMeta.Name, manifestDigest, //nolint:contextcheck
manifestMetadataMap)
if err != nil {
return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, map[string]mTypes.IndexData{},
pageInfo,
fmt.Errorf("metadb: error while unmashaling manifest metadata for digest %s %w", descriptor.Digest, err)
}
imageFilterData, err := collectImageManifestFilterData(manifestDigest, repoMeta, manifestMeta)
if err != nil {
return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, map[string]mTypes.IndexData{},
pageInfo,
fmt.Errorf("%w", err)
}
if !common.AcceptedByFilter(filter, imageFilterData) {
delete(matchedTags, tag)
continue
}
manifestMetadataMap[descriptor.Digest] = manifestMeta
case ispec.MediaTypeImageIndex:
indexDigest := descriptor.Digest
indexData, err := dwr.fetchIndexDataWithCheck(indexDigest, indexDataMap) //nolint:contextcheck
if err != nil {
return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, map[string]mTypes.IndexData{},
pageInfo,
fmt.Errorf("%w", err)
}
var indexContent ispec.Index
err = json.Unmarshal(indexData.IndexBlob, &indexContent)
if err != nil {
return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, map[string]mTypes.IndexData{},
pageInfo,
fmt.Errorf("metadb: error while unmashaling index content for digest %s %w", indexDigest, err)
}
manifestHasBeenMatched := false
for _, manifest := range indexContent.Manifests {
manifestDigest := manifest.Digest.String()
manifestMeta, err := dwr.fetchManifestMetaWithCheck(repoMeta.Name, manifestDigest, //nolint:contextcheck
manifestMetadataMap)
if err != nil {
return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, map[string]mTypes.IndexData{},
pageInfo,
fmt.Errorf("%w", err)
}
manifestFilterData, err := collectImageManifestFilterData(manifestDigest, repoMeta, manifestMeta)
if err != nil {
return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, map[string]mTypes.IndexData{},
pageInfo,
fmt.Errorf("%w", err)
}
manifestMetadataMap[manifestDigest] = manifestMeta
if common.AcceptedByFilter(filter, manifestFilterData) {
manifestHasBeenMatched = true
}
}
if !manifestHasBeenMatched {
delete(matchedTags, tag)
for _, manifest := range indexContent.Manifests {
delete(manifestMetadataMap, manifest.Digest.String())
}
continue
}
indexDataMap[indexDigest] = indexData
default:
dwr.Log.Error().Str("mediaType", descriptor.MediaType).Msg("Unsupported media type")
continue
manifestMetadataMap[manifestDigest] = manifestMeta
}
}
if len(matchedTags) == 0 {
indexDataMap[indexDigest] = indexData
default:
dwr.Log.Error().Str("mediaType", descriptor.MediaType).Msg("Unsupported media type")
continue
}
repoMeta.Tags = matchedTags
pageFinder.Add(mTypes.DetailedRepoMeta{
RepoMetadata: repoMeta,
})
}
foundRepos, pageInfo := pageFinder.Page()
if len(matchedTags) == 0 {
return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, map[string]mTypes.IndexData{},
err
}
foundManifestMetadataMap, foundindexDataMap, err := common.FilterDataByRepo(foundRepos, manifestMetadataMap,
indexDataMap)
repoMeta.Tags = matchedTags
return foundRepos, foundManifestMetadataMap, foundindexDataMap, pageInfo, err
foundRepos = append(foundRepos, repoMeta)
return foundRepos, manifestMetadataMap, indexDataMap, err
}
func (dwr *DynamoDB) PatchDB() error {
pkg/meta/dynamodb/dynamo_internal_test.go → pkg/meta/dynamodb/dynamodb_internal_test.go
View File
pkg/meta/dynamodb/dynamo_test.go → pkg/meta/dynamodb/dynamodb_test.go
+93 -223
View File
@@ -823,8 +823,7 @@ func TestWrapperErrors(t *testing.T) {
err = setBadRepoMeta(dynamoWrapper.Client, repoMetaTablename, "repo") //nolint:contextcheck
So(err, ShouldBeNil)
_, err = dynamoWrapper.GetMultipleRepoMeta(ctx, func(repoMeta mTypes.RepoMetadata) bool { return true },
mTypes.PageInput{})
_, err = dynamoWrapper.GetMultipleRepoMeta(ctx, func(repoMeta mTypes.RepoMetadata) bool { return true })
So(err, ShouldNotBeNil)
})
@@ -833,7 +832,50 @@ func TestWrapperErrors(t *testing.T) {
err = setBadRepoMeta(dynamoWrapper.Client, repoMetaTablename, "repo") //nolint:contextcheck
So(err, ShouldBeNil)
_, _, _, _, err = dynamoWrapper.SearchRepos(ctx, "", mTypes.Filter{}, mTypes.PageInput{})
_, _, _, err = dynamoWrapper.SearchRepos(ctx, "")
So(err, ShouldNotBeNil)
})
Convey("SearchRepos bad tablename", func() {
dynamoWrapper.RepoMetaTablename = badTablename
_, _, _, err = dynamoWrapper.SearchRepos(ctx, "")
So(err, ShouldNotBeNil)
})
Convey("GetMultipleRepoMeta bad tablename", func() {
dynamoWrapper.RepoMetaTablename = badTablename
_, err = dynamoWrapper.GetMultipleRepoMeta(ctx, func(repoMeta mTypes.RepoMetadata) bool { return true })
So(err, ShouldNotBeNil)
})
Convey("FilterTags bad tablename", func() {
dynamoWrapper.RepoMetaTablename = badTablename
_, _, _, err = dynamoWrapper.FilterTags(ctx,
func(repoMeta mTypes.RepoMetadata, manifestMeta mTypes.ManifestMetadata) bool {
return true
})
So(err, ShouldNotBeNil)
})
Convey("FilterRepos bad tablename", func() {
dynamoWrapper.RepoMetaTablename = badTablename
_, _, _, err = dynamoWrapper.FilterRepos(ctx, func(repoMeta mTypes.RepoMetadata) bool { return true })
So(err, ShouldNotBeNil)
})
Convey("SearchTags bad tablename", func() {
dynamoWrapper.RepoMetaTablename = badTablename
_, _, _, err = dynamoWrapper.SearchTags(ctx, "repo:tag")
So(err, ShouldNotBeNil)
})
@@ -843,22 +885,7 @@ func TestWrapperErrors(t *testing.T) {
ispec.MediaTypeImageManifest)
So(err, ShouldBeNil)
_, _, _, _, err = dynamoWrapper.SearchRepos(ctx, "", mTypes.Filter{}, mTypes.PageInput{})
So(err, ShouldNotBeNil)
})
Convey("SearchRepos config unmarshal error", func() {
err := dynamoWrapper.SetRepoReference("repo", "tag1", "dig1", ispec.MediaTypeImageManifest) //nolint:contextcheck
So(err, ShouldBeNil)
err = dynamoWrapper.SetManifestData("dig1", mTypes.ManifestData{ //nolint:contextcheck
ManifestBlob: []byte("{}"),
ConfigBlob: []byte("bad json"),
})
So(err, ShouldBeNil)
_, _, _, _, err = dynamoWrapper.SearchRepos(ctx, "", mTypes.Filter{}, mTypes.PageInput{})
_, _, _, err = dynamoWrapper.SearchRepos(ctx, "")
So(err, ShouldNotBeNil)
})
@@ -869,18 +896,14 @@ func TestWrapperErrors(t *testing.T) {
err := dynamoWrapper.SetRepoReference("repo", "tag1", digest, "invalid type") //nolint:contextcheck
So(err, ShouldBeNil)
_, _, _, _, err = dynamoWrapper.SearchRepos(ctx, "", mTypes.Filter{}, mTypes.PageInput{})
_, _, _, err = dynamoWrapper.SearchRepos(ctx, "")
So(err, ShouldBeNil)
_, _, _, _, err = dynamoWrapper.SearchTags(ctx, "repo:", mTypes.Filter{}, mTypes.PageInput{})
_, _, _, err = dynamoWrapper.SearchTags(ctx, "repo:")
So(err, ShouldBeNil)
_, _, _, _, err = dynamoWrapper.FilterTags(
ctx,
func(repoMeta mTypes.RepoMetadata, manifestMeta mTypes.ManifestMetadata) bool { return true },
mTypes.Filter{},
mTypes.PageInput{},
)
_, _, _, err = dynamoWrapper.FilterTags(ctx,
func(repoMeta mTypes.RepoMetadata, manifestMeta mTypes.ManifestMetadata) bool { return true })
So(err, ShouldBeNil)
})
@@ -893,7 +916,7 @@ func TestWrapperErrors(t *testing.T) {
err = setBadIndexData(dynamoWrapper.Client, indexDataTablename, indexDigest.String()) //nolint:contextcheck
So(err, ShouldBeNil)
_, _, _, _, err = dynamoWrapper.SearchRepos(ctx, "", mTypes.Filter{}, mTypes.PageInput{})
_, _, _, err = dynamoWrapper.SearchRepos(ctx, "")
So(err, ShouldNotBeNil)
})
@@ -908,43 +931,7 @@ func TestWrapperErrors(t *testing.T) {
})
So(err, ShouldBeNil)
_, _, _, _, err = dynamoWrapper.SearchRepos(ctx, "", mTypes.Filter{}, mTypes.PageInput{})
So(err, ShouldNotBeNil)
})
Convey("SearchRepos good index data, bad manifest inside index", func() {
var (
indexDigest = digest.FromString("indexDigest")
manifestDigestFromIndex1 = digest.FromString("manifestDigestFromIndex1")
manifestDigestFromIndex2 = digest.FromString("manifestDigestFromIndex2")
)
err := dynamoWrapper.SetRepoReference("repo", "tag1", indexDigest, ispec.MediaTypeImageIndex) //nolint:contextcheck
So(err, ShouldBeNil)
indexBlob, err := test.GetIndexBlobWithManifests([]digest.Digest{
manifestDigestFromIndex1, manifestDigestFromIndex2,
})
So(err, ShouldBeNil)
err = dynamoWrapper.SetIndexData(indexDigest, mTypes.IndexData{ //nolint:contextcheck
IndexBlob: indexBlob,
})
So(err, ShouldBeNil)
err = dynamoWrapper.SetManifestData(manifestDigestFromIndex1, mTypes.ManifestData{ //nolint:contextcheck
ManifestBlob: []byte("Bad Manifest"),
ConfigBlob: []byte("Bad Manifest"),
})
So(err, ShouldBeNil)
err = dynamoWrapper.SetManifestData(manifestDigestFromIndex2, mTypes.ManifestData{ //nolint:contextcheck
ManifestBlob: []byte("Bad Manifest"),
ConfigBlob: []byte("Bad Manifest"),
})
So(err, ShouldBeNil)
_, _, _, _, err = dynamoWrapper.SearchRepos(ctx, "", mTypes.Filter{}, mTypes.PageInput{})
_, _, _, err = dynamoWrapper.SearchRepos(ctx, "")
So(err, ShouldNotBeNil)
})
@@ -952,7 +939,7 @@ func TestWrapperErrors(t *testing.T) {
err = setBadRepoMeta(dynamoWrapper.Client, repoMetaTablename, "repo") //nolint:contextcheck
So(err, ShouldBeNil)
_, _, _, _, err = dynamoWrapper.SearchTags(ctx, "repo:", mTypes.Filter{}, mTypes.PageInput{})
_, _, _, err = dynamoWrapper.SearchTags(ctx, "repo:")
So(err, ShouldNotBeNil)
})
@@ -962,25 +949,7 @@ func TestWrapperErrors(t *testing.T) {
ispec.MediaTypeImageManifest)
So(err, ShouldBeNil)
_, _, _, _, err = dynamoWrapper.SearchTags(ctx, "repo:", mTypes.Filter{}, mTypes.PageInput{})
So(err, ShouldNotBeNil)
})
Convey("SearchTags config unmarshal error", func() {
err := dynamoWrapper.SetRepoReference("repo", "tag1", "dig1", ispec.MediaTypeImageManifest) //nolint:contextcheck
So(err, ShouldBeNil)
err = dynamoWrapper.SetManifestData( //nolint:contextcheck
"dig1",
mTypes.ManifestData{
ManifestBlob: []byte("{}"),
ConfigBlob: []byte("bad json"),
},
)
So(err, ShouldBeNil)
_, _, _, _, err = dynamoWrapper.SearchTags(ctx, "repo:", mTypes.Filter{}, mTypes.PageInput{})
_, _, _, err = dynamoWrapper.SearchTags(ctx, "repo:")
So(err, ShouldNotBeNil)
})
@@ -994,7 +963,7 @@ func TestWrapperErrors(t *testing.T) {
err = setBadIndexData(dynamoWrapper.Client, indexDataTablename, indexDigest.String()) //nolint:contextcheck
So(err, ShouldBeNil)
_, _, _, _, err = dynamoWrapper.SearchTags(ctx, "repo:", mTypes.Filter{}, mTypes.PageInput{})
_, _, _, err = dynamoWrapper.SearchTags(ctx, "repo:")
So(err, ShouldNotBeNil)
})
@@ -1009,57 +978,31 @@ func TestWrapperErrors(t *testing.T) {
})
So(err, ShouldBeNil)
_, _, _, _, err = dynamoWrapper.SearchTags(ctx, "repo:", mTypes.Filter{}, mTypes.PageInput{})
_, _, _, err = dynamoWrapper.SearchTags(ctx, "repo:")
So(err, ShouldNotBeNil)
})
Convey("SearchTags good index data, bad manifest inside index", func() {
var (
indexDigest = digest.FromString("indexDigest")
manifestDigestFromIndex1 = digest.FromString("manifestDigestFromIndex1")
manifestDigestFromIndex2 = digest.FromString("manifestDigestFromIndex2")
)
err := dynamoWrapper.SetRepoReference("repo", "tag1", indexDigest, ispec.MediaTypeImageIndex) //nolint:contextcheck
Convey("SearchRepos attr", func() {
err = setBadRepoMeta(dynamoWrapper.Client, repoMetaTablename, "repo") //nolint:contextcheck
So(err, ShouldBeNil)
indexBlob, err := test.GetIndexBlobWithManifests([]digest.Digest{
manifestDigestFromIndex1, manifestDigestFromIndex2,
})
So(err, ShouldBeNil)
err = dynamoWrapper.SetIndexData(indexDigest, mTypes.IndexData{ //nolint:contextcheck
IndexBlob: indexBlob,
})
So(err, ShouldBeNil)
err = dynamoWrapper.SetManifestData(manifestDigestFromIndex1, mTypes.ManifestData{ //nolint:contextcheck
ManifestBlob: []byte("Bad Manifest"),
ConfigBlob: []byte("Bad Manifest"),
})
So(err, ShouldBeNil)
err = dynamoWrapper.SetManifestData(manifestDigestFromIndex2, mTypes.ManifestData{ //nolint:contextcheck
ManifestBlob: []byte("Bad Manifest"),
ConfigBlob: []byte("Bad Manifest"),
})
So(err, ShouldBeNil)
_, _, _, _, err = dynamoWrapper.SearchTags(ctx, "repo:", mTypes.Filter{}, mTypes.PageInput{})
So(err, ShouldNotBeNil)
})
Convey("FilterRepos NewBaseRepoPageFinder errors", func() {
_, _, _, _, err := dynamoWrapper.SearchRepos(ctx, "text", mTypes.Filter{},
mTypes.PageInput{Offset: -2, Limit: -2})
_, _, _, err := dynamoWrapper.SearchRepos(ctx, "repo")
So(err, ShouldNotBeNil)
})
Convey("FilterRepos attributevalue.Unmarshal(repoMetaAttribute) errors", func() {
err = setBadRepoMeta(dynamoWrapper.Client, repoMetaTablename, "repo") //nolint:contextcheck
So(err, ShouldBeNil)
dynamoWrapper.RepoMetaTablename = "bad-table-FilterRepos"
_, _, _, _, err := dynamoWrapper.SearchRepos(ctx, "repo", mTypes.Filter{}, mTypes.PageInput{})
_, _, _, err := dynamoWrapper.FilterRepos(ctx, func(repoMeta mTypes.RepoMetadata) bool {
return true
})
So(err, ShouldNotBeNil)
})
Convey("SearchRepos bad RepoMeta table name", func() {
dynamoWrapper.RepoMetaTablename = "SearchRepos-bad-table"
_, _, _, err := dynamoWrapper.SearchRepos(ctx, "repo")
So(err, ShouldNotBeNil)
})
@@ -1067,14 +1010,21 @@ func TestWrapperErrors(t *testing.T) {
err = setBadRepoMeta(dynamoWrapper.Client, repoMetaTablename, "repo") //nolint:contextcheck
So(err, ShouldBeNil)
_, _, _, _, err = dynamoWrapper.FilterTags(
ctx,
_, _, _, err = dynamoWrapper.FilterTags(ctx,
func(repoMeta mTypes.RepoMetadata, manifestMeta mTypes.ManifestMetadata) bool {
return true
},
mTypes.Filter{},
mTypes.PageInput{},
)
})
So(err, ShouldNotBeNil)
})
Convey("FilterTags bad RepoMeta table name", func() {
dynamoWrapper.RepoMetaTablename = "bad-table"
_, _, _, err := dynamoWrapper.FilterTags(ctx,
func(repoMeta mTypes.RepoMetadata, manifestMeta mTypes.ManifestMetadata) bool {
return true
})
So(err, ShouldNotBeNil)
})
@@ -1084,14 +1034,10 @@ func TestWrapperErrors(t *testing.T) {
ispec.MediaTypeImageManifest)
So(err, ShouldBeNil)
_, _, _, _, err = dynamoWrapper.FilterTags(
ctx,
_, _, _, err = dynamoWrapper.FilterTags(ctx,
func(repoMeta mTypes.RepoMetadata, manifestMeta mTypes.ManifestMetadata) bool {
return true
},
mTypes.Filter{},
mTypes.PageInput{},
)
})
So(err, ShouldNotBeNil)
})
@@ -1103,14 +1049,11 @@ func TestWrapperErrors(t *testing.T) {
err = setBadManifestData(dynamoWrapper.Client, manifestDataTablename, "dig") //nolint:contextcheck
So(err, ShouldBeNil)
_, _, _, _, err = dynamoWrapper.FilterTags(
_, _, _, err = dynamoWrapper.FilterTags(
ctx,
func(repoMeta mTypes.RepoMetadata, manifestMeta mTypes.ManifestMetadata) bool {
return true
},
mTypes.Filter{},
mTypes.PageInput{},
)
})
So(err, ShouldNotBeNil)
})
@@ -1124,11 +1067,8 @@ func TestWrapperErrors(t *testing.T) {
err = setBadIndexData(dynamoWrapper.Client, indexDataTablename, indexDigest.String()) //nolint:contextcheck
So(err, ShouldBeNil)
_, _, _, _, err = dynamoWrapper.FilterTags(ctx,
func(repoMeta mTypes.RepoMetadata, manifestMeta mTypes.ManifestMetadata) bool { return true },
mTypes.Filter{},
mTypes.PageInput{},
)
_, _, _, err = dynamoWrapper.FilterTags(ctx,
func(repoMeta mTypes.RepoMetadata, manifestMeta mTypes.ManifestMetadata) bool { return true })
So(err, ShouldNotBeNil)
})
@@ -1143,11 +1083,8 @@ func TestWrapperErrors(t *testing.T) {
})
So(err, ShouldBeNil)
_, _, _, _, err = dynamoWrapper.FilterTags(ctx,
func(repoMeta mTypes.RepoMetadata, manifestMeta mTypes.ManifestMetadata) bool { return true },
mTypes.Filter{},
mTypes.PageInput{},
)
_, _, _, err = dynamoWrapper.FilterTags(ctx,
func(repoMeta mTypes.RepoMetadata, manifestMeta mTypes.ManifestMetadata) bool { return true })
So(err, ShouldNotBeNil)
})
@@ -1183,78 +1120,11 @@ func TestWrapperErrors(t *testing.T) {
})
So(err, ShouldBeNil)
_, _, _, _, err = dynamoWrapper.FilterTags(ctx,
func(repoMeta mTypes.RepoMetadata, manifestMeta mTypes.ManifestMetadata) bool { return false },
mTypes.Filter{},
mTypes.PageInput{},
)
_, _, _, err = dynamoWrapper.FilterTags(ctx,
func(repoMeta mTypes.RepoMetadata, manifestMeta mTypes.ManifestMetadata) bool { return false })
So(err, ShouldBeNil)
})
Convey("FilterTags bad config blob in image with a single manifest", func() {
manifestDigest := digest.FromString("manifestDigestBadConfig")
err := dynamoWrapper.SetRepoReference( //nolint:contextcheck
"repo", "tag1", manifestDigest, ispec.MediaTypeImageManifest,
)
So(err, ShouldBeNil)
err = dynamoWrapper.SetManifestData(manifestDigest, mTypes.ManifestData{ //nolint:contextcheck
ManifestBlob: []byte("{}"),
ConfigBlob: []byte("bad blob"),
})
So(err, ShouldBeNil)
_, _, _, _, err = dynamoWrapper.FilterTags(ctx,
func(repoMeta mTypes.RepoMetadata, manifestMeta mTypes.ManifestMetadata) bool { return true },
mTypes.Filter{},
mTypes.PageInput{},
)
So(err, ShouldNotBeNil)
})
Convey("FilterTags bad config blob in index", func() {
var (
indexDigest = digest.FromString("indexDigest")
manifestDigestFromIndex1 = digest.FromString("manifestDigestFromIndexGoodConfig")
manifestDigestFromIndex2 = digest.FromString("manifestDigestFromIndexBadConfig")
)
err := dynamoWrapper.SetRepoReference( //nolint:contextcheck
"repo", "tag1", indexDigest, ispec.MediaTypeImageIndex,
)
So(err, ShouldBeNil)
indexBlob, err := test.GetIndexBlobWithManifests([]digest.Digest{
manifestDigestFromIndex1, manifestDigestFromIndex2,
})
So(err, ShouldBeNil)
err = dynamoWrapper.SetIndexData(indexDigest, mTypes.IndexData{ //nolint:contextcheck
IndexBlob: indexBlob,
})
So(err, ShouldBeNil)
err = dynamoWrapper.SetManifestData(manifestDigestFromIndex1, mTypes.ManifestData{ //nolint:contextcheck
ManifestBlob: []byte("{}"),
ConfigBlob: []byte("{}"),
})
So(err, ShouldBeNil)
err = dynamoWrapper.SetManifestData(manifestDigestFromIndex2, mTypes.ManifestData{ //nolint:contextcheck
ManifestBlob: []byte("{}"),
ConfigBlob: []byte("bad blob"),
})
So(err, ShouldBeNil)
_, _, _, _, err = dynamoWrapper.FilterTags(ctx,
func(repoMeta mTypes.RepoMetadata, manifestMeta mTypes.ManifestMetadata) bool { return true },
mTypes.Filter{},
mTypes.PageInput{},
)
So(err, ShouldNotBeNil)
})
Convey("PatchDB dwr.getDBVersion errors", func() {
dynamoWrapper.VersionTablename = badTablename
pkg/meta/dynamodb/iterator.go
+1 -1
View File
@@ -54,7 +54,7 @@ func (dii *BaseAttributesIterator) First(ctx context.Context) (types.AttributeVa
Limit: dii.readLimit,
})
if err != nil {
return nil, err
return &types.AttributeValueMemberBOOL{}, err
}
if len(scanOutput.Items) == 0 {
pkg/meta/update.go → pkg/meta/hooks.go
+4 -4
View File
@@ -58,7 +58,7 @@ func OnUpdateManifest(repo, reference, mediaType string, digest godigest.Digest,
}
}
} else {
err := SetImageMetaFromInput(repo, reference, mediaType, digest, body,
err = SetImageMetaFromInput(repo, reference, mediaType, digest, body,
imgStore, metaDB, log)
if err != nil {
metadataSuccessfullySet = false
@@ -66,7 +66,7 @@ func OnUpdateManifest(repo, reference, mediaType string, digest godigest.Digest,
}
if !metadataSuccessfullySet {
log.Info().Str("tag", reference).Str("repository", repo).Msg("uploding image meta was unsuccessful for tag in repo")
log.Info().Str("tag", reference).Str("repository", repo).Msg("uploading image meta was unsuccessful for tag in repo")
if err := imgStore.DeleteImageManifest(repo, reference, false); err != nil {
log.Error().Err(err).Str("reference", reference).Str("repository", repo).
@@ -122,8 +122,8 @@ func OnDeleteManifest(repo, reference, mediaType string, digest godigest.Digest,
manageRepoMetaSuccessfully = false
}
if refferredDigest, hasSubject := common.GetReferredSubject(manifestBlob); hasSubject {
err := metaDB.DeleteReferrer(repo, refferredDigest, digest)
if referredDigest, hasSubject := common.GetReferredSubject(manifestBlob); hasSubject {
err := metaDB.DeleteReferrer(repo, referredDigest, digest)
if err != nil {
log.Error().Err(err).Msg("metadb: error while deleting referrer")
pkg/meta/update_test.go → pkg/meta/hooks_test.go
View File
pkg/meta/meta_test.go
+68 -811
View File
File diff suppressed because it is too large Load Diff
pkg/meta/pagination/pagination.go
-273
View File
@@ -1,273 +0,0 @@
package pagination
import (
"fmt"
"sort"
zerr "zotregistry.io/zot/errors"
"zotregistry.io/zot/pkg/common"
mTypes "zotregistry.io/zot/pkg/meta/types"
)
// PageFinder permits keeping a pool of objects using Add
// and returning a specific page.
type PageFinder interface {
Add(detailedRepoMeta mTypes.DetailedRepoMeta)
Page() ([]mTypes.RepoMetadata, common.PageInfo)
Reset()
}
// RepoPageFinder implements PageFinder. It manages RepoMeta objects and calculates the page
// using the given limit, offset and sortBy option.
type RepoPageFinder struct {
limit int
offset int
sortBy mTypes.SortCriteria
pageBuffer []mTypes.DetailedRepoMeta
}
func NewBaseRepoPageFinder(limit, offset int, sortBy mTypes.SortCriteria) (*RepoPageFinder, error) {
if sortBy == "" {
sortBy = mTypes.AlphabeticAsc
}
if limit < 0 {
return nil, zerr.ErrLimitIsNegative
}
if offset < 0 {
return nil, zerr.ErrOffsetIsNegative
}
if _, found := mTypes.SortFunctions()[sortBy]; !found {
return nil, fmt.Errorf("sorting repos by '%s' is not supported %w",
sortBy, zerr.ErrSortCriteriaNotSupported)
}
return &RepoPageFinder{
limit: limit,
offset: offset,
sortBy: sortBy,
pageBuffer: make([]mTypes.DetailedRepoMeta, 0, limit),
}, nil
}
func (bpt *RepoPageFinder) Reset() {
bpt.pageBuffer = []mTypes.DetailedRepoMeta{}
}
func (bpt *RepoPageFinder) Add(namedRepoMeta mTypes.DetailedRepoMeta) {
bpt.pageBuffer = append(bpt.pageBuffer, namedRepoMeta)
}
func (bpt *RepoPageFinder) Page() ([]mTypes.RepoMetadata, common.PageInfo) {
if len(bpt.pageBuffer) == 0 {
return []mTypes.RepoMetadata{}, common.PageInfo{}
}
pageInfo := &common.PageInfo{}
sort.Slice(bpt.pageBuffer, mTypes.SortFunctions()[bpt.sortBy](bpt.pageBuffer))
// the offset and limit are calculatd in terms of repos counted
start := bpt.offset
end := bpt.offset + bpt.limit
// we'll return an empty array when the offset is greater than the number of elements
if start >= len(bpt.pageBuffer) {
start = len(bpt.pageBuffer)
end = start
}
if end >= len(bpt.pageBuffer) {
end = len(bpt.pageBuffer)
}
detailedReposPage := bpt.pageBuffer[start:end]
pageInfo.ItemCount = len(detailedReposPage)
if start == 0 && end == 0 {
detailedReposPage = bpt.pageBuffer
pageInfo.ItemCount = len(detailedReposPage)
}
repos := make([]mTypes.RepoMetadata, 0, len(detailedReposPage))
for _, drm := range detailedReposPage {
repos = append(repos, drm.RepoMetadata)
}
pageInfo.TotalCount = len(bpt.pageBuffer)
return repos, *pageInfo
}
type ImagePageFinder struct {
limit int
offset int
sortBy mTypes.SortCriteria
pageBuffer []mTypes.DetailedRepoMeta
}
func NewBaseImagePageFinder(limit, offset int, sortBy mTypes.SortCriteria) (*ImagePageFinder, error) {
if sortBy == "" {
sortBy = mTypes.AlphabeticAsc
}
if limit < 0 {
return nil, zerr.ErrLimitIsNegative
}
if offset < 0 {
return nil, zerr.ErrOffsetIsNegative
}
if _, found := mTypes.SortFunctions()[sortBy]; !found {
return nil, fmt.Errorf("sorting repos by '%s' is not supported %w",
sortBy, zerr.ErrSortCriteriaNotSupported)
}
return &ImagePageFinder{
limit: limit,
offset: offset,
sortBy: sortBy,
pageBuffer: make([]mTypes.DetailedRepoMeta, 0, limit),
}, nil
}
func (bpt *ImagePageFinder) Reset() {
bpt.pageBuffer = []mTypes.DetailedRepoMeta{}
}
func (bpt *ImagePageFinder) Add(namedRepoMeta mTypes.DetailedRepoMeta) {
bpt.pageBuffer = append(bpt.pageBuffer, namedRepoMeta)
}
func (bpt *ImagePageFinder) Page() ([]mTypes.RepoMetadata, common.PageInfo) {
if len(bpt.pageBuffer) == 0 {
return []mTypes.RepoMetadata{}, common.PageInfo{}
}
pageInfo := common.PageInfo{}
for _, drm := range bpt.pageBuffer {
repo := drm.RepoMetadata
pageInfo.TotalCount += len(repo.Tags)
}
sort.Slice(bpt.pageBuffer, mTypes.SortFunctions()[bpt.sortBy](bpt.pageBuffer))
repoStartIndex := 0
tagStartIndex := 0
// the offset and limit are calculatd in terms of tags counted
remainingOffset := bpt.offset
remainingLimit := bpt.limit
repos := make([]mTypes.RepoMetadata, 0)
if remainingOffset == 0 && remainingLimit == 0 {
for _, drm := range bpt.pageBuffer {
repo := drm.RepoMetadata
repos = append(repos, repo)
pageInfo.ItemCount += len(repo.Tags)
}
return repos, pageInfo
}
// bring cursor to position in RepoMeta array
for _, drm := range bpt.pageBuffer {
if remainingOffset < len(drm.Tags) {
tagStartIndex = remainingOffset
break
}
remainingOffset -= len(drm.Tags)
repoStartIndex++
}
// offset is larger than the number of tags
if repoStartIndex >= len(bpt.pageBuffer) {
return []mTypes.RepoMetadata{}, common.PageInfo{}
}
// finish counting remaining tags inside the first repo meta
partialTags := map[string]mTypes.Descriptor{}
firstRepoMeta := bpt.pageBuffer[repoStartIndex].RepoMetadata
tags := make([]string, 0, len(firstRepoMeta.Tags))
for k := range firstRepoMeta.Tags {
tags = append(tags, k)
}
sort.Strings(tags)
for i := tagStartIndex; i < len(tags); i++ {
tag := tags[i]
partialTags[tag] = firstRepoMeta.Tags[tag]
remainingLimit--
if remainingLimit == 0 {
firstRepoMeta.Tags = partialTags
repos = append(repos, firstRepoMeta)
pageInfo.ItemCount = len(partialTags)
return repos, pageInfo
}
}
firstRepoMeta.Tags = partialTags
pageInfo.ItemCount += len(firstRepoMeta.Tags)
repos = append(repos, firstRepoMeta)
repoStartIndex++
// continue with the remaining repos
for i := repoStartIndex; i < len(bpt.pageBuffer); i++ {
repoMeta := bpt.pageBuffer[i].RepoMetadata
if len(repoMeta.Tags) > remainingLimit {
partialTags := map[string]mTypes.Descriptor{}
tags := make([]string, 0, len(repoMeta.Tags))
for k := range repoMeta.Tags {
tags = append(tags, k)
}
sort.Strings(tags)
for _, tag := range tags {
partialTags[tag] = repoMeta.Tags[tag]
remainingLimit--
if remainingLimit == 0 {
repoMeta.Tags = partialTags
repos = append(repos, repoMeta)
pageInfo.ItemCount += len(partialTags)
break
}
}
return repos, pageInfo
}
// add the whole repo
repos = append(repos, repoMeta)
pageInfo.ItemCount += len(repoMeta.Tags)
remainingLimit -= len(repoMeta.Tags)
if remainingLimit == 0 {
return repos, pageInfo
}
}
// we arrive here when the limit is bigger than the number of tags
return repos, pageInfo
}
pkg/meta/pagination/pagination_test.go
-241
View File
@@ -1,241 +0,0 @@
package pagination_test
import (
"testing"
ispec "github.com/opencontainers/image-spec/specs-go/v1"
. "github.com/smartystreets/goconvey/convey"
"zotregistry.io/zot/pkg/meta/pagination"
mTypes "zotregistry.io/zot/pkg/meta/types"
)
func TestPagination(t *testing.T) {
Convey("Repo Pagination", t, func() {
Convey("reset", func() {
pageFinder, err := pagination.NewBaseRepoPageFinder(1, 0, mTypes.AlphabeticAsc)
So(err, ShouldBeNil)
So(pageFinder, ShouldNotBeNil)
pageFinder.Add(mTypes.DetailedRepoMeta{})
pageFinder.Add(mTypes.DetailedRepoMeta{})
pageFinder.Add(mTypes.DetailedRepoMeta{})
pageFinder.Reset()
result, _ := pageFinder.Page()
So(result, ShouldBeEmpty)
})
})
Convey("Image Pagination", t, func() {
Convey("create new pageFinder errors", func() {
pageFinder, err := pagination.NewBaseImagePageFinder(-1, 10, mTypes.AlphabeticAsc)
So(pageFinder, ShouldBeNil)
So(err, ShouldNotBeNil)
pageFinder, err = pagination.NewBaseImagePageFinder(2, -1, mTypes.AlphabeticAsc)
So(pageFinder, ShouldBeNil)
So(err, ShouldNotBeNil)
pageFinder, err = pagination.NewBaseImagePageFinder(2, 1, "wrong sorting criteria")
So(pageFinder, ShouldBeNil)
So(err, ShouldNotBeNil)
})
Convey("Reset", func() {
pageFinder, err := pagination.NewBaseImagePageFinder(1, 0, mTypes.AlphabeticAsc)
So(err, ShouldBeNil)
So(pageFinder, ShouldNotBeNil)
pageFinder.Add(mTypes.DetailedRepoMeta{})
pageFinder.Add(mTypes.DetailedRepoMeta{})
pageFinder.Add(mTypes.DetailedRepoMeta{})
pageFinder.Reset()
result, _ := pageFinder.Page()
So(result, ShouldBeEmpty)
})
Convey("Page", func() {
Convey("no limit or offset", func() {
pageFinder, err := pagination.NewBaseImagePageFinder(0, 0, mTypes.AlphabeticAsc)
So(err, ShouldBeNil)
So(pageFinder, ShouldNotBeNil)
pageFinder.Add(mTypes.DetailedRepoMeta{
RepoMetadata: mTypes.RepoMetadata{
Name: "repo1",
Tags: map[string]mTypes.Descriptor{
"tag1": {Digest: "dig1", MediaType: ispec.MediaTypeImageManifest},
},
},
})
pageFinder.Add(mTypes.DetailedRepoMeta{
RepoMetadata: mTypes.RepoMetadata{
Name: "repo2",
Tags: map[string]mTypes.Descriptor{
"Tag1": {Digest: "dig1", MediaType: ispec.MediaTypeImageManifest},
"Tag2": {Digest: "dig2", MediaType: ispec.MediaTypeImageManifest},
"Tag3": {Digest: "dig3", MediaType: ispec.MediaTypeImageManifest},
"Tag4": {Digest: "dig4", MediaType: ispec.MediaTypeImageManifest},
},
},
})
_, pageInfo := pageFinder.Page()
So(pageInfo.ItemCount, ShouldEqual, 5)
So(pageInfo.TotalCount, ShouldEqual, 5)
})
Convey("Test 1 limit < len(tags)", func() {
pageFinder, err := pagination.NewBaseImagePageFinder(5, 2, mTypes.AlphabeticAsc)
So(err, ShouldBeNil)
So(pageFinder, ShouldNotBeNil)
pageFinder.Add(mTypes.DetailedRepoMeta{
RepoMetadata: mTypes.RepoMetadata{
Name: "repo1",
Tags: map[string]mTypes.Descriptor{
"tag1": {Digest: "dig1", MediaType: ispec.MediaTypeImageManifest},
},
},
})
pageFinder.Add(mTypes.DetailedRepoMeta{
RepoMetadata: mTypes.RepoMetadata{
Name: "repo2",
Tags: map[string]mTypes.Descriptor{
"Tag1": {Digest: "dig1", MediaType: ispec.MediaTypeImageManifest},
"Tag2": {Digest: "dig2", MediaType: ispec.MediaTypeImageManifest},
"Tag3": {Digest: "dig3", MediaType: ispec.MediaTypeImageManifest},
"Tag4": {Digest: "dig4", MediaType: ispec.MediaTypeImageManifest},
},
},
})
_, pageInfo := pageFinder.Page()
So(pageInfo.ItemCount, ShouldEqual, 3)
So(pageInfo.TotalCount, ShouldEqual, 5)
})
Convey("Test 2 limit < len(tags)", func() {
pageFinder, err := pagination.NewBaseImagePageFinder(5, 2, mTypes.AlphabeticAsc)
So(err, ShouldBeNil)
So(pageFinder, ShouldNotBeNil)
pageFinder.Add(mTypes.DetailedRepoMeta{
RepoMetadata: mTypes.RepoMetadata{
Name: "repo1",
Tags: map[string]mTypes.Descriptor{
"tag1": {
Digest: "dig1",
MediaType: ispec.MediaTypeImageManifest,
},
},
},
})
pageFinder.Add(mTypes.DetailedRepoMeta{
RepoMetadata: mTypes.RepoMetadata{
Name: "repo2",
Tags: map[string]mTypes.Descriptor{
"Tag1": {
Digest: "dig1",
MediaType: ispec.MediaTypeImageManifest,
},
"Tag2": {
Digest: "dig2",
MediaType: ispec.MediaTypeImageManifest,
},
"Tag3": {
Digest: "dig3",
MediaType: ispec.MediaTypeImageManifest,
},
"Tag4": {
Digest: "dig4",
MediaType: ispec.MediaTypeImageManifest,
},
},
},
})
pageFinder.Add(mTypes.DetailedRepoMeta{
RepoMetadata: mTypes.RepoMetadata{
Name: "repo3",
Tags: map[string]mTypes.Descriptor{
"Tag11": {
Digest: "dig11",
MediaType: ispec.MediaTypeImageManifest,
},
"Tag12": {
Digest: "dig12",
MediaType: ispec.MediaTypeImageManifest,
},
"Tag13": {
Digest: "dig13",
MediaType: ispec.MediaTypeImageManifest,
},
"Tag14": {
Digest: "dig14",
MediaType: ispec.MediaTypeImageManifest,
},
},
},
})
result, pageInfo := pageFinder.Page()
So(result[0].Tags, ShouldContainKey, "Tag2")
So(result[0].Tags, ShouldContainKey, "Tag3")
So(result[0].Tags, ShouldContainKey, "Tag4")
So(result[1].Tags, ShouldContainKey, "Tag11")
So(result[1].Tags, ShouldContainKey, "Tag12")
So(pageInfo.ItemCount, ShouldEqual, 5)
So(pageInfo.TotalCount, ShouldEqual, 9)
})
Convey("Test 2 limit > len(tags)", func() {
pageFinder, err := pagination.NewBaseImagePageFinder(3, 0, mTypes.AlphabeticAsc)
So(err, ShouldBeNil)
So(pageFinder, ShouldNotBeNil)
pageFinder.Add(mTypes.DetailedRepoMeta{
RepoMetadata: mTypes.RepoMetadata{
Name: "repo1",
Tags: map[string]mTypes.Descriptor{
"tag1": {
Digest: "dig1",
MediaType: ispec.MediaTypeImageManifest,
},
},
},
})
pageFinder.Add(mTypes.DetailedRepoMeta{
RepoMetadata: mTypes.RepoMetadata{
Name: "repo2",
Tags: map[string]mTypes.Descriptor{
"Tag1": {
Digest: "dig1",
MediaType: ispec.MediaTypeImageManifest,
},
},
},
})
pageFinder.Add(mTypes.DetailedRepoMeta{
RepoMetadata: mTypes.RepoMetadata{
Name: "repo3",
Tags: map[string]mTypes.Descriptor{
"Tag11": {
Digest: "dig11",
MediaType: ispec.MediaTypeImageManifest,
},
},
},
})
result, _ := pageFinder.Page()
So(result[0].Tags, ShouldContainKey, "tag1")
So(result[1].Tags, ShouldContainKey, "Tag1")
So(result[2].Tags, ShouldContainKey, "Tag11")
})
})
})
}
pkg/meta/storage_parsing.go → pkg/meta/parse.go
+7 -5
View File
@@ -332,9 +332,11 @@ func NewManifestData(repoName string, manifestBlob []byte, imageStore storageTyp
return mTypes.ManifestData{}, err
}
err = json.Unmarshal(configBlob, &configContent)
if err != nil {
return mTypes.ManifestData{}, err
if manifestContent.Config.MediaType == ispec.MediaTypeImageConfig {
err = json.Unmarshal(configBlob, &configContent)
if err != nil {
return mTypes.ManifestData{}, err
}
}
manifestData.ManifestBlob = manifestBlob
@@ -381,9 +383,9 @@ func SetImageMetaFromInput(repo, reference, mediaType string, digest godigest.Di
}
}
refferredDigest, referrerInfo, hasSubject, err := GetReferredInfo(descriptorBlob, digest.String(), mediaType)
referredDigest, referrerInfo, hasSubject, err := GetReferredInfo(descriptorBlob, digest.String(), mediaType)
if hasSubject && err == nil {
err := metaDB.SetReferrer(repo, refferredDigest, referrerInfo)
err := metaDB.SetReferrer(repo, referredDigest, referrerInfo)
if err != nil {
log.Error().Err(err).Msg("metadb: error while settingg referrer")
pkg/meta/storage_parsing_test.go → pkg/meta/parse_test.go
+2 -15
View File
@@ -223,15 +223,6 @@ func TestParseStorageErrors(t *testing.T) {
err = meta.ParseRepo("repo", metaDB, storeController, log)
So(err, ShouldNotBeNil)
})
Convey("json.Unmarshal(configBlob errors", func() {
imageStore.GetBlobContentFn = func(repo string, digest godigest.Digest) ([]byte, error) {
return []byte("invalid JSON"), nil
}
err = meta.ParseRepo("repo", metaDB, storeController, log)
So(err, ShouldNotBeNil)
})
})
Convey("CheckIsImageSignature -> is signature", func() {
@@ -474,11 +465,8 @@ func RunParseStorageTests(rootDir string, metaDB mTypes.MetaDB) {
err = meta.ParseStorage(metaDB, storeController, log.NewLogger("debug", ""))
So(err, ShouldBeNil)
repos, err := metaDB.GetMultipleRepoMeta(
context.Background(),
func(repoMeta mTypes.RepoMetadata) bool { return true },
mTypes.PageInput{},
)
repos, err := metaDB.GetMultipleRepoMeta(context.Background(),
func(repoMeta mTypes.RepoMetadata) bool { return true })
So(err, ShouldBeNil)
So(len(repos), ShouldEqual, 1)
@@ -540,7 +528,6 @@ func RunParseStorageTests(rootDir string, metaDB mTypes.MetaDB) {
repos, err := metaDB.GetMultipleRepoMeta(
context.Background(),
func(repoMeta mTypes.RepoMetadata) bool { return true },
mTypes.PageInput{},
)
So(err, ShouldBeNil)
pkg/meta/types/sort_criteria.go
-54
View File
@@ -1,54 +0,0 @@
package types
type SortCriteria string
const (
Relevance = SortCriteria("RELEVANCE")
UpdateTime = SortCriteria("UPDATE_TIME")
AlphabeticAsc = SortCriteria("ALPHABETIC_ASC")
AlphabeticDsc = SortCriteria("ALPHABETIC_DSC")
Stars = SortCriteria("STARS")
Downloads = SortCriteria("DOWNLOADS")
)
func SortFunctions() map[SortCriteria]func(pageBuffer []DetailedRepoMeta) func(i, j int) bool {
return map[SortCriteria]func(pageBuffer []DetailedRepoMeta) func(i, j int) bool{
AlphabeticAsc: SortByAlphabeticAsc,
AlphabeticDsc: SortByAlphabeticDsc,
Relevance: SortByRelevance,
UpdateTime: SortByUpdateTime,
Downloads: SortByDownloads,
}
}
func SortByAlphabeticAsc(pageBuffer []DetailedRepoMeta) func(i, j int) bool {
return func(i, j int) bool {
return pageBuffer[i].Name < pageBuffer[j].Name
}
}
func SortByAlphabeticDsc(pageBuffer []DetailedRepoMeta) func(i, j int) bool {
return func(i, j int) bool {
return pageBuffer[i].Name > pageBuffer[j].Name
}
}
func SortByRelevance(pageBuffer []DetailedRepoMeta) func(i, j int) bool {
return func(i, j int) bool {
return pageBuffer[i].Rank < pageBuffer[j].Rank
}
}
// SortByUpdateTime sorting descending by time.
func SortByUpdateTime(pageBuffer []DetailedRepoMeta) func(i, j int) bool {
return func(i, j int) bool {
return pageBuffer[i].UpdateTime.After(pageBuffer[j].UpdateTime)
}
}
// SortByDownloads returns a comparison function for descendant sorting by downloads.
func SortByDownloads(pageBuffer []DetailedRepoMeta) func(i, j int) bool {
return func(i, j int) bool {
return pageBuffer[i].Downloads > pageBuffer[j].Downloads
}
}
pkg/meta/types/types.go
+10 -17
View File
@@ -5,8 +5,6 @@ import (
"time"
godigest "github.com/opencontainers/go-digest"
"zotregistry.io/zot/pkg/common"
)
// DetailedRepoMeta is a auxiliary structure used for sorting RepoMeta arrays by information
@@ -62,7 +60,7 @@ type MetaDB interface { //nolint:interfacebloat
// GetMultipleRepoMeta returns information about all repositories as map[string]RepoMetadata filtered by the filter
// function
GetMultipleRepoMeta(ctx context.Context, filter func(repoMeta RepoMetadata) bool, requestedPage PageInput) (
GetMultipleRepoMeta(ctx context.Context, filter func(repoMeta RepoMetadata) bool) (
[]RepoMetadata, error)
// SetManifestData sets ManifestData for a given manifest in the database
@@ -107,20 +105,20 @@ type MetaDB interface { //nolint:interfacebloat
UpdateSignaturesValidity(repo string, manifestDigest godigest.Digest) error
// SearchRepos searches for repos given a search string
SearchRepos(ctx context.Context, searchText string, filter Filter, requestedPage PageInput) (
[]RepoMetadata, map[string]ManifestMetadata, map[string]IndexData, common.PageInfo, error)
SearchRepos(ctx context.Context, searchText string) (
[]RepoMetadata, map[string]ManifestMetadata, map[string]IndexData, error)
// SearchTags searches for images(repo:tag) given a search string
SearchTags(ctx context.Context, searchText string, filter Filter, requestedPage PageInput) (
[]RepoMetadata, map[string]ManifestMetadata, map[string]IndexData, common.PageInfo, error)
SearchTags(ctx context.Context, searchText string) (
[]RepoMetadata, map[string]ManifestMetadata, map[string]IndexData, error)
// FilterRepos filters for repos given a filter function
FilterRepos(ctx context.Context, filter FilterRepoFunc, requestedPage PageInput) (
[]RepoMetadata, map[string]ManifestMetadata, map[string]IndexData, common.PageInfo, error)
FilterRepos(ctx context.Context, filter FilterRepoFunc) (
[]RepoMetadata, map[string]ManifestMetadata, map[string]IndexData, error)
// FilterTags filters for images given a filter function
FilterTags(ctx context.Context, filterFunc FilterFunc, filter Filter,
requestedPage PageInput) ([]RepoMetadata, map[string]ManifestMetadata, map[string]IndexData, common.PageInfo, error)
FilterTags(ctx context.Context, filterFunc FilterFunc) (
[]RepoMetadata, map[string]ManifestMetadata, map[string]IndexData, error)
PatchDB() error
}
@@ -204,6 +202,7 @@ type RepoMetadata struct {
IsStarred bool
IsBookmarked bool
Rank int
Stars int
}
@@ -234,12 +233,6 @@ type UserData struct {
APIKeys map[string]APIKeyDetails
}
type PageInput struct {
Limit int
Offset int
SortBy SortCriteria
}
type Filter struct {
Os []*string
Arch []*string