feat(repodb): Multiarch Image support (#1147)

* feat(repodb): index logic + tests

Signed-off-by: Laurentiu Niculae <niculae.laurentiu1@gmail.com>

* feat(cli): printing indexes support using the rest api

Signed-off-by: Laurentiu Niculae <niculae.laurentiu1@gmail.com>

---------

Signed-off-by: Laurentiu Niculae <niculae.laurentiu1@gmail.com>
This commit is contained in:
LaurentiuNiculae
2023-02-27 21:23:18 +02:00
committed by GitHub
parent a561d0bad5
commit d62c09e2cc
54 changed files with 8656 additions and 2988 deletions
+522 -130
View File
@@ -56,6 +56,11 @@ func NewBoltDBWrapper(params DBParameters) (*DBWrapper, error) {
return err
}
_, err = transaction.CreateBucketIfNotExists([]byte(repodb.IndexDataBucket))
if err != nil {
return err
}
_, err = transaction.CreateBucketIfNotExists([]byte(repodb.RepoMetadataBucket))
if err != nil {
return err
@@ -209,6 +214,51 @@ func (bdw DBWrapper) GetManifestMeta(repo string, manifestDigest godigest.Digest
return manifestMetadata, err
}
func (bdw DBWrapper) SetIndexData(indexDigest godigest.Digest, indexMetadata repodb.IndexData) error {
// we make the assumption that the oci layout is consistent and all manifests refferenced inside the
// index are present
err := bdw.DB.Update(func(tx *bolt.Tx) error {
buck := tx.Bucket([]byte(repodb.IndexDataBucket))
imBlob, err := json.Marshal(indexMetadata)
if err != nil {
return errors.Wrapf(err, "repodb: error while calculating blob for manifest with digest %s", indexDigest)
}
err = buck.Put([]byte(indexDigest), imBlob)
if err != nil {
return errors.Wrapf(err, "repodb: error while setting manifest meta with for digest %s", indexDigest)
}
return nil
})
return err
}
func (bdw DBWrapper) GetIndexData(indexDigest godigest.Digest) (repodb.IndexData, error) {
var indexMetadata repodb.IndexData
err := bdw.DB.View(func(tx *bolt.Tx) error {
buck := tx.Bucket([]byte(repodb.IndexDataBucket))
mmBlob := buck.Get([]byte(indexDigest))
if len(mmBlob) == 0 {
return zerr.ErrManifestMetaNotFound
}
err := json.Unmarshal(mmBlob, &indexMetadata)
if err != nil {
return errors.Wrapf(err, "repodb: error while unmashaling manifest meta for digest %s", indexDigest)
}
return nil
})
return indexMetadata, err
}
func (bdw DBWrapper) SetRepoTag(repo string, tag string, manifestDigest godigest.Digest,
mediaType string,
) error {
@@ -622,24 +672,30 @@ func (bdw DBWrapper) DeleteSignature(repo string, signedManifestDigest godigest.
func (bdw DBWrapper) SearchRepos(ctx context.Context, searchText string, filter repodb.Filter,
requestedPage repodb.PageInput,
) ([]repodb.RepoMetadata, map[string]repodb.ManifestMetadata, repodb.PageInfo, error) {
) ([]repodb.RepoMetadata, map[string]repodb.ManifestMetadata, map[string]repodb.IndexData, repodb.PageInfo,
error,
) {
var (
foundRepos = make([]repodb.RepoMetadata, 0)
foundManifestMetadataMap = make(map[string]repodb.ManifestMetadata)
foundindexDataMap = make(map[string]repodb.IndexData)
pageFinder repodb.PageFinder
pageInfo repodb.PageInfo
)
pageFinder, err := repodb.NewBaseRepoPageFinder(requestedPage.Limit, requestedPage.Offset, requestedPage.SortBy)
if err != nil {
return []repodb.RepoMetadata{}, map[string]repodb.ManifestMetadata{}, repodb.PageInfo{}, err
return []repodb.RepoMetadata{}, map[string]repodb.ManifestMetadata{}, map[string]repodb.IndexData{},
repodb.PageInfo{}, err
}
err = bdw.DB.View(func(tx *bolt.Tx) error {
err = bdw.DB.View(func(transaction *bolt.Tx) error {
var (
manifestMetadataMap = make(map[string]repodb.ManifestMetadata)
repoBuck = tx.Bucket([]byte(repodb.RepoMetadataBucket))
dataBuck = tx.Bucket([]byte(repodb.ManifestDataBucket))
indexDataMap = make(map[string]repodb.IndexData)
repoBuck = transaction.Bucket([]byte(repodb.RepoMetadataBucket))
indexBuck = transaction.Bucket([]byte(repodb.IndexDataBucket))
manifestBuck = transaction.Bucket([]byte(repodb.ManifestDataBucket))
)
cursor := repoBuck.Cursor()
@@ -667,47 +723,87 @@ func (bdw DBWrapper) SearchRepos(ctx context.Context, searchText string, filter
isSigned = false
)
for _, descriptor := range repoMeta.Tags {
var manifestMeta repodb.ManifestMetadata
for tag, descriptor := range repoMeta.Tags {
switch descriptor.MediaType {
case ispec.MediaTypeImageManifest:
manifestDigest := descriptor.Digest
manifestMeta, manifestDownloaded := manifestMetadataMap[descriptor.Digest]
if !manifestDownloaded {
manifestMetaBlob := dataBuck.Get([]byte(descriptor.Digest))
if manifestMetaBlob == nil {
return zerr.ErrManifestMetaNotFound
}
err := json.Unmarshal(manifestMetaBlob, &manifestMeta)
manifestMeta, err := fetchManifestMetaWithCheck(repoMeta, manifestDigest,
manifestMetadataMap, manifestBuck)
if err != nil {
return errors.Wrapf(err, "repodb: error while unmarshaling manifest metadata for digest %s", descriptor.Digest)
return errors.Wrapf(err, "repodb: error fetching manifest meta for manifest with digest %s",
manifestDigest)
}
manifestFilterData, err := collectImageManifestFilterData(manifestDigest, repoMeta, manifestMeta)
if err != nil {
return errors.Wrapf(err, "repodb: error collecting filter data for manifest with digest %s",
manifestDigest)
}
repoDownloads += manifestFilterData.DownloadCount
for _, os := range manifestFilterData.OsList {
osSet[os] = true
}
for _, arch := range manifestFilterData.ArchList {
archSet[arch] = true
}
if firstImageChecked || repoLastUpdated.Before(manifestFilterData.LastUpdated) {
repoLastUpdated = manifestFilterData.LastUpdated
firstImageChecked = false
isSigned = manifestFilterData.IsSigned
}
manifestMetadataMap[descriptor.Digest] = manifestMeta
case ispec.MediaTypeImageIndex:
var indexLastUpdated time.Time
indexDigest := descriptor.Digest
indexData, err := fetchIndexDataWithCheck(indexDigest, indexDataMap, indexBuck)
if err != nil {
return errors.Wrapf(err, "repodb: error fetching index data for index with digest %s",
indexDigest)
}
var indexContent ispec.Index
err = json.Unmarshal(indexData.IndexBlob, &indexContent)
if err != nil {
return errors.Wrapf(err, "repodb: error while unmashaling index content for %s:%s", repoName, tag)
}
// this also updates manifestMetadataMap
imageFilterData, err := collectImageIndexFilterInfo(indexDigest, repoMeta, indexData, manifestMetadataMap,
manifestBuck)
if err != nil {
return errors.Wrapf(err, "repodb: error collecting filter data for index with digest %s",
indexDigest)
}
for _, arch := range imageFilterData.ArchList {
archSet[arch] = true
}
for _, os := range imageFilterData.OsList {
osSet[os] = true
}
repoDownloads += imageFilterData.DownloadCount
if repoLastUpdated.Before(imageFilterData.LastUpdated) {
repoLastUpdated = indexLastUpdated
isSigned = imageFilterData.IsSigned
}
indexDataMap[indexDigest] = indexData
default:
bdw.Log.Error().Msgf("Unsupported type: %s", descriptor.MediaType)
}
// get fields related to filtering
var configContent ispec.Image
err = json.Unmarshal(manifestMeta.ConfigBlob, &configContent)
if err != nil {
return errors.Wrapf(err, "repodb: error while unmarshaling config content for digest %s", descriptor.Digest)
}
osSet[configContent.OS] = true
archSet[configContent.Architecture] = true
// get fields related to sorting
repoDownloads += repoMeta.Statistics[descriptor.Digest].DownloadCount
imageLastUpdated := common.GetImageLastUpdatedTimestamp(configContent)
if firstImageChecked || repoLastUpdated.Before(imageLastUpdated) {
repoLastUpdated = imageLastUpdated
firstImageChecked = false
isSigned = common.CheckIsSigned(repoMeta.Signatures[descriptor.Digest])
}
manifestMetadataMap[descriptor.Digest] = manifestMeta
}
repoFilterData := repodb.FilterData{
@@ -731,40 +827,226 @@ func (bdw DBWrapper) SearchRepos(ctx context.Context, searchText string, filter
foundRepos, pageInfo = pageFinder.Page()
// keep just the manifestMeta we need
// keep just the manifestMeta and indexData we need
for _, repoMeta := range foundRepos {
for _, manifestDigest := range repoMeta.Tags {
foundManifestMetadataMap[manifestDigest.Digest] = manifestMetadataMap[manifestDigest.Digest]
for _, descriptor := range repoMeta.Tags {
switch descriptor.MediaType {
case ispec.MediaTypeImageManifest:
foundManifestMetadataMap[descriptor.Digest] = manifestMetadataMap[descriptor.Digest]
case ispec.MediaTypeImageIndex:
indexData := indexDataMap[descriptor.Digest]
var indexContent ispec.Index
err := json.Unmarshal(indexData.IndexBlob, &indexContent)
if err != nil {
return err
}
for _, manifestDescriptor := range indexContent.Manifests {
manifestDigest := manifestDescriptor.Digest.String()
foundManifestMetadataMap[manifestDigest] = manifestMetadataMap[manifestDigest]
}
foundindexDataMap[descriptor.Digest] = indexData
default:
bdw.Log.Error().Msgf("Unsupported type: %s", descriptor.MediaType)
}
}
}
return nil
})
return foundRepos, foundManifestMetadataMap, pageInfo, err
return foundRepos, foundManifestMetadataMap, foundindexDataMap, pageInfo, err
}
func fetchManifestMetaWithCheck(repoMeta repodb.RepoMetadata, manifestDigest string,
manifestMetadataMap map[string]repodb.ManifestMetadata, manifestBuck *bolt.Bucket,
) (repodb.ManifestMetadata, error) {
manifestMeta, manifestDownloaded := manifestMetadataMap[manifestDigest]
if !manifestDownloaded {
var manifestData repodb.ManifestData
manifestDataBlob := manifestBuck.Get([]byte(manifestDigest))
if manifestDataBlob == nil {
return repodb.ManifestMetadata{}, zerr.ErrManifestMetaNotFound
}
err := json.Unmarshal(manifestDataBlob, &manifestData)
if err != nil {
return repodb.ManifestMetadata{}, errors.Wrapf(err,
"repodb: error while unmarshaling manifest metadata for digest %s", manifestDigest)
}
manifestMeta = NewManifestMetadata(manifestDigest, repoMeta, manifestData)
}
return manifestMeta, nil
}
func fetchIndexDataWithCheck(indexDigest string, indexDataMap map[string]repodb.IndexData,
indexBuck *bolt.Bucket,
) (repodb.IndexData, error) {
var (
indexData repodb.IndexData
err error
)
indexData, indexExists := indexDataMap[indexDigest]
if !indexExists {
indexDataBlob := indexBuck.Get([]byte(indexDigest))
if indexDataBlob == nil {
return repodb.IndexData{}, zerr.ErrIndexDataNotFount
}
err := json.Unmarshal(indexDataBlob, &indexData)
if err != nil {
return repodb.IndexData{},
errors.Wrapf(err, "repodb: error while unmashaling index data for digest %s", indexDigest)
}
}
return indexData, err
}
func collectImageManifestFilterData(digest string, repoMeta repodb.RepoMetadata,
manifestMeta repodb.ManifestMetadata,
) (repodb.FilterData, error) {
// get fields related to filtering
var (
configContent ispec.Image
osList []string
archList []string
)
err := json.Unmarshal(manifestMeta.ConfigBlob, &configContent)
if err != nil {
return repodb.FilterData{},
errors.Wrapf(err, "repodb: error while unmarshaling config content")
}
if configContent.OS != "" {
osList = append(osList, configContent.OS)
}
if configContent.Architecture != "" {
archList = append(archList, configContent.Architecture)
}
return repodb.FilterData{
DownloadCount: repoMeta.Statistics[digest].DownloadCount,
OsList: osList,
ArchList: archList,
LastUpdated: common.GetImageLastUpdatedTimestamp(configContent),
IsSigned: common.CheckIsSigned(repoMeta.Signatures[digest]),
}, nil
}
func collectImageIndexFilterInfo(indexDigest string, repoMeta repodb.RepoMetadata,
indexData repodb.IndexData, manifestMetadataMap map[string]repodb.ManifestMetadata,
manifestBuck *bolt.Bucket,
) (repodb.FilterData, error) {
var indexContent ispec.Index
err := json.Unmarshal(indexData.IndexBlob, &indexContent)
if err != nil {
return repodb.FilterData{},
errors.Wrapf(err, "repodb: error while unmarshaling index content for digest %s", indexDigest)
}
var (
indexLastUpdated time.Time
firstManifestChecked = false
indexOsList = []string{}
indexArchList = []string{}
)
for _, manifest := range indexContent.Manifests {
manifestDigest := manifest.Digest
manifestMeta, err := fetchManifestMetaWithCheck(repoMeta, manifestDigest.String(),
manifestMetadataMap, manifestBuck)
if err != nil {
return repodb.FilterData{},
errors.Wrapf(err, "")
}
manifestFilterData, err := collectImageManifestFilterData(manifestDigest.String(), repoMeta,
manifestMeta)
if err != nil {
return repodb.FilterData{},
errors.Wrapf(err, "")
}
indexOsList = append(indexOsList, manifestFilterData.OsList...)
indexArchList = append(indexArchList, manifestFilterData.ArchList...)
if !firstManifestChecked || indexLastUpdated.Before(manifestFilterData.LastUpdated) {
indexLastUpdated = manifestFilterData.LastUpdated
firstManifestChecked = true
}
manifestMetadataMap[manifest.Digest.String()] = manifestMeta
}
return repodb.FilterData{
DownloadCount: repoMeta.Statistics[indexDigest].DownloadCount,
LastUpdated: indexLastUpdated,
OsList: indexOsList,
ArchList: indexArchList,
IsSigned: common.CheckIsSigned(repoMeta.Signatures[indexDigest]),
}, nil
}
func NewManifestMetadata(manifestDigest string, repoMeta repodb.RepoMetadata,
manifestData repodb.ManifestData,
) repodb.ManifestMetadata {
manifestMeta := repodb.ManifestMetadata{
ManifestBlob: manifestData.ManifestBlob,
ConfigBlob: manifestData.ConfigBlob,
}
manifestMeta.DownloadCount = repoMeta.Statistics[manifestDigest].DownloadCount
manifestMeta.Signatures = repodb.ManifestSignatures{}
if repoMeta.Signatures[manifestDigest] != nil {
manifestMeta.Signatures = repoMeta.Signatures[manifestDigest]
}
return manifestMeta
}
func (bdw DBWrapper) FilterTags(ctx context.Context, filter repodb.FilterFunc,
requestedPage repodb.PageInput,
) ([]repodb.RepoMetadata, map[string]repodb.ManifestMetadata, repodb.PageInfo, error) {
) ([]repodb.RepoMetadata, map[string]repodb.ManifestMetadata, map[string]repodb.IndexData,
repodb.PageInfo, error,
) {
var (
foundRepos = make([]repodb.RepoMetadata, 0)
manifestMetadataMap = make(map[string]repodb.ManifestMetadata)
indexDataMap = make(map[string]repodb.IndexData)
foundManifestMetadataMap = make(map[string]repodb.ManifestMetadata)
foundindexDataMap = make(map[string]repodb.IndexData)
pageFinder repodb.PageFinder
pageInfo repodb.PageInfo
)
pageFinder, err := repodb.NewBaseImagePageFinder(requestedPage.Limit, requestedPage.Offset, requestedPage.SortBy)
if err != nil {
return []repodb.RepoMetadata{}, map[string]repodb.ManifestMetadata{}, repodb.PageInfo{}, err
return []repodb.RepoMetadata{}, map[string]repodb.ManifestMetadata{}, map[string]repodb.IndexData{},
repodb.PageInfo{}, err
}
err = bdw.DB.View(func(tx *bolt.Tx) error {
var (
manifestMetadataMap = make(map[string]repodb.ManifestMetadata)
repoBuck = tx.Bucket([]byte(repodb.RepoMetadataBucket))
dataBuck = tx.Bucket([]byte(repodb.ManifestDataBucket))
cursor = repoBuck.Cursor()
repoBuck = tx.Bucket([]byte(repodb.RepoMetadataBucket))
indexBuck = tx.Bucket([]byte(repodb.IndexDataBucket))
manifestBuck = tx.Bucket([]byte(repodb.ManifestDataBucket))
cursor = repoBuck.Cursor()
)
repoName, repoMetaBlob := cursor.First()
@@ -784,46 +1066,69 @@ func (bdw DBWrapper) FilterTags(ctx context.Context, filter repodb.FilterFunc,
matchedTags := make(map[string]repodb.Descriptor)
// take all manifestMetas
for tag, descriptor := range repoMeta.Tags {
manifestDigest := descriptor.Digest
matchedTags[tag] = descriptor
switch descriptor.MediaType {
case ispec.MediaTypeImageManifest:
manifestDigest := descriptor.Digest
// in case tags reference the same manifest we don't download from DB multiple times
manifestMeta, manifestExists := manifestMetadataMap[manifestDigest]
if !manifestExists {
manifestDataBlob := dataBuck.Get([]byte(manifestDigest))
if manifestDataBlob == nil {
return zerr.ErrManifestMetaNotFound
}
var manifestData repodb.ManifestData
err := json.Unmarshal(manifestDataBlob, &manifestData)
manifestMeta, err := fetchManifestMetaWithCheck(repoMeta, manifestDigest, manifestMetadataMap, manifestBuck)
if err != nil {
return errors.Wrapf(err, "repodb: error while unmashaling manifest metadata for digest %s", manifestDigest)
}
var configContent ispec.Image
if !filter(repoMeta, manifestMeta) {
delete(matchedTags, tag)
err = json.Unmarshal(manifestData.ConfigBlob, &configContent)
continue
}
manifestMetadataMap[manifestDigest] = manifestMeta
case ispec.MediaTypeImageIndex:
indexDigest := descriptor.Digest
indexData, err := fetchIndexDataWithCheck(indexDigest, indexDataMap, indexBuck)
if err != nil {
return errors.Wrapf(err, "repodb: error while unmashaling config for manifest with digest %s", manifestDigest)
return errors.Wrapf(err, "repodb: error while getting index data for digest %s", indexDigest)
}
manifestMeta = repodb.ManifestMetadata{
ConfigBlob: manifestData.ConfigBlob,
ManifestBlob: manifestData.ManifestBlob,
var indexContent ispec.Index
err = json.Unmarshal(indexData.IndexBlob, &indexContent)
if err != nil {
return errors.Wrapf(err, "repodb: error while unmashaling index content for digest %s", indexDigest)
}
manifestHasBeenMatched := false
for _, manifest := range indexContent.Manifests {
manifestDigest := manifest.Digest.String()
manifestMeta, err := fetchManifestMetaWithCheck(repoMeta, manifestDigest, manifestMetadataMap, manifestBuck)
if err != nil {
return errors.Wrapf(err, "repodb: error while getting manifest data for digest %s", manifestDigest)
}
manifestMetadataMap[manifestDigest] = manifestMeta
if filter(repoMeta, manifestMeta) {
manifestHasBeenMatched = true
}
}
if !manifestHasBeenMatched {
delete(matchedTags, tag)
for _, manifest := range indexContent.Manifests {
delete(manifestMetadataMap, manifest.Digest.String())
}
continue
}
indexDataMap[indexDigest] = indexData
default:
bdw.Log.Error().Msgf("Unsupported type: %s", descriptor.MediaType)
}
if !filter(repoMeta, manifestMeta) {
delete(matchedTags, tag)
continue
}
manifestMetadataMap[manifestDigest] = manifestMeta
}
if len(matchedTags) == 0 {
@@ -839,25 +1144,50 @@ func (bdw DBWrapper) FilterTags(ctx context.Context, filter repodb.FilterFunc,
foundRepos, pageInfo = pageFinder.Page()
// keep just the manifestMeta we need
// keep just the manifestMeta and indexData we need
for _, repoMeta := range foundRepos {
for _, descriptor := range repoMeta.Tags {
foundManifestMetadataMap[descriptor.Digest] = manifestMetadataMap[descriptor.Digest]
switch descriptor.MediaType {
case ispec.MediaTypeImageManifest:
foundManifestMetadataMap[descriptor.Digest] = manifestMetadataMap[descriptor.Digest]
case ispec.MediaTypeImageIndex:
indexData := indexDataMap[descriptor.Digest]
var indexContent ispec.Index
err := json.Unmarshal(indexData.IndexBlob, &indexContent)
if err != nil {
return err
}
for _, manifestDescriptor := range indexContent.Manifests {
manifestDigest := manifestDescriptor.Digest.String()
foundManifestMetadataMap[manifestDigest] = manifestMetadataMap[manifestDigest]
}
foundindexDataMap[descriptor.Digest] = indexData
default:
bdw.Log.Error().Msgf("Unsupported type: %s", descriptor.MediaType)
}
}
}
return nil
})
return foundRepos, foundManifestMetadataMap, pageInfo, err
return foundRepos, foundManifestMetadataMap, foundindexDataMap, pageInfo, err
}
func (bdw DBWrapper) SearchTags(ctx context.Context, searchText string, filter repodb.Filter,
requestedPage repodb.PageInput,
) ([]repodb.RepoMetadata, map[string]repodb.ManifestMetadata, repodb.PageInfo, error) {
) ([]repodb.RepoMetadata, map[string]repodb.ManifestMetadata, map[string]repodb.IndexData, repodb.PageInfo, error) {
var (
foundRepos = make([]repodb.RepoMetadata, 0)
manifestMetadataMap = make(map[string]repodb.ManifestMetadata)
indexDataMap = make(map[string]repodb.IndexData)
foundManifestMetadataMap = make(map[string]repodb.ManifestMetadata)
foundindexDataMap = make(map[string]repodb.IndexData)
pageInfo repodb.PageInfo
pageFinder repodb.PageFinder
@@ -865,21 +1195,23 @@ func (bdw DBWrapper) SearchTags(ctx context.Context, searchText string, filter r
pageFinder, err := repodb.NewBaseImagePageFinder(requestedPage.Limit, requestedPage.Offset, requestedPage.SortBy)
if err != nil {
return []repodb.RepoMetadata{}, map[string]repodb.ManifestMetadata{}, repodb.PageInfo{}, err
return []repodb.RepoMetadata{}, map[string]repodb.ManifestMetadata{}, map[string]repodb.IndexData{},
repodb.PageInfo{}, err
}
searchedRepo, searchedTag, err := common.GetRepoTag(searchText)
if err != nil {
return []repodb.RepoMetadata{}, map[string]repodb.ManifestMetadata{}, repodb.PageInfo{},
return []repodb.RepoMetadata{}, map[string]repodb.ManifestMetadata{}, map[string]repodb.IndexData{},
repodb.PageInfo{},
errors.Wrap(err, "repodb: error while parsing search text, invalid format")
}
err = bdw.DB.View(func(tx *bolt.Tx) error {
var (
manifestMetadataMap = make(map[string]repodb.ManifestMetadata)
repoBuck = tx.Bucket([]byte(repodb.RepoMetadataBucket))
dataBuck = tx.Bucket([]byte(repodb.ManifestDataBucket))
cursor = repoBuck.Cursor()
repoBuck = tx.Bucket([]byte(repodb.RepoMetadataBucket))
indexBuck = tx.Bucket([]byte(repodb.IndexDataBucket))
manifestBuck = tx.Bucket([]byte(repodb.ManifestDataBucket))
cursor = repoBuck.Cursor()
)
repoName, repoMetaBlob := cursor.Seek([]byte(searchedRepo))
@@ -906,46 +1238,84 @@ func (bdw DBWrapper) SearchTags(ctx context.Context, searchText string, filter r
matchedTags[tag] = descriptor
// in case tags reference the same manifest we don't download from DB multiple times
if manifestMeta, manifestExists := manifestMetadataMap[descriptor.Digest]; manifestExists {
switch descriptor.MediaType {
case ispec.MediaTypeImageManifest:
manifestDigest := descriptor.Digest
manifestMeta, err := fetchManifestMetaWithCheck(repoMeta, manifestDigest, manifestMetadataMap, manifestBuck)
if err != nil {
return errors.Wrapf(err, "repodb: error fetching manifest meta for manifest with digest %s",
manifestDigest)
}
imageFilterData, err := collectImageManifestFilterData(manifestDigest, repoMeta, manifestMeta)
if err != nil {
return errors.Wrapf(err, "repodb: error collecting filter data for manifest with digest %s",
manifestDigest)
}
if !common.AcceptedByFilter(filter, imageFilterData) {
delete(matchedTags, tag)
continue
}
manifestMetadataMap[descriptor.Digest] = manifestMeta
case ispec.MediaTypeImageIndex:
indexDigest := descriptor.Digest
continue
indexData, err := fetchIndexDataWithCheck(indexDigest, indexDataMap, indexBuck)
if err != nil {
return errors.Wrapf(err, "repodb: error fetching index data for index with digest %s",
indexDigest)
}
var indexContent ispec.Index
err = json.Unmarshal(indexData.IndexBlob, &indexContent)
if err != nil {
return errors.Wrapf(err, "repodb: error collecting filter data for index with digest %s",
indexDigest)
}
manifestHasBeenMatched := false
for _, manifest := range indexContent.Manifests {
manifestDigest := manifest.Digest.String()
manifestMeta, err := fetchManifestMetaWithCheck(repoMeta, manifestDigest, manifestMetadataMap, manifestBuck)
if err != nil {
return errors.Wrapf(err, "repodb: error fetching from db manifest meta for manifest with digest %s",
manifestDigest)
}
manifestFilterData, err := collectImageManifestFilterData(manifestDigest, repoMeta, manifestMeta)
if err != nil {
return errors.Wrapf(err, "repodb: error collecting filter data for manifest with digest %s",
manifestDigest)
}
manifestMetadataMap[manifestDigest] = manifestMeta
if common.AcceptedByFilter(filter, manifestFilterData) {
manifestHasBeenMatched = true
}
}
if !manifestHasBeenMatched {
delete(matchedTags, tag)
for _, manifest := range indexContent.Manifests {
delete(manifestMetadataMap, manifest.Digest.String())
}
continue
}
indexDataMap[indexDigest] = indexData
default:
bdw.Log.Error().Msgf("Unsupported type: %s", descriptor.MediaType)
}
manifestMetaBlob := dataBuck.Get([]byte(descriptor.Digest))
if manifestMetaBlob == nil {
return zerr.ErrManifestMetaNotFound
}
var manifestMeta repodb.ManifestMetadata
err := json.Unmarshal(manifestMetaBlob, &manifestMeta)
if err != nil {
return errors.Wrapf(err, "repodb: error while unmashaling manifest metadata for digest %s", descriptor.Digest)
}
var configContent ispec.Image
err = json.Unmarshal(manifestMeta.ConfigBlob, &configContent)
if err != nil {
return errors.Wrapf(err, "repodb: error while unmashaling config for manifest with digest %s", descriptor.Digest)
}
imageFilterData := repodb.FilterData{
OsList: []string{configContent.OS},
ArchList: []string{configContent.Architecture},
IsSigned: false,
}
if !common.AcceptedByFilter(filter, imageFilterData) {
delete(matchedTags, tag)
delete(manifestMetadataMap, descriptor.Digest)
continue
}
manifestMetadataMap[descriptor.Digest] = manifestMeta
}
if len(matchedTags) == 0 {
@@ -962,17 +1332,39 @@ func (bdw DBWrapper) SearchTags(ctx context.Context, searchText string, filter r
foundRepos, pageInfo = pageFinder.Page()
// keep just the manifestMeta we need
// keep just the manifestMeta and indexData we need
for _, repoMeta := range foundRepos {
for _, descriptor := range repoMeta.Tags {
foundManifestMetadataMap[descriptor.Digest] = manifestMetadataMap[descriptor.Digest]
switch descriptor.MediaType {
case ispec.MediaTypeImageManifest:
foundManifestMetadataMap[descriptor.Digest] = manifestMetadataMap[descriptor.Digest]
case ispec.MediaTypeImageIndex:
indexData := indexDataMap[descriptor.Digest]
var indexContent ispec.Index
err := json.Unmarshal(indexData.IndexBlob, &indexContent)
if err != nil {
return err
}
for _, manifestDescriptor := range indexContent.Manifests {
manifestDigest := manifestDescriptor.Digest.String()
foundManifestMetadataMap[manifestDigest] = manifestMetadataMap[manifestDigest]
}
foundindexDataMap[descriptor.Digest] = indexData
default:
bdw.Log.Error().Msgf("Unsupported type: %s", descriptor.MediaType)
}
}
}
return nil
})
return foundRepos, foundManifestMetadataMap, pageInfo, err
return foundRepos, foundManifestMetadataMap, foundindexDataMap, pageInfo, err
}
func (bdw *DBWrapper) PatchDB() error {
@@ -13,10 +13,12 @@ import (
"zotregistry.io/zot/pkg/meta/repodb"
bolt "zotregistry.io/zot/pkg/meta/repodb/boltdb-wrapper"
"zotregistry.io/zot/pkg/test"
)
func TestWrapperErrors(t *testing.T) {
Convey("Errors", t, func() {
ctx := context.Background()
tmpDir := t.TempDir()
boltDBParams := bolt.DBParameters{RootDir: tmpDir}
boltdbWrapper, err := bolt.NewBoltDBWrapper(boltDBParams)
@@ -300,7 +302,7 @@ func TestWrapperErrors(t *testing.T) {
})
So(err, ShouldBeNil)
_, _, _, err = boltdbWrapper.SearchRepos(context.Background(), "", repodb.Filter{}, repodb.PageInput{})
_, _, _, _, err = boltdbWrapper.SearchRepos(context.Background(), "", repodb.Filter{}, repodb.PageInput{})
So(err, ShouldNotBeNil)
err = boltdbWrapper.DB.Update(func(tx *bbolt.Tx) error {
@@ -339,10 +341,10 @@ func TestWrapperErrors(t *testing.T) {
})
So(err, ShouldBeNil)
_, _, _, err = boltdbWrapper.SearchRepos(context.Background(), "repo1", repodb.Filter{}, repodb.PageInput{})
_, _, _, _, err = boltdbWrapper.SearchRepos(context.Background(), "repo1", repodb.Filter{}, repodb.PageInput{})
So(err, ShouldNotBeNil)
_, _, _, err = boltdbWrapper.SearchRepos(context.Background(), "repo2", repodb.Filter{}, repodb.PageInput{})
_, _, _, _, err = boltdbWrapper.SearchRepos(context.Background(), "repo2", repodb.Filter{}, repodb.PageInput{})
So(err, ShouldNotBeNil)
err = boltdbWrapper.DB.Update(func(tx *bbolt.Tx) error {
@@ -378,10 +380,85 @@ func TestWrapperErrors(t *testing.T) {
})
So(err, ShouldBeNil)
_, _, _, err = boltdbWrapper.SearchRepos(context.Background(), "repo1", repodb.Filter{}, repodb.PageInput{})
_, _, _, _, err = boltdbWrapper.SearchRepos(context.Background(), "repo1", repodb.Filter{}, repodb.PageInput{})
So(err, ShouldNotBeNil)
})
Convey("Index Errors", func() {
Convey("Bad index data", func() {
indexDigest := digest.FromString("indexDigest")
err := boltdbWrapper.SetRepoTag("repo", "tag1", indexDigest, ispec.MediaTypeImageIndex) //nolint:contextcheck
So(err, ShouldBeNil)
err = setBadIndexData(boltdbWrapper.DB, indexDigest.String())
So(err, ShouldBeNil)
_, _, _, _, err = boltdbWrapper.SearchRepos(ctx, "", repodb.Filter{}, repodb.PageInput{})
So(err, ShouldNotBeNil)
_, _, _, _, err = boltdbWrapper.SearchTags(ctx, "repo:", repodb.Filter{}, repodb.PageInput{})
So(err, ShouldNotBeNil)
})
Convey("Bad indexBlob in IndexData", func() {
indexDigest := digest.FromString("indexDigest")
err := boltdbWrapper.SetRepoTag("repo", "tag1", indexDigest, ispec.MediaTypeImageIndex) //nolint:contextcheck
So(err, ShouldBeNil)
err = boltdbWrapper.SetIndexData(indexDigest, repodb.IndexData{
IndexBlob: []byte("bad json"),
})
So(err, ShouldBeNil)
_, _, _, _, err = boltdbWrapper.SearchRepos(ctx, "", repodb.Filter{}, repodb.PageInput{})
So(err, ShouldNotBeNil)
_, _, _, _, err = boltdbWrapper.SearchTags(ctx, "repo:", repodb.Filter{}, repodb.PageInput{})
So(err, ShouldNotBeNil)
})
Convey("Good index data, bad manifest inside index", func() {
var (
indexDigest = digest.FromString("indexDigest")
manifestDigestFromIndex1 = digest.FromString("manifestDigestFromIndex1")
manifestDigestFromIndex2 = digest.FromString("manifestDigestFromIndex2")
)
err := boltdbWrapper.SetRepoTag("repo", "tag1", indexDigest, ispec.MediaTypeImageIndex) //nolint:contextcheck
So(err, ShouldBeNil)
indexBlob, err := test.GetIndexBlobWithManifests([]digest.Digest{
manifestDigestFromIndex1, manifestDigestFromIndex2,
})
So(err, ShouldBeNil)
err = boltdbWrapper.SetIndexData(indexDigest, repodb.IndexData{
IndexBlob: indexBlob,
})
So(err, ShouldBeNil)
err = boltdbWrapper.SetManifestData(manifestDigestFromIndex1, repodb.ManifestData{
ManifestBlob: []byte("Bad Manifest"),
ConfigBlob: []byte("Bad Manifest"),
})
So(err, ShouldBeNil)
err = boltdbWrapper.SetManifestData(manifestDigestFromIndex2, repodb.ManifestData{
ManifestBlob: []byte("Bad Manifest"),
ConfigBlob: []byte("Bad Manifest"),
})
So(err, ShouldBeNil)
_, _, _, _, err = boltdbWrapper.SearchRepos(ctx, "", repodb.Filter{}, repodb.PageInput{})
So(err, ShouldNotBeNil)
_, _, _, _, err = boltdbWrapper.SearchTags(ctx, "repo:", repodb.Filter{}, repodb.PageInput{})
So(err, ShouldNotBeNil)
})
})
Convey("SearchTags", func() {
ctx := context.Background()
@@ -392,10 +469,10 @@ func TestWrapperErrors(t *testing.T) {
})
So(err, ShouldBeNil)
_, _, _, err = boltdbWrapper.SearchTags(ctx, "", repodb.Filter{}, repodb.PageInput{})
_, _, _, _, err = boltdbWrapper.SearchTags(ctx, "", repodb.Filter{}, repodb.PageInput{})
So(err, ShouldNotBeNil)
_, _, _, err = boltdbWrapper.SearchTags(ctx, "repo1:", repodb.Filter{}, repodb.PageInput{})
_, _, _, _, err = boltdbWrapper.SearchTags(ctx, "repo1:", repodb.Filter{}, repodb.PageInput{})
So(err, ShouldNotBeNil)
err = boltdbWrapper.DB.Update(func(tx *bbolt.Tx) error {
@@ -466,14 +543,117 @@ func TestWrapperErrors(t *testing.T) {
})
So(err, ShouldBeNil)
_, _, _, err = boltdbWrapper.SearchTags(ctx, "repo1:", repodb.Filter{}, repodb.PageInput{})
_, _, _, _, err = boltdbWrapper.SearchTags(ctx, "repo1:", repodb.Filter{}, repodb.PageInput{})
So(err, ShouldNotBeNil)
_, _, _, err = boltdbWrapper.SearchTags(ctx, "repo2:", repodb.Filter{}, repodb.PageInput{})
_, _, _, _, err = boltdbWrapper.SearchTags(ctx, "repo2:", repodb.Filter{}, repodb.PageInput{})
So(err, ShouldNotBeNil)
_, _, _, err = boltdbWrapper.SearchTags(ctx, "repo3:", repodb.Filter{}, repodb.PageInput{})
_, _, _, _, err = boltdbWrapper.SearchTags(ctx, "repo3:", repodb.Filter{}, repodb.PageInput{})
So(err, ShouldNotBeNil)
})
Convey("FilterTags Index errors", func() {
Convey("FilterTags bad IndexData", func() {
indexDigest := digest.FromString("indexDigest")
err := boltdbWrapper.SetRepoTag("repo", "tag1", indexDigest, ispec.MediaTypeImageIndex) //nolint:contextcheck
So(err, ShouldBeNil)
err = setBadIndexData(boltdbWrapper.DB, indexDigest.String())
So(err, ShouldBeNil)
_, _, _, _, err = boltdbWrapper.FilterTags(ctx,
func(repoMeta repodb.RepoMetadata, manifestMeta repodb.ManifestMetadata) bool { return true },
repodb.PageInput{},
)
So(err, ShouldNotBeNil)
})
Convey("FilterTags bad indexBlob in IndexData", func() {
indexDigest := digest.FromString("indexDigest")
err := boltdbWrapper.SetRepoTag("repo", "tag1", indexDigest, ispec.MediaTypeImageIndex) //nolint:contextcheck
So(err, ShouldBeNil)
err = boltdbWrapper.SetIndexData(indexDigest, repodb.IndexData{
IndexBlob: []byte("bad json"),
})
So(err, ShouldBeNil)
_, _, _, _, err = boltdbWrapper.FilterTags(ctx,
func(repoMeta repodb.RepoMetadata, manifestMeta repodb.ManifestMetadata) bool { return true },
repodb.PageInput{},
)
So(err, ShouldNotBeNil)
})
Convey("FilterTags didn't match any index manifest", func() {
var (
indexDigest = digest.FromString("indexDigest")
manifestDigestFromIndex1 = digest.FromString("manifestDigestFromIndex1")
manifestDigestFromIndex2 = digest.FromString("manifestDigestFromIndex2")
)
err := boltdbWrapper.SetRepoTag("repo", "tag1", indexDigest, ispec.MediaTypeImageIndex) //nolint:contextcheck
So(err, ShouldBeNil)
indexBlob, err := test.GetIndexBlobWithManifests([]digest.Digest{
manifestDigestFromIndex1, manifestDigestFromIndex2,
})
So(err, ShouldBeNil)
err = boltdbWrapper.SetIndexData(indexDigest, repodb.IndexData{
IndexBlob: indexBlob,
})
So(err, ShouldBeNil)
err = boltdbWrapper.SetManifestData(manifestDigestFromIndex1, repodb.ManifestData{
ManifestBlob: []byte("{}"),
ConfigBlob: []byte("{}"),
})
So(err, ShouldBeNil)
err = boltdbWrapper.SetManifestData(manifestDigestFromIndex2, repodb.ManifestData{
ManifestBlob: []byte("{}"),
ConfigBlob: []byte("{}"),
})
So(err, ShouldBeNil)
_, _, _, _, err = boltdbWrapper.FilterTags(ctx,
func(repoMeta repodb.RepoMetadata, manifestMeta repodb.ManifestMetadata) bool { return false },
repodb.PageInput{},
)
So(err, ShouldBeNil)
})
})
Convey("Unsuported type", func() {
digest := digest.FromString("digest")
err := boltdbWrapper.SetRepoTag("repo", "tag1", digest, "invalid type") //nolint:contextcheck
So(err, ShouldBeNil)
_, _, _, _, err = boltdbWrapper.SearchRepos(ctx, "", repodb.Filter{}, repodb.PageInput{})
So(err, ShouldBeNil)
_, _, _, _, err = boltdbWrapper.SearchTags(ctx, "repo:", repodb.Filter{}, repodb.PageInput{})
So(err, ShouldBeNil)
_, _, _, _, err = boltdbWrapper.FilterTags(
ctx,
func(repoMeta repodb.RepoMetadata, manifestMeta repodb.ManifestMetadata) bool { return true },
repodb.PageInput{},
)
So(err, ShouldBeNil)
})
})
}
func setBadIndexData(dB *bbolt.DB, digest string) error {
return dB.Update(func(tx *bbolt.Tx) error {
indexDataBuck := tx.Bucket([]byte(repodb.IndexDataBucket))
return indexDataBuck.Put([]byte(digest), []byte("bad json"))
})
}
@@ -64,6 +64,9 @@ func TestWrapperErrors(t *testing.T) {
err = dynamoWrapper.createManifestDataTable()
So(err, ShouldNotBeNil)
err = dynamoWrapper.createIndexDataTable()
So(err, ShouldNotBeNil)
err = dynamoWrapper.createVersionTable()
So(err, ShouldNotBeNil)
})
+328 -35
View File
@@ -12,6 +12,8 @@ import (
"github.com/aws/aws-sdk-go-v2/service/dynamodb"
"github.com/aws/aws-sdk-go-v2/service/dynamodb/types"
guuid "github.com/gofrs/uuid"
"github.com/opencontainers/go-digest"
ispec "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/rs/zerolog"
. "github.com/smartystreets/goconvey/convey"
@@ -20,6 +22,7 @@ import (
dynamo "zotregistry.io/zot/pkg/meta/repodb/dynamodb-wrapper"
"zotregistry.io/zot/pkg/meta/repodb/dynamodb-wrapper/iterator"
dynamoParams "zotregistry.io/zot/pkg/meta/repodb/dynamodb-wrapper/params"
"zotregistry.io/zot/pkg/test"
)
func TestIterator(t *testing.T) {
@@ -36,6 +39,7 @@ func TestIterator(t *testing.T) {
repoMetaTablename := "RepoMetadataTable" + uuid.String()
manifestDataTablename := "ManifestDataTable" + uuid.String()
versionTablename := "Version" + uuid.String()
indexDataTablename := "IndexDataTable" + uuid.String()
Convey("TestIterator", t, func() {
dynamoWrapper, err := dynamo.NewDynamoDBWrapper(dynamoParams.DBDriverParameters{
@@ -43,6 +47,7 @@ func TestIterator(t *testing.T) {
Region: region,
RepoMetaTablename: repoMetaTablename,
ManifestDataTablename: manifestDataTablename,
IndexDataTablename: indexDataTablename,
VersionTablename: versionTablename,
})
So(err, ShouldBeNil)
@@ -127,6 +132,7 @@ func TestWrapperErrors(t *testing.T) {
repoMetaTablename := "RepoMetadataTable" + uuid.String()
manifestDataTablename := "ManifestDataTable" + uuid.String()
versionTablename := "Version" + uuid.String()
indexDataTablename := "IndexDataTable" + uuid.String()
ctx := context.Background()
@@ -136,6 +142,7 @@ func TestWrapperErrors(t *testing.T) {
Region: region,
RepoMetaTablename: repoMetaTablename,
ManifestDataTablename: manifestDataTablename,
IndexDataTablename: indexDataTablename,
VersionTablename: versionTablename,
})
So(err, ShouldBeNil)
@@ -144,7 +151,7 @@ func TestWrapperErrors(t *testing.T) {
So(dynamoWrapper.ResetRepoMetaTable(), ShouldBeNil) //nolint:contextcheck
Convey("SetManifestData", func() {
dynamoWrapper.ManifestDataTablename = "WRONG table"
dynamoWrapper.ManifestDataTablename = "WRONG tables"
err := dynamoWrapper.SetManifestData("dig", repodb.ManifestData{})
So(err, ShouldNotBeNil)
@@ -165,6 +172,21 @@ func TestWrapperErrors(t *testing.T) {
So(err, ShouldNotBeNil)
})
Convey("GetIndexData", func() {
dynamoWrapper.IndexDataTablename = "WRONG table"
_, err := dynamoWrapper.GetIndexData("dig")
So(err, ShouldNotBeNil)
})
Convey("GetIndexData unmarshal error", func() {
err := setBadIndexData(dynamoWrapper.Client, indexDataTablename, "dig")
So(err, ShouldBeNil)
_, err = dynamoWrapper.GetManifestData("dig")
So(err, ShouldNotBeNil)
})
Convey("SetManifestMeta GetRepoMeta error", func() {
err := setBadRepoMeta(dynamoWrapper.Client, repoMetaTablename, "repo1")
So(err, ShouldBeNil)
@@ -255,14 +277,6 @@ func TestWrapperErrors(t *testing.T) {
So(err, ShouldNotBeNil)
})
Convey("IncrementImageDownloads GetManifestMeta error", func() {
err := dynamoWrapper.SetRepoTag("repo", "tag", "dig", "")
So(err, ShouldBeNil)
err = dynamoWrapper.IncrementImageDownloads("repo", "tag")
So(err, ShouldNotBeNil)
})
Convey("AddManifestSignature GetRepoMeta error", func() {
err := dynamoWrapper.SetRepoTag("repo", "tag", "dig", "")
So(err, ShouldBeNil)
@@ -329,22 +343,22 @@ func TestWrapperErrors(t *testing.T) {
err = setBadRepoMeta(dynamoWrapper.Client, repoMetaTablename, "repo") //nolint:contextcheck
So(err, ShouldBeNil)
_, _, _, err = dynamoWrapper.SearchRepos(ctx, "", repodb.Filter{}, repodb.PageInput{})
_, _, _, _, err = dynamoWrapper.SearchRepos(ctx, "", repodb.Filter{}, repodb.PageInput{})
So(err, ShouldNotBeNil)
})
Convey("SearchRepos GetManifestMeta error", func() {
err := dynamoWrapper.SetRepoTag("repo", "tag1", "notFoundDigest", "") //nolint:contextcheck
err := dynamoWrapper.SetRepoTag("repo", "tag1", "notFoundDigest", ispec.MediaTypeImageManifest) //nolint:contextcheck
So(err, ShouldBeNil)
_, _, _, err = dynamoWrapper.SearchRepos(ctx, "", repodb.Filter{}, repodb.PageInput{})
_, _, _, _, err = dynamoWrapper.SearchRepos(ctx, "", repodb.Filter{}, repodb.PageInput{})
So(err, ShouldNotBeNil)
})
Convey("SearchRepos config unmarshal error", func() {
err := dynamoWrapper.SetRepoTag("repo", "tag1", "dig1", "") //nolint:contextcheck
err := dynamoWrapper.SetRepoTag("repo", "tag1", "dig1", ispec.MediaTypeImageManifest) //nolint:contextcheck
So(err, ShouldBeNil)
err = dynamoWrapper.SetManifestData("dig1", repodb.ManifestData{ //nolint:contextcheck
@@ -353,31 +367,116 @@ func TestWrapperErrors(t *testing.T) {
})
So(err, ShouldBeNil)
_, _, _, err = dynamoWrapper.SearchRepos(ctx, "", repodb.Filter{}, repodb.PageInput{})
_, _, _, _, err = dynamoWrapper.SearchRepos(ctx, "", repodb.Filter{}, repodb.PageInput{})
So(err, ShouldNotBeNil)
})
Convey("Unsuported type", func() {
digest := digest.FromString("digest")
err := dynamoWrapper.SetRepoTag("repo", "tag1", digest, "invalid type") //nolint:contextcheck
So(err, ShouldBeNil)
_, _, _, _, err = dynamoWrapper.SearchRepos(ctx, "", repodb.Filter{}, repodb.PageInput{})
So(err, ShouldBeNil)
_, _, _, _, err = dynamoWrapper.SearchTags(ctx, "repo:", repodb.Filter{}, repodb.PageInput{})
So(err, ShouldBeNil)
_, _, _, _, err = dynamoWrapper.FilterTags(
ctx,
func(repoMeta repodb.RepoMetadata, manifestMeta repodb.ManifestMetadata) bool { return true },
repodb.PageInput{},
)
So(err, ShouldBeNil)
})
Convey("SearchRepos bad index data", func() {
indexDigest := digest.FromString("indexDigest")
err := dynamoWrapper.SetRepoTag("repo", "tag1", indexDigest, ispec.MediaTypeImageIndex) //nolint:contextcheck
So(err, ShouldBeNil)
err = setBadIndexData(dynamoWrapper.Client, indexDataTablename, indexDigest.String()) //nolint:contextcheck
So(err, ShouldBeNil)
_, _, _, _, err = dynamoWrapper.SearchRepos(ctx, "", repodb.Filter{}, repodb.PageInput{})
So(err, ShouldNotBeNil)
})
Convey("SearchRepos bad indexBlob in IndexData", func() {
indexDigest := digest.FromString("indexDigest")
err := dynamoWrapper.SetRepoTag("repo", "tag1", indexDigest, ispec.MediaTypeImageIndex) //nolint:contextcheck
So(err, ShouldBeNil)
err = dynamoWrapper.SetIndexData(indexDigest, repodb.IndexData{ //nolint:contextcheck
IndexBlob: []byte("bad json"),
})
So(err, ShouldBeNil)
_, _, _, _, err = dynamoWrapper.SearchRepos(ctx, "", repodb.Filter{}, repodb.PageInput{})
So(err, ShouldNotBeNil)
})
Convey("SearchRepos good index data, bad manifest inside index", func() {
var (
indexDigest = digest.FromString("indexDigest")
manifestDigestFromIndex1 = digest.FromString("manifestDigestFromIndex1")
manifestDigestFromIndex2 = digest.FromString("manifestDigestFromIndex2")
)
err := dynamoWrapper.SetRepoTag("repo", "tag1", indexDigest, ispec.MediaTypeImageIndex) //nolint:contextcheck
So(err, ShouldBeNil)
indexBlob, err := test.GetIndexBlobWithManifests([]digest.Digest{
manifestDigestFromIndex1, manifestDigestFromIndex2,
})
So(err, ShouldBeNil)
err = dynamoWrapper.SetIndexData(indexDigest, repodb.IndexData{ //nolint:contextcheck
IndexBlob: indexBlob,
})
So(err, ShouldBeNil)
err = dynamoWrapper.SetManifestData(manifestDigestFromIndex1, repodb.ManifestData{ //nolint:contextcheck
ManifestBlob: []byte("Bad Manifest"),
ConfigBlob: []byte("Bad Manifest"),
})
So(err, ShouldBeNil)
err = dynamoWrapper.SetManifestData(manifestDigestFromIndex2, repodb.ManifestData{ //nolint:contextcheck
ManifestBlob: []byte("Bad Manifest"),
ConfigBlob: []byte("Bad Manifest"),
})
So(err, ShouldBeNil)
_, _, _, _, err = dynamoWrapper.SearchRepos(ctx, "", repodb.Filter{}, repodb.PageInput{})
So(err, ShouldNotBeNil)
})
Convey("SearchTags repoMeta unmarshal error", func() {
err = setBadRepoMeta(dynamoWrapper.Client, repoMetaTablename, "repo") //nolint:contextcheck
So(err, ShouldBeNil)
_, _, _, err = dynamoWrapper.SearchTags(ctx, "repo:", repodb.Filter{}, repodb.PageInput{})
_, _, _, _, err = dynamoWrapper.SearchTags(ctx, "repo:", repodb.Filter{}, repodb.PageInput{})
So(err, ShouldNotBeNil)
})
Convey("SearchTags GetManifestMeta error", func() {
err := dynamoWrapper.SetRepoTag("repo", "tag1", "manifestNotFound", "") //nolint:contextcheck
err := dynamoWrapper.SetRepoTag("repo", "tag1", "manifestNotFound", //nolint:contextcheck
ispec.MediaTypeImageManifest)
So(err, ShouldBeNil)
_, _, _, err = dynamoWrapper.SearchTags(ctx, "repo:", repodb.Filter{}, repodb.PageInput{})
_, _, _, _, err = dynamoWrapper.SearchTags(ctx, "repo:", repodb.Filter{}, repodb.PageInput{})
So(err, ShouldNotBeNil)
})
Convey("SearchTags config unmarshal error", func() {
err := dynamoWrapper.SetRepoTag("repo", "tag1", "dig1", "") //nolint:contextcheck
err := dynamoWrapper.SetRepoTag("repo", "tag1", "dig1", ispec.MediaTypeImageManifest) //nolint:contextcheck
So(err, ShouldBeNil)
err = dynamoWrapper.SetManifestData( //nolint:contextcheck
@@ -389,16 +488,80 @@ func TestWrapperErrors(t *testing.T) {
)
So(err, ShouldBeNil)
_, _, _, err = dynamoWrapper.SearchTags(ctx, "repo:", repodb.Filter{}, repodb.PageInput{})
_, _, _, _, err = dynamoWrapper.SearchTags(ctx, "repo:", repodb.Filter{}, repodb.PageInput{})
So(err, ShouldNotBeNil)
})
Convey("SearchTags bad index data", func() {
indexDigest := digest.FromString("indexDigest")
err := dynamoWrapper.SetRepoTag("repo", "tag1", indexDigest, ispec.MediaTypeImageIndex) //nolint:contextcheck
So(err, ShouldBeNil)
err = setBadIndexData(dynamoWrapper.Client, indexDataTablename, indexDigest.String()) //nolint:contextcheck
So(err, ShouldBeNil)
_, _, _, _, err = dynamoWrapper.SearchTags(ctx, "repo:", repodb.Filter{}, repodb.PageInput{})
So(err, ShouldNotBeNil)
})
Convey("SearchTags bad indexBlob in IndexData", func() {
indexDigest := digest.FromString("indexDigest")
err := dynamoWrapper.SetRepoTag("repo", "tag1", indexDigest, ispec.MediaTypeImageIndex) //nolint:contextcheck
So(err, ShouldBeNil)
err = dynamoWrapper.SetIndexData(indexDigest, repodb.IndexData{ //nolint:contextcheck
IndexBlob: []byte("bad json"),
})
So(err, ShouldBeNil)
_, _, _, _, err = dynamoWrapper.SearchTags(ctx, "repo:", repodb.Filter{}, repodb.PageInput{})
So(err, ShouldNotBeNil)
})
Convey("SearchTags good index data, bad manifest inside index", func() {
var (
indexDigest = digest.FromString("indexDigest")
manifestDigestFromIndex1 = digest.FromString("manifestDigestFromIndex1")
manifestDigestFromIndex2 = digest.FromString("manifestDigestFromIndex2")
)
err := dynamoWrapper.SetRepoTag("repo", "tag1", indexDigest, ispec.MediaTypeImageIndex) //nolint:contextcheck
So(err, ShouldBeNil)
indexBlob, err := test.GetIndexBlobWithManifests([]digest.Digest{
manifestDigestFromIndex1, manifestDigestFromIndex2,
})
So(err, ShouldBeNil)
err = dynamoWrapper.SetIndexData(indexDigest, repodb.IndexData{ //nolint:contextcheck
IndexBlob: indexBlob,
})
So(err, ShouldBeNil)
err = dynamoWrapper.SetManifestData(manifestDigestFromIndex1, repodb.ManifestData{ //nolint:contextcheck
ManifestBlob: []byte("Bad Manifest"),
ConfigBlob: []byte("Bad Manifest"),
})
So(err, ShouldBeNil)
err = dynamoWrapper.SetManifestData(manifestDigestFromIndex2, repodb.ManifestData{ //nolint:contextcheck
ManifestBlob: []byte("Bad Manifest"),
ConfigBlob: []byte("Bad Manifest"),
})
So(err, ShouldBeNil)
_, _, _, _, err = dynamoWrapper.SearchTags(ctx, "repo:", repodb.Filter{}, repodb.PageInput{})
So(err, ShouldNotBeNil)
})
Convey("FilterTags repoMeta unmarshal error", func() {
err = setBadRepoMeta(dynamoWrapper.Client, repoMetaTablename, "repo") //nolint:contextcheck
So(err, ShouldBeNil)
_, _, _, err = dynamoWrapper.FilterTags(
_, _, _, _, err = dynamoWrapper.FilterTags(
ctx,
func(repoMeta repodb.RepoMetadata, manifestMeta repodb.ManifestMetadata) bool {
return true
@@ -410,10 +573,11 @@ func TestWrapperErrors(t *testing.T) {
})
Convey("FilterTags manifestMeta not found", func() {
err := dynamoWrapper.SetRepoTag("repo", "tag1", "manifestNotFound", "") //nolint:contextcheck
err := dynamoWrapper.SetRepoTag("repo", "tag1", "manifestNotFound", //nolint:contextcheck
ispec.MediaTypeImageManifest)
So(err, ShouldBeNil)
_, _, _, err = dynamoWrapper.FilterTags(
_, _, _, _, err = dynamoWrapper.FilterTags(
ctx,
func(repoMeta repodb.RepoMetadata, manifestMeta repodb.ManifestMetadata) bool {
return true
@@ -425,13 +589,13 @@ func TestWrapperErrors(t *testing.T) {
})
Convey("FilterTags manifestMeta unmarshal error", func() {
err := dynamoWrapper.SetRepoTag("repo", "tag1", "dig", "") //nolint:contextcheck
err := dynamoWrapper.SetRepoTag("repo", "tag1", "dig", ispec.MediaTypeImageManifest) //nolint:contextcheck
So(err, ShouldBeNil)
err = setBadManifestData(dynamoWrapper.Client, manifestDataTablename, "dig") //nolint:contextcheck
So(err, ShouldBeNil)
_, _, _, err = dynamoWrapper.FilterTags(
_, _, _, _, err = dynamoWrapper.FilterTags(
ctx,
func(repoMeta repodb.RepoMetadata, manifestMeta repodb.ManifestMetadata) bool {
return true
@@ -442,26 +606,130 @@ func TestWrapperErrors(t *testing.T) {
So(err, ShouldNotBeNil)
})
Convey("FilterTags config unmarshal error", func() {
err := dynamoWrapper.SetRepoTag("repo", "tag1", "dig1", "") //nolint:contextcheck
Convey("FilterTags bad IndexData", func() {
indexDigest := digest.FromString("indexDigest")
err := dynamoWrapper.SetRepoTag("repo", "tag1", indexDigest, ispec.MediaTypeImageIndex) //nolint:contextcheck
So(err, ShouldBeNil)
err = dynamoWrapper.SetManifestData("dig1", repodb.ManifestData{ //nolint:contextcheck
ManifestBlob: []byte("{}"),
ConfigBlob: []byte("bad json"),
err = setBadIndexData(dynamoWrapper.Client, indexDataTablename, indexDigest.String()) //nolint:contextcheck
So(err, ShouldBeNil)
_, _, _, _, err = dynamoWrapper.FilterTags(ctx,
func(repoMeta repodb.RepoMetadata, manifestMeta repodb.ManifestMetadata) bool { return true },
repodb.PageInput{},
)
So(err, ShouldNotBeNil)
})
Convey("FilterTags bad indexBlob in IndexData", func() {
indexDigest := digest.FromString("indexDigest")
err := dynamoWrapper.SetRepoTag("repo", "tag1", indexDigest, ispec.MediaTypeImageIndex) //nolint:contextcheck
So(err, ShouldBeNil)
err = dynamoWrapper.SetIndexData(indexDigest, repodb.IndexData{ //nolint:contextcheck
IndexBlob: []byte("bad json"),
})
So(err, ShouldBeNil)
_, _, _, err = dynamoWrapper.FilterTags(
ctx,
func(repoMeta repodb.RepoMetadata, manifestMeta repodb.ManifestMetadata) bool {
return true
},
_, _, _, _, err = dynamoWrapper.FilterTags(ctx,
func(repoMeta repodb.RepoMetadata, manifestMeta repodb.ManifestMetadata) bool { return true },
repodb.PageInput{},
)
So(err, ShouldNotBeNil)
})
Convey("FilterTags didn't match any index manifest", func() {
var (
indexDigest = digest.FromString("indexDigest")
manifestDigestFromIndex1 = digest.FromString("manifestDigestFromIndex1")
manifestDigestFromIndex2 = digest.FromString("manifestDigestFromIndex2")
)
err := dynamoWrapper.SetRepoTag("repo", "tag1", indexDigest, ispec.MediaTypeImageIndex) //nolint:contextcheck
So(err, ShouldBeNil)
indexBlob, err := test.GetIndexBlobWithManifests([]digest.Digest{
manifestDigestFromIndex1, manifestDigestFromIndex2,
})
So(err, ShouldBeNil)
err = dynamoWrapper.SetIndexData(indexDigest, repodb.IndexData{ //nolint:contextcheck
IndexBlob: indexBlob,
})
So(err, ShouldBeNil)
err = dynamoWrapper.SetManifestData(manifestDigestFromIndex1, repodb.ManifestData{ //nolint:contextcheck
ManifestBlob: []byte("{}"),
ConfigBlob: []byte("{}"),
})
So(err, ShouldBeNil)
err = dynamoWrapper.SetManifestData(manifestDigestFromIndex2, repodb.ManifestData{ //nolint:contextcheck
ManifestBlob: []byte("{}"),
ConfigBlob: []byte("{}"),
})
So(err, ShouldBeNil)
_, _, _, _, err = dynamoWrapper.FilterTags(ctx,
func(repoMeta repodb.RepoMetadata, manifestMeta repodb.ManifestMetadata) bool { return false },
repodb.PageInput{},
)
So(err, ShouldBeNil)
})
})
Convey("NewDynamoDBWrapper errors", t, func() {
_, err := dynamo.NewDynamoDBWrapper(dynamoParams.DBDriverParameters{ //nolint:contextcheck
Endpoint: endpoint,
Region: region,
RepoMetaTablename: "",
ManifestDataTablename: manifestDataTablename,
IndexDataTablename: indexDataTablename,
VersionTablename: versionTablename,
})
So(err, ShouldNotBeNil)
_, err = dynamo.NewDynamoDBWrapper(dynamoParams.DBDriverParameters{ //nolint:contextcheck
Endpoint: endpoint,
Region: region,
RepoMetaTablename: repoMetaTablename,
ManifestDataTablename: "",
IndexDataTablename: indexDataTablename,
VersionTablename: versionTablename,
})
So(err, ShouldNotBeNil)
_, err = dynamo.NewDynamoDBWrapper(dynamoParams.DBDriverParameters{ //nolint:contextcheck
Endpoint: endpoint,
Region: region,
RepoMetaTablename: repoMetaTablename,
ManifestDataTablename: manifestDataTablename,
IndexDataTablename: "",
VersionTablename: versionTablename,
})
So(err, ShouldNotBeNil)
_, err = dynamo.NewDynamoDBWrapper(dynamoParams.DBDriverParameters{ //nolint:contextcheck
Endpoint: endpoint,
Region: region,
RepoMetaTablename: repoMetaTablename,
ManifestDataTablename: manifestDataTablename,
IndexDataTablename: indexDataTablename,
VersionTablename: "",
})
So(err, ShouldNotBeNil)
_, err = dynamo.NewDynamoDBWrapper(dynamoParams.DBDriverParameters{ //nolint:contextcheck
Endpoint: endpoint,
Region: region,
RepoMetaTablename: repoMetaTablename,
ManifestDataTablename: manifestDataTablename,
IndexDataTablename: indexDataTablename,
VersionTablename: versionTablename,
})
So(err, ShouldBeNil)
})
}
@@ -490,6 +758,31 @@ func setBadManifestData(client *dynamodb.Client, manifestDataTableName, digest s
return err
}
func setBadIndexData(client *dynamodb.Client, indexDataTableName, digest string) error {
mdAttributeValue, err := attributevalue.Marshal("string")
if err != nil {
return err
}
_, err = client.UpdateItem(context.TODO(), &dynamodb.UpdateItemInput{
ExpressionAttributeNames: map[string]string{
"#ID": "IndexData",
},
ExpressionAttributeValues: map[string]types.AttributeValue{
":IndexData": mdAttributeValue,
},
Key: map[string]types.AttributeValue{
"IndexDigest": &types.AttributeValueMemberS{
Value: digest,
},
},
TableName: aws.String(indexDataTableName),
UpdateExpression: aws.String("SET #ID = :IndexData"),
})
return err
}
func setBadRepoMeta(client *dynamodb.Client, repoMetadataTableName, repoName string) error {
repoAttributeValue, err := attributevalue.Marshal("string")
if err != nil {
+505 -128
View File
@@ -30,6 +30,7 @@ import (
type DBWrapper struct {
Client *dynamodb.Client
RepoMetaTablename string
IndexDataTablename string
ManifestDataTablename string
VersionTablename string
Patches []func(client *dynamodb.Client, tableNames map[string]string) error
@@ -60,6 +61,7 @@ func NewDynamoDBWrapper(params dynamoParams.DBDriverParameters) (*DBWrapper, err
Client: dynamodb.NewFromConfig(cfg),
RepoMetaTablename: params.RepoMetaTablename,
ManifestDataTablename: params.ManifestDataTablename,
IndexDataTablename: params.IndexDataTablename,
VersionTablename: params.VersionTablename,
Patches: version.GetDynamoDBPatches(),
Log: log.Logger{Logger: zerolog.New(os.Stdout)},
@@ -80,6 +82,11 @@ func NewDynamoDBWrapper(params dynamoParams.DBDriverParameters) (*DBWrapper, err
return nil, err
}
err = dynamoWrapper.createIndexDataTable()
if err != nil {
return nil, err
}
// Using the Config value, create the DynamoDB client
return &dynamoWrapper, nil
}
@@ -248,6 +255,58 @@ func (dwr DBWrapper) GetRepoStars(repo string) (int, error) {
return repoMeta.Stars, nil
}
func (dwr DBWrapper) SetIndexData(indexDigest godigest.Digest, indexData repodb.IndexData) error {
indexAttributeValue, err := attributevalue.Marshal(indexData)
if err != nil {
return err
}
_, err = dwr.Client.UpdateItem(context.TODO(), &dynamodb.UpdateItemInput{
ExpressionAttributeNames: map[string]string{
"#ID": "IndexData",
},
ExpressionAttributeValues: map[string]types.AttributeValue{
":IndexData": indexAttributeValue,
},
Key: map[string]types.AttributeValue{
"IndexDigest": &types.AttributeValueMemberS{
Value: indexDigest.String(),
},
},
TableName: aws.String(dwr.IndexDataTablename),
UpdateExpression: aws.String("SET #ID = :IndexData"),
})
return err
}
func (dwr DBWrapper) GetIndexData(indexDigest godigest.Digest) (repodb.IndexData, error) {
resp, err := dwr.Client.GetItem(context.TODO(), &dynamodb.GetItemInput{
TableName: aws.String(dwr.IndexDataTablename),
Key: map[string]types.AttributeValue{
"IndexDigest": &types.AttributeValueMemberS{
Value: indexDigest.String(),
},
},
})
if err != nil {
return repodb.IndexData{}, err
}
if resp.Item == nil {
return repodb.IndexData{}, zerr.ErrRepoMetaNotFound
}
var indexData repodb.IndexData
err = attributevalue.Unmarshal(resp.Item["IndexData"], &indexData)
if err != nil {
return repodb.IndexData{}, err
}
return indexData, nil
}
func (dwr DBWrapper) SetRepoTag(repo string, tag string, manifestDigest godigest.Digest, mediaType string) error {
if err := common.ValidateRepoTagInput(repo, tag, manifestDigest); err != nil {
return err
@@ -377,7 +436,7 @@ func (dwr DBWrapper) IncrementImageDownloads(repo string, reference string) erro
return err
}
manifestDigest := reference
descriptorDigest := reference
if !common.ReferenceIsDigest(reference) {
// search digest for tag
@@ -387,19 +446,14 @@ func (dwr DBWrapper) IncrementImageDownloads(repo string, reference string) erro
return zerr.ErrManifestMetaNotFound
}
manifestDigest = descriptor.Digest
descriptorDigest = descriptor.Digest
}
manifestMeta, err := dwr.GetManifestMeta(repo, godigest.Digest(manifestDigest))
if err != nil {
return err
}
manifestStatistics := repoMeta.Statistics[descriptorDigest]
manifestStatistics.DownloadCount++
repoMeta.Statistics[descriptorDigest] = manifestStatistics
manifestMeta.DownloadCount++
err = dwr.SetManifestMeta(repo, godigest.Digest(manifestDigest), manifestMeta)
return err
return dwr.setRepoMeta(repo, repoMeta)
}
func (dwr DBWrapper) AddManifestSignature(repo string, signedManifestDigest godigest.Digest,
@@ -531,11 +585,10 @@ func (dwr DBWrapper) GetMultipleRepoMeta(ctx context.Context,
func (dwr DBWrapper) SearchRepos(ctx context.Context, searchText string, filter repodb.Filter,
requestedPage repodb.PageInput,
) ([]repodb.RepoMetadata, map[string]repodb.ManifestMetadata, repodb.PageInfo, error) {
) ([]repodb.RepoMetadata, map[string]repodb.ManifestMetadata, map[string]repodb.IndexData, repodb.PageInfo, error) {
var (
foundManifestMetadataMap = make(map[string]repodb.ManifestMetadata)
manifestMetadataMap = make(map[string]repodb.ManifestMetadata)
manifestMetadataMap = make(map[string]repodb.ManifestMetadata)
indexDataMap = make(map[string]repodb.IndexData)
repoMetaAttributeIterator iterator.AttributesIterator
pageFinder repodb.PageFinder
pageInfo repodb.PageInfo
@@ -547,7 +600,8 @@ func (dwr DBWrapper) SearchRepos(ctx context.Context, searchText string, filter
pageFinder, err := repodb.NewBaseRepoPageFinder(requestedPage.Limit, requestedPage.Offset, requestedPage.SortBy)
if err != nil {
return []repodb.RepoMetadata{}, map[string]repodb.ManifestMetadata{}, pageInfo, err
return []repodb.RepoMetadata{}, map[string]repodb.ManifestMetadata{}, map[string]repodb.IndexData{},
pageInfo, err
}
repoMetaAttribute, err := repoMetaAttributeIterator.First(ctx)
@@ -555,14 +609,16 @@ func (dwr DBWrapper) SearchRepos(ctx context.Context, searchText string, filter
for ; repoMetaAttribute != nil; repoMetaAttribute, err = repoMetaAttributeIterator.Next(ctx) {
if err != nil {
// log
return []repodb.RepoMetadata{}, map[string]repodb.ManifestMetadata{}, pageInfo, err
return []repodb.RepoMetadata{}, map[string]repodb.ManifestMetadata{}, map[string]repodb.IndexData{},
pageInfo, err
}
var repoMeta repodb.RepoMetadata
err := attributevalue.Unmarshal(repoMetaAttribute, &repoMeta)
if err != nil {
return []repodb.RepoMetadata{}, map[string]repodb.ManifestMetadata{}, pageInfo, err
return []repodb.RepoMetadata{}, map[string]repodb.ManifestMetadata{}, map[string]repodb.IndexData{},
pageInfo, err
}
if ok, err := localCtx.RepoIsUserAvailable(ctx, repoMeta.Name); !ok || err != nil {
@@ -581,43 +637,84 @@ func (dwr DBWrapper) SearchRepos(ctx context.Context, searchText string, filter
)
for _, descriptor := range repoMeta.Tags {
var manifestMeta repodb.ManifestMetadata
switch descriptor.MediaType {
case ispec.MediaTypeImageManifest:
manifestDigest := descriptor.Digest
manifestMeta, manifestDownloaded := manifestMetadataMap[descriptor.Digest]
if !manifestDownloaded {
manifestMeta, err = dwr.GetManifestMeta(repoMeta.Name, godigest.Digest(descriptor.Digest)) //nolint:contextcheck
manifestMeta, err := dwr.fetchManifestMetaWithCheck(repoMeta.Name, manifestDigest, //nolint:contextcheck
manifestMetadataMap)
if err != nil {
return []repodb.RepoMetadata{}, map[string]repodb.ManifestMetadata{}, pageInfo,
errors.Wrapf(err, "repodb: error while unmarshaling manifest metadata for digest %s", descriptor.Digest)
return []repodb.RepoMetadata{}, map[string]repodb.ManifestMetadata{}, map[string]repodb.IndexData{},
pageInfo,
errors.Wrapf(err, "")
}
manifestFilterData, err := collectImageManifestFilterData(manifestDigest, repoMeta, manifestMeta)
if err != nil {
return []repodb.RepoMetadata{}, map[string]repodb.ManifestMetadata{}, map[string]repodb.IndexData{},
pageInfo,
errors.Wrapf(err, "")
}
repoDownloads += manifestFilterData.DownloadCount
for _, os := range manifestFilterData.OsList {
osSet[os] = true
}
for _, arch := range manifestFilterData.ArchList {
archSet[arch] = true
}
if firstImageChecked || repoLastUpdated.Before(manifestFilterData.LastUpdated) {
repoLastUpdated = manifestFilterData.LastUpdated
firstImageChecked = false
isSigned = manifestFilterData.IsSigned
}
manifestMetadataMap[descriptor.Digest] = manifestMeta
case ispec.MediaTypeImageIndex:
var indexLastUpdated time.Time
indexDigest := descriptor.Digest
indexData, err := dwr.fetchIndexDataWithCheck(indexDigest, indexDataMap) //nolint:contextcheck
if err != nil {
return []repodb.RepoMetadata{}, map[string]repodb.ManifestMetadata{}, map[string]repodb.IndexData{},
pageInfo,
errors.Wrapf(err, "")
}
// this also updates manifestMetadataMap
imageFilterData, err := dwr.collectImageIndexFilterInfo(indexDigest, repoMeta, indexData, //nolint:contextcheck
manifestMetadataMap)
if err != nil {
return []repodb.RepoMetadata{}, map[string]repodb.ManifestMetadata{}, map[string]repodb.IndexData{},
pageInfo,
errors.Wrapf(err, "")
}
for _, arch := range imageFilterData.ArchList {
archSet[arch] = true
}
for _, os := range imageFilterData.OsList {
osSet[os] = true
}
repoDownloads += imageFilterData.DownloadCount
if repoLastUpdated.Before(imageFilterData.LastUpdated) {
repoLastUpdated = indexLastUpdated
isSigned = imageFilterData.IsSigned
}
indexDataMap[indexDigest] = indexData
default:
dwr.Log.Error().Msgf("Unsupported type: %s", descriptor.MediaType)
}
// get fields related to filtering
var configContent ispec.Image
err = json.Unmarshal(manifestMeta.ConfigBlob, &configContent)
if err != nil {
return []repodb.RepoMetadata{}, map[string]repodb.ManifestMetadata{}, pageInfo,
errors.Wrapf(err, "repodb: error while unmarshaling config content for digest %s", descriptor.Digest)
}
osSet[configContent.OS] = true
archSet[configContent.Architecture] = true
// get fields related to sorting
repoDownloads += repoMeta.Statistics[descriptor.Digest].DownloadCount
imageLastUpdated := common.GetImageLastUpdatedTimestamp(configContent)
if firstImageChecked || repoLastUpdated.Before(imageLastUpdated) {
repoLastUpdated = imageLastUpdated
firstImageChecked = false
isSigned = common.CheckIsSigned(manifestMeta.Signatures)
}
manifestMetadataMap[descriptor.Digest] = manifestMeta
}
repoFilterData := repodb.FilterData{
@@ -641,22 +738,145 @@ func (dwr DBWrapper) SearchRepos(ctx context.Context, searchText string, filter
foundRepos, pageInfo := pageFinder.Page()
// keep just the manifestMeta we need
for _, repoMeta := range foundRepos {
for _, descriptor := range repoMeta.Tags {
foundManifestMetadataMap[descriptor.Digest] = manifestMetadataMap[descriptor.Digest]
foundManifestMetadataMap, foundindexDataMap, err := filterFoundData(foundRepos, manifestMetadataMap, indexDataMap)
return foundRepos, foundManifestMetadataMap, foundindexDataMap, pageInfo, err
}
func (dwr DBWrapper) fetchManifestMetaWithCheck(repoName string, manifestDigest string,
manifestMetadataMap map[string]repodb.ManifestMetadata,
) (repodb.ManifestMetadata, error) {
var (
manifestMeta repodb.ManifestMetadata
err error
)
manifestMeta, manifestDownloaded := manifestMetadataMap[manifestDigest]
if !manifestDownloaded {
manifestMeta, err = dwr.GetManifestMeta(repoName, godigest.Digest(manifestDigest)) //nolint:contextcheck
if err != nil {
return repodb.ManifestMetadata{}, err
}
}
return foundRepos, foundManifestMetadataMap, pageInfo, err
return manifestMeta, nil
}
func collectImageManifestFilterData(digest string, repoMeta repodb.RepoMetadata,
manifestMeta repodb.ManifestMetadata,
) (repodb.FilterData, error) {
// get fields related to filtering
var (
configContent ispec.Image
osList []string
archList []string
)
err := json.Unmarshal(manifestMeta.ConfigBlob, &configContent)
if err != nil {
return repodb.FilterData{},
errors.Wrapf(err, "repodb: error while unmarshaling config content")
}
if configContent.OS != "" {
osList = append(osList, configContent.OS)
}
if configContent.Architecture != "" {
archList = append(archList, configContent.Architecture)
}
return repodb.FilterData{
DownloadCount: repoMeta.Statistics[digest].DownloadCount,
OsList: osList,
ArchList: archList,
LastUpdated: common.GetImageLastUpdatedTimestamp(configContent),
IsSigned: common.CheckIsSigned(repoMeta.Signatures[digest]),
}, nil
}
func (dwr DBWrapper) fetchIndexDataWithCheck(indexDigest string, indexDataMap map[string]repodb.IndexData,
) (repodb.IndexData, error) {
var (
indexData repodb.IndexData
err error
)
indexData, indexExists := indexDataMap[indexDigest]
if !indexExists {
indexData, err = dwr.GetIndexData(godigest.Digest(indexDigest)) //nolint:contextcheck
if err != nil {
return repodb.IndexData{},
errors.Wrapf(err, "repodb: error while unmarshaling index data for digest %s", indexDigest)
}
}
return indexData, err
}
func (dwr DBWrapper) collectImageIndexFilterInfo(indexDigest string, repoMeta repodb.RepoMetadata,
indexData repodb.IndexData, manifestMetadataMap map[string]repodb.ManifestMetadata,
) (repodb.FilterData, error) {
var indexContent ispec.Index
err := json.Unmarshal(indexData.IndexBlob, &indexContent)
if err != nil {
return repodb.FilterData{},
errors.Wrapf(err, "repodb: error while unmarshaling index content for digest %s", indexDigest)
}
var (
indexLastUpdated time.Time
firstManifestChecked = false
indexOsList = []string{}
indexArchList = []string{}
)
for _, manifest := range indexContent.Manifests {
manifestDigest := manifest.Digest
manifestMeta, err := dwr.fetchManifestMetaWithCheck(repoMeta.Name, manifestDigest.String(),
manifestMetadataMap)
if err != nil {
return repodb.FilterData{},
errors.Wrapf(err, "")
}
manifestFilterData, err := collectImageManifestFilterData(manifestDigest.String(), repoMeta,
manifestMeta)
if err != nil {
return repodb.FilterData{},
errors.Wrapf(err, "")
}
indexOsList = append(indexOsList, manifestFilterData.OsList...)
indexArchList = append(indexArchList, manifestFilterData.ArchList...)
if !firstManifestChecked || indexLastUpdated.Before(manifestFilterData.LastUpdated) {
indexLastUpdated = manifestFilterData.LastUpdated
firstManifestChecked = true
}
manifestMetadataMap[manifest.Digest.String()] = manifestMeta
}
return repodb.FilterData{
DownloadCount: repoMeta.Statistics[indexDigest].DownloadCount,
LastUpdated: indexLastUpdated,
OsList: indexOsList,
ArchList: indexArchList,
IsSigned: common.CheckIsSigned(repoMeta.Signatures[indexDigest]),
}, nil
}
func (dwr DBWrapper) FilterTags(ctx context.Context, filter repodb.FilterFunc,
requestedPage repodb.PageInput,
) ([]repodb.RepoMetadata, map[string]repodb.ManifestMetadata, repodb.PageInfo, error) {
) ([]repodb.RepoMetadata, map[string]repodb.ManifestMetadata, map[string]repodb.IndexData, repodb.PageInfo, error) {
var (
foundManifestMetadataMap = make(map[string]repodb.ManifestMetadata)
manifestMetadataMap = make(map[string]repodb.ManifestMetadata)
indexDataMap = make(map[string]repodb.IndexData)
pageFinder repodb.PageFinder
repoMetaAttributeIterator iterator.AttributesIterator
pageInfo repodb.PageInfo
@@ -668,7 +888,8 @@ func (dwr DBWrapper) FilterTags(ctx context.Context, filter repodb.FilterFunc,
pageFinder, err := repodb.NewBaseImagePageFinder(requestedPage.Limit, requestedPage.Offset, requestedPage.SortBy)
if err != nil {
return []repodb.RepoMetadata{}, map[string]repodb.ManifestMetadata{}, pageInfo, err
return []repodb.RepoMetadata{}, map[string]repodb.ManifestMetadata{}, map[string]repodb.IndexData{},
pageInfo, err
}
repoMetaAttribute, err := repoMetaAttributeIterator.First(ctx)
@@ -676,14 +897,16 @@ func (dwr DBWrapper) FilterTags(ctx context.Context, filter repodb.FilterFunc,
for ; repoMetaAttribute != nil; repoMetaAttribute, err = repoMetaAttributeIterator.Next(ctx) {
if err != nil {
// log
return []repodb.RepoMetadata{}, map[string]repodb.ManifestMetadata{}, pageInfo, err
return []repodb.RepoMetadata{}, map[string]repodb.ManifestMetadata{}, map[string]repodb.IndexData{},
pageInfo, err
}
var repoMeta repodb.RepoMetadata
err := attributevalue.Unmarshal(repoMetaAttribute, &repoMeta)
if err != nil {
return []repodb.RepoMetadata{}, map[string]repodb.ManifestMetadata{}, pageInfo, err
return []repodb.RepoMetadata{}, map[string]repodb.ManifestMetadata{}, map[string]repodb.IndexData{},
pageInfo, err
}
if ok, err := localCtx.RepoIsUserAvailable(ctx, repoMeta.Name); !ok || err != nil {
@@ -692,36 +915,80 @@ func (dwr DBWrapper) FilterTags(ctx context.Context, filter repodb.FilterFunc,
matchedTags := make(map[string]repodb.Descriptor)
// take all manifestMetas
for tag, descriptor := range repoMeta.Tags {
manifestDigest := descriptor.Digest
matchedTags[tag] = descriptor
// in case tags reference the same manifest we don't download from DB multiple times
manifestMeta, manifestExists := manifestMetadataMap[manifestDigest]
switch descriptor.MediaType {
case ispec.MediaTypeImageManifest:
manifestDigest := descriptor.Digest
if !manifestExists {
manifestMeta, err := dwr.GetManifestMeta(repoMeta.Name, godigest.Digest(manifestDigest)) //nolint:contextcheck
manifestMeta, err := dwr.fetchManifestMetaWithCheck(repoMeta.Name, manifestDigest, //nolint:contextcheck
manifestMetadataMap)
if err != nil {
return []repodb.RepoMetadata{}, map[string]repodb.ManifestMetadata{}, pageInfo,
return []repodb.RepoMetadata{}, map[string]repodb.ManifestMetadata{}, map[string]repodb.IndexData{},
pageInfo,
errors.Wrapf(err, "repodb: error while unmashaling manifest metadata for digest %s", manifestDigest)
}
var configContent ispec.Image
if !filter(repoMeta, manifestMeta) {
delete(matchedTags, tag)
err = json.Unmarshal(manifestMeta.ConfigBlob, &configContent)
if err != nil {
return []repodb.RepoMetadata{}, map[string]repodb.ManifestMetadata{}, pageInfo,
errors.Wrapf(err, "repodb: error while unmashaling config for manifest with digest %s", manifestDigest)
continue
}
manifestMetadataMap[manifestDigest] = manifestMeta
case ispec.MediaTypeImageIndex:
indexDigest := descriptor.Digest
indexData, err := dwr.fetchIndexDataWithCheck(indexDigest, indexDataMap) //nolint:contextcheck
if err != nil {
return []repodb.RepoMetadata{}, map[string]repodb.ManifestMetadata{}, map[string]repodb.IndexData{},
pageInfo,
errors.Wrapf(err, "repodb: error while getting index data for digest %s", indexDigest)
}
var indexContent ispec.Index
err = json.Unmarshal(indexData.IndexBlob, &indexContent)
if err != nil {
return []repodb.RepoMetadata{}, map[string]repodb.ManifestMetadata{}, map[string]repodb.IndexData{},
pageInfo,
errors.Wrapf(err, "repodb: error while unmashaling index content for digest %s", indexDigest)
}
manifestHasBeenMatched := false
for _, manifest := range indexContent.Manifests {
manifestDigest := manifest.Digest.String()
manifestMeta, err := dwr.fetchManifestMetaWithCheck(repoMeta.Name, manifestDigest, //nolint:contextcheck
manifestMetadataMap)
if err != nil {
return []repodb.RepoMetadata{}, map[string]repodb.ManifestMetadata{}, map[string]repodb.IndexData{},
pageInfo,
errors.Wrapf(err, "repodb: error while getting manifest data for digest %s", manifestDigest)
}
manifestMetadataMap[manifestDigest] = manifestMeta
if filter(repoMeta, manifestMeta) {
manifestHasBeenMatched = true
}
}
if !manifestHasBeenMatched {
delete(matchedTags, tag)
for _, manifest := range indexContent.Manifests {
delete(manifestMetadataMap, manifest.Digest.String())
}
continue
}
indexDataMap[indexDigest] = indexData
default:
dwr.Log.Error().Msgf("Unsupported type: %s", descriptor.MediaType)
}
if !filter(repoMeta, manifestMeta) {
delete(matchedTags, tag)
continue
}
manifestMetadataMap[manifestDigest] = manifestMeta
}
if len(matchedTags) == 0 {
@@ -737,22 +1004,17 @@ func (dwr DBWrapper) FilterTags(ctx context.Context, filter repodb.FilterFunc,
foundRepos, pageInfo := pageFinder.Page()
// keep just the manifestMeta we need
for _, repoMeta := range foundRepos {
for _, descriptor := range repoMeta.Tags {
foundManifestMetadataMap[descriptor.Digest] = manifestMetadataMap[descriptor.Digest]
}
}
foundManifestMetadataMap, foundindexDataMap, err := filterFoundData(foundRepos, manifestMetadataMap, indexDataMap)
return foundRepos, foundManifestMetadataMap, pageInfo, err
return foundRepos, foundManifestMetadataMap, foundindexDataMap, pageInfo, err
}
func (dwr DBWrapper) SearchTags(ctx context.Context, searchText string, filter repodb.Filter,
requestedPage repodb.PageInput,
) ([]repodb.RepoMetadata, map[string]repodb.ManifestMetadata, repodb.PageInfo, error) {
) ([]repodb.RepoMetadata, map[string]repodb.ManifestMetadata, map[string]repodb.IndexData, repodb.PageInfo, error) {
var (
foundManifestMetadataMap = make(map[string]repodb.ManifestMetadata)
manifestMetadataMap = make(map[string]repodb.ManifestMetadata)
indexDataMap = make(map[string]repodb.IndexData)
repoMetaAttributeIterator = iterator.NewBaseDynamoAttributesIterator(
dwr.Client, dwr.RepoMetaTablename, "RepoMetadata", 0, dwr.Log,
)
@@ -763,12 +1025,14 @@ func (dwr DBWrapper) SearchTags(ctx context.Context, searchText string, filter r
pageFinder, err := repodb.NewBaseImagePageFinder(requestedPage.Limit, requestedPage.Offset, requestedPage.SortBy)
if err != nil {
return []repodb.RepoMetadata{}, map[string]repodb.ManifestMetadata{}, pageInfo, err
return []repodb.RepoMetadata{}, map[string]repodb.ManifestMetadata{}, map[string]repodb.IndexData{},
pageInfo, err
}
searchedRepo, searchedTag, err := common.GetRepoTag(searchText)
if err != nil {
return []repodb.RepoMetadata{}, map[string]repodb.ManifestMetadata{}, pageInfo,
return []repodb.RepoMetadata{}, map[string]repodb.ManifestMetadata{}, map[string]repodb.IndexData{},
pageInfo,
errors.Wrap(err, "repodb: error while parsing search text, invalid format")
}
@@ -777,14 +1041,16 @@ func (dwr DBWrapper) SearchTags(ctx context.Context, searchText string, filter r
for ; repoMetaAttribute != nil; repoMetaAttribute, err = repoMetaAttributeIterator.Next(ctx) {
if err != nil {
// log
return []repodb.RepoMetadata{}, map[string]repodb.ManifestMetadata{}, pageInfo, err
return []repodb.RepoMetadata{}, map[string]repodb.ManifestMetadata{}, map[string]repodb.IndexData{},
pageInfo, err
}
var repoMeta repodb.RepoMetadata
err := attributevalue.Unmarshal(repoMetaAttribute, &repoMeta)
if err != nil {
return []repodb.RepoMetadata{}, map[string]repodb.ManifestMetadata{}, pageInfo, err
return []repodb.RepoMetadata{}, map[string]repodb.ManifestMetadata{}, map[string]repodb.IndexData{},
pageInfo, err
}
if ok, err := localCtx.RepoIsUserAvailable(ctx, repoMeta.Name); !ok || err != nil {
@@ -801,41 +1067,92 @@ func (dwr DBWrapper) SearchTags(ctx context.Context, searchText string, filter r
matchedTags[tag] = descriptor
// in case tags reference the same manifest we don't download from DB multiple times
if manifestMeta, manifestExists := manifestMetadataMap[descriptor.Digest]; manifestExists {
switch descriptor.MediaType {
case ispec.MediaTypeImageManifest:
manifestDigest := descriptor.Digest
manifestMeta, err := dwr.fetchManifestMetaWithCheck(repoMeta.Name, manifestDigest, //nolint:contextcheck
manifestMetadataMap)
if err != nil {
return []repodb.RepoMetadata{}, map[string]repodb.ManifestMetadata{}, map[string]repodb.IndexData{},
pageInfo,
errors.Wrapf(err, "repodb: error while unmashaling manifest metadata for digest %s", descriptor.Digest)
}
imageFilterData, err := collectImageManifestFilterData(manifestDigest, repoMeta, manifestMeta)
if err != nil {
return []repodb.RepoMetadata{}, map[string]repodb.ManifestMetadata{}, map[string]repodb.IndexData{},
pageInfo,
errors.Wrapf(err, "")
}
if !common.AcceptedByFilter(filter, imageFilterData) {
delete(matchedTags, tag)
continue
}
manifestMetadataMap[descriptor.Digest] = manifestMeta
case ispec.MediaTypeImageIndex:
indexDigest := descriptor.Digest
continue
indexData, err := dwr.fetchIndexDataWithCheck(indexDigest, indexDataMap) //nolint:contextcheck
if err != nil {
return []repodb.RepoMetadata{}, map[string]repodb.ManifestMetadata{}, map[string]repodb.IndexData{},
pageInfo,
errors.Wrapf(err, "")
}
var indexContent ispec.Index
err = json.Unmarshal(indexData.IndexBlob, &indexContent)
if err != nil {
return []repodb.RepoMetadata{}, map[string]repodb.ManifestMetadata{}, map[string]repodb.IndexData{},
pageInfo,
errors.Wrapf(err, "repodb: error while unmashaling index content for digest %s", indexDigest)
}
manifestHasBeenMatched := false
for _, manifest := range indexContent.Manifests {
manifestDigest := manifest.Digest.String()
manifestMeta, err := dwr.fetchManifestMetaWithCheck(repoMeta.Name, manifestDigest, //nolint:contextcheck
manifestMetadataMap)
if err != nil {
return []repodb.RepoMetadata{}, map[string]repodb.ManifestMetadata{}, map[string]repodb.IndexData{},
pageInfo,
errors.Wrapf(err, "")
}
manifestFilterData, err := collectImageManifestFilterData(manifestDigest, repoMeta, manifestMeta)
if err != nil {
return []repodb.RepoMetadata{}, map[string]repodb.ManifestMetadata{}, map[string]repodb.IndexData{},
pageInfo,
errors.Wrapf(err, "")
}
manifestMetadataMap[manifestDigest] = manifestMeta
if common.AcceptedByFilter(filter, manifestFilterData) {
manifestHasBeenMatched = true
}
}
if !manifestHasBeenMatched {
delete(matchedTags, tag)
for _, manifest := range indexContent.Manifests {
delete(manifestMetadataMap, manifest.Digest.String())
}
continue
}
indexDataMap[indexDigest] = indexData
default:
dwr.Log.Error().Msgf("Unsupported type: %s", descriptor.MediaType)
}
manifestMeta, err := dwr.GetManifestMeta(repoMeta.Name, godigest.Digest(descriptor.Digest)) //nolint:contextcheck
if err != nil {
return []repodb.RepoMetadata{}, map[string]repodb.ManifestMetadata{}, pageInfo,
errors.Wrapf(err, "repodb: error while unmashaling manifest metadata for digest %s", descriptor.Digest)
}
var configContent ispec.Image
err = json.Unmarshal(manifestMeta.ConfigBlob, &configContent)
if err != nil {
return []repodb.RepoMetadata{}, map[string]repodb.ManifestMetadata{}, pageInfo,
errors.Wrapf(err, "repodb: error while unmashaling config for manifest with digest %s", descriptor.Digest)
}
imageFilterData := repodb.FilterData{
OsList: []string{configContent.OS},
ArchList: []string{configContent.Architecture},
IsSigned: false,
}
if !common.AcceptedByFilter(filter, imageFilterData) {
delete(matchedTags, tag)
delete(manifestMetadataMap, descriptor.Digest)
continue
}
manifestMetadataMap[descriptor.Digest] = manifestMeta
}
if len(matchedTags) == 0 {
@@ -852,14 +1169,49 @@ func (dwr DBWrapper) SearchTags(ctx context.Context, searchText string, filter r
foundRepos, pageInfo := pageFinder.Page()
foundManifestMetadataMap, foundindexDataMap, err := filterFoundData(foundRepos, manifestMetadataMap, indexDataMap)
return foundRepos, foundManifestMetadataMap, foundindexDataMap, pageInfo, err
}
func filterFoundData(foundRepos []repodb.RepoMetadata, manifestMetadataMap map[string]repodb.ManifestMetadata,
indexDataMap map[string]repodb.IndexData,
) (map[string]repodb.ManifestMetadata, map[string]repodb.IndexData, error) {
var (
foundManifestMetadataMap = make(map[string]repodb.ManifestMetadata)
foundindexDataMap = make(map[string]repodb.IndexData)
)
// keep just the manifestMeta we need
for _, repoMeta := range foundRepos {
for _, descriptor := range repoMeta.Tags {
foundManifestMetadataMap[descriptor.Digest] = manifestMetadataMap[descriptor.Digest]
switch descriptor.MediaType {
case ispec.MediaTypeImageManifest:
foundManifestMetadataMap[descriptor.Digest] = manifestMetadataMap[descriptor.Digest]
case ispec.MediaTypeImageIndex:
indexData := indexDataMap[descriptor.Digest]
var indexContent ispec.Index
err := json.Unmarshal(indexData.IndexBlob, &indexContent)
if err != nil {
return map[string]repodb.ManifestMetadata{}, map[string]repodb.IndexData{},
errors.Wrapf(err, "repodb: error while getting manifest data for digest %s", descriptor.Digest)
}
for _, manifestDescriptor := range indexContent.Manifests {
manifestDigest := manifestDescriptor.Digest.String()
foundManifestMetadataMap[manifestDigest] = manifestMetadataMap[manifestDigest]
}
foundindexDataMap[descriptor.Digest] = indexData
default:
}
}
}
return foundRepos, foundManifestMetadataMap, pageInfo, err
return foundManifestMetadataMap, foundindexDataMap, nil
}
func (dwr *DBWrapper) PatchDB() error {
@@ -1008,6 +1360,31 @@ func (dwr DBWrapper) createManifestDataTable() error {
return dwr.waitTableToBeCreated(dwr.ManifestDataTablename)
}
func (dwr DBWrapper) createIndexDataTable() error {
_, err := dwr.Client.CreateTable(context.Background(), &dynamodb.CreateTableInput{
TableName: aws.String(dwr.IndexDataTablename),
AttributeDefinitions: []types.AttributeDefinition{
{
AttributeName: aws.String("IndexDigest"),
AttributeType: types.ScalarAttributeTypeS,
},
},
KeySchema: []types.KeySchemaElement{
{
AttributeName: aws.String("IndexDigest"),
KeyType: types.KeyTypeHash,
},
},
BillingMode: types.BillingModePayPerRequest,
})
if err != nil && strings.Contains(err.Error(), "Table already exists") {
return nil
}
return dwr.waitTableToBeCreated(dwr.IndexDataTablename)
}
func (dwr *DBWrapper) createVersionTable() error {
_, err := dwr.Client.CreateTable(context.Background(), &dynamodb.CreateTableInput{
TableName: aws.String(dwr.VersionTablename),
@@ -1,5 +1,6 @@
package params
type DBDriverParameters struct {
Endpoint, Region, RepoMetaTablename, ManifestDataTablename, VersionTablename string
Endpoint, Region, RepoMetaTablename, ManifestDataTablename, IndexDataTablename,
VersionTablename string
}
+20 -6
View File
@@ -2,6 +2,7 @@ package repodb
import (
"context"
"time"
godigest "github.com/opencontainers/go-digest"
)
@@ -9,6 +10,7 @@ import (
// MetadataDB.
const (
ManifestDataBucket = "ManifestData"
IndexDataBucket = "IndexData"
UserMetadataBucket = "UserMeta"
RepoMetadataBucket = "RepoMetadata"
VersionBucket = "Version"
@@ -59,6 +61,12 @@ type RepoDB interface { //nolint:interfacebloat
// GetManifestMeta sets ManifestMetadata for a given manifest in the database
SetManifestMeta(repo string, manifestDigest godigest.Digest, mm ManifestMetadata) error
// SetIndexData sets indexData for a given index in the database
SetIndexData(digest godigest.Digest, indexData IndexData) error
// GetIndexData returns indexData for a given Index from the database
GetIndexData(indexDigest godigest.Digest) (IndexData, error)
// IncrementManifestDownloads adds 1 to the download count of a manifest
IncrementImageDownloads(repo string, reference string) error
@@ -70,15 +78,15 @@ type RepoDB interface { //nolint:interfacebloat
// SearchRepos searches for repos given a search string
SearchRepos(ctx context.Context, searchText string, filter Filter, requestedPage PageInput) (
[]RepoMetadata, map[string]ManifestMetadata, PageInfo, error)
[]RepoMetadata, map[string]ManifestMetadata, map[string]IndexData, PageInfo, error)
// SearchTags searches for images(repo:tag) given a search string
SearchTags(ctx context.Context, searchText string, filter Filter, requestedPage PageInput) (
[]RepoMetadata, map[string]ManifestMetadata, PageInfo, error)
[]RepoMetadata, map[string]ManifestMetadata, map[string]IndexData, PageInfo, error)
// FilterTags filters for images given a filter function
FilterTags(ctx context.Context, filter FilterFunc,
requestedPage PageInput) ([]RepoMetadata, map[string]ManifestMetadata, PageInfo, error)
requestedPage PageInput) ([]RepoMetadata, map[string]ManifestMetadata, map[string]IndexData, PageInfo, error)
PatchDB() error
}
@@ -90,6 +98,10 @@ type ManifestMetadata struct {
Signatures ManifestSignatures
}
type IndexData struct {
IndexBlob []byte
}
type ManifestData struct {
ManifestBlob []byte
ConfigBlob []byte
@@ -163,7 +175,9 @@ type Filter struct {
}
type FilterData struct {
OsList []string
ArchList []string
IsSigned bool
DownloadCount int
LastUpdated time.Time
OsList []string
ArchList []string
IsSigned bool
}
File diff suppressed because it is too large Load Diff
@@ -19,6 +19,7 @@ func TestCreateDynamo(t *testing.T) {
Endpoint: os.Getenv("DYNAMODBMOCK_ENDPOINT"),
RepoMetaTablename: "RepoMetadataTable",
ManifestDataTablename: "ManifestDataTable",
IndexDataTablename: "IndexDataTable",
VersionTablename: "Version",
Region: "us-east-2",
}
+65 -28
View File
@@ -74,12 +74,6 @@ func SyncRepo(repo string, repoDB RepoDB, storeController storage.StoreControlle
for _, manifest := range indexContent.Manifests {
tag, hasTag := manifest.Annotations[ispec.AnnotationRefName]
if !hasTag {
log.Warn().Msgf("sync-repo: image without tag found, will not be synced into RepoDB")
continue
}
manifestMetaIsPresent, err := isManifestMetaPresent(repo, manifest, repoDB)
if err != nil {
log.Error().Err(err).Msgf("sync-repo: error checking manifestMeta in RepoDB")
@@ -87,7 +81,7 @@ func SyncRepo(repo string, repoDB RepoDB, storeController storage.StoreControlle
return err
}
if manifestMetaIsPresent {
if manifestMetaIsPresent && hasTag {
err = repoDB.SetRepoTag(repo, tag, manifest.Digest, manifest.MediaType)
if err != nil {
log.Error().Err(err).Msgf("sync-repo: failed to set repo tag for %s:%s", repo, tag)
@@ -131,31 +125,16 @@ func SyncRepo(repo string, repoDB RepoDB, storeController storage.StoreControlle
continue
}
manifestData, err := NewManifestData(repo, manifestBlob, storeController)
if err != nil {
log.Error().Err(err).Msgf("sync-repo: failed to create manifest data for image %s:%s manifest digest %s ",
repo, tag, manifest.Digest.String())
reference := tag
return err
if tag == "" {
reference = manifest.Digest.String()
}
err = repoDB.SetManifestMeta(repo, manifest.Digest, ManifestMetadata{
ManifestBlob: manifestData.ManifestBlob,
ConfigBlob: manifestData.ConfigBlob,
DownloadCount: 0,
Signatures: ManifestSignatures{},
})
err = SetMetadataFromInput(repo, reference, manifest.MediaType, manifest.Digest, manifestBlob,
storeController, repoDB, log)
if err != nil {
log.Error().Err(err).Msgf("sync-repo: failed to set manifest meta for image %s:%s manifest digest %s ",
repo, tag, manifest.Digest.String())
return err
}
err = repoDB.SetRepoTag(repo, tag, manifest.Digest, manifest.MediaType)
if err != nil {
log.Error().Err(err).Msgf("sync-repo: failed to repo tag for repo %s and tag %s",
repo, tag)
log.Error().Err(err).Msgf("sync-repo: failed to set metadata for %s:%s", repo, tag)
return err
}
@@ -271,3 +250,61 @@ func NewManifestData(repoName string, manifestBlob []byte, storeController stora
return manifestData, nil
}
func NewIndexData(repoName string, indexBlob []byte, storeController storage.StoreController,
) IndexData {
indexData := IndexData{}
indexData.IndexBlob = indexBlob
return indexData
}
// SetMetadataFromInput tries to set manifest metadata and update repo metadata by adding the current tag
// (in case the reference is a tag). The function expects image manifests and indexes (multi arch images).
func SetMetadataFromInput(repo, reference, mediaType string, digest godigest.Digest, descriptorBlob []byte,
storeController storage.StoreController, repoDB RepoDB, log log.Logger,
) error {
switch mediaType {
case ispec.MediaTypeImageManifest:
imageData, err := NewManifestData(repo, descriptorBlob, storeController)
if err != nil {
return err
}
err = repoDB.SetManifestData(digest, imageData)
if err != nil {
log.Error().Err(err).Msg("repodb: error while putting manifest meta")
return err
}
case ispec.MediaTypeImageIndex:
indexData := NewIndexData(repo, descriptorBlob, storeController)
err := repoDB.SetIndexData(digest, indexData)
if err != nil {
log.Error().Err(err).Msg("repodb: error while putting index data")
return err
}
}
if refferenceIsDigest(reference) {
return nil
}
err := repoDB.SetRepoTag(repo, reference, digest, mediaType)
if err != nil {
log.Error().Err(err).Msg("repodb: error while putting repo meta")
return err
}
return nil
}
func refferenceIsDigest(reference string) bool {
_, err := godigest.Parse(reference)
return err == nil
}
+34 -32
View File
@@ -298,10 +298,10 @@ func TestSyncRepoDBWithStorage(t *testing.T) {
err = test.WriteImageToFileSystem(
test.Image{
Config: config,
Layers: layers,
Manifest: manifest,
Tag: fmt.Sprintf("tag%d", i),
Config: config,
Layers: layers,
Manifest: manifest,
Reference: fmt.Sprintf("tag%d", i),
},
repo,
storeController)
@@ -322,10 +322,10 @@ func TestSyncRepoDBWithStorage(t *testing.T) {
err = test.WriteImageToFileSystem(
test.Image{
Config: config,
Layers: layers,
Manifest: manifest,
Tag: signatureTag,
Config: config,
Layers: layers,
Manifest: manifest,
Reference: signatureTag,
},
repo,
storeController)
@@ -398,10 +398,10 @@ func TestSyncRepoDBWithStorage(t *testing.T) {
err = test.WriteImageToFileSystem(
test.Image{
Config: config,
Layers: layers,
Manifest: manifest,
Tag: "tag1",
Config: config,
Layers: layers,
Manifest: manifest,
Reference: "tag1",
},
repo,
storeController)
@@ -420,10 +420,10 @@ func TestSyncRepoDBWithStorage(t *testing.T) {
err = test.WriteImageToFileSystem(
test.Image{
Config: config,
Layers: layers,
Manifest: manifest,
Tag: signatureTag,
Config: config,
Layers: layers,
Manifest: manifest,
Reference: signatureTag,
},
repo,
storeController)
@@ -470,10 +470,10 @@ func TestSyncRepoDBDynamoWrapper(t *testing.T) {
err = test.WriteImageToFileSystem(
test.Image{
Config: config,
Layers: layers,
Manifest: manifest,
Tag: fmt.Sprintf("tag%d", i),
Config: config,
Layers: layers,
Manifest: manifest,
Reference: fmt.Sprintf("tag%d", i),
},
repo,
storeController)
@@ -494,10 +494,10 @@ func TestSyncRepoDBDynamoWrapper(t *testing.T) {
err = test.WriteImageToFileSystem(
test.Image{
Config: config,
Layers: layers,
Manifest: manifest,
Tag: signatureTag,
Config: config,
Layers: layers,
Manifest: manifest,
Reference: signatureTag,
},
repo,
storeController)
@@ -531,6 +531,7 @@ func TestSyncRepoDBDynamoWrapper(t *testing.T) {
Region: "us-east-2",
RepoMetaTablename: "RepoMetadataTable",
ManifestDataTablename: "ManifestDataTable",
IndexDataTablename: "IndexDataTable",
VersionTablename: "Version",
})
So(err, ShouldBeNil)
@@ -580,10 +581,10 @@ func TestSyncRepoDBDynamoWrapper(t *testing.T) {
err = test.WriteImageToFileSystem(
test.Image{
Config: config,
Layers: layers,
Manifest: manifest,
Tag: "tag1",
Config: config,
Layers: layers,
Manifest: manifest,
Reference: "tag1",
},
repo,
storeController)
@@ -602,10 +603,10 @@ func TestSyncRepoDBDynamoWrapper(t *testing.T) {
err = test.WriteImageToFileSystem(
test.Image{
Config: config,
Layers: layers,
Manifest: manifest,
Tag: signatureTag,
Config: config,
Layers: layers,
Manifest: manifest,
Reference: signatureTag,
},
repo,
storeController)
@@ -617,6 +618,7 @@ func TestSyncRepoDBDynamoWrapper(t *testing.T) {
Region: "us-east-2",
RepoMetaTablename: "RepoMetadataTable",
ManifestDataTablename: "ManifestDataTable",
IndexDataTablename: "IndexDataTable",
VersionTablename: "Version",
})
So(err, ShouldBeNil)
+1 -44
View File
@@ -51,7 +51,7 @@ func OnUpdateManifest(name, reference, mediaType string, digest godigest.Digest,
metadataSuccessfullySet = false
}
} else {
err := SetMetadataFromInput(name, reference, mediaType, digest, body,
err := repodb.SetMetadataFromInput(name, reference, mediaType, digest, body,
storeController, repoDB, log)
if err != nil {
metadataSuccessfullySet = false
@@ -160,46 +160,3 @@ func OnGetManifest(name, reference string, digest godigest.Digest, body []byte,
return nil
}
// SetMetadataFromInput receives raw information about the manifest pushed and tries to set manifest metadata
// and update repo metadata by adding the current tag (in case the reference is a tag).
// The function expects image manifest.
func SetMetadataFromInput(repo, reference, mediaType string, digest godigest.Digest, manifestBlob []byte,
storeController storage.StoreController, repoDB repodb.RepoDB, log log.Logger,
) error {
imageMetadata, err := repodb.NewManifestData(repo, manifestBlob, storeController)
if err != nil {
return err
}
err = repoDB.SetManifestMeta(repo, digest, repodb.ManifestMetadata{
ManifestBlob: imageMetadata.ManifestBlob,
ConfigBlob: imageMetadata.ConfigBlob,
DownloadCount: 0,
Signatures: repodb.ManifestSignatures{},
})
if err != nil {
log.Error().Err(err).Msg("repodb: error while putting image meta")
return err
}
if refferenceIsDigest(reference) {
return nil
}
err = repoDB.SetRepoTag(repo, reference, digest, mediaType)
if err != nil {
log.Error().Err(err).Msg("repodb: error while putting repo meta")
return err
}
return nil
}
func refferenceIsDigest(reference string) bool {
_, err := godigest.Parse(reference)
return err == nil
}
+30 -5
View File
@@ -14,6 +14,7 @@ import (
zerr "zotregistry.io/zot/errors"
"zotregistry.io/zot/pkg/extensions/monitoring"
"zotregistry.io/zot/pkg/log"
"zotregistry.io/zot/pkg/meta/repodb"
bolt_wrapper "zotregistry.io/zot/pkg/meta/repodb/boltdb-wrapper"
repoDBUpdate "zotregistry.io/zot/pkg/meta/repodb/update"
"zotregistry.io/zot/pkg/storage"
@@ -42,8 +43,12 @@ func TestOnUpdateManifest(t *testing.T) {
config, layers, manifest, err := test.GetRandomImageComponents(100)
So(err, ShouldBeNil)
err = test.WriteImageToFileSystem(test.Image{Config: config, Manifest: manifest, Layers: layers, Tag: "tag1"},
"repo", storeController)
err = test.WriteImageToFileSystem(
test.Image{
Config: config, Manifest: manifest, Layers: layers, Reference: "tag1",
},
"repo",
storeController)
So(err, ShouldBeNil)
manifestBlob, err := json.Marshal(manifest)
@@ -59,6 +64,26 @@ func TestOnUpdateManifest(t *testing.T) {
So(repoMeta.Tags, ShouldContainKey, "tag1")
})
Convey("metadataSuccessfullySet is false", t, func() {
rootDir := t.TempDir()
storeController := storage.StoreController{}
log := log.NewLogger("debug", "")
metrics := monitoring.NewMetricsServer(false, log)
storeController.DefaultStore = local.NewImageStore(rootDir, true, 1*time.Second,
true, true, log, metrics, nil, nil,
)
repoDB := mocks.RepoDBMock{
SetManifestDataFn: func(manifestDigest godigest.Digest, mm repodb.ManifestData) error {
return ErrTestError
},
}
err := repoDBUpdate.OnUpdateManifest("repo", "tag1", ispec.MediaTypeImageManifest, "digest",
[]byte("{}"), storeController, repoDB, log)
So(err, ShouldNotBeNil)
})
}
func TestUpdateErrors(t *testing.T) {
@@ -160,8 +185,8 @@ func TestUpdateErrors(t *testing.T) {
repoDB := mocks.RepoDBMock{}
log := log.NewLogger("debug", "")
err := repoDBUpdate.SetMetadataFromInput("repo", "ref", "digest", "", []byte("BadManifestBlob"),
storeController, repoDB, log)
err := repodb.SetMetadataFromInput("repo", "ref", ispec.MediaTypeImageManifest, "digest",
[]byte("BadManifestBlob"), storeController, repoDB, log)
So(err, ShouldNotBeNil)
// reference is digest
@@ -177,7 +202,7 @@ func TestUpdateErrors(t *testing.T) {
return []byte("{}"), nil
}
err = repoDBUpdate.SetMetadataFromInput("repo", string(godigest.FromString("reference")), "", "digest",
err = repodb.SetMetadataFromInput("repo", string(godigest.FromString("reference")), "", "digest",
manifestBlob, storeController, repoDB, log)
So(err, ShouldBeNil)
})
+1
View File
@@ -119,6 +119,7 @@ func TestVersioningDynamoDB(t *testing.T) {
Region: region,
RepoMetaTablename: "RepoMetadataTable",
ManifestDataTablename: "ManifestDataTable",
IndexDataTablename: "IndexDataTable",
VersionTablename: "Version",
})
So(err, ShouldBeNil)