github/zot
1
0
Fork 0
mirror of https://github.com/project-zot/zot.git synced 2026-06-16 20:38:08 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(repodb): DerivedImageList and BaseImageList make use of RepoDB (#1135)

Browse Source 
- derivedImageList and baseImageList now use FilterTags to obtain results,
each with its own filter function
- images that have the exact same manifest as the one provided as a
parameter are no longer considered base images or derived images
- both calls can be made with specific pagination parameters, and the
response will include PageInfo

Signed-off-by: Alex Stan <alexandrustan96@yahoo.ro>

fix(tests): fix one of the pagination tests

The results were not reliable as the 2 returned tags were sorted by created date/time
which was not set, resulting in an unpredictable order

Signed-off-by: Andrei Aaron <andaaron@cisco.com>
(cherry picked from commit be504200a1127371422aeb0e5c0219e2a1ead20a)
(cherry picked from commit ed8d797e639f262a63840120afe92da7db9a7600)
Signed-off-by: Andrei Aaron <aaaron@luxoft.com>

Signed-off-by: Andrei Aaron <andaaron@cisco.com>
Signed-off-by: Andrei Aaron <aaaron@luxoft.com>
Co-authored-by: Alex Stan <alexandrustan96@yahoo.ro>
This commit is contained in:
Andrei Aaron
2023-01-26 00:06:02 +02:00
committed by GitHub
parent be4b8c6243
commit feb7328f50
15 changed files with 1525 additions and 482 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/meta/repodb/boltdb-wrapper/boltdb_wrapper.go
+5 -4
View File
@@ -746,16 +746,17 @@ func (bdw DBWrapper) SearchRepos(ctx context.Context, searchText string, filter
func (bdw DBWrapper) FilterTags(ctx context.Context, filter repodb.FilterFunc,
requestedPage repodb.PageInput,
) ([]repodb.RepoMetadata, map[string]repodb.ManifestMetadata, error) {
) ([]repodb.RepoMetadata, map[string]repodb.ManifestMetadata, repodb.PageInfo, error) {
var (
foundRepos = make([]repodb.RepoMetadata, 0)
foundManifestMetadataMap = make(map[string]repodb.ManifestMetadata)
pageFinder repodb.PageFinder
pageInfo repodb.PageInfo
)
pageFinder, err := repodb.NewBaseImagePageFinder(requestedPage.Limit, requestedPage.Offset, requestedPage.SortBy)
if err != nil {
return []repodb.RepoMetadata{}, map[string]repodb.ManifestMetadata{}, err
return []repodb.RepoMetadata{}, map[string]repodb.ManifestMetadata{}, repodb.PageInfo{}, err
}
err = bdw.DB.View(func(tx *bolt.Tx) error {
@@ -836,7 +837,7 @@ func (bdw DBWrapper) FilterTags(ctx context.Context, filter repodb.FilterFunc,
})
}
foundRepos, _ = pageFinder.Page()
foundRepos, pageInfo = pageFinder.Page()
// keep just the manifestMeta we need
for _, repoMeta := range foundRepos {
@@ -848,7 +849,7 @@ func (bdw DBWrapper) FilterTags(ctx context.Context, filter repodb.FilterFunc,
return nil
})
return foundRepos, foundManifestMetadataMap, err
return foundRepos, foundManifestMetadataMap, pageInfo, err
}
func (bdw DBWrapper) SearchTags(ctx context.Context, searchText string, filter repodb.Filter,
pkg/meta/repodb/dynamodb-wrapper/dynamo_test.go
+4 -4
View File
@@ -398,7 +398,7 @@ func TestWrapperErrors(t *testing.T) {
err = setBadRepoMeta(dynamoWrapper.Client, repoMetaTablename, "repo") //nolint:contextcheck
So(err, ShouldBeNil)
_, _, err = dynamoWrapper.FilterTags(
_, _, _, err = dynamoWrapper.FilterTags(
ctx,
func(repoMeta repodb.RepoMetadata, manifestMeta repodb.ManifestMetadata) bool {
return true
@@ -413,7 +413,7 @@ func TestWrapperErrors(t *testing.T) {
err := dynamoWrapper.SetRepoTag("repo", "tag1", "manifestNotFound", "") //nolint:contextcheck
So(err, ShouldBeNil)
_, _, err = dynamoWrapper.FilterTags(
_, _, _, err = dynamoWrapper.FilterTags(
ctx,
func(repoMeta repodb.RepoMetadata, manifestMeta repodb.ManifestMetadata) bool {
return true
@@ -431,7 +431,7 @@ func TestWrapperErrors(t *testing.T) {
err = setBadManifestData(dynamoWrapper.Client, manifestDataTablename, "dig") //nolint:contextcheck
So(err, ShouldBeNil)
_, _, err = dynamoWrapper.FilterTags(
_, _, _, err = dynamoWrapper.FilterTags(
ctx,
func(repoMeta repodb.RepoMetadata, manifestMeta repodb.ManifestMetadata) bool {
return true
@@ -452,7 +452,7 @@ func TestWrapperErrors(t *testing.T) {
})
So(err, ShouldBeNil)
_, _, err = dynamoWrapper.FilterTags(
_, _, _, err = dynamoWrapper.FilterTags(
ctx,
func(repoMeta repodb.RepoMetadata, manifestMeta repodb.ManifestMetadata) bool {
return true
pkg/meta/repodb/dynamodb-wrapper/dynamo_wrapper.go
+9 -8
View File
@@ -653,12 +653,13 @@ func (dwr DBWrapper) SearchRepos(ctx context.Context, searchText string, filter
func (dwr DBWrapper) FilterTags(ctx context.Context, filter repodb.FilterFunc,
requestedPage repodb.PageInput,
) ([]repodb.RepoMetadata, map[string]repodb.ManifestMetadata, error) {
) ([]repodb.RepoMetadata, map[string]repodb.ManifestMetadata, repodb.PageInfo, error) {
var (
foundManifestMetadataMap = make(map[string]repodb.ManifestMetadata)
manifestMetadataMap = make(map[string]repodb.ManifestMetadata)
pageFinder repodb.PageFinder
repoMetaAttributeIterator iterator.AttributesIterator
pageInfo repodb.PageInfo
)
repoMetaAttributeIterator = iterator.NewBaseDynamoAttributesIterator(
@@ -667,7 +668,7 @@ func (dwr DBWrapper) FilterTags(ctx context.Context, filter repodb.FilterFunc,
pageFinder, err := repodb.NewBaseImagePageFinder(requestedPage.Limit, requestedPage.Offset, requestedPage.SortBy)
if err != nil {
return []repodb.RepoMetadata{}, map[string]repodb.ManifestMetadata{}, err
return []repodb.RepoMetadata{}, map[string]repodb.ManifestMetadata{}, pageInfo, err
}
repoMetaAttribute, err := repoMetaAttributeIterator.First(ctx)
@@ -675,14 +676,14 @@ func (dwr DBWrapper) FilterTags(ctx context.Context, filter repodb.FilterFunc,
for ; repoMetaAttribute != nil; repoMetaAttribute, err = repoMetaAttributeIterator.Next(ctx) {
if err != nil {
// log
return []repodb.RepoMetadata{}, map[string]repodb.ManifestMetadata{}, err
return []repodb.RepoMetadata{}, map[string]repodb.ManifestMetadata{}, pageInfo, err
}
var repoMeta repodb.RepoMetadata
err := attributevalue.Unmarshal(repoMetaAttribute, &repoMeta)
if err != nil {
return []repodb.RepoMetadata{}, map[string]repodb.ManifestMetadata{}, err
return []repodb.RepoMetadata{}, map[string]repodb.ManifestMetadata{}, pageInfo, err
}
if ok, err := localCtx.RepoIsUserAvailable(ctx, repoMeta.Name); !ok || err != nil {
@@ -701,7 +702,7 @@ func (dwr DBWrapper) FilterTags(ctx context.Context, filter repodb.FilterFunc,
if !manifestExists {
manifestMeta, err := dwr.GetManifestMeta(repoMeta.Name, godigest.Digest(manifestDigest)) //nolint:contextcheck
if err != nil {
return []repodb.RepoMetadata{}, map[string]repodb.ManifestMetadata{},
return []repodb.RepoMetadata{}, map[string]repodb.ManifestMetadata{}, pageInfo,
errors.Wrapf(err, "repodb: error while unmashaling manifest metadata for digest %s", manifestDigest)
}
@@ -709,7 +710,7 @@ func (dwr DBWrapper) FilterTags(ctx context.Context, filter repodb.FilterFunc,
err = json.Unmarshal(manifestMeta.ConfigBlob, &configContent)
if err != nil {
return []repodb.RepoMetadata{}, map[string]repodb.ManifestMetadata{},
return []repodb.RepoMetadata{}, map[string]repodb.ManifestMetadata{}, pageInfo,
errors.Wrapf(err, "repodb: error while unmashaling config for manifest with digest %s", manifestDigest)
}
}
@@ -734,7 +735,7 @@ func (dwr DBWrapper) FilterTags(ctx context.Context, filter repodb.FilterFunc,
})
}
foundRepos, _ := pageFinder.Page()
foundRepos, pageInfo := pageFinder.Page()
// keep just the manifestMeta we need
for _, repoMeta := range foundRepos {
@@ -743,7 +744,7 @@ func (dwr DBWrapper) FilterTags(ctx context.Context, filter repodb.FilterFunc,
}
}
return foundRepos, foundManifestMetadataMap, err
return foundRepos, foundManifestMetadataMap, pageInfo, err
}
func (dwr DBWrapper) SearchTags(ctx context.Context, searchText string, filter repodb.Filter,
pkg/meta/repodb/repodb.go
+1 -1
View File
@@ -78,7 +78,7 @@ type RepoDB interface { //nolint:interfacebloat
// FilterTags filters for images given a filter function
FilterTags(ctx context.Context, filter FilterFunc,
requestedPage PageInput) ([]RepoMetadata, map[string]ManifestMetadata, error)
requestedPage PageInput) ([]RepoMetadata, map[string]ManifestMetadata, PageInfo, error)
PatchDB() error
}
pkg/meta/repodb/repodb_test.go
+13 -5
View File
@@ -1335,7 +1335,7 @@ func RunRepoDBTests(repoDB repodb.RepoDB, preparationFuncs ...func() error) {
So(err, ShouldBeNil)
Convey("Return all tags", func() {
repos, manifesMetaMap, err := repoDB.FilterTags(
repos, manifesMetaMap, pageInfo, err := repoDB.FilterTags(
ctx,
func(repoMeta repodb.RepoMetadata, manifestMeta repodb.ManifestMetadata) bool {
return true
@@ -1358,10 +1358,12 @@ func RunRepoDBTests(repoDB repodb.RepoDB, preparationFuncs ...func() error) {
So(manifesMetaMap, ShouldContainKey, manifestDigest1.String())
So(manifesMetaMap, ShouldContainKey, manifestDigest2.String())
So(manifesMetaMap, ShouldContainKey, manifestDigest3.String())
So(pageInfo.ItemCount, ShouldEqual, 6)
So(pageInfo.TotalCount, ShouldEqual, 6)
})
Convey("Return all tags in a specific repo", func() {
repos, manifesMetaMap, err := repoDB.FilterTags(
repos, manifesMetaMap, pageInfo, err := repoDB.FilterTags(
ctx,
func(repoMeta repodb.RepoMetadata, manifestMeta repodb.ManifestMetadata) bool {
return repoMeta.Name == repo1
@@ -1381,10 +1383,12 @@ func RunRepoDBTests(repoDB repodb.RepoDB, preparationFuncs ...func() error) {
So(manifesMetaMap, ShouldContainKey, manifestDigest1.String())
So(manifesMetaMap, ShouldContainKey, manifestDigest2.String())
So(manifesMetaMap, ShouldContainKey, manifestDigest3.String())
So(pageInfo.ItemCount, ShouldEqual, 5)
So(pageInfo.TotalCount, ShouldEqual, 5)
})
Convey("Filter everything out", func() {
repos, manifesMetaMap, err := repoDB.FilterTags(
repos, manifesMetaMap, pageInfo, err := repoDB.FilterTags(
ctx,
func(repoMeta repodb.RepoMetadata, manifestMeta repodb.ManifestMetadata) bool {
return false
@@ -1395,6 +1399,8 @@ func RunRepoDBTests(repoDB repodb.RepoDB, preparationFuncs ...func() error) {
So(err, ShouldBeNil)
So(len(repos), ShouldEqual, 0)
So(len(manifesMetaMap), ShouldEqual, 0)
So(pageInfo.ItemCount, ShouldEqual, 0)
So(pageInfo.TotalCount, ShouldEqual, 0)
})
Convey("Search with access control", func() {
@@ -1409,7 +1415,7 @@ func RunRepoDBTests(repoDB repodb.RepoDB, preparationFuncs ...func() error) {
authzCtxKey := localCtx.GetContextKey()
ctx := context.WithValue(context.Background(), authzCtxKey, acCtx)
repos, manifesMetaMap, err := repoDB.FilterTags(
repos, manifesMetaMap, pageInfo, err := repoDB.FilterTags(
ctx,
func(repoMeta repodb.RepoMetadata, manifestMeta repodb.ManifestMetadata) bool {
return true
@@ -1423,10 +1429,12 @@ func RunRepoDBTests(repoDB repodb.RepoDB, preparationFuncs ...func() error) {
So(len(repos[0].Tags), ShouldEqual, 1)
So(repos[0].Tags, ShouldContainKey, "0.0.1")
So(manifesMetaMap, ShouldContainKey, manifestDigest3.String())
So(pageInfo.ItemCount, ShouldEqual, 1)
So(pageInfo.TotalCount, ShouldEqual, 1)
})
Convey("With wrong pagination input", func() {
repos, _, err := repoDB.FilterTags(
repos, _, _, err := repoDB.FilterTags(
ctx,
func(repoMeta repodb.RepoMetadata, manifestMeta repodb.ManifestMetadata) bool {
return true