feat(repodb): add user related information to repodb (#1317)

Initial code was contributed by Bogdan BIVOLARU <104334+bogdanbiv@users.noreply.github.com> Moved implementation from a separate db to repodb by Andrei Aaron <aaaron@luxoft.com> Not done yet: - run/test dynamodb implementation, only boltdb was tested - add additional coverage for existing functionality - add web-based APIs to toggle the stars/bookmarks on/off Initially graphql mutation was discussed for the missing API but we decided REST endpoints would be better suited for configuration feat(userdb): complete functionality for userdb integration - dynamodb rollback changes to user starred repos in case increasing the total star count fails - dynamodb increment/decrement repostars in repometa when user stars/unstars a repo - dynamodb check anonymous user permissions are working as intendend - common test handle anonymous users - RepoMeta2RepoSummary set IsStarred and IsBookmarked feat(userdb): rest api calls for toggling stars/bookmarks on/off test(userdb): blackbox tests test(userdb): move preferences tests in a different file with specific build tags feat(repodb): add is-starred and is-bookmarked fields to repo-meta - removed duplicated logic for determining if a repo is starred/bookmarked Signed-off-by: Laurentiu Niculae <niculae.laurentiu1@gmail.com> Co-authored-by: Andrei Aaron <aaaron@luxoft.com>
2026-06-16 20:38:08 +08:00 · 2023-04-24 21:13:15 +03:00
parent ef51fd692d
commit 9cc990d7ca
50 changed files with 4357 additions and 648 deletions
@@ -2,6 +2,7 @@ package common

 import (
 	"encoding/json"
+	"fmt"
 	"strings"
 	"time"

@@ -286,3 +287,98 @@ func CheckImageLastUpdated(repoLastUpdated time.Time, isSigned bool, noImageChec

 	return repoLastUpdated, noImageChecked, isSigned
 }
+
+func FilterDataByRepo(foundRepos []repodb.RepoMetadata, manifestMetadataMap map[string]repodb.ManifestMetadata,
+	indexDataMap map[string]repodb.IndexData,
+) (map[string]repodb.ManifestMetadata, map[string]repodb.IndexData, error) {
+	var (
+		foundManifestMetadataMap = make(map[string]repodb.ManifestMetadata)
+		foundindexDataMap        = make(map[string]repodb.IndexData)
+	)
+
+	// keep just the manifestMeta we need
+	for _, repoMeta := range foundRepos {
+		for _, descriptor := range repoMeta.Tags {
+			switch descriptor.MediaType {
+			case ispec.MediaTypeImageManifest:
+				foundManifestMetadataMap[descriptor.Digest] = manifestMetadataMap[descriptor.Digest]
+			case ispec.MediaTypeImageIndex:
+				indexData := indexDataMap[descriptor.Digest]
+
+				var indexContent ispec.Index
+
+				err := json.Unmarshal(indexData.IndexBlob, &indexContent)
+				if err != nil {
+					return map[string]repodb.ManifestMetadata{}, map[string]repodb.IndexData{},
+						fmt.Errorf("repodb: error while getting manifest data for digest %s %w", descriptor.Digest, err)
+				}
+
+				for _, manifestDescriptor := range indexContent.Manifests {
+					manifestDigest := manifestDescriptor.Digest.String()
+
+					foundManifestMetadataMap[manifestDigest] = manifestMetadataMap[manifestDigest]
+				}
+
+				foundindexDataMap[descriptor.Digest] = indexData
+			default:
+			}
+		}
+	}
+
+	return foundManifestMetadataMap, foundindexDataMap, nil
+}
+
+func FetchDataForRepos(repoDB repodb.RepoDB, foundRepos []repodb.RepoMetadata,
+) (map[string]repodb.ManifestMetadata, map[string]repodb.IndexData, error) {
+	foundManifestMetadataMap := map[string]repodb.ManifestMetadata{}
+	foundIndexDataMap := map[string]repodb.IndexData{}
+
+	for idx := range foundRepos {
+		for _, descriptor := range foundRepos[idx].Tags {
+			switch descriptor.MediaType {
+			case ispec.MediaTypeImageManifest:
+				manifestData, err := repoDB.GetManifestData(godigest.Digest(descriptor.Digest))
+				if err != nil {
+					return map[string]repodb.ManifestMetadata{}, map[string]repodb.IndexData{}, err
+				}
+
+				foundManifestMetadataMap[descriptor.Digest] = repodb.ManifestMetadata{
+					ManifestBlob: manifestData.ManifestBlob,
+					ConfigBlob:   manifestData.ConfigBlob,
+				}
+			case ispec.MediaTypeImageIndex:
+				indexData, err := repoDB.GetIndexData(godigest.Digest(descriptor.Digest))
+				if err != nil {
+					return map[string]repodb.ManifestMetadata{}, map[string]repodb.IndexData{}, err
+				}
+
+				var indexContent ispec.Index
+
+				err = json.Unmarshal(indexData.IndexBlob, &indexContent)
+				if err != nil {
+					return map[string]repodb.ManifestMetadata{},
+						map[string]repodb.IndexData{},
+						fmt.Errorf("repodb: error while getting index data for digest %s %w", descriptor.Digest, err)
+				}
+
+				for _, manifestDescriptor := range indexContent.Manifests {
+					manifestDigest := manifestDescriptor.Digest.String()
+
+					manifestData, err := repoDB.GetManifestData(manifestDescriptor.Digest)
+					if err != nil {
+						return map[string]repodb.ManifestMetadata{}, map[string]repodb.IndexData{}, err
+					}
+
+					foundManifestMetadataMap[manifestDigest] = repodb.ManifestMetadata{
+						ManifestBlob: manifestData.ManifestBlob,
+						ConfigBlob:   manifestData.ConfigBlob,
+					}
+				}
+
+				foundIndexDataMap[descriptor.Digest] = indexData
+			}
+		}
+	}
+
+	return foundManifestMetadataMap, foundIndexDataMap, nil
+}
@@ -1,15 +1,21 @@
 package common_test

 import (
+	"errors"
 	"testing"
 	"time"

+	"github.com/opencontainers/go-digest"
+	ispec "github.com/opencontainers/image-spec/specs-go/v1"
 	. "github.com/smartystreets/goconvey/convey"

 	"zotregistry.io/zot/pkg/meta/common"
 	"zotregistry.io/zot/pkg/meta/repodb"
+	"zotregistry.io/zot/pkg/test/mocks"
 )

+var ErrTestError = errors.New("test error")
+
 func TestUtils(t *testing.T) {
 	Convey("GetReferredSubject", t, func() {
 		_, err := common.GetReferredSubject([]byte("bad json"))
@@ -94,4 +100,128 @@ func TestUtils(t *testing.T) {
 			So(noImageChecked, ShouldEqual, false)
 		})
 	})
+
+	Convey("SignatureAlreadyExists", t, func() {
+		res := common.SignatureAlreadyExists(
+			[]repodb.SignatureInfo{{SignatureManifestDigest: "digest"}},
+			repodb.SignatureMetadata{SignatureDigest: "digest"},
+		)
+
+		So(res, ShouldEqual, true)
+
+		res = common.SignatureAlreadyExists(
+			[]repodb.SignatureInfo{{SignatureManifestDigest: "digest"}},
+			repodb.SignatureMetadata{SignatureDigest: "digest2"},
+		)
+
+		So(res, ShouldEqual, false)
+	})
+
+	Convey("FilterDataByRepo", t, func() {
+		Convey("Errors", func() {
+			// Unmarshal index data error
+			_, _, err := common.FilterDataByRepo(
+				[]repodb.RepoMetadata{{
+					Tags: map[string]repodb.Descriptor{
+						"tag": {
+							Digest:    "indexDigest",
+							MediaType: ispec.MediaTypeImageIndex,
+						},
+					},
+				}},
+				map[string]repodb.ManifestMetadata{},
+				map[string]repodb.IndexData{
+					"indexDigest": {
+						IndexBlob: []byte("bad blob"),
+					},
+				},
+			)
+
+			So(err, ShouldNotBeNil)
+		})
+	})
+
+	Convey("FetchDataForRepos", t, func() {
+		Convey("Errors", func() {
+			// Unmarshal index data error
+			_, _, err := common.FetchDataForRepos(
+				mocks.RepoDBMock{
+					GetIndexDataFn: func(indexDigest digest.Digest) (repodb.IndexData, error) {
+						return repodb.IndexData{
+							IndexBlob: []byte("bad blob"),
+						}, nil
+					},
+				},
+				[]repodb.RepoMetadata{{
+					Tags: map[string]repodb.Descriptor{
+						"tag": {
+							Digest:    "indexDigest",
+							MediaType: ispec.MediaTypeImageIndex,
+						},
+					},
+				}},
+			)
+			So(err, ShouldNotBeNil)
+		})
+	})
+}
+
+func TestFetchDataForRepos(t *testing.T) {
+	Convey("GetReferredSubject", t, func() {
+		mockRepoDB := mocks.RepoDBMock{}
+
+		Convey("GetManifestData errors", func() {
+			mockRepoDB.GetManifestDataFn = func(manifestDigest digest.Digest) (repodb.ManifestData, error) {
+				return repodb.ManifestData{}, ErrTestError
+			}
+
+			_, _, err := common.FetchDataForRepos(mockRepoDB, []repodb.RepoMetadata{
+				{
+					Tags: map[string]repodb.Descriptor{
+						"tag1": {Digest: "dig1", MediaType: ispec.MediaTypeImageManifest},
+					},
+				},
+			})
+			So(err, ShouldNotBeNil)
+		})
+
+		Convey("GetIndexData errors", func() {
+			mockRepoDB.GetIndexDataFn = func(indexDigest digest.Digest) (repodb.IndexData, error) {
+				return repodb.IndexData{}, ErrTestError
+			}
+
+			_, _, err := common.FetchDataForRepos(mockRepoDB, []repodb.RepoMetadata{
+				{
+					Tags: map[string]repodb.Descriptor{
+						"tag1": {Digest: "dig1", MediaType: ispec.MediaTypeImageIndex},
+					},
+				},
+			})
+			So(err, ShouldNotBeNil)
+		})
+
+		Convey("GetIndexData ok, GetManifestData errors", func() {
+			mockRepoDB.GetIndexDataFn = func(indexDigest digest.Digest) (repodb.IndexData, error) {
+				return repodb.IndexData{
+					IndexBlob: []byte(`{
+						"manifests": [
+							{"digest": "dig1"}
+						]
+					}`),
+				}, nil
+			}
+			mockRepoDB.GetManifestDataFn = func(manifestDigest digest.Digest) (repodb.ManifestData, error) {
+				return repodb.ManifestData{}, ErrTestError
+			}
+
+			_, _, err := common.FetchDataForRepos(mockRepoDB, []repodb.RepoMetadata{
+				{
+					Tags: map[string]repodb.Descriptor{
+						"tag1": {Digest: "dig1", MediaType: ispec.MediaTypeImageIndex},
+					},
+				},
+			})
+			So(err, ShouldNotBeNil)
+		})
+	})
 }