github/zot
1
0
Fork 0
mirror of https://github.com/project-zot/zot.git synced 2026-06-15 20:07:55 +08:00
Code Issues Packages Projects Releases Wiki Activity

test(meta): add push-pull-read tests for metadb (#2022)

Browse Source 
Signed-off-by: Laurentiu Niculae <niculae.laurentiu1@gmail.com>
This commit is contained in:
LaurentiuNiculae
2023-11-08 23:35:51 +02:00
committed by GitHub
parent 7f52f58e3c
commit c9cc5b9acb
5 changed files with 395 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/extensions/search/convert/metadb.go
+4 -3
View File
@@ -321,7 +321,10 @@ func RepoMeta2RepoSummary(ctx context.Context, repoMeta mTypes.RepoMeta,
) *gql_generated.RepoSummary {
var (
repoName = repoMeta.Name
repoLastUpdatedTimestamp = deref(repoMeta.LastUpdatedImage, mTypes.LastUpdatedImage{}).LastUpdated
lastUpdatedImage = deref(repoMeta.LastUpdatedImage, mTypes.LastUpdatedImage{})
lastUpdatedImageMeta = imageMetaMap[lastUpdatedImage.Digest]
lastUpdatedTag = lastUpdatedImage.Tag
repoLastUpdatedTimestamp = lastUpdatedImage.LastUpdated
repoPlatforms = repoMeta.Platforms
repoVendors = repoMeta.Vendors
repoDownloadCount = repoMeta.DownloadCount
@@ -329,8 +332,6 @@ func RepoMeta2RepoSummary(ctx context.Context, repoMeta mTypes.RepoMeta,
repoIsUserStarred = repoMeta.IsStarred // value specific to the current user
repoIsUserBookMarked = repoMeta.IsBookmarked // value specific to the current user
repoSize = repoMeta.Size
lastUpdatedImageMeta = imageMetaMap[repoMeta.LastUpdatedImage.Digest]
lastUpdatedTag = repoMeta.LastUpdatedImage.Tag
)
if repoLastUpdatedTimestamp == nil {
pkg/extensions/search/search_test.go
+380
View File
@@ -19,6 +19,7 @@ import (
"time"
dbTypes "github.com/aquasecurity/trivy-db/pkg/types"
guuid "github.com/gofrs/uuid"
regTypes "github.com/google/go-containerregistry/pkg/v1/types"
notreg "github.com/notaryproject/notation-go/registry"
godigest "github.com/opencontainers/go-digest"
@@ -47,6 +48,7 @@ import (
"zotregistry.io/zot/pkg/test/mocks"
ociutils "zotregistry.io/zot/pkg/test/oci-utils"
"zotregistry.io/zot/pkg/test/signature"
tskip "zotregistry.io/zot/pkg/test/skip"
)
const (
@@ -6406,3 +6408,381 @@ func TestUploadingArtifactsWithDifferentMediaType(t *testing.T) {
imageWithIncompatibleConfig.ConfigDescriptor.Digest.String())
})
}
func TestReadUploadDeleteDynamoDB(t *testing.T) {
tskip.SkipDynamo(t)
uuid, err := guuid.NewV4()
if err != nil {
panic(err)
}
cacheTablename := "BlobTable" + uuid.String()
repoMetaTablename := "RepoMetadataTable" + uuid.String()
versionTablename := "Version" + uuid.String()
userDataTablename := "UserDataTable" + uuid.String()
apiKeyTablename := "ApiKeyTable" + uuid.String()
imageMetaTablename := "ImageMeta" + uuid.String()
repoBlobsTablename := "RepoBlobs" + uuid.String()
cacheDriverParams := map[string]interface{}{
"name": "dynamoDB",
"endpoint": os.Getenv("DYNAMODBMOCK_ENDPOINT"),
"region": "us-east-2",
"cachetablename": cacheTablename,
"repometatablename": repoMetaTablename,
"imagemetatablename": imageMetaTablename,
"repoblobsinfotablename": repoBlobsTablename,
"userdatatablename": userDataTablename,
"apikeytablename": apiKeyTablename,
"versiontablename": versionTablename,
}
port := GetFreePort()
baseURL := GetBaseURL(port)
conf := config.New()
conf.HTTP.Port = port
conf.Storage.RootDirectory = t.TempDir()
conf.Storage.GC = false
conf.Storage.CacheDriver = cacheDriverParams
conf.Storage.RemoteCache = true
conf.Log = &config.LogConfig{Level: "debug", Output: "/dev/null"}
defaultVal := true
conf.Extensions = &extconf.ExtensionConfig{
Search: &extconf.SearchConfig{BaseConfig: extconf.BaseConfig{Enable: &defaultVal}, CVE: nil},
}
ctlr := api.NewController(conf)
ctlrManager := NewControllerManager(ctlr)
ctlrManager.StartAndWait(port)
defer ctlrManager.StopServer()
RunReadUploadDeleteTests(t, baseURL)
}
func TestReadUploadDeleteBoltDB(t *testing.T) {
port := GetFreePort()
baseURL := GetBaseURL(port)
conf := config.New()
conf.HTTP.Port = port
conf.Storage.RootDirectory = t.TempDir()
conf.Storage.GC = false
conf.Log = &config.LogConfig{Level: "debug", Output: "/dev/null"}
defaultVal := true
conf.Extensions = &extconf.ExtensionConfig{
Search: &extconf.SearchConfig{BaseConfig: extconf.BaseConfig{Enable: &defaultVal}, CVE: nil},
}
ctlr := api.NewController(conf)
ctlrManager := NewControllerManager(ctlr)
ctlrManager.StartAndWait(port)
defer ctlrManager.StopServer()
RunReadUploadDeleteTests(t, baseURL)
}
func RunReadUploadDeleteTests(t *testing.T, baseURL string) {
t.Helper()
repo1 := "repo1"
image := CreateRandomImage()
tag1 := "tag1"
imageWithoutTag := CreateRandomImage()
usedImages := []repoRef{
{repo1, tag1},
{repo1, imageWithoutTag.DigestStr()},
}
Convey("Push-Read-Delete", t, func() {
results := GlobalSearchGQL("", baseURL)
So(len(results.Images), ShouldEqual, 0)
So(len(results.Repos), ShouldEqual, 0)
Convey("Push an image without tag", func() {
err := UploadImage(imageWithoutTag, baseURL, repo1, imageWithoutTag.DigestStr())
So(err, ShouldBeNil)
results := GlobalSearchGQL("", baseURL)
So(len(results.Repos), ShouldEqual, 0)
Convey("Add tag and delete it", func() {
err := UploadImage(image, baseURL, repo1, tag1)
So(err, ShouldBeNil)
results := GlobalSearchGQL("", baseURL)
So(len(results.Repos), ShouldEqual, 1)
status, err := DeleteImage(repo1, tag1, baseURL)
So(status, ShouldEqual, http.StatusAccepted)
So(err, ShouldBeNil)
results = GlobalSearchGQL("", baseURL)
So(len(results.Repos), ShouldEqual, 0)
})
})
Convey("Push a random image", func() {
err := UploadImage(image, baseURL, repo1, tag1)
So(err, ShouldBeNil)
results := GlobalSearchGQL("", baseURL)
So(len(results.Repos), ShouldEqual, 1)
Convey("Delete the image pushed", func() {
status, err := DeleteImage(repo1, tag1, baseURL)
So(status, ShouldEqual, http.StatusAccepted)
So(err, ShouldBeNil)
results := GlobalSearchGQL("", baseURL)
So(len(results.Repos), ShouldEqual, 0)
Convey("Push an image without tag", func() {
err := UploadImage(imageWithoutTag, baseURL, repo1, imageWithoutTag.DigestStr())
So(err, ShouldBeNil)
results := GlobalSearchGQL("", baseURL)
So(len(results.Repos), ShouldEqual, 0)
})
})
Convey("Delete the image pushed multiple times", func() {
for i := 0; i < 3; i++ {
status, err := DeleteImage(repo1, tag1, baseURL)
So(status, ShouldBeIn, []int{http.StatusAccepted, http.StatusNotFound, http.StatusBadRequest})
So(err, ShouldBeNil)
results := GlobalSearchGQL("", baseURL)
So(len(results.Repos), ShouldEqual, 0)
}
})
Convey("Upload same image multiple times", func() {
for i := 0; i < 3; i++ {
err := UploadImage(image, baseURL, repo1, tag1)
So(err, ShouldBeNil)
}
results := GlobalSearchGQL("", baseURL)
So(len(results.Repos), ShouldEqual, 1)
})
})
deleteUsedImages(usedImages, baseURL)
})
// Images with create time
repoLatest := "repo-latest"
afterImage := CreateImageWith().DefaultLayers().
ImageConfig(ispec.Image{Created: DateRef(2010, 1, 1, 1, 1, 1, 0, time.UTC)}).Build()
tagAfter := "after"
middleImage := CreateImageWith().DefaultLayers().
ImageConfig(ispec.Image{Created: DateRef(2005, 1, 1, 1, 1, 1, 0, time.UTC)}).Build()
tagMiddle := "middle"
beforeImage := CreateImageWith().DefaultLayers().
ImageConfig(ispec.Image{Created: DateRef(2000, 1, 1, 1, 1, 1, 0, time.UTC)}).Build()
tagBefore := "before"
imageWithoutTag = CreateImageWith().DefaultLayers().
ImageConfig(ispec.Image{Created: DateRef(2020, 1, 1, 1, 1, 1, 0, time.UTC)}).Build()
imageWithoutCreateTime := CreateImageWith().DefaultLayers().
ImageConfig(ispec.Image{Created: nil}).Build()
tagWithoutTime := "without-time"
usedImages = []repoRef{
{repoLatest, tagAfter},
{repoLatest, tagMiddle},
{repoLatest, tagBefore},
{repoLatest, tagWithoutTime},
{repoLatest, imageWithoutTag.DigestStr()},
}
Convey("Last Updated Image", t, func() {
results := GlobalSearchGQL("", baseURL)
So(len(results.Images), ShouldEqual, 0)
So(len(results.Repos), ShouldEqual, 0)
Convey("Without time", func() {
err := UploadImage(imageWithoutCreateTime, baseURL, repoLatest, tagWithoutTime)
So(err, ShouldBeNil)
results := GlobalSearchGQL("", baseURL)
So(len(results.Repos), ShouldEqual, 1)
So(results.Repos[0].NewestImage.Digest, ShouldResemble, imageWithoutCreateTime.DigestStr())
Convey("Add an image with create time and delete it", func() {
err := UploadImage(beforeImage, baseURL, repoLatest, tagBefore)
So(err, ShouldBeNil)
results := GlobalSearchGQL("", baseURL)
So(len(results.Repos), ShouldEqual, 1)
So(results.Repos[0].NewestImage.Digest, ShouldResemble, beforeImage.DigestStr())
status, err := DeleteImage(repoLatest, tagBefore, baseURL)
So(status, ShouldEqual, http.StatusAccepted)
So(err, ShouldBeNil)
results = GlobalSearchGQL("", baseURL)
So(len(results.Repos), ShouldEqual, 1)
So(results.Repos[0].NewestImage.Digest, ShouldResemble, imageWithoutCreateTime.DigestStr())
})
})
Convey("Upload middle image", func() {
err := UploadImage(middleImage, baseURL, repoLatest, tagMiddle)
So(err, ShouldBeNil)
results := GlobalSearchGQL("", baseURL)
So(len(results.Repos), ShouldEqual, 1)
So(results.Repos[0].NewestImage.Digest, ShouldResemble, middleImage.DigestStr())
Convey("Upload an image created before", func() {
err := UploadImage(beforeImage, baseURL, repoLatest, tagBefore)
So(err, ShouldBeNil)
results := GlobalSearchGQL("", baseURL)
So(len(results.Repos), ShouldEqual, 1)
So(results.Repos[0].NewestImage.Digest, ShouldResemble, middleImage.DigestStr())
Convey("Upload an image created after", func() {
err := UploadImage(afterImage, baseURL, repoLatest, tagAfter)
So(err, ShouldBeNil)
results := GlobalSearchGQL("", baseURL)
So(len(results.Repos), ShouldEqual, 1)
So(results.Repos[0].NewestImage.Digest, ShouldResemble, afterImage.DigestStr())
Convey("Delete middle then after", func() {
status, err := DeleteImage(repoLatest, tagMiddle, baseURL)
So(status, ShouldEqual, http.StatusAccepted)
So(err, ShouldBeNil)
results := GlobalSearchGQL("", baseURL)
So(len(results.Repos), ShouldEqual, 1)
So(results.Repos[0].NewestImage.Digest, ShouldResemble, afterImage.DigestStr())
status, err = DeleteImage(repoLatest, tagAfter, baseURL)
So(status, ShouldEqual, http.StatusAccepted)
So(err, ShouldBeNil)
results = GlobalSearchGQL("", baseURL)
So(len(results.Repos), ShouldEqual, 1)
So(results.Repos[0].NewestImage.Digest, ShouldResemble, beforeImage.DigestStr())
})
})
})
Convey("Upload an image created after", func() {
err := UploadImage(afterImage, baseURL, repoLatest, tagAfter)
So(err, ShouldBeNil)
results := GlobalSearchGQL("", baseURL)
So(len(results.Repos), ShouldEqual, 1)
So(results.Repos[0].NewestImage.Digest, ShouldResemble, afterImage.DigestStr())
Convey("Add newer image without tag", func() {
err := UploadImage(imageWithoutTag, baseURL, repoLatest, imageWithoutTag.DigestStr())
So(err, ShouldBeNil)
results := GlobalSearchGQL("", baseURL)
So(len(results.Repos), ShouldEqual, 1)
So(results.Repos[0].NewestImage.Digest, ShouldResemble, afterImage.DigestStr())
})
Convey("Delete afterImage", func() {
status, err := DeleteImage(repoLatest, tagAfter, baseURL)
So(status, ShouldEqual, http.StatusAccepted)
So(err, ShouldBeNil)
results := GlobalSearchGQL("", baseURL)
So(len(results.Repos), ShouldEqual, 1)
So(results.Repos[0].NewestImage.Digest, ShouldResemble, middleImage.DigestStr())
})
})
})
deleteUsedImages(usedImages, baseURL)
})
}
type repoRef struct {
Repo string
Tag string
}
func deleteUsedImages(repoTags []repoRef, baseURL string) {
for _, image := range repoTags {
status, err := DeleteImage(image.Repo, image.Tag, baseURL)
So(status, ShouldBeIn, []int{http.StatusAccepted, http.StatusNotFound, http.StatusBadRequest})
So(err, ShouldBeNil)
}
}
func GlobalSearchGQL(query, baseURL string) *zcommon.GlobalSearchResultResp {
queryStr := `
{
GlobalSearch(query:"` + query + `"){
Images {
RepoName Tag Digest MediaType Size DownloadCount LastUpdated IsSigned
Description Licenses Labels Title Source Documentation Authors Vendor
Manifests {
Digest ConfigDigest LastUpdated Size IsSigned
DownloadCount
SignatureInfo {Tool IsTrusted Author}
Platform {Os Arch}
Layers {Size Digest}
History {
Layer { Size Digest }
HistoryDescription { Author Comment Created CreatedBy EmptyLayer }
}
Vulnerabilities {Count MaxSeverity}
Referrers {MediaType ArtifactType Size Digest Annotations {Key Value}}
}
Referrers {MediaType ArtifactType Size Digest Annotations {Key Value}}
Vulnerabilities { Count MaxSeverity }
SignatureInfo {Tool IsTrusted Author}
}
Repos {
Name LastUpdated Size DownloadCount StarCount IsBookmarked IsStarred
Platforms { Os Arch }
Vendors
NewestImage {
RepoName Tag Digest MediaType Size DownloadCount LastUpdated IsSigned
Description Licenses Labels Title Source Documentation Authors Vendor
Manifests {
Digest ConfigDigest LastUpdated Size IsSigned
DownloadCount
SignatureInfo {Tool IsTrusted Author}
Platform {Os Arch}
Layers {Size Digest}
History {
Layer { Size Digest }
HistoryDescription { Author Comment Created CreatedBy EmptyLayer }
}
Vulnerabilities {Count MaxSeverity}
Referrers {MediaType ArtifactType Size Digest Annotations {Key Value}}
}
Referrers {MediaType ArtifactType Size Digest Annotations {Key Value}}
Vulnerabilities { Count MaxSeverity }
SignatureInfo {Tool IsTrusted Author}
}
}
}
}`
resp, err := resty.R().Get(baseURL + graphqlQueryPrefix + "?query=" + url.QueryEscape(queryStr))
So(resp, ShouldNotBeNil)
So(err, ShouldBeNil)
So(resp.StatusCode(), ShouldEqual, 200)
responseStruct := &zcommon.GlobalSearchResultResp{}
err = json.Unmarshal(resp.Body(), responseStruct)
So(err, ShouldBeNil)
return responseStruct
}
pkg/meta/boltdb/boltdb.go
+4
View File
@@ -385,6 +385,10 @@ func (bdw *BoltDB) SearchRepos(ctx context.Context, searchText string,
delete(protoRepoMeta.Tags, "")
if len(protoRepoMeta.Tags) == 0 {
continue
}
protoRepoMeta.Rank = int32(rank)
protoRepoMeta.IsStarred = zcommon.Contains(userStars, protoRepoMeta.Name)
protoRepoMeta.IsBookmarked = zcommon.Contains(userBookmarks, protoRepoMeta.Name)
pkg/meta/common/common.go
+1 -1
View File
@@ -277,7 +277,7 @@ func RemoveImageFromRepoMeta(repoMeta *proto_go.RepoMeta, repoBlobs *proto_go.Re
queue := []string{descriptor.Digest}
mConvert.GetProtoEarlierUpdatedImage(updatedLastImage, &proto_go.RepoLastUpdatedImage{
updatedLastImage = mConvert.GetProtoEarlierUpdatedImage(updatedLastImage, &proto_go.RepoLastUpdatedImage{
LastUpdated: repoBlobs.Blobs[descriptor.Digest].LastUpdated,
MediaType: descriptor.MediaType,
Digest: descriptor.Digest,
pkg/meta/dynamodb/dynamodb.go
+6
View File
@@ -459,6 +459,12 @@ func (dwr *DynamoDB) SearchRepos(ctx context.Context, searchText string) ([]mTyp
continue
}
delete(protoRepoMeta.Tags, "")
if len(protoRepoMeta.Tags) == 0 {
continue
}
rank := common.RankRepoName(searchText, protoRepoMeta.Name)
if rank == -1 {
continue