github/zot
1
0
Fork 0
mirror of https://github.com/project-zot/zot.git synced 2026-06-17 21:09:23 +08:00
Code Issues Packages Projects Releases Wiki Activity

refactor(artifact): remove oci artifact support (#1359)

Browse Source 
* refactor(artifact): remove oci artifact support
- add header to referrers call to indicated applied artifact type filters

Signed-off-by: Laurentiu Niculae <niculae.laurentiu1@gmail.com>

* feat(gc): simplify gc logic to increase coverage

Signed-off-by: Laurentiu Niculae <niculae.laurentiu1@gmail.com>

---------

Signed-off-by: Laurentiu Niculae <niculae.laurentiu1@gmail.com>
This commit is contained in:
LaurentiuNiculae
2023-05-10 20:15:33 +03:00
committed by GitHub
parent 3be690c2ac
commit ea79be64da
54 changed files with 604 additions and 1608 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/api/controller_test.go
+5 -135
View File
@@ -51,7 +51,6 @@ import (
"zotregistry.io/zot/pkg/log"
"zotregistry.io/zot/pkg/meta/repodb/repodbfactory"
"zotregistry.io/zot/pkg/storage"
storageConstants "zotregistry.io/zot/pkg/storage/constants"
"zotregistry.io/zot/pkg/storage/local"
"zotregistry.io/zot/pkg/test"
)
@@ -157,7 +156,6 @@ func TestCreateCacheDatabaseDriver(t *testing.T) {
"cacheTablename": "BlobTable",
"repoMetaTablename": "RepoMetadataTable",
"manifestDataTablename": "ManifestDataTable",
"artifactDataTablename": "ArtifactDataTable",
"userDataTablename": "ZotUserDataTable",
"versionTablename": "Version",
}
@@ -173,7 +171,6 @@ func TestCreateCacheDatabaseDriver(t *testing.T) {
"cacheTablename": "BlobTable",
"repoMetaTablename": "RepoMetadataTable",
"manifestDataTablename": "ManifestDataTable",
"artifactDataTablename": "ArtifactDataTable",
"userDataTablename": "ZotUserDataTable",
"versionTablename": "Version",
}
@@ -188,7 +185,6 @@ func TestCreateCacheDatabaseDriver(t *testing.T) {
"cacheTablename": "BlobTable",
"repoMetaTablename": "RepoMetadataTable",
"manifestDataTablename": "ManifestDataTable",
"artifactDataTablename": "ArtifactDataTable",
"userDataTablename": "ZotUserDataTable",
"versionTablename": "Version",
}
@@ -222,7 +218,6 @@ func TestCreateRepoDBDriver(t *testing.T) {
"cachetablename": "BlobTable",
"repometatablename": "RepoMetadataTable",
"manifestdatatablename": "ManifestDataTable",
"artifactDataTablename": "ArtifactDataTable",
"userdatatablename": "UserDatatable",
}
@@ -236,7 +231,6 @@ func TestCreateRepoDBDriver(t *testing.T) {
"cachetablename": "",
"repometatablename": "RepoMetadataTable",
"manifestdatatablename": "ManifestDataTable",
"artifactDataTablename": "ArtifactDataTable",
"userDataTablename": "ZotUserDataTable",
"versiontablename": 1,
}
@@ -252,7 +246,6 @@ func TestCreateRepoDBDriver(t *testing.T) {
"repometatablename": "RepoMetadataTable",
"manifestdatatablename": "ManifestDataTable",
"indexdatatablename": "IndexDataTable",
"artifactdatatablename": "ArtifactDataTable",
"userdatatablename": "ZotUserDataTable",
"versiontablename": "1",
}
@@ -4395,138 +4388,15 @@ func TestArtifactReferences(t *testing.T) {
So(err, ShouldBeNil)
So(resp.StatusCode(), ShouldEqual, http.StatusOK)
So(resp.Header().Get("Content-Type"), ShouldEqual, ispec.MediaTypeImageIndex)
})
})
So(resp.Header().Get("OCI-Filters-Applied"), ShouldEqual, artifactType)
Convey("Validate Artifact Manifest Reference", func() {
resp, err := resty.R().Get(baseURL + fmt.Sprintf("/v2/%s/referrers/%s", repoName, digest.String()))
So(err, ShouldBeNil)
So(resp.StatusCode(), ShouldEqual, http.StatusOK)
var referrers ispec.Index
err = json.Unmarshal(resp.Body(), &referrers)
So(err, ShouldBeNil)
So(referrers.Manifests, ShouldBeEmpty)
// now upload a reference
// upload image config blob
resp, err = resty.R().Post(baseURL + fmt.Sprintf("/v2/%s/blobs/uploads/", repoName))
So(err, ShouldBeNil)
So(resp.StatusCode(), ShouldEqual, http.StatusAccepted)
loc := test.Location(baseURL, resp)
cblob, cdigest := test.GetEmptyImageConfig()
resp, err = resty.R().
SetContentLength(true).
SetHeader("Content-Length", fmt.Sprintf("%d", len(cblob))).
SetHeader("Content-Type", "application/octet-stream").
SetQueryParam("digest", cdigest.String()).
SetBody(cblob).
Put(loc)
So(err, ShouldBeNil)
So(resp.StatusCode(), ShouldEqual, http.StatusCreated)
// create a artifact
manifest := ispec.Artifact{
MediaType: ispec.MediaTypeArtifactManifest,
ArtifactType: artifactType,
Blobs: []ispec.Descriptor{
{
MediaType: "application/vnd.oci.image.layer.v1.tar",
Digest: digest,
Size: int64(len(content)),
},
},
Subject: &ispec.Descriptor{
MediaType: ispec.MediaTypeImageManifest,
Digest: digest,
Size: int64(len(content)),
},
Annotations: map[string]string{
"key": "val",
},
}
Convey("Using invalid content", func() {
content := []byte("invalid data")
So(err, ShouldBeNil)
mdigest := godigest.FromBytes(content)
So(mdigest, ShouldNotBeNil)
resp, err = resty.R().SetHeader("Content-Type", ispec.MediaTypeArtifactManifest).
SetBody(content).Put(baseURL + fmt.Sprintf("/v2/%s/manifests/%s", repoName, mdigest.String()))
So(err, ShouldBeNil)
So(resp.StatusCode(), ShouldEqual, http.StatusBadRequest)
// unknown repo will return status not found
resp, err = resty.R().Get(baseURL + fmt.Sprintf("/v2/%s/referrers/%s", "unknown", digest.String()))
So(err, ShouldBeNil)
So(resp.StatusCode(), ShouldEqual, http.StatusNotFound)
resp, err = resty.R().Get(baseURL + fmt.Sprintf("/v2/%s/referrers/%s", repoName, digest.String()))
So(err, ShouldBeNil)
So(resp.StatusCode(), ShouldEqual, http.StatusOK)
resp, err = resty.R().SetQueryParams(map[string]string{"artifactType": artifactType}).
Get(baseURL + fmt.Sprintf("/v2/%s/referrers/%s", repoName, digest.String()))
So(err, ShouldBeNil)
So(resp.StatusCode(), ShouldEqual, http.StatusOK)
})
Convey("Using valid content", func() {
content, err = json.Marshal(manifest)
So(err, ShouldBeNil)
mdigest := godigest.FromBytes(content)
So(mdigest, ShouldNotBeNil)
resp, err = resty.R().SetHeader("Content-Type", ispec.MediaTypeArtifactManifest).
SetBody(content).Put(baseURL + fmt.Sprintf("/v2/%s/manifests/%s", repoName, mdigest.String()))
So(err, ShouldBeNil)
So(resp.StatusCode(), ShouldEqual, http.StatusCreated)
resp, err = resty.R().Get(baseURL + fmt.Sprintf("/v2/%s/referrers/%s", repoName, digest.String()))
So(err, ShouldBeNil)
So(resp.StatusCode(), ShouldEqual, http.StatusOK)
resp, err = resty.R().SetQueryParams(map[string]string{"artifact": "invalid"}).
Get(baseURL + fmt.Sprintf("/v2/%s/referrers/%s", repoName, digest.String()))
So(err, ShouldBeNil)
So(resp.StatusCode(), ShouldEqual, http.StatusOK)
resp, err = resty.R().SetQueryParams(map[string]string{"artifactType": "invalid"}).
Get(baseURL + fmt.Sprintf("/v2/%s/referrers/%s", repoName, digest.String()))
So(err, ShouldBeNil)
So(resp.StatusCode(), ShouldEqual, http.StatusOK)
resp, err = resty.R().SetQueryParams(map[string]string{"artifactType": artifactType}).
Get(baseURL + fmt.Sprintf("/v2/%s/referrers/%s", repoName, digest.String()))
resp, err = resty.R().SetQueryParams(map[string]string{"artifactType": artifactType +
",otherArtType"}).Get(baseURL + fmt.Sprintf("/v2/%s/referrers/%s", repoName,
digest.String()))
So(err, ShouldBeNil)
So(resp.StatusCode(), ShouldEqual, http.StatusOK)
So(resp.Header().Get("Content-Type"), ShouldEqual, ispec.MediaTypeImageIndex)
var index ispec.Index
err = json.Unmarshal(resp.Body(), &index)
So(err, ShouldBeNil)
So(index.Manifests, ShouldNotBeEmpty)
So(index.Annotations[storageConstants.ReferrerFilterAnnotation], ShouldNotBeEmpty)
// filter by multiple artifactTypes
req, err := http.NewRequestWithContext(context.TODO(), http.MethodGet,
baseURL+fmt.Sprintf("/v2/%s/referrers/%s", repoName, digest.String()), nil)
So(err, ShouldBeNil)
values := url.Values{}
values.Add("artifactType", artifactType)
values.Add("artifactType", "foobar")
req.URL.RawQuery = values.Encode()
rsp, err := http.DefaultClient.Do(req)
So(err, ShouldBeNil)
defer rsp.Body.Close()
So(rsp.StatusCode, ShouldEqual, http.StatusOK)
So(rsp.Header.Get("Content-Type"), ShouldEqual, ispec.MediaTypeImageIndex)
body, err := io.ReadAll(rsp.Body)
So(err, ShouldBeNil)
err = json.Unmarshal(body, &index)
So(err, ShouldBeNil)
So(index.Manifests, ShouldNotBeEmpty)
So(index.Annotations[storageConstants.ReferrerFilterAnnotation], ShouldNotBeEmpty)
So(len(strings.Split(index.Annotations[storageConstants.ReferrerFilterAnnotation], ",")), ShouldEqual, 2)
So(resp.Header().Get("OCI-Filters-Applied"), ShouldEqual, artifactType+",otherArtType")
})
})
})
pkg/api/routes.go
+5 -1
View File
@@ -433,7 +433,7 @@ func (rh *RouteHandler) GetManifest(response http.ResponseWriter, request *http.
}
if rh.c.RepoDB != nil {
err := meta.OnGetManifest(name, reference, digest, content, rh.c.StoreController, rh.c.RepoDB, rh.c.Log)
err := meta.OnGetManifest(name, reference, content, rh.c.StoreController, rh.c.RepoDB, rh.c.Log)
if errors.Is(err, zerr.ErrOrphanSignature) {
rh.c.Log.Error().Err(err).Msg("image is an orphan signature")
@@ -546,6 +546,10 @@ func (rh *RouteHandler) GetReferrers(response http.ResponseWriter, request *http
return
}
if len(artifactTypes) > 0 {
response.Header().Set("OCI-Filters-Applied", strings.Join(artifactTypes, ","))
}
WriteData(response, http.StatusOK, ispec.MediaTypeImageIndex, out)
}
pkg/cli/client.go
+2 -4
View File
@@ -490,10 +490,8 @@ func isNotationSigned(ctx context.Context, repo, digestStr string, searchConf se
return false
}
for _, reference := range referrers.Manifests {
if reference.ArtifactType == notreg.ArtifactTypeNotation {
return true
}
if len(referrers.Manifests) > 0 {
return true
}
return false
pkg/cli/client_utils_test.go
+18 -61
View File
@@ -5,7 +5,6 @@ package cli //nolint:testpackage
import (
"context"
"errors"
"fmt"
"io"
"net/http"
@@ -20,48 +19,6 @@ import (
"zotregistry.io/zot/pkg/test"
)
type RouteHandler struct {
Route string
// HandlerFunc is the HTTP handler function that receives a writer for output and an HTTP request as input.
HandlerFunc http.HandlerFunc
// AllowedMethods specifies the HTTP methods allowed for the current route.
AllowedMethods []string
}
// Routes is a map that associates HTTP paths to their corresponding HTTP handlers.
type HTTPRoutes []RouteHandler
func StartTestHTTPServer(routes HTTPRoutes, port string) *http.Server {
baseURL := test.GetBaseURL(port)
mux := mux.NewRouter()
mux.HandleFunc("/test", func(w http.ResponseWriter, r *http.Request) {
_, err := w.Write([]byte("{}"))
if err != nil {
return
}
}).Methods(http.MethodGet)
for _, routeHandler := range routes {
mux.HandleFunc(routeHandler.Route, routeHandler.HandlerFunc).Methods(routeHandler.AllowedMethods...)
}
server := &http.Server{ //nolint:gosec
Addr: fmt.Sprintf(":%s", port),
Handler: mux,
}
go func() {
if err := server.ListenAndServe(); !errors.Is(err, http.ErrServerClosed) {
return
}
}()
test.WaitTillServerReady(baseURL + "/test")
return server
}
func getDefaultSearchConf(baseURL string) searchConfig {
verifyTLS := false
debug := false
@@ -81,7 +38,7 @@ func getDefaultSearchConf(baseURL string) searchConfig {
func TestDoHTTPRequest(t *testing.T) {
Convey("doHTTPRequest nil result pointer", t, func() {
port := test.GetFreePort()
server := StartTestHTTPServer(nil, port)
server := test.StartTestHTTPServer(nil, port)
defer server.Close()
url := fmt.Sprintf("http://127.0.0.1:%s/asd", port)
@@ -93,7 +50,7 @@ func TestDoHTTPRequest(t *testing.T) {
Convey("doHTTPRequest bad return json", t, func() {
port := test.GetFreePort()
server := StartTestHTTPServer(HTTPRoutes{
server := test.StartTestHTTPServer(test.HTTPRoutes{
{
Route: "/test",
HandlerFunc: func(w http.ResponseWriter, r *http.Request) {
@@ -135,7 +92,7 @@ func TestDoHTTPRequest(t *testing.T) {
searchConf := getDefaultSearchConf(baseURL)
// 404 erorr will appear
server := StartTestHTTPServer(HTTPRoutes{}, port)
server := test.StartTestHTTPServer(test.HTTPRoutes{}, port)
defer server.Close()
URL := baseURL + "/v2/repo/manifests/tag"
@@ -158,7 +115,7 @@ func TestDoHTTPRequest(t *testing.T) {
searchConf := getDefaultSearchConf(baseURL)
Convey("makeGETRequest manifest error, context is done", func() {
server := StartTestHTTPServer(HTTPRoutes{}, port)
server := test.StartTestHTTPServer(test.HTTPRoutes{}, port)
defer server.Close()
ctx, cancel := context.WithCancel(context.Background())
@@ -172,7 +129,7 @@ func TestDoHTTPRequest(t *testing.T) {
})
Convey("makeGETRequest manifest error, context is not done", func() {
server := StartTestHTTPServer(HTTPRoutes{}, port)
server := test.StartTestHTTPServer(test.HTTPRoutes{}, port)
defer server.Close()
_, err := fetchManifestStruct(context.Background(), "repo", "tag", searchConf,
@@ -182,7 +139,7 @@ func TestDoHTTPRequest(t *testing.T) {
})
Convey("makeGETRequest config error, context is not done", func() {
server := StartTestHTTPServer(HTTPRoutes{
server := test.StartTestHTTPServer(test.HTTPRoutes{
{
Route: "/v2/{name}/manifests/{reference}",
HandlerFunc: func(w http.ResponseWriter, r *http.Request) {
@@ -203,7 +160,7 @@ func TestDoHTTPRequest(t *testing.T) {
})
Convey("Platforms on config", func() {
server := StartTestHTTPServer(HTTPRoutes{
server := test.StartTestHTTPServer(test.HTTPRoutes{
{
Route: "/v2/{name}/manifests/{reference}",
HandlerFunc: func(w http.ResponseWriter, r *http.Request) {
@@ -258,7 +215,7 @@ func TestDoHTTPRequest(t *testing.T) {
})
Convey("fetchImageIndexStruct no errors", func() {
server := StartTestHTTPServer(HTTPRoutes{
server := test.StartTestHTTPServer(test.HTTPRoutes{
{
Route: "/v2/{name}/manifests/{reference}",
HandlerFunc: func(writer http.ResponseWriter, req *http.Request) {
@@ -327,7 +284,7 @@ func TestDoHTTPRequest(t *testing.T) {
})
Convey("fetchImageIndexStruct makeGETRequest errors context done", func() {
server := StartTestHTTPServer(HTTPRoutes{}, port)
server := test.StartTestHTTPServer(test.HTTPRoutes{}, port)
defer server.Close()
ctx, cancel := context.WithCancel(context.Background())
@@ -349,7 +306,7 @@ func TestDoHTTPRequest(t *testing.T) {
})
Convey("fetchImageIndexStruct makeGETRequest errors context not done", func() {
server := StartTestHTTPServer(HTTPRoutes{}, port)
server := test.StartTestHTTPServer(test.HTTPRoutes{}, port)
defer server.Close()
URL := baseURL + "/v2/repo/manifests/indexRef"
@@ -384,7 +341,7 @@ func TestDoJobErrors(t *testing.T) {
reqPool.wtgrp.Add(1)
Convey("Do Job makeHEADRequest error context done", func() {
server := StartTestHTTPServer(HTTPRoutes{}, port)
server := test.StartTestHTTPServer(test.HTTPRoutes{}, port)
defer server.Close()
URL := baseURL + "/v2/repo/manifests/manifestRef"
@@ -404,7 +361,7 @@ func TestDoJobErrors(t *testing.T) {
})
Convey("Do Job makeHEADRequest error context not done", func() {
server := StartTestHTTPServer(HTTPRoutes{}, port)
server := test.StartTestHTTPServer(test.HTTPRoutes{}, port)
defer server.Close()
URL := baseURL + "/v2/repo/manifests/manifestRef"
@@ -426,7 +383,7 @@ func TestDoJobErrors(t *testing.T) {
})
Convey("Do Job fetchManifestStruct errors context canceled", func() {
server := StartTestHTTPServer(HTTPRoutes{
server := test.StartTestHTTPServer(test.HTTPRoutes{
{
Route: "/v2/{name}/manifests/{reference}",
HandlerFunc: func(w http.ResponseWriter, r *http.Request) {
@@ -459,7 +416,7 @@ func TestDoJobErrors(t *testing.T) {
})
Convey("Do Job fetchManifestStruct errors context not canceled", func() {
server := StartTestHTTPServer(HTTPRoutes{
server := test.StartTestHTTPServer(test.HTTPRoutes{
{
Route: "/v2/{name}/manifests/{reference}",
HandlerFunc: func(w http.ResponseWriter, r *http.Request) {
@@ -493,7 +450,7 @@ func TestDoJobErrors(t *testing.T) {
})
Convey("Do Job fetchIndexStruct errors context canceled", func() {
server := StartTestHTTPServer(HTTPRoutes{
server := test.StartTestHTTPServer(test.HTTPRoutes{
{
Route: "/v2/{name}/manifests/{reference}",
HandlerFunc: func(w http.ResponseWriter, r *http.Request) {
@@ -526,7 +483,7 @@ func TestDoJobErrors(t *testing.T) {
})
Convey("Do Job fetchIndexStruct errors context not canceled", func() {
server := StartTestHTTPServer(HTTPRoutes{
server := test.StartTestHTTPServer(test.HTTPRoutes{
{
Route: "/v2/{name}/manifests/{reference}",
HandlerFunc: func(w http.ResponseWriter, r *http.Request) {
@@ -559,7 +516,7 @@ func TestDoJobErrors(t *testing.T) {
So(result.StrValue, ShouldResemble, "")
})
Convey("Do Job fetchIndexStruct not supported content type", func() {
server := StartTestHTTPServer(HTTPRoutes{
server := test.StartTestHTTPServer(test.HTTPRoutes{
{
Route: "/v2/{name}/manifests/{reference}",
HandlerFunc: func(w http.ResponseWriter, r *http.Request) {
@@ -589,7 +546,7 @@ func TestDoJobErrors(t *testing.T) {
})
Convey("Media type is MediaTypeImageIndex image.string erorrs", func() {
server := StartTestHTTPServer(HTTPRoutes{
server := test.StartTestHTTPServer(test.HTTPRoutes{
{
Route: "/v2/{name}/manifests/{reference}",
HandlerFunc: func(w http.ResponseWriter, r *http.Request) {
pkg/cli/cve_cmd_test.go
+4 -4
View File
@@ -713,7 +713,7 @@ func TestServerCVEResponse(t *testing.T) {
str := space.ReplaceAllString(buff.String(), " ")
str = strings.TrimSpace(str)
So(err, ShouldBeNil)
So(str, ShouldEqual, "IMAGE NAME TAG OS/ARCH DIGEST SIGNED SIZE zot-cve-test 0.0.1 82836dd7 false 548B")
So(str, ShouldEqual, "IMAGE NAME TAG OS/ARCH DIGEST SIGNED SIZE zot-cve-test 0.0.1 40d1f749 false 605B")
})
Convey("Test images by CVE ID - GQL - invalid CVE ID", t, func() {
@@ -829,7 +829,7 @@ func TestServerCVEResponse(t *testing.T) {
str := space.ReplaceAllString(buff.String(), " ")
So(err, ShouldBeNil)
So(strings.TrimSpace(str), ShouldEqual,
"IMAGE NAME TAG OS/ARCH DIGEST SIGNED SIZE zot-cve-test 0.0.1 82836dd7 false 548B")
"IMAGE NAME TAG OS/ARCH DIGEST SIGNED SIZE zot-cve-test 0.0.1 40d1f749 false 605B")
})
Convey("Test CVE by name and CVE ID - GQL - invalid name and CVE ID", t, func() {
@@ -907,7 +907,7 @@ func TestServerCVEResponse(t *testing.T) {
str := space.ReplaceAllString(buff.String(), " ")
str = strings.TrimSpace(str)
So(err, ShouldBeNil)
So(str, ShouldEqual, "IMAGE NAME TAG OS/ARCH DIGEST SIGNED SIZE zot-cve-test 0.0.1 linux/amd64 82836dd7 false 548B")
So(str, ShouldEqual, "IMAGE NAME TAG OS/ARCH DIGEST SIGNED SIZE zot-cve-test 0.0.1 linux/amd64 40d1f749 false 605B")
})
Convey("Test images by CVE ID - invalid CVE ID", t, func() {
@@ -992,7 +992,7 @@ func TestServerCVEResponse(t *testing.T) {
str := space.ReplaceAllString(buff.String(), " ")
So(err, ShouldBeNil)
So(strings.TrimSpace(str), ShouldEqual,
"IMAGE NAME TAG OS/ARCH DIGEST SIGNED SIZE zot-cve-test 0.0.1 linux/amd64 82836dd7 false 548B")
"IMAGE NAME TAG OS/ARCH DIGEST SIGNED SIZE zot-cve-test 0.0.1 linux/amd64 40d1f749 false 605B")
})
Convey("Test CVE by name and CVE ID - invalid name and CVE ID", t, func() {
pkg/cli/image_cmd_test.go
+15 -15
View File
@@ -333,7 +333,7 @@ func TestSignature(t *testing.T) {
str := space.ReplaceAllString(buff.String(), " ")
actual := strings.TrimSpace(str)
So(actual, ShouldContainSubstring, "IMAGE NAME TAG OS/ARCH DIGEST SIGNED SIZE")
So(actual, ShouldContainSubstring, "repo7 test:1.0 linux/amd64 6742241d true 447B")
So(actual, ShouldContainSubstring, "repo7 test:1.0 linux/amd64 8e59ed3b true 504B")
t.Log("Test getting all images using rest calls to get catalog and individual manifests")
cmd = MockNewImageCommand(new(searchService))
@@ -346,7 +346,7 @@ func TestSignature(t *testing.T) {
str = space.ReplaceAllString(buff.String(), " ")
actual = strings.TrimSpace(str)
So(actual, ShouldContainSubstring, "IMAGE NAME TAG OS/ARCH DIGEST SIGNED SIZE")
So(actual, ShouldContainSubstring, "repo7 test:1.0 linux/amd64 6742241d true 447B")
So(actual, ShouldContainSubstring, "repo7 test:1.0 linux/amd64 8e59ed3b true 504B")
err = os.Chdir(currentWorkingDir)
So(err, ShouldBeNil)
@@ -410,7 +410,7 @@ func TestSignature(t *testing.T) {
str := space.ReplaceAllString(buff.String(), " ")
actual := strings.TrimSpace(str)
So(actual, ShouldContainSubstring, "IMAGE NAME TAG OS/ARCH DIGEST SIGNED SIZE")
So(actual, ShouldContainSubstring, "repo7 0.0.1 linux/amd64 6742241d true 447B")
So(actual, ShouldContainSubstring, "repo7 0.0.1 linux/amd64 8e59ed3b true 504B")
t.Log("Test getting all images using rest calls to get catalog and individual manifests")
cmd = MockNewImageCommand(new(searchService))
@@ -423,7 +423,7 @@ func TestSignature(t *testing.T) {
str = space.ReplaceAllString(buff.String(), " ")
actual = strings.TrimSpace(str)
So(actual, ShouldContainSubstring, "IMAGE NAME TAG OS/ARCH DIGEST SIGNED SIZE")
So(actual, ShouldContainSubstring, "repo7 0.0.1 linux/amd64 6742241d true 447B")
So(actual, ShouldContainSubstring, "repo7 0.0.1 linux/amd64 8e59ed3b true 504B")
err = os.Chdir(currentWorkingDir)
So(err, ShouldBeNil)
@@ -1333,13 +1333,13 @@ func runDisplayIndexTests(baseURL string) {
actual := strings.TrimSpace(str)
// Actual cli output should be something similar to (order of images may differ):
// IMAGE NAME TAG OS/ARCH DIGEST SIGNED SIZE
// repo multi-arch * 46b78b06 false 1.4kB
// linux/amd64 97b0d65c false 577B
// repo multi-arch * 59b25ae4 false 1.5kB
// linux/amd64 97b0d65c false 634B
// windows/arm64/v6 dcfa3a9c false 444B
So(actual, ShouldContainSubstring, "IMAGE NAME TAG OS/ARCH DIGEST SIGNED SIZE")
So(actual, ShouldContainSubstring, "repo multi-arch * 46b78b06 false 1.4kB ")
So(actual, ShouldContainSubstring, "linux/amd64 97b0d65c false 577B ")
So(actual, ShouldContainSubstring, "windows/arm64/v6 dcfa3a9c false 444B")
So(actual, ShouldContainSubstring, "repo multi-arch * 59b25ae4 false 1.5kB ")
So(actual, ShouldContainSubstring, "linux/amd64 2ab1a275 false 634B ")
So(actual, ShouldContainSubstring, "windows/arm64/v6 55fdd23a false 501B")
})
Convey("Test Image Index Verbose", func() {
@@ -1361,18 +1361,18 @@ func runDisplayIndexTests(baseURL string) {
actual := strings.TrimSpace(str)
// Actual cli output should be something similar to (order of images may differ):
// IMAGE NAME TAG OS/ARCH DIGEST CONFIG SIGNED LAYERS SIZE
// repo multi-arch * 46b78b06 false 1.4kB
// linux/amd64 97b0d65c 58cc9abe false 577B
// repo multi-arch * 59b25ae4 false 1.5kB
// linux/amd64 2ab1a275 58cc9abe false 634B
// cbb5b121 4B
// a00291e8 4B
// windows/arm64/v6 dcfa3a9c 5132a1cd false 444B
// windows/arm64/v6 55fdd23a 5132a1cd false 501B
// 7d08ce29 4B
So(actual, ShouldContainSubstring, "IMAGE NAME TAG OS/ARCH DIGEST CONFIG SIGNED LAYERS SIZE")
So(actual, ShouldContainSubstring, "repo multi-arch * 46b78b06 false 1.4kB")
So(actual, ShouldContainSubstring, "linux/amd64 97b0d65c 58cc9abe false 577B")
So(actual, ShouldContainSubstring, "repo multi-arch * 59b25ae4 false 1.5kB")
So(actual, ShouldContainSubstring, "linux/amd64 2ab1a275 58cc9abe false 634B")
So(actual, ShouldContainSubstring, "cbb5b121 4B")
So(actual, ShouldContainSubstring, "a00291e8 4B")
So(actual, ShouldContainSubstring, "windows/arm64/v6 dcfa3a9c 5132a1cd false 444B")
So(actual, ShouldContainSubstring, "windows/arm64/v6 55fdd23a 5132a1cd false 501B")
So(actual, ShouldContainSubstring, "7d08ce29 4B")
})
}
pkg/common/common.go
+9
View File
@@ -17,6 +17,7 @@ import (
"unicode/utf8"
"github.com/opencontainers/go-digest"
ispec "github.com/opencontainers/image-spec/specs-go/v1"
"zotregistry.io/zot/pkg/log"
)
@@ -274,3 +275,11 @@ func DContains(slice []digest.Digest, item digest.Digest) bool {
return false
}
func GetManifestArtifactType(manifestContent ispec.Manifest) string {
if manifestContent.ArtifactType != "" {
return manifestContent.ArtifactType
}
return manifestContent.Config.MediaType
}
pkg/extensions/lint/lint.go
+2 -1
View File
@@ -78,7 +78,8 @@ func (linter *Linter) CheckMandatoryAnnotations(repo string, manifestDigest godi
content, err = imgStore.GetBlobContent(repo, configDigest)
if err != nil {
linter.log.Error().Err(err).Msg("linter: couldn't get config JSON " + configDigest.String())
linter.log.Error().Err(err).Msg("linter: couldn't get config JSON " +
configDigest.String())
return false, err
}
pkg/extensions/search/convert/convert_test.go
+1 -1
View File
@@ -183,7 +183,7 @@ func TestConvertErrors(t *testing.T) {
Referrers: map[string][]repodb.ReferrerInfo{},
},
repodb.ManifestMetadata{
ManifestBlob: []byte("{}"),
ManifestBlob: []byte(`{}`),
ConfigBlob: []byte("bad json"),
},
nil,
pkg/extensions/search/cve/cve_test.go
+1 -1
View File
@@ -923,7 +923,7 @@ func TestCVEStruct(t *testing.T) {
},
Layers: []ispec.Descriptor{
{
MediaType: ispec.MediaTypeImageLayerNonDistributableGzip,
MediaType: ispec.MediaTypeImageLayerNonDistributableGzip, //nolint:staticcheck
Size: 0,
Digest: godigest.NewDigestFromEncoded(godigest.SHA256, "digest"),
},
pkg/extensions/search/resolver_test.go
+1 -1
View File
@@ -1449,7 +1449,7 @@ func TestGetReferrers(t *testing.T) {
Convey("GetReferrers return index of descriptors", func() {
testLogger := log.NewLogger("debug", "")
referrerDescriptor := ispec.Descriptor{
MediaType: ispec.MediaTypeArtifactManifest,
MediaType: ispec.MediaTypeImageManifest,
ArtifactType: "com.artifact.test",
Size: 403,
Digest: godigest.FromString("test"),
pkg/extensions/search/search_test.go
+80 -53
View File
@@ -21,6 +21,7 @@ import (
dbTypes "github.com/aquasecurity/trivy-db/pkg/types"
"github.com/gobwas/glob"
regTypes "github.com/google/go-containerregistry/pkg/v1/types"
notreg "github.com/notaryproject/notation-go/registry"
godigest "github.com/opencontainers/go-digest"
"github.com/opencontainers/image-spec/specs-go"
ispec "github.com/opencontainers/image-spec/specs-go/v1"
@@ -31,7 +32,7 @@ import (
"zotregistry.io/zot/pkg/api"
"zotregistry.io/zot/pkg/api/config"
"zotregistry.io/zot/pkg/api/constants"
"zotregistry.io/zot/pkg/common"
zcommon "zotregistry.io/zot/pkg/common"
extconf "zotregistry.io/zot/pkg/extensions/config"
"zotregistry.io/zot/pkg/extensions/monitoring"
cveinfo "zotregistry.io/zot/pkg/extensions/search/cve"
@@ -97,7 +98,7 @@ type ReferrersResp struct {
}
type ReferrersResult struct {
Referrers []common.Referrer `json:"referrers"`
Referrers []zcommon.Referrer `json:"referrers"`
}
type GlobalSearchResultResp struct {
GlobalSearchResult GlobalSearchResult `json:"data"`
@@ -109,24 +110,24 @@ type GlobalSearchResult struct {
}
type GlobalSearch struct {
Images []common.ImageSummary `json:"images"`
Repos []common.RepoSummary `json:"repos"`
Layers []common.LayerSummary `json:"layers"`
Page repodb.PageInfo `json:"page"`
Images []zcommon.ImageSummary `json:"images"`
Repos []zcommon.RepoSummary `json:"repos"`
Layers []zcommon.LayerSummary `json:"layers"`
Page repodb.PageInfo `json:"page"`
}
type ExpandedRepoInfo struct {
RepoInfo common.RepoInfo `json:"expandedRepoInfo"`
RepoInfo zcommon.RepoInfo `json:"expandedRepoInfo"`
}
type PaginatedReposResult struct {
Results []common.RepoSummary `json:"results"`
Page repodb.PageInfo `json:"page"`
Results []zcommon.RepoSummary `json:"results"`
Page repodb.PageInfo `json:"page"`
}
type PaginatedImagesResult struct {
Results []common.ImageSummary `json:"results"`
Page repodb.PageInfo `json:"page"`
Results []zcommon.ImageSummary `json:"results"`
Page repodb.PageInfo `json:"page"`
}
//nolint:tagliatelle // graphQL schema
@@ -140,7 +141,7 @@ type ErrorGQL struct {
}
type SingleImageSummary struct {
ImageSummary common.ImageSummary `json:"Image"` //nolint:tagliatelle
ImageSummary zcommon.ImageSummary `json:"Image"` //nolint:tagliatelle
}
type ImageSummaryResult struct {
SingleImageSummary SingleImageSummary `json:"data"`
@@ -189,7 +190,7 @@ func readFileAndSearchString(filePath string, stringToMatch string, timeout time
}
func verifyRepoSummaryFields(t *testing.T,
actualRepoSummary, expectedRepoSummary *common.RepoSummary,
actualRepoSummary, expectedRepoSummary *zcommon.RepoSummary,
) {
t.Helper()
@@ -218,7 +219,7 @@ func verifyRepoSummaryFields(t *testing.T,
}
func verifyImageSummaryFields(t *testing.T,
actualImageSummary, expectedImageSummary *common.ImageSummary,
actualImageSummary, expectedImageSummary *zcommon.ImageSummary,
) {
t.Helper()
@@ -972,30 +973,37 @@ func TestGetReferrersGQL(t *testing.T) {
artifactContentBlobDigest := godigest.FromBytes(artifactContentBlob)
artifactType := "com.artifact.test"
err = UploadBlob(baseURL, repo, artifactContentBlob, artifactContentType)
So(err, ShouldBeNil)
artifact := &ispec.Artifact{
Blobs: []ispec.Descriptor{
{
MediaType: artifactContentType,
Digest: artifactContentBlobDigest,
Size: artifactContentBlobSize,
artifactImg := Image{
Manifest: ispec.Manifest{
Layers: []ispec.Descriptor{
{
MediaType: artifactContentType,
Digest: artifactContentBlobDigest,
Size: artifactContentBlobSize,
},
},
Subject: subjectDescriptor,
ArtifactType: artifactType,
Config: ispec.Descriptor{
MediaType: ispec.MediaTypeScratch,
Digest: ispec.ScratchDescriptor.Digest,
Data: ispec.ScratchDescriptor.Data,
},
MediaType: ispec.MediaTypeImageManifest,
Annotations: map[string]string{
"com.artifact.format": "test",
},
},
Subject: subjectDescriptor,
ArtifactType: artifactType,
MediaType: ispec.MediaTypeArtifactManifest,
Annotations: map[string]string{
"com.artifact.format": "test",
},
Config: ispec.Image{},
Layers: [][]byte{artifactContentBlob},
}
artifactManifestBlob, err := json.Marshal(artifact)
artifactManifestBlob, err := json.Marshal(artifactImg.Manifest)
So(err, ShouldBeNil)
artifactManifestDigest := godigest.FromBytes(artifactManifestBlob)
artifactImg.Reference = artifactManifestDigest.String()
err = UploadArtifactManifest(artifact, nil, baseURL, repo)
err = UploadImage(artifactImg, baseURL, repo)
So(err, ShouldBeNil)
gqlQuery := `
@@ -1029,7 +1037,7 @@ func TestGetReferrersGQL(t *testing.T) {
So(err, ShouldBeNil)
So(referrersResp.Errors, ShouldBeNil)
So(referrersResp.ReferrersResult.Referrers[0].ArtifactType, ShouldEqual, artifactType)
So(referrersResp.ReferrersResult.Referrers[0].MediaType, ShouldEqual, ispec.MediaTypeArtifactManifest)
So(referrersResp.ReferrersResult.Referrers[0].MediaType, ShouldEqual, ispec.MediaTypeImageManifest)
So(referrersResp.ReferrersResult.Referrers[0].Annotations[0].Key, ShouldEqual, "com.artifact.format")
So(referrersResp.ReferrersResult.Referrers[0].Annotations[0].Value, ShouldEqual, "test")
@@ -1505,6 +1513,7 @@ func TestExpandedRepoInfo(t *testing.T) {
})
Convey("Test expanded repo info with tagged referrers", t, func() {
const test = "test"
rootDir := t.TempDir()
port := GetFreePort()
baseURL := GetBaseURL(port)
@@ -1524,7 +1533,7 @@ func TestExpandedRepoInfo(t *testing.T) {
ctlrManager.StartAndWait(port)
defer ctlrManager.StopServer()
image, err := GetRandomImage("test")
image, err := GetRandomImage(test)
So(err, ShouldBeNil)
manifestDigest, err := image.Digest()
So(err, ShouldBeNil)
@@ -1532,17 +1541,15 @@ func TestExpandedRepoInfo(t *testing.T) {
err = UploadImage(image, baseURL, "repo")
So(err, ShouldBeNil)
referrer, err := GetRandomArtifact(&ispec.Descriptor{
Digest: manifestDigest,
MediaType: ispec.MediaTypeImageManifest,
})
referrer, err := GetImageWithSubject(manifestDigest, ispec.MediaTypeImageManifest)
So(err, ShouldBeNil)
tag := "test-ref-tag"
err = UploadArtifactManifest(&referrer.Manifest, &tag, baseURL, "repo")
referrer.Reference = tag
err = UploadImage(referrer, baseURL, "repo")
So(err, ShouldBeNil)
// ------- Make the call to GQL and see that it doesn't crash and that the referrer isn't in the list of tags
// ------- Make the call to GQL and see that it doesn't crash
responseStruct := &ExpandedRepoInfoResp{}
query := `
{
@@ -1564,10 +1571,23 @@ func TestExpandedRepoInfo(t *testing.T) {
err = json.Unmarshal(resp.Body(), responseStruct)
So(err, ShouldBeNil)
So(len(responseStruct.ExpandedRepoInfo.RepoInfo.ImageSummaries), ShouldEqual, 1)
So(len(responseStruct.ExpandedRepoInfo.RepoInfo.ImageSummaries), ShouldEqual, 2)
repoInfo := responseStruct.ExpandedRepoInfo.RepoInfo
So(repoInfo.ImageSummaries[0].Tag, ShouldEqual, "test")
foundTagTest := false
foundTagRefTag := false
for _, imgSum := range repoInfo.ImageSummaries {
switch imgSum.Tag {
case test:
foundTagTest = true
case "test-ref-tag":
foundTagRefTag = true
}
}
So(foundTagTest || foundTagRefTag, ShouldEqual, true)
})
Convey("Test image tags order", t, func() {
@@ -3241,8 +3261,8 @@ func TestGlobalSearch(t *testing.T) {
repos, err := olu.GetRepositories()
So(err, ShouldBeNil)
allExpectedRepoInfoMap := make(map[string]common.RepoInfo)
allExpectedImageSummaryMap := make(map[string]common.ImageSummary)
allExpectedRepoInfoMap := make(map[string]zcommon.RepoInfo)
allExpectedImageSummaryMap := make(map[string]zcommon.ImageSummary)
for _, repo := range repos {
repoInfo, err := olu.GetExpandedRepoInfo(repo)
So(err, ShouldBeNil)
@@ -3310,8 +3330,8 @@ func TestGlobalSearch(t *testing.T) {
t.Logf("returned layers: %v", responseStruct.GlobalSearchResult.GlobalSearch.Layers)
So(responseStruct.GlobalSearchResult.GlobalSearch.Layers, ShouldBeEmpty)
newestImageMap := make(map[string]common.ImageSummary)
actualRepoMap := make(map[string]common.RepoSummary)
newestImageMap := make(map[string]zcommon.ImageSummary)
actualRepoMap := make(map[string]zcommon.RepoSummary)
for _, repo := range responseStruct.GlobalSearchResult.GlobalSearch.Repos {
newestImageMap[repo.Name] = repo.NewestImage
actualRepoMap[repo.Name] = repo
@@ -3570,8 +3590,8 @@ func TestGlobalSearch(t *testing.T) {
repos, err := olu.GetRepositories()
So(err, ShouldBeNil)
allExpectedRepoInfoMap := make(map[string]common.RepoInfo)
allExpectedImageSummaryMap := make(map[string]common.ImageSummary)
allExpectedRepoInfoMap := make(map[string]zcommon.RepoInfo)
allExpectedImageSummaryMap := make(map[string]zcommon.ImageSummary)
for _, repo := range repos {
repoInfo, err := olu.GetExpandedRepoInfo(repo)
So(err, ShouldBeNil)
@@ -3637,8 +3657,8 @@ func TestGlobalSearch(t *testing.T) {
t.Logf("returned layers: %v", responseStruct.GlobalSearchResult.GlobalSearch.Layers)
So(responseStruct.GlobalSearchResult.GlobalSearch.Layers, ShouldBeEmpty)
newestImageMap := make(map[string]common.ImageSummary)
actualRepoMap := make(map[string]common.RepoSummary)
newestImageMap := make(map[string]zcommon.ImageSummary)
actualRepoMap := make(map[string]zcommon.RepoSummary)
for _, repo := range responseStruct.GlobalSearchResult.GlobalSearch.Repos {
newestImageMap[repo.Name] = repo.NewestImage
actualRepoMap[repo.Name] = repo
@@ -5626,10 +5646,17 @@ func TestRepoDBWhenDeletingImages(t *testing.T) {
signatureReference := ""
var sigManifestContent ispec.Artifact
var sigManifestContent ispec.Manifest
for _, manifest := range indexContent.Manifests {
if manifest.MediaType == ispec.MediaTypeArtifactManifest {
manifestBlob, _, _, err := storage.GetImageManifest(repo, manifest.Digest.String())
So(err, ShouldBeNil)
var manifestContent ispec.Manifest
err = json.Unmarshal(manifestBlob, &manifestContent)
So(err, ShouldBeNil)
if zcommon.GetManifestArtifactType(manifestContent) == notreg.ArtifactTypeNotation {
signatureReference = manifest.Digest.String()
manifestBlob, _, _, err := storage.GetImageManifest(repo, signatureReference)
So(err, ShouldBeNil)
@@ -6233,11 +6260,11 @@ func TestImageSummary(t *testing.T) {
So(imgSummary.Vulnerabilities.Count, ShouldEqual, 0)
So(imgSummary.Vulnerabilities.MaxSeverity, ShouldEqual, "")
So(len(imgSummary.Referrers), ShouldEqual, 1)
So(imgSummary.Referrers[0], ShouldResemble, common.Referrer{
So(imgSummary.Referrers[0], ShouldResemble, zcommon.Referrer{
MediaType: ispec.MediaTypeImageManifest,
ArtifactType: "test.artifact.type",
Digest: referrerManifestDigest.String(),
Annotations: []common.Annotation{{Key: "testAnnotationKey", Value: "testAnnotationValue"}},
Annotations: []zcommon.Annotation{{Key: "testAnnotationKey", Value: "testAnnotationValue"}},
})
t.Log("starting Test retrieve duplicated image same layers based on image identifier")
pkg/extensions/sync/signatures.go
+2 -130
View File
@@ -355,23 +355,8 @@ func (sig *signaturesCopier) syncOCIRefs(localRepo, remoteRepo, digestStr string
if err := syncBlob(sig, imageStore, localRepo, remoteRepo, manifest.Config.Digest); err != nil {
return err
}
} else if ref.MediaType == ispec.MediaTypeArtifactManifest {
// read manifest
var manifest ispec.Artifact
err = json.Unmarshal(OCIRefBody, &manifest)
if err != nil {
sig.log.Error().Str("errorType", common.TypeOf(err)).Err(err).
Str("manifest", getRefManifestURL.String()).Msg("couldn't unmarshal oci reference manifest")
return err
}
for _, layer := range manifest.Blobs {
if err := syncBlob(sig, imageStore, localRepo, remoteRepo, layer.Digest); err != nil {
return err
}
}
} else {
continue
}
digest, err := imageStore.PutImageManifest(localRepo, ref.Digest.String(),
@@ -404,78 +389,6 @@ func (sig *signaturesCopier) syncOCIRefs(localRepo, remoteRepo, digestStr string
return nil
}
func (sig *signaturesCopier) syncOCIArtifact(localRepo, remoteRepo, reference string,
ociArtifactBuf []byte,
) error {
var ociArtifact ispec.Artifact
err := json.Unmarshal(ociArtifactBuf, &ociArtifact)
if err != nil {
sig.log.Error().Err(err).Str("repository", remoteRepo).Str("reference", reference).
Msg("couldn't unmarshal OCI artifact")
return err
}
canSkipOCIArtifact, err := sig.canSkipOCIArtifact(localRepo, reference, ociArtifact)
if err != nil {
sig.log.Error().Err(err).Str("repository", remoteRepo).Str("reference", reference).
Msg("couldn't check if OCI artifact can be skipped")
}
if canSkipOCIArtifact {
return nil
}
imageStore := sig.storeController.GetImageStore(localRepo)
sig.log.Info().Msg("syncing OCI artifacts")
for _, blob := range ociArtifact.Blobs {
if err := syncBlob(sig, imageStore, localRepo, remoteRepo, blob.Digest); err != nil {
return err
}
}
artifactManifestBuf, err := json.Marshal(ociArtifact)
if err != nil {
sig.log.Error().Str("errorType", common.TypeOf(err)).
Err(err).Msg("couldn't marshal OCI artifact")
return err
}
// push manifest
digest, err := imageStore.PutImageManifest(localRepo, reference,
ispec.MediaTypeArtifactManifest, artifactManifestBuf)
if err != nil {
sig.log.Error().Str("errorType", common.TypeOf(err)).
Err(err).Msg("couldn't upload OCI artifact manifest")
return err
}
if sig.repoDB != nil {
sig.log.Debug().Str("repository", localRepo).Str("digest", digest.String()).
Msg("trying to OCI refs for repo digest")
err = repodb.SetMetadataFromInput(localRepo, reference, ispec.MediaTypeArtifactManifest,
digest, artifactManifestBuf, sig.storeController.GetImageStore(localRepo),
sig.repoDB, sig.log)
if err != nil {
return fmt.Errorf("failed to set metadata for OCI Artifact '%s@%s': %w", localRepo, digest.String(), err)
}
sig.log.Info().Str("repository", localRepo).Str("digest", digest.String()).
Msg("successfully added oci artifacts to RepoDB for repo digest")
}
sig.log.Info().Str("repository", localRepo).Str("tag", reference).
Msg("successfully synced OCI artifact for repo tag")
return nil
}
func (sig *signaturesCopier) canSkipORASRefs(localRepo, digestStr string, refs ReferenceList,
) (bool, error) {
imageStore := sig.storeController.GetImageStore(localRepo)
@@ -509,47 +422,6 @@ func (sig *signaturesCopier) canSkipORASRefs(localRepo, digestStr string, refs R
return true, nil
}
func (sig *signaturesCopier) canSkipOCIArtifact(localRepo, reference string, artifact ispec.Artifact,
) (bool, error) {
imageStore := sig.storeController.GetImageStore(localRepo)
var localArtifactManifest ispec.Artifact
localArtifactBuf, _, _, err := imageStore.GetImageManifest(localRepo, reference)
if err != nil {
if errors.Is(err, zerr.ErrManifestNotFound) || errors.Is(err, zerr.ErrRepoNotFound) {
return false, nil
}
sig.log.Error().Str("errorType", common.TypeOf(err)).Err(err).
Str("repository", localRepo).Str("reference", reference).
Msg("couldn't get local OCI artifact manifest")
return false, err
}
err = json.Unmarshal(localArtifactBuf, &localArtifactManifest)
if err != nil {
sig.log.Error().Str("errorType", common.TypeOf(err)).Err(err).
Str("repository", localRepo).Str("reference", reference).
Msg("couldn't unmarshal local OCI artifact manifest")
return false, err
}
if !artifactsEqual(localArtifactManifest, artifact) {
sig.log.Info().Str("repository", localRepo).Str("reference", reference).
Msg("upstream OCI artifact changed, syncing again")
return false, nil
}
sig.log.Info().Str("repository", localRepo).Str("reference", reference).
Msg("skipping OCI artifact, already synced")
return true, nil
}
func (sig *signaturesCopier) canSkipCosignSignature(localRepo, digestStr string, cosignManifest *ispec.Manifest,
) (bool, error) {
imageStore := sig.storeController.GetImageStore(localRepo)
pkg/extensions/sync/sync_internal_test.go
+56 -61
View File
@@ -361,7 +361,7 @@ func TestSyncInternal(t *testing.T) {
So(regURL, ShouldNotBeNil)
ref := ispec.Descriptor{
MediaType: ispec.MediaTypeArtifactManifest,
MediaType: ispec.MediaTypeImageManifest,
Digest: "fakeDigest",
ArtifactType: "application/vnd.cncf.notary.signature",
}
@@ -406,7 +406,7 @@ func TestSyncInternal(t *testing.T) {
refs := ispec.Index{Manifests: []ispec.Descriptor{
{
MediaType: ispec.MediaTypeArtifactManifest,
MediaType: ispec.MediaTypeImageManifest,
Digest: "fakeDigest",
ArtifactType: "application/vnd.cncf.notary.signature",
},
@@ -446,7 +446,7 @@ func TestSyncInternal(t *testing.T) {
err = json.Unmarshal(buf, &index)
So(err, ShouldBeNil)
index.Manifests = append(index.Manifests, ispec.Descriptor{
MediaType: ispec.MediaTypeArtifactManifest,
MediaType: ispec.MediaTypeImageManifest,
Digest: godigest.FromString(""),
ArtifactType: "application/vnd.cncf.notary.signature",
})
@@ -548,6 +548,59 @@ func TestSyncInternal(t *testing.T) {
So(canBeSkipped, ShouldBeFalse)
})
Convey("Test syncOCIRefs with bad mediaType", t, func() {
downPort, upPort := test.GetFreePort(), test.GetFreePort()
downStream := test.StartTestHTTPServer(
[]test.RouteHandler{
{
Route: "",
HandlerFunc: func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusOK)
},
AllowedMethods: []string{},
},
},
downPort,
)
defer downStream.Close()
upStream := test.StartTestHTTPServer(
[]test.RouteHandler{
{
Route: "/v2/{name}/manifests/{digest}",
HandlerFunc: func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusOK)
_, err := w.Write([]byte("{}"))
if err != nil {
t.FailNow()
}
},
AllowedMethods: []string{"GET"},
},
},
upPort,
)
defer upStream.Close()
upStreamURL, err := url.Parse(fmt.Sprintf("http://127.0.0.1:%s", upPort))
So(err, ShouldBeNil)
client := &http.Client{}
mockRepoDB := mocks.RepoDBMock{}
mockImageStore := mocks.MockedImageStore{}
sig := newSignaturesCopier(client, syncconf.Credentials{},
*upStreamURL, mockRepoDB, storage.StoreController{DefaultStore: mockImageStore},
log.NewLogger("debug", ""),
)
digest := godigest.FromString("1")
err = sig.syncOCIRefs("repo", "repo", digest.String(),
ispec.Index{Manifests: []ispec.Descriptor{{MediaType: "bad media type", Digest: digest}}})
So(err, ShouldBeNil)
})
Convey("Test filterRepos()", t, func() {
repos := []string{"repo", "repo1", "repo2", "repo/repo2", "repo/repo2/repo3/repo4"}
contents := []syncconf.Content{
@@ -1265,61 +1318,3 @@ func TestCompareArtifactRefs(t *testing.T) {
}
})
}
func TestCompareArtifactManifests(t *testing.T) {
testCases := []struct {
refs1 ispec.Artifact
refs2 ispec.Artifact
expected bool
}{
{
refs1: ispec.Artifact{
MediaType: "mediatype",
ArtifactType: "signature",
Blobs: []ispec.Descriptor{
{
Digest: "digest1",
},
},
},
refs2: ispec.Artifact{
MediaType: "mediatype",
ArtifactType: "signature",
Blobs: []ispec.Descriptor{
{
Digest: "digest1",
},
},
},
expected: true,
},
{
refs1: ispec.Artifact{
MediaType: "mediatype",
ArtifactType: "signature",
Blobs: []ispec.Descriptor{
{
Digest: "digest1",
},
},
},
refs2: ispec.Artifact{
MediaType: "mediatype",
ArtifactType: "signature",
Blobs: []ispec.Descriptor{
{
Digest: "digest2",
},
},
},
expected: false,
},
}
Convey("Test artifactsEqual()", t, func() {
for _, test := range testCases {
actualResult := artifactsEqual(test.refs1, test.refs2)
So(actualResult, ShouldEqual, test.expected)
}
})
}
pkg/extensions/sync/sync_test.go
+86 -362
View File
@@ -400,19 +400,24 @@ func TestORAS(t *testing.T) {
digest = godigest.FromBytes(resp.Body())
content := []byte("blob content")
adigest := pushBlob(srcBaseURL, repoName, content)
// layer
layer := []byte("blob content")
blobDigest := pushBlob(srcBaseURL, repoName, layer)
artifactManifest := ispec.Artifact{
// config
_ = pushBlob(srcBaseURL, repoName, ispec.ScratchDescriptor.Data)
artifactManifest := ispec.Manifest{
MediaType: artifactspec.MediaTypeArtifactManifest,
ArtifactType: "application/vnd.oras.artifact",
Blobs: []ispec.Descriptor{
Layers: []ispec.Descriptor{
{
MediaType: "application/octet-stream",
Digest: adigest,
Size: int64(len(content)),
Digest: blobDigest,
Size: int64(len(layer)),
},
},
Config: ispec.ScratchDescriptor,
Subject: &ispec.Descriptor{
MediaType: "application/vnd.oci.image.manifest.v1+json",
Digest: digest,
@@ -420,19 +425,15 @@ func TestORAS(t *testing.T) {
},
}
content, err = json.Marshal(artifactManifest)
if err != nil {
panic(err)
}
artManifestBlob, err := json.Marshal(artifactManifest)
So(err, ShouldBeNil)
adigest = godigest.FromBytes(content)
artifactDigest := godigest.FromBytes(artManifestBlob)
// put OCI reference artifact mediaType artifact
_, err = resty.R().SetHeader("Content-Type", artifactspec.MediaTypeArtifactManifest).
SetBody(content).Put(srcBaseURL + fmt.Sprintf("/v2/%s/manifests/%s", repoName, adigest.String()))
if err != nil {
panic(err)
}
SetBody(artManifestBlob).Put(srcBaseURL + fmt.Sprintf("/v2/%s/manifests/%s", repoName, artifactDigest.String()))
So(err, ShouldBeNil)
err = os.Chmod(path.Join(destDir, testImage, "index.json"), 0o000)
So(err, ShouldBeNil)
@@ -447,7 +448,7 @@ func TestORAS(t *testing.T) {
So(err, ShouldBeNil)
// trigger getORASRefs err
err = os.Chmod(path.Join(srcDir, testImage, "blobs/sha256", adigest.Encoded()), 0o000)
err = os.Chmod(path.Join(srcDir, testImage, "blobs/sha256", artifactDigest.Encoded()), 0o000)
So(err, ShouldBeNil)
resp, err = resty.R().Get(destBaseURL + "/v2/" + testImage + "/manifests/" + digest.String())
@@ -455,7 +456,7 @@ func TestORAS(t *testing.T) {
So(resp, ShouldNotBeEmpty)
So(resp.StatusCode(), ShouldEqual, http.StatusOK)
err = os.Chmod(path.Join(srcDir, testImage, "blobs/sha256", adigest.Encoded()), 0o755)
err = os.Chmod(path.Join(srcDir, testImage, "blobs/sha256", artifactDigest.Encoded()), 0o755)
So(err, ShouldBeNil)
resp, err = resty.R().Get(getORASReferrersURL)
@@ -473,11 +474,12 @@ func TestORAS(t *testing.T) {
err = os.RemoveAll(path.Join(destDir, repoName))
So(err, ShouldBeNil)
err = os.WriteFile(path.Join(srcDir, repoName, "blobs", "sha256", adigest.Encoded()), []byte("wrong content"), 0o600)
err = os.WriteFile(path.Join(srcDir, repoName, "blobs", "sha256", artifactDigest.Encoded()),
[]byte("wrong content"), 0o600)
So(err, ShouldBeNil)
_, err = resty.R().SetHeader("Content-Type", artifactspec.MediaTypeArtifactManifest).
SetBody(content).Put(srcBaseURL + fmt.Sprintf("/v2/%s/manifests/%s", repoName, adigest.String()))
SetBody(artManifestBlob).Put(srcBaseURL + fmt.Sprintf("/v2/%s/manifests/%s", repoName, artifactDigest.String()))
if err != nil {
panic(err)
}
@@ -665,20 +667,33 @@ func TestOnDemand(t *testing.T) {
So(err, ShouldBeNil)
// add OCI Ref
OCIRefManifest := ispec.Artifact{
_ = pushBlob(srcBaseURL, "remote-repo", ispec.ScratchDescriptor.Data)
OCIRefManifest := ispec.Manifest{
Subject: &ispec.Descriptor{
MediaType: ispec.MediaTypeImageManifest,
Digest: manifestDigest,
},
Blobs: []ispec.Descriptor{},
MediaType: ispec.MediaTypeArtifactManifest,
Config: ispec.Descriptor{
MediaType: ispec.MediaTypeScratch,
Digest: ispec.ScratchDescriptor.Digest,
Size: 2,
},
Layers: []ispec.Descriptor{
{
MediaType: ispec.MediaTypeScratch,
Digest: ispec.ScratchDescriptor.Digest,
Size: 2,
},
},
MediaType: ispec.MediaTypeImageManifest,
}
OCIRefManifestBlob, err := json.Marshal(OCIRefManifest)
So(err, ShouldBeNil)
resp, err := resty.R().
SetHeader("Content-type", ispec.MediaTypeArtifactManifest).
SetHeader("Content-type", ispec.MediaTypeImageManifest).
SetBody(OCIRefManifestBlob).
Put(srcBaseURL + "/v2/remote-repo/manifests/oci.ref")
@@ -763,7 +778,7 @@ func TestOnDemand(t *testing.T) {
dctlr.RepoDB = mocks.RepoDBMock{
SetRepoReferenceFn: func(repo, Reference string, manifestDigest godigest.Digest, mediaType string) error {
if mediaType == ispec.MediaTypeArtifactManifest {
if mediaType == ispec.MediaTypeImageManifest {
return sync.ErrTestError
}
@@ -854,7 +869,7 @@ func TestSyncWithNonDistributableBlob(t *testing.T) {
nonDistributableLayer := make([]byte, 10)
nonDistributableDigest := godigest.FromBytes(nonDistributableLayer)
nonDistributableLayerDesc := ispec.Descriptor{
MediaType: ispec.MediaTypeImageLayerNonDistributableGzip,
MediaType: ispec.MediaTypeImageLayerNonDistributableGzip, //nolint:staticcheck
Digest: nonDistributableDigest,
Size: int64(len(nonDistributableLayer)),
URLs: []string{
@@ -3018,7 +3033,7 @@ func TestPeriodicallySignaturesErr(t *testing.T) {
So(err, ShouldBeNil)
// read manifest
var artifactManifest ispec.Artifact
var artifactManifest ispec.Manifest
for _, ref := range referrers.Manifests {
refPath := path.Join(srcDir, repoName, "blobs", string(ref.Digest.Algorithm()), ref.Digest.Encoded())
body, err := os.ReadFile(refPath)
@@ -3028,7 +3043,7 @@ func TestPeriodicallySignaturesErr(t *testing.T) {
So(err, ShouldBeNil)
// triggers perm denied on sig blobs
for _, blob := range artifactManifest.Blobs {
for _, blob := range artifactManifest.Layers {
blobPath := path.Join(srcDir, repoName, "blobs", string(blob.Digest.Algorithm()), blob.Digest.Encoded())
err := os.Chmod(blobPath, 0o000)
So(err, ShouldBeNil)
@@ -3145,7 +3160,7 @@ func TestPeriodicallySignaturesErr(t *testing.T) {
Convey("of type OCI artifact", func() { //nolint: dupl
// read manifest
var artifactManifest ispec.Artifact
var artifactManifest ispec.Manifest
for _, ref := range referrers.Manifests {
refPath := path.Join(srcDir, repoName, "blobs", string(ref.Digest.Algorithm()), ref.Digest.Encoded())
body, err := os.ReadFile(refPath)
@@ -3155,7 +3170,7 @@ func TestPeriodicallySignaturesErr(t *testing.T) {
So(err, ShouldBeNil)
// triggers perm denied on artifact blobs
for _, blob := range artifactManifest.Blobs {
for _, blob := range artifactManifest.Layers {
blobPath := path.Join(srcDir, repoName, "blobs", string(blob.Digest.Algorithm()), blob.Digest.Encoded())
err := os.Chmod(blobPath, 0o000)
So(err, ShouldBeNil)
@@ -3349,7 +3364,7 @@ func TestSignatures(t *testing.T) {
err = os.RemoveAll(path.Join(destDir, repoName))
So(err, ShouldBeNil)
var artifactManifest ispec.Artifact
var artifactManifest ispec.Manifest
for _, ref := range referrers.Manifests {
refPath := path.Join(srcDir, repoName, "blobs", string(ref.Digest.Algorithm()), ref.Digest.Encoded())
body, err := os.ReadFile(refPath)
@@ -3359,7 +3374,7 @@ func TestSignatures(t *testing.T) {
So(err, ShouldBeNil)
// triggers perm denied on notary sig blobs on downstream
for _, blob := range artifactManifest.Blobs {
for _, blob := range artifactManifest.Layers {
blobPath := path.Join(destDir, repoName, "blobs", string(blob.Digest.Algorithm()), blob.Digest.Encoded())
err := os.MkdirAll(blobPath, 0o755)
So(err, ShouldBeNil)
@@ -3392,7 +3407,7 @@ func TestSignatures(t *testing.T) {
So(resp.StatusCode(), ShouldEqual, http.StatusOK)
// triggers perm denied on sig blobs
for _, blob := range artifactManifest.Blobs {
for _, blob := range artifactManifest.Layers {
blobPath := path.Join(srcDir, repoName, "blobs", string(blob.Digest.Algorithm()), blob.Digest.Encoded())
err := os.Chmod(blobPath, 0o000)
So(err, ShouldBeNil)
@@ -5390,331 +5405,6 @@ func TestSyncImageIndex(t *testing.T) {
})
}
func TestSyncOCIArtifactsWithTag(t *testing.T) {
Convey("Verify syncing tagged OCI artifacts", t, func() {
updateDuration, _ := time.ParseDuration("10s")
sctlr, srcBaseURL, _, _, _ := makeUpstreamServer(t, false, false)
scm := test.NewControllerManager(sctlr)
scm.StartAndWait(sctlr.Config.HTTP.Port)
defer scm.StopServer()
regex := ".*"
var semver bool
tlsVerify := false
repoName := "artifact"
syncRegistryConfig := syncconf.RegistryConfig{
Content: []syncconf.Content{
{
Prefix: repoName,
Tags: &syncconf.Tags{
Regex: &regex,
Semver: &semver,
},
},
},
URLs: []string{srcBaseURL},
OnDemand: false,
PollInterval: updateDuration,
TLSVerify: &tlsVerify,
}
defaultVal := true
syncConfig := &syncconf.Config{
Enable: &defaultVal,
Registries: []syncconf.RegistryConfig{syncRegistryConfig},
}
// create artifact blob
buf := []byte("this is an artifact")
digest := pushBlob(srcBaseURL, repoName, buf)
// create artifact config blob
cbuf := []byte("{}")
cdigest := pushBlob(srcBaseURL, repoName, cbuf)
// push a referrer artifact
manifest := ispec.Manifest{
MediaType: ispec.MediaTypeImageManifest,
Config: ispec.Descriptor{
MediaType: "application/vnd.cncf.icecream",
Digest: cdigest,
Size: int64(len(cbuf)),
},
Layers: []ispec.Descriptor{
{
MediaType: "application/octet-stream",
Digest: digest,
Size: int64(len(buf)),
},
},
}
artifactManifest := ispec.Artifact{
MediaType: ispec.MediaTypeArtifactManifest,
ArtifactType: "application/vnd.cncf.icecream",
Blobs: []ispec.Descriptor{
{
MediaType: "application/octet-stream",
Digest: digest,
Size: int64(len(buf)),
},
},
}
manifest.SchemaVersion = 2
content, err := json.Marshal(manifest)
So(err, ShouldBeNil)
// put OCI artifact mediatype oci image
_, err = resty.R().SetHeader("Content-Type", ispec.MediaTypeImageManifest).
SetBody(content).Put(srcBaseURL + fmt.Sprintf("/v2/%s/manifests/%s", repoName, "1.0"))
So(err, ShouldBeNil)
content, err = json.Marshal(artifactManifest)
So(err, ShouldBeNil)
artifactDigest := godigest.FromBytes(content)
// put OCI artifact mediatype artifact
_, err = resty.R().SetHeader("Content-Type", ispec.MediaTypeArtifactManifest).
SetBody(content).Put(srcBaseURL + fmt.Sprintf("/v2/%s/manifests/%s", repoName, "2.0"))
So(err, ShouldBeNil)
Convey("sync periodically", func() {
// start downstream server
dctlr, destBaseURL, _, _ := makeDownstreamServer(t, false, syncConfig)
dcm := test.NewControllerManager(dctlr)
dcm.StartAndWait(dctlr.Config.HTTP.Port)
defer dcm.StopServer()
// give it time to set up sync
t.Logf("waitsync(%s, %s)", dctlr.Config.Storage.RootDirectory, repoName)
waitSync(dctlr.Config.Storage.RootDirectory, repoName)
resp, err := resty.R().SetHeader("Content-Type", ispec.MediaTypeImageManifest).
Get(destBaseURL + fmt.Sprintf("/v2/%s/manifests/%s", repoName, "1.0"))
So(err, ShouldBeNil)
So(resp.StatusCode(), ShouldEqual, http.StatusOK)
So(resp.Body(), ShouldNotBeEmpty)
So(resp.Header().Get("Content-Type"), ShouldNotBeEmpty)
var syncedManifest ispec.Manifest
err = json.Unmarshal(resp.Body(), &syncedManifest)
So(err, ShouldBeNil)
So(reflect.DeepEqual(syncedManifest, manifest), ShouldEqual, true)
resp, err = resty.R().SetHeader("Content-Type", ispec.MediaTypeArtifactManifest).
Get(destBaseURL + fmt.Sprintf("/v2/%s/manifests/%s", repoName, "2.0"))
So(err, ShouldBeNil)
So(resp.StatusCode(), ShouldEqual, http.StatusOK)
So(resp.Body(), ShouldNotBeEmpty)
So(resp.Header().Get("Content-Type"), ShouldNotBeEmpty)
var syncedArtifact ispec.Artifact
err = json.Unmarshal(resp.Body(), &syncedArtifact)
So(err, ShouldBeNil)
So(reflect.DeepEqual(syncedArtifact, artifactManifest), ShouldEqual, true)
// for coverage
found, err := test.ReadLogFileAndSearchString(dctlr.Config.Log.Output,
"skipping OCI artifact", 15*time.Second)
if err != nil {
panic(err)
}
if !found {
data, err := os.ReadFile(dctlr.Config.Log.Output)
So(err, ShouldBeNil)
t.Logf("downstream log: %s", string(data))
}
So(found, ShouldBeTrue)
waitSyncFinish(dctlr.Config.Log.Output)
})
Convey("sync on demand", func() {
// start downstream server
syncConfig.Registries[0].OnDemand = true
syncConfig.Registries[0].PollInterval = 0
dctlr, destBaseURL, _, _ := makeDownstreamServer(t, false, syncConfig)
dcm := test.NewControllerManager(dctlr)
dcm.StartAndWait(dctlr.Config.HTTP.Port)
defer dcm.StopServer()
resp, err := resty.R().SetHeader("Content-Type", ispec.MediaTypeArtifactManifest).
Get(destBaseURL + fmt.Sprintf("/v2/%s/manifests/%s", repoName, "2.0"))
So(err, ShouldBeNil)
So(resp.StatusCode(), ShouldEqual, http.StatusOK)
So(resp.Body(), ShouldNotBeEmpty)
So(resp.Header().Get("Content-Type"), ShouldNotBeEmpty)
var syncedArtifact ispec.Artifact
err = json.Unmarshal(resp.Body(), &syncedArtifact)
So(err, ShouldBeNil)
So(reflect.DeepEqual(syncedArtifact, artifactManifest), ShouldEqual, true)
resp, err = resty.R().SetHeader("Content-Type", ispec.MediaTypeImageManifest).
Get(destBaseURL + fmt.Sprintf("/v2/%s/manifests/%s", repoName, "1.0"))
So(err, ShouldBeNil)
So(resp.StatusCode(), ShouldEqual, http.StatusOK)
So(resp.Body(), ShouldNotBeEmpty)
So(resp.Header().Get("Content-Type"), ShouldNotBeEmpty)
var syncedManifest ispec.Manifest
err = json.Unmarshal(resp.Body(), &syncedManifest)
So(err, ShouldBeNil)
So(reflect.DeepEqual(syncedManifest, manifest), ShouldEqual, true)
})
Convey("sync periodically error on mediatype", func() {
manifestPath := path.Join(sctlr.Config.Storage.RootDirectory, repoName, "blobs", "sha256", artifactDigest.Encoded())
So(os.Chmod(manifestPath, 0o000), ShouldBeNil)
// start downstream server
dctlr, destBaseURL, _, _ := makeDownstreamServer(t, false, syncConfig)
dcm := test.NewControllerManager(dctlr)
dcm.StartAndWait(dctlr.Config.HTTP.Port)
defer dcm.StopServer()
defer func() {
err := os.Chmod(manifestPath, 0o755)
So(err, ShouldBeNil)
}()
found, err := test.ReadLogFileAndSearchString(dctlr.Config.Log.Output,
"finished syncing", 15*time.Second)
if err != nil {
panic(err)
}
if !found {
data, err := os.ReadFile(dctlr.Config.Log.Output)
So(err, ShouldBeNil)
t.Logf("downstream log: %s", string(data))
}
So(found, ShouldBeTrue)
resp, err := resty.R().SetHeader("Content-Type", ispec.MediaTypeArtifactManifest).
Get(destBaseURL + fmt.Sprintf("/v2/%s/manifests/%s", repoName, artifactDigest.String()))
So(err, ShouldBeNil)
So(resp.StatusCode(), ShouldEqual, http.StatusNotFound)
found, err = test.ReadLogFileAndSearchString(dctlr.Config.Log.Output,
"couldn't get upstream image", 5*time.Second)
if err != nil {
panic(err)
}
if !found {
data, err := os.ReadFile(dctlr.Config.Log.Output)
So(err, ShouldBeNil)
t.Logf("downstream log: %s", string(data))
}
So(found, ShouldBeTrue)
})
Convey("sync on demand error on mediatype", func() {
// start downstream server
syncConfig.Registries[0].OnDemand = true
syncConfig.Registries[0].PollInterval = 0
dctlr, destBaseURL, _, _ := makeDownstreamServer(t, false, syncConfig)
dcm := test.NewControllerManager(dctlr)
dcm.StartAndWait(dctlr.Config.HTTP.Port)
defer dcm.StopServer()
manifestPath := path.Join(sctlr.Config.Storage.RootDirectory, repoName, "blobs", "sha256", artifactDigest.Encoded())
So(os.Chmod(manifestPath, 0o000), ShouldBeNil)
defer func() {
err := os.Chmod(manifestPath, 0o755)
So(err, ShouldBeNil)
}()
resp, err := resty.R().SetHeader("Content-Type", ispec.MediaTypeArtifactManifest).
Get(destBaseURL + fmt.Sprintf("/v2/%s/manifests/%s", repoName, artifactDigest.String()))
So(err, ShouldBeNil)
So(resp.StatusCode(), ShouldEqual, http.StatusNotFound)
})
Convey("sync on demand and periodically error on PutImageManifest", func() {
// start downstream server
syncConfig.Registries[0].OnDemand = true
destDir := t.TempDir()
destConfig := config.New()
destConfig.HTTP.Port = test.GetFreePort()
destBaseURL := test.GetBaseURL(destConfig.HTTP.Port)
destConfig.Storage.RootDirectory = destDir
destConfig.Extensions = &extconf.ExtensionConfig{}
destConfig.Extensions.Search = nil
destConfig.Extensions.Sync = syncConfig
destConfig.Log.Output = path.Join(destDir, "sync.log")
dctlr := api.NewController(destConfig)
dcm := test.NewControllerManager(dctlr)
manifestPath := path.Join(destDir, repoName, "blobs", "sha256", artifactDigest.Encoded())
So(os.MkdirAll(manifestPath, 0o755), ShouldBeNil)
So(os.Chmod(manifestPath, 0o000), ShouldBeNil)
dcm.StartAndWait(destConfig.HTTP.Port)
defer dcm.StopServer()
defer func() {
err := os.Chmod(manifestPath, 0o755)
So(err, ShouldBeNil)
}()
found, err := test.ReadLogFileAndSearchString(dctlr.Config.Log.Output,
"couldn't upload OCI artifact manifest", 15*time.Second)
if err != nil {
panic(err)
}
if !found {
data, err := os.ReadFile(dctlr.Config.Log.Output)
So(err, ShouldBeNil)
t.Logf("downstream log: %s", string(data))
}
So(found, ShouldBeTrue)
resp, err := resty.R().SetHeader("Content-Type", ispec.MediaTypeArtifactManifest).
Get(destBaseURL + fmt.Sprintf("/v2/%s/manifests/%s", repoName, artifactDigest.String()))
So(err, ShouldBeNil)
So(resp.StatusCode(), ShouldEqual, http.StatusNotFound)
waitSyncFinish(dctlr.Config.Log.Output)
})
})
}
func generateKeyPairs(tdir string) {
// generate a keypair
os.Setenv("COSIGN_PASSWORD", "")
@@ -5817,6 +5507,35 @@ func pushRepo(url, repoName string) godigest.Digest {
panic(err)
}
// upload scratch image config
resp, err = resty.R().
Post(fmt.Sprintf("%s/v2/%s/blobs/uploads/", url, repoName))
if err != nil {
panic(err)
}
if resp.StatusCode() != http.StatusAccepted {
panic(fmt.Errorf("invalid status code: %d %w", resp.StatusCode(), errBadStatus))
}
loc = test.Location(url, resp)
cblob, cdigest := ispec.ScratchDescriptor.Data, ispec.ScratchDescriptor.Digest
resp, err = resty.R().
SetContentLength(true).
SetHeader("Content-Length", fmt.Sprintf("%d", len(cblob))).
SetHeader("Content-Type", "application/octet-stream").
SetQueryParam("digest", cdigest.String()).
SetBody(cblob).
Put(loc)
if err != nil {
panic(err)
}
if resp.StatusCode() != http.StatusCreated {
panic(fmt.Errorf("invalid status code: %d %w", resp.StatusCode(), errBadStatus))
}
// upload image config blob
resp, err = resty.R().
Post(fmt.Sprintf("%s/v2/%s/blobs/uploads/", url, repoName))
@@ -5829,7 +5548,7 @@ func pushRepo(url, repoName string) godigest.Digest {
}
loc = test.Location(url, resp)
cblob, cdigest := test.GetRandomImageConfig()
cblob, cdigest = test.GetRandomImageConfig()
resp, err = resty.R().
SetContentLength(true).
@@ -5907,10 +5626,15 @@ func pushRepo(url, repoName string) godigest.Digest {
},
}
artifactManifest := ispec.Artifact{
MediaType: ispec.MediaTypeArtifactManifest,
artifactManifest := ispec.Manifest{
MediaType: ispec.MediaTypeImageManifest,
ArtifactType: "application/vnd.cncf.icecream",
Blobs: []ispec.Descriptor{
Config: ispec.Descriptor{
MediaType: ispec.MediaTypeScratch,
Digest: ispec.ScratchDescriptor.Digest,
Size: 2,
},
Layers: []ispec.Descriptor{
{
MediaType: "application/octet-stream",
Digest: adigest,
@@ -5948,7 +5672,7 @@ func pushRepo(url, repoName string) godigest.Digest {
adigest = godigest.FromBytes(content)
// put OCI reference artifact mediaType artifact
_, err = resty.R().SetHeader("Content-Type", ispec.MediaTypeArtifactManifest).
_, err = resty.R().SetHeader("Content-Type", ispec.MediaTypeImageManifest).
SetBody(content).Put(url + fmt.Sprintf("/v2/%s/manifests/%s", repoName, adigest.String()))
if err != nil {
panic(err)
pkg/extensions/sync/utils.go
-21
View File
@@ -590,17 +590,6 @@ func manifestsEqual(manifest1, manifest2 ispec.Manifest) bool {
return false
}
func artifactsEqual(manifest1, manifest2 ispec.Artifact) bool {
if manifest1.ArtifactType == manifest2.ArtifactType &&
manifest1.MediaType == manifest2.MediaType {
if descriptorsEqual(manifest1.Blobs, manifest2.Blobs) {
return true
}
}
return false
}
func artifactDescriptorsEqual(desc1, desc2 []artifactspec.Descriptor) bool {
if len(desc1) != len(desc2) {
return false
@@ -690,16 +679,6 @@ func syncImageWithRefs(ctx context.Context, localRepo, upstreamRepo, reference s
upstreamImageDigest := godigest.FromBytes(manifestBuf)
if !isSupportedMediaType(mediaType) {
if mediaType == ispec.MediaTypeArtifactManifest {
err = sig.syncOCIArtifact(localRepo, upstreamRepo, reference, manifestBuf) //nolint
if err != nil {
log.Error().Err(err).Str("image", upstreamImageRef.DockerReference().String()).
Msg("couldn't sync oci artifact with artifact mediaType")
return skipped, err
}
}
return skipped, nil
}
pkg/log/log.go
+1 -1
View File
@@ -92,7 +92,7 @@ func GoroutineID() int {
type goroutineHook struct{}
func (h goroutineHook) Run(e *zerolog.Event, level zerolog.Level, msg string) {
func (h goroutineHook) Run(e *zerolog.Event, level zerolog.Level, _ string) {
if level != zerolog.NoLevel {
e.Int("goroutine", GoroutineID())
}
pkg/meta/bolt/buckets.go
-1
View File
@@ -4,7 +4,6 @@ package bolt
const (
ManifestDataBucket = "ManifestData"
IndexDataBucket = "IndexData"
ArtifactDataBucket = "ArtifactData"
RepoMetadataBucket = "RepoMetadata"
UserDataBucket = "UserData"
VersionBucket = "Version"
pkg/meta/dynamo/parameters.go
+1 -1
View File
@@ -10,7 +10,7 @@ import (
type DBDriverParameters struct {
Endpoint, Region, RepoMetaTablename, ManifestDataTablename, IndexDataTablename,
ArtifactDataTablename, VersionTablename, UserDataTablename string
VersionTablename, UserDataTablename string
}
func GetDynamoClient(params DBDriverParameters) (*dynamodb.Client, error) {
pkg/meta/repodb/boltdb-wrapper/boltdb_wrapper.go
-48
View File
@@ -49,11 +49,6 @@ func NewBoltDBWrapper(boltDB *bbolt.DB, log log.Logger) (*DBWrapper, error) {
return err
}
_, err = transaction.CreateBucketIfNotExists([]byte(bolt.ArtifactDataBucket))
if err != nil {
return err
}
_, err = transaction.CreateBucketIfNotExists([]byte(bolt.RepoMetadataBucket))
if err != nil {
return err
@@ -258,49 +253,6 @@ func (bdw *DBWrapper) GetIndexData(indexDigest godigest.Digest) (repodb.IndexDat
return indexMetadata, err
}
func (bdw DBWrapper) SetArtifactData(artifactDigest godigest.Digest, artifactData repodb.ArtifactData) error {
err := bdw.DB.Update(func(tx *bbolt.Tx) error {
buck := tx.Bucket([]byte(bolt.ArtifactDataBucket))
imBlob, err := json.Marshal(artifactData)
if err != nil {
return fmt.Errorf("repodb: error while calculating blob for artifact with digest %s %w", artifactDigest, err)
}
err = buck.Put([]byte(artifactDigest), imBlob)
if err != nil {
return fmt.Errorf("repodb: error while setting artifact blob for digest %s %w", artifactDigest, err)
}
return nil
})
return err
}
func (bdw DBWrapper) GetArtifactData(artifactDigest godigest.Digest) (repodb.ArtifactData, error) {
var artifactData repodb.ArtifactData
err := bdw.DB.View(func(tx *bbolt.Tx) error {
buck := tx.Bucket([]byte(bolt.ArtifactDataBucket))
blob := buck.Get([]byte(artifactDigest))
if len(blob) == 0 {
return zerr.ErrArtifactDataNotFound
}
err := json.Unmarshal(blob, &artifactData)
if err != nil {
return fmt.Errorf("repodb: error while unmashaling artifact data for digest %s %w", artifactDigest, err)
}
return nil
})
return artifactData, err
}
func (bdw DBWrapper) SetReferrer(repo string, referredDigest godigest.Digest, referrer repodb.ReferrerInfo) error {
err := bdw.DB.Update(func(tx *bbolt.Tx) error {
buck := tx.Bucket([]byte(bolt.RepoMetadataBucket))
pkg/meta/repodb/boltdb-wrapper/boltdb_wrapper_test.go
-12
View File
@@ -91,18 +91,6 @@ func TestWrapperErrors(t *testing.T) {
So(err, ShouldNotBeNil)
})
Convey("GetArtifactData", func() {
err := boltdbWrapper.DB.Update(func(tx *bbolt.Tx) error {
artifactBuck := tx.Bucket([]byte(bolt.ArtifactDataBucket))
return artifactBuck.Put([]byte("artifactDigest"), []byte("wrong json"))
})
So(err, ShouldBeNil)
_, err = boltdbWrapper.GetArtifactData("artifactDigest")
So(err, ShouldNotBeNil)
})
Convey("SetReferrer", func() {
err := boltdbWrapper.DB.Update(func(tx *bbolt.Tx) error {
repoBuck := tx.Bucket([]byte(bolt.RepoMetadataBucket))
pkg/meta/repodb/dynamodb-wrapper/dynamo_internal_test.go
-3
View File
@@ -30,7 +30,6 @@ func TestWrapperErrors(t *testing.T) {
repoMetaTablename := "RepoMetadataTable" + uuid.String()
manifestDataTablename := "ManifestDataTable" + uuid.String()
indexDataTablename := "IndexDataTable" + uuid.String()
artifactDataTablename := "ArtifactDataTable" + uuid.String()
userDataTablename := "UserDataTable" + uuid.String()
versionTablename := "Version" + uuid.String()
@@ -57,7 +56,6 @@ func TestWrapperErrors(t *testing.T) {
RepoMetaTablename: repoMetaTablename,
ManifestDataTablename: manifestDataTablename,
IndexDataTablename: indexDataTablename,
ArtifactDataTablename: artifactDataTablename,
VersionTablename: versionTablename,
UserDataTablename: userDataTablename,
Patches: version.GetDynamoDBPatches(),
@@ -99,7 +97,6 @@ func TestWrapperErrors(t *testing.T) {
ManifestDataTablename: manifestDataTablename,
VersionTablename: versionTablename,
IndexDataTablename: indexDataTablename,
ArtifactDataTablename: artifactDataTablename,
UserDataTablename: userDataTablename,
Patches: version.GetDynamoDBPatches(),
Log: log.Logger{Logger: zerolog.New(os.Stdout)},
pkg/meta/repodb/dynamodb-wrapper/dynamo_test.go
+11 -97
View File
@@ -42,7 +42,6 @@ func TestIterator(t *testing.T) {
manifestDataTablename := "ManifestDataTable" + uuid.String()
versionTablename := "Version" + uuid.String()
indexDataTablename := "IndexDataTable" + uuid.String()
artifactDataTablename := "ArtifactDataTable" + uuid.String()
userDataTablename := "UserDataTable" + uuid.String()
log := log.NewLogger("debug", "")
@@ -54,7 +53,6 @@ func TestIterator(t *testing.T) {
RepoMetaTablename: repoMetaTablename,
ManifestDataTablename: manifestDataTablename,
IndexDataTablename: indexDataTablename,
ArtifactDataTablename: artifactDataTablename,
VersionTablename: versionTablename,
UserDataTablename: userDataTablename,
}
@@ -145,7 +143,6 @@ func TestWrapperErrors(t *testing.T) {
manifestDataTablename := "ManifestDataTable" + uuid.String()
versionTablename := "Version" + uuid.String()
indexDataTablename := "IndexDataTable" + uuid.String()
artifactDataTablename := "ArtifactData" + uuid.String()
userDataTablename := "UserDataTable" + uuid.String()
ctx := context.Background()
@@ -159,7 +156,6 @@ func TestWrapperErrors(t *testing.T) {
RepoMetaTablename: repoMetaTablename,
ManifestDataTablename: manifestDataTablename,
IndexDataTablename: indexDataTablename,
ArtifactDataTablename: artifactDataTablename,
UserDataTablename: userDataTablename,
VersionTablename: versionTablename,
}
@@ -417,20 +413,6 @@ func TestWrapperErrors(t *testing.T) {
So(err, ShouldNotBeNil)
})
Convey("GetArtifactData", func() {
dynamoWrapper.ArtifactDataTablename = badTablename
_, err = dynamoWrapper.GetArtifactData("dig")
So(err, ShouldNotBeNil)
})
Convey("GetArtifactData unmarhsal error", func() {
err = setBadArtifactData(dynamoWrapper.Client, artifactDataTablename, "dig")
So(err, ShouldBeNil)
_, err = dynamoWrapper.GetArtifactData("dig")
So(err, ShouldNotBeNil)
})
Convey("SetRepoReference client error", func() {
dynamoWrapper.RepoMetaTablename = badTablename
digest := digest.FromString("str")
@@ -488,7 +470,6 @@ func TestWrapperErrors(t *testing.T) {
Convey("GetReferrersInfo getData fails", func() {
dynamoWrapper.ManifestDataTablename = badTablename
dynamoWrapper.ArtifactDataTablename = badTablename
err = dynamoWrapper.SetReferrer("repo", "rf", repodb.ReferrerInfo{
Digest: "dig1",
MediaType: ispec.MediaTypeImageManifest,
@@ -497,7 +478,7 @@ func TestWrapperErrors(t *testing.T) {
err = dynamoWrapper.SetReferrer("repo", "rf", repodb.ReferrerInfo{
Digest: "dig2",
MediaType: ispec.MediaTypeArtifactManifest,
MediaType: ispec.MediaTypeImageManifest,
})
So(err, ShouldBeNil)
@@ -506,11 +487,6 @@ func TestWrapperErrors(t *testing.T) {
})
Convey("GetReferrersInfo bad descriptor blob", func() {
err = dynamoWrapper.SetArtifactData("dig2", repodb.ArtifactData{
ManifestBlob: []byte("bad json"),
})
So(err, ShouldBeNil)
err = dynamoWrapper.SetManifestData("dig3", repodb.ManifestData{
ManifestBlob: []byte("bad json"),
})
@@ -518,7 +494,7 @@ func TestWrapperErrors(t *testing.T) {
err = dynamoWrapper.SetReferrer("repo", "rf", repodb.ReferrerInfo{
Digest: "dig2",
MediaType: ispec.MediaTypeArtifactManifest,
MediaType: ispec.MediaTypeImageManifest,
})
So(err, ShouldBeNil)
@@ -1095,7 +1071,6 @@ func TestWrapperErrors(t *testing.T) {
RepoMetaTablename: "",
ManifestDataTablename: manifestDataTablename,
IndexDataTablename: indexDataTablename,
ArtifactDataTablename: artifactDataTablename,
UserDataTablename: userDataTablename,
VersionTablename: versionTablename,
}
@@ -1111,7 +1086,6 @@ func TestWrapperErrors(t *testing.T) {
RepoMetaTablename: repoMetaTablename,
ManifestDataTablename: "",
IndexDataTablename: indexDataTablename,
ArtifactDataTablename: artifactDataTablename,
UserDataTablename: userDataTablename,
VersionTablename: versionTablename,
}
@@ -1127,7 +1101,6 @@ func TestWrapperErrors(t *testing.T) {
RepoMetaTablename: repoMetaTablename,
ManifestDataTablename: manifestDataTablename,
IndexDataTablename: "",
ArtifactDataTablename: artifactDataTablename,
UserDataTablename: userDataTablename,
VersionTablename: versionTablename,
}
@@ -1143,7 +1116,6 @@ func TestWrapperErrors(t *testing.T) {
RepoMetaTablename: repoMetaTablename,
ManifestDataTablename: manifestDataTablename,
IndexDataTablename: indexDataTablename,
ArtifactDataTablename: artifactDataTablename,
UserDataTablename: userDataTablename,
VersionTablename: "",
}
@@ -1159,41 +1131,8 @@ func TestWrapperErrors(t *testing.T) {
RepoMetaTablename: repoMetaTablename,
ManifestDataTablename: manifestDataTablename,
IndexDataTablename: indexDataTablename,
ArtifactDataTablename: "",
UserDataTablename: userDataTablename,
VersionTablename: versionTablename,
}
client, err = dynamo.GetDynamoClient(params)
So(err, ShouldBeNil)
_, err = dynamoWrapper.NewDynamoDBWrapper(client, params, log)
So(err, ShouldNotBeNil)
params = dynamo.DBDriverParameters{ //nolint:contextcheck
Endpoint: endpoint,
Region: region,
RepoMetaTablename: repoMetaTablename,
ManifestDataTablename: manifestDataTablename,
IndexDataTablename: indexDataTablename,
VersionTablename: versionTablename,
UserDataTablename: userDataTablename,
ArtifactDataTablename: artifactDataTablename,
}
client, err = dynamo.GetDynamoClient(params)
So(err, ShouldBeNil)
_, err = dynamoWrapper.NewDynamoDBWrapper(client, params, log)
So(err, ShouldBeNil)
params = dynamo.DBDriverParameters{ //nolint:contextcheck
Endpoint: endpoint,
Region: region,
RepoMetaTablename: repoMetaTablename,
ManifestDataTablename: manifestDataTablename,
IndexDataTablename: indexDataTablename,
VersionTablename: versionTablename,
UserDataTablename: "",
ArtifactDataTablename: artifactDataTablename,
VersionTablename: versionTablename,
}
client, err = dynamo.GetDynamoClient(params)
So(err, ShouldBeNil)
@@ -1228,26 +1167,26 @@ func setBadManifestData(client *dynamodb.Client, manifestDataTableName, digest s
return err
}
func setBadArtifactData(client *dynamodb.Client, artifactDataTablename, digest string) error {
mdAttributeValue, err := attributevalue.Marshal("string")
func setBadRepoMeta(client *dynamodb.Client, repoMetadataTableName, repoName string) error {
repoAttributeValue, err := attributevalue.Marshal("string")
if err != nil {
return err
}
_, err = client.UpdateItem(context.TODO(), &dynamodb.UpdateItemInput{
ExpressionAttributeNames: map[string]string{
"#AD": "ArtifactData",
"#RM": "RepoMetadata",
},
ExpressionAttributeValues: map[string]types.AttributeValue{
":ArtifactData": mdAttributeValue,
":RepoMetadata": repoAttributeValue,
},
Key: map[string]types.AttributeValue{
"ArtifactDigest": &types.AttributeValueMemberS{
Value: digest,
"RepoName": &types.AttributeValueMemberS{
Value: repoName,
},
},
TableName: aws.String(artifactDataTablename),
UpdateExpression: aws.String("SET #AD = :ArtifactData"),
TableName: aws.String(repoMetadataTableName),
UpdateExpression: aws.String("SET #RM = :RepoMetadata"),
})
return err
@@ -1278,31 +1217,6 @@ func setBadIndexData(client *dynamodb.Client, indexDataTableName, digest string)
return err
}
func setBadRepoMeta(client *dynamodb.Client, repoMetadataTableName, repoName string) error {
repoAttributeValue, err := attributevalue.Marshal("string")
if err != nil {
return err
}
_, err = client.UpdateItem(context.TODO(), &dynamodb.UpdateItemInput{
ExpressionAttributeNames: map[string]string{
"#RM": "RepoMetadata",
},
ExpressionAttributeValues: map[string]types.AttributeValue{
":RepoMetadata": repoAttributeValue,
},
Key: map[string]types.AttributeValue{
"RepoName": &types.AttributeValueMemberS{
Value: repoName,
},
},
TableName: aws.String(repoMetadataTableName),
UpdateExpression: aws.String("SET #RM = :RepoMetadata"),
})
return err
}
func setBadUserData(client *dynamodb.Client, userDataTablename, userID string) error {
userAttributeValue, err := attributevalue.Marshal("string")
if err != nil {
pkg/meta/repodb/dynamodb-wrapper/dynamo_wrapper.go
-84
View File
@@ -32,7 +32,6 @@ type DBWrapper struct {
RepoMetaTablename string
IndexDataTablename string
ManifestDataTablename string
ArtifactDataTablename string
UserDataTablename string
VersionTablename string
Patches []func(client *dynamodb.Client, tableNames map[string]string) error
@@ -45,7 +44,6 @@ func NewDynamoDBWrapper(client *dynamodb.Client, params dynamo.DBDriverParameter
RepoMetaTablename: params.RepoMetaTablename,
ManifestDataTablename: params.ManifestDataTablename,
IndexDataTablename: params.IndexDataTablename,
ArtifactDataTablename: params.ArtifactDataTablename,
VersionTablename: params.VersionTablename,
UserDataTablename: params.UserDataTablename,
Patches: version.GetDynamoDBPatches(),
@@ -67,11 +65,6 @@ func NewDynamoDBWrapper(client *dynamodb.Client, params dynamo.DBDriverParameter
return nil, err
}
err = dynamoWrapper.createArtifactDataTable()
if err != nil {
return nil, err
}
err = dynamoWrapper.createIndexDataTable()
if err != nil {
return nil, err
@@ -301,58 +294,6 @@ func (dwr *DBWrapper) GetIndexData(indexDigest godigest.Digest) (repodb.IndexDat
return indexData, nil
}
func (dwr DBWrapper) SetArtifactData(artifactDigest godigest.Digest, artifactData repodb.ArtifactData) error {
artifactAttributeValue, err := attributevalue.Marshal(artifactData)
if err != nil {
return err
}
_, err = dwr.Client.UpdateItem(context.TODO(), &dynamodb.UpdateItemInput{
ExpressionAttributeNames: map[string]string{
"#AD": "ArtifactData",
},
ExpressionAttributeValues: map[string]types.AttributeValue{
":ArtifactData": artifactAttributeValue,
},
Key: map[string]types.AttributeValue{
"ArtifactDigest": &types.AttributeValueMemberS{
Value: artifactDigest.String(),
},
},
TableName: aws.String(dwr.ArtifactDataTablename),
UpdateExpression: aws.String("SET #AD = :ArtifactData"),
})
return err
}
func (dwr DBWrapper) GetArtifactData(artifactDigest godigest.Digest) (repodb.ArtifactData, error) {
resp, err := dwr.Client.GetItem(context.TODO(), &dynamodb.GetItemInput{
TableName: aws.String(dwr.ArtifactDataTablename),
Key: map[string]types.AttributeValue{
"ArtifactDigest": &types.AttributeValueMemberS{
Value: artifactDigest.String(),
},
},
})
if err != nil {
return repodb.ArtifactData{}, err
}
if resp.Item == nil {
return repodb.ArtifactData{}, zerr.ErrRepoMetaNotFound
}
var artifactData repodb.ArtifactData
err = attributevalue.Unmarshal(resp.Item["ArtifactData"], &artifactData)
if err != nil {
return repodb.ArtifactData{}, err
}
return artifactData, nil
}
func (dwr DBWrapper) SetReferrer(repo string, referredDigest godigest.Digest, referrer repodb.ReferrerInfo) error {
resp, err := dwr.Client.GetItem(context.TODO(), &dynamodb.GetItemInput{
TableName: aws.String(dwr.RepoMetaTablename),
@@ -1664,31 +1605,6 @@ func (dwr *DBWrapper) createIndexDataTable() error {
return dwr.waitTableToBeCreated(dwr.IndexDataTablename)
}
func (dwr DBWrapper) createArtifactDataTable() error {
_, err := dwr.Client.CreateTable(context.Background(), &dynamodb.CreateTableInput{
TableName: aws.String(dwr.ArtifactDataTablename),
AttributeDefinitions: []types.AttributeDefinition{
{
AttributeName: aws.String("ArtifactDigest"),
AttributeType: types.ScalarAttributeTypeS,
},
},
KeySchema: []types.KeySchemaElement{
{
AttributeName: aws.String("ArtifactDigest"),
KeyType: types.KeyTypeHash,
},
},
BillingMode: types.BillingModePayPerRequest,
})
if err != nil && !strings.Contains(err.Error(), "Table already exists") {
return err
}
return dwr.waitTableToBeCreated(dwr.ManifestDataTablename)
}
func (dwr *DBWrapper) createVersionTable() error {
_, err := dwr.Client.CreateTable(context.Background(), &dynamodb.CreateTableInput{
TableName: aws.String(dwr.VersionTablename),
pkg/meta/repodb/repodb.go
-10
View File
@@ -77,12 +77,6 @@ type RepoDB interface { //nolint:interfacebloat
// GetIndexData returns indexData for a given Index from the database
GetIndexData(indexDigest godigest.Digest) (IndexData, error)
// SetArtifactData sets artifactData for a given artifact in the database
SetArtifactData(artifactDigest godigest.Digest, artifactData ArtifactData) error
// GetArtifactData returns artifactData for a given artifact from the database
GetArtifactData(artifactDigest godigest.Digest) (ArtifactData, error)
// SetReferrer adds a referrer to the referrers list of a manifest inside a repo
SetReferrer(repo string, referredDigest godigest.Digest, referrer ReferrerInfo) error
@@ -150,10 +144,6 @@ type ManifestData struct {
ConfigBlob []byte
}
type ArtifactData struct {
ManifestBlob []byte
}
type ReferrerInfo struct {
Digest string
MediaType string
pkg/meta/repodb/repodb_test.go
+14 -48
View File
@@ -87,7 +87,6 @@ func TestDynamoDBWrapper(t *testing.T) {
manifestDataTablename := "ManifestDataTable" + uuid.String()
versionTablename := "Version" + uuid.String()
indexDataTablename := "IndexDataTable" + uuid.String()
artifactDataTablename := "ArtifactDataTable" + uuid.String()
userDataTablename := "UserDataTable" + uuid.String()
Convey("DynamoDB Wrapper", t, func() {
@@ -96,7 +95,6 @@ func TestDynamoDBWrapper(t *testing.T) {
RepoMetaTablename: repoMetaTablename,
ManifestDataTablename: manifestDataTablename,
IndexDataTablename: indexDataTablename,
ArtifactDataTablename: artifactDataTablename,
VersionTablename: versionTablename,
UserDataTablename: userDataTablename,
Region: "us-east-2",
@@ -2174,27 +2172,6 @@ func RunRepoDBTests(repoDB repodb.RepoDB, preparationFuncs ...func() error) {
So(err, ShouldNotBeNil)
})
Convey("Test artifact logic", func() {
artifact, err := test.GetRandomArtifact(nil)
So(err, ShouldBeNil)
artifactDigest, err := artifact.Digest()
So(err, ShouldBeNil)
artifactData, err := artifact.ArtifactData()
So(err, ShouldBeNil)
err = repoDB.SetArtifactData(artifactDigest, artifactData)
So(err, ShouldBeNil)
result, err := repoDB.GetArtifactData(artifactDigest)
So(err, ShouldBeNil)
So(result, ShouldResemble, artifactData)
_, err = repoDB.GetArtifactData(godigest.FromString("inexistent"))
So(err, ShouldNotBeNil)
})
Convey("Test Referrers", func() {
image, err := test.GetRandomImage("tag")
So(err, ShouldBeNil)
@@ -2221,10 +2198,10 @@ func RunRepoDBTests(repoDB repodb.RepoDB, preparationFuncs ...func() error) {
// ------- Add Artifact 1
artifact1, err := test.GetRandomArtifact(&ispec.Descriptor{
Digest: referredDigest,
MediaType: ispec.MediaTypeImageManifest,
})
artifact1, err := test.GetImageWithSubject(
referredDigest,
ispec.MediaTypeImageManifest,
)
So(err, ShouldBeNil)
artifactDigest1, err := artifact1.Digest()
@@ -2238,10 +2215,10 @@ func RunRepoDBTests(repoDB repodb.RepoDB, preparationFuncs ...func() error) {
// ------- Add Artifact 2
artifact2, err := test.GetRandomArtifact(&ispec.Descriptor{
Digest: referredDigest,
MediaType: ispec.MediaTypeImageManifest,
})
artifact2, err := test.GetImageWithSubject(
referredDigest,
ispec.MediaTypeImageManifest,
)
So(err, ShouldBeNil)
artifactDigest2, err := artifact2.Digest()
@@ -2249,7 +2226,7 @@ func RunRepoDBTests(repoDB repodb.RepoDB, preparationFuncs ...func() error) {
err = repoDB.SetReferrer("repo", referredDigest, repodb.ReferrerInfo{
Digest: artifactDigest2.String(),
MediaType: ispec.MediaTypeArtifactManifest,
MediaType: ispec.MediaTypeImageManifest,
})
So(err, ShouldBeNil)
@@ -2263,7 +2240,7 @@ func RunRepoDBTests(repoDB repodb.RepoDB, preparationFuncs ...func() error) {
})
So(referrers, ShouldContain, repodb.ReferrerInfo{
Digest: artifactDigest2.String(),
MediaType: ispec.MediaTypeArtifactManifest,
MediaType: ispec.MediaTypeImageManifest,
})
So(err, ShouldBeNil)
@@ -2333,12 +2310,6 @@ func RunRepoDBTests(repoDB repodb.RepoDB, preparationFuncs ...func() error) {
})
So(err, ShouldBeNil)
err = repoDB.SetReferrer("repo", referredDigest, repodb.ReferrerInfo{
Digest: "inexistendArtifactManifestDigest",
MediaType: ispec.MediaTypeArtifactManifest,
})
So(err, ShouldBeNil)
// ------- Set existent manifest and artifact manifest
err = repoDB.SetManifestData("goodManifest", repodb.ManifestData{
ManifestBlob: []byte(`{"artifactType": "unwantedType"}`),
@@ -2347,20 +2318,15 @@ func RunRepoDBTests(repoDB repodb.RepoDB, preparationFuncs ...func() error) {
So(err, ShouldBeNil)
err = repoDB.SetReferrer("repo", referredDigest, repodb.ReferrerInfo{
Digest: "goodManifest",
Digest: "goodManifestUnwanted",
MediaType: ispec.MediaTypeImageManifest,
ArtifactType: "unwantedType",
})
So(err, ShouldBeNil)
err = repoDB.SetArtifactData("goodArtifact", repodb.ArtifactData{
ManifestBlob: []byte(`{"artifactType": "wantedType"}`),
})
So(err, ShouldBeNil)
err = repoDB.SetReferrer("repo", referredDigest, repodb.ReferrerInfo{
Digest: "goodArtifact",
MediaType: ispec.MediaTypeArtifactManifest,
Digest: "goodManifest",
MediaType: ispec.MediaTypeImageManifest,
ArtifactType: "wantedType",
})
So(err, ShouldBeNil)
@@ -2369,7 +2335,7 @@ func RunRepoDBTests(repoDB repodb.RepoDB, preparationFuncs ...func() error) {
So(err, ShouldBeNil)
So(len(referrerInfo), ShouldEqual, 1)
So(referrerInfo[0].ArtifactType, ShouldResemble, "wantedType")
So(referrerInfo[0].Digest, ShouldResemble, "goodArtifact")
So(referrerInfo[0].Digest, ShouldResemble, "goodManifest")
})
Convey("FilterRepos", func() {
pkg/meta/repodb/repodbfactory/repodb_factory.go
-4
View File
@@ -89,9 +89,6 @@ func getDynamoParams(cacheDriverConfig map[string]interface{}, log log.Logger) d
indexDataTablename, ok := toStringIfOk(cacheDriverConfig, "indexdatatablename", log)
allParametersOk = allParametersOk && ok
artifactDataTablename, ok := toStringIfOk(cacheDriverConfig, "artifactdatatablename", log)
allParametersOk = allParametersOk && ok
versionTablename, ok := toStringIfOk(cacheDriverConfig, "versiontablename", log)
allParametersOk = allParametersOk && ok
@@ -108,7 +105,6 @@ func getDynamoParams(cacheDriverConfig map[string]interface{}, log log.Logger) d
RepoMetaTablename: repoMetaTablename,
ManifestDataTablename: manifestDataTablename,
IndexDataTablename: indexDataTablename,
ArtifactDataTablename: artifactDataTablename,
UserDataTablename: userDataTablename,
VersionTablename: versionTablename,
}
pkg/meta/repodb/repodbfactory/repodb_factory_test.go
-1
View File
@@ -22,7 +22,6 @@ func TestCreateDynamo(t *testing.T) {
RepoMetaTablename: "RepoMetadataTable",
ManifestDataTablename: "ManifestDataTable",
IndexDataTablename: "IndexDataTable",
ArtifactDataTablename: "ArtifactDataTable",
UserDataTablename: "UserDataTable",
VersionTablename: "Version",
Region: "us-east-2",
pkg/meta/repodb/storage_parsing.go
+14 -50
View File
@@ -9,6 +9,7 @@ import (
ispec "github.com/opencontainers/image-spec/specs-go/v1"
zerr "zotregistry.io/zot/errors"
zcommon "zotregistry.io/zot/pkg/common"
"zotregistry.io/zot/pkg/log"
"zotregistry.io/zot/pkg/storage"
)
@@ -263,13 +264,6 @@ func NewIndexData(repoName string, indexBlob []byte, imageStore storage.ImageSto
return indexData
}
func NewArtifactData(repo string, descriptorBlob []byte, imageStore storage.ImageStore,
) ArtifactData {
return ArtifactData{
ManifestBlob: descriptorBlob,
}
}
// SetMetadataFromInput tries to set manifest metadata and update repo metadata by adding the current tag
// (in case the reference is a tag). The function expects image manifests and indexes (multi arch images).
func SetMetadataFromInput(repo, reference, mediaType string, digest godigest.Digest, descriptorBlob []byte,
@@ -295,15 +289,6 @@ func SetMetadataFromInput(repo, reference, mediaType string, digest godigest.Dig
if err != nil {
log.Error().Err(err).Msg("repodb: error while putting index data")
return err
}
case ispec.MediaTypeArtifactManifest:
artifactData := NewArtifactData(repo, descriptorBlob, imageStore)
err := repoDB.SetArtifactData(digest, artifactData)
if err != nil {
log.Error().Err(err).Msg("repodb: error while putting artifact data")
return err
}
}
@@ -335,43 +320,22 @@ func GetReferredSubject(descriptorBlob []byte, referrerDigest, mediaType string,
referrerSubject *ispec.Descriptor
)
switch mediaType {
case ispec.MediaTypeImageManifest:
var manifestContent ispec.Manifest
var manifestContent ispec.Manifest
err := json.Unmarshal(descriptorBlob, &manifestContent)
if err != nil {
return "", referrerInfo, false,
fmt.Errorf("repodb: can't unmarhsal manifest for digest %s: %w", referrerDigest, err)
}
err := json.Unmarshal(descriptorBlob, &manifestContent)
if err != nil {
return "", referrerInfo, false,
fmt.Errorf("repodb: can't unmarshal manifest for digest %s: %w", referrerDigest, err)
}
referrerSubject = manifestContent.Subject
referrerSubject = manifestContent.Subject
referrerInfo = ReferrerInfo{
Digest: referrerDigest,
MediaType: mediaType,
ArtifactType: manifestContent.Config.MediaType,
Size: len(descriptorBlob),
Annotations: manifestContent.Annotations,
}
case ispec.MediaTypeArtifactManifest:
manifestContent := ispec.Artifact{}
err := json.Unmarshal(descriptorBlob, &manifestContent)
if err != nil {
return "", referrerInfo, false,
fmt.Errorf("repodb: can't unmarhsal artifact manifest for digest %s: %w", referrerDigest, err)
}
referrerSubject = manifestContent.Subject
referrerInfo = ReferrerInfo{
Digest: referrerDigest,
MediaType: manifestContent.MediaType,
ArtifactType: manifestContent.ArtifactType,
Size: len(descriptorBlob),
Annotations: manifestContent.Annotations,
}
referrerInfo = ReferrerInfo{
Digest: referrerDigest,
MediaType: mediaType,
ArtifactType: zcommon.GetManifestArtifactType(manifestContent),
Size: len(descriptorBlob),
Annotations: manifestContent.Annotations,
}
if referrerSubject == nil || referrerSubject.Digest.String() == "" {
pkg/meta/repodb/storage_parsing_test.go
+2 -6
View File
@@ -237,7 +237,7 @@ func TestParseStorageErrors(t *testing.T) {
})
Convey("CheckIsImageSignature -> is signature", func() {
manifestContent := ispec.Artifact{
manifestContent := ispec.Manifest{
Subject: &ispec.Descriptor{
Digest: "123",
},
@@ -292,7 +292,6 @@ func TestParseStorageDynamoWrapper(t *testing.T) {
RepoMetaTablename: "RepoMetadataTable",
ManifestDataTablename: "ManifestDataTable",
IndexDataTablename: "IndexDataTable",
ArtifactDataTablename: "ArtifactDataTable",
UserDataTablename: "UserDataTable",
VersionTablename: "Version",
}
@@ -515,10 +514,7 @@ func RunParseStorageTests(rootDir string, repoDB repodb.RepoDB) {
func TestGetReferredSubject(t *testing.T) {
Convey("GetReferredSubject error", t, func() {
_, _, _, err := repodb.GetReferredSubject([]byte("bad json"), "digest", ispec.MediaTypeArtifactManifest)
So(err, ShouldNotBeNil)
_, _, _, err = repodb.GetReferredSubject([]byte("bad json"), "digest", ispec.MediaTypeImageManifest)
_, _, _, err := repodb.GetReferredSubject([]byte("bad json"), "digest", ispec.MediaTypeImageManifest)
So(err, ShouldNotBeNil)
})
}
pkg/meta/update.go
+1 -1
View File
@@ -144,7 +144,7 @@ func OnDeleteManifest(repo, reference, mediaType string, digest godigest.Digest,
}
// OnDeleteManifest is called when a manifest is downloaded. It increments the download couter on that manifest.
func OnGetManifest(name, reference string, digest godigest.Digest, body []byte,
func OnGetManifest(name, reference string, body []byte,
storeController storage.StoreController, repoDB repodb.RepoDB, log log.Logger,
) error {
// check if image is a signature
pkg/meta/update_test.go
+12 -16
View File
@@ -100,7 +100,7 @@ func TestUpdateErrors(t *testing.T) {
log := log.NewLogger("debug", "")
Convey("zerr.ErrOrphanSignature", func() {
manifestContent := ispec.Artifact{
manifestContent := ispec.Manifest{
Subject: &ispec.Descriptor{
Digest: "123",
},
@@ -126,7 +126,7 @@ func TestUpdateErrors(t *testing.T) {
log := log.NewLogger("debug", "")
Convey("CheckIsImageSignature errors", func() {
manifestContent := ispec.Artifact{
manifestContent := ispec.Manifest{
Subject: &ispec.Descriptor{
Digest: "123",
},
@@ -179,7 +179,7 @@ func TestUpdateErrors(t *testing.T) {
log := log.NewLogger("debug", "")
Convey("CheckIsImageSignature errors", func() {
manifestContent := ispec.Artifact{
manifestContent := ispec.Manifest{
Subject: &ispec.Descriptor{
Digest: "123",
},
@@ -192,7 +192,7 @@ func TestUpdateErrors(t *testing.T) {
return []byte{}, "", "", zerr.ErrManifestNotFound
}
err = meta.OnGetManifest("repo", "tag1", "digest", manifestBlob,
err = meta.OnGetManifest("repo", "tag1", manifestBlob,
storeController, repoDB, log)
So(err, ShouldNotBeNil)
@@ -200,7 +200,7 @@ func TestUpdateErrors(t *testing.T) {
return []byte{}, "", "", ErrTestError
}
err = meta.OnGetManifest("repo", "tag1", "media", manifestBlob,
err = meta.OnGetManifest("repo", "tag1", manifestBlob,
storeController, repoDB, log)
So(err, ShouldNotBeNil)
})
@@ -245,22 +245,18 @@ func TestUpdateErrors(t *testing.T) {
err := repodb.SetMetadataFromInput("repo", "ref", ispec.MediaTypeImageManifest, "digest",
[]byte("{}"), imageStore, repoDB, log)
So(err, ShouldNotBeNil)
})
repoDB = mocks.RepoDBMock{
Convey("SetMetadataFromInput SetIndexData errors", func() {
imageStore := mocks.MockedImageStore{}
log := log.NewLogger("debug", "")
repoDB := mocks.RepoDBMock{
SetIndexDataFn: func(digest godigest.Digest, indexData repodb.IndexData) error {
return ErrTestError
},
}
err = repodb.SetMetadataFromInput("repo", "ref", ispec.MediaTypeImageIndex, "digest",
[]byte("{}"), imageStore, repoDB, log)
So(err, ShouldNotBeNil)
repoDB = mocks.RepoDBMock{
SetArtifactDataFn: func(digest godigest.Digest, artifactData repodb.ArtifactData) error {
return ErrTestError
},
}
err = repodb.SetMetadataFromInput("repo", "ref", ispec.MediaTypeArtifactManifest, "digest",
err := repodb.SetMetadataFromInput("repo", "ref", ispec.MediaTypeImageIndex, "digest",
[]byte("{}"), imageStore, repoDB, log)
So(err, ShouldNotBeNil)
})
pkg/meta/version/version_test.go
-1
View File
@@ -125,7 +125,6 @@ func TestVersioningDynamoDB(t *testing.T) {
Region: region,
RepoMetaTablename: "RepoMetadataTable",
ManifestDataTablename: "ManifestDataTable",
ArtifactDataTablename: "ArtifactDataTable",
IndexDataTablename: "IndexDataTable",
UserDataTablename: "UserDataTable",
VersionTablename: "Version",
pkg/storage/common.go
+78 -98
View File
@@ -16,10 +16,16 @@ import (
"github.com/sigstore/cosign/v2/pkg/oci/remote"
zerr "zotregistry.io/zot/errors"
zcommon "zotregistry.io/zot/pkg/common"
"zotregistry.io/zot/pkg/scheduler"
storageConstants "zotregistry.io/zot/pkg/storage/constants"
)
const (
CosignType = "cosign"
NotationType = "notation"
)
func SignatureMediaTypes() map[string]bool {
return map[string]bool{
notreg.ArtifactTypeNotation: true,
@@ -106,22 +112,6 @@ func ValidateManifest(imgStore ImageStore, repo, reference, mediaType string, bo
return "", zerr.ErrBadManifest
}
case ispec.MediaTypeArtifactManifest:
var artifact ispec.Artifact
if err := json.Unmarshal(body, &artifact); err != nil {
log.Error().Err(err).Msg("unable to unmarshal JSON")
return "", zerr.ErrBadManifest
}
if artifact.Subject != nil {
var m ispec.Descriptor
if err := json.Unmarshal(body, &m); err != nil {
log.Error().Err(err).Msg("unable to unmarshal JSON")
return "", zerr.ErrBadManifest
}
}
}
return "", nil
@@ -303,6 +293,27 @@ func GetImageIndex(imgStore ImageStore, repo string, digest godigest.Digest, log
return imageIndex, nil
}
func GetImageManifest(imgStore ImageStore, repo string, digest godigest.Digest, log zerolog.Logger,
) (ispec.Manifest, error) {
var manifestContent ispec.Manifest
manifestBlob, err := imgStore.GetBlobContent(repo, digest)
if err != nil {
return manifestContent, err
}
manifestPath := path.Join(imgStore.RootDir(), repo, "blobs",
digest.Algorithm().String(), digest.Encoded())
if err := json.Unmarshal(manifestBlob, &manifestContent); err != nil {
log.Error().Err(err).Str("path", manifestPath).Msg("invalid JSON")
return manifestContent, err
}
return manifestContent, nil
}
func RemoveManifestDescByReference(index *ispec.Index, reference string, detectCollisions bool,
) (ispec.Descriptor, error) {
var removedManifest ispec.Descriptor
@@ -456,31 +467,50 @@ func PruneImageManifestsFromIndex(imgStore ImageStore, repo string, digest godig
return prunedManifests, nil
}
func ApplyLinter(imgStore ImageStore, linter Lint, repo string, manifestDesc ispec.Descriptor) (bool, error) {
func ApplyLinter(imgStore ImageStore, linter Lint, repo string, descriptor ispec.Descriptor) (bool, error) {
pass := true
if linter != nil {
tag := manifestDesc.Annotations[ispec.AnnotationRefName]
// apply linter only on images, not signatures
if manifestDesc.MediaType == ispec.MediaTypeImageManifest &&
// check that image manifest is not cosign signature
!strings.HasPrefix(tag, "sha256-") &&
!strings.HasSuffix(tag, remote.SignatureTagSuffix) {
// lint new index with new manifest before writing to disk
pass, err := linter.Lint(repo, manifestDesc.Digest, imgStore)
if err != nil {
return false, err
}
// we'll skip anything that's not a image manifest
if descriptor.MediaType != ispec.MediaTypeImageManifest {
return pass, nil
}
if !pass {
return false, zerr.ErrImageLintAnnotations
}
if linter != nil && !IsSignature(descriptor) {
// lint new index with new manifest before writing to disk
pass, err := linter.Lint(repo, descriptor.Digest, imgStore)
if err != nil {
return false, err
}
if !pass {
return false, zerr.ErrImageLintAnnotations
}
}
return pass, nil
}
func IsSignature(descriptor ispec.Descriptor) bool {
tag := descriptor.Annotations[ispec.AnnotationRefName]
switch descriptor.MediaType {
case ispec.MediaTypeImageManifest:
// is cosgin signature
if strings.HasPrefix(tag, "sha256-") && strings.HasSuffix(tag, remote.SignatureTagSuffix) {
return true
}
// is notation signature
if descriptor.ArtifactType == notreg.ArtifactTypeNotation {
return true
}
default:
return false
}
return false
}
func GetOrasReferrers(imgStore ImageStore, repo string, gdigest godigest.Digest, artifactType string,
log zerolog.Logger,
) ([]oras.Descriptor, error) {
@@ -609,67 +639,18 @@ func GetReferrers(imgStore ImageStore, repo string, gdigest godigest.Digest, art
}
// filter by artifact type
if len(artifactTypes) > 0 {
found := false
manifestArtifactType := zcommon.GetManifestArtifactType(mfst)
for _, artifactType := range artifactTypes {
if artifactType != "" && mfst.Config.MediaType != artifactType {
continue
}
found = true
break
}
if !found {
continue
}
}
result = append(result, ispec.Descriptor{
MediaType: manifest.MediaType,
ArtifactType: mfst.Config.MediaType,
Size: manifest.Size,
Digest: manifest.Digest,
Annotations: mfst.Annotations,
})
} else if manifest.MediaType == ispec.MediaTypeArtifactManifest {
var art ispec.Artifact
if err := json.Unmarshal(buf, &art); err != nil {
log.Error().Err(err).Str("manifest digest", manifest.Digest.String()).Msg("invalid JSON")
return nilIndex, err
}
if art.Subject == nil || art.Subject.Digest != gdigest {
if len(artifactTypes) > 0 && !zcommon.Contains(artifactTypes, manifestArtifactType) {
continue
}
// filter by artifact type
if len(artifactTypes) > 0 {
found := false
for _, artifactType := range artifactTypes {
if artifactType != "" && art.ArtifactType != artifactType {
continue
}
found = true
break
}
if !found {
continue
}
}
result = append(result, ispec.Descriptor{
MediaType: manifest.MediaType,
ArtifactType: art.ArtifactType,
ArtifactType: manifestArtifactType,
Size: manifest.Size,
Digest: manifest.Digest,
Annotations: art.Annotations,
Annotations: mfst.Annotations,
})
}
}
@@ -719,14 +700,13 @@ func GetOrasManifestByDigest(imgStore ImageStore, repo string, digest godigest.D
func IsSupportedMediaType(mediaType string) bool {
return mediaType == ispec.MediaTypeImageIndex ||
mediaType == ispec.MediaTypeImageManifest ||
mediaType == ispec.MediaTypeArtifactManifest ||
mediaType == oras.MediaTypeArtifactManifest
}
func IsNonDistributable(mediaType string) bool {
return mediaType == ispec.MediaTypeImageLayerNonDistributable ||
mediaType == ispec.MediaTypeImageLayerNonDistributableGzip ||
mediaType == ispec.MediaTypeImageLayerNonDistributableZstd
return mediaType == ispec.MediaTypeImageLayerNonDistributable || //nolint:staticcheck
mediaType == ispec.MediaTypeImageLayerNonDistributableGzip || //nolint:staticcheck
mediaType == ispec.MediaTypeImageLayerNonDistributableZstd //nolint:staticcheck
}
// CheckIsImageSignature checks if the given image (repo:tag) represents a signature. The function
@@ -742,30 +722,30 @@ func IsNonDistributable(mediaType string) bool {
func CheckIsImageSignature(repoName string, manifestBlob []byte, reference string,
storeController StoreController,
) (bool, string, godigest.Digest, error) {
const cosign = "cosign"
var manifestContent ispec.Artifact
var manifestContent ispec.Manifest
err := json.Unmarshal(manifestBlob, &manifestContent)
if err != nil {
return false, "", "", err
}
manifestArtifactType := zcommon.GetManifestArtifactType(manifestContent)
// check notation signature
if _, ok := SignatureMediaTypes()[manifestContent.ArtifactType]; ok && manifestContent.Subject != nil {
if _, ok := SignatureMediaTypes()[manifestArtifactType]; ok && manifestContent.Subject != nil {
imgStore := storeController.GetImageStore(repoName)
_, signedImageManifestDigest, _, err := imgStore.GetImageManifest(repoName,
manifestContent.Subject.Digest.String())
if err != nil {
if errors.Is(err, zerr.ErrManifestNotFound) {
return true, "notation", signedImageManifestDigest, zerr.ErrOrphanSignature
return true, NotationType, signedImageManifestDigest, zerr.ErrOrphanSignature
}
return false, "", "", err
}
return true, "notation", signedImageManifestDigest, nil
return true, NotationType, signedImageManifestDigest, nil
}
// check cosign
@@ -785,17 +765,17 @@ func CheckIsImageSignature(repoName string, manifestBlob []byte, reference strin
signedImageManifestDigest.String())
if err != nil {
if errors.Is(err, zerr.ErrManifestNotFound) {
return true, cosign, signedImageManifestDigest, zerr.ErrOrphanSignature
return true, CosignType, signedImageManifestDigest, zerr.ErrOrphanSignature
}
return false, "", "", err
}
if signedImageManifestDigest.String() == "" {
return true, cosign, signedImageManifestDigest, zerr.ErrOrphanSignature
return true, CosignType, signedImageManifestDigest, zerr.ErrOrphanSignature
}
return true, cosign, signedImageManifestDigest, nil
return true, CosignType, signedImageManifestDigest, nil
}
return false, "", "", nil
pkg/storage/common_test.go
+11 -29
View File
@@ -118,7 +118,7 @@ func TestValidateManifest(t *testing.T) {
},
Layers: []ispec.Descriptor{
{
MediaType: ispec.MediaTypeImageLayerNonDistributable,
MediaType: ispec.MediaTypeImageLayerNonDistributable, //nolint:staticcheck
Digest: digest,
Size: int64(len(content)),
},
@@ -299,38 +299,11 @@ func TestGetReferrersErrors(t *testing.T) {
So(err, ShouldNotBeNil)
})
Convey("Trigger unmarshal error on artifact mediaType", func(c C) {
index = ispec.Index{
Manifests: []ispec.Descriptor{
{
MediaType: ispec.MediaTypeArtifactManifest,
Digest: digest,
},
},
}
indexBuf, err = json.Marshal(index)
So(err, ShouldBeNil)
imgStore = &mocks.MockedImageStore{
GetIndexContentFn: func(repo string) ([]byte, error) {
return indexBuf, nil
},
GetBlobContentFn: func(repo string, digest godigest.Digest) ([]byte, error) {
return []byte{}, nil
},
}
_, err = storage.GetReferrers(imgStore, "zot-test", validDigest,
[]string{artifactType}, log.With().Caller().Logger())
So(err, ShouldNotBeNil)
})
Convey("Trigger nil subject", func(c C) {
index = ispec.Index{
Manifests: []ispec.Descriptor{
{
MediaType: ispec.MediaTypeArtifactManifest,
MediaType: ispec.MediaTypeImageManifest,
Digest: digest,
},
},
@@ -398,3 +371,12 @@ func TestGetImageIndexErrors(t *testing.T) {
So(err, ShouldNotBeNil)
})
}
func TestIsSignature(t *testing.T) {
Convey("Unknown media type", t, func(c C) {
isSingature := storage.IsSignature(ispec.Descriptor{
MediaType: "unknown media type",
})
So(isSingature, ShouldBeFalse)
})
}
pkg/storage/local/local.go
+48 -36
View File
@@ -29,7 +29,7 @@ import (
"github.com/sigstore/cosign/v2/pkg/oci/remote"
zerr "zotregistry.io/zot/errors"
"zotregistry.io/zot/pkg/common"
zcommon "zotregistry.io/zot/pkg/common"
"zotregistry.io/zot/pkg/extensions/monitoring"
zlog "zotregistry.io/zot/pkg/log"
zreg "zotregistry.io/zot/pkg/regexp"
@@ -66,7 +66,7 @@ func (is *ImageStoreLocal) RootDir() string {
}
func (is *ImageStoreLocal) DirExists(d string) bool {
return common.DirExists(d)
return zcommon.DirExists(d)
}
// NewImageStore returns a new image store backed by a file storage.
@@ -535,7 +535,18 @@ func (is *ImageStoreLocal) PutImageManifest(repo, reference, mediaType string, /
return "", err
}
// apply linter only on images, not signatures
if mediaType == ispec.MediaTypeImageManifest {
var manifest ispec.Manifest
err := json.Unmarshal(body, &manifest)
if err != nil {
return "", err
}
desc.ArtifactType = zcommon.GetManifestArtifactType(manifest)
}
// apply linter only on images, not signatures or indexes
pass, err := storage.ApplyLinter(is, is.linter, repo, desc)
if !pass {
is.log.Error().Err(err).Str("repository", repo).Str("reference", reference).Msg("linter didn't pass")
@@ -1440,6 +1451,12 @@ func ensureDir(dir string, log zerolog.Logger) error {
return nil
}
type extendedManifest struct {
ispec.Manifest
Digest godigest.Digest
}
func (is *ImageStoreLocal) garbageCollect(dir string, repo string) error {
oci, err := umoci.OpenLayout(dir)
if err := test.Error(err); err != nil {
@@ -1455,7 +1472,7 @@ func (is *ImageStoreLocal) garbageCollect(dir string, repo string) error {
referencedByImageIndex := []string{}
cosignDescriptors := []ispec.Descriptor{}
notationDescriptors := []ispec.Descriptor{}
notationManifests := []extendedManifest{}
/* gather manifests references by multiarch images (to skip gc)
gather cosign and notation signatures descriptors */
@@ -1479,10 +1496,27 @@ func (is *ImageStoreLocal) garbageCollect(dir string, repo string) error {
// gather cosign signatures
if strings.HasPrefix(tag, "sha256-") && strings.HasSuffix(tag, remote.SignatureTagSuffix) {
cosignDescriptors = append(cosignDescriptors, desc)
continue
}
}
case ispec.MediaTypeArtifactManifest:
notationDescriptors = append(notationDescriptors, desc)
manifestContent, err := storage.GetImageManifest(is, repo, desc.Digest, is.log)
if err != nil {
is.log.Error().Err(err).Str("repo", repo).Str("digest", desc.Digest.String()).
Msg("gc: failed to read manifest image")
return err
}
if zcommon.GetManifestArtifactType(manifestContent) == notreg.ArtifactTypeNotation {
notationManifests = append(notationManifests, extendedManifest{
Digest: desc.Digest,
Manifest: manifestContent,
})
continue
}
}
}
@@ -1500,7 +1534,7 @@ func (is *ImageStoreLocal) garbageCollect(dir string, repo string) error {
is.log.Info().Msg("gc: notation signatures")
if err := gcNotationSignatures(is, oci, &index, repo, notationDescriptors); err != nil {
if err := gcNotationSignatures(is, oci, &index, repo, notationManifests); err != nil {
return err
}
@@ -1519,7 +1553,7 @@ func gcUntaggedManifests(imgStore *ImageStoreLocal, oci casext.Engine, index *is
) error {
for _, desc := range index.Manifests {
// skip manifests referenced in image indexex
if common.Contains(referencedByImageIndex, desc.Digest.String()) {
if zcommon.Contains(referencedByImageIndex, desc.Digest.String()) {
continue
}
@@ -1616,46 +1650,24 @@ func gcCosignSignatures(imgStore *ImageStoreLocal, oci casext.Engine, index *isp
}
func gcNotationSignatures(imgStore *ImageStoreLocal, oci casext.Engine, index *ispec.Index, repo string,
notationDescriptors []ispec.Descriptor,
notationManifests []extendedManifest,
) error {
for _, notationDesc := range notationDescriptors {
for _, notationManifest := range notationManifests {
foundSubject := false
// check if we can find the manifest which the signature points to
var artManifest ispec.Artifact
buf, err := imgStore.GetBlobContent(repo, notationDesc.Digest)
if err != nil {
imgStore.log.Error().Err(err).Str("repository", repo).Str("digest", notationDesc.Digest.String()).
Msg("gc: failed to get oras artifact manifest")
return err
}
if err := json.Unmarshal(buf, &artManifest); err != nil {
imgStore.log.Error().Err(err).Str("repository", repo).Str("digest", notationDesc.Digest.String()).
Msg("gc: failed to get oras artifact manifest")
return err
}
// skip oci artifacts which are not signatures
if artManifest.ArtifactType != notreg.ArtifactTypeNotation {
continue
}
for _, desc := range index.Manifests {
if desc.Digest == artManifest.Subject.Digest {
if desc.Digest == notationManifest.Subject.Digest {
foundSubject = true
}
}
if !foundSubject {
// remove manifest
imgStore.log.Info().Str("repository", repo).Str("digest", notationDesc.Digest.String()).
imgStore.log.Info().Str("repository", repo).Str("digest", notationManifest.Digest.String()).
Msg("gc: removing notation signature without subject")
// no need to check for manifest conflict, if one doesn't have a subject, then none with same digest will have
_, _ = storage.RemoveManifestDescByReference(index, notationDesc.Digest.String(), false)
_, _ = storage.RemoveManifestDescByReference(index, notationManifest.Digest.String(), false)
err := oci.PutIndex(context.Background(), *index)
if err != nil {
@@ -1808,7 +1820,7 @@ func (is *ImageStoreLocal) GetNextDigestWithBlobPaths(lastDigests []godigest.Dig
return nil //nolint:nilerr // ignore files which are not blobs
}
if digest == "" && !common.DContains(lastDigests, blobDigest) {
if digest == "" && !zcommon.DContains(lastDigests, blobDigest) {
digest = blobDigest
}
pkg/storage/s3/s3.go
+13 -2
View File
@@ -24,7 +24,7 @@ import (
"github.com/rs/zerolog"
zerr "zotregistry.io/zot/errors"
"zotregistry.io/zot/pkg/common"
zcommon "zotregistry.io/zot/pkg/common"
"zotregistry.io/zot/pkg/extensions/monitoring"
zlog "zotregistry.io/zot/pkg/log"
zreg "zotregistry.io/zot/pkg/regexp"
@@ -438,6 +438,17 @@ func (is *ObjectStorage) PutImageManifest(repo, reference, mediaType string, //n
return "", err
}
if mediaType == ispec.MediaTypeImageManifest {
var manifest ispec.Manifest
err := json.Unmarshal(body, &manifest)
if err != nil {
return "", err
}
desc.ArtifactType = zcommon.GetManifestArtifactType(manifest)
}
// apply linter only on images, not signatures
pass, err := storage.ApplyLinter(is, is.linter, repo, desc)
if !pass {
@@ -1419,7 +1430,7 @@ func (is *ObjectStorage) GetNextDigestWithBlobPaths(lastDigests []godigest.Diges
return nil //nolint:nilerr // ignore files which are not blobs
}
if digest == "" && !common.DContains(lastDigests, blobDigest) {
if digest == "" && !zcommon.DContains(lastDigests, blobDigest) {
digest = blobDigest
}
pkg/storage/s3/s3_test.go
-38
View File
@@ -584,44 +584,6 @@ func TestGetOrasAndOCIReferrers(t *testing.T) {
So(index.Manifests[0].Digest, ShouldEqual, manDigest)
})
Convey("Get oci referrers - application/vnd.oci.artifact.manifest.v1+json", func(c C) {
artifactType := "application/vnd.example.icecream.v1"
artifactManifest := ispec.Artifact{
MediaType: ispec.MediaTypeArtifactManifest,
ArtifactType: artifactType,
Blobs: []ispec.Descriptor{
{
MediaType: "application/octet-stream",
Size: int64(buflen),
Digest: digest,
},
},
Subject: &ispec.Descriptor{
MediaType: ispec.MediaTypeImageManifest,
Size: int64(mbuflen),
Digest: mdigest,
},
}
manBuf, err := json.Marshal(artifactManifest)
So(err, ShouldBeNil)
manBufLen := len(manBuf)
manDigest := godigest.FromBytes(manBuf)
_, err = imgStore.PutImageManifest(repo, manDigest.Encoded(), ispec.MediaTypeArtifactManifest, manBuf)
So(err, ShouldBeNil)
index, err := imgStore.GetReferrers(repo, mdigest, []string{artifactType})
So(err, ShouldBeNil)
So(index, ShouldNotBeEmpty)
So(index.Manifests[1].ArtifactType, ShouldEqual, artifactType)
So(index.Manifests[1].MediaType, ShouldEqual, ispec.MediaTypeArtifactManifest)
So(index.Manifests[1].Size, ShouldEqual, manBufLen)
So(index.Manifests[1].Digest, ShouldEqual, manDigest)
})
Convey("Get oras referrers", func(c C) {
artifactManifest := artifactspec.Manifest{}
artifactManifest.ArtifactType = "signature-example"
pkg/test/common.go
+26 -95
View File
@@ -105,32 +105,6 @@ func (img Image) Digest() (godigest.Digest, error) {
return godigest.FromBytes(blob), nil
}
type Artifact struct {
Manifest ispec.Artifact
Blobs []ArtifactBlobs
Reference string
}
func (a Artifact) Digest() (godigest.Digest, error) {
blob, err := json.Marshal(a.Manifest)
if err != nil {
return "", err
}
return godigest.FromBytes(blob), nil
}
func (a Artifact) ArtifactData() (repodb.ArtifactData, error) {
blob, err := json.Marshal(a.Manifest)
if err != nil {
return repodb.ArtifactData{}, err
}
return repodb.ArtifactData{
ManifestBlob: blob,
}, nil
}
type ArtifactBlobs struct {
Blob []byte
MediaType string
@@ -588,6 +562,7 @@ func GetImageComponents(layerSize int) (ispec.Image, [][]byte, ispec.Manifest, e
schemaVersion := 2
manifest := ispec.Manifest{
MediaType: ispec.MediaTypeImageManifest,
Versioned: specs.Versioned{
SchemaVersion: schemaVersion,
},
@@ -642,6 +617,7 @@ func GetRandomImageComponents(layerSize int) (ispec.Image, [][]byte, ispec.Manif
schemaVersion := 2
manifest := ispec.Manifest{
MediaType: ispec.MediaTypeImageManifest,
Versioned: specs.Versioned{
SchemaVersion: schemaVersion,
},
@@ -657,7 +633,6 @@ func GetRandomImageComponents(layerSize int) (ispec.Image, [][]byte, ispec.Manif
Size: int64(len(layers[0])),
},
},
MediaType: ispec.MediaTypeImageManifest,
}
return config, layers, manifest, nil
@@ -702,6 +677,7 @@ func GetImageComponentsWithConfig(conf ispec.Image) (ispec.Image, [][]byte, ispe
schemaVersion := 2
manifest := ispec.Manifest{
MediaType: ispec.MediaTypeImageManifest,
Versioned: specs.Versioned{
SchemaVersion: schemaVersion,
},
@@ -760,6 +736,7 @@ func GetImageWithComponents(config ispec.Image, layers [][]byte) (Image, error)
const schemaVersion = 2
manifest := ispec.Manifest{
MediaType: ispec.MediaTypeImageManifest,
Versioned: specs.Versioned{
SchemaVersion: schemaVersion,
},
@@ -784,49 +761,6 @@ func GetImageWithComponents(config ispec.Image, layers [][]byte) (Image, error)
}, nil
}
func GetRandomArtifact(subject *ispec.Descriptor) (Artifact, error) {
var randBlob [10]byte
_, err := rand.Read(randBlob[:])
if err != nil {
return Artifact{}, err
}
artifactBlobs := []ArtifactBlobs{
{
Blob: randBlob[:],
MediaType: "application/octet-stream",
},
}
blobsDescriptors := make([]ispec.Descriptor, 0, len(artifactBlobs))
for _, artifactBlob := range artifactBlobs {
blobsDescriptors = append(blobsDescriptors, ispec.Descriptor{
Digest: godigest.FromBytes(artifactBlob.Blob),
MediaType: artifactBlob.MediaType,
Size: int64(len(artifactBlob.Blob)),
})
}
artifactManifest := ispec.Artifact{
MediaType: ispec.MediaTypeArtifactManifest,
Blobs: blobsDescriptors,
Subject: subject,
}
artifactManifestBlob, err := json.Marshal(artifactManifest)
if err != nil {
return Artifact{}, err
}
return Artifact{
Manifest: artifactManifest,
Blobs: artifactBlobs,
Reference: godigest.FromBytes(artifactManifestBlob).String(),
}, nil
}
func GetCosignSignatureTagForManifest(manifest ispec.Manifest) (string, error) {
manifestBlob, err := json.Marshal(manifest)
if err != nil {
@@ -906,6 +840,11 @@ func UploadImage(img Image, baseURL, repo string) error {
cdigest := godigest.FromBytes(cblob)
if img.Manifest.Config.MediaType == ispec.MediaTypeScratch {
cblob = ispec.ScratchDescriptor.Data
cdigest = ispec.ScratchDescriptor.Digest
}
resp, err := resty.R().
Post(baseURL + "/v2/" + repo + "/blobs/uploads/")
if err = Error(err); err != nil {
@@ -940,7 +879,7 @@ func UploadImage(img Image, baseURL, repo string) error {
}
resp, err = resty.R().
SetHeader("Content-type", "application/vnd.oci.image.manifest.v1+json").
SetHeader("Content-type", ispec.MediaTypeImageManifest).
SetBody(manifestBlob).
Put(baseURL + "/v2/" + repo + "/manifests/" + img.Reference)
@@ -966,27 +905,6 @@ func DeleteImage(repo, reference, baseURL string) (int, error) {
return resp.StatusCode(), err
}
// UploadArtifactManifest is used in tests where we don't need to upload the blobs of the artifact.
func UploadArtifactManifest(artifactManifest *ispec.Artifact, ref *string, baseURL, repo string) error {
// put manifest
artifactManifestBlob, err := json.Marshal(artifactManifest)
if err != nil {
return err
}
reference := godigest.FromBytes(artifactManifestBlob).String()
if ref != nil {
reference = *ref
}
_, err = resty.R().
SetHeader("Content-type", ispec.MediaTypeArtifactManifest).
SetBody(artifactManifestBlob).
Put(baseURL + "/v2/" + repo + "/manifests/" + reference)
return err
}
func UploadBlob(baseURL, repo string, blob []byte, artifactBlobMediaType string) error {
resp, err := resty.R().Post(baseURL + "/v2/" + repo + "/blobs/uploads/")
if err != nil {
@@ -1252,7 +1170,11 @@ func SignWithNotation(keyName string, reference string, tdir string) error {
PlainHTTP: plainHTTP,
}
sigRepo := notreg.NewRepository(remoteRepo)
repositoryOpts := notreg.RepositoryOptions{
OCIImageManifest: true,
}
sigRepo := notreg.NewRepositoryWithOptions(remoteRepo, repositoryOpts)
sigOpts := notation.RemoteSignOptions{
SignOptions: notation.SignOptions{
@@ -1334,7 +1256,11 @@ func VerifyWithNotation(reference string, tdir string) error {
PlainHTTP: plainHTTP,
}
repo := notreg.NewRepository(remoteRepo)
repositoryOpts := notreg.RepositoryOptions{
OCIImageManifest: true,
}
repo := notreg.NewRepositoryWithOptions(remoteRepo, repositoryOpts)
manifestDesc, err := repo.Resolve(ctx, ref.Reference)
if err != nil {
@@ -1357,7 +1283,7 @@ func VerifyWithNotation(reference string, tdir string) error {
PlainHTTP: plainHTTP,
}
repo = notreg.NewRepository(remoteRepo)
repo = notreg.NewRepositoryWithOptions(remoteRepo, repositoryOpts)
configs := map[string]string{}
@@ -1587,6 +1513,11 @@ func UploadImageWithBasicAuth(img Image, baseURL, repo, user, password string) e
cdigest := godigest.FromBytes(cblob)
if img.Manifest.Config.MediaType == ispec.MediaTypeScratch {
cblob = ispec.ScratchDescriptor.Data
cdigest = ispec.ScratchDescriptor.Digest
}
resp, err := resty.R().
SetBasicAuth(user, password).
Post(baseURL + "/v2/" + repo + "/blobs/uploads/")
pkg/test/common_test.go
-12
View File
@@ -234,18 +234,6 @@ func TestControllerManager(t *testing.T) {
})
}
func TestUploadArtifact(t *testing.T) {
Convey("Put request results in an error", t, func() {
port := test.GetFreePort()
baseURL := test.GetBaseURL(port)
artifact := ispec.Artifact{}
err := test.UploadArtifactManifest(&artifact, nil, baseURL, "test")
So(err, ShouldNotBeNil)
})
}
func TestUploadBlob(t *testing.T) {
Convey("Post request results in an error", t, func() {
port := test.GetFreePort()
pkg/test/mocks/repo_db_mock.go
-20
View File
@@ -44,10 +44,6 @@ type RepoDBMock struct {
GetIndexDataFn func(indexDigest godigest.Digest) (repodb.IndexData, error)
SetArtifactDataFn func(digest godigest.Digest, artifactData repodb.ArtifactData) error
GetArtifactDataFn func(artifactDigest godigest.Digest) (repodb.ArtifactData, error)
SetReferrerFn func(repo string, referredDigest godigest.Digest, referrer repodb.ReferrerInfo) error
DeleteReferrerFn func(repo string, referredDigest godigest.Digest, referrerDigest godigest.Digest) error
@@ -348,22 +344,6 @@ func (sdm RepoDBMock) PatchDB() error {
return nil
}
func (sdm RepoDBMock) SetArtifactData(digest godigest.Digest, artifactData repodb.ArtifactData) error {
if sdm.SetArtifactDataFn != nil {
return sdm.SetArtifactDataFn(digest, artifactData)
}
return nil
}
func (sdm RepoDBMock) GetArtifactData(artifactDigest godigest.Digest) (repodb.ArtifactData, error) {
if sdm.GetArtifactDataFn != nil {
return sdm.GetArtifactDataFn(artifactDigest)
}
return repodb.ArtifactData{}, nil
}
func (sdm RepoDBMock) SetReferrer(repo string, referredDigest godigest.Digest, referrer repodb.ReferrerInfo) error {
if sdm.SetReferrerFn != nil {
return sdm.SetReferrerFn(repo, referredDigest, referrer)
pkg/test/oci-layout/oci_layout_test.go
+1 -1
View File
@@ -376,7 +376,7 @@ func TestExtractImageDetails(t *testing.T) {
olu := ocilayout.NewBaseOciLayoutUtils(storeController, testLogger)
resDigest, resManifest, resIspecImage, resErr := olu.ExtractImageDetails("zot-test", "latest", testLogger)
So(string(resDigest), ShouldContainSubstring, "sha256:c52f15d2d4")
So(string(resDigest), ShouldContainSubstring, "sha256:8492645f16")
So(resManifest.Config.Digest.String(), ShouldContainSubstring, configDigest.Encoded())
So(resIspecImage.Architecture, ShouldContainSubstring, "amd64")
pkg/test/test_http_server.go
+51
View File
@@ -0,0 +1,51 @@
package test
import (
"errors"
"fmt"
"net/http"
"github.com/gorilla/mux"
)
type RouteHandler struct {
Route string
// HandlerFunc is the HTTP handler function that receives a writer for output and an HTTP request as input.
HandlerFunc http.HandlerFunc
// AllowedMethods specifies the HTTP methods allowed for the current route.
AllowedMethods []string
}
// Routes is a map that associates HTTP paths to their corresponding HTTP handlers.
type HTTPRoutes []RouteHandler
func StartTestHTTPServer(routes HTTPRoutes, port string) *http.Server {
baseURL := GetBaseURL(port)
mux := mux.NewRouter()
mux.HandleFunc("/test", func(w http.ResponseWriter, r *http.Request) {
_, err := w.Write([]byte("{}"))
if err != nil {
return
}
}).Methods(http.MethodGet)
for _, routeHandler := range routes {
mux.HandleFunc(routeHandler.Route, routeHandler.HandlerFunc).Methods(routeHandler.AllowedMethods...)
}
server := &http.Server{ //nolint:gosec
Addr: fmt.Sprintf(":%s", port),
Handler: mux,
}
go func() {
if err := server.ListenAndServe(); !errors.Is(err, http.ErrServerClosed) {
return
}
}()
WaitTillServerReady(baseURL + "/test")
return server
}