fix: add support for uploaded index when signing using notation (#1882)

ci(notation): update to latest notation version
fix(sync): add layers info when syncing signatures

Signed-off-by: Andreea-Lupu <andreealupu1470@yahoo.com>

pkg/extensions/sync/constants/consts.go

@@ -5,5 +5,6 @@ const (
 	Oras              = "OrasReference"
 	Cosign            = "CosignSignature"
 	OCI               = "OCIReference"
+	Tag               = "TagReference"
 	SyncBlobUploadDir = ".sync"
 )

pkg/extensions/sync/references/cosign.go

@@ -161,10 +161,8 @@ func (ref CosignReference) SyncReferences(ctx context.Context, localRepo, remote
 			}
 
 			if isSig {
-				err = ref.metaDB.AddManifestSignature(localRepo, signedManifestDig, mTypes.SignatureMetadata{
-					SignatureType:   sigType,
-					SignatureDigest: referenceDigest.String(),
-				})
+				err = addSigToMeta(ref.metaDB, localRepo, sigType, cosignTag, signedManifestDig, referenceDigest,
+					manifestBuf, imageStore, ref.log)
 			} else {
 				err = meta.SetImageMetaFromInput(localRepo, cosignTag, ispec.MediaTypeImageManifest,
 					referenceDigest, manifestBuf, ref.storeController.GetImageStore(localRepo),

pkg/extensions/sync/references/oci.go

@@ -145,10 +145,8 @@ func (ref OciReferences) SyncReferences(ctx context.Context, localRepo, remoteRe
 			}
 
 			if isSig {
-				err = ref.metaDB.AddManifestSignature(localRepo, signedManifestDig, mTypes.SignatureMetadata{
-					SignatureType:   sigType,
-					SignatureDigest: referenceDigest.String(),
-				})
+				err = addSigToMeta(ref.metaDB, localRepo, sigType, referrer.Digest.String(), signedManifestDig, referenceDigest,
+					referenceBuf, imageStore, ref.log)
 			} else {
 				err = meta.SetImageMetaFromInput(localRepo, referenceDigest.String(), referrer.MediaType,
 					referenceDigest, referenceBuf, ref.storeController.GetImageStore(localRepo),

pkg/extensions/sync/references/references.go

@@ -17,6 +17,7 @@ import (
 	"zotregistry.io/zot/pkg/common"
 	client "zotregistry.io/zot/pkg/extensions/sync/httpclient"
 	"zotregistry.io/zot/pkg/log"
+	"zotregistry.io/zot/pkg/meta"
 	mTypes "zotregistry.io/zot/pkg/meta/types"
 	"zotregistry.io/zot/pkg/storage"
 	storageTypes "zotregistry.io/zot/pkg/storage/types"
@@ -42,6 +43,7 @@ func NewReferences(httpClient *client.Client, storeController storage.StoreContr
 	refs := References{log: log}
 
 	refs.referenceList = append(refs.referenceList, NewCosignReference(httpClient, storeController, metaDB, log))
+	refs.referenceList = append(refs.referenceList, NewTagReferences(httpClient, storeController, metaDB, log))
 	refs.referenceList = append(refs.referenceList, NewOciReferences(httpClient, storeController, metaDB, log))
 	refs.referenceList = append(refs.referenceList, NewORASReferences(httpClient, storeController, metaDB, log))
 
@@ -215,3 +217,21 @@ func getNotationManifestsFromOCIRefs(ociRefs ispec.Index) []ispec.Descriptor {
 
 	return notaryManifests
 }
+
+func addSigToMeta(
+	metaDB mTypes.MetaDB, repo, sigType, tag string, signedManifestDig, referenceDigest godigest.Digest,
+	referenceBuf []byte, imageStore storageTypes.ImageStore, log log.Logger,
+) error {
+	layersInfo, errGetLayers := meta.GetSignatureLayersInfo(repo, tag, referenceDigest.String(),
+		sigType, referenceBuf, imageStore, log)
+
+	if errGetLayers != nil {
+		return errGetLayers
+	}
+
+	return metaDB.AddManifestSignature(repo, signedManifestDig, mTypes.SignatureMetadata{
+		SignatureType:   sigType,
+		SignatureDigest: referenceDigest.String(),
+		LayersInfo:      layersInfo,
+	})
+}

pkg/extensions/sync/references/references_internal_test.go

@@ -86,6 +86,54 @@ func TestOci(t *testing.T) {
 	})
 }
 
+func TestReferrersTag(t *testing.T) {
+	Convey("trigger errors", t, func() {
+		cfg := client.Config{
+			URL:       "url",
+			TLSVerify: false,
+		}
+
+		client, err := client.New(cfg, log.NewLogger("debug", ""))
+		So(err, ShouldBeNil)
+
+		referrersTag := NewTagReferences(client, storage.StoreController{DefaultStore: mocks.MockedImageStore{
+			GetImageManifestFn: func(repo, reference string) ([]byte, godigest.Digest, string, error) {
+				return []byte{}, "", "", errRef
+			},
+		}}, nil, log.NewLogger("debug", ""))
+
+		ok := referrersTag.IsSigned(context.Background(), "repo", "")
+		So(ok, ShouldBeFalse)
+
+		// trigger GetImageManifest err
+		ok, err = referrersTag.canSkipReferences("repo", "subjectdigest", "digest")
+		So(err, ShouldNotBeNil)
+		So(ok, ShouldBeFalse)
+
+		referrersTag = NewTagReferences(client, storage.StoreController{DefaultStore: mocks.MockedImageStore{
+			GetImageManifestFn: func(repo, reference string) ([]byte, godigest.Digest, string, error) {
+				return []byte{}, "", "", zerr.ErrManifestNotFound
+			},
+		}}, nil, log.NewLogger("debug", ""))
+
+		// trigger GetImageManifest err
+		ok, err = referrersTag.canSkipReferences("repo", "subjectdigest", "digest")
+		So(err, ShouldBeNil)
+		So(ok, ShouldBeFalse)
+
+		referrersTag = NewTagReferences(client, storage.StoreController{DefaultStore: mocks.MockedImageStore{
+			GetImageManifestFn: func(repo, reference string) ([]byte, godigest.Digest, string, error) {
+				return []byte{}, "digest", "", nil
+			},
+		}}, nil, log.NewLogger("debug", ""))
+
+		// different digest
+		ok, err = referrersTag.canSkipReferences("repo", "subjectdigest", "newdigest")
+		So(err, ShouldBeNil)
+		So(ok, ShouldBeFalse)
+	})
+}
+
 func TestORAS(t *testing.T) {
 	Convey("trigger errors", t, func() {
 		cfg := client.Config{
@@ -392,3 +440,14 @@ func TestCompareArtifactRefs(t *testing.T) {
 		}
 	})
 }
+
+func TestAddSigToMeta(t *testing.T) {
+	Convey("Test addSigToMeta", t, func() {
+		imageStore := mocks.MockedImageStore{}
+		metaDB := mocks.MetaDBMock{}
+
+		err := addSigToMeta(metaDB, "repo", "cosign", "tag", godigest.FromString("signedmanifest"),
+			godigest.FromString("reference"), []byte("bad"), imageStore, log.Logger{})
+		So(err, ShouldNotBeNil)
+	})
+}

pkg/extensions/sync/references/referrers_tag.go

@@ -0,0 +1,185 @@
+//go:build sync
+// +build sync
+
+package references
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"net/http"
+	"strings"
+
+	godigest "github.com/opencontainers/go-digest"
+	ispec "github.com/opencontainers/image-spec/specs-go/v1"
+
+	zerr "zotregistry.io/zot/errors"
+	"zotregistry.io/zot/pkg/common"
+	"zotregistry.io/zot/pkg/extensions/sync/constants"
+	client "zotregistry.io/zot/pkg/extensions/sync/httpclient"
+	"zotregistry.io/zot/pkg/log"
+	"zotregistry.io/zot/pkg/meta"
+	mTypes "zotregistry.io/zot/pkg/meta/types"
+	"zotregistry.io/zot/pkg/storage"
+)
+
+type TagReferences struct {
+	client          *client.Client
+	storeController storage.StoreController
+	metaDB          mTypes.MetaDB
+	log             log.Logger
+}
+
+func NewTagReferences(httpClient *client.Client, storeController storage.StoreController,
+	metaDB mTypes.MetaDB, log log.Logger,
+) TagReferences {
+	return TagReferences{
+		client:          httpClient,
+		storeController: storeController,
+		metaDB:          metaDB,
+		log:             log,
+	}
+}
+
+func (ref TagReferences) Name() string {
+	return constants.Tag
+}
+
+func (ref TagReferences) IsSigned(ctx context.Context, remoteRepo, subjectDigestStr string) bool {
+	return false
+}
+
+func (ref TagReferences) canSkipReferences(localRepo, subjectDigestStr, digest string) (bool, error) {
+	imageStore := ref.storeController.GetImageStore(localRepo)
+
+	_, localDigest, _, err := imageStore.GetImageManifest(localRepo, getReferrersTagFromSubjectDigest(subjectDigestStr))
+	if err != nil {
+		if errors.Is(err, zerr.ErrManifestNotFound) {
+			return false, nil
+		}
+
+		ref.log.Error().Str("errorType", common.TypeOf(err)).
+			Str("repository", localRepo).Str("subject", subjectDigestStr).
+			Err(err).Msg("couldn't get local index with referrers tag for image")
+
+		return false, err
+	}
+
+	if localDigest.String() != digest {
+		return false, nil
+	}
+
+	ref.log.Info().Str("repository", localRepo).Str("reference", subjectDigestStr).
+		Msg("skipping index with referrers tag for image, already synced")
+
+	return true, nil
+}
+
+func (ref TagReferences) SyncReferences(ctx context.Context, localRepo, remoteRepo, subjectDigestStr string) (
+	[]godigest.Digest, error,
+) {
+	refsDigests := make([]godigest.Digest, 0, 10)
+
+	index, indexContent, err := ref.getIndex(ctx, remoteRepo, subjectDigestStr)
+	if err != nil {
+		return refsDigests, err
+	}
+
+	skipTagRefs, err := ref.canSkipReferences(localRepo, subjectDigestStr, string(godigest.FromBytes(indexContent)))
+	if err != nil {
+		ref.log.Error().Err(err).Str("repository", localRepo).Str("subject", subjectDigestStr).
+			Msg("couldn't check if the upstream index with referrers tag for image can be skipped")
+	}
+
+	if skipTagRefs {
+		return refsDigests, nil
+	}
+
+	imageStore := ref.storeController.GetImageStore(localRepo)
+
+	ref.log.Info().Str("repository", localRepo).Str("subject", subjectDigestStr).
+		Msg("syncing oci references for image")
+
+	for _, referrer := range index.Manifests {
+		referenceBuf, referenceDigest, err := syncManifest(ctx, ref.client, imageStore, localRepo, remoteRepo,
+			referrer, subjectDigestStr, ref.log)
+		if err != nil {
+			return refsDigests, err
+		}
+
+		refsDigests = append(refsDigests, referenceDigest)
+
+		if ref.metaDB != nil {
+			ref.log.Debug().Str("repository", localRepo).Str("subject", subjectDigestStr).
+				Msg("metaDB: trying to add oci references for image")
+
+			isSig, sigType, signedManifestDig, err := storage.CheckIsImageSignature(localRepo, referenceBuf,
+				referrer.Digest.String())
+			if err != nil {
+				return refsDigests, fmt.Errorf("failed to check if oci reference '%s@%s' is a signature: %w", localRepo,
+					referrer.Digest.String(), err)
+			}
+
+			if isSig {
+				err = addSigToMeta(ref.metaDB, localRepo, sigType, referrer.Digest.String(), signedManifestDig, referenceDigest,
+					referenceBuf, imageStore, ref.log)
+			} else {
+				err = meta.SetImageMetaFromInput(localRepo, referenceDigest.String(), referrer.MediaType,
+					referenceDigest, referenceBuf, ref.storeController.GetImageStore(localRepo),
+					ref.metaDB, ref.log)
+			}
+
+			if err != nil {
+				return refsDigests, fmt.Errorf("failed to set metadata for oci reference in '%s@%s': %w",
+					localRepo, subjectDigestStr, err)
+			}
+
+			ref.log.Info().Str("repository", localRepo).Str("subject", subjectDigestStr).
+				Msg("metaDB: successfully added oci references to MetaDB for image")
+		}
+	}
+
+	ref.log.Info().Str("repository", localRepo).Str("subject", subjectDigestStr).
+		Msg("syncing index with referrers tag for image")
+
+	referrersTag := getReferrersTagFromSubjectDigest(subjectDigestStr)
+
+	_, _, err = imageStore.PutImageManifest(localRepo, referrersTag, index.MediaType, indexContent)
+	if err != nil {
+		ref.log.Error().Str("errorType", common.TypeOf(err)).
+			Str("repository", localRepo).Str("subject", subjectDigestStr).
+			Err(err).Msg("couldn't upload index with referrers tag for image")
+
+		return refsDigests, err
+	}
+
+	ref.log.Info().Str("repository", localRepo).Str("subject", subjectDigestStr).
+		Msg("successfully synced index with referrers tag for image")
+
+	return refsDigests, nil
+}
+
+func (ref TagReferences) getIndex(
+	ctx context.Context, repo, subjectDigestStr string,
+) (ispec.Index, []byte, error) {
+	var index ispec.Index
+
+	content, _, statusCode, err := ref.client.MakeGetRequest(ctx, &index, ispec.MediaTypeImageIndex,
+		"v2", repo, "manifests", getReferrersTagFromSubjectDigest(subjectDigestStr))
+	if err != nil {
+		if statusCode == http.StatusNotFound {
+			ref.log.Debug().Str("repository", repo).Str("subject", subjectDigestStr).
+				Msg("couldn't find any index with referrers tag for image, skipping")
+
+			return index, []byte{}, zerr.ErrSyncReferrerNotFound
+		}
+
+		return index, []byte{}, err
+	}
+
+	return index, content, nil
+}
+
+func getReferrersTagFromSubjectDigest(digestStr string) string {
+	return strings.Replace(digestStr, ":", "-", 1)
+}

pkg/extensions/sync/service.go

@@ -298,7 +298,7 @@ func (service *BaseService) SyncRepo(ctx context.Context, repo string) error {
 		default:
 		}
 
-		if references.IsCosignTag(tag) {
+		if references.IsCosignTag(tag) || common.IsReferrersTag(tag) {
 			continue
 		}
 
@@ -374,7 +374,8 @@ func (service *BaseService) syncTag(ctx context.Context, localRepo, remoteRepo,
 		return "", zerr.ErrMediaTypeNotSupported
 	}
 
-	if service.config.OnlySigned != nil && *service.config.OnlySigned && !references.IsCosignTag(tag) {
+	if service.config.OnlySigned != nil && *service.config.OnlySigned &&
+		!references.IsCosignTag(tag) && !common.IsReferrersTag(tag) {
 		signed := service.references.IsSigned(ctx, remoteRepo, manifestDigest.String())
 		if !signed {
 			// skip unsigned images

pkg/extensions/sync/sync_test.go

@@ -898,6 +898,146 @@ func TestOnDemand(t *testing.T) {
 			So(err, ShouldBeNil)
 			So(resp.StatusCode(), ShouldEqual, http.StatusOK)
 		})
+
+		Convey("Sync referrers tag errors", func() {
+			// start upstream server
+			rootDir := t.TempDir()
+			port := test.GetFreePort()
+			srcBaseURL := test.GetBaseURL(port)
+			conf := config.New()
+			conf.HTTP.Port = port
+			conf.Storage.GC = false
+			ctlr := api.NewController(conf)
+			ctlr.Config.Storage.RootDirectory = rootDir
+
+			cm := test.NewControllerManager(ctlr)
+			cm.StartAndWait(conf.HTTP.Port)
+			defer cm.StopServer()
+
+			image := CreateRandomImage()
+			manifestBlob := image.ManifestDescriptor.Data
+			manifestDigest := image.ManifestDescriptor.Digest
+
+			err := UploadImage(image, srcBaseURL, "remote-repo", "test")
+			So(err, ShouldBeNil)
+
+			subjectDesc := ispec.Descriptor{
+				MediaType: ispec.MediaTypeImageManifest,
+				Digest:    manifestDigest,
+				Size:      int64(len(manifestBlob)),
+			}
+
+			ociRefImage := CreateDefaultImageWith().Subject(&subjectDesc).Build()
+
+			err = UploadImage(ociRefImage, srcBaseURL, "remote-repo", ociRefImage.ManifestDescriptor.Digest.String())
+			So(err, ShouldBeNil)
+
+			tag := strings.Replace(manifestDigest.String(), ":", "-", 1)
+
+			// add index with referrers tag
+			tagRefIndex := ispec.Index{
+				MediaType: ispec.MediaTypeImageIndex,
+				Manifests: []ispec.Descriptor{
+					{
+						MediaType: ispec.MediaTypeImageManifest,
+						Digest:    ociRefImage.ManifestDescriptor.Digest,
+						Size:      int64(len(ociRefImage.ManifestDescriptor.Data)),
+					},
+				},
+				Annotations: map[string]string{ispec.AnnotationRefName: tag},
+			}
+
+			tagRefIndex.SchemaVersion = 2
+
+			tagRefIndexBlob, err := json.Marshal(tagRefIndex)
+			So(err, ShouldBeNil)
+
+			resp, err := resty.R().
+				SetHeader("Content-type", ispec.MediaTypeImageIndex).
+				SetBody(tagRefIndexBlob).
+				Put(srcBaseURL + "/v2/remote-repo/manifests/" + tag)
+
+			So(err, ShouldBeNil)
+			So(resp.StatusCode(), ShouldEqual, http.StatusCreated)
+
+			//------- Start downstream server
+
+			var tlsVerify bool
+
+			regex := ".*"
+			semver := true
+
+			destPort := test.GetFreePort()
+			destConfig := config.New()
+
+			destBaseURL := test.GetBaseURL(destPort)
+
+			hostname, err := os.Hostname()
+			So(err, ShouldBeNil)
+
+			syncRegistryConfig := syncconf.RegistryConfig{
+				Content: []syncconf.Content{
+					{
+						Prefix: "remote-repo",
+						Tags: &syncconf.Tags{
+							Regex:  &regex,
+							Semver: &semver,
+						},
+					},
+				},
+				// include self url, should be ignored
+				URLs: []string{
+					fmt.Sprintf("http://%s", hostname), destBaseURL,
+					srcBaseURL, fmt.Sprintf("http://localhost:%s", destPort),
+				},
+				TLSVerify: &tlsVerify,
+				CertDir:   "",
+				OnDemand:  true,
+			}
+
+			defaultVal := true
+			syncConfig := &syncconf.Config{
+				Enable:     &defaultVal,
+				Registries: []syncconf.RegistryConfig{syncRegistryConfig},
+			}
+
+			destConfig.HTTP.Port = destPort
+
+			destDir := t.TempDir()
+
+			destConfig.Storage.RootDirectory = destDir
+			destConfig.Storage.Dedupe = false
+			destConfig.Storage.GC = false
+
+			destConfig.Extensions = &extconf.ExtensionConfig{}
+
+			destConfig.Extensions.Sync = syncConfig
+
+			dctlr := api.NewController(destConfig)
+
+			// metadb fails for syncReferrersTag"
+			dctlr.MetaDB = mocks.MetaDBMock{
+				SetManifestDataFn: func(manifestDigest godigest.Digest, mm mTypes.ManifestData) error {
+					if manifestDigest.String() == ociRefImage.ManifestDescriptor.Digest.String() {
+						return sync.ErrTestError
+					}
+
+					return nil
+				},
+			}
+
+			dcm := test.NewControllerManager(dctlr)
+			dcm.StartAndWait(destPort)
+			defer dcm.StopServer()
+
+			resp, err = resty.R().Get(destBaseURL + "/v2/remote-repo/manifests/test")
+			So(err, ShouldBeNil)
+			So(resp.StatusCode(), ShouldEqual, http.StatusOK)
+
+			resp, err = resty.R().Get(destBaseURL + "/v2/remote-repo/manifests/" + tag)
+			So(err, ShouldBeNil)
+			So(resp.StatusCode(), ShouldEqual, http.StatusNotFound)
+		})
 	})
 }
 
@@ -3818,7 +3958,10 @@ func TestPeriodicallySignaturesErr(t *testing.T) {
 				So(found, ShouldBeTrue)
 
 				// should not be synced nor sync on demand
-				resp, err = resty.R().Get(destBaseURL + artifactURLPath)
+				resp, err = resty.R().
+					SetHeader("Content-Type", "application/json").
+					SetQueryParam("artifactType", "application/vnd.cncf.icecream").
+					Get(destBaseURL + artifactURLPath)
 				So(err, ShouldBeNil)
 				So(resp.StatusCode(), ShouldEqual, http.StatusOK)
 
@@ -3871,7 +4014,10 @@ func TestPeriodicallySignaturesErr(t *testing.T) {
 				So(found, ShouldBeTrue)
 
 				// should not be synced nor sync on demand
-				resp, err = resty.R().Get(destBaseURL + artifactURLPath)
+				resp, err = resty.R().
+					SetHeader("Content-Type", "application/json").
+					SetQueryParam("artifactType", "application/vnd.cncf.icecream").
+					Get(destBaseURL + artifactURLPath)
 				So(err, ShouldBeNil)
 				So(resp.StatusCode(), ShouldEqual, http.StatusOK)
 
@@ -4475,7 +4621,7 @@ func TestSyncedSignaturesMetaDB(t *testing.T) {
 		err := UploadImage(signedImage, srcBaseURL, repoName, tag)
 		So(err, ShouldBeNil)
 
-		err = signature.SignImageUsingNotary(repoName+":"+tag, srcPort)
+		err = signature.SignImageUsingNotary(repoName+":"+tag, srcPort, true)
 		So(err, ShouldBeNil)
 
 		err = signature.SignImageUsingCosign(repoName+":"+tag, srcPort)
@@ -5715,7 +5861,7 @@ func TestSyncSignaturesDiff(t *testing.T) {
 		So(func() { signImage(tdir, srcPort, repoName, digest) }, ShouldNotPanic)
 
 		// wait for signatures
-		time.Sleep(10 * time.Second)
+		time.Sleep(12 * time.Second)
 
 		// notation verify the image
 		image = fmt.Sprintf("localhost:%s/%s:%s", destPort, repoName, testImageTag)
@@ -6301,8 +6447,8 @@ func signImage(tdir, port, repoName string, digest godigest.Digest) {
 	// sign the image
 	image := fmt.Sprintf("localhost:%s/%s@%s", port, repoName, digest.String())
 
-	err = signature.SignWithNotation("good", image, tdir)
-	if err != nil {
+	err = signature.SignWithNotation("good", image, tdir, false)
+	if err != nil && !strings.Contains(err.Error(), "failed to delete dangling referrers index") {
 		panic(err)
 	}