fix(oras)!: remove ORAS artifact references support (#2294)

* fix(oras)!: remove ORAS artifact references support

ORAS artifacts/references predated OCI dist-spec 1.1.0 which now has the
same functionality and likely to see wider adoption.

Signed-off-by: Ramkumar Chinchani <rchincha@cisco.com>

* test: update to released official images

So that they are unlikely to be deleted.
*-rc images may be cleaned up over time.

Signed-off-by: Ramkumar Chinchani <rchincha@cisco.com>

---------

Signed-off-by: Ramkumar Chinchani <rchincha@cisco.com>

pkg/extensions/sync/references/oras.go

@@ -1,194 +0,0 @@
-//go:build sync
-// +build sync
-
-package references
-
-import (
-	"context"
-	"errors"
-	"fmt"
-	"net/http"
-
-	godigest "github.com/opencontainers/go-digest"
-	oras "github.com/oras-project/artifacts-spec/specs-go/v1"
-
-	zerr "zotregistry.dev/zot/errors"
-	apiConstants "zotregistry.dev/zot/pkg/api/constants"
-	"zotregistry.dev/zot/pkg/common"
-	"zotregistry.dev/zot/pkg/extensions/sync/constants"
-	client "zotregistry.dev/zot/pkg/extensions/sync/httpclient"
-	"zotregistry.dev/zot/pkg/log"
-	"zotregistry.dev/zot/pkg/meta"
-	mTypes "zotregistry.dev/zot/pkg/meta/types"
-	"zotregistry.dev/zot/pkg/storage"
-)
-
-type ReferenceList struct {
-	References []oras.Descriptor `json:"references"`
-}
-
-type ORASReferences struct {
-	client          *client.Client
-	storeController storage.StoreController
-	metaDB          mTypes.MetaDB
-	log             log.Logger
-}
-
-func NewORASReferences(httpClient *client.Client, storeController storage.StoreController,
-	metaDB mTypes.MetaDB, log log.Logger,
-) ORASReferences {
-	return ORASReferences{
-		client:          httpClient,
-		storeController: storeController,
-		metaDB:          metaDB,
-		log:             log,
-	}
-}
-
-func (ref ORASReferences) Name() string {
-	return constants.Oras
-}
-
-func (ref ORASReferences) IsSigned(ctx context.Context, remoteRepo, subjectDigestStr string) bool {
-	return false
-}
-
-func (ref ORASReferences) canSkipReferences(localRepo, subjectDigestStr string, referrers ReferenceList) (bool, error) {
-	imageStore := ref.storeController.GetImageStore(localRepo)
-	digest := godigest.Digest(subjectDigestStr)
-
-	// check oras artifacts already synced
-	if len(referrers.References) > 0 {
-		localRefs, err := imageStore.GetOrasReferrers(localRepo, digest, "")
-		if err != nil {
-			if errors.Is(err, zerr.ErrManifestNotFound) {
-				return false, nil
-			}
-
-			ref.log.Error().Str("errorType", common.TypeOf(err)).Str("repository", localRepo).
-				Str("subject", subjectDigestStr).
-				Err(err).Msg("couldn't get local ORAS artifact for image")
-
-			return false, err
-		}
-
-		if !artifactDescriptorsEqual(localRefs, referrers.References) {
-			ref.log.Info().Str("repository", localRepo).Str("subject", subjectDigestStr).
-				Msg("upstream ORAS artifacts for image changed, syncing again")
-
-			return false, nil
-		}
-	}
-
-	ref.log.Info().Str("repository", localRepo).Str("subject", subjectDigestStr).
-		Msg("skipping ORAS artifact for image, already synced")
-
-	return true, nil
-}
-
-func (ref ORASReferences) SyncReferences(ctx context.Context, localRepo, remoteRepo, subjectDigestStr string) (
-	[]godigest.Digest, error,
-) {
-	refsDigests := make([]godigest.Digest, 0, 10)
-
-	referrers, err := ref.getReferenceList(ctx, remoteRepo, subjectDigestStr)
-	if err != nil {
-		return refsDigests, err
-	}
-
-	skipORASRefs, err := ref.canSkipReferences(localRepo, subjectDigestStr, referrers)
-	if err != nil {
-		ref.log.Error().Err(err).Str("repository", localRepo).Str("subject", subjectDigestStr).
-			Msg("couldn't check if ORAS artifact for image can be skipped")
-	}
-
-	if skipORASRefs {
-		for _, man := range referrers.References {
-			refsDigests = append(refsDigests, man.Digest)
-		}
-
-		return refsDigests, nil
-	}
-
-	imageStore := ref.storeController.GetImageStore(localRepo)
-
-	ref.log.Info().Str("repository", localRepo).Str("subject", subjectDigestStr).
-		Msg("syncing ORAS artifacts for image")
-
-	for _, referrer := range referrers.References {
-		var artifactManifest oras.Manifest
-
-		orasBuf, _, statusCode, err := ref.client.MakeGetRequest(ctx, &artifactManifest, oras.MediaTypeDescriptor,
-			"v2", remoteRepo, "manifests", referrer.Digest.String())
-		if err != nil {
-			if statusCode == http.StatusNotFound {
-				return refsDigests, zerr.ErrSyncReferrerNotFound
-			}
-
-			ref.log.Error().Str("errorType", common.TypeOf(err)).
-				Str("repository", localRepo).Str("subject", subjectDigestStr).
-				Err(err).Msg("couldn't get ORAS artifact for image")
-
-			return refsDigests, err
-		}
-
-		for _, blob := range artifactManifest.Blobs {
-			if err := syncBlob(ctx, ref.client, imageStore, localRepo, remoteRepo, blob.Digest, ref.log); err != nil {
-				return refsDigests, err
-			}
-		}
-
-		referenceDigest, _, err := imageStore.PutImageManifest(localRepo, referrer.Digest.String(),
-			oras.MediaTypeArtifactManifest, orasBuf)
-		if err != nil {
-			ref.log.Error().Str("errorType", common.TypeOf(err)).
-				Str("repository", localRepo).Str("subject", subjectDigestStr).
-				Err(err).Msg("couldn't upload ORAS artifact for image")
-
-			return refsDigests, err
-		}
-
-		refsDigests = append(refsDigests, referenceDigest)
-
-		if ref.metaDB != nil {
-			ref.log.Debug().Str("repository", localRepo).Str("subject", subjectDigestStr).Str("component", "metadb").
-				Msg("trying to sync oras artifact for image")
-
-			err := meta.SetImageMetaFromInput(context.Background(), localRepo, //nolint:contextcheck
-				referenceDigest.String(), referrer.MediaType,
-				referenceDigest, orasBuf, ref.storeController.GetImageStore(localRepo),
-				ref.metaDB, ref.log)
-			if err != nil {
-				return refsDigests, fmt.Errorf("failed to set metadata in db for oras artifact '%s@%s': %w",
-					localRepo, subjectDigestStr, err)
-			}
-
-			ref.log.Info().Str("repository", localRepo).Str("subject", subjectDigestStr).Str("component", "metadb").
-				Msg("successfully added oras artifacts to MetaDB for image")
-		}
-	}
-
-	ref.log.Info().Str("repository", localRepo).Str("subject", subjectDigestStr).
-		Msg("successfully synced oras artifacts for image")
-
-	return refsDigests, nil
-}
-
-func (ref ORASReferences) getReferenceList(ctx context.Context, repo, subjectDigestStr string) (ReferenceList, error) {
-	var referrers ReferenceList
-
-	_, _, statusCode, err := ref.client.MakeGetRequest(ctx, &referrers, "application/json",
-		apiConstants.ArtifactSpecRoutePrefix, repo, "manifests", subjectDigestStr, "referrers")
-	if err != nil {
-		if statusCode == http.StatusNotFound || statusCode == http.StatusBadRequest {
-			ref.log.Debug().Str("repository", repo).Str("subject", subjectDigestStr).Err(err).
-				Msg("couldn't find any ORAS artifact for image")
-
-			return referrers, zerr.ErrSyncReferrerNotFound
-		}
-
-		return referrers, err
-	}
-
-	return referrers, nil
-}

pkg/extensions/sync/references/references.go

@@ -12,7 +12,6 @@ import (
 
 	godigest "github.com/opencontainers/go-digest"
 	ispec "github.com/opencontainers/image-spec/specs-go/v1"
-	artifactspec "github.com/oras-project/artifacts-spec/specs-go/v1"
 	"github.com/sigstore/cosign/v2/pkg/oci/static"
 
 	zerr "zotregistry.dev/zot/errors"
@@ -27,7 +26,7 @@ import (
 )
 
 type Reference interface {
-	// Returns name of reference (OCIReference/CosignReference/OrasReference)
+	// Returns name of reference (OCIReference/CosignReference)
 	Name() string
 	// Returns whether or not image is signed
 	IsSigned(ctx context.Context, upstreamRepo, subjectDigestStr string) bool
@@ -49,7 +48,6 @@ func NewReferences(httpClient *client.Client, storeController storage.StoreContr
 	refs.referenceList = append(refs.referenceList, NewCosignReference(httpClient, storeController, metaDB, log))
 	refs.referenceList = append(refs.referenceList, NewTagReferences(httpClient, storeController, metaDB, log))
 	refs.referenceList = append(refs.referenceList, NewOciReferences(httpClient, storeController, metaDB, log))
-	refs.referenceList = append(refs.referenceList, NewORASReferences(httpClient, storeController, metaDB, log))
 
 	return refs
 }
@@ -81,7 +79,7 @@ func (refs References) syncAll(ctx context.Context, localRepo, upstreamRepo,
 	// mark subject digest as seen as soon as it comes in
 	*seen = append(*seen, godigest.Digest(subjectDigestStr))
 
-	// for each reference type(cosign/oci/oras reference)
+	// for each reference type(cosign/oci reference)
 	for _, ref := range refs.referenceList {
 		supported, ok := refs.features.Get(ref.Name(), upstreamRepo)
 		if !supported && ok {
@@ -186,23 +184,6 @@ func manifestsEqual(manifest1, manifest2 ispec.Manifest) bool {
 	return false
 }
 
-func artifactDescriptorsEqual(desc1, desc2 []artifactspec.Descriptor) bool {
-	if len(desc1) != len(desc2) {
-		return false
-	}
-
-	for id, desc := range desc1 {
-		if desc.Digest != desc2[id].Digest ||
-			desc.Size != desc2[id].Size ||
-			desc.MediaType != desc2[id].MediaType ||
-			desc.ArtifactType != desc2[id].ArtifactType {
-			return false
-		}
-	}
-
-	return true
-}
-
 func descriptorsEqual(desc1, desc2 []ispec.Descriptor) bool {
 	if len(desc1) != len(desc2) {
 		return false

pkg/extensions/sync/references/references_internal_test.go

@@ -10,7 +10,6 @@ import (
 
 	godigest "github.com/opencontainers/go-digest"
 	ispec "github.com/opencontainers/image-spec/specs-go/v1"
-	artifactspec "github.com/oras-project/artifacts-spec/specs-go/v1"
 	. "github.com/smartystreets/goconvey/convey"
 
 	zerr "zotregistry.dev/zot/errors"
@@ -134,45 +133,6 @@ func TestReferrersTag(t *testing.T) {
 	})
 }
 
-func TestORAS(t *testing.T) {
-	Convey("trigger errors", t, func() {
-		cfg := client.Config{
-			URL:       "url",
-			TLSVerify: false,
-		}
-
-		client, err := client.New(cfg, log.NewLogger("debug", ""))
-		So(err, ShouldBeNil)
-
-		orasRefs := []artifactspec.Descriptor{
-			{
-				MediaType:    "oras",
-				ArtifactType: "oras",
-				Digest:       "digest1",
-			},
-		}
-
-		oras := NewORASReferences(client, storage.StoreController{DefaultStore: mocks.MockedImageStore{
-			GetOrasReferrersFn: func(repo string, digest godigest.Digest, artifactType string) (
-				[]artifactspec.Descriptor, error,
-			) {
-				return orasRefs, nil
-			},
-		}}, nil, log.NewLogger("debug", ""))
-
-		// trigger artifactDescriptors not equal
-		ok, err := oras.canSkipReferences("repo", "tag", ReferenceList{[]artifactspec.Descriptor{
-			{
-				MediaType:    "oras",
-				ArtifactType: "oras",
-				Digest:       "digest2",
-			},
-		}})
-		So(err, ShouldBeNil)
-		So(ok, ShouldBeFalse)
-	})
-}
-
 func TestSyncManifest(t *testing.T) {
 	Convey("sync manifest not found err", t, func() {
 		cfg := client.Config{
@@ -344,99 +304,3 @@ func TestCompareManifest(t *testing.T) {
 		}
 	})
 }
-
-func TestCompareArtifactRefs(t *testing.T) {
-	testCases := []struct {
-		refs1    []artifactspec.Descriptor
-		refs2    []artifactspec.Descriptor
-		expected bool
-	}{
-		{
-			refs1: []artifactspec.Descriptor{
-				{
-					Digest: "digest1",
-				},
-			},
-			refs2: []artifactspec.Descriptor{
-				{
-					Digest: "digest2",
-				},
-			},
-			expected: false,
-		},
-		{
-			refs1: []artifactspec.Descriptor{
-				{
-					Digest: "digest",
-				},
-			},
-			refs2: []artifactspec.Descriptor{
-				{
-					Digest: "digest",
-				},
-			},
-			expected: true,
-		},
-		{
-			refs1: []artifactspec.Descriptor{
-				{
-					Digest: "digest",
-				},
-				{
-					Digest: "digest2",
-				},
-			},
-			refs2: []artifactspec.Descriptor{
-				{
-					Digest: "digest",
-				},
-			},
-			expected: false,
-		},
-		{
-			refs1: []artifactspec.Descriptor{
-				{
-					Digest: "digest1",
-				},
-				{
-					Digest: "digest2",
-				},
-			},
-			refs2: []artifactspec.Descriptor{
-				{
-					Digest: "digest1",
-				},
-				{
-					Digest: "digest2",
-				},
-			},
-			expected: true,
-		},
-		{
-			refs1: []artifactspec.Descriptor{
-				{
-					Digest: "digest",
-				},
-				{
-					Digest: "digest1",
-				},
-			},
-			refs2: []artifactspec.Descriptor{
-				{
-					Digest: "digest1",
-				},
-				{
-					Digest: "digest2",
-				},
-			},
-			expected: false,
-		},
-	}
-
-	Convey("Test manifestsEqual()", t, func() {
-		for _, test := range testCases {
-			actualResult := artifactDescriptorsEqual(test.refs1, test.refs2)
-			So(actualResult, ShouldEqual, test.expected)
-		}
-	})
-}