Catalog content discovery (#2782)

fix(sync): use pagination when querying remote catalog

feat(api): added /v2/_catalog pagination, fixes #2715

Signed-off-by: Eusebiu Petu <petu.eusebiu@gmail.com>

pkg/extensions/sync/httpclient/client.go

@@ -177,9 +177,9 @@ func (httpClient *Client) Ping() bool {
 	return false
 }
 
-func (httpClient *Client) MakeGetRequest(ctx context.Context, resultPtr interface{}, mediaType string,
+func (httpClient *Client) MakeGetRequest(ctx context.Context, resultPtr interface{}, mediaType string, rawQuery string,
 	route ...string,
-) ([]byte, string, int, error) {
+) ([]byte, http.Header, int, error) {
 	httpClient.lock.RLock()
 	defer httpClient.lock.RUnlock()
 
@@ -192,11 +192,12 @@ func (httpClient *Client) MakeGetRequest(ctx context.Context, resultPtr interfac
 		// we know that the second route argument is always the repo name.
 		// need it for caching tokens, it's not used in requests made to authz server.
 		if idx == 1 {
-			namespace = path
+			namespace = strings.Trim(path, "/")
 		}
 	}
 
-	url.RawQuery = url.Query().Encode()
+	url.RawQuery = rawQuery
+
 	//nolint: bodyclose,contextcheck
 	resp, body, err := httpClient.makeAndDoRequest(http.MethodGet, mediaType, namespace, url.String())
 	if err != nil {
@@ -204,11 +205,11 @@ func (httpClient *Client) MakeGetRequest(ctx context.Context, resultPtr interfac
 			Str("errorType", common.TypeOf(err)).
 			Msg("failed to make request")
 
-		return nil, "", -1, err
+		return nil, nil, -1, err
 	}
 
 	if resp.StatusCode != http.StatusOK {
-		return nil, "", resp.StatusCode, errors.New(string(body)) //nolint:goerr113
+		return nil, nil, resp.StatusCode, errors.New(string(body)) //nolint:goerr113
 	}
 
 	// read blob
@@ -216,7 +217,7 @@ func (httpClient *Client) MakeGetRequest(ctx context.Context, resultPtr interfac
 		err = json.Unmarshal(body, &resultPtr)
 	}
 
-	return body, resp.Header.Get("Content-Type"), resp.StatusCode, err
+	return body, resp.Header, resp.StatusCode, err
 }
 
 func (httpClient *Client) getAuthType(resp *http.Response) {

pkg/extensions/sync/references/cosign.go

@@ -172,7 +172,7 @@ func (ref CosignReference) SyncReferences(ctx context.Context, localRepo, remote
 func (ref CosignReference) getManifest(ctx context.Context, repo, cosignTag string) (*ispec.Manifest, []byte, error) {
 	var cosignManifest ispec.Manifest
 
-	body, _, statusCode, err := ref.client.MakeGetRequest(ctx, &cosignManifest, ispec.MediaTypeImageManifest,
+	body, _, statusCode, err := ref.client.MakeGetRequest(ctx, &cosignManifest, ispec.MediaTypeImageManifest, "",
 		"v2", repo, "manifests", cosignTag)
 	if err != nil {
 		if statusCode == http.StatusNotFound {

pkg/extensions/sync/references/oci.go

@@ -159,7 +159,7 @@ func (ref OciReferences) SyncReferences(ctx context.Context, localRepo, remoteRe
 func (ref OciReferences) getIndex(ctx context.Context, repo, subjectDigestStr string) (ispec.Index, error) {
 	var index ispec.Index
 
-	_, _, statusCode, err := ref.client.MakeGetRequest(ctx, &index, ispec.MediaTypeImageIndex,
+	_, _, statusCode, err := ref.client.MakeGetRequest(ctx, &index, ispec.MediaTypeImageIndex, "",
 		"v2", repo, "referrers", subjectDigestStr)
 	if err != nil {
 		if statusCode == http.StatusNotFound {
@@ -182,7 +182,7 @@ func syncManifest(ctx context.Context, client *client.Client, imageStore storage
 
 	var refDigest godigest.Digest
 
-	OCIRefBuf, _, statusCode, err := client.MakeGetRequest(ctx, &manifest, ispec.MediaTypeImageManifest,
+	OCIRefBuf, _, statusCode, err := client.MakeGetRequest(ctx, &manifest, ispec.MediaTypeImageManifest, "",
 		"v2", remoteRepo, "manifests", desc.Digest.String())
 	if err != nil {
 		if statusCode == http.StatusNotFound {

pkg/extensions/sync/references/references.go

@@ -152,7 +152,7 @@ func syncBlob(ctx context.Context, client *client.Client, imageStore storageType
 ) error {
 	var resultPtr interface{}
 
-	body, _, statusCode, err := client.MakeGetRequest(ctx, resultPtr, "", "v2", remoteRepo, "blobs", digest.String())
+	body, _, statusCode, err := client.MakeGetRequest(ctx, resultPtr, "", "", "v2", remoteRepo, "blobs", digest.String())
 	if err != nil {
 		if statusCode != http.StatusOK {
 			log.Info().Str("repo", remoteRepo).Str("digest", digest.String()).Msg("couldn't get remote blob")

pkg/extensions/sync/references/referrers_tag.go

@@ -151,7 +151,7 @@ func (ref TagReferences) getIndex(
 ) (ispec.Index, []byte, error) {
 	var index ispec.Index
 
-	content, _, statusCode, err := ref.client.MakeGetRequest(ctx, &index, ispec.MediaTypeImageIndex,
+	content, _, statusCode, err := ref.client.MakeGetRequest(ctx, &index, ispec.MediaTypeImageIndex, "",
 		"v2", repo, "manifests", getReferrersTagFromSubjectDigest(subjectDigestStr))
 	if err != nil {
 		if statusCode == http.StatusNotFound {

pkg/extensions/sync/remote.go

@@ -6,6 +6,7 @@ package sync
 import (
 	"context"
 	"fmt"
+	"net/url"
 	"strings"
 
 	"github.com/containers/image/v5/docker"
@@ -50,13 +51,37 @@ func (registry *RemoteRegistry) GetContext() *types.SystemContext {
 func (registry *RemoteRegistry) GetRepositories(ctx context.Context) ([]string, error) {
 	var catalog catalog
 
-	_, _, _, err := registry.client.MakeGetRequest(ctx, &catalog, "application/json", //nolint: dogsled
+	_, header, _, err := registry.client.MakeGetRequest(ctx, &catalog, "application/json", "", //nolint: dogsled
 		constants.RoutePrefix, constants.ExtCatalogPrefix)
 	if err != nil {
 		return []string{}, err
 	}
 
-	return catalog.Repositories, nil
+	var repos []string
+
+	repos = append(repos, catalog.Repositories...)
+
+	link := header.Get("Link")
+	for link != "" {
+		linkURLPart, _, _ := strings.Cut(link, ";")
+
+		linkURL, err := url.Parse(strings.Trim(linkURLPart, "<>"))
+		if err != nil {
+			return catalog.Repositories, err
+		}
+
+		_, header, _, err := registry.client.MakeGetRequest(ctx, &catalog, "application/json",
+			linkURL.RawQuery, constants.RoutePrefix, constants.ExtCatalogPrefix) //nolint: dogsled
+		if err != nil {
+			return repos, err
+		}
+
+		repos = append(repos, catalog.Repositories...)
+
+		link = header.Get("Link")
+	}
+
+	return repos, nil
 }
 
 func (registry *RemoteRegistry) GetDockerRemoteRepo(repo string) string {