feat(repodb): Multiarch Image support (#1147)

* feat(repodb): index logic + tests Signed-off-by: Laurentiu Niculae <niculae.laurentiu1@gmail.com> * feat(cli): printing indexes support using the rest api Signed-off-by: Laurentiu Niculae <niculae.laurentiu1@gmail.com> --------- Signed-off-by: Laurentiu Niculae <niculae.laurentiu1@gmail.com>
2026-06-16 20:38:08 +08:00 · 2023-02-27 21:23:18 +02:00
parent a561d0bad5
commit d62c09e2cc
54 changed files with 8656 additions and 2988 deletions
@@ -52,6 +52,19 @@ func makeGETRequest(ctx context.Context, url, username, password string,
 	return doHTTPRequest(req, verifyTLS, debug, resultsPtr, configWriter)
 }

+func makeHEADRequest(ctx context.Context, url, username, password string, verifyTLS bool,
+	debug bool,
+) (http.Header, error) {
+	req, err := http.NewRequestWithContext(ctx, http.MethodHead, url, nil)
+	if err != nil {
+		return nil, err
+	}
+
+	req.SetBasicAuth(username, password)
+
+	return doHTTPRequest(req, verifyTLS, debug, nil, io.Discard)
+}
+
 func makeGraphQLRequest(ctx context.Context, url, query, username,
 	password string, verifyTLS bool, debug bool, resultsPtr interface{}, configWriter io.Writer,
 ) error {
@@ -126,6 +139,10 @@ func doHTTPRequest(req *http.Request, verifyTLS bool, debug bool,
 		return nil, errors.New(string(bodyBytes)) //nolint: goerr113
 	}

+	if resultsPtr == nil {
+		return resp.Header, nil
+	}
+
 	if err := json.NewDecoder(resp.Body).Decode(resultsPtr); err != nil {
 		return nil, err
 	}
@@ -140,26 +157,25 @@ func isURL(str string) bool {
 } // from https://stackoverflow.com/a/55551215

 type requestsPool struct {
-	jobs     chan *manifestJob
+	jobs     chan *httpJob
 	done     chan struct{}
 	wtgrp    *sync.WaitGroup
 	outputCh chan stringResult
 }

-type manifestJob struct {
-	url          string
-	username     string
-	password     string
-	imageName    string
-	tagName      string
-	config       searchConfig
-	manifestResp manifestResponse
+type httpJob struct {
+	url       string
+	username  string
+	password  string
+	imageName string
+	tagName   string
+	config    searchConfig
 }

 const rateLimiterBuffer = 5000

 func newSmoothRateLimiter(wtgrp *sync.WaitGroup, opch chan stringResult) *requestsPool {
-	ch := make(chan *manifestJob, rateLimiterBuffer)
+	ch := make(chan *httpJob, rateLimiterBuffer)

 	return &requestsPool{
 		jobs:     ch,
@@ -188,11 +204,12 @@ func (p *requestsPool) startRateLimiter(ctx context.Context) {
 	}
 }

-func (p *requestsPool) doJob(ctx context.Context, job *manifestJob) {
+func (p *requestsPool) doJob(ctx context.Context, job *httpJob) {
 	defer p.wtgrp.Done()

-	header, err := makeGETRequest(ctx, job.url, job.username, job.password,
-		*job.config.verifyTLS, *job.config.debug, &job.manifestResp, job.config.resultWriter)
+	// Check manifest media type
+	header, err := makeHEADRequest(ctx, job.url, job.username, job.password, *job.config.verifyTLS,
+		*job.config.debug)
 	if err != nil {
 		if isContextDone(ctx) {
 			return
@@ -200,88 +217,298 @@ func (p *requestsPool) doJob(ctx context.Context, job *manifestJob) {
 		p.outputCh <- stringResult{"", err}
 	}

-	digestStr := header.Get("docker-content-digest")
-	configDigest := job.manifestResp.Config.Digest
+	switch header.Get("Content-Type") {
+	case ispec.MediaTypeImageManifest:
+		image, err := fetchImageManifestStruct(ctx, job)
+		if err != nil {
+			if isContextDone(ctx) {
+				return
+			}
+			p.outputCh <- stringResult{"", err}

-	var size uint64
+			return
+		}
+		platformStr := getPlatformStr(image.Manifests[0].Platform)
+
+		str, err := image.string(*job.config.outputFormat, len(job.imageName), len(job.tagName), len(platformStr))
+		if err != nil {
+			if isContextDone(ctx) {
+				return
+			}
+			p.outputCh <- stringResult{"", err}
+
+			return
+		}
+
+		if isContextDone(ctx) {
+			return
+		}
+
+		p.outputCh <- stringResult{str, nil}
+	case ispec.MediaTypeImageIndex:
+		image, err := fetchImageIndexStruct(ctx, job)
+		if err != nil {
+			if isContextDone(ctx) {
+				return
+			}
+			p.outputCh <- stringResult{"", err}
+
+			return
+		}
+
+		platformStr := getPlatformStr(image.Manifests[0].Platform)
+
+		str, err := image.string(*job.config.outputFormat, len(job.imageName), len(job.tagName), len(platformStr))
+		if err != nil {
+			if isContextDone(ctx) {
+				return
+			}
+			p.outputCh <- stringResult{"", err}
+
+			return
+		}
+
+		if isContextDone(ctx) {
+			return
+		}
+
+		p.outputCh <- stringResult{str, nil}
+	default:
+		return
+	}
+}
+
+func fetchImageIndexStruct(ctx context.Context, job *httpJob) (*imageStruct, error) {
+	var indexContent ispec.Index
+
+	header, err := makeGETRequest(ctx, job.url, job.username, job.password,
+		*job.config.verifyTLS, *job.config.debug, &indexContent, job.config.resultWriter)
+	if err != nil {
+		if isContextDone(ctx) {
+			return nil, context.Canceled
+		}
+
+		return nil, err
+	}
+
+	indexDigest := header.Get("docker-content-digest")
+
+	indexSize, err := strconv.ParseInt(header.Get("Content-Length"), 10, 64)
+	if err != nil {
+		return nil, err
+	}
+
+	imageSize := indexSize
+
+	manifestList := make([]manifestStruct, 0, len(indexContent.Manifests))
+
+	for _, manifestDescriptor := range indexContent.Manifests {
+		manifest, err := fetchManifestStruct(ctx, job.imageName, manifestDescriptor.Digest.String(),
+			job.config, job.username, job.password)
+		if err != nil {
+			return nil, err
+		}
+
+		imageSize += int64(atoiWithDefault(manifest.Size, 0))
+
+		if manifestDescriptor.Platform != nil {
+			manifest.Platform = platform{
+				Os:      manifestDescriptor.Platform.OS,
+				Arch:    manifestDescriptor.Platform.Architecture,
+				Variant: manifestDescriptor.Platform.Variant,
+			}
+		}
+
+		manifestList = append(manifestList, manifest)
+	}
+
+	isIndexSigned := isCosignSigned(ctx, job.imageName, indexDigest, job.config, job.username, job.password) ||
+		isNotationSigned(ctx, job.imageName, indexDigest, job.config, job.username, job.password)
+
+	return &imageStruct{
+		verbose:   *job.config.verbose,
+		RepoName:  job.imageName,
+		Tag:       job.tagName,
+		Size:      strconv.FormatInt(imageSize, 10),
+		IsSigned:  isIndexSigned,
+		Manifests: manifestList,
+	}, nil
+}
+
+func atoiWithDefault(size string, defaultVal int) int {
+	val, err := strconv.Atoi(size)
+	if err != nil {
+		return defaultVal
+	}
+
+	return val
+}
+
+func fetchImageManifestStruct(ctx context.Context, job *httpJob) (*imageStruct, error) {
+	manifest, err := fetchManifestStruct(ctx, job.imageName, job.tagName, job.config, job.username, job.password)
+	if err != nil {
+		return nil, err
+	}
+
+	return &imageStruct{
+		verbose:  *job.config.verbose,
+		RepoName: job.imageName,
+		Tag:      job.tagName,
+		Size:     manifest.Size,
+		IsSigned: manifest.IsSigned,
+		Manifests: []manifestStruct{
+			manifest,
+		},
+	}, nil
+}
+
+func fetchManifestStruct(ctx context.Context, repo, manifestReference string, searchConf searchConfig,
+	username, password string,
+) (manifestStruct, error) {
+	manifestResp := ispec.Manifest{}
+
+	URL := fmt.Sprintf("%s/v2/%s/manifests/%s",
+		*searchConf.servURL, repo, manifestReference)
+
+	header, err := makeGETRequest(ctx, URL, username, password,
+		*searchConf.verifyTLS, *searchConf.debug, &manifestResp, searchConf.resultWriter)
+	if err != nil {
+		if isContextDone(ctx) {
+			return manifestStruct{}, context.Canceled
+		}
+
+		return manifestStruct{}, err
+	}
+
+	manifestDigest := header.Get("docker-content-digest")
+	configDigest := manifestResp.Config.Digest.String()
+
+	configContent, err := fetchConfig(ctx, repo, configDigest, searchConf, username, password)
+	if err != nil {
+		if isContextDone(ctx) {
+			return manifestStruct{}, context.Canceled
+		}
+
+		return manifestStruct{}, err
+	}
+
+	opSys := ""
+	arch := ""
+	variant := ""
+
+	if manifestResp.Config.Platform != nil {
+		opSys = manifestResp.Config.Platform.OS
+		arch = manifestResp.Config.Platform.Architecture
+		variant = manifestResp.Config.Platform.Variant
+	}
+
+	if opSys == "" {
+		opSys = configContent.OS
+	}
+
+	if arch == "" {
+		arch = configContent.Architecture
+	}
+
+	if variant == "" {
+		variant = configContent.Variant
+	}
+
+	manifestSize, err := strconv.ParseInt(header.Get("Content-Length"), 10, 64)
+	if err != nil {
+		return manifestStruct{}, err
+	}
+
+	var imageSize int64
+
+	imageSize += manifestResp.Config.Size
+	imageSize += manifestSize

 	layers := []layer{}

-	for _, entry := range job.manifestResp.Layers {
-		size += entry.Size
+	for _, entry := range manifestResp.Layers {
+		imageSize += entry.Size

 		layers = append(
 			layers,
 			layer{
 				Size:   entry.Size,
-				Digest: entry.Digest,
+				Digest: entry.Digest.String(),
 			},
 		)
 	}

-	size += uint64(job.manifestResp.Config.Size)
+	isSigned := isCosignSigned(ctx, repo, manifestDigest, searchConf, username, password) ||
+		isNotationSigned(ctx, repo, manifestDigest, searchConf, username, password)

-	manifestSize, err := strconv.Atoi(header.Get("Content-Length"))
-	if err != nil {
-		p.outputCh <- stringResult{"", err}
-	}
+	return manifestStruct{
+		ConfigDigest: configDigest,
+		Digest:       manifestDigest,
+		Layers:       layers,
+		Platform:     platform{Os: opSys, Arch: arch, Variant: variant},
+		Size:         strconv.FormatInt(imageSize, 10),
+		IsSigned:     isSigned,
+	}, nil
+}

-	isSigned := false
-	cosignTag := strings.Replace(digestStr, ":", "-", 1) + "." + remote.SignatureTagSuffix
+func fetchConfig(ctx context.Context, repo, configDigest string, searchConf searchConfig,
+	username, password string,
+) (ispec.Image, error) {
+	configContent := ispec.Image{}

-	_, err = makeGETRequest(ctx, *job.config.servURL+"/v2/"+job.imageName+
-		"/manifests/"+cosignTag, job.username, job.password,
-		*job.config.verifyTLS, *job.config.debug, &job.manifestResp, job.config.resultWriter)
-	if err == nil {
-		isSigned = true
-	}
+	URL := fmt.Sprintf("%s/v2/%s/blobs/%s",
+		*searchConf.servURL, repo, configDigest)

-	var referrers ispec.Index
-
-	if !isSigned {
-		_, err = makeGETRequest(ctx, fmt.Sprintf("%s/v2/%s/referrers/%s?artifactType=%s",
-			*job.config.servURL, job.imageName, digestStr, notreg.ArtifactTypeNotation), job.username, job.password,
-			*job.config.verifyTLS, *job.config.debug, &referrers, job.config.resultWriter)
-		if err == nil {
-			for _, reference := range referrers.Manifests {
-				if reference.ArtifactType == notreg.ArtifactTypeNotation {
-					isSigned = true
-
-					break
-				}
-			}
-		}
-	}
-
-	size += uint64(manifestSize)
-
-	image := &imageStruct{}
-	image.verbose = *job.config.verbose
-	image.RepoName = job.imageName
-	image.Tag = job.tagName
-	image.Digest = digestStr
-	image.Size = strconv.Itoa(int(size))
-	image.ConfigDigest = configDigest
-	image.Layers = layers
-	image.IsSigned = isSigned
-
-	str, err := image.string(*job.config.outputFormat, len(job.imageName), len(job.tagName))
+	_, err := makeGETRequest(ctx, URL, username, password,
+		*searchConf.verifyTLS, *searchConf.debug, &configContent, searchConf.resultWriter)
 	if err != nil {
 		if isContextDone(ctx) {
-			return
+			return ispec.Image{}, context.Canceled
 		}
-		p.outputCh <- stringResult{"", err}

-		return
+		return ispec.Image{}, err
 	}

-	if isContextDone(ctx) {
-		return
-	}
-
-	p.outputCh <- stringResult{str, nil}
+	return configContent, nil
 }

-func (p *requestsPool) submitJob(job *manifestJob) {
+func isNotationSigned(ctx context.Context, repo, digestStr string, searchConf searchConfig,
+	username, password string,
+) bool {
+	var referrers ispec.Index
+
+	URL := fmt.Sprintf("%s/v2/%s/referrers/%s?artifactType=%s",
+		*searchConf.servURL, repo, digestStr, notreg.ArtifactTypeNotation)
+
+	_, err := makeGETRequest(ctx, URL, username, password,
+		*searchConf.verifyTLS, *searchConf.debug, &referrers, searchConf.resultWriter)
+	if err != nil {
+		return false
+	}
+
+	for _, reference := range referrers.Manifests {
+		if reference.ArtifactType == notreg.ArtifactTypeNotation {
+			return true
+		}
+	}
+
+	return false
+}
+
+func isCosignSigned(ctx context.Context, repo, digestStr string, searchConf searchConfig,
+	username, password string,
+) bool {
+	var result interface{}
+	cosignTag := strings.Replace(digestStr, ":", "-", 1) + "." + remote.SignatureTagSuffix
+
+	URL := fmt.Sprintf("%s/v2/%s/manifests/%s", *searchConf.servURL, repo, cosignTag)
+
+	_, err := makeGETRequest(ctx, URL, username, password, *searchConf.verifyTLS,
+		*searchConf.debug, &result, searchConf.resultWriter)
+
+	return err == nil
+}
+
+func (p *requestsPool) submitJob(job *httpJob) {
 	p.jobs <- job
 }