mirror of
https://github.com/project-zot/zot.git
synced 2026-06-18 05:28:07 +08:00
b7ab9dab16
fix: make sure metadb statistics are initialized on image download, and minor metadb fixes for Docker v2 manifest compatibility Looking into potential causes of https://github.com/project-zot/zot/issues/3163 1. One possible reason is the statistics were not properly initialized in the first place because of (unknown and/or unavoidable) errors on image push. To workaround this add logic to initialize the statistics on the call to download them. 2. Some images have the download statistics while others dont, one cause could be a bug in the logic handling manifest mediatypes in the search extension. Add compatibility checks for Docker v2 manifest types in metadb convert functions, and more tests for covering the Docker mediatype use case. Side fixes: - Ensure PushedBy Statistics entries are properly initialized in SetRepoReference - Fix and issue in the image upload test functions, they were uploading docker images with oci mediatypes in call headers Signed-off-by: Andrei Aaron <andreifdaaron@gmail.com>
303 lines
7.1 KiB
Go
303 lines
7.1 KiB
Go
package image
|
|
|
|
import (
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"net/http"
|
|
"strconv"
|
|
|
|
godigest "github.com/opencontainers/go-digest"
|
|
ispec "github.com/opencontainers/image-spec/specs-go/v1"
|
|
"gopkg.in/resty.v1"
|
|
|
|
tcommon "zotregistry.dev/zot/v2/pkg/test/common"
|
|
"zotregistry.dev/zot/v2/pkg/test/inject"
|
|
)
|
|
|
|
var (
|
|
ErrPostBlob = errors.New("can't post blob")
|
|
ErrPutBlob = errors.New("can't put blob")
|
|
ErrPutIndex = errors.New("can't put index")
|
|
)
|
|
|
|
func UploadImage(img Image, baseURL, repo, ref string) error {
|
|
if ref == "" {
|
|
ref = img.DigestStr()
|
|
}
|
|
|
|
digestAlgorithm := img.digestAlgorithm
|
|
|
|
if digestAlgorithm == "" {
|
|
digestAlgorithm = godigest.Canonical
|
|
}
|
|
|
|
for _, blob := range img.Layers {
|
|
resp, err := resty.R().Post(baseURL + "/v2/" + repo + "/blobs/uploads/")
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
if resp.StatusCode() != http.StatusAccepted {
|
|
return ErrPostBlob
|
|
}
|
|
|
|
loc := resp.Header().Get("Location")
|
|
|
|
digest := digestAlgorithm.FromBytes(blob).String()
|
|
|
|
resp, err = resty.R().
|
|
SetHeader("Content-Length", strconv.Itoa(len(blob))).
|
|
SetHeader("Content-Type", "application/octet-stream").
|
|
SetQueryParam("digest", digest).
|
|
SetBody(blob).
|
|
Put(baseURL + loc)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
if resp.StatusCode() != http.StatusCreated {
|
|
return ErrPutBlob
|
|
}
|
|
}
|
|
|
|
var err error
|
|
|
|
cblob := img.ConfigDescriptor.Data
|
|
|
|
// we'll remove this check once we make the full transition to the new way of generating test images
|
|
if len(cblob) == 0 {
|
|
cblob, err = json.Marshal(img.Config)
|
|
if err = inject.Error(err); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
|
|
cdigest := digestAlgorithm.FromBytes(cblob)
|
|
|
|
if img.Manifest.Config.MediaType == ispec.MediaTypeEmptyJSON ||
|
|
img.Manifest.Config.Digest == ispec.DescriptorEmptyJSON.Digest {
|
|
cblob = ispec.DescriptorEmptyJSON.Data
|
|
cdigest = ispec.DescriptorEmptyJSON.Digest
|
|
}
|
|
|
|
resp, err := resty.R().
|
|
Post(baseURL + "/v2/" + repo + "/blobs/uploads/")
|
|
if err = inject.Error(err); err != nil {
|
|
return err
|
|
}
|
|
|
|
if inject.ErrStatusCode(resp.StatusCode()) != http.StatusAccepted || inject.ErrStatusCode(resp.StatusCode()) == -1 {
|
|
return ErrPostBlob
|
|
}
|
|
|
|
loc := tcommon.Location(baseURL, resp)
|
|
|
|
// uploading blob should get 201
|
|
resp, err = resty.R().
|
|
SetHeader("Content-Length", strconv.Itoa(len(cblob))).
|
|
SetHeader("Content-Type", "application/octet-stream").
|
|
SetQueryParam("digest", cdigest.String()).
|
|
SetBody(cblob).
|
|
Put(loc)
|
|
if err = inject.Error(err); err != nil {
|
|
return err
|
|
}
|
|
|
|
if inject.ErrStatusCode(resp.StatusCode()) != http.StatusCreated || inject.ErrStatusCode(resp.StatusCode()) == -1 {
|
|
return ErrPostBlob
|
|
}
|
|
|
|
manifestBlob := img.ManifestDescriptor.Data
|
|
|
|
// we'll remove this check once we make the full transition to the new way of generating test images
|
|
if len(manifestBlob) == 0 {
|
|
manifestBlob, err = json.Marshal(img.Manifest)
|
|
if err = inject.Error(err); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
|
|
// Use the media type from ManifestDescriptor, or fall back to Manifest.MediaType, or default to OCI
|
|
mediaType := img.ManifestDescriptor.MediaType
|
|
|
|
if mediaType == "" {
|
|
mediaType = img.Manifest.MediaType
|
|
}
|
|
|
|
if mediaType == "" {
|
|
mediaType = ispec.MediaTypeImageManifest
|
|
}
|
|
|
|
resp, err = resty.R().
|
|
SetHeader("Content-type", mediaType).
|
|
SetBody(manifestBlob).
|
|
Put(baseURL + "/v2/" + repo + "/manifests/" + ref)
|
|
|
|
if inject.ErrStatusCode(resp.StatusCode()) != http.StatusCreated {
|
|
return ErrPutBlob
|
|
}
|
|
|
|
return err
|
|
}
|
|
|
|
func UploadImageWithBasicAuth(img Image, baseURL, repo, ref, user, password string) error {
|
|
digestAlgorithm := img.digestAlgorithm
|
|
|
|
if digestAlgorithm == "" {
|
|
digestAlgorithm = godigest.Canonical
|
|
}
|
|
|
|
for _, blob := range img.Layers {
|
|
resp, err := resty.R().
|
|
SetBasicAuth(user, password).
|
|
Post(baseURL + "/v2/" + repo + "/blobs/uploads/")
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
if resp.StatusCode() != http.StatusAccepted {
|
|
return ErrPostBlob
|
|
}
|
|
|
|
loc := resp.Header().Get("Location")
|
|
|
|
digest := digestAlgorithm.FromBytes(blob).String()
|
|
|
|
resp, err = resty.R().
|
|
SetBasicAuth(user, password).
|
|
SetHeader("Content-Length", strconv.Itoa(len(blob))).
|
|
SetHeader("Content-Type", "application/octet-stream").
|
|
SetQueryParam("digest", digest).
|
|
SetBody(blob).
|
|
Put(baseURL + loc)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
if resp.StatusCode() != http.StatusCreated {
|
|
return ErrPutBlob
|
|
}
|
|
}
|
|
// upload config
|
|
cblob, err := json.Marshal(img.Config)
|
|
if err = inject.Error(err); err != nil {
|
|
return err
|
|
}
|
|
|
|
cdigest := digestAlgorithm.FromBytes(cblob)
|
|
|
|
if img.Manifest.Config.MediaType == ispec.MediaTypeEmptyJSON {
|
|
cblob = ispec.DescriptorEmptyJSON.Data
|
|
cdigest = ispec.DescriptorEmptyJSON.Digest
|
|
}
|
|
|
|
resp, err := resty.R().
|
|
SetBasicAuth(user, password).
|
|
Post(baseURL + "/v2/" + repo + "/blobs/uploads/")
|
|
if err = inject.Error(err); err != nil {
|
|
return err
|
|
}
|
|
|
|
if inject.ErrStatusCode(resp.StatusCode()) != http.StatusAccepted || inject.ErrStatusCode(resp.StatusCode()) == -1 {
|
|
return ErrPostBlob
|
|
}
|
|
|
|
loc := tcommon.Location(baseURL, resp)
|
|
|
|
// uploading blob should get 201
|
|
resp, err = resty.R().
|
|
SetBasicAuth(user, password).
|
|
SetHeader("Content-Length", strconv.Itoa(len(cblob))).
|
|
SetHeader("Content-Type", "application/octet-stream").
|
|
SetQueryParam("digest", cdigest.String()).
|
|
SetBody(cblob).
|
|
Put(loc)
|
|
if err = inject.Error(err); err != nil {
|
|
return err
|
|
}
|
|
|
|
if inject.ErrStatusCode(resp.StatusCode()) != http.StatusCreated || inject.ErrStatusCode(resp.StatusCode()) == -1 {
|
|
return ErrPostBlob
|
|
}
|
|
|
|
// put manifest
|
|
manifestBlob, err := json.Marshal(img.Manifest)
|
|
if err = inject.Error(err); err != nil {
|
|
return err
|
|
}
|
|
|
|
// Use the media type from ManifestDescriptor, or fall back to Manifest.MediaType, or default to OCI
|
|
mediaType := img.ManifestDescriptor.MediaType
|
|
|
|
if mediaType == "" {
|
|
mediaType = img.Manifest.MediaType
|
|
}
|
|
|
|
if mediaType == "" {
|
|
mediaType = ispec.MediaTypeImageManifest
|
|
}
|
|
|
|
_, err = resty.R().
|
|
SetBasicAuth(user, password).
|
|
SetHeader("Content-type", mediaType).
|
|
SetBody(manifestBlob).
|
|
Put(baseURL + "/v2/" + repo + "/manifests/" + ref)
|
|
|
|
return err
|
|
}
|
|
|
|
func UploadMultiarchImage(multiImage MultiarchImage, baseURL string, repo, ref string) error {
|
|
for _, image := range multiImage.Images {
|
|
err := UploadImage(image, baseURL, repo, image.DigestStr())
|
|
if err != nil {
|
|
return err
|
|
}
|
|
}
|
|
|
|
// put manifest
|
|
indexBlob := multiImage.IndexDescriptor.Data
|
|
|
|
if len(indexBlob) == 0 {
|
|
var err error
|
|
|
|
indexBlob, err = json.Marshal(multiImage.Index)
|
|
if err = inject.Error(err); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
|
|
// Use the media type from IndexDescriptor, or fall back to Index.MediaType, or default to OCI
|
|
mediaType := multiImage.IndexDescriptor.MediaType
|
|
|
|
if mediaType == "" {
|
|
mediaType = multiImage.Index.MediaType
|
|
}
|
|
|
|
if mediaType == "" {
|
|
mediaType = ispec.MediaTypeImageIndex
|
|
}
|
|
|
|
resp, err := resty.R().
|
|
SetHeader("Content-type", mediaType).
|
|
SetBody(indexBlob).
|
|
Put(baseURL + "/v2/" + repo + "/manifests/" + ref)
|
|
|
|
if resp.StatusCode() != http.StatusCreated {
|
|
return ErrPutIndex
|
|
}
|
|
|
|
return err
|
|
}
|
|
|
|
func DeleteImage(repo, reference, baseURL string) (int, error) {
|
|
resp, err := resty.R().Delete(
|
|
fmt.Sprintf(baseURL+"/v2/%s/manifests/%s", repo, reference),
|
|
)
|
|
if err != nil {
|
|
return -1, err
|
|
}
|
|
|
|
return resp.StatusCode(), err
|
|
}
|