mirror of
https://github.com/project-zot/zot.git
synced 2026-06-17 12:58:02 +08:00
refactor(storage): refactoring storage (#1459)
Signed-off-by: Laurentiu Niculae <niculae.laurentiu1@gmail.com>
This commit is contained in:
LaurentiuNiculae
@@ -0,0 +1,786 @@
|
||||
package storage
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"path"
|
||||
"strings"
|
||||
|
||||
notreg "github.com/notaryproject/notation-go/registry"
|
||||
godigest "github.com/opencontainers/go-digest"
|
||||
imeta "github.com/opencontainers/image-spec/specs-go"
|
||||
ispec "github.com/opencontainers/image-spec/specs-go/v1"
|
||||
oras "github.com/oras-project/artifacts-spec/specs-go/v1"
|
||||
"github.com/rs/zerolog"
|
||||
"github.com/sigstore/cosign/v2/pkg/oci/remote"
|
||||
|
||||
zerr "zotregistry.io/zot/errors"
|
||||
zcommon "zotregistry.io/zot/pkg/common"
|
||||
"zotregistry.io/zot/pkg/scheduler"
|
||||
storageConstants "zotregistry.io/zot/pkg/storage/constants"
|
||||
storageTypes "zotregistry.io/zot/pkg/storage/types"
|
||||
)
|
||||
|
||||
func GetTagsByIndex(index ispec.Index) []string {
|
||||
tags := make([]string, 0)
|
||||
|
||||
for _, manifest := range index.Manifests {
|
||||
v, ok := manifest.Annotations[ispec.AnnotationRefName]
|
||||
if ok {
|
||||
tags = append(tags, v)
|
||||
}
|
||||
}
|
||||
|
||||
return tags
|
||||
}
|
||||
|
||||
func GetManifestDescByReference(index ispec.Index, reference string) (ispec.Descriptor, bool) {
|
||||
var manifestDesc ispec.Descriptor
|
||||
|
||||
for _, manifest := range index.Manifests {
|
||||
if reference == manifest.Digest.String() {
|
||||
return manifest, true
|
||||
}
|
||||
|
||||
v, ok := manifest.Annotations[ispec.AnnotationRefName]
|
||||
if ok && v == reference {
|
||||
return manifest, true
|
||||
}
|
||||
}
|
||||
|
||||
return manifestDesc, false
|
||||
}
|
||||
|
||||
func ValidateManifest(imgStore storageTypes.ImageStore, repo, reference, mediaType string, body []byte,
|
||||
log zerolog.Logger,
|
||||
) (godigest.Digest, error) {
|
||||
// validate the manifest
|
||||
if !IsSupportedMediaType(mediaType) {
|
||||
log.Debug().Interface("actual", mediaType).
|
||||
Msg("bad manifest media type")
|
||||
|
||||
return "", zerr.ErrBadManifest
|
||||
}
|
||||
|
||||
if len(body) == 0 {
|
||||
log.Debug().Int("len", len(body)).Msg("invalid body length")
|
||||
|
||||
return "", zerr.ErrBadManifest
|
||||
}
|
||||
|
||||
switch mediaType {
|
||||
case ispec.MediaTypeImageManifest:
|
||||
var manifest ispec.Manifest
|
||||
if err := json.Unmarshal(body, &manifest); err != nil {
|
||||
log.Error().Err(err).Msg("unable to unmarshal JSON")
|
||||
|
||||
return "", zerr.ErrBadManifest
|
||||
}
|
||||
|
||||
if manifest.Config.MediaType == ispec.MediaTypeImageConfig {
|
||||
digest, err := validateOCIManifest(imgStore, repo, reference, &manifest, log)
|
||||
if err != nil {
|
||||
log.Error().Err(err).Msg("invalid oci image manifest")
|
||||
|
||||
return digest, err
|
||||
}
|
||||
}
|
||||
|
||||
if manifest.Subject != nil {
|
||||
var m ispec.Descriptor
|
||||
if err := json.Unmarshal(body, &m); err != nil {
|
||||
log.Error().Err(err).Msg("unable to unmarshal JSON")
|
||||
|
||||
return "", zerr.ErrBadManifest
|
||||
}
|
||||
}
|
||||
case oras.MediaTypeArtifactManifest:
|
||||
var m oras.Descriptor
|
||||
if err := json.Unmarshal(body, &m); err != nil {
|
||||
log.Error().Err(err).Msg("unable to unmarshal JSON")
|
||||
|
||||
return "", zerr.ErrBadManifest
|
||||
}
|
||||
}
|
||||
|
||||
return "", nil
|
||||
}
|
||||
|
||||
func validateOCIManifest(imgStore storageTypes.ImageStore, repo, reference string, //nolint:unparam
|
||||
manifest *ispec.Manifest, log zerolog.Logger,
|
||||
) (godigest.Digest, error) {
|
||||
if manifest.SchemaVersion != storageConstants.SchemaVersion {
|
||||
log.Error().Int("SchemaVersion", manifest.SchemaVersion).Msg("invalid manifest")
|
||||
|
||||
return "", zerr.ErrBadManifest
|
||||
}
|
||||
|
||||
// validate image config
|
||||
config := manifest.Config
|
||||
|
||||
blobBuf, err := imgStore.GetBlobContent(repo, config.Digest)
|
||||
if err != nil {
|
||||
return config.Digest, zerr.ErrBlobNotFound
|
||||
}
|
||||
|
||||
var cspec ispec.Image
|
||||
|
||||
err = json.Unmarshal(blobBuf, &cspec)
|
||||
if err != nil {
|
||||
return "", zerr.ErrBadManifest
|
||||
}
|
||||
|
||||
// validate the layers
|
||||
for _, layer := range manifest.Layers {
|
||||
if IsNonDistributable(layer.MediaType) {
|
||||
log.Warn().Str("digest", layer.Digest.String()).Str("mediaType", layer.MediaType).Msg("not validating layer exists")
|
||||
|
||||
continue
|
||||
}
|
||||
|
||||
_, err := imgStore.GetBlobContent(repo, layer.Digest)
|
||||
if err != nil {
|
||||
return layer.Digest, zerr.ErrBlobNotFound
|
||||
}
|
||||
}
|
||||
|
||||
return "", nil
|
||||
}
|
||||
|
||||
func GetAndValidateRequestDigest(body []byte, digestStr string, log zerolog.Logger) (godigest.Digest, error) {
|
||||
bodyDigest := godigest.FromBytes(body)
|
||||
|
||||
d, err := godigest.Parse(digestStr)
|
||||
if err == nil {
|
||||
if d.String() != bodyDigest.String() {
|
||||
log.Error().Str("actual", bodyDigest.String()).Str("expected", d.String()).
|
||||
Msg("manifest digest is not valid")
|
||||
|
||||
return "", zerr.ErrBadManifest
|
||||
}
|
||||
}
|
||||
|
||||
return bodyDigest, err
|
||||
}
|
||||
|
||||
/*
|
||||
CheckIfIndexNeedsUpdate verifies if an index needs to be updated given a new manifest descriptor.
|
||||
|
||||
Returns whether or not index needs update, in the latter case it will also return the previous digest.
|
||||
*/
|
||||
func CheckIfIndexNeedsUpdate(index *ispec.Index, desc *ispec.Descriptor,
|
||||
log zerolog.Logger,
|
||||
) (bool, godigest.Digest, error) {
|
||||
var oldDgst godigest.Digest
|
||||
|
||||
var reference string
|
||||
|
||||
tag, ok := desc.Annotations[ispec.AnnotationRefName]
|
||||
if ok {
|
||||
reference = tag
|
||||
} else {
|
||||
reference = desc.Digest.String()
|
||||
}
|
||||
|
||||
updateIndex := true
|
||||
|
||||
for midx, manifest := range index.Manifests {
|
||||
manifest := manifest
|
||||
if reference == manifest.Digest.String() {
|
||||
// nothing changed, so don't update
|
||||
updateIndex = false
|
||||
|
||||
break
|
||||
}
|
||||
|
||||
v, ok := manifest.Annotations[ispec.AnnotationRefName]
|
||||
if ok && v == reference {
|
||||
if manifest.Digest.String() == desc.Digest.String() {
|
||||
// nothing changed, so don't update
|
||||
updateIndex = false
|
||||
|
||||
break
|
||||
}
|
||||
|
||||
// manifest contents have changed for the same tag,
|
||||
// so update index.json descriptor
|
||||
log.Info().
|
||||
Int64("old size", manifest.Size).
|
||||
Int64("new size", desc.Size).
|
||||
Str("old digest", manifest.Digest.String()).
|
||||
Str("new digest", desc.Digest.String()).
|
||||
Str("old mediaType", manifest.MediaType).
|
||||
Str("new mediaType", desc.MediaType).
|
||||
Msg("updating existing tag with new manifest contents")
|
||||
|
||||
// changing media-type is disallowed!
|
||||
if manifest.MediaType != desc.MediaType {
|
||||
err := zerr.ErrBadManifest
|
||||
log.Error().Err(err).
|
||||
Str("old mediaType", manifest.MediaType).
|
||||
Str("new mediaType", desc.MediaType).Msg("cannot change media-type")
|
||||
|
||||
return false, "", err
|
||||
}
|
||||
|
||||
oldDesc := *desc
|
||||
|
||||
desc = &manifest
|
||||
oldDgst = manifest.Digest
|
||||
desc.Size = oldDesc.Size
|
||||
desc.Digest = oldDesc.Digest
|
||||
|
||||
index.Manifests = append(index.Manifests[:midx], index.Manifests[midx+1:]...)
|
||||
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
return updateIndex, oldDgst, nil
|
||||
}
|
||||
|
||||
// GetIndex returns the contents of index.json.
|
||||
func GetIndex(imgStore storageTypes.ImageStore, repo string, log zerolog.Logger) (ispec.Index, error) {
|
||||
var index ispec.Index
|
||||
|
||||
buf, err := imgStore.GetIndexContent(repo)
|
||||
if err != nil {
|
||||
return index, err
|
||||
}
|
||||
|
||||
if err := json.Unmarshal(buf, &index); err != nil {
|
||||
log.Error().Err(err).Str("dir", path.Join(imgStore.RootDir(), repo)).Msg("invalid JSON")
|
||||
|
||||
return index, zerr.ErrRepoBadVersion
|
||||
}
|
||||
|
||||
return index, nil
|
||||
}
|
||||
|
||||
// GetImageIndex returns a multiarch type image.
|
||||
func GetImageIndex(imgStore storageTypes.ImageStore, repo string, digest godigest.Digest, log zerolog.Logger,
|
||||
) (ispec.Index, error) {
|
||||
var imageIndex ispec.Index
|
||||
|
||||
if err := digest.Validate(); err != nil {
|
||||
return imageIndex, err
|
||||
}
|
||||
|
||||
buf, err := imgStore.GetBlobContent(repo, digest)
|
||||
if err != nil {
|
||||
return imageIndex, err
|
||||
}
|
||||
|
||||
indexPath := path.Join(imgStore.RootDir(), repo, "blobs",
|
||||
digest.Algorithm().String(), digest.Encoded())
|
||||
|
||||
if err := json.Unmarshal(buf, &imageIndex); err != nil {
|
||||
log.Error().Err(err).Str("path", indexPath).Msg("invalid JSON")
|
||||
|
||||
return imageIndex, err
|
||||
}
|
||||
|
||||
return imageIndex, nil
|
||||
}
|
||||
|
||||
func GetImageManifest(imgStore storageTypes.ImageStore, repo string, digest godigest.Digest, log zerolog.Logger,
|
||||
) (ispec.Manifest, error) {
|
||||
var manifestContent ispec.Manifest
|
||||
|
||||
manifestBlob, err := imgStore.GetBlobContent(repo, digest)
|
||||
if err != nil {
|
||||
return manifestContent, err
|
||||
}
|
||||
|
||||
manifestPath := path.Join(imgStore.RootDir(), repo, "blobs",
|
||||
digest.Algorithm().String(), digest.Encoded())
|
||||
|
||||
if err := json.Unmarshal(manifestBlob, &manifestContent); err != nil {
|
||||
log.Error().Err(err).Str("path", manifestPath).Msg("invalid JSON")
|
||||
|
||||
return manifestContent, err
|
||||
}
|
||||
|
||||
return manifestContent, nil
|
||||
}
|
||||
|
||||
func RemoveManifestDescByReference(index *ispec.Index, reference string, detectCollisions bool,
|
||||
) (ispec.Descriptor, error) {
|
||||
var removedManifest ispec.Descriptor
|
||||
|
||||
var found bool
|
||||
|
||||
foundCount := 0
|
||||
|
||||
var outIndex ispec.Index
|
||||
|
||||
for _, manifest := range index.Manifests {
|
||||
tag, ok := manifest.Annotations[ispec.AnnotationRefName]
|
||||
if ok && tag == reference {
|
||||
removedManifest = manifest
|
||||
found = true
|
||||
foundCount++
|
||||
|
||||
continue
|
||||
} else if reference == manifest.Digest.String() {
|
||||
removedManifest = manifest
|
||||
found = true
|
||||
foundCount++
|
||||
|
||||
continue
|
||||
}
|
||||
|
||||
outIndex.Manifests = append(outIndex.Manifests, manifest)
|
||||
}
|
||||
|
||||
if foundCount > 1 && detectCollisions {
|
||||
return ispec.Descriptor{}, zerr.ErrManifestConflict
|
||||
} else if !found {
|
||||
return ispec.Descriptor{}, zerr.ErrManifestNotFound
|
||||
}
|
||||
|
||||
index.Manifests = outIndex.Manifests
|
||||
|
||||
return removedManifest, nil
|
||||
}
|
||||
|
||||
/*
|
||||
Unmarshal an image index and for all manifests in that
|
||||
index, ensure that they do not have a name or they are not in other
|
||||
manifest indexes else GC can never clean them.
|
||||
*/
|
||||
func UpdateIndexWithPrunedImageManifests(imgStore storageTypes.ImageStore, index *ispec.Index, repo string,
|
||||
desc ispec.Descriptor, oldDgst godigest.Digest, log zerolog.Logger,
|
||||
) error {
|
||||
if (desc.MediaType == ispec.MediaTypeImageIndex) && (oldDgst != "") {
|
||||
otherImgIndexes := []ispec.Descriptor{}
|
||||
|
||||
for _, manifest := range index.Manifests {
|
||||
if manifest.MediaType == ispec.MediaTypeImageIndex {
|
||||
otherImgIndexes = append(otherImgIndexes, manifest)
|
||||
}
|
||||
}
|
||||
|
||||
otherImgIndexes = append(otherImgIndexes, desc)
|
||||
|
||||
prunedManifests, err := PruneImageManifestsFromIndex(imgStore, repo, oldDgst, *index, otherImgIndexes, log)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
index.Manifests = prunedManifests
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
/*
|
||||
Before an image index manifest is pushed to a repo, its constituent manifests
|
||||
are pushed first, so when updating/removing this image index manifest, we also
|
||||
need to determine if there are other image index manifests which refer to the
|
||||
same constitutent manifests so that they can be garbage-collected correctly
|
||||
|
||||
PruneImageManifestsFromIndex is a helper routine to achieve this.
|
||||
*/
|
||||
func PruneImageManifestsFromIndex(imgStore storageTypes.ImageStore, repo string, digest godigest.Digest, //nolint:gocyclo,lll
|
||||
outIndex ispec.Index, otherImgIndexes []ispec.Descriptor, log zerolog.Logger,
|
||||
) ([]ispec.Descriptor, error) {
|
||||
dir := path.Join(imgStore.RootDir(), repo)
|
||||
|
||||
indexPath := path.Join(dir, "blobs", digest.Algorithm().String(), digest.Encoded())
|
||||
|
||||
buf, err := imgStore.GetBlobContent(repo, digest)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var imgIndex ispec.Index
|
||||
if err := json.Unmarshal(buf, &imgIndex); err != nil {
|
||||
log.Error().Err(err).Str("path", indexPath).Msg("invalid JSON")
|
||||
|
||||
return nil, err
|
||||
}
|
||||
|
||||
inUse := map[string]uint{}
|
||||
|
||||
for _, manifest := range imgIndex.Manifests {
|
||||
inUse[manifest.Digest.Encoded()]++
|
||||
}
|
||||
|
||||
for _, otherIndex := range otherImgIndexes {
|
||||
oindex, err := GetImageIndex(imgStore, repo, otherIndex.Digest, log)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
for _, omanifest := range oindex.Manifests {
|
||||
_, ok := inUse[omanifest.Digest.Encoded()]
|
||||
if ok {
|
||||
inUse[omanifest.Digest.Encoded()]++
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
prunedManifests := []ispec.Descriptor{}
|
||||
|
||||
// for all manifests in the index, skip those that either have a tag or
|
||||
// are used in other imgIndexes
|
||||
for _, outManifest := range outIndex.Manifests {
|
||||
if outManifest.MediaType != ispec.MediaTypeImageManifest {
|
||||
prunedManifests = append(prunedManifests, outManifest)
|
||||
|
||||
continue
|
||||
}
|
||||
|
||||
_, ok := outManifest.Annotations[ispec.AnnotationRefName]
|
||||
if ok {
|
||||
prunedManifests = append(prunedManifests, outManifest)
|
||||
|
||||
continue
|
||||
}
|
||||
|
||||
count, ok := inUse[outManifest.Digest.Encoded()]
|
||||
if !ok {
|
||||
prunedManifests = append(prunedManifests, outManifest)
|
||||
|
||||
continue
|
||||
}
|
||||
|
||||
if count != 1 {
|
||||
// this manifest is in use in other image indexes
|
||||
prunedManifests = append(prunedManifests, outManifest)
|
||||
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
return prunedManifests, nil
|
||||
}
|
||||
|
||||
func ApplyLinter(imgStore storageTypes.ImageStore, linter Lint, repo string, descriptor ispec.Descriptor,
|
||||
) (bool, error) {
|
||||
pass := true
|
||||
|
||||
// we'll skip anything that's not a image manifest
|
||||
if descriptor.MediaType != ispec.MediaTypeImageManifest {
|
||||
return pass, nil
|
||||
}
|
||||
|
||||
if linter != nil && !IsSignature(descriptor) {
|
||||
// lint new index with new manifest before writing to disk
|
||||
pass, err := linter.Lint(repo, descriptor.Digest, imgStore)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
|
||||
if !pass {
|
||||
return false, zerr.ErrImageLintAnnotations
|
||||
}
|
||||
}
|
||||
|
||||
return pass, nil
|
||||
}
|
||||
|
||||
func IsSignature(descriptor ispec.Descriptor) bool {
|
||||
tag := descriptor.Annotations[ispec.AnnotationRefName]
|
||||
|
||||
switch descriptor.MediaType {
|
||||
case ispec.MediaTypeImageManifest:
|
||||
// is cosgin signature
|
||||
if strings.HasPrefix(tag, "sha256-") && strings.HasSuffix(tag, remote.SignatureTagSuffix) {
|
||||
return true
|
||||
}
|
||||
|
||||
// is notation signature
|
||||
if descriptor.ArtifactType == notreg.ArtifactTypeNotation {
|
||||
return true
|
||||
}
|
||||
default:
|
||||
return false
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
func GetOrasReferrers(imgStore storageTypes.ImageStore, repo string, gdigest godigest.Digest, artifactType string,
|
||||
log zerolog.Logger,
|
||||
) ([]oras.Descriptor, error) {
|
||||
if err := gdigest.Validate(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
dir := path.Join(imgStore.RootDir(), repo)
|
||||
if !imgStore.DirExists(dir) {
|
||||
return nil, zerr.ErrRepoNotFound
|
||||
}
|
||||
|
||||
index, err := GetIndex(imgStore, repo, log)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
found := false
|
||||
|
||||
result := []oras.Descriptor{}
|
||||
|
||||
for _, manifest := range index.Manifests {
|
||||
if manifest.MediaType != oras.MediaTypeArtifactManifest {
|
||||
continue
|
||||
}
|
||||
|
||||
artManifest, err := GetOrasManifestByDigest(imgStore, repo, manifest.Digest, log)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if artManifest.Subject.Digest != gdigest {
|
||||
continue
|
||||
}
|
||||
|
||||
// filter by artifact type
|
||||
if artifactType != "" && artManifest.ArtifactType != artifactType {
|
||||
continue
|
||||
}
|
||||
|
||||
result = append(result, oras.Descriptor{
|
||||
MediaType: manifest.MediaType,
|
||||
ArtifactType: artManifest.ArtifactType,
|
||||
Digest: manifest.Digest,
|
||||
Size: manifest.Size,
|
||||
Annotations: manifest.Annotations,
|
||||
})
|
||||
|
||||
found = true
|
||||
}
|
||||
|
||||
if !found {
|
||||
return nil, zerr.ErrManifestNotFound
|
||||
}
|
||||
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func getReferrerFilterAnnotation(artifactTypes []string) string {
|
||||
// as per spec, return what filters were applied as an annotation if artifactTypes
|
||||
annotation := ""
|
||||
|
||||
for _, artifactType := range artifactTypes {
|
||||
if artifactType == "" {
|
||||
// ignore empty artifactTypes
|
||||
continue
|
||||
}
|
||||
|
||||
if annotation == "" {
|
||||
annotation = artifactType
|
||||
} else {
|
||||
annotation += "," + artifactType
|
||||
}
|
||||
}
|
||||
|
||||
return annotation
|
||||
}
|
||||
|
||||
func GetReferrers(imgStore storageTypes.ImageStore, repo string, gdigest godigest.Digest, artifactTypes []string,
|
||||
log zerolog.Logger,
|
||||
) (ispec.Index, error) {
|
||||
nilIndex := ispec.Index{}
|
||||
|
||||
if err := gdigest.Validate(); err != nil {
|
||||
return nilIndex, err
|
||||
}
|
||||
|
||||
dir := path.Join(imgStore.RootDir(), repo)
|
||||
if !imgStore.DirExists(dir) {
|
||||
return nilIndex, zerr.ErrRepoNotFound
|
||||
}
|
||||
|
||||
index, err := GetIndex(imgStore, repo, log)
|
||||
if err != nil {
|
||||
return nilIndex, err
|
||||
}
|
||||
|
||||
result := []ispec.Descriptor{}
|
||||
|
||||
for _, manifest := range index.Manifests {
|
||||
if manifest.Digest == gdigest {
|
||||
continue
|
||||
}
|
||||
|
||||
buf, err := imgStore.GetBlobContent(repo, manifest.Digest)
|
||||
if err != nil {
|
||||
log.Error().Err(err).Str("blob", imgStore.BlobPath(repo, manifest.Digest)).Msg("failed to read manifest")
|
||||
|
||||
if errors.Is(err, zerr.ErrBlobNotFound) {
|
||||
return nilIndex, zerr.ErrManifestNotFound
|
||||
}
|
||||
|
||||
return nilIndex, err
|
||||
}
|
||||
|
||||
if manifest.MediaType == ispec.MediaTypeImageManifest {
|
||||
var mfst ispec.Manifest
|
||||
if err := json.Unmarshal(buf, &mfst); err != nil {
|
||||
log.Error().Err(err).Str("manifest digest", manifest.Digest.String()).Msg("invalid JSON")
|
||||
|
||||
return nilIndex, err
|
||||
}
|
||||
|
||||
if mfst.Subject == nil || mfst.Subject.Digest != gdigest {
|
||||
continue
|
||||
}
|
||||
|
||||
// filter by artifact type
|
||||
manifestArtifactType := zcommon.GetManifestArtifactType(mfst)
|
||||
|
||||
if len(artifactTypes) > 0 && !zcommon.Contains(artifactTypes, manifestArtifactType) {
|
||||
continue
|
||||
}
|
||||
|
||||
result = append(result, ispec.Descriptor{
|
||||
MediaType: manifest.MediaType,
|
||||
ArtifactType: manifestArtifactType,
|
||||
Size: manifest.Size,
|
||||
Digest: manifest.Digest,
|
||||
Annotations: mfst.Annotations,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
index = ispec.Index{
|
||||
Versioned: imeta.Versioned{SchemaVersion: storageConstants.SchemaVersion},
|
||||
MediaType: ispec.MediaTypeImageIndex,
|
||||
Manifests: result,
|
||||
Annotations: map[string]string{},
|
||||
}
|
||||
|
||||
// as per spec, return what filters were applied as an annotation if artifactTypes
|
||||
if annotation := getReferrerFilterAnnotation(artifactTypes); annotation != "" {
|
||||
index.Annotations[storageConstants.ReferrerFilterAnnotation] = annotation
|
||||
log.Info().Str("annotation", annotation).Msg("filters applied")
|
||||
}
|
||||
|
||||
return index, nil
|
||||
}
|
||||
|
||||
func GetOrasManifestByDigest(imgStore storageTypes.ImageStore, repo string, digest godigest.Digest, log zerolog.Logger,
|
||||
) (oras.Manifest, error) {
|
||||
var artManifest oras.Manifest
|
||||
|
||||
blobPath := imgStore.BlobPath(repo, digest)
|
||||
|
||||
buf, err := imgStore.GetBlobContent(repo, digest)
|
||||
if err != nil {
|
||||
log.Error().Err(err).Str("blob", blobPath).Msg("failed to read manifest")
|
||||
|
||||
if errors.Is(err, zerr.ErrBlobNotFound) {
|
||||
return artManifest, zerr.ErrManifestNotFound
|
||||
}
|
||||
|
||||
return artManifest, err
|
||||
}
|
||||
|
||||
if err := json.Unmarshal(buf, &artManifest); err != nil {
|
||||
log.Error().Err(err).Str("blob", blobPath).Msg("invalid JSON")
|
||||
|
||||
return artManifest, err
|
||||
}
|
||||
|
||||
return artManifest, nil
|
||||
}
|
||||
|
||||
func IsSupportedMediaType(mediaType string) bool {
|
||||
return mediaType == ispec.MediaTypeImageIndex ||
|
||||
mediaType == ispec.MediaTypeImageManifest ||
|
||||
mediaType == oras.MediaTypeArtifactManifest
|
||||
}
|
||||
|
||||
func IsNonDistributable(mediaType string) bool {
|
||||
return mediaType == ispec.MediaTypeImageLayerNonDistributable || //nolint:staticcheck
|
||||
mediaType == ispec.MediaTypeImageLayerNonDistributableGzip || //nolint:staticcheck
|
||||
mediaType == ispec.MediaTypeImageLayerNonDistributableZstd //nolint:staticcheck
|
||||
}
|
||||
|
||||
/*
|
||||
DedupeTaskGenerator takes all blobs paths found in the storage.imagestore and groups them by digest
|
||||
|
||||
for each digest and based on the dedupe value it will dedupe or restore deduped blobs to the original state(undeduped)\
|
||||
by creating a task for each digest and pushing it to the task scheduler.
|
||||
*/
|
||||
type DedupeTaskGenerator struct {
|
||||
ImgStore storageTypes.ImageStore
|
||||
// storage dedupe value
|
||||
Dedupe bool
|
||||
// store blobs paths grouped by digest
|
||||
digest godigest.Digest
|
||||
duplicateBlobs []string
|
||||
/* store processed digest, used for iterating duplicateBlobs one by one
|
||||
and generating a task for each unprocessed one*/
|
||||
lastDigests []godigest.Digest
|
||||
done bool
|
||||
Log zerolog.Logger
|
||||
}
|
||||
|
||||
func (gen *DedupeTaskGenerator) GenerateTask() (scheduler.Task, error) {
|
||||
var err error
|
||||
|
||||
// get all blobs from storage.imageStore and group them by digest
|
||||
gen.digest, gen.duplicateBlobs, err = gen.ImgStore.GetNextDigestWithBlobPaths(gen.lastDigests)
|
||||
if err != nil {
|
||||
gen.Log.Error().Err(err).Msg("dedupe rebuild: failed to get next digest")
|
||||
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// if no digests left, then mark the task generator as done
|
||||
if gen.digest == "" {
|
||||
gen.Log.Info().Msg("dedupe rebuild: finished")
|
||||
|
||||
gen.done = true
|
||||
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
// mark digest as processed before running its task
|
||||
gen.lastDigests = append(gen.lastDigests, gen.digest)
|
||||
|
||||
// generate rebuild dedupe task for this digest
|
||||
return newDedupeTask(gen.ImgStore, gen.digest, gen.Dedupe, gen.duplicateBlobs, gen.Log), nil
|
||||
}
|
||||
|
||||
func (gen *DedupeTaskGenerator) IsDone() bool {
|
||||
return gen.done
|
||||
}
|
||||
|
||||
func (gen *DedupeTaskGenerator) Reset() {
|
||||
gen.lastDigests = []godigest.Digest{}
|
||||
gen.duplicateBlobs = []string{}
|
||||
gen.digest = ""
|
||||
gen.done = false
|
||||
}
|
||||
|
||||
type dedupeTask struct {
|
||||
imgStore storageTypes.ImageStore
|
||||
// digest of duplicateBLobs
|
||||
digest godigest.Digest
|
||||
// blobs paths with the same digest ^
|
||||
duplicateBlobs []string
|
||||
dedupe bool
|
||||
log zerolog.Logger
|
||||
}
|
||||
|
||||
func newDedupeTask(imgStore storageTypes.ImageStore, digest godigest.Digest, dedupe bool,
|
||||
duplicateBlobs []string, log zerolog.Logger,
|
||||
) *dedupeTask {
|
||||
return &dedupeTask{imgStore, digest, duplicateBlobs, dedupe, log}
|
||||
}
|
||||
|
||||
func (dt *dedupeTask) DoWork() error {
|
||||
// run task
|
||||
err := dt.imgStore.RunDedupeForDigest(dt.digest, dt.dedupe, dt.duplicateBlobs)
|
||||
if err != nil {
|
||||
// log it
|
||||
dt.log.Error().Err(err).Str("digest", dt.digest.String()).Msg("rebuild dedupe: failed to rebuild digest")
|
||||
}
|
||||
|
||||
return err
|
||||
}
|
||||
@@ -0,0 +1,384 @@
|
||||
package storage_test
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path"
|
||||
"testing"
|
||||
|
||||
godigest "github.com/opencontainers/go-digest"
|
||||
ispec "github.com/opencontainers/image-spec/specs-go/v1"
|
||||
artifactspec "github.com/oras-project/artifacts-spec/specs-go/v1"
|
||||
"github.com/rs/zerolog"
|
||||
. "github.com/smartystreets/goconvey/convey"
|
||||
|
||||
"zotregistry.io/zot/errors"
|
||||
"zotregistry.io/zot/pkg/extensions/monitoring"
|
||||
"zotregistry.io/zot/pkg/log"
|
||||
"zotregistry.io/zot/pkg/storage"
|
||||
"zotregistry.io/zot/pkg/storage/cache"
|
||||
common "zotregistry.io/zot/pkg/storage/common"
|
||||
storageConstants "zotregistry.io/zot/pkg/storage/constants"
|
||||
"zotregistry.io/zot/pkg/storage/local"
|
||||
"zotregistry.io/zot/pkg/test"
|
||||
"zotregistry.io/zot/pkg/test/mocks"
|
||||
)
|
||||
|
||||
func TestValidateManifest(t *testing.T) {
|
||||
Convey("Make manifest", t, func(c C) {
|
||||
dir := t.TempDir()
|
||||
|
||||
log := log.Logger{Logger: zerolog.New(os.Stdout)}
|
||||
metrics := monitoring.NewMetricsServer(false, log)
|
||||
cacheDriver, _ := storage.Create("boltdb", cache.BoltDBDriverParameters{
|
||||
RootDir: dir,
|
||||
Name: "cache",
|
||||
UseRelPaths: true,
|
||||
}, log)
|
||||
imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, true,
|
||||
true, log, metrics, nil, cacheDriver)
|
||||
|
||||
content := []byte("this is a blob")
|
||||
digest := godigest.FromBytes(content)
|
||||
So(digest, ShouldNotBeNil)
|
||||
|
||||
_, blen, err := imgStore.FullBlobUpload("test", bytes.NewReader(content), digest)
|
||||
So(err, ShouldBeNil)
|
||||
So(blen, ShouldEqual, len(content))
|
||||
|
||||
cblob, cdigest := test.GetRandomImageConfig()
|
||||
_, clen, err := imgStore.FullBlobUpload("test", bytes.NewReader(cblob), cdigest)
|
||||
So(err, ShouldBeNil)
|
||||
So(clen, ShouldEqual, len(cblob))
|
||||
|
||||
Convey("bad manifest schema version", func() {
|
||||
manifest := ispec.Manifest{
|
||||
Config: ispec.Descriptor{
|
||||
MediaType: ispec.MediaTypeImageConfig,
|
||||
Digest: cdigest,
|
||||
Size: int64(len(cblob)),
|
||||
},
|
||||
Layers: []ispec.Descriptor{
|
||||
{
|
||||
MediaType: ispec.MediaTypeImageLayer,
|
||||
Digest: digest,
|
||||
Size: int64(len(content)),
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
manifest.SchemaVersion = 999
|
||||
|
||||
body, err := json.Marshal(manifest)
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
_, _, err = imgStore.PutImageManifest("test", "1.0", ispec.MediaTypeImageManifest, body)
|
||||
So(err, ShouldNotBeNil)
|
||||
})
|
||||
|
||||
Convey("bad config blob", func() {
|
||||
manifest := ispec.Manifest{
|
||||
Config: ispec.Descriptor{
|
||||
MediaType: ispec.MediaTypeImageConfig,
|
||||
Digest: cdigest,
|
||||
Size: int64(len(cblob)),
|
||||
},
|
||||
Layers: []ispec.Descriptor{
|
||||
{
|
||||
MediaType: ispec.MediaTypeImageLayer,
|
||||
Digest: digest,
|
||||
Size: int64(len(content)),
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
manifest.SchemaVersion = 2
|
||||
|
||||
configBlobPath := imgStore.BlobPath("test", cdigest)
|
||||
|
||||
err := os.WriteFile(configBlobPath, []byte("bad config blob"), 0o000)
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
body, err := json.Marshal(manifest)
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
_, _, err = imgStore.PutImageManifest("test", "1.0", ispec.MediaTypeImageManifest, body)
|
||||
So(err, ShouldNotBeNil)
|
||||
})
|
||||
|
||||
Convey("manifest with non-distributable layers", func() {
|
||||
content := []byte("this blob doesn't exist")
|
||||
digest := godigest.FromBytes(content)
|
||||
So(digest, ShouldNotBeNil)
|
||||
|
||||
manifest := ispec.Manifest{
|
||||
Config: ispec.Descriptor{
|
||||
MediaType: ispec.MediaTypeImageConfig,
|
||||
Digest: cdigest,
|
||||
Size: int64(len(cblob)),
|
||||
},
|
||||
Layers: []ispec.Descriptor{
|
||||
{
|
||||
MediaType: ispec.MediaTypeImageLayerNonDistributable, //nolint:staticcheck
|
||||
Digest: digest,
|
||||
Size: int64(len(content)),
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
manifest.SchemaVersion = 2
|
||||
|
||||
body, err := json.Marshal(manifest)
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
_, _, err = imgStore.PutImageManifest("test", "1.0", ispec.MediaTypeImageManifest, body)
|
||||
So(err, ShouldBeNil)
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
func TestGetReferrersErrors(t *testing.T) {
|
||||
Convey("make storage", t, func(c C) {
|
||||
dir := t.TempDir()
|
||||
|
||||
log := log.Logger{Logger: zerolog.New(os.Stdout)}
|
||||
metrics := monitoring.NewMetricsServer(false, log)
|
||||
cacheDriver, _ := storage.Create("boltdb", cache.BoltDBDriverParameters{
|
||||
RootDir: dir,
|
||||
Name: "cache",
|
||||
UseRelPaths: true,
|
||||
}, log)
|
||||
|
||||
imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, false,
|
||||
true, log, metrics, nil, cacheDriver)
|
||||
|
||||
artifactType := "application/vnd.example.icecream.v1"
|
||||
validDigest := godigest.FromBytes([]byte("blob"))
|
||||
|
||||
Convey("Trigger invalid digest error", func(c C) {
|
||||
_, err := common.GetReferrers(imgStore, "zot-test", "invalidDigest",
|
||||
[]string{artifactType}, log.With().Caller().Logger())
|
||||
So(err, ShouldNotBeNil)
|
||||
|
||||
_, err = common.GetOrasReferrers(imgStore, "zot-test", "invalidDigest",
|
||||
artifactType, log.With().Caller().Logger())
|
||||
So(err, ShouldNotBeNil)
|
||||
})
|
||||
|
||||
Convey("Trigger repo not found error", func(c C) {
|
||||
_, err := common.GetReferrers(imgStore, "zot-test", validDigest,
|
||||
[]string{artifactType}, log.With().Caller().Logger())
|
||||
So(err, ShouldNotBeNil)
|
||||
|
||||
_, err = common.GetOrasReferrers(imgStore, "zot-test", validDigest,
|
||||
artifactType, log.With().Caller().Logger())
|
||||
So(err, ShouldNotBeNil)
|
||||
})
|
||||
|
||||
err := test.CopyFiles("../../../test/data/zot-test", path.Join(dir, "zot-test"))
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
digest := godigest.FromBytes([]byte("{}"))
|
||||
|
||||
index := ispec.Index{
|
||||
Manifests: []ispec.Descriptor{
|
||||
{
|
||||
MediaType: artifactspec.MediaTypeArtifactManifest,
|
||||
Digest: digest,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
indexBuf, err := json.Marshal(index)
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
Convey("Trigger GetBlobContent() not found", func(c C) {
|
||||
imgStore = &mocks.MockedImageStore{
|
||||
GetIndexContentFn: func(repo string) ([]byte, error) {
|
||||
return indexBuf, nil
|
||||
},
|
||||
GetBlobContentFn: func(repo string, digest godigest.Digest) ([]byte, error) {
|
||||
return []byte{}, errors.ErrBlobNotFound
|
||||
},
|
||||
}
|
||||
|
||||
_, err = common.GetReferrers(imgStore, "zot-test", validDigest,
|
||||
[]string{artifactType}, log.With().Caller().Logger())
|
||||
So(err, ShouldNotBeNil)
|
||||
|
||||
_, err = common.GetOrasReferrers(imgStore, "zot-test", validDigest,
|
||||
artifactType, log.With().Caller().Logger())
|
||||
So(err, ShouldNotBeNil)
|
||||
})
|
||||
|
||||
Convey("Trigger GetBlobContent() generic error", func(c C) {
|
||||
imgStore = &mocks.MockedImageStore{
|
||||
GetIndexContentFn: func(repo string) ([]byte, error) {
|
||||
return indexBuf, nil
|
||||
},
|
||||
GetBlobContentFn: func(repo string, digest godigest.Digest) ([]byte, error) {
|
||||
return []byte{}, errors.ErrBadBlob
|
||||
},
|
||||
}
|
||||
|
||||
_, err = common.GetReferrers(imgStore, "zot-test", validDigest,
|
||||
[]string{artifactType}, log.With().Caller().Logger())
|
||||
So(err, ShouldNotBeNil)
|
||||
|
||||
_, err = common.GetOrasReferrers(imgStore, "zot-test", validDigest,
|
||||
artifactType, log.With().Caller().Logger())
|
||||
So(err, ShouldNotBeNil)
|
||||
})
|
||||
|
||||
Convey("Trigger continue on different artifactType", func(c C) {
|
||||
orasManifest := artifactspec.Manifest{
|
||||
Subject: &artifactspec.Descriptor{
|
||||
Digest: digest,
|
||||
ArtifactType: "unknown",
|
||||
},
|
||||
}
|
||||
|
||||
orasBuf, err := json.Marshal(orasManifest)
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
imgStore = &mocks.MockedImageStore{
|
||||
GetIndexContentFn: func(repo string) ([]byte, error) {
|
||||
return indexBuf, nil
|
||||
},
|
||||
GetBlobContentFn: func(repo string, digest godigest.Digest) ([]byte, error) {
|
||||
return orasBuf, nil
|
||||
},
|
||||
}
|
||||
|
||||
_, err = common.GetOrasReferrers(imgStore, "zot-test", validDigest,
|
||||
artifactType, log.With().Caller().Logger())
|
||||
So(err, ShouldNotBeNil)
|
||||
|
||||
_, err = common.GetOrasReferrers(imgStore, "zot-test", digest,
|
||||
artifactType, log.With().Caller().Logger())
|
||||
So(err, ShouldNotBeNil)
|
||||
})
|
||||
|
||||
Convey("Unmarshal oras artifact error", func(c C) {
|
||||
imgStore = &mocks.MockedImageStore{
|
||||
GetIndexContentFn: func(repo string) ([]byte, error) {
|
||||
return indexBuf, nil
|
||||
},
|
||||
GetBlobContentFn: func(repo string, digest godigest.Digest) ([]byte, error) {
|
||||
return []byte("wrong content"), nil
|
||||
},
|
||||
}
|
||||
|
||||
_, err = common.GetOrasReferrers(imgStore, "zot-test", validDigest, artifactType, log.With().Caller().Logger())
|
||||
So(err, ShouldNotBeNil)
|
||||
})
|
||||
|
||||
Convey("Trigger unmarshal error on manifest image mediaType", func(c C) {
|
||||
index = ispec.Index{
|
||||
Manifests: []ispec.Descriptor{
|
||||
{
|
||||
MediaType: ispec.MediaTypeImageManifest,
|
||||
Digest: digest,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
indexBuf, err = json.Marshal(index)
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
imgStore = &mocks.MockedImageStore{
|
||||
GetIndexContentFn: func(repo string) ([]byte, error) {
|
||||
return indexBuf, nil
|
||||
},
|
||||
GetBlobContentFn: func(repo string, digest godigest.Digest) ([]byte, error) {
|
||||
return []byte{}, nil
|
||||
},
|
||||
}
|
||||
|
||||
_, err = common.GetReferrers(imgStore, "zot-test", validDigest,
|
||||
[]string{artifactType}, log.With().Caller().Logger())
|
||||
So(err, ShouldNotBeNil)
|
||||
})
|
||||
|
||||
Convey("Trigger nil subject", func(c C) {
|
||||
index = ispec.Index{
|
||||
Manifests: []ispec.Descriptor{
|
||||
{
|
||||
MediaType: ispec.MediaTypeImageManifest,
|
||||
Digest: digest,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
indexBuf, err = json.Marshal(index)
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
ociManifest := ispec.Manifest{
|
||||
Subject: nil,
|
||||
}
|
||||
|
||||
ociManifestBuf, err := json.Marshal(ociManifest)
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
imgStore = &mocks.MockedImageStore{
|
||||
GetIndexContentFn: func(repo string) ([]byte, error) {
|
||||
return indexBuf, nil
|
||||
},
|
||||
GetBlobContentFn: func(repo string, digest godigest.Digest) ([]byte, error) {
|
||||
return ociManifestBuf, nil
|
||||
},
|
||||
}
|
||||
|
||||
_, err = common.GetReferrers(imgStore, "zot-test", validDigest,
|
||||
[]string{artifactType}, log.With().Caller().Logger())
|
||||
So(err, ShouldBeNil)
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
func TestGetImageIndexErrors(t *testing.T) {
|
||||
log := zerolog.New(os.Stdout)
|
||||
|
||||
Convey("Trigger invalid digest error", t, func(c C) {
|
||||
imgStore := &mocks.MockedImageStore{}
|
||||
|
||||
_, err := common.GetImageIndex(imgStore, "zot-test", "invalidDigest", log)
|
||||
So(err, ShouldNotBeNil)
|
||||
})
|
||||
|
||||
Convey("Trigger GetBlobContent error", t, func(c C) {
|
||||
imgStore := &mocks.MockedImageStore{
|
||||
GetBlobContentFn: func(repo string, digest godigest.Digest) ([]byte, error) {
|
||||
return []byte{}, errors.ErrBlobNotFound
|
||||
},
|
||||
}
|
||||
|
||||
validDigest := godigest.FromBytes([]byte("blob"))
|
||||
|
||||
_, err := common.GetImageIndex(imgStore, "zot-test", validDigest, log)
|
||||
So(err, ShouldNotBeNil)
|
||||
})
|
||||
|
||||
Convey("Trigger unmarshal error", t, func(c C) {
|
||||
imgStore := &mocks.MockedImageStore{
|
||||
GetBlobContentFn: func(repo string, digest godigest.Digest) ([]byte, error) {
|
||||
return []byte{}, nil
|
||||
},
|
||||
}
|
||||
|
||||
validDigest := godigest.FromBytes([]byte("blob"))
|
||||
|
||||
_, err := common.GetImageIndex(imgStore, "zot-test", validDigest, log)
|
||||
So(err, ShouldNotBeNil)
|
||||
})
|
||||
}
|
||||
|
||||
func TestIsSignature(t *testing.T) {
|
||||
Convey("Unknown media type", t, func(c C) {
|
||||
isSingature := common.IsSignature(ispec.Descriptor{
|
||||
MediaType: "unknown media type",
|
||||
})
|
||||
So(isSingature, ShouldBeFalse)
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
package storage
|
||||
|
||||
import (
|
||||
godigest "github.com/opencontainers/go-digest"
|
||||
|
||||
storageTypes "zotregistry.io/zot/pkg/storage/types"
|
||||
)
|
||||
|
||||
type Lint interface {
|
||||
Lint(repo string, manifestDigest godigest.Digest, imageStore storageTypes.ImageStore) (bool, error)
|
||||
}
|
||||
Reference in New Issue
Block a user