github/zot
1
0
Fork 0
mirror of https://github.com/project-zot/zot.git synced 2026-06-17 04:48:26 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(scheduler): pass the shutdown/reload ctx to running tasks (#1671)

Browse Source 
Signed-off-by: Petu Eusebiu <peusebiu@cisco.com>
This commit is contained in:
peusebiu
2023-09-05 19:48:56 +03:00
committed by GitHub
parent a0290b4b37
commit 59dc4c3229
28 changed files with 190 additions and 143 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/extensions/sync/references/cosign.go
+11 -8
View File
@@ -4,6 +4,7 @@
package references
import (
"context"
"errors"
"fmt"
"net/http"
@@ -45,9 +46,9 @@ func (ref CosignReference) Name() string {
return constants.Cosign
}
func (ref CosignReference) IsSigned(upstreamRepo, subjectDigestStr string) bool {
func (ref CosignReference) IsSigned(ctx context.Context, upstreamRepo, subjectDigestStr string) bool {
cosignSignatureTag := getCosignSignatureTagFromSubjectDigest(subjectDigestStr)
_, _, err := ref.getManifest(upstreamRepo, cosignSignatureTag)
_, _, err := ref.getManifest(ctx, upstreamRepo, cosignSignatureTag)
return err == nil
}
@@ -85,13 +86,15 @@ func (ref CosignReference) canSkipReferences(localRepo, digest string, manifest
return true, nil
}
func (ref CosignReference) SyncReferences(localRepo, remoteRepo, subjectDigestStr string) ([]godigest.Digest, error) {
func (ref CosignReference) SyncReferences(ctx context.Context, localRepo, remoteRepo, subjectDigestStr string) (
[]godigest.Digest, error,
) {
cosignTags := getCosignTagsFromSubjectDigest(subjectDigestStr)
refsDigests := make([]godigest.Digest, 0, len(cosignTags))
for _, cosignTag := range cosignTags {
manifest, manifestBuf, err := ref.getManifest(remoteRepo, cosignTag)
manifest, manifestBuf, err := ref.getManifest(ctx, remoteRepo, cosignTag)
if err != nil {
if errors.Is(err, zerr.ErrSyncReferrerNotFound) {
continue
@@ -120,13 +123,13 @@ func (ref CosignReference) SyncReferences(localRepo, remoteRepo, subjectDigestSt
Msg("syncing cosign reference for image")
for _, blob := range manifest.Layers {
if err := syncBlob(ref.client, imageStore, localRepo, remoteRepo, blob.Digest, ref.log); err != nil {
if err := syncBlob(ctx, ref.client, imageStore, localRepo, remoteRepo, blob.Digest, ref.log); err != nil {
return refsDigests, err
}
}
// sync config blob
if err := syncBlob(ref.client, imageStore, localRepo, remoteRepo, manifest.Config.Digest, ref.log); err != nil {
if err := syncBlob(ctx, ref.client, imageStore, localRepo, remoteRepo, manifest.Config.Digest, ref.log); err != nil {
return refsDigests, err
}
@@ -181,10 +184,10 @@ func (ref CosignReference) SyncReferences(localRepo, remoteRepo, subjectDigestSt
return refsDigests, nil
}
func (ref CosignReference) getManifest(repo, cosignTag string) (*ispec.Manifest, []byte, error) {
func (ref CosignReference) getManifest(ctx context.Context, repo, cosignTag string) (*ispec.Manifest, []byte, error) {
var cosignManifest ispec.Manifest
body, _, statusCode, err := ref.client.MakeGetRequest(&cosignManifest, ispec.MediaTypeImageManifest,
body, _, statusCode, err := ref.client.MakeGetRequest(ctx, &cosignManifest, ispec.MediaTypeImageManifest,
"v2", repo, "manifests", cosignTag)
if err != nil {
if statusCode == http.StatusNotFound {
pkg/extensions/sync/references/oci.go
+15 -12
View File
@@ -4,6 +4,7 @@
package references
import (
"context"
"encoding/json"
"errors"
"fmt"
@@ -45,9 +46,9 @@ func (ref OciReferences) Name() string {
return constants.OCI
}
func (ref OciReferences) IsSigned(remoteRepo, subjectDigestStr string) bool {
func (ref OciReferences) IsSigned(ctx context.Context, remoteRepo, subjectDigestStr string) bool {
// use artifactTypeFilter
index, err := ref.getIndex(remoteRepo, subjectDigestStr)
index, err := ref.getIndex(ctx, remoteRepo, subjectDigestStr)
if err != nil {
return false
}
@@ -92,10 +93,12 @@ func (ref OciReferences) canSkipReferences(localRepo, subjectDigestStr string, i
return true, nil
}
func (ref OciReferences) SyncReferences(localRepo, remoteRepo, subjectDigestStr string) ([]godigest.Digest, error) {
func (ref OciReferences) SyncReferences(ctx context.Context, localRepo, remoteRepo, subjectDigestStr string) (
[]godigest.Digest, error,
) {
refsDigests := make([]godigest.Digest, 0, 10)
index, err := ref.getIndex(remoteRepo, subjectDigestStr)
index, err := ref.getIndex(ctx, remoteRepo, subjectDigestStr)
if err != nil {
return refsDigests, err
}
@@ -122,7 +125,7 @@ func (ref OciReferences) SyncReferences(localRepo, remoteRepo, subjectDigestStr
Msg("syncing oci references for image")
for _, referrer := range index.Manifests {
referenceBuf, referenceDigest, err := syncManifest(ref.client, imageStore, localRepo, remoteRepo,
referenceBuf, referenceDigest, err := syncManifest(ctx, ref.client, imageStore, localRepo, remoteRepo,
referrer, subjectDigestStr, ref.log)
if err != nil {
return refsDigests, err
@@ -168,10 +171,10 @@ func (ref OciReferences) SyncReferences(localRepo, remoteRepo, subjectDigestStr
return refsDigests, nil
}
func (ref OciReferences) getIndex(repo, subjectDigestStr string) (ispec.Index, error) {
func (ref OciReferences) getIndex(ctx context.Context, repo, subjectDigestStr string) (ispec.Index, error) {
var index ispec.Index
_, _, statusCode, err := ref.client.MakeGetRequest(&index, ispec.MediaTypeImageIndex,
_, _, statusCode, err := ref.client.MakeGetRequest(ctx, &index, ispec.MediaTypeImageIndex,
"v2", repo, "referrers", subjectDigestStr)
if err != nil {
if statusCode == http.StatusNotFound {
@@ -191,14 +194,14 @@ func (ref OciReferences) getIndex(repo, subjectDigestStr string) (ispec.Index, e
return index, nil
}
func syncManifest(client *client.Client, imageStore storageTypes.ImageStore, localRepo, remoteRepo string,
desc ispec.Descriptor, subjectDigestStr string, log log.Logger,
func syncManifest(ctx context.Context, client *client.Client, imageStore storageTypes.ImageStore, localRepo,
remoteRepo string, desc ispec.Descriptor, subjectDigestStr string, log log.Logger,
) ([]byte, godigest.Digest, error) {
var manifest ispec.Manifest
var refDigest godigest.Digest
OCIRefBuf, _, statusCode, err := client.MakeGetRequest(&manifest, ispec.MediaTypeImageManifest,
OCIRefBuf, _, statusCode, err := client.MakeGetRequest(ctx, &manifest, ispec.MediaTypeImageManifest,
"v2", remoteRepo, "manifests", desc.Digest.String())
if err != nil {
if statusCode == http.StatusNotFound {
@@ -226,13 +229,13 @@ func syncManifest(client *client.Client, imageStore storageTypes.ImageStore, loc
}
for _, layer := range manifest.Layers {
if err := syncBlob(client, imageStore, localRepo, remoteRepo, layer.Digest, log); err != nil {
if err := syncBlob(ctx, client, imageStore, localRepo, remoteRepo, layer.Digest, log); err != nil {
return []byte{}, refDigest, err
}
}
// sync config blob
if err := syncBlob(client, imageStore, localRepo, remoteRepo, manifest.Config.Digest, log); err != nil {
if err := syncBlob(ctx, client, imageStore, localRepo, remoteRepo, manifest.Config.Digest, log); err != nil {
return []byte{}, refDigest, err
}
} else {
pkg/extensions/sync/references/oras.go
+10 -7
View File
@@ -4,6 +4,7 @@
package references
import (
"context"
"errors"
"fmt"
"net/http"
@@ -48,7 +49,7 @@ func (ref ORASReferences) Name() string {
return constants.Oras
}
func (ref ORASReferences) IsSigned(remoteRepo, subjectDigestStr string) bool {
func (ref ORASReferences) IsSigned(ctx context.Context, remoteRepo, subjectDigestStr string) bool {
return false
}
@@ -85,10 +86,12 @@ func (ref ORASReferences) canSkipReferences(localRepo, subjectDigestStr string,
return true, nil
}
func (ref ORASReferences) SyncReferences(localRepo, remoteRepo, subjectDigestStr string) ([]godigest.Digest, error) {
func (ref ORASReferences) SyncReferences(ctx context.Context, localRepo, remoteRepo, subjectDigestStr string) (
[]godigest.Digest, error,
) {
refsDigests := make([]godigest.Digest, 0, 10)
referrers, err := ref.getReferenceList(remoteRepo, subjectDigestStr)
referrers, err := ref.getReferenceList(ctx, remoteRepo, subjectDigestStr)
if err != nil {
return refsDigests, err
}
@@ -115,7 +118,7 @@ func (ref ORASReferences) SyncReferences(localRepo, remoteRepo, subjectDigestStr
for _, referrer := range referrers.References {
var artifactManifest oras.Manifest
orasBuf, _, statusCode, err := ref.client.MakeGetRequest(&artifactManifest, oras.MediaTypeDescriptor,
orasBuf, _, statusCode, err := ref.client.MakeGetRequest(ctx, &artifactManifest, oras.MediaTypeDescriptor,
"v2", remoteRepo, "manifests", referrer.Digest.String())
if err != nil {
if statusCode == http.StatusNotFound {
@@ -130,7 +133,7 @@ func (ref ORASReferences) SyncReferences(localRepo, remoteRepo, subjectDigestStr
}
for _, blob := range artifactManifest.Blobs {
if err := syncBlob(ref.client, imageStore, localRepo, remoteRepo, blob.Digest, ref.log); err != nil {
if err := syncBlob(ctx, ref.client, imageStore, localRepo, remoteRepo, blob.Digest, ref.log); err != nil {
return refsDigests, err
}
}
@@ -170,10 +173,10 @@ func (ref ORASReferences) SyncReferences(localRepo, remoteRepo, subjectDigestStr
return refsDigests, nil
}
func (ref ORASReferences) getReferenceList(repo, subjectDigestStr string) (ReferenceList, error) {
func (ref ORASReferences) getReferenceList(ctx context.Context, repo, subjectDigestStr string) (ReferenceList, error) {
var referrers ReferenceList
_, _, statusCode, err := ref.client.MakeGetRequest(&referrers, "application/json",
_, _, statusCode, err := ref.client.MakeGetRequest(ctx, &referrers, "application/json",
apiConstants.ArtifactSpecRoutePrefix, repo, "manifests", subjectDigestStr, "referrers")
if err != nil {
if statusCode == http.StatusNotFound || statusCode == http.StatusBadRequest {
pkg/extensions/sync/references/references.go
+20 -15
View File
@@ -5,6 +5,7 @@ package references
import (
"bytes"
"context"
"fmt"
"net/http"
@@ -26,9 +27,9 @@ type Reference interface {
// Returns name of reference (OCIReference/CosignReference/OrasReference)
Name() string
// Returns whether or not image is signed
IsSigned(upstreamRepo, subjectDigestStr string) bool
IsSigned(ctx context.Context, upstreamRepo, subjectDigestStr string) bool
// Sync recursively all references for a subject digest (can be image/artifacts/signatures)
SyncReferences(localRepo, upstreamRepo, subjectDigestStr string) ([]godigest.Digest, error)
SyncReferences(ctx context.Context, localRepo, upstreamRepo, subjectDigestStr string) ([]godigest.Digest, error)
}
type References struct {
@@ -48,9 +49,9 @@ func NewReferences(httpClient *client.Client, storeController storage.StoreContr
return refs
}
func (refs References) IsSigned(upstreamRepo, subjectDigestStr string) bool {
func (refs References) IsSigned(ctx context.Context, upstreamRepo, subjectDigestStr string) bool {
for _, ref := range refs.referenceList {
ok := ref.IsSigned(upstreamRepo, subjectDigestStr)
ok := ref.IsSigned(ctx, upstreamRepo, subjectDigestStr)
if ok {
return true
}
@@ -59,13 +60,15 @@ func (refs References) IsSigned(upstreamRepo, subjectDigestStr string) bool {
return false
}
func (refs References) SyncAll(localRepo, upstreamRepo, subjectDigestStr string) error {
func (refs References) SyncAll(ctx context.Context, localRepo, upstreamRepo, subjectDigestStr string) error {
seen := &[]godigest.Digest{}
return refs.syncAll(localRepo, upstreamRepo, subjectDigestStr, seen)
return refs.syncAll(ctx, localRepo, upstreamRepo, subjectDigestStr, seen)
}
func (refs References) syncAll(localRepo, upstreamRepo, subjectDigestStr string, seen *[]godigest.Digest) error {
func (refs References) syncAll(ctx context.Context, localRepo, upstreamRepo,
subjectDigestStr string, seen *[]godigest.Digest,
) error {
var err error
var syncedRefsDigests []godigest.Digest
@@ -75,7 +78,7 @@ func (refs References) syncAll(localRepo, upstreamRepo, subjectDigestStr string,
// for each reference type(cosign/oci/oras reference)
for _, ref := range refs.referenceList {
syncedRefsDigests, err = ref.SyncReferences(localRepo, upstreamRepo, subjectDigestStr)
syncedRefsDigests, err = ref.SyncReferences(ctx, localRepo, upstreamRepo, subjectDigestStr)
if err != nil {
refs.log.Debug().Err(err).
Str("reference type", ref.Name()).
@@ -87,7 +90,7 @@ func (refs References) syncAll(localRepo, upstreamRepo, subjectDigestStr string,
for _, refDigest := range syncedRefsDigests {
if !common.Contains(*seen, refDigest) {
// sync all references pointing to this one
err = refs.syncAll(localRepo, upstreamRepo, refDigest.String(), seen)
err = refs.syncAll(ctx, localRepo, upstreamRepo, refDigest.String(), seen)
}
}
}
@@ -95,14 +98,16 @@ func (refs References) syncAll(localRepo, upstreamRepo, subjectDigestStr string,
return err
}
func (refs References) SyncReference(localRepo, upstreamRepo, subjectDigestStr, referenceType string) error {
func (refs References) SyncReference(ctx context.Context, localRepo, upstreamRepo,
subjectDigestStr, referenceType string,
) error {
var err error
var syncedRefsDigests []godigest.Digest
for _, ref := range refs.referenceList {
if ref.Name() == referenceType {
syncedRefsDigests, err = ref.SyncReferences(localRepo, upstreamRepo, subjectDigestStr)
syncedRefsDigests, err = ref.SyncReferences(ctx, localRepo, upstreamRepo, subjectDigestStr)
if err != nil {
refs.log.Error().Err(err).
Str("reference type", ref.Name()).
@@ -113,7 +118,7 @@ func (refs References) SyncReference(localRepo, upstreamRepo, subjectDigestStr,
}
for _, refDigest := range syncedRefsDigests {
err = refs.SyncAll(localRepo, upstreamRepo, refDigest.String())
err = refs.SyncAll(ctx, localRepo, upstreamRepo, refDigest.String())
}
}
}
@@ -121,12 +126,12 @@ func (refs References) SyncReference(localRepo, upstreamRepo, subjectDigestStr,
return err
}
func syncBlob(client *client.Client, imageStore storageTypes.ImageStore, localRepo, remoteRepo string,
digest godigest.Digest, log log.Logger,
func syncBlob(ctx context.Context, client *client.Client, imageStore storageTypes.ImageStore,
localRepo, remoteRepo string, digest godigest.Digest, log log.Logger,
) error {
var resultPtr interface{}
body, _, statusCode, err := client.MakeGetRequest(resultPtr, "", "v2", remoteRepo, "blobs", digest.String())
body, _, statusCode, err := client.MakeGetRequest(ctx, resultPtr, "", "v2", remoteRepo, "blobs", digest.String())
if err != nil {
if statusCode != http.StatusOK {
log.Info().Str("repo", remoteRepo).Str("digest", digest.String()).Msg("couldn't get remote blob")
pkg/extensions/sync/references/references_internal_test.go
+8 -6
View File
@@ -4,6 +4,7 @@
package references
import (
"context"
"errors"
"testing"
@@ -75,7 +76,7 @@ func TestOci(t *testing.T) {
},
}}, nil, log.NewLogger("debug", ""))
ok := oci.IsSigned("repo", "")
ok := oci.IsSigned(context.Background(), "repo", "")
So(ok, ShouldBeFalse)
// trigger GetReferrers err
@@ -136,11 +137,12 @@ func TestSyncManifest(t *testing.T) {
digest := godigest.FromString("test")
buf, refDigest, err := syncManifest(client, mocks.MockedImageStore{}, "repo", "repo", ispec.Descriptor{
Digest: digest,
Size: 10,
MediaType: ispec.MediaTypeImageManifest,
}, digest.String(), log.Logger{})
buf, refDigest, err := syncManifest(context.Background(), client, mocks.MockedImageStore{},
"repo", "repo", ispec.Descriptor{
Digest: digest,
Size: 10,
MediaType: ispec.MediaTypeImageManifest,
}, digest.String(), log.Logger{})
So(buf, ShouldBeEmpty)
So(refDigest, ShouldBeEmpty)