diff --git a/pkg/extensions/sync/on_demand.go b/pkg/extensions/sync/on_demand.go index 01387c2d..60c44778 100644 --- a/pkg/extensions/sync/on_demand.go +++ b/pkg/extensions/sync/on_demand.go @@ -3,7 +3,6 @@ package sync import ( "context" "fmt" - "net/url" "os" "sync" "time" @@ -13,21 +12,16 @@ import ( "github.com/containers/image/v5/docker" "github.com/containers/image/v5/signature" "github.com/containers/image/v5/types" - "gopkg.in/resty.v1" "zotregistry.io/zot/pkg/log" "zotregistry.io/zot/pkg/storage" ) type syncContextUtils struct { - imageStore storage.ImageStore policyCtx *signature.PolicyContext localCtx *types.SystemContext upstreamCtx *types.SystemContext - client *resty.Client - url *url.URL upstreamAddr string - retryOptions *retry.RetryOptions copyOptions copy.Options } @@ -115,8 +109,6 @@ func syncOneImage(imageChannel chan error, cfg Config, storeController storage.S return } - imageStore := storeController.GetImageStore(localRepo) - for _, registryCfg := range cfg.Registries { regCfg := registryCfg if !regCfg.OnDemand { @@ -125,7 +117,7 @@ func syncOneImage(imageChannel chan error, cfg Config, storeController storage.S continue } - remoteRepo := localRepo + upstreamRepo := localRepo // if content config is not specified, then don't filter, just sync demanded image if len(regCfg.Content) != 0 { @@ -137,7 +129,7 @@ func syncOneImage(imageChannel chan error, cfg Config, storeController storage.S continue } - remoteRepo = getRepoSource(localRepo, regCfg.Content[contentID]) + upstreamRepo = getRepoSource(localRepo, regCfg.Content[contentID]) } retryOptions := &retry.RetryOptions{} @@ -163,7 +155,8 @@ func syncOneImage(imageChannel chan error, cfg Config, storeController storage.S return } - // it's an image + sig := newSignaturesCopier(httpClient, *registryURL, storeController, log) + upstreamCtx := getUpstreamContext(®Cfg, credentialsFile[upstreamAddr]) options := getCopyOptions(upstreamCtx, localCtx) @@ -171,18 +164,18 @@ func syncOneImage(imageChannel chan error, cfg Config, storeController storage.S if isCosignTag(tag) { // at tis point we should already have images synced, but not their signatures. // is cosign signature - cosignManifest, err := getCosignManifest(httpClient, *registryURL, remoteRepo, tag, log) + cosignManifest, err := sig.getCosignManifest(upstreamRepo, tag) if err != nil { log.Error().Str("errorType", TypeOf(err)). - Err(err).Msgf("couldn't get upstream image %s:%s:%s cosign manifest", upstreamURL, remoteRepo, tag) + Err(err).Msgf("couldn't get upstream image %s:%s:%s cosign manifest", upstreamURL, upstreamRepo, tag) continue } - err = syncCosignSignature(httpClient, imageStore, *registryURL, localRepo, remoteRepo, tag, cosignManifest, log) + err = sig.syncCosignSignature(localRepo, upstreamRepo, tag, cosignManifest) if err != nil { log.Error().Str("errorType", TypeOf(err)). - Err(err).Msgf("couldn't copy upstream image cosign signature %s/%s:%s", upstreamURL, remoteRepo, tag) + Err(err).Msgf("couldn't copy upstream image cosign signature %s/%s:%s", upstreamURL, upstreamRepo, tag) continue } @@ -192,18 +185,18 @@ func syncOneImage(imageChannel chan error, cfg Config, storeController storage.S return } else if isArtifact { // is notary signature - refs, err := getNotaryRefs(httpClient, *registryURL, remoteRepo, tag, log) + refs, err := sig.getNotaryRefs(upstreamRepo, tag) if err != nil { log.Error().Str("errorType", TypeOf(err)). - Err(err).Msgf("couldn't get upstream image %s/%s:%s notary references", upstreamURL, remoteRepo, tag) + Err(err).Msgf("couldn't get upstream image %s/%s:%s notary references", upstreamURL, upstreamRepo, tag) continue } - err = syncNotarySignature(httpClient, imageStore, *registryURL, localRepo, remoteRepo, tag, refs, log) + err = sig.syncNotarySignature(localRepo, upstreamRepo, tag, refs) if err != nil { log.Error().Str("errorType", TypeOf(err)). - Err(err).Msgf("couldn't copy image signature %s/%s:%s", upstreamURL, remoteRepo, tag) + Err(err).Msgf("couldn't copy image signature %s/%s:%s", upstreamURL, upstreamRepo, tag) continue } @@ -214,24 +207,20 @@ func syncOneImage(imageChannel chan error, cfg Config, storeController storage.S } syncContextUtils := syncContextUtils{ - imageStore: imageStore, policyCtx: policyCtx, localCtx: localCtx, upstreamCtx: upstreamCtx, - client: httpClient, - url: registryURL, upstreamAddr: upstreamAddr, - retryOptions: retryOptions, copyOptions: options, } - skipped, copyErr := syncRun(regCfg, localRepo, remoteRepo, tag, syncContextUtils, log) + skipped, copyErr := syncRun(regCfg, localRepo, upstreamRepo, tag, syncContextUtils, sig, log) if skipped { continue } // key used to check if we already have a go routine syncing this image - demandedImageRef := fmt.Sprintf("%s/%s:%s", upstreamAddr, remoteRepo, tag) + demandedImageRef := fmt.Sprintf("%s/%s:%s", upstreamAddr, upstreamRepo, tag) if copyErr != nil { // don't retry in background if maxretry is 0 @@ -260,7 +249,7 @@ func syncOneImage(imageChannel chan error, cfg Config, storeController storage.S time.Sleep(retryOptions.Delay) if err = retry.RetryIfNecessary(context.Background(), func() error { - _, err := syncRun(regCfg, localRepo, remoteRepo, tag, syncContextUtils, log) + _, err := syncRun(regCfg, localRepo, upstreamRepo, tag, syncContextUtils, sig, log) return err }, retryOptions); err != nil { @@ -276,14 +265,14 @@ func syncOneImage(imageChannel chan error, cfg Config, storeController storage.S } func syncRun(regCfg RegistryConfig, - localRepo, remoteRepo, tag string, utils syncContextUtils, + localRepo, upstreamRepo, tag string, utils syncContextUtils, sig *signaturesCopier, log log.Logger, ) (bool, error) { - upstreamImageRef, err := getImageRef(utils.upstreamAddr, remoteRepo, tag) + upstreamImageRef, err := getImageRef(utils.upstreamAddr, upstreamRepo, tag) if err != nil { log.Error().Str("errorType", TypeOf(err)). Err(err).Msgf("error creating docker reference for repository %s/%s:%s", - utils.upstreamAddr, remoteRepo, tag) + utils.upstreamAddr, upstreamRepo, tag) return false, err } @@ -297,14 +286,13 @@ func syncRun(regCfg RegistryConfig, } // get upstream signatures - cosignManifest, err := getCosignManifest(utils.client, *utils.url, remoteRepo, - upstreamImageDigest.String(), log) + cosignManifest, err := sig.getCosignManifest(upstreamRepo, upstreamImageDigest.String()) if err != nil { log.Error().Str("errorType", TypeOf(err)). Err(err).Msgf("couldn't get upstream image %s cosign manifest", upstreamImageRef.DockerReference()) } - refs, err := getNotaryRefs(utils.client, *utils.url, remoteRepo, upstreamImageDigest.String(), log) + refs, err := sig.getNotaryRefs(upstreamRepo, upstreamImageDigest.String()) if err != nil { log.Error().Str("errorType", TypeOf(err)). Err(err).Msgf("couldn't get upstream image %s notary references", upstreamImageRef.DockerReference()) @@ -321,7 +309,9 @@ func syncRun(regCfg RegistryConfig, } } - localCachePath, err := getLocalCachePath(utils.imageStore, localRepo) + imageStore := sig.storeController.GetImageStore(localRepo) + + localCachePath, err := getLocalCachePath(imageStore, localRepo) if err != nil { log.Error().Err(err).Msgf("couldn't get localCachePath for %s", localRepo) } @@ -348,7 +338,7 @@ func syncRun(regCfg RegistryConfig, return false, err } - err = pushSyncedLocalImage(localRepo, tag, localCachePath, utils.imageStore, log) + err = pushSyncedLocalImage(localRepo, tag, localCachePath, imageStore, log) if err != nil { log.Error().Str("errorType", TypeOf(err)). Err(err).Msgf("error while pushing synced cached image %s", @@ -357,25 +347,23 @@ func syncRun(regCfg RegistryConfig, return false, err } - err = syncCosignSignature(utils.client, utils.imageStore, *utils.url, localRepo, remoteRepo, - upstreamImageDigest.String(), cosignManifest, log) + err = sig.syncCosignSignature(localRepo, upstreamRepo, upstreamImageDigest.String(), cosignManifest) if err != nil { log.Error().Str("errorType", TypeOf(err)). - Err(err).Msgf("couldn't copy image cosign signature %s/%s:%s", utils.upstreamAddr, remoteRepo, tag) + Err(err).Msgf("couldn't copy image cosign signature %s/%s:%s", utils.upstreamAddr, upstreamRepo, tag) return false, err } - err = syncNotarySignature(utils.client, utils.imageStore, *utils.url, localRepo, remoteRepo, - upstreamImageDigest.String(), refs, log) + err = sig.syncNotarySignature(localRepo, upstreamRepo, upstreamImageDigest.String(), refs) if err != nil { log.Error().Str("errorType", TypeOf(err)). - Err(err).Msgf("couldn't copy image notary signature %s/%s:%s", utils.upstreamAddr, remoteRepo, tag) + Err(err).Msgf("couldn't copy image notary signature %s/%s:%s", utils.upstreamAddr, upstreamRepo, tag) return false, err } - log.Info().Msgf("successfully synced %s/%s:%s", utils.upstreamAddr, remoteRepo, tag) + log.Info().Msgf("successfully synced %s/%s:%s", utils.upstreamAddr, upstreamRepo, tag) return false, nil } diff --git a/pkg/extensions/sync/signatures.go b/pkg/extensions/sync/signatures.go index ae920811..373f1c2f 100644 --- a/pkg/extensions/sync/signatures.go +++ b/pkg/extensions/sync/signatures.go @@ -20,24 +20,40 @@ import ( "zotregistry.io/zot/pkg/storage" ) -func getCosignManifest(client *resty.Client, regURL url.URL, repo, digest string, - log log.Logger, -) (*ispec.Manifest, error) { +type signaturesCopier struct { + client *resty.Client + upstreamURL url.URL + storeController storage.StoreController + log log.Logger +} + +func newSignaturesCopier(httpClient *resty.Client, upstreamURL url.URL, + storeController storage.StoreController, log log.Logger, +) *signaturesCopier { + return &signaturesCopier{ + client: httpClient, + upstreamURL: upstreamURL, + storeController: storeController, + log: log, + } +} + +func (sig *signaturesCopier) getCosignManifest(repo, digest string) (*ispec.Manifest, error) { var cosignManifest ispec.Manifest cosignTag := getCosignTagFromImageDigest(digest) - getCosignManifestURL := regURL + getCosignManifestURL := sig.upstreamURL getCosignManifestURL.Path = path.Join(getCosignManifestURL.Path, "v2", repo, "manifests", cosignTag) getCosignManifestURL.RawQuery = getCosignManifestURL.Query().Encode() - resp, err := client.R(). + resp, err := sig.client.R(). SetHeader("Content-Type", "application/vnd.oci.image.manifest.v1+json"). Get(getCosignManifestURL.String()) if err != nil { - log.Error().Str("errorType", TypeOf(err)). + sig.log.Error().Str("errorType", TypeOf(err)). Err(err).Str("url", getCosignManifestURL.String()). Msgf("couldn't get cosign manifest: %s", cosignTag) @@ -45,12 +61,12 @@ func getCosignManifest(client *resty.Client, regURL url.URL, repo, digest string } if resp.StatusCode() == http.StatusNotFound { - log.Info().Msgf("couldn't find any cosign signature from %s, status code: %d skipping", + sig.log.Info().Msgf("couldn't find any cosign signature from %s, status code: %d skipping", getCosignManifestURL.String(), resp.StatusCode()) return nil, zerr.ErrSyncSignatureNotFound } else if resp.IsError() { - log.Error().Str("errorType", TypeOf(zerr.ErrSyncSignature)). + sig.log.Error().Str("errorType", TypeOf(zerr.ErrSyncSignature)). Err(zerr.ErrSyncSignature).Msgf("couldn't get cosign signature from %s, status code: %d skipping", getCosignManifestURL.String(), resp.StatusCode()) @@ -59,7 +75,7 @@ func getCosignManifest(client *resty.Client, regURL url.URL, repo, digest string err = json.Unmarshal(resp.Body(), &cosignManifest) if err != nil { - log.Error().Str("errorType", TypeOf(err)). + sig.log.Error().Str("errorType", TypeOf(err)). Err(err).Str("url", getCosignManifestURL.String()). Msgf("couldn't unmarshal cosign manifest %s", cosignTag) @@ -69,10 +85,10 @@ func getCosignManifest(client *resty.Client, regURL url.URL, repo, digest string return &cosignManifest, nil } -func getNotaryRefs(client *resty.Client, regURL url.URL, repo, digest string, log log.Logger) (ReferenceList, error) { +func (sig *signaturesCopier) getNotaryRefs(repo, digest string) (ReferenceList, error) { var referrers ReferenceList - getReferrersURL := regURL + getReferrersURL := sig.upstreamURL // based on manifest digest get referrers getReferrersURL.Path = path.Join(getReferrersURL.Path, constants.ArtifactSpecRoutePrefix, @@ -80,24 +96,24 @@ func getNotaryRefs(client *resty.Client, regURL url.URL, repo, digest string, lo getReferrersURL.RawQuery = getReferrersURL.Query().Encode() - resp, err := client.R(). + resp, err := sig.client.R(). SetHeader("Content-Type", "application/json"). SetQueryParam("artifactType", notreg.ArtifactTypeNotation). Get(getReferrersURL.String()) if err != nil { - log.Error().Str("errorType", TypeOf(err)). + sig.log.Error().Str("errorType", TypeOf(err)). Err(err).Str("url", getReferrersURL.String()).Msg("couldn't get referrers") return referrers, err } if resp.StatusCode() == http.StatusNotFound || resp.StatusCode() == http.StatusBadRequest { - log.Info().Msgf("couldn't find any notary signature from %s, status code: %d, skipping", + sig.log.Info().Msgf("couldn't find any notary signature from %s, status code: %d, skipping", getReferrersURL.String(), resp.StatusCode()) return ReferenceList{}, zerr.ErrSyncSignatureNotFound } else if resp.IsError() { - log.Error().Str("errorType", TypeOf(zerr.ErrSyncSignature)). + sig.log.Error().Str("errorType", TypeOf(zerr.ErrSyncSignature)). Err(zerr.ErrSyncSignature).Msgf("couldn't get notary signature from %s, status code: %d skipping", getReferrersURL.String(), resp.StatusCode()) @@ -106,7 +122,7 @@ func getNotaryRefs(client *resty.Client, regURL url.URL, repo, digest string, lo err = json.Unmarshal(resp.Body(), &referrers) if err != nil { - log.Error().Str("errorType", TypeOf(err)). + sig.log.Error().Str("errorType", TypeOf(err)). Err(err).Str("url", getReferrersURL.String()). Msgf("couldn't unmarshal notary signature") @@ -116,8 +132,7 @@ func getNotaryRefs(client *resty.Client, regURL url.URL, repo, digest string, lo return referrers, nil } -func syncCosignSignature(client *resty.Client, imageStore storage.ImageStore, - regURL url.URL, localRepo, remoteRepo, digest string, cosignManifest *ispec.Manifest, log log.Logger, +func (sig *signaturesCopier) syncCosignSignature(localRepo, remoteRepo, digest string, cosignManifest *ispec.Manifest, ) error { cosignTag := getCosignTagFromImageDigest(digest) @@ -126,24 +141,36 @@ func syncCosignSignature(client *resty.Client, imageStore storage.ImageStore, return nil } - log.Info().Msg("syncing cosign signatures") + skipCosignSig, err := sig.canSkipCosignSignature(localRepo, digest, cosignManifest) + if err != nil { + sig.log.Error().Err(err).Msgf("couldn't check if the upstream image %s:%s cosign signature can be skipped", + remoteRepo, digest) + } + + if skipCosignSig { + return nil + } + + imageStore := sig.storeController.GetImageStore(localRepo) + + sig.log.Info().Msg("syncing cosign signatures") for _, blob := range cosignManifest.Layers { // get blob - getBlobURL := regURL + getBlobURL := sig.upstreamURL getBlobURL.Path = path.Join(getBlobURL.Path, "v2", remoteRepo, "blobs", blob.Digest.String()) getBlobURL.RawQuery = getBlobURL.Query().Encode() - resp, err := client.R().SetDoNotParseResponse(true).Get(getBlobURL.String()) + resp, err := sig.client.R().SetDoNotParseResponse(true).Get(getBlobURL.String()) if err != nil { - log.Error().Str("errorType", TypeOf(err)). + sig.log.Error().Str("errorType", TypeOf(err)). Err(err).Msgf("couldn't get cosign blob: %s", blob.Digest.String()) return err } if resp.IsError() { - log.Info().Msgf("couldn't find cosign blob from %s, status code: %d", getBlobURL.String(), resp.StatusCode()) + sig.log.Info().Msgf("couldn't find cosign blob from %s, status code: %d", getBlobURL.String(), resp.StatusCode()) return zerr.ErrSyncSignature } @@ -153,7 +180,7 @@ func syncCosignSignature(client *resty.Client, imageStore storage.ImageStore, // push blob _, _, err = imageStore.FullBlobUpload(localRepo, resp.RawBody(), blob.Digest.String()) if err != nil { - log.Error().Str("errorType", TypeOf(err)). + sig.log.Error().Str("errorType", TypeOf(err)). Err(err).Msg("couldn't upload cosign blob") return err @@ -161,20 +188,21 @@ func syncCosignSignature(client *resty.Client, imageStore storage.ImageStore, } // get config blob - getBlobURL := regURL + getBlobURL := sig.upstreamURL getBlobURL.Path = path.Join(getBlobURL.Path, "v2", remoteRepo, "blobs", cosignManifest.Config.Digest.String()) getBlobURL.RawQuery = getBlobURL.Query().Encode() - resp, err := client.R().SetDoNotParseResponse(true).Get(getBlobURL.String()) + resp, err := sig.client.R().SetDoNotParseResponse(true).Get(getBlobURL.String()) if err != nil { - log.Error().Str("errorType", TypeOf(err)). + sig.log.Error().Str("errorType", TypeOf(err)). Err(err).Msgf("couldn't get cosign config blob: %s", getBlobURL.String()) return err } if resp.IsError() { - log.Info().Msgf("couldn't find cosign config blob from %s, status code: %d", getBlobURL.String(), resp.StatusCode()) + sig.log.Info().Msgf("couldn't find cosign config blob from %s, status code: %d", + getBlobURL.String(), resp.StatusCode()) return zerr.ErrSyncSignature } @@ -184,7 +212,7 @@ func syncCosignSignature(client *resty.Client, imageStore storage.ImageStore, // push config blob _, _, err = imageStore.FullBlobUpload(localRepo, resp.RawBody(), cosignManifest.Config.Digest.String()) if err != nil { - log.Error().Str("errorType", TypeOf(err)). + sig.log.Error().Str("errorType", TypeOf(err)). Err(err).Msg("couldn't upload cosign config blob") return err @@ -192,7 +220,7 @@ func syncCosignSignature(client *resty.Client, imageStore storage.ImageStore, cosignManifestBuf, err := json.Marshal(cosignManifest) if err != nil { - log.Error().Str("errorType", TypeOf(err)). + sig.log.Error().Str("errorType", TypeOf(err)). Err(err).Msg("couldn't marshal cosign manifest") } @@ -200,36 +228,47 @@ func syncCosignSignature(client *resty.Client, imageStore storage.ImageStore, _, err = imageStore.PutImageManifest(localRepo, cosignTag, ispec.MediaTypeImageManifest, cosignManifestBuf) if err != nil { - log.Error().Str("errorType", TypeOf(err)). + sig.log.Error().Str("errorType", TypeOf(err)). Err(err).Msg("couldn't upload cosign manifest") return err } - log.Info().Msgf("successfully synced cosign signature for repo %s digest %s", localRepo, digest) + sig.log.Info().Msgf("successfully synced cosign signature for repo %s digest %s", localRepo, digest) return nil } -func syncNotarySignature(client *resty.Client, imageStore storage.ImageStore, - regURL url.URL, localRepo, remoteRepo, digest string, referrers ReferenceList, log log.Logger, +func (sig *signaturesCopier) syncNotarySignature(localRepo, remoteRepo, digest string, referrers ReferenceList, ) error { if len(referrers.References) == 0 { return nil } - log.Info().Msg("syncing notary signatures") + skipNotarySig, err := sig.canSkipNotarySignature(localRepo, digest, referrers) + if skipNotarySig || err != nil { + sig.log.Error().Err(err).Msgf("couldn't check if the upstream image %s:%s notary signature can be skipped", + remoteRepo, digest) + } + + if skipNotarySig { + return nil + } + + imageStore := sig.storeController.GetImageStore(localRepo) + + sig.log.Info().Msg("syncing notary signatures") for _, ref := range referrers.References { // get referrer manifest - getRefManifestURL := regURL + getRefManifestURL := sig.upstreamURL getRefManifestURL.Path = path.Join(getRefManifestURL.Path, "v2", remoteRepo, "manifests", ref.Digest.String()) getRefManifestURL.RawQuery = getRefManifestURL.Query().Encode() - resp, err := client.R(). + resp, err := sig.client.R(). Get(getRefManifestURL.String()) if err != nil { - log.Error().Str("errorType", TypeOf(err)). + sig.log.Error().Str("errorType", TypeOf(err)). Err(err).Msgf("couldn't get notary manifest: %s", getRefManifestURL.String()) return err @@ -240,20 +279,20 @@ func syncNotarySignature(client *resty.Client, imageStore storage.ImageStore, err = json.Unmarshal(resp.Body(), &artifactManifest) if err != nil { - log.Error().Str("errorType", TypeOf(err)). + sig.log.Error().Str("errorType", TypeOf(err)). Err(err).Msgf("couldn't unmarshal notary manifest: %s", getRefManifestURL.String()) return err } for _, blob := range artifactManifest.Blobs { - getBlobURL := regURL + getBlobURL := sig.upstreamURL getBlobURL.Path = path.Join(getBlobURL.Path, "v2", remoteRepo, "blobs", blob.Digest.String()) getBlobURL.RawQuery = getBlobURL.Query().Encode() - resp, err := client.R().SetDoNotParseResponse(true).Get(getBlobURL.String()) + resp, err := sig.client.R().SetDoNotParseResponse(true).Get(getBlobURL.String()) if err != nil { - log.Error().Str("errorType", TypeOf(err)). + sig.log.Error().Str("errorType", TypeOf(err)). Err(err).Msgf("couldn't get notary blob: %s", getBlobURL.String()) return err @@ -262,7 +301,7 @@ func syncNotarySignature(client *resty.Client, imageStore storage.ImageStore, defer resp.RawBody().Close() if resp.IsError() { - log.Info().Msgf("couldn't find notary blob from %s, status code: %d", + sig.log.Info().Msgf("couldn't find notary blob from %s, status code: %d", getBlobURL.String(), resp.StatusCode()) return zerr.ErrSyncSignature @@ -270,7 +309,7 @@ func syncNotarySignature(client *resty.Client, imageStore storage.ImageStore, _, _, err = imageStore.FullBlobUpload(localRepo, resp.RawBody(), blob.Digest.String()) if err != nil { - log.Error().Str("errorType", TypeOf(err)). + sig.log.Error().Str("errorType", TypeOf(err)). Err(err).Msg("couldn't upload notary sig blob") return err @@ -280,50 +319,51 @@ func syncNotarySignature(client *resty.Client, imageStore storage.ImageStore, _, err = imageStore.PutImageManifest(localRepo, ref.Digest.String(), artifactspec.MediaTypeArtifactManifest, resp.Body()) if err != nil { - log.Error().Str("errorType", TypeOf(err)). + sig.log.Error().Str("errorType", TypeOf(err)). Err(err).Msg("couldn't upload notary sig manifest") return err } } - log.Info().Msgf("successfully synced notary signature for repo %s digest %s", localRepo, digest) + sig.log.Info().Msgf("successfully synced notary signature for repo %s digest %s", localRepo, digest) return nil } -func canSkipNotarySignature(repo, tag, digest string, refs ReferenceList, imageStore storage.ImageStore, - log log.Logger, +func (sig *signaturesCopier) canSkipNotarySignature(localRepo, digest string, refs ReferenceList, ) (bool, error) { + imageStore := sig.storeController.GetImageStore(localRepo) + // check notary signature already synced if len(refs.References) > 0 { - localRefs, err := imageStore.GetReferrers(repo, digest, notreg.ArtifactTypeNotation) + localRefs, err := imageStore.GetReferrers(localRepo, digest, notreg.ArtifactTypeNotation) if err != nil { if errors.Is(err, zerr.ErrManifestNotFound) { return false, nil } - log.Error().Str("errorType", TypeOf(err)). - Err(err).Msgf("couldn't get local notary signature %s:%s manifest", repo, tag) + sig.log.Error().Str("errorType", TypeOf(err)). + Err(err).Msgf("couldn't get local notary signature %s:%s manifest", localRepo, digest) return false, err } if !artifactDescriptorsEqual(localRefs, refs.References) { - log.Info().Msgf("upstream notary signatures %s:%s changed, syncing again", repo, tag) + sig.log.Info().Msgf("upstream notary signatures %s:%s changed, syncing again", localRepo, digest) return false, nil } } - log.Info().Msgf("skipping notary signature %s:%s, already synced", repo, tag) + sig.log.Info().Msgf("skipping notary signature %s:%s, already synced", localRepo, digest) return true, nil } -func canSkipCosignSignature(repo, tag, digest string, cosignManifest *ispec.Manifest, imageStore storage.ImageStore, - log log.Logger, +func (sig *signaturesCopier) canSkipCosignSignature(localRepo, digest string, cosignManifest *ispec.Manifest, ) (bool, error) { + imageStore := sig.storeController.GetImageStore(localRepo) // check cosign signature already synced if cosignManifest != nil { var localCosignManifest ispec.Manifest @@ -332,34 +372,34 @@ func canSkipCosignSignature(repo, tag, digest string, cosignManifest *ispec.Mani because of an issue where cosign digests differs between upstream and downstream */ cosignManifestTag := getCosignTagFromImageDigest(digest) - localCosignManifestBuf, _, _, err := imageStore.GetImageManifest(repo, cosignManifestTag) + localCosignManifestBuf, _, _, err := imageStore.GetImageManifest(localRepo, cosignManifestTag) if err != nil { if errors.Is(err, zerr.ErrManifestNotFound) { return false, nil } - log.Error().Str("errorType", TypeOf(err)). - Err(err).Msgf("couldn't get local cosign %s:%s manifest", repo, tag) + sig.log.Error().Str("errorType", TypeOf(err)). + Err(err).Msgf("couldn't get local cosign %s:%s manifest", localRepo, digest) return false, err } err = json.Unmarshal(localCosignManifestBuf, &localCosignManifest) if err != nil { - log.Error().Str("errorType", TypeOf(err)). - Err(err).Msgf("couldn't unmarshal local cosign signature %s:%s manifest", repo, tag) + sig.log.Error().Str("errorType", TypeOf(err)). + Err(err).Msgf("couldn't unmarshal local cosign signature %s:%s manifest", localRepo, digest) return false, err } if !manifestsEqual(localCosignManifest, *cosignManifest) { - log.Info().Msgf("upstream cosign signatures %s:%s changed, syncing again", repo, tag) + sig.log.Info().Msgf("upstream cosign signatures %s:%s changed, syncing again", localRepo, digest) return false, nil } } - log.Info().Msgf("skipping cosign signature %s:%s, already synced", repo, tag) + sig.log.Info().Msgf("skipping cosign signature %s:%s, already synced", localRepo, digest) return true, nil } diff --git a/pkg/extensions/sync/sync.go b/pkg/extensions/sync/sync.go index 279e0956..59c03e87 100644 --- a/pkg/extensions/sync/sync.go +++ b/pkg/extensions/sync/sync.go @@ -7,11 +7,9 @@ import ( "fmt" "io" "os" - "regexp" goSync "sync" "time" - "github.com/Masterminds/semver" "github.com/containers/common/pkg/retry" "github.com/containers/image/v5/copy" "github.com/containers/image/v5/docker" @@ -76,6 +74,12 @@ type Tags struct { Semver *bool } +type RepoReferences struct { + contentID int // matched registry config content + name string // repo name + imageReferences []types.ImageReference // contained images(tags) +} + // getUpstreamCatalog gets all repos from a registry. func getUpstreamCatalog(client *resty.Client, upstreamURL string, log log.Logger) (catalog, error) { var catalog catalog @@ -106,159 +110,70 @@ func getUpstreamCatalog(client *resty.Client, upstreamURL string, log log.Logger return catalog, nil } -// getImageTags lists all tags in a repository. -// It returns a string slice of tags and any error encountered. -func getImageTags(ctx context.Context, sysCtx *types.SystemContext, repoRef reference.Named) ([]string, error) { - dockerRef, err := docker.NewReference(reference.TagNameOnly(repoRef)) - // hard to reach test case, injected error, see pkg/test/dev.go - if err = test.Error(err); err != nil { - return nil, err // Should never happen for a reference with tag and no digest - } - - tags, err := docker.GetRepositoryTags(ctx, sysCtx, dockerRef) - if err != nil { - return nil, err - } - - return tags, nil -} - -// filterImagesByTagRegex filters images by tag regex given in the config. -func filterImagesByTagRegex(upstreamReferences *[]types.ImageReference, content Content, log log.Logger) error { - refs := *upstreamReferences - - if content.Tags == nil { - // no need to filter anything - return nil - } - - if content.Tags.Regex != nil { - log.Info().Msgf("start filtering using the regular expression: %s", *content.Tags.Regex) - - tagReg, err := regexp.Compile(*content.Tags.Regex) - if err != nil { - return err - } - - numTags := 0 - - for _, ref := range refs { - tagged := getTagFromRef(ref, log) - if tagged != nil { - if tagReg.MatchString(tagged.Tag()) { - refs[numTags] = ref - numTags++ - } - } - } - - refs = refs[:numTags] - } - - *upstreamReferences = refs - - return nil -} - -// filterImagesBySemver filters images by checking if their tags are semver compliant. -func filterImagesBySemver(upstreamReferences *[]types.ImageReference, content Content, log log.Logger) { - refs := *upstreamReferences - - if content.Tags == nil { - return - } - - if content.Tags.Semver != nil && *content.Tags.Semver { - log.Info().Msg("start filtering using semver compliant rule") - - numTags := 0 - - for _, ref := range refs { - tagged := getTagFromRef(ref, log) - if tagged != nil { - _, ok := semver.NewVersion(tagged.Tag()) - if ok == nil { - refs[numTags] = ref - numTags++ - } - } - } - - refs = refs[:numTags] - } - - *upstreamReferences = refs -} - // imagesToCopyFromRepos lists all images given a registry name and its repos. -func imagesToCopyFromUpstream(ctx context.Context, registryName string, repos []string, +func imagesToCopyFromUpstream(ctx context.Context, registryName string, repoName string, upstreamCtx *types.SystemContext, content Content, log log.Logger, -) (map[string][]types.ImageReference, error) { - upstreamReferences := make(map[string][]types.ImageReference) +) ([]types.ImageReference, error) { + imageRefs := []types.ImageReference{} - for _, repoName := range repos { - repoUpstreamReferences := make([]types.ImageReference, 0) + repoRef, err := parseRepositoryReference(fmt.Sprintf("%s/%s", registryName, repoName)) + if err != nil { + log.Error().Str("errorType", TypeOf(err)). + Err(err).Msgf("couldn't parse repository reference: %s", repoRef) - repoRef, err := parseRepositoryReference(fmt.Sprintf("%s/%s", registryName, repoName)) - if err != nil { - log.Error().Str("errorType", TypeOf(err)). - Err(err).Msgf("couldn't parse repository reference: %s", repoRef) - - return nil, err - } - - tags, err := getImageTags(ctx, upstreamCtx, repoRef) - if err != nil { - log.Error().Str("errorType", TypeOf(err)). - Err(err).Msgf("couldn't fetch tags for %s", repoRef) - - return nil, err - } - - for _, tag := range tags { - // don't copy cosign signature, containers/image doesn't support it - // we will copy it manually later - if isCosignTag(tag) { - continue - } - - taggedRef, err := reference.WithTag(repoRef, tag) - if err != nil { - log.Err(err).Msgf("error creating a reference for repository %s and tag %q", repoRef.Name(), tag) - - return nil, err - } - - ref, err := docker.NewReference(taggedRef) - if err != nil { - log.Err(err).Msgf("cannot obtain a valid image reference for transport %q and reference %s", - docker.Transport.Name(), taggedRef.String()) - - return nil, err - } - - repoUpstreamReferences = append(repoUpstreamReferences, ref) - } - - upstreamReferences[repoName] = repoUpstreamReferences - - log.Debug().Msgf("repo: %s - upstream refs to be copied: %v", repoName, upstreamReferences) - - err = filterImagesByTagRegex(&repoUpstreamReferences, content, log) - if err != nil { - return map[string][]types.ImageReference{}, err - } - - log.Debug().Msgf("repo: %s - remaining upstream refs to be copied: %v", repoName, repoUpstreamReferences) - - filterImagesBySemver(&repoUpstreamReferences, content, log) - - log.Debug().Msgf("repo: %s - remaining upstream refs to be copied: %v", repoName, repoUpstreamReferences) - - upstreamReferences[repoName] = repoUpstreamReferences + return imageRefs, err } - return upstreamReferences, nil + tags, err := getImageTags(ctx, upstreamCtx, repoRef) + if err != nil { + log.Error().Str("errorType", TypeOf(err)). + Err(err).Msgf("couldn't fetch tags for %s", repoRef) + + return imageRefs, err + } + + // filter based on tags rules + if content.Tags != nil { + if content.Tags.Regex != nil { + tags, err = filterTagsByRegex(tags, *content.Tags.Regex, log) + if err != nil { + return imageRefs, err + } + } + + if content.Tags.Semver != nil && *content.Tags.Semver { + tags = filterTagsBySemver(tags, log) + } + } + + log.Debug().Msgf("repo: %s - upstream tags to be copied: %v", repoName, tags) + + for _, tag := range tags { + // don't copy cosign signature, containers/image doesn't support it + // we will copy it manually later + if isCosignTag(tag) { + continue + } + + taggedRef, err := reference.WithTag(repoRef, tag) + if err != nil { + log.Err(err).Msgf("error creating a reference for repository %s and tag %q", repoRef.Name(), tag) + + return imageRefs, err + } + + ref, err := docker.NewReference(taggedRef) + if err != nil { + log.Err(err).Msgf("cannot obtain a valid image reference for transport %q and reference %s", + docker.Transport.Name(), taggedRef.String()) + + return imageRefs, err + } + + imageRefs = append(imageRefs, ref) + } + + return imageRefs, nil } func getCopyOptions(upstreamCtx, localCtx *types.SystemContext) copy.Options { @@ -339,69 +254,53 @@ func syncRegistry(ctx context.Context, regCfg RegistryConfig, upstreamAddr := StripRegistryTransport(upstreamURL) - reposWithContentID := make(map[string][]struct { - ref types.ImageReference - content Content - }) + reposReferences := []RepoReferences{} for contentID, repos := range repos { - r := repos - contentID := contentID + for _, repoName := range repos { + var imageReferences []types.ImageReference - if err = retry.RetryIfNecessary(ctx, func() error { - for _, repo := range r { - refs, err := imagesToCopyFromUpstream(ctx, upstreamAddr, r, upstreamCtx, regCfg.Content[contentID], log) - if err != nil { - return err - } + if err = retry.RetryIfNecessary(ctx, func() error { + imageReferences, err = imagesToCopyFromUpstream(ctx, upstreamAddr, repoName, upstreamCtx, + regCfg.Content[contentID], log) - for _, ref := range refs[repo] { - reposWithContentID[repo] = append(reposWithContentID[repo], struct { - ref types.ImageReference - content Content - }{ - ref: ref, - content: regCfg.Content[contentID], - }) - } - } - - return nil - }, retryOptions); err != nil { - log.Error().Str("errorType", TypeOf(err)). - Err(err).Msg("error while getting images references from upstream, retrying...") - - return err - } - } - - for remoteRepo, imageList := range reposWithContentID { - select { - case <-ctx.Done(): - return ctx.Err() - default: - break - } - - remoteRepoCopy := remoteRepo - - for _, image := range imageList { - localRepo := getRepoDestination(remoteRepo, image.content) - - imageStore := storeController.GetImageStore(localRepo) - - localCachePath, err := getLocalCachePath(imageStore, localRepo) - if err != nil { + return err + }, retryOptions); err != nil { log.Error().Str("errorType", TypeOf(err)). - Err(err).Msgf("couldn't get localCachePath for %s", remoteRepoCopy) + Err(err).Msg("error while getting images references from upstream, retrying...") return err } - defer os.RemoveAll(localCachePath) + reposReferences = append(reposReferences, RepoReferences{ + contentID: contentID, + name: repoName, + imageReferences: imageReferences, + }) + } + } - upstreamImageRef := image.ref + sig := newSignaturesCopier(httpClient, *registryURL, storeController, log) + for _, repoReference := range reposReferences { + upstreamRepo := repoReference.name + content := regCfg.Content[repoReference.contentID] + + localRepo := getRepoDestination(upstreamRepo, content) + + imageStore := storeController.GetImageStore(localRepo) + + localCachePath, err := getLocalCachePath(imageStore, localRepo) + if err != nil { + log.Error().Str("errorType", TypeOf(err)). + Err(err).Msgf("couldn't get localCachePath for %s", localRepo) + + return err + } + + defer os.RemoveAll(localCachePath) + + for _, upstreamImageRef := range repoReference.imageReferences { upstreamImageDigest, err := docker.GetDigest(ctx, upstreamCtx, upstreamImageRef) if err != nil { log.Error().Err(err).Msgf("couldn't get upstream image %s manifest", upstreamImageRef.DockerReference()) @@ -409,17 +308,15 @@ func syncRegistry(ctx context.Context, regCfg RegistryConfig, return err } - tag := getTagFromRef(upstreamImageRef, log).Tag() // get upstream signatures - cosignManifest, err := getCosignManifest(httpClient, *registryURL, remoteRepoCopy, - upstreamImageDigest.String(), log) + cosignManifest, err := sig.getCosignManifest(upstreamRepo, upstreamImageDigest.String()) if err != nil && !errors.Is(err, zerr.ErrSyncSignatureNotFound) { log.Error().Err(err).Msgf("couldn't get upstream image %s cosign manifest", upstreamImageRef.DockerReference()) return err } - refs, err := getNotaryRefs(httpClient, *registryURL, remoteRepoCopy, upstreamImageDigest.String(), log) + refs, err := sig.getNotaryRefs(upstreamRepo, upstreamImageDigest.String()) if err != nil && !errors.Is(err, zerr.ErrSyncSignatureNotFound) { log.Error().Err(err).Msgf("couldn't get upstream image %s notary references", upstreamImageRef.DockerReference()) @@ -437,6 +334,8 @@ func syncRegistry(ctx context.Context, regCfg RegistryConfig, } } + tag := getTagFromRef(upstreamImageRef, log).Tag() + skipImage, err := canSkipImage(localRepo, tag, upstreamImageDigest.String(), imageStore, log) if err != nil { log.Error().Err(err).Msgf("couldn't check if the upstream image %s can be skipped", @@ -445,103 +344,60 @@ func syncRegistry(ctx context.Context, regCfg RegistryConfig, return err } - // sync only differences - if skipImage { + if !skipImage { + // sync image + localImageRef, err := getLocalImageRef(localCachePath, localRepo, tag) + if err != nil { + log.Error().Str("errorType", TypeOf(err)). + Err(err).Msgf("couldn't obtain a valid image reference for reference %s/%s:%s", + localCachePath, localRepo, tag) + + return err + } + + log.Info().Msgf("copying image %s to %s", upstreamImageRef.DockerReference(), localCachePath) + + if err = retry.RetryIfNecessary(ctx, func() error { + _, err = copy.Image(ctx, policyCtx, localImageRef, upstreamImageRef, &options) + + return err + }, retryOptions); err != nil { + log.Error().Str("errorType", TypeOf(err)). + Err(err).Msgf("error while copying image %s to %s", + upstreamImageRef.DockerReference(), localCachePath) + + return err + } + // push from cache to repo + err = pushSyncedLocalImage(localRepo, tag, localCachePath, imageStore, log) + if err != nil { + log.Error().Str("errorType", TypeOf(err)). + Err(err).Msgf("error while pushing synced cached image %s", + fmt.Sprintf("%s/%s:%s", localCachePath, localRepo, tag)) + + return err + } + } else { log.Info().Msgf("already synced image %s, checking its signatures", upstreamImageRef.DockerReference()) - - skipNotarySig, err := canSkipNotarySignature(localRepo, tag, upstreamImageDigest.String(), - refs, imageStore, log) - if err != nil { - log.Error().Err(err).Msgf("couldn't check if the upstream image %s notary signature can be skipped", - upstreamImageRef.DockerReference()) - } - - if !skipNotarySig { - if err = retry.RetryIfNecessary(ctx, func() error { - err = syncNotarySignature(httpClient, imageStore, *registryURL, localRepo, remoteRepoCopy, - upstreamImageDigest.String(), refs, log) - - return err - }, retryOptions); err != nil { - log.Error().Err(err).Msgf("couldn't copy notary signature for %s", upstreamImageRef.DockerReference()) - } - } - - skipCosignSig, err := canSkipCosignSignature(localRepo, tag, upstreamImageDigest.String(), - cosignManifest, imageStore, log) - if err != nil { - log.Error().Err(err).Msgf("couldn't check if the upstream image %s cosign signature can be skipped", - upstreamImageRef.DockerReference()) - } - - if !skipCosignSig { - if err = retry.RetryIfNecessary(ctx, func() error { - err = syncCosignSignature(httpClient, imageStore, *registryURL, localRepo, remoteRepoCopy, - upstreamImageDigest.String(), cosignManifest, log) - - return err - }, retryOptions); err != nil { - log.Error().Err(err).Msgf("couldn't copy cosign signature for %s", upstreamImageRef.DockerReference()) - } - } - - continue } - localImageRef, err := getLocalImageRef(localCachePath, localRepo, tag) - if err != nil { - log.Error().Str("errorType", TypeOf(err)). - Err(err).Msgf("couldn't obtain a valid image reference for reference %s/%s:%s", - localCachePath, localRepo, tag) - - return err - } - - log.Info().Msgf("copying image %s to %s", upstreamImageRef.DockerReference(), localCachePath) - + // sync signatures if err = retry.RetryIfNecessary(ctx, func() error { - _, err = copy.Image(ctx, policyCtx, localImageRef, upstreamImageRef, &options) + err = sig.syncNotarySignature(localRepo, upstreamRepo, upstreamImageDigest.String(), refs) + if err != nil { + return err + } - return err - }, retryOptions); err != nil { - log.Error().Str("errorType", TypeOf(err)). - Err(err).Msgf("error while copying image %s to %s", - upstreamImageRef.DockerReference(), localCachePath) + err = sig.syncCosignSignature(localRepo, upstreamRepo, upstreamImageDigest.String(), cosignManifest) + if err != nil { + return err + } - return err - } - // push from cache to repo - err = pushSyncedLocalImage(localRepo, tag, localCachePath, imageStore, log) - if err != nil { - log.Error().Str("errorType", TypeOf(err)). - Err(err).Msgf("error while pushing synced cached image %s", - fmt.Sprintf("%s/%s:%s", localCachePath, localRepo, tag)) - - return err - } - - refs, err = getNotaryRefs(httpClient, *registryURL, remoteRepoCopy, upstreamImageDigest.String(), log) - if err = retry.RetryIfNecessary(ctx, func() error { - err = syncNotarySignature(httpClient, imageStore, *registryURL, localRepo, - remoteRepoCopy, upstreamImageDigest.String(), refs, log) - - return err + return nil }, retryOptions); err != nil { log.Error().Str("errorType", TypeOf(err)). Err(err).Msgf("couldn't copy notary signature for %s", upstreamImageRef.DockerReference()) } - - cosignManifest, err = getCosignManifest(httpClient, *registryURL, remoteRepoCopy, - upstreamImageDigest.String(), log) - if err = retry.RetryIfNecessary(ctx, func() error { - err = syncCosignSignature(httpClient, imageStore, *registryURL, localRepo, - remoteRepoCopy, upstreamImageDigest.String(), cosignManifest, log) - - return err - }, retryOptions); err != nil { - log.Error().Str("errorType", TypeOf(err)). - Err(err).Msgf("couldn't copy cosign signature for %s", upstreamImageRef.DockerReference()) - } } } diff --git a/pkg/extensions/sync/sync_internal_test.go b/pkg/extensions/sync/sync_internal_test.go index 72444491..00d902e0 100644 --- a/pkg/extensions/sync/sync_internal_test.go +++ b/pkg/extensions/sync/sync_internal_test.go @@ -290,14 +290,13 @@ func TestSyncInternal(t *testing.T) { }) Convey("Test imagesToCopyFromUpstream()", t, func() { - repos := []string{"repo1"} upstreamCtx := &types.SystemContext{} - _, err := imagesToCopyFromUpstream(context.Background(), "localhost:4566", repos, upstreamCtx, + _, err := imagesToCopyFromUpstream(context.Background(), "localhost:4566", "repo1", upstreamCtx, Content{}, log.NewLogger("debug", "")) So(err, ShouldNotBeNil) - _, err = imagesToCopyFromUpstream(context.Background(), "docker://localhost:4566", repos, upstreamCtx, + _, err = imagesToCopyFromUpstream(context.Background(), "docker://localhost:4566", "repo1", upstreamCtx, Content{}, log.NewLogger("debug", "")) So(err, ShouldNotBeNil) }) @@ -323,16 +322,15 @@ func TestSyncInternal(t *testing.T) { Layers: []ispec.Descriptor{desc}, } - err = syncCosignSignature(client, &local.ImageStoreLocal{}, *regURL, testImage, testImage, - testImageTag, &ispec.Manifest{}, log) + sig := newSignaturesCopier(client, *regURL, storage.StoreController{DefaultStore: &local.ImageStoreLocal{}}, log) + + err = sig.syncCosignSignature(testImage, testImage, testImageTag, &ispec.Manifest{}) So(err, ShouldNotBeNil) - err = syncCosignSignature(client, &local.ImageStoreLocal{}, *regURL, testImage, testImage, - testImageTag, &manifest, log) + err = sig.syncCosignSignature(testImage, testImage, testImageTag, &manifest) So(err, ShouldNotBeNil) - err = syncNotarySignature(client, &local.ImageStoreLocal{}, *regURL, testImage, testImage, - "invalidDigest", ReferenceList{[]artifactspec.Descriptor{ref}}, log) + err = sig.syncNotarySignature(testImage, testImage, "invalidDigest", ReferenceList{[]artifactspec.Descriptor{ref}}) So(err, ShouldNotBeNil) }) @@ -370,16 +368,20 @@ func TestSyncInternal(t *testing.T) { So(err, ShouldBeNil) So(testImageManifestDigest, ShouldNotBeEmpty) - canBeSkipped, err = canSkipNotarySignature(testImage, testImageTag, - testImageManifestDigest, refs, imageStore, log) + regURL, err := url.Parse("http://zot") + So(err, ShouldBeNil) + So(regURL, ShouldNotBeNil) + + sig := newSignaturesCopier(resty.New(), *regURL, storage.StoreController{DefaultStore: imageStore}, log) + + canBeSkipped, err = sig.canSkipNotarySignature(testImage, testImageManifestDigest, refs) So(err, ShouldBeNil) So(canBeSkipped, ShouldBeFalse) err = os.Chmod(path.Join(imageStore.RootDir(), testImage, "index.json"), 0o000) So(err, ShouldBeNil) - canBeSkipped, err = canSkipNotarySignature(testImage, testImageTag, - testImageManifestDigest, refs, imageStore, log) + canBeSkipped, err = sig.canSkipNotarySignature(testImage, testImageManifestDigest, refs) So(err, ShouldNotBeNil) So(canBeSkipped, ShouldBeFalse) @@ -390,8 +392,7 @@ func TestSyncInternal(t *testing.T) { err = os.Chmod(path.Join(imageStore.RootDir(), testImage, "index.json"), 0o755) So(err, ShouldBeNil) - canBeSkipped, err = canSkipCosignSignature(testImage, testImageTag, - testImageManifestDigest, &cosignManifest, imageStore, log) + canBeSkipped, err = sig.canSkipCosignSignature(testImage, testImageManifestDigest, &cosignManifest) So(err, ShouldBeNil) So(canBeSkipped, ShouldBeFalse) }) @@ -424,6 +425,13 @@ func TestSyncInternal(t *testing.T) { So(len(filteredRepos), ShouldEqual, 0) }) + Convey("Test filterTagsByRegex()", t, func() { + tags := []string{"one"} + filteredTags, err := filterTagsByRegex(tags, ".*", log.NewLogger("", "")) + So(err, ShouldBeNil) + So(filteredTags, ShouldResemble, tags) + }) + Convey("Verify pushSyncedLocalImage func", t, func() { storageDir := t.TempDir() diff --git a/pkg/extensions/sync/utils.go b/pkg/extensions/sync/utils.go index 88d57c2e..483f3ff0 100644 --- a/pkg/extensions/sync/utils.go +++ b/pkg/extensions/sync/utils.go @@ -1,6 +1,7 @@ package sync import ( + "context" "crypto/tls" "crypto/x509" "encoding/json" @@ -9,8 +10,10 @@ import ( "net/url" "os" "path" + "regexp" "strings" + "github.com/Masterminds/semver" glob "github.com/bmatcuk/doublestar/v4" "github.com/containers/image/v5/docker" "github.com/containers/image/v5/docker/reference" @@ -49,6 +52,65 @@ func getTagFromRef(ref types.ImageReference, log log.Logger) reference.Tagged { return tagged } +// getImageTags lists all tags in a repository. +// It returns a string slice of tags and any error encountered. +func getImageTags(ctx context.Context, sysCtx *types.SystemContext, repoRef reference.Named) ([]string, error) { + dockerRef, err := docker.NewReference(reference.TagNameOnly(repoRef)) + // hard to reach test case, injected error, see pkg/test/dev.go + if err = test.Error(err); err != nil { + return nil, err // Should never happen for a reference with tag and no digest + } + + tags, err := docker.GetRepositoryTags(ctx, sysCtx, dockerRef) + if err != nil { + return nil, err + } + + return tags, nil +} + +// filterTagsByRegex filters images by tag regex given in the config. +func filterTagsByRegex(tags []string, regex string, log log.Logger) ([]string, error) { + filteredTags := []string{} + + if len(tags) == 0 || regex == "" { + return filteredTags, nil + } + + log.Info().Msgf("start filtering using the regular expression: %s", regex) + + tagReg, err := regexp.Compile(regex) + if err != nil { + log.Error().Err(err).Str("regex", regex).Msg("couldn't compile regex") + + return filteredTags, err + } + + for _, tag := range tags { + if tagReg.MatchString(tag) { + filteredTags = append(filteredTags, tag) + } + } + + return filteredTags, nil +} + +// filterTagsBySemver filters tags by checking if they are semver compliant. +func filterTagsBySemver(tags []string, log log.Logger) []string { + filteredTags := []string{} + + log.Info().Msg("start filtering using semver compliant rule") + + for _, tag := range tags { + _, err := semver.NewVersion(tag) + if err == nil { + filteredTags = append(filteredTags, tag) + } + } + + return filteredTags +} + // parseRepositoryReference parses input into a reference.Named, and verifies that it names a repository, not an image. func parseRepositoryReference(input string) (reference.Named, error) { ref, err := reference.ParseNormalizedNamed(input)