lint: upgrade golangci-lint

Signed-off-by: Ramkumar Chinchani <rchincha@cisco.com>
This commit is contained in:
Ramkumar Chinchani
2021-12-13 19:23:31 +00:00
committed by Ravi Chamarthy
parent 5f04092e71
commit ac3801ea2d
71 changed files with 3038 additions and 2575 deletions
+8 -6
View File
@@ -16,7 +16,7 @@ type PostHandler struct {
Log log.Logger
}
func (h *PostHandler) Handler(w http.ResponseWriter, r *http.Request) {
func (h *PostHandler) Handler(response http.ResponseWriter, request *http.Request) {
var credentialsFile CredentialsFile
var err error
@@ -25,7 +25,7 @@ func (h *PostHandler) Handler(w http.ResponseWriter, r *http.Request) {
credentialsFile, err = getFileCredentials(h.Cfg.CredentialsFile)
if err != nil {
h.Log.Error().Err(err).Msgf("sync http handler: couldn't get registry credentials from %s", h.Cfg.CredentialsFile)
WriteData(w, http.StatusInternalServerError, err.Error())
WriteData(response, http.StatusInternalServerError, err.Error())
return
}
@@ -33,7 +33,7 @@ func (h *PostHandler) Handler(w http.ResponseWriter, r *http.Request) {
localCtx, policyCtx, err := getLocalContexts(h.Log)
if err != nil {
WriteData(w, http.StatusInternalServerError, err.Error())
WriteData(response, http.StatusInternalServerError, err.Error())
return
}
@@ -42,7 +42,7 @@ func (h *PostHandler) Handler(w http.ResponseWriter, r *http.Request) {
uuid, err := guuid.NewV4()
if err != nil {
WriteData(w, http.StatusInternalServerError, err.Error())
WriteData(response, http.StatusInternalServerError, err.Error())
return
}
@@ -51,12 +51,14 @@ func (h *PostHandler) Handler(w http.ResponseWriter, r *http.Request) {
// if content not provided, don't run periodically sync
if len(regCfg.Content) == 0 {
h.Log.Info().Msgf("sync config content not configured for %s, will not run periodically sync", regCfg.URL)
continue
}
// if pollInterval is not provided, don't run periodically sync
if regCfg.PollInterval == 0 {
h.Log.Warn().Msgf("sync config PollInterval not configured for %s, will not run periodically sync", regCfg.URL)
continue
}
@@ -65,13 +67,13 @@ func (h *PostHandler) Handler(w http.ResponseWriter, r *http.Request) {
if err := syncRegistry(regCfg, h.StoreController, h.Log, localCtx, policyCtx,
credentialsFile[upstreamRegistryName], uuid.String()); err != nil {
h.Log.Err(err).Msg("sync http handler: error while syncing in")
WriteData(w, http.StatusInternalServerError, err.Error())
WriteData(response, http.StatusInternalServerError, err.Error())
return
}
}
WriteData(w, http.StatusOK, "")
WriteData(response, http.StatusOK, "")
}
func WriteData(w http.ResponseWriter, status int, msg string) {
+11 -1
View File
@@ -27,6 +27,7 @@ func OneImage(cfg Config, storeController storage.StoreController,
credentialsFile, err = getFileCredentials(cfg.CredentialsFile)
if err != nil {
log.Error().Err(err).Msgf("couldn't get registry credentials from %s", cfg.CredentialsFile)
return err
}
}
@@ -48,6 +49,7 @@ func OneImage(cfg Config, storeController storage.StoreController,
for _, regCfg := range cfg.Registries {
if !regCfg.OnDemand {
log.Info().Msgf("skipping syncing on demand from %s, onDemand flag is false", regCfg.URL)
continue
}
@@ -57,6 +59,7 @@ func OneImage(cfg Config, storeController storage.StoreController,
if len(repos) == 0 {
log.Info().Msgf("skipping syncing on demand %s from %s registry because it's filtered out by content config",
repo, regCfg.URL)
continue
}
}
@@ -71,6 +74,7 @@ func OneImage(cfg Config, storeController storage.StoreController,
upstreamRepoRef, err := parseRepositoryReference(fmt.Sprintf("%s/%s", upstreamRegistryName, repo))
if err != nil {
log.Error().Err(err).Msgf("error parsing repository reference %s/%s", upstreamRegistryName, repo)
return err
}
@@ -78,6 +82,7 @@ func OneImage(cfg Config, storeController storage.StoreController,
if err != nil {
log.Error().Err(err).Msgf("error creating a reference for repository %s and tag %q",
upstreamRepoRef.Name(), tag)
return err
}
@@ -85,6 +90,7 @@ func OneImage(cfg Config, storeController storage.StoreController,
if err != nil {
log.Error().Err(err).Msgf("error creating docker reference for repository %s and tag %q",
upstreamRepoRef.Name(), tag)
return err
}
@@ -92,8 +98,9 @@ func OneImage(cfg Config, storeController storage.StoreController,
localRepo := path.Join(imageStore.RootDir(), imageName, SyncBlobUploadDir, uuid.String(), imageName)
if err = os.MkdirAll(localRepo, 0755); err != nil {
if err = os.MkdirAll(localRepo, storage.DefaultDirPerms); err != nil {
log.Error().Err(err).Str("dir", localRepo).Msg("couldn't create temporary dir")
return err
}
@@ -104,6 +111,7 @@ func OneImage(cfg Config, storeController storage.StoreController,
localRef, err := layout.ParseReference(localTaggedRepo)
if err != nil {
log.Error().Err(err).Msgf("cannot obtain a valid image reference for reference %q", localRepo)
return err
}
@@ -118,6 +126,7 @@ func OneImage(cfg Config, storeController storage.StoreController,
if err = retry.RetryIfNecessary(context.Background(), func() error {
_, copyErr = copy.Image(context.Background(), policyCtx, localRef, upstreamRef, &options)
return err
}, retryOptions); copyErr != nil {
log.Error().Err(copyErr).Msgf("error while copying image %s to %s",
@@ -129,6 +138,7 @@ func OneImage(cfg Config, storeController storage.StoreController,
if err != nil {
log.Error().Err(err).Msgf("error while pushing synced cached image %s",
localTaggedRepo)
return err
}
+41 -18
View File
@@ -90,17 +90,19 @@ func getUpstreamCatalog(regCfg *RegistryConfig, credentials Credentials, log log
caCert, err := ioutil.ReadFile(caCertPath)
if err != nil {
log.Error().Err(err).Msg("couldn't read CA certificate")
return c, err
}
caCertPool := x509.NewCertPool()
caCertPool.AppendCertsFromPEM(caCert)
client.SetTLSClientConfig(&tls.Config{RootCAs: caCertPool})
client.SetTLSClientConfig(&tls.Config{RootCAs: caCertPool, MinVersion: tls.VersionTLS12})
cert, err := tls.LoadX509KeyPair(clientCert, clientKey)
if err != nil {
log.Error().Err(err).Msg("couldn't read certificates key pairs")
return c, err
}
@@ -120,18 +122,21 @@ func getUpstreamCatalog(regCfg *RegistryConfig, credentials Credentials, log log
resp, err := client.R().SetHeader("Content-Type", "application/json").Get(registryCatalogURL)
if err != nil {
log.Err(err).Msgf("couldn't query %s", registryCatalogURL)
return c, err
}
if resp.IsError() {
log.Error().Msgf("couldn't query %s, status code: %d, body: %s", registryCatalogURL,
resp.StatusCode(), resp.Body())
return c, errors.ErrSyncMissingCatalog
}
err = json.Unmarshal(resp.Body(), &c)
if err != nil {
log.Err(err).Str("body", string(resp.Body())).Msg("couldn't unmarshal registry's catalog")
return c, err
}
@@ -171,19 +176,19 @@ func filterImagesByTagRegex(upstreamReferences *[]types.ImageReference, content
return err
}
n := 0
numTags := 0
for _, ref := range refs {
tagged := getTagFromRef(ref, log)
if tagged != nil {
if tagReg.MatchString(tagged.Tag()) {
refs[n] = ref
n++
refs[numTags] = ref
numTags++
}
}
}
refs = refs[:n]
refs = refs[:numTags]
}
*upstreamReferences = refs
@@ -202,20 +207,20 @@ func filterImagesBySemver(upstreamReferences *[]types.ImageReference, content Co
if content.Tags.Semver != nil && *content.Tags.Semver {
log.Info().Msg("start filtering using semver compliant rule")
n := 0
numTags := 0
for _, ref := range refs {
tagged := getTagFromRef(ref, log)
if tagged != nil {
_, ok := semver.NewVersion(tagged.Tag())
if ok == nil {
refs[n] = ref
n++
refs[numTags] = ref
numTags++
}
}
}
refs = refs[:n]
refs = refs[:numTags]
}
*upstreamReferences = refs
@@ -230,12 +235,14 @@ func imagesToCopyFromUpstream(registryName string, repos []string, upstreamCtx *
repoRef, err := parseRepositoryReference(fmt.Sprintf("%s/%s", registryName, repoName))
if err != nil {
log.Error().Err(err).Msgf("couldn't parse repository reference: %s", repoRef)
return nil, err
}
tags, err := getImageTags(context.Background(), upstreamCtx, repoRef)
if err != nil {
log.Error().Err(err).Msgf("couldn't fetch tags for %s", repoRef)
return nil, err
}
@@ -243,6 +250,7 @@ func imagesToCopyFromUpstream(registryName string, repos []string, upstreamCtx *
taggedRef, err := reference.WithTag(repoRef, tag)
if err != nil {
log.Err(err).Msgf("error creating a reference for repository %s and tag %q", repoRef.Name(), tag)
return nil, err
}
@@ -250,6 +258,7 @@ func imagesToCopyFromUpstream(registryName string, repos []string, upstreamCtx *
if err != nil {
log.Err(err).Msgf("cannot obtain a valid image reference for transport %q and reference %s",
docker.Transport.Name(), taggedRef.String())
return nil, err
}
@@ -329,9 +338,11 @@ func syncRegistry(regCfg RegistryConfig, storeController storage.StoreController
if err = retry.RetryIfNecessary(context.Background(), func() error {
catalog, err = getUpstreamCatalog(&regCfg, credentials, log)
return err
}, retryOptions); err != nil {
log.Error().Err(err).Msg("error while getting upstream catalog, retrying...")
return err
}
@@ -352,15 +363,18 @@ func syncRegistry(regCfg RegistryConfig, storeController storage.StoreController
if err = retry.RetryIfNecessary(context.Background(), func() error {
refs, err := imagesToCopyFromUpstream(upstreamRegistryName, r, upstreamCtx, regCfg.Content[id], log)
images = append(images, refs...)
return err
}, retryOptions); err != nil {
log.Error().Err(err).Msg("error while getting images references from upstream, retrying...")
return err
}
}
if len(images) == 0 {
log.Info().Msg("no images to copy, no need to sync")
return nil
}
@@ -374,8 +388,9 @@ func syncRegistry(regCfg RegistryConfig, storeController storage.StoreController
localRepo := path.Join(imageStore.RootDir(), imageName, SyncBlobUploadDir, uuid, imageName)
if err = os.MkdirAll(localRepo, 0755); err != nil {
if err = os.MkdirAll(localRepo, storage.DefaultDirPerms); err != nil {
log.Error().Err(err).Str("dir", localRepo).Msg("couldn't create temporary dir")
return err
}
@@ -388,6 +403,7 @@ func syncRegistry(regCfg RegistryConfig, storeController storage.StoreController
localRef, err := layout.ParseReference(localTaggedRepo)
if err != nil {
log.Error().Err(err).Msgf("Cannot obtain a valid image reference for reference %q", localTaggedRepo)
return err
}
@@ -396,10 +412,12 @@ func syncRegistry(regCfg RegistryConfig, storeController storage.StoreController
if err = retry.RetryIfNecessary(context.Background(), func() error {
_, err = copy.Image(context.Background(), policyCtx, localRef, upstreamRef, &options)
return err
}, retryOptions); err != nil {
log.Error().Err(err).Msgf("error while copying image %s:%s to %s",
upstreamRef.DockerReference().Name(), upstreamTaggedRef.Tag(), localTaggedRepo)
return err
}
@@ -409,6 +427,7 @@ func syncRegistry(regCfg RegistryConfig, storeController storage.StoreController
if err != nil {
log.Error().Err(err).Msgf("error while pushing synced cached image %s",
localTaggedRepo)
return err
}
}
@@ -433,13 +452,14 @@ func getLocalContexts(log log.Logger) (*types.SystemContext, *signature.PolicyCo
policyContext, err := signature.NewPolicyContext(policy)
if err != nil {
log.Error().Err(err).Msg("couldn't create policy context")
return &types.SystemContext{}, &signature.PolicyContext{}, err
}
return localCtx, policyContext, nil
}
func Run(cfg Config, storeController storage.StoreController, wg *goSync.WaitGroup, logger log.Logger) error {
func Run(cfg Config, storeController storage.StoreController, wtgrp *goSync.WaitGroup, logger log.Logger) error {
var credentialsFile CredentialsFile
var err error
@@ -448,6 +468,7 @@ func Run(cfg Config, storeController storage.StoreController, wg *goSync.WaitGro
credentialsFile, err = getFileCredentials(cfg.CredentialsFile)
if err != nil {
logger.Error().Err(err).Msgf("couldn't get registry credentials from %s", cfg.CredentialsFile)
return err
}
}
@@ -467,38 +488,40 @@ func Run(cfg Config, storeController storage.StoreController, wg *goSync.WaitGro
// if content not provided, don't run periodically sync
if len(regCfg.Content) == 0 {
logger.Info().Msgf("sync config content not configured for %s, will not run periodically sync", regCfg.URL)
continue
}
// if pollInterval is not provided, don't run periodically sync
if regCfg.PollInterval == 0 {
logger.Warn().Msgf("sync config PollInterval not configured for %s, will not run periodically sync", regCfg.URL)
continue
}
ticker := time.NewTicker(regCfg.PollInterval)
// fork a new zerolog child to avoid data race
l := log.Logger{Logger: logger.With().Caller().Timestamp().Logger()}
tlogger := log.Logger{Logger: logger.With().Caller().Timestamp().Logger()}
upstreamRegistry := strings.Replace(strings.Replace(regCfg.URL, "http://", "", 1), "https://", "", 1)
// schedule each registry sync
go func(regCfg RegistryConfig, l log.Logger) {
go func(regCfg RegistryConfig, logger log.Logger) {
// run on intervals
for ; true; <-ticker.C {
// increment reference since will be busy, so shutdown has to wait
wg.Add(1)
wtgrp.Add(1)
if err := syncRegistry(regCfg, storeController, l, localCtx, policyCtx,
if err := syncRegistry(regCfg, storeController, logger, localCtx, policyCtx,
credentialsFile[upstreamRegistry], uuid.String()); err != nil {
l.Error().Err(err).Msg("sync exited with error, stopping it...")
logger.Error().Err(err).Msg("sync exited with error, stopping it...")
ticker.Stop()
}
// mark as done after a single sync run
wg.Done()
wtgrp.Done()
}
}(regCfg, l)
}(regCfg, tlogger)
}
logger.Info().Msg("finished setting up sync")
+15 -15
View File
@@ -54,17 +54,17 @@ func TestSyncInternal(t *testing.T) {
_, err = getFileCredentials("/path/to/inexistent/file")
So(err, ShouldNotBeNil)
f, err := ioutil.TempFile("", "sync-credentials-")
tempFile, err := ioutil.TempFile("", "sync-credentials-")
if err != nil {
panic(err)
}
content := []byte(`{`)
if err := ioutil.WriteFile(f.Name(), content, 0600); err != nil {
if err := ioutil.WriteFile(tempFile.Name(), content, 0o600); err != nil {
panic(err)
}
_, err = getFileCredentials(f.Name())
_, err = getFileCredentials(tempFile.Name())
So(err, ShouldNotBeNil)
srcCtx := &types.SystemContext{}
@@ -80,7 +80,7 @@ func TestSyncInternal(t *testing.T) {
dockerRef, err := docker.NewReference(taggedRef)
So(err, ShouldBeNil)
//tag := getTagFromRef(dockerRef, log.NewLogger("", ""))
// tag := getTagFromRef(dockerRef, log.NewLogger("", ""))
So(getTagFromRef(dockerRef, log.NewLogger("debug", "")), ShouldNotBeNil)
@@ -133,7 +133,7 @@ func TestSyncInternal(t *testing.T) {
panic(err)
}
if err := os.WriteFile(path.Join(badCertsDir, "ca.crt"), []byte("certificate"), 0755); err != nil {
if err := os.WriteFile(path.Join(badCertsDir, "ca.crt"), []byte("certificate"), 0o600); err != nil {
panic(err)
}
@@ -217,9 +217,9 @@ func TestSyncInternal(t *testing.T) {
So(err, ShouldNotBeNil)
testRootDir := path.Join(imageStore.RootDir(), testImage, SyncBlobUploadDir)
//testImagePath := path.Join(testRootDir, testImage)
// testImagePath := path.Join(testRootDir, testImage)
err = os.MkdirAll(testRootDir, 0755)
err = os.MkdirAll(testRootDir, 0o755)
if err != nil {
panic(err)
}
@@ -239,7 +239,7 @@ func TestSyncInternal(t *testing.T) {
panic(err)
}
if err := os.Chmod(storageDir, 0000); err != nil {
if err := os.Chmod(storageDir, 0o000); err != nil {
panic(err)
}
@@ -250,12 +250,12 @@ func TestSyncInternal(t *testing.T) {
ShouldPanic)
}
if err := os.Chmod(storageDir, 0755); err != nil {
if err := os.Chmod(storageDir, 0o755); err != nil {
panic(err)
}
if err := os.Chmod(path.Join(testRootDir, testImage, "blobs", "sha256",
manifest.Layers[0].Digest.Hex()), 0000); err != nil {
manifest.Layers[0].Digest.Hex()), 0o000); err != nil {
panic(err)
}
@@ -263,25 +263,25 @@ func TestSyncInternal(t *testing.T) {
So(err, ShouldNotBeNil)
if err := os.Chmod(path.Join(testRootDir, testImage, "blobs", "sha256",
manifest.Layers[0].Digest.Hex()), 0755); err != nil {
manifest.Layers[0].Digest.Hex()), 0o755); err != nil {
panic(err)
}
cachedManifestConfigPath := path.Join(imageStore.RootDir(), testImage, SyncBlobUploadDir,
testImage, "blobs", "sha256", manifest.Config.Digest.Hex())
if err := os.Chmod(cachedManifestConfigPath, 0000); err != nil {
if err := os.Chmod(cachedManifestConfigPath, 0o000); err != nil {
panic(err)
}
err = pushSyncedLocalImage(testImage, testImageTag, "", storeController, log)
So(err, ShouldNotBeNil)
if err := os.Chmod(cachedManifestConfigPath, 0755); err != nil {
if err := os.Chmod(cachedManifestConfigPath, 0o755); err != nil {
panic(err)
}
manifestConfigPath := path.Join(imageStore.RootDir(), testImage, "blobs", "sha256", manifest.Config.Digest.Hex())
if err := os.MkdirAll(manifestConfigPath, 0000); err != nil {
if err := os.MkdirAll(manifestConfigPath, 0o000); err != nil {
panic(err)
}
@@ -295,7 +295,7 @@ func TestSyncInternal(t *testing.T) {
mDigest := godigest.FromBytes(manifestContent)
manifestPath := path.Join(imageStore.RootDir(), testImage, "blobs", mDigest.Algorithm().String(), mDigest.Encoded())
if err := os.MkdirAll(manifestPath, 0000); err != nil {
if err := os.MkdirAll(manifestPath, 0o000); err != nil {
panic(err)
}
+69 -56
View File
@@ -96,7 +96,7 @@ func startUpstreamServer(secure, basicAuth bool) (*api.Controller, string, strin
caCertPool := x509.NewCertPool()
caCertPool.AppendCertsFromPEM(caCert)
client.SetTLSClientConfig(&tls.Config{RootCAs: caCertPool})
client.SetTLSClientConfig(&tls.Config{RootCAs: caCertPool, MinVersion: tls.VersionTLS12})
cert, err := tls.LoadX509KeyPair("../../../test/data/client.cert", "../../../test/data/client.key")
if err != nil {
@@ -132,11 +132,11 @@ func startUpstreamServer(secure, basicAuth bool) (*api.Controller, string, strin
srcConfig.Storage.RootDirectory = srcDir
sc := api.NewController(srcConfig)
sctlr := api.NewController(srcConfig)
go func() {
// this blocks
if err := sc.Run(); err != nil {
if err := sctlr.Run(); err != nil {
return
}
}()
@@ -151,7 +151,7 @@ func startUpstreamServer(secure, basicAuth bool) (*api.Controller, string, strin
time.Sleep(100 * time.Millisecond)
}
return sc, srcBaseURL, srcDir, htpasswdPath, client
return sctlr, srcBaseURL, srcDir, htpasswdPath, client
}
func startDownstreamServer(secure bool, syncConfig *sync.Config) (*api.Controller, string, string, *resty.Client) {
@@ -179,7 +179,7 @@ func startDownstreamServer(secure bool, syncConfig *sync.Config) (*api.Controlle
caCertPool := x509.NewCertPool()
caCertPool.AppendCertsFromPEM(caCert)
client.SetTLSClientConfig(&tls.Config{RootCAs: caCertPool})
client.SetTLSClientConfig(&tls.Config{RootCAs: caCertPool, MinVersion: tls.VersionTLS12})
cert, err := tls.LoadX509KeyPair("../../../test/data/client.cert", "../../../test/data/client.key")
if err != nil {
@@ -204,11 +204,11 @@ func startDownstreamServer(secure bool, syncConfig *sync.Config) (*api.Controlle
destConfig.Extensions.Search = nil
destConfig.Extensions.Sync = syncConfig
dc := api.NewController(destConfig)
dctlr := api.NewController(destConfig)
go func() {
// this blocks
if err := dc.Run(); err != nil {
if err := dctlr.Run(); err != nil {
return
}
}()
@@ -223,7 +223,7 @@ func startDownstreamServer(secure bool, syncConfig *sync.Config) (*api.Controlle
time.Sleep(100 * time.Millisecond)
}
return dc, destBaseURL, destDir, client
return dctlr, destBaseURL, destDir, client
}
func TestSyncOnDemand(t *testing.T) {
@@ -257,13 +257,14 @@ func TestSyncOnDemand(t *testing.T) {
}
syncConfig := &sync.Config{
Registries: []sync.RegistryConfig{syncRegistryConfig}}
Registries: []sync.RegistryConfig{syncRegistryConfig},
}
dc, destBaseURL, destDir, destClient := startDownstreamServer(false, syncConfig)
dctlr, destBaseURL, destDir, destClient := startDownstreamServer(false, syncConfig)
defer os.RemoveAll(destDir)
defer func() {
dc.Shutdown()
dctlr.Shutdown()
}()
var srcTagsList TagsList
@@ -286,7 +287,7 @@ func TestSyncOnDemand(t *testing.T) {
So(err, ShouldBeNil)
So(resp.StatusCode(), ShouldEqual, 404)
err = os.Chmod(path.Join(destDir, testImage), 0000)
err = os.Chmod(path.Join(destDir, testImage), 0o000)
if err != nil {
panic(err)
}
@@ -295,7 +296,7 @@ func TestSyncOnDemand(t *testing.T) {
So(err, ShouldBeNil)
So(resp.StatusCode(), ShouldEqual, 500)
err = os.Chmod(path.Join(destDir, testImage), 0755)
err = os.Chmod(path.Join(destDir, testImage), 0o755)
if err != nil {
panic(err)
}
@@ -304,7 +305,7 @@ func TestSyncOnDemand(t *testing.T) {
So(err, ShouldBeNil)
So(resp.StatusCode(), ShouldEqual, 404)
err = os.Chmod(path.Join(destDir, testImage, sync.SyncBlobUploadDir), 0000)
err = os.Chmod(path.Join(destDir, testImage, sync.SyncBlobUploadDir), 0o000)
if err != nil {
panic(err)
}
@@ -317,12 +318,12 @@ func TestSyncOnDemand(t *testing.T) {
So(err, ShouldBeNil)
So(resp.StatusCode(), ShouldEqual, 404)
err = os.Chmod(path.Join(destDir, testImage, sync.SyncBlobUploadDir), 0755)
err = os.Chmod(path.Join(destDir, testImage, sync.SyncBlobUploadDir), 0o755)
if err != nil {
panic(err)
}
err = os.MkdirAll(path.Join(destDir, testImage, "blobs"), 0000)
err = os.MkdirAll(path.Join(destDir, testImage, "blobs"), 0o000)
if err != nil {
panic(err)
}
@@ -331,7 +332,7 @@ func TestSyncOnDemand(t *testing.T) {
So(err, ShouldBeNil)
So(resp.StatusCode(), ShouldEqual, 404)
err = os.Chmod(path.Join(destDir, testImage, "blobs"), 0755)
err = os.Chmod(path.Join(destDir, testImage, "blobs"), 0o755)
if err != nil {
panic(err)
}
@@ -357,11 +358,11 @@ func TestSync(t *testing.T) {
Convey("Verify sync feature", t, func() {
updateDuration, _ := time.ParseDuration("30m")
sc, srcBaseURL, srcDir, _, srcClient := startUpstreamServer(false, false)
sctlr, srcBaseURL, srcDir, _, srcClient := startUpstreamServer(false, false)
defer os.RemoveAll(srcDir)
defer func() {
sc.Shutdown()
sctlr.Shutdown()
}()
regex := ".*"
@@ -569,7 +570,7 @@ func TestSyncPermsDenied(t *testing.T) {
dc.Shutdown()
}()
err := os.Chmod(path.Join(destDir, testImage, sync.SyncBlobUploadDir), 0000)
err := os.Chmod(path.Join(destDir, testImage, sync.SyncBlobUploadDir), 0o000)
if err != nil {
panic(err)
}
@@ -776,8 +777,10 @@ func TestSyncBasicAuth(t *testing.T) {
CertDir: "",
}
syncConfig := &sync.Config{CredentialsFile: credentialsFile,
Registries: []sync.RegistryConfig{syncRegistryConfig}}
syncConfig := &sync.Config{
CredentialsFile: credentialsFile,
Registries: []sync.RegistryConfig{syncRegistryConfig},
}
dc, destBaseURL, destDir, destClient := startDownstreamServer(false, syncConfig)
defer os.RemoveAll(destDir)
@@ -872,20 +875,22 @@ func TestSyncBasicAuth(t *testing.T) {
destConfig.Extensions = &extconf.ExtensionConfig{}
destConfig.Extensions.Search = nil
destConfig.Extensions.Sync = &sync.Config{CredentialsFile: credentialsFile,
Registries: []sync.RegistryConfig{syncRegistryConfig}}
destConfig.Extensions.Sync = &sync.Config{
CredentialsFile: credentialsFile,
Registries: []sync.RegistryConfig{syncRegistryConfig},
}
dc := api.NewController(destConfig)
dctlr := api.NewController(destConfig)
go func() {
// this blocks
if err := dc.Run(); err != nil {
if err := dctlr.Run(); err != nil {
return
}
}()
defer func() {
dc.Shutdown()
dctlr.Shutdown()
}()
// wait till ready
@@ -915,11 +920,11 @@ func TestSyncBasicAuth(t *testing.T) {
credentialsFile := makeCredentialsFile(fmt.Sprintf(`{"%s":{"username": "test", "password": "test"}}`,
registryName))
err := os.Chmod(credentialsFile, 0000)
err := os.Chmod(credentialsFile, 0o000)
So(err, ShouldBeNil)
defer func() {
So(os.Chmod(credentialsFile, 0755), ShouldBeNil)
So(os.Chmod(credentialsFile, 0o755), ShouldBeNil)
So(os.RemoveAll(credentialsFile), ShouldBeNil)
}()
@@ -943,8 +948,10 @@ func TestSyncBasicAuth(t *testing.T) {
CertDir: "",
}
syncConfig := &sync.Config{CredentialsFile: credentialsFile,
Registries: []sync.RegistryConfig{syncRegistryConfig}}
syncConfig := &sync.Config{
CredentialsFile: credentialsFile,
Registries: []sync.RegistryConfig{syncRegistryConfig},
}
dc, destBaseURL, destDir, destClient := startDownstreamServer(false, syncConfig)
defer os.RemoveAll(destDir)
@@ -985,16 +992,20 @@ func TestSyncBasicAuth(t *testing.T) {
}
// add file path to the credentials
syncConfig := &sync.Config{CredentialsFile: credentialsFile,
Registries: []sync.RegistryConfig{unreacheableSyncRegistryConfig1,
syncConfig := &sync.Config{
CredentialsFile: credentialsFile,
Registries: []sync.RegistryConfig{
unreacheableSyncRegistryConfig1,
unreacheableSyncRegistryConfig2,
syncRegistryConfig}}
syncRegistryConfig,
},
}
dc, destBaseURL, destDir, destClient := startDownstreamServer(false, syncConfig)
dctlr, destBaseURL, destDir, destClient := startDownstreamServer(false, syncConfig)
defer os.RemoveAll(destDir)
defer func() {
dc.Shutdown()
dctlr.Shutdown()
}()
var srcTagsList TagsList
@@ -1021,7 +1032,7 @@ func TestSyncBasicAuth(t *testing.T) {
So(err, ShouldBeNil)
So(resp.StatusCode(), ShouldEqual, 200)
err = dc.StoreController.DefaultStore.DeleteImageManifest(testImage, testImageTag)
err = dctlr.StoreController.DefaultStore.DeleteImageManifest(testImage, testImageTag)
So(err, ShouldBeNil)
resp, err = destClient.R().Get(destBaseURL + "/v2/" + testImage + "/manifests/" + "1.1.1")
@@ -1300,14 +1311,14 @@ func TestSyncInvalidCerts(t *testing.T) {
panic(err)
}
f, err := os.OpenFile(destFilePath, os.O_TRUNC|os.O_WRONLY|os.O_CREATE, 0600)
dstfile, err := os.OpenFile(destFilePath, os.O_TRUNC|os.O_WRONLY|os.O_CREATE, 0o600)
if err != nil {
panic(err)
}
defer f.Close()
defer dstfile.Close()
if _, err = f.WriteString("Add Invalid Text In Cert"); err != nil {
if _, err = dstfile.WriteString("Add Invalid Text In Cert"); err != nil {
panic(err)
}
@@ -1358,17 +1369,17 @@ func TestSyncInvalidCerts(t *testing.T) {
}
func makeCredentialsFile(fileContent string) string {
f, err := ioutil.TempFile("", "sync-credentials-")
tmpfile, err := ioutil.TempFile("", "sync-credentials-")
if err != nil {
panic(err)
}
content := []byte(fileContent)
if err := ioutil.WriteFile(f.Name(), content, 0600); err != nil {
if err := ioutil.WriteFile(tmpfile.Name(), content, 0o600); err != nil {
panic(err)
}
return f.Name()
return tmpfile.Name()
}
func TestSyncInvalidUrl(t *testing.T) {
@@ -1445,7 +1456,8 @@ func TestSyncInvalidTags(t *testing.T) {
}
syncConfig := &sync.Config{
Registries: []sync.RegistryConfig{syncRegistryConfig}}
Registries: []sync.RegistryConfig{syncRegistryConfig},
}
dc, destBaseURL, destDir, destClient := startDownstreamServer(false, syncConfig)
defer os.RemoveAll(destDir)
@@ -1485,11 +1497,11 @@ func TestSyncSubPaths(t *testing.T) {
srcConfig.Storage.RootDirectory = srcDir
sc := api.NewController(srcConfig)
sctlr := api.NewController(srcConfig)
go func() {
// this blocks
if err := sc.Run(); err != nil {
if err := sctlr.Run(); err != nil {
return
}
}()
@@ -1505,7 +1517,7 @@ func TestSyncSubPaths(t *testing.T) {
}
defer func() {
sc.Shutdown()
sctlr.Shutdown()
}()
regex := ".*"
@@ -1530,7 +1542,8 @@ func TestSyncSubPaths(t *testing.T) {
}
syncConfig := &sync.Config{
Registries: []sync.RegistryConfig{syncRegistryConfig}}
Registries: []sync.RegistryConfig{syncRegistryConfig},
}
destPort := GetFreePort()
destConfig := config.New()
@@ -1564,11 +1577,11 @@ func TestSyncSubPaths(t *testing.T) {
destConfig.Extensions.Search = nil
destConfig.Extensions.Sync = syncConfig
dc := api.NewController(destConfig)
dctlr := api.NewController(destConfig)
go func() {
// this blocks
if err := dc.Run(); err != nil {
if err := dctlr.Run(); err != nil {
return
}
}()
@@ -1584,7 +1597,7 @@ func TestSyncSubPaths(t *testing.T) {
}
defer func() {
dc.Shutdown()
dctlr.Shutdown()
}()
var destTagsList TagsList
@@ -1608,13 +1621,13 @@ func TestSyncSubPaths(t *testing.T) {
}
// synced image should get into subpath instead of rootDir
fi, err := os.Stat(path.Join(subPathDestDir, subpath, testImage, "blobs/sha256"))
So(fi, ShouldNotBeNil)
binfo, err := os.Stat(path.Join(subPathDestDir, subpath, testImage, "blobs/sha256"))
So(binfo, ShouldNotBeNil)
So(err, ShouldBeNil)
// check rootDir is not populated with any image.
fi, err = os.Stat(path.Join(destDir, subpath))
So(fi, ShouldBeNil)
binfo, err = os.Stat(path.Join(destDir, subpath))
So(binfo, ShouldBeNil)
So(err, ShouldNotBeNil)
})
}
@@ -1636,7 +1649,7 @@ func TestSyncOnDemandContentFiltering(t *testing.T) {
syncRegistryConfig := sync.RegistryConfig{
Content: []sync.Content{
{
//should be filtered out
// should be filtered out
Prefix: "dummy",
Tags: &sync.Tags{
Regex: &regex,
+17 -7
View File
@@ -42,28 +42,30 @@ func parseRepositoryReference(input string) (reference.Named, error) {
}
// filterRepos filters repos based on prefix given in the config.
func filterRepos(repos []string, content []Content, log log.Logger) map[int][]string {
func filterRepos(repos []string, contentList []Content, log log.Logger) map[int][]string {
filtered := make(map[int][]string)
for _, repo := range repos {
for contentID, c := range content {
for contentID, content := range contentList {
var prefix string
// handle prefixes starting with '/'
if strings.HasPrefix(c.Prefix, "/") {
prefix = c.Prefix[1:]
if strings.HasPrefix(content.Prefix, "/") {
prefix = content.Prefix[1:]
} else {
prefix = c.Prefix
prefix = content.Prefix
}
matched, err := glob.Match(prefix, repo)
if err != nil {
log.Error().Err(err).Str("pattern",
prefix).Msg("error while parsing glob pattern, skipping it...")
continue
}
if matched {
filtered[contentID] = append(filtered[contentID], repo)
break
}
}
@@ -74,14 +76,14 @@ func filterRepos(repos []string, content []Content, log log.Logger) map[int][]st
// Get sync.FileCredentials from file.
func getFileCredentials(filepath string) (CredentialsFile, error) {
f, err := ioutil.ReadFile(filepath)
credsFile, err := ioutil.ReadFile(filepath)
if err != nil {
return nil, err
}
var creds CredentialsFile
err = json.Unmarshal(f, &creds)
err = json.Unmarshal(credsFile, &creds)
if err != nil {
return nil, err
}
@@ -102,6 +104,7 @@ func pushSyncedLocalImage(repo, tag, uuid string,
manifestContent, _, _, err := cacheImageStore.GetImageManifest(repo, tag)
if err != nil {
log.Error().Err(err).Str("dir", path.Join(cacheImageStore.RootDir(), repo)).Msg("couldn't find index.json")
return err
}
@@ -109,6 +112,7 @@ func pushSyncedLocalImage(repo, tag, uuid string,
if err := json.Unmarshal(manifestContent, &manifest); err != nil {
log.Error().Err(err).Str("dir", path.Join(cacheImageStore.RootDir(), repo)).Msg("invalid JSON")
return err
}
@@ -117,12 +121,14 @@ func pushSyncedLocalImage(repo, tag, uuid string,
if err != nil {
log.Error().Err(err).Str("dir", path.Join(cacheImageStore.RootDir(),
repo)).Str("blob digest", blob.Digest.String()).Msg("couldn't read blob")
return err
}
_, _, err = imageStore.FullBlobUpload(repo, blobReader, blob.Digest.String())
if err != nil {
log.Error().Err(err).Str("blob digest", blob.Digest.String()).Msg("couldn't upload blob")
return err
}
}
@@ -131,18 +137,21 @@ func pushSyncedLocalImage(repo, tag, uuid string,
if err != nil {
log.Error().Err(err).Str("dir", path.Join(cacheImageStore.RootDir(),
repo)).Str("blob digest", manifest.Config.Digest.String()).Msg("couldn't read config blob")
return err
}
_, _, err = imageStore.FullBlobUpload(repo, blobReader, manifest.Config.Digest.String())
if err != nil {
log.Error().Err(err).Str("blob digest", manifest.Config.Digest.String()).Msg("couldn't upload config blob")
return err
}
_, err = imageStore.PutImageManifest(repo, tag, ispec.MediaTypeImageManifest, manifestContent)
if err != nil {
log.Error().Err(err).Msg("couldn't upload manifest")
return err
}
@@ -150,6 +159,7 @@ func pushSyncedLocalImage(repo, tag, uuid string,
if err := os.RemoveAll(path.Join(cacheImageStore.RootDir(), repo)); err != nil {
log.Error().Err(err).Msg("couldn't remove locally cached sync repo")
return err
}