diff --git a/examples/README.md b/examples/README.md index 23d7322f..a974ab07 100644 --- a/examples/README.md +++ b/examples/README.md @@ -357,10 +357,10 @@ Configure each registry sync: "registries": [{ "url": "https://registry1:5000", "onDemand": false, # pull any image which the local registry doesn't have - "pollInterval": "6h", # polling interval + "pollInterval": "6h", # polling interval, if not set then periodically polling will not run "tlsVerify": true, # whether or not to verify tls "certDir": "/home/user/certs", # use certificates at certDir path, if not specified then use the default certs dir - "content":[ # which content to periodically pull + "content":[ # which content to periodically pull, also it's used for filtering ondemand images, if not set then periodically polling will not run { "prefix":"/repo1/repo", # pull image repo1/repo "tags":{ # filter by tags diff --git a/pkg/cli/root_test.go b/pkg/cli/root_test.go index 963d9cbb..97419552 100644 --- a/pkg/cli/root_test.go +++ b/pkg/cli/root_test.go @@ -93,6 +93,37 @@ func TestVerify(t *testing.T) { So(func() { _ = cli.NewRootCmd().Execute() }, ShouldPanic) }) + Convey("Test verify storage driver different than s3", t, func(c C) { + tmpfile, err := ioutil.TempFile("", "zot-test*.json") + So(err, ShouldBeNil) + defer os.Remove(tmpfile.Name()) // clean up + content := []byte(`{"storage":{"rootDirectory":"/tmp/zot", "storageDriver": {"name": "gcs"}}, + "http":{"address":"127.0.0.1","port":"8080","realm":"zot", + "auth":{"htpasswd":{"path":"test/data/htpasswd"},"failDelay":1}}}`) + _, err = tmpfile.Write(content) + So(err, ShouldBeNil) + err = tmpfile.Close() + So(err, ShouldBeNil) + os.Args = []string{"cli_test", "verify", tmpfile.Name()} + So(func() { _ = cli.NewRootCmd().Execute() }, ShouldPanic) + }) + + Convey("Test verify subpath storage driver different than s3", t, func(c C) { + tmpfile, err := ioutil.TempFile("", "zot-test*.json") + So(err, ShouldBeNil) + defer os.Remove(tmpfile.Name()) // clean up + content := []byte(`{"storage":{"rootDirectory":"/tmp/zot", "storageDriver": {"name": "s3"}, + "subPaths": {"/a": {"rootDirectory": "/zot-a","storageDriver": {"name": "gcs"}}}}, + "http":{"address":"127.0.0.1","port":"8080","realm":"zot", + "auth":{"htpasswd":{"path":"test/data/htpasswd"},"failDelay":1}}}`) + _, err = tmpfile.Write(content) + So(err, ShouldBeNil) + err = tmpfile.Close() + So(err, ShouldBeNil) + os.Args = []string{"cli_test", "verify", tmpfile.Name()} + So(func() { _ = cli.NewRootCmd().Execute() }, ShouldPanic) + }) + Convey("Test verify w/ authorization and w/o authentication", t, func(c C) { tmpfile, err := ioutil.TempFile("", "zot-test*.json") So(err, ShouldBeNil) diff --git a/pkg/extensions/extensions.go b/pkg/extensions/extensions.go index 5a426ca9..11fb2e97 100644 --- a/pkg/extensions/extensions.go +++ b/pkg/extensions/extensions.go @@ -72,19 +72,6 @@ func EnableExtensions(config *config.Config, log log.Logger, rootDir string) { func EnableSyncExtension(config *config.Config, wg *goSync.WaitGroup, storeController storage.StoreController, log log.Logger) { if config.Extensions.Sync != nil { - defaultPollInterval, _ := time.ParseDuration("1h") - for id, registryCfg := range config.Extensions.Sync.Registries { - if registryCfg.Content != nil && - len(registryCfg.Content) > 0 && - registryCfg.PollInterval < defaultPollInterval { - config.Extensions.Sync.Registries[id].PollInterval = defaultPollInterval - - log.Warn().Str("registry", registryCfg.URL). - Msg("Sync registries interval set to too-short interval < 1h," + - "changing update duration to 1 hour and continuing.") - } - } - if err := sync.Run(*config.Extensions.Sync, storeController, wg, log); err != nil { log.Error().Err(err).Msg("Error encountered while setting up syncing") } diff --git a/pkg/extensions/sync/http_handler.go b/pkg/extensions/sync/http_handler.go index 6d1b01f7..c5993a76 100644 --- a/pkg/extensions/sync/http_handler.go +++ b/pkg/extensions/sync/http_handler.go @@ -48,8 +48,15 @@ func (h *PostHandler) Handler(w http.ResponseWriter, r *http.Request) { } for _, regCfg := range h.Cfg.Registries { + // if content not provided, don't run periodically sync if len(regCfg.Content) == 0 { - h.Log.Info().Msgf("no content found for %s, will not run periodically sync", regCfg.URL) + h.Log.Info().Msgf("sync config content not configured for %s, will not run periodically sync", regCfg.URL) + continue + } + + // if pollInterval is not provided, don't run periodically sync + if regCfg.PollInterval == 0 { + h.Log.Warn().Msgf("sync config PollInterval not configured for %s, will not run periodically sync", regCfg.URL) continue } diff --git a/pkg/extensions/sync/on_demand.go b/pkg/extensions/sync/on_demand.go index b4ae1b5a..0619b109 100644 --- a/pkg/extensions/sync/on_demand.go +++ b/pkg/extensions/sync/on_demand.go @@ -51,6 +51,16 @@ func OneImage(cfg Config, storeController storage.StoreController, continue } + // if content config is not specified, then don't filter, just sync demanded image + if len(regCfg.Content) != 0 { + repos := filterRepos([]string{repo}, regCfg.Content, log) + if len(repos) == 0 { + log.Info().Msgf("skipping syncing on demand %s from %s registry because it's filtered out by content config", + repo, regCfg.URL) + continue + } + } + registryConfig := regCfg log.Info().Msgf("syncing on demand with %s", registryConfig.URL) diff --git a/pkg/extensions/sync/sync.go b/pkg/extensions/sync/sync.go index b96c5d16..df992e06 100644 --- a/pkg/extensions/sync/sync.go +++ b/pkg/extensions/sync/sync.go @@ -464,15 +464,21 @@ func Run(cfg Config, storeController storage.StoreController, wg *goSync.WaitGro // for each upstream registry, start a go routine. for _, regCfg := range cfg.Registries { + // if content not provided, don't run periodically sync if len(regCfg.Content) == 0 { - logger.Info().Msgf("no content found for %s, will not run periodically sync", regCfg.URL) + logger.Info().Msgf("sync config content not configured for %s, will not run periodically sync", regCfg.URL) + continue + } + + // if pollInterval is not provided, don't run periodically sync + if regCfg.PollInterval == 0 { + logger.Warn().Msgf("sync config PollInterval not configured for %s, will not run periodically sync", regCfg.URL) continue } // increment reference since will be busy, so shutdown has to wait wg.Add(1) - // schedule each registry sync ticker := time.NewTicker(regCfg.PollInterval) // fork a new zerolog child to avoid data race @@ -480,6 +486,7 @@ func Run(cfg Config, storeController storage.StoreController, wg *goSync.WaitGro upstreamRegistry := strings.Replace(strings.Replace(regCfg.URL, "http://", "", 1), "https://", "", 1) + // schedule each registry sync go func(regCfg RegistryConfig, l log.Logger) { // run on intervals for ; true; <-ticker.C { diff --git a/pkg/extensions/sync/sync_test.go b/pkg/extensions/sync/sync_test.go index f292973b..027966f3 100644 --- a/pkg/extensions/sync/sync_test.go +++ b/pkg/extensions/sync/sync_test.go @@ -228,8 +228,6 @@ func startDownstreamServer(secure bool, syncConfig *sync.Config) (*api.Controlle func TestSyncOnDemand(t *testing.T) { Convey("Verify sync on demand feature", t, func() { - updateDuration, _ := time.ParseDuration("30m") - sc, srcBaseURL, srcDir, _, srcClient := startUpstreamServer(false, false) defer os.RemoveAll(srcDir) @@ -237,26 +235,25 @@ func TestSyncOnDemand(t *testing.T) { sc.Shutdown() }() - regex := ".*" - var semver bool var tlsVerify bool + regex := ".*" + semver := true + syncRegistryConfig := sync.RegistryConfig{ Content: []sync.Content{ { - // won't match any image on source registry, we will sync on demand - Prefix: "dummy", + Prefix: testImage, Tags: &sync.Tags{ Regex: ®ex, Semver: &semver, }, }, }, - URL: srcBaseURL, - PollInterval: updateDuration, - TLSVerify: &tlsVerify, - CertDir: "", - OnDemand: true, + URL: srcBaseURL, + TLSVerify: &tlsVerify, + CertDir: "", + OnDemand: true, } syncConfig := &sync.Config{ @@ -1621,3 +1618,193 @@ func TestSyncSubPaths(t *testing.T) { So(err, ShouldNotBeNil) }) } + +func TestSyncOnDemandContentFiltering(t *testing.T) { + Convey("Verify sync on demand feature", t, func() { + sc, srcBaseURL, srcDir, _, _ := startUpstreamServer(false, false) + defer os.RemoveAll(srcDir) + + defer func() { + sc.Shutdown() + }() + + Convey("Test image is filtered out by content", func() { + regex := ".*" + var semver bool + var tlsVerify bool + + syncRegistryConfig := sync.RegistryConfig{ + Content: []sync.Content{ + { + //should be filtered out + Prefix: "dummy", + Tags: &sync.Tags{ + Regex: ®ex, + Semver: &semver, + }, + }, + }, + URL: srcBaseURL, + TLSVerify: &tlsVerify, + CertDir: "", + OnDemand: true, + } + + syncConfig := &sync.Config{Registries: []sync.RegistryConfig{syncRegistryConfig}} + + dc, destBaseURL, destDir, _ := startDownstreamServer(false, syncConfig) + defer os.RemoveAll(destDir) + + defer func() { + dc.Shutdown() + }() + + resp, err := resty.R().Get(destBaseURL + "/v2/" + testImage + "/manifests/" + testImageTag) + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, 404) + }) + + Convey("Test image is not filtered out by content", func() { + regex := ".*" + semver := true + var tlsVerify bool + + syncRegistryConfig := sync.RegistryConfig{ + Content: []sync.Content{ + { + // will sync on demand, should not be filtered out + Prefix: testImage, + Tags: &sync.Tags{ + Regex: ®ex, + Semver: &semver, + }, + }, + }, + URL: srcBaseURL, + TLSVerify: &tlsVerify, + CertDir: "", + OnDemand: true, + } + + syncConfig := &sync.Config{Registries: []sync.RegistryConfig{syncRegistryConfig}} + + dc, destBaseURL, destDir, _ := startDownstreamServer(false, syncConfig) + defer os.RemoveAll(destDir) + + defer func() { + dc.Shutdown() + }() + + resp, err := resty.R().Get(destBaseURL + "/v2/" + testImage + "/manifests/" + testImageTag) + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, 200) + }) + }) +} + +func TestSyncConfigRules(t *testing.T) { + Convey("Verify sync config rules", t, func() { + sc, srcBaseURL, srcDir, _, _ := startUpstreamServer(false, false) + defer os.RemoveAll(srcDir) + + defer func() { + sc.Shutdown() + }() + + Convey("Test periodically sync is disabled when pollInterval is not set", func() { + regex := ".*" + var semver bool + var tlsVerify bool + + syncRegistryConfig := sync.RegistryConfig{ + Content: []sync.Content{ + { + Prefix: testImage, + Tags: &sync.Tags{ + Regex: ®ex, + Semver: &semver, + }, + }, + }, + URL: srcBaseURL, + TLSVerify: &tlsVerify, + CertDir: "", + OnDemand: false, + } + + syncConfig := &sync.Config{Registries: []sync.RegistryConfig{syncRegistryConfig}} + + dc, destBaseURL, destDir, _ := startDownstreamServer(false, syncConfig) + defer os.RemoveAll(destDir) + + defer func() { + dc.Shutdown() + }() + + // trigger sync, this way we can be sure periodically sync ran + resp, _ := resty.R().Post(destBaseURL + "/sync") + So(resp, ShouldNotBeNil) + So(resp.StatusCode(), ShouldEqual, 200) + + // image should not be synced + resp, err := resty.R().Get(destBaseURL + "/v2/" + testImage + "/manifests/" + testImageTag) + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, 404) + }) + + Convey("Test periodically sync is disabled when content is not set", func() { + var tlsVerify bool + updateDuration, _ := time.ParseDuration("30m") + + syncRegistryConfig := sync.RegistryConfig{ + PollInterval: updateDuration, + URL: srcBaseURL, + TLSVerify: &tlsVerify, + CertDir: "", + OnDemand: false, + } + + syncConfig := &sync.Config{Registries: []sync.RegistryConfig{syncRegistryConfig}} + + dc, destBaseURL, destDir, _ := startDownstreamServer(false, syncConfig) + defer os.RemoveAll(destDir) + + defer func() { + dc.Shutdown() + }() + + // trigger sync, this way we can be sure periodically sync ran + resp, _ := resty.R().Post(destBaseURL + "/sync") + So(resp, ShouldNotBeNil) + So(resp.StatusCode(), ShouldEqual, 200) + + resp, err := resty.R().Get(destBaseURL + "/v2/" + testImage + "/manifests/" + testImageTag) + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, 404) + }) + + Convey("Test ondemand sync is disabled when ondemand is false", func() { + var tlsVerify bool + + syncRegistryConfig := sync.RegistryConfig{ + URL: srcBaseURL, + TLSVerify: &tlsVerify, + CertDir: "", + OnDemand: false, + } + + syncConfig := &sync.Config{Registries: []sync.RegistryConfig{syncRegistryConfig}} + + dc, destBaseURL, destDir, _ := startDownstreamServer(false, syncConfig) + defer os.RemoveAll(destDir) + + defer func() { + dc.Shutdown() + }() + + resp, err := resty.R().Get(destBaseURL + "/v2/" + testImage + "/manifests/" + testImageTag) + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, 404) + }) + }) +}