Disable sync periodically polling when pollInterval is not configured

Filtering out sync on demand images based on content configuration

Signed-off-by: Petu Eusebiu <peusebiu@cisco.com>

pkg/extensions/sync/http_handler.go

@@ -48,8 +48,15 @@ func (h *PostHandler) Handler(w http.ResponseWriter, r *http.Request) {
 	}
 
 	for _, regCfg := range h.Cfg.Registries {
+		// if content not provided, don't run periodically sync
 		if len(regCfg.Content) == 0 {
-			h.Log.Info().Msgf("no content found for %s, will not run periodically sync", regCfg.URL)
+			h.Log.Info().Msgf("sync config content not configured for %s, will not run periodically sync", regCfg.URL)
+			continue
+		}
+
+		// if pollInterval is not provided, don't run periodically sync
+		if regCfg.PollInterval == 0 {
+			h.Log.Warn().Msgf("sync config PollInterval not configured for %s, will not run periodically sync", regCfg.URL)
 			continue
 		}
 

pkg/extensions/sync/on_demand.go

@@ -51,6 +51,16 @@ func OneImage(cfg Config, storeController storage.StoreController,
 			continue
 		}
 
+		// if content config is not specified, then don't filter, just sync demanded image
+		if len(regCfg.Content) != 0 {
+			repos := filterRepos([]string{repo}, regCfg.Content, log)
+			if len(repos) == 0 {
+				log.Info().Msgf("skipping syncing on demand %s from %s registry because it's filtered out by content config",
+					repo, regCfg.URL)
+				continue
+			}
+		}
+
 		registryConfig := regCfg
 		log.Info().Msgf("syncing on demand with %s", registryConfig.URL)
 

pkg/extensions/sync/sync.go

@@ -464,15 +464,21 @@ func Run(cfg Config, storeController storage.StoreController, wg *goSync.WaitGro
 
 	// for each upstream registry, start a go routine.
 	for _, regCfg := range cfg.Registries {
+		// if content not provided, don't run periodically sync
 		if len(regCfg.Content) == 0 {
-			logger.Info().Msgf("no content found for %s, will not run periodically sync", regCfg.URL)
+			logger.Info().Msgf("sync config content not configured for %s, will not run periodically sync", regCfg.URL)
+			continue
+		}
+
+		// if pollInterval is not provided, don't run periodically sync
+		if regCfg.PollInterval == 0 {
+			logger.Warn().Msgf("sync config PollInterval not configured for %s, will not run periodically sync", regCfg.URL)
 			continue
 		}
 
 		// increment reference since will be busy, so shutdown has to wait
 		wg.Add(1)
 
-		// schedule each registry sync
 		ticker := time.NewTicker(regCfg.PollInterval)
 
 		// fork a new zerolog child to avoid data race
@@ -480,6 +486,7 @@ func Run(cfg Config, storeController storage.StoreController, wg *goSync.WaitGro
 
 		upstreamRegistry := strings.Replace(strings.Replace(regCfg.URL, "http://", "", 1), "https://", "", 1)
 
+		// schedule each registry sync
 		go func(regCfg RegistryConfig, l log.Logger) {
 			// run on intervals
 			for ; true; <-ticker.C {

pkg/extensions/sync/sync_test.go

@@ -228,8 +228,6 @@ func startDownstreamServer(secure bool, syncConfig *sync.Config) (*api.Controlle
 
 func TestSyncOnDemand(t *testing.T) {
 	Convey("Verify sync on demand feature", t, func() {
-		updateDuration, _ := time.ParseDuration("30m")
-
 		sc, srcBaseURL, srcDir, _, srcClient := startUpstreamServer(false, false)
 		defer os.RemoveAll(srcDir)
 
@@ -237,26 +235,25 @@ func TestSyncOnDemand(t *testing.T) {
 			sc.Shutdown()
 		}()
 
-		regex := ".*"
-		var semver bool
 		var tlsVerify bool
 
+		regex := ".*"
+		semver := true
+
 		syncRegistryConfig := sync.RegistryConfig{
 			Content: []sync.Content{
 				{
-					// won't match any image on source registry, we will sync on demand
-					Prefix: "dummy",
+					Prefix: testImage,
 					Tags: &sync.Tags{
 						Regex:  &regex,
 						Semver: &semver,
 					},
 				},
 			},
-			URL:          srcBaseURL,
-			PollInterval: updateDuration,
-			TLSVerify:    &tlsVerify,
-			CertDir:      "",
-			OnDemand:     true,
+			URL:       srcBaseURL,
+			TLSVerify: &tlsVerify,
+			CertDir:   "",
+			OnDemand:  true,
 		}
 
 		syncConfig := &sync.Config{
@@ -1621,3 +1618,193 @@ func TestSyncSubPaths(t *testing.T) {
 		So(err, ShouldNotBeNil)
 	})
 }
+
+func TestSyncOnDemandContentFiltering(t *testing.T) {
+	Convey("Verify sync on demand feature", t, func() {
+		sc, srcBaseURL, srcDir, _, _ := startUpstreamServer(false, false)
+		defer os.RemoveAll(srcDir)
+
+		defer func() {
+			sc.Shutdown()
+		}()
+
+		Convey("Test image is filtered out by content", func() {
+			regex := ".*"
+			var semver bool
+			var tlsVerify bool
+
+			syncRegistryConfig := sync.RegistryConfig{
+				Content: []sync.Content{
+					{
+						//should be filtered out
+						Prefix: "dummy",
+						Tags: &sync.Tags{
+							Regex:  &regex,
+							Semver: &semver,
+						},
+					},
+				},
+				URL:       srcBaseURL,
+				TLSVerify: &tlsVerify,
+				CertDir:   "",
+				OnDemand:  true,
+			}
+
+			syncConfig := &sync.Config{Registries: []sync.RegistryConfig{syncRegistryConfig}}
+
+			dc, destBaseURL, destDir, _ := startDownstreamServer(false, syncConfig)
+			defer os.RemoveAll(destDir)
+
+			defer func() {
+				dc.Shutdown()
+			}()
+
+			resp, err := resty.R().Get(destBaseURL + "/v2/" + testImage + "/manifests/" + testImageTag)
+			So(err, ShouldBeNil)
+			So(resp.StatusCode(), ShouldEqual, 404)
+		})
+
+		Convey("Test image is not filtered out by content", func() {
+			regex := ".*"
+			semver := true
+			var tlsVerify bool
+
+			syncRegistryConfig := sync.RegistryConfig{
+				Content: []sync.Content{
+					{
+						// will sync on demand, should not be filtered out
+						Prefix: testImage,
+						Tags: &sync.Tags{
+							Regex:  &regex,
+							Semver: &semver,
+						},
+					},
+				},
+				URL:       srcBaseURL,
+				TLSVerify: &tlsVerify,
+				CertDir:   "",
+				OnDemand:  true,
+			}
+
+			syncConfig := &sync.Config{Registries: []sync.RegistryConfig{syncRegistryConfig}}
+
+			dc, destBaseURL, destDir, _ := startDownstreamServer(false, syncConfig)
+			defer os.RemoveAll(destDir)
+
+			defer func() {
+				dc.Shutdown()
+			}()
+
+			resp, err := resty.R().Get(destBaseURL + "/v2/" + testImage + "/manifests/" + testImageTag)
+			So(err, ShouldBeNil)
+			So(resp.StatusCode(), ShouldEqual, 200)
+		})
+	})
+}
+
+func TestSyncConfigRules(t *testing.T) {
+	Convey("Verify sync config rules", t, func() {
+		sc, srcBaseURL, srcDir, _, _ := startUpstreamServer(false, false)
+		defer os.RemoveAll(srcDir)
+
+		defer func() {
+			sc.Shutdown()
+		}()
+
+		Convey("Test periodically sync is disabled when pollInterval is not set", func() {
+			regex := ".*"
+			var semver bool
+			var tlsVerify bool
+
+			syncRegistryConfig := sync.RegistryConfig{
+				Content: []sync.Content{
+					{
+						Prefix: testImage,
+						Tags: &sync.Tags{
+							Regex:  &regex,
+							Semver: &semver,
+						},
+					},
+				},
+				URL:       srcBaseURL,
+				TLSVerify: &tlsVerify,
+				CertDir:   "",
+				OnDemand:  false,
+			}
+
+			syncConfig := &sync.Config{Registries: []sync.RegistryConfig{syncRegistryConfig}}
+
+			dc, destBaseURL, destDir, _ := startDownstreamServer(false, syncConfig)
+			defer os.RemoveAll(destDir)
+
+			defer func() {
+				dc.Shutdown()
+			}()
+
+			// trigger sync, this way we can be sure periodically sync ran
+			resp, _ := resty.R().Post(destBaseURL + "/sync")
+			So(resp, ShouldNotBeNil)
+			So(resp.StatusCode(), ShouldEqual, 200)
+
+			// image should not be synced
+			resp, err := resty.R().Get(destBaseURL + "/v2/" + testImage + "/manifests/" + testImageTag)
+			So(err, ShouldBeNil)
+			So(resp.StatusCode(), ShouldEqual, 404)
+		})
+
+		Convey("Test periodically sync is disabled when content is not set", func() {
+			var tlsVerify bool
+			updateDuration, _ := time.ParseDuration("30m")
+
+			syncRegistryConfig := sync.RegistryConfig{
+				PollInterval: updateDuration,
+				URL:          srcBaseURL,
+				TLSVerify:    &tlsVerify,
+				CertDir:      "",
+				OnDemand:     false,
+			}
+
+			syncConfig := &sync.Config{Registries: []sync.RegistryConfig{syncRegistryConfig}}
+
+			dc, destBaseURL, destDir, _ := startDownstreamServer(false, syncConfig)
+			defer os.RemoveAll(destDir)
+
+			defer func() {
+				dc.Shutdown()
+			}()
+
+			// trigger sync, this way we can be sure periodically sync ran
+			resp, _ := resty.R().Post(destBaseURL + "/sync")
+			So(resp, ShouldNotBeNil)
+			So(resp.StatusCode(), ShouldEqual, 200)
+
+			resp, err := resty.R().Get(destBaseURL + "/v2/" + testImage + "/manifests/" + testImageTag)
+			So(err, ShouldBeNil)
+			So(resp.StatusCode(), ShouldEqual, 404)
+		})
+
+		Convey("Test ondemand sync is disabled when ondemand is false", func() {
+			var tlsVerify bool
+
+			syncRegistryConfig := sync.RegistryConfig{
+				URL:       srcBaseURL,
+				TLSVerify: &tlsVerify,
+				CertDir:   "",
+				OnDemand:  false,
+			}
+
+			syncConfig := &sync.Config{Registries: []sync.RegistryConfig{syncRegistryConfig}}
+
+			dc, destBaseURL, destDir, _ := startDownstreamServer(false, syncConfig)
+			defer os.RemoveAll(destDir)
+
+			defer func() {
+				dc.Shutdown()
+			}()
+
+			resp, err := resty.R().Get(destBaseURL + "/v2/" + testImage + "/manifests/" + testImageTag)
+			So(err, ShouldBeNil)
+			So(resp.StatusCode(), ShouldEqual, 404)
+		})
+	})
+}