Sync prefix can be an exact match or a glob pattern, closes #297

Signed-off-by: Petu Eusebiu <peusebiu@cisco.com>

pkg/extensions/sync/http_handler.go

@@ -48,6 +48,11 @@ func (h *PostHandler) Handler(w http.ResponseWriter, r *http.Request) {
 	}
 
 	for _, regCfg := range h.Cfg.Registries {
+		if len(regCfg.Content) == 0 {
+			h.Log.Info().Msgf("no content found for %s, will not run periodically sync", regCfg.URL)
+			continue
+		}
+
 		upstreamRegistryName := strings.Replace(strings.Replace(regCfg.URL, "http://", "", 1), "https://", "", 1)
 
 		if err := syncRegistry(regCfg, h.StoreController, h.Log, localCtx, policyCtx,

pkg/extensions/sync/sync.go

@@ -310,11 +310,6 @@ func getUpstreamContext(regCfg *RegistryConfig, credentials Credentials) *types.
 func syncRegistry(regCfg RegistryConfig, storeController storage.StoreController,
 	log log.Logger, localCtx *types.SystemContext,
 	policyCtx *signature.PolicyContext, credentials Credentials, uuid string) error {
-	if len(regCfg.Content) == 0 {
-		log.Info().Msgf("no content found for %s, will not run periodically sync", regCfg.URL)
-		return nil
-	}
-
 	log.Info().Msgf("syncing registry: %s", regCfg.URL)
 
 	var err error
@@ -341,9 +336,9 @@ func syncRegistry(regCfg RegistryConfig, storeController storage.StoreController
 
 	upstreamRegistryName := strings.Replace(strings.Replace(regCfg.URL, "http://", "", 1), "https://", "", 1)
 
-	log.Info().Msg("filtering repos based on sync prefixes")
+	log.Info().Msgf("filtering %d repos based on sync prefixes", len(catalog.Repositories))
 
-	repos := filterRepos(catalog.Repositories, regCfg.Content)
+	repos := filterRepos(catalog.Repositories, regCfg.Content, log)
 
 	log.Info().Msgf("got repos: %v", repos)
 
@@ -467,6 +462,11 @@ func Run(cfg Config, storeController storage.StoreController, logger log.Logger)
 
 	// for each upstream registry, start a go routine.
 	for _, regCfg := range cfg.Registries {
+		if len(regCfg.Content) == 0 {
+			logger.Info().Msgf("no content found for %s, will not run periodically sync", regCfg.URL)
+			continue
+		}
+
 		// schedule each registry sync
 		ticker := time.NewTicker(regCfg.PollInterval)
 

pkg/extensions/sync/sync_internal_test.go

@@ -168,6 +168,34 @@ func TestSyncInternal(t *testing.T) {
 		So(err, ShouldNotBeNil)
 	})
 
+	Convey("Test filterRepos()", t, func() {
+		repos := []string{"repo", "repo1", "repo2", "repo/repo2", "repo/repo2/repo3/repo4"}
+		contents := []Content{
+			{
+				Prefix: "repo",
+			},
+			{
+				Prefix: "/repo/**",
+			},
+			{
+				Prefix: "repo*",
+			},
+		}
+		filteredRepos := filterRepos(repos, contents, log.NewLogger("", ""))
+		So(filteredRepos[0], ShouldResemble, []string{"repo"})
+		So(filteredRepos[1], ShouldResemble, []string{"repo/repo2", "repo/repo2/repo3/repo4"})
+		So(filteredRepos[2], ShouldResemble, []string{"repo1", "repo2"})
+
+		contents = []Content{
+			{
+				Prefix: "[repo%#@",
+			},
+		}
+
+		filteredRepos = filterRepos(repos, contents, log.NewLogger("", ""))
+		So(len(filteredRepos), ShouldEqual, 0)
+	})
+
 	Convey("Verify pushSyncedLocalImage func", t, func() {
 		storageDir, err := ioutil.TempDir("", "oci-dest-repo-test")
 		if err != nil {

pkg/extensions/sync/utils.go

@@ -11,6 +11,7 @@ import (
 	"github.com/anuvu/zot/pkg/extensions/monitoring"
 	"github.com/anuvu/zot/pkg/log"
 	"github.com/anuvu/zot/pkg/storage"
+	glob "github.com/bmatcuk/doublestar/v4"
 	"github.com/containers/image/v5/docker/reference"
 	"github.com/containers/image/v5/types"
 	ispec "github.com/opencontainers/image-spec/specs-go/v1"
@@ -42,44 +43,26 @@ func parseRepositoryReference(input string) (reference.Named, error) {
 }
 
 // filterRepos filters repos based on prefix given in the config.
-func filterRepos(repos []string, content []Content) map[int][]string {
-	// prefix: repo
+func filterRepos(repos []string, content []Content, log log.Logger) map[int][]string {
 	filtered := make(map[int][]string)
 
 	for _, repo := range repos {
-		matched := false
-		// we use contentID to figure out tags filtering
 		for contentID, c := range content {
-			// handle prefixes starting with '/'
 			var prefix string
+			// handle prefixes starting with '/'
 			if strings.HasPrefix(c.Prefix, "/") {
 				prefix = c.Prefix[1:]
 			} else {
 				prefix = c.Prefix
 			}
 
-			// split both prefix and repository and compare each part
-			splittedPrefix := strings.Split(prefix, "/")
-			// split at most n + 1
-			splittedRepo := strings.SplitN(repo, "/", len(splittedPrefix)+1)
-
-			// if prefix is longer than a repository, no match
-			if len(splittedPrefix) > len(splittedRepo) {
+			matched, err := glob.Match(prefix, repo)
+			if err != nil {
+				log.Error().Err(err).Str("pattern",
+					prefix).Msg("error while parsing glob pattern, skipping it...")
 				continue
 			}
 
-			// check if matched each part of prefix and repository
-			for i := 0; i < len(splittedPrefix); i++ {
-				if splittedRepo[i] == splittedPrefix[i] {
-					matched = true
-				} else {
-					// if a part doesn't match, check next prefix
-					matched = false
-					break
-				}
-			}
-
-			// if matched no need to check the next prefixes
 			if matched {
 				filtered[contentID] = append(filtered[contentID], repo)
 				break