diff --git a/pkg/cli/root.go b/pkg/cli/root.go index 98f74a22..bb2b9855 100644 --- a/pkg/cli/root.go +++ b/pkg/cli/root.go @@ -598,10 +598,18 @@ func validateSync(config *config.Config) error { for _, content := range regCfg.Content { ok := glob.ValidatePattern(content.Prefix) if !ok { - log.Error().Err(glob.ErrBadPattern).Str("pattern", content.Prefix).Msg("sync pattern could not be compiled") + log.Error().Err(glob.ErrBadPattern).Str("prefix", content.Prefix).Msg("sync prefix could not be compiled") return glob.ErrBadPattern } + + if content.StripPrefix && !strings.Contains(content.Prefix, "/*") && content.Destination == "/" { + log.Error().Err(errors.ErrBadConfig). + Interface("sync content", content). + Msg("sync config: can not use stripPrefix true and destination '/' without using glob patterns in prefix") + + return errors.ErrBadConfig + } } } } diff --git a/pkg/cli/root_test.go b/pkg/cli/root_test.go index b362cb4b..ddeedc01 100644 --- a/pkg/cli/root_test.go +++ b/pkg/cli/root_test.go @@ -357,6 +357,43 @@ func TestVerify(t *testing.T) { So(func() { _ = cli.NewServerRootCmd().Execute() }, ShouldPanic) }) + Convey("Test verify with bad sync content config", t, func(c C) { + tmpfile, err := os.CreateTemp("", "zot-test*.json") + So(err, ShouldBeNil) + defer os.Remove(tmpfile.Name()) // clean up + content := []byte(`{"storage":{"rootDirectory":"/tmp/zot"}, + "http":{"address":"127.0.0.1","port":"8080","realm":"zot", + "auth":{"htpasswd":{"path":"test/data/htpasswd"},"failDelay":1}}, + "extensions":{"sync": {"registries": [{"urls":["localhost:9999"], + "maxRetries": 1, "retryDelay": "10s", + "content": [{"prefix":"zot-repo","stripPrefix":true,"destination":"/"}]}]}}}`) + _, err = tmpfile.Write(content) + So(err, ShouldBeNil) + err = tmpfile.Close() + So(err, ShouldBeNil) + os.Args = []string{"cli_test", "verify", tmpfile.Name()} + So(func() { _ = cli.NewServerRootCmd().Execute() }, ShouldPanic) + }) + + Convey("Test verify with good sync content config", t, func(c C) { + tmpfile, err := os.CreateTemp("", "zot-test*.json") + So(err, ShouldBeNil) + defer os.Remove(tmpfile.Name()) // clean up + content := []byte(`{"storage":{"rootDirectory":"/tmp/zot"}, + "http":{"address":"127.0.0.1","port":"8080","realm":"zot", + "auth":{"htpasswd":{"path":"test/data/htpasswd"},"failDelay":1}}, + "extensions":{"sync": {"registries": [{"urls":["localhost:9999"], + "maxRetries": 1, "retryDelay": "10s", + "content": [{"prefix":"zot-repo/*","stripPrefix":true,"destination":"/"}]}]}}}`) + _, err = tmpfile.Write(content) + So(err, ShouldBeNil) + err = tmpfile.Close() + So(err, ShouldBeNil) + os.Args = []string{"cli_test", "verify", tmpfile.Name()} + err = cli.NewServerRootCmd().Execute() + So(err, ShouldBeNil) + }) + Convey("Test verify with bad authorization repo patterns", t, func(c C) { tmpfile, err := os.CreateTemp("", "zot-test*.json") So(err, ShouldBeNil) diff --git a/pkg/extensions/sync/sync.go b/pkg/extensions/sync/sync.go index 81240a2a..cd130cae 100644 --- a/pkg/extensions/sync/sync.go +++ b/pkg/extensions/sync/sync.go @@ -382,22 +382,22 @@ func syncRegistry(ctx context.Context, regCfg RegistryConfig, } remoteRepoCopy := remoteRepo - imageStore := storeController.GetImageStore(remoteRepoCopy) - - localCachePath, err := getLocalCachePath(imageStore, remoteRepoCopy) - if err != nil { - log.Error().Str("errorType", TypeOf(err)). - Err(err).Msgf("couldn't get localCachePath for %s", remoteRepoCopy) - - return err - } - - if localCachePath != "" { - defer os.RemoveAll(localCachePath) - } for _, image := range imageList { - localRepo := remoteRepoCopy + localRepo := getRepoDestination(remoteRepo, image.content) + + imageStore := storeController.GetImageStore(localRepo) + + localCachePath, err := getLocalCachePath(imageStore, localRepo) + if err != nil { + log.Error().Str("errorType", TypeOf(err)). + Err(err).Msgf("couldn't get localCachePath for %s", remoteRepoCopy) + + return err + } + + defer os.RemoveAll(localCachePath) + upstreamImageRef := image.ref upstreamImageDigest, err := docker.GetDigest(ctx, upstreamCtx, upstreamImageRef) diff --git a/pkg/extensions/sync/sync_test.go b/pkg/extensions/sync/sync_test.go index d7ff94a9..01767227 100644 --- a/pkg/extensions/sync/sync_test.go +++ b/pkg/extensions/sync/sync_test.go @@ -42,6 +42,7 @@ import ( extconf "zotregistry.io/zot/pkg/extensions/config" "zotregistry.io/zot/pkg/extensions/sync" "zotregistry.io/zot/pkg/storage" + "zotregistry.io/zot/pkg/storage/local" "zotregistry.io/zot/pkg/test" ) @@ -948,7 +949,7 @@ func TestMandatoryAnnotations(t *testing.T) { // give it time to set up sync time.Sleep(3 * time.Second) - resp, err := destClient.R().Get(destBaseURL + "/v2/" + testImage + "/manifest/0.0.1") + resp, err := destClient.R().Get(destBaseURL + "/v2/" + testImage + "/manifests/0.0.1") So(err, ShouldBeNil) So(resp, ShouldNotBeNil) So(resp.StatusCode(), ShouldEqual, 404) @@ -3952,6 +3953,140 @@ func TestOnlySignedFlag(t *testing.T) { }) } +func TestSyncWithDestination(t *testing.T) { + Convey("Test sync computes destination option correctly", t, func() { + testCases := []struct { + content sync.Content + expected string + repo string + }{ + { + expected: "zot-test/zot-fold/zot-test", + content: sync.Content{Prefix: "zot-fold/zot-test", Destination: "/zot-test", StripPrefix: false}, + repo: "zot-fold/zot-test", + }, + { + expected: "zot-fold/zot-test", + content: sync.Content{Prefix: "zot-fold/zot-test", Destination: "/", StripPrefix: false}, + repo: "zot-fold/zot-test", + }, + { + expected: "zot-test", + content: sync.Content{Prefix: "zot-fold/zot-test", Destination: "/zot-test", StripPrefix: true}, + repo: "zot-fold/zot-test", + }, + { + expected: "zot-test", + content: sync.Content{Prefix: "zot-fold/*", Destination: "/", StripPrefix: true}, + repo: "zot-fold/zot-test", + }, + { + expected: "zot-test", + content: sync.Content{Prefix: "zot-fold/zot-test", Destination: "/zot-test", StripPrefix: true}, + repo: "zot-fold/zot-test", + }, + { + expected: "zot-test", + content: sync.Content{Prefix: "zot-fold/*", Destination: "/", StripPrefix: true}, + repo: "zot-fold/zot-test", + }, + { + expected: "zot-test", + content: sync.Content{Prefix: "zot-fold/**", Destination: "/", StripPrefix: true}, + repo: "zot-fold/zot-test", + }, + { + expected: "zot-fold/zot-test", + content: sync.Content{Prefix: "zot-fold/**", Destination: "/", StripPrefix: false}, + repo: "zot-fold/zot-test", + }, + } + + sctlr, srcBaseURL, _, _, _ := startUpstreamServer(t, false, false) + + defer func() { + sctlr.Shutdown() + }() + + err := os.MkdirAll(path.Join(sctlr.Config.Storage.RootDirectory, "/zot-fold"), local.DefaultDirPerms) + So(err, ShouldBeNil) + + // move upstream images under /zot-fold + err = os.Rename( + path.Join(sctlr.Config.Storage.RootDirectory, "zot-test"), + path.Join(sctlr.Config.Storage.RootDirectory, "/zot-fold/zot-test"), + ) + + So(err, ShouldBeNil) + + Convey("Test peridiocally sync", func() { + for _, testCase := range testCases { + updateDuration, _ := time.ParseDuration("30m") + tlsVerify := false + syncRegistryConfig := sync.RegistryConfig{ + Content: []sync.Content{testCase.content}, + URLs: []string{srcBaseURL}, + OnDemand: false, + PollInterval: updateDuration, + TLSVerify: &tlsVerify, + } + + defaultVal := true + syncConfig := &sync.Config{ + Enable: &defaultVal, + Registries: []sync.RegistryConfig{syncRegistryConfig}, + } + + dctlr, destBaseURL, _, destClient := startDownstreamServer(t, false, syncConfig) + + defer func() { + dctlr.Shutdown() + }() + + // give it time to set up sync + waitSync(dctlr.Config.Storage.RootDirectory, testCase.expected) + + resp, err := destClient.R().Get(destBaseURL + "/v2/" + testCase.expected + "/manifests/0.0.1") + t.Logf("testcase: %#v", testCase) + So(err, ShouldBeNil) + So(resp, ShouldNotBeNil) + So(resp.StatusCode(), ShouldEqual, 200) + } + }) + + // this is the inverse function of getRepoDestination() + Convey("Test ondemand sync", func() { + for _, testCase := range testCases { + tlsVerify := false + syncRegistryConfig := sync.RegistryConfig{ + Content: []sync.Content{testCase.content}, + URLs: []string{srcBaseURL}, + OnDemand: true, + TLSVerify: &tlsVerify, + } + + defaultVal := true + syncConfig := &sync.Config{ + Enable: &defaultVal, + Registries: []sync.RegistryConfig{syncRegistryConfig}, + } + + dctlr, destBaseURL, _, destClient := startDownstreamServer(t, false, syncConfig) + + defer func() { + dctlr.Shutdown() + }() + + resp, err := destClient.R().Get(destBaseURL + "/v2/" + testCase.expected + "/manifests/0.0.1") + t.Logf("testcase: %#v", testCase) + So(err, ShouldBeNil) + So(resp, ShouldNotBeNil) + So(resp.StatusCode(), ShouldEqual, 200) + } + }) + }) +} + func generateKeyPairs(tdir string) { // generate a keypair os.Setenv("COSIGN_PASSWORD", "")