feat(cluster): Add support for request proxying for scale out (#2385)

* feat(cluster): initial commit for scale-out cluster Signed-off-by: Ramkumar Chinchani <rchincha@cisco.com> * feat(cluster): support shared storage scale out This change introduces support for shared storage backed zot cluster scale out. New feature Multiple stateless zot instances can run using the same shared storage backend where each instance looks at a specific set of repositories based on a siphash of the repository name to improve scale as the load is distributed across multiple instances. For a given config, there will only be one instance that can perform dist-spec read/write on a given repository. What's changed? - introduced a transparent request proxy for dist-spec endpoints based on siphash of repository name. - new config for scale out cluster that specifies list of cluster members. Signed-off-by: Vishwas Rajashekar <vrajashe@cisco.com> --------- Signed-off-by: Ramkumar Chinchani <rchincha@cisco.com> Signed-off-by: Vishwas Rajashekar <vrajashe@cisco.com> Co-authored-by: Ramkumar Chinchani <rchincha@cisco.com>
2026-06-16 04:17:55 +08:00 · 2024-05-20 21:35:21 +05:30
parent be5ad66797
commit 5ae7a028d9
30 changed files with 2320 additions and 24 deletions
@@ -457,6 +457,11 @@ func validateConfiguration(config *config.Config, log zlog.Logger) error {
 		}
 	}

+	// check validity of scale out cluster config
+	if err := validateClusterConfig(config, log); err != nil {
+		return err
+	}
+
 	return nil
 }

@@ -1103,3 +1108,27 @@ func validateSync(config *config.Config, log zlog.Logger) error {

 	return nil
 }
+
+func validateClusterConfig(config *config.Config, log zlog.Logger) error {
+	if config.Cluster != nil {
+		if len(config.Cluster.Members) == 0 {
+			log.Error().Err(zerr.ErrBadConfig).
+				Msg("cannot have 0 members in a scale out cluster")
+
+			return zerr.ErrBadConfig
+		}
+
+		// the allowed length is 16 as the siphash requires a 128 bit key.
+		// that translates to 16 characters * 8 bits each.
+		allowedHashKeyLength := 16
+		if len(config.Cluster.HashKey) != allowedHashKeyLength {
+			log.Error().Err(zerr.ErrBadConfig).
+				Str("hashkey", config.Cluster.HashKey).
+				Msg(fmt.Sprintf("hashKey for scale out cluster must have %d characters", allowedHashKeyLength))
+
+			return zerr.ErrBadConfig
+		}
+	}
+
+	return nil
+}
@@ -2028,6 +2028,150 @@ func TestUpdateLDAPConfig(t *testing.T) {
 	})
 }

+func TestClusterConfig(t *testing.T) {
+	baseExamplePath := "../../../examples/scale-out-cluster-cloud/"
+
+	Convey("Should successfully load example configs for cloud", t, func() {
+		for memberIdx := 0; memberIdx < 3; memberIdx++ {
+			cfgFileToLoad := fmt.Sprintf("%s/config-cluster-member%d.json", baseExamplePath, memberIdx)
+			cfg := config.New()
+			err := cli.LoadConfiguration(cfg, cfgFileToLoad)
+			So(err, ShouldBeNil)
+		}
+	})
+
+	Convey("Should successfully load example TLS configs for cloud", t, func() {
+		for memberIdx := 0; memberIdx < 3; memberIdx++ {
+			cfgFileToLoad := fmt.Sprintf("%s/tls/config-cluster-member%d.json", baseExamplePath, memberIdx)
+			cfg := config.New()
+			err := cli.LoadConfiguration(cfg, cfgFileToLoad)
+			So(err, ShouldBeNil)
+		}
+	})
+
+	Convey("Should reject scale out cluster invalid cases", t, func() {
+		cfgFileContents, err := os.ReadFile(baseExamplePath + "config-cluster-member0.json")
+		So(err, ShouldBeNil)
+
+		Convey("Should reject empty members list", func() {
+			cfg := config.New()
+			err := json.Unmarshal(cfgFileContents, cfg)
+			So(err, ShouldBeNil)
+
+			// set the members to an empty list
+			cfg.Cluster.Members = []string{}
+
+			file, err := os.CreateTemp("", "cluster-config-*.json")
+			So(err, ShouldBeNil)
+			defer os.Remove(file.Name())
+
+			cfgFileContents, err := json.MarshalIndent(cfg, "", " ")
+			So(err, ShouldBeNil)
+
+			err = os.WriteFile(file.Name(), cfgFileContents, 0o600)
+			So(err, ShouldBeNil)
+			err = cli.LoadConfiguration(cfg, file.Name())
+			So(err, ShouldNotBeNil)
+		})
+
+		Convey("Should reject missing members list", func() {
+			cfg := config.New()
+
+			configStr := `
+			{
+				"storage": {
+					"RootDirectory": "/tmp/example"
+				},
+				"http": {
+					"address": "127.0.0.1",
+					"port": "800"
+				},
+				"cluster" {
+					"hashKey": "loremipsumdolors"
+				}
+			}`
+
+			file, err := os.CreateTemp("", "cluster-config-*.json")
+			So(err, ShouldBeNil)
+			defer os.Remove(file.Name())
+
+			err = os.WriteFile(file.Name(), []byte(configStr), 0o600)
+			So(err, ShouldBeNil)
+			err = cli.LoadConfiguration(cfg, file.Name())
+			So(err, ShouldNotBeNil)
+		})
+
+		Convey("Should reject missing hashkey", func() {
+			cfg := config.New()
+
+			configStr := `
+			{
+				"storage": {
+					"RootDirectory": "/tmp/example"
+				},
+				"http": {
+					"address": "127.0.0.1",
+					"port": "800"
+				},
+				"cluster" {
+					"members": ["127.0.0.1:9000"]
+				}
+			}`
+
+			file, err := os.CreateTemp("", "cluster-config-*.json")
+			So(err, ShouldBeNil)
+			defer os.Remove(file.Name())
+
+			err = os.WriteFile(file.Name(), []byte(configStr), 0o600)
+			So(err, ShouldBeNil)
+			err = cli.LoadConfiguration(cfg, file.Name())
+			So(err, ShouldNotBeNil)
+		})
+
+		Convey("Should reject a hashkey that is too short", func() {
+			cfg := config.New()
+			err := json.Unmarshal(cfgFileContents, cfg)
+			So(err, ShouldBeNil)
+
+			// set the hashkey to a string shorter than 16 characters
+			cfg.Cluster.HashKey = "fifteencharacte"
+
+			file, err := os.CreateTemp("", "cluster-config-*.json")
+			So(err, ShouldBeNil)
+			defer os.Remove(file.Name())
+
+			cfgFileContents, err := json.MarshalIndent(cfg, "", " ")
+			So(err, ShouldBeNil)
+
+			err = os.WriteFile(file.Name(), cfgFileContents, 0o600)
+			So(err, ShouldBeNil)
+			err = cli.LoadConfiguration(cfg, file.Name())
+			So(err, ShouldNotBeNil)
+		})
+
+		Convey("Should reject a hashkey that is too long", func() {
+			cfg := config.New()
+			err := json.Unmarshal(cfgFileContents, cfg)
+			So(err, ShouldBeNil)
+
+			// set the hashkey to a string longer than 16 characters
+			cfg.Cluster.HashKey = "seventeencharacte"
+
+			file, err := os.CreateTemp("", "cluster-config-*.json")
+			So(err, ShouldBeNil)
+			defer os.Remove(file.Name())
+
+			cfgFileContents, err := json.MarshalIndent(cfg, "", " ")
+			So(err, ShouldBeNil)
+
+			err = os.WriteFile(file.Name(), cfgFileContents, 0o600)
+			So(err, ShouldBeNil)
+			err = cli.LoadConfiguration(cfg, file.Name())
+			So(err, ShouldNotBeNil)
+		})
+	})
+}
+
 // run cli and return output.
 func runCLIWithConfig(tempDir string, config string) (string, error) {
 	port := GetFreePort()