feat(cluster): Add support for request proxying for scale out (#2385)

* feat(cluster): initial commit for scale-out cluster

Signed-off-by: Ramkumar Chinchani <rchincha@cisco.com>

* feat(cluster): support shared storage scale out

This change introduces support for shared storage backed
zot cluster scale out.

New feature
Multiple stateless zot instances can run using the same shared
storage backend where each instance looks at a specific set
of repositories based on a siphash of the repository name to improve
scale as the load is distributed across multiple instances.
For a given config, there will only be one instance that can perform
dist-spec read/write on a given repository.

What's changed?
- introduced a transparent request proxy for dist-spec endpoints based on
siphash of repository name.
- new config for scale out cluster that specifies list of
cluster members.

Signed-off-by: Vishwas Rajashekar <vrajashe@cisco.com>

---------

Signed-off-by: Ramkumar Chinchani <rchincha@cisco.com>
Signed-off-by: Vishwas Rajashekar <vrajashe@cisco.com>
Co-authored-by: Ramkumar Chinchani <rchincha@cisco.com>

pkg/api/config/config.go

@@ -121,6 +121,32 @@ type SchedulerConfig struct {
 	NumWorkers int
 }
 
+// contains the scale-out configuration which is identical for all zot replicas.
+type ClusterConfig struct {
+	// contains the "host:port" of all the zot instances participating
+	// in the cluster.
+	Members []string `json:"members" mapstructure:"members"`
+
+	// contains the hash key that is required for siphash.
+	// must be a 128-bit (16-byte) key
+	// https://github.com/dchest/siphash?tab=readme-ov-file#func-newkey-byte-hashhash64
+	HashKey string `json:"hashKey" mapstructure:"hashKey"`
+
+	// contains client TLS config.
+	TLS *TLSConfig `json:"tls" mapstructure:"tls"`
+
+	// private field for storing Proxy details such as internal socket list.
+	Proxy *ClusterRequestProxyConfig `json:"-" mapstructure:"-"`
+}
+
+type ClusterRequestProxyConfig struct {
+	// holds the cluster socket (IP:port) derived from the host's
+	// interface configuration and the listening port of the HTTP server.
+	LocalMemberClusterSocket string
+	// index of the local member cluster socket in the members array.
+	LocalMemberClusterSocketIndex uint64
+}
+
 type LDAPCredentials struct {
 	BindDN       string
 	BindPassword string
@@ -230,6 +256,7 @@ type Config struct {
 	Log             *LogConfig
 	Extensions      *extconf.ExtensionConfig
 	Scheduler       *SchedulerConfig `json:"scheduler" mapstructure:",omitempty"`
+	Cluster         *ClusterConfig   `json:"cluster"   mapstructure:",omitempty"`
 }
 
 func New() *Config {