feat(cluster): Add support for request proxying for scale out (#2385)

* feat(cluster): initial commit for scale-out cluster Signed-off-by: Ramkumar Chinchani <rchincha@cisco.com> * feat(cluster): support shared storage scale out This change introduces support for shared storage backed zot cluster scale out. New feature Multiple stateless zot instances can run using the same shared storage backend where each instance looks at a specific set of repositories based on a siphash of the repository name to improve scale as the load is distributed across multiple instances. For a given config, there will only be one instance that can perform dist-spec read/write on a given repository. What's changed? - introduced a transparent request proxy for dist-spec endpoints based on siphash of repository name. - new config for scale out cluster that specifies list of cluster members. Signed-off-by: Vishwas Rajashekar <vrajashe@cisco.com> --------- Signed-off-by: Ramkumar Chinchani <rchincha@cisco.com> Signed-off-by: Vishwas Rajashekar <vrajashe@cisco.com> Co-authored-by: Ramkumar Chinchani <rchincha@cisco.com>
2026-06-16 04:17:55 +08:00 · 2024-05-20 21:35:21 +05:30
parent be5ad66797
commit 5ae7a028d9
30 changed files with 2320 additions and 24 deletions
@@ -6,6 +6,7 @@ import (
 	"errors"
 	"fmt"
 	"io/fs"
+	"net"
 	"os"
 	"regexp"
 	"strings"
@@ -145,3 +146,93 @@ func IsContextDone(ctx context.Context) bool {
 		return false
 	}
 }
+
+// get a list of IP addresses configured on the host's
+// interfaces.
+func GetLocalIPs() ([]string, error) {
+	var localIPs []string
+
+	ifaces, err := net.Interfaces()
+	if err != nil {
+		return []string{}, err
+	}
+
+	for _, i := range ifaces {
+		addrs, err := i.Addrs()
+		if err != nil {
+			return localIPs, err
+		}
+
+		for _, addr := range addrs {
+			if localIP, ok := addr.(*net.IPNet); ok {
+				localIPs = append(localIPs, localIP.IP.String())
+			}
+		}
+	}
+
+	return localIPs, nil
+}
+
+// get a list of listening sockets on the host (IP:port).
+// IPv6 is returned as [host]:port.
+func GetLocalSockets(port string) ([]string, error) {
+	localIPs, err := GetLocalIPs()
+	if err != nil {
+		return []string{}, err
+	}
+
+	localSockets := make([]string, len(localIPs))
+
+	for idx, ip := range localIPs {
+		// JoinHostPort automatically wraps IPv6 addresses in []
+		localSockets[idx] = net.JoinHostPort(ip, port)
+	}
+
+	return localSockets, nil
+}
+
+func GetIPFromHostName(host string) ([]string, error) {
+	addrs, err := net.LookupIP(host)
+	if err != nil {
+		return []string{}, err
+	}
+
+	ips := make([]string, 0, len(addrs))
+
+	for _, ip := range addrs {
+		ips = append(ips, ip.String())
+	}
+
+	return ips, nil
+}
+
+// checks if 2 sockets are equal at the host port level.
+func AreSocketsEqual(socketA string, socketB string) (bool, error) {
+	hostA, portA, err := net.SplitHostPort(socketA)
+	if err != nil {
+		return false, err
+	}
+
+	hostB, portB, err := net.SplitHostPort(socketB)
+	if err != nil {
+		return false, err
+	}
+
+	hostAIP := net.ParseIP(hostA)
+	if hostAIP == nil {
+		// this could be a fully-qualified domain name (FQDN)
+		// for FQDN, just a normal compare is enough
+		return hostA == hostB, nil
+	}
+
+	hostBIP := net.ParseIP(hostB)
+	if hostBIP == nil {
+		// if the host part of socketA was parsed successfully, it was an IP
+		// if the host part of socketA was an FQDN, then the comparison is
+		// already done as the host of socketB is also assumed to be an FQDN.
+		// since the parsing failed, assume that A and B are not equal.
+		return false, nil
+	}
+
+	return (hostAIP.Equal(hostBIP) && (portA == portB)), nil
+}
@@ -3,6 +3,7 @@ package common_test
 import (
 	"os"
 	"path"
+	"strings"
 	"testing"

 	notreg "github.com/notaryproject/notation-go/registry"
@@ -61,4 +62,107 @@ func TestCommon(t *testing.T) {
 	Convey("Test ArtifactTypeNotation const has same value as in notaryproject", t, func() {
 		So(common.ArtifactTypeNotation, ShouldEqual, notreg.ArtifactTypeNotation)
 	})
+
+	Convey("Test GetLocalIPs", t, func() {
+		localIPs, err := common.GetLocalIPs()
+		So(err, ShouldBeNil)
+		So(localIPs, ShouldNotBeEmpty)
+		So(localIPs, ShouldContain, "127.0.0.1")
+	})
+
+	Convey("Test GetLocalSockets IPv4", t, func() {
+		localSockets, err := common.GetLocalSockets("8765")
+		So(err, ShouldBeNil)
+		So(localSockets, ShouldNotBeEmpty)
+		So(localSockets, ShouldContain, "127.0.0.1:8765")
+		for _, socket := range localSockets {
+			lastColonIndex := strings.LastIndex(socket, ":")
+			So(socket[lastColonIndex+1:], ShouldEqual, "8765")
+		}
+	})
+
+	Convey("Test GetLocalSockets IPv6", t, func() {
+		localSockets, err := common.GetLocalSockets("8766")
+		So(err, ShouldBeNil)
+		So(localSockets, ShouldNotBeEmpty)
+		So(localSockets, ShouldContain, "[::1]:8766")
+		for _, socket := range localSockets {
+			lastColonIndex := strings.LastIndex(socket, ":")
+			So(socket[lastColonIndex+1:], ShouldEqual, "8766")
+		}
+	})
+
+	Convey("Test GetIPFromHostName with valid hostname", t, func() {
+		addrs, err := common.GetIPFromHostName("github.com")
+		So(err, ShouldBeNil)
+		So(addrs, ShouldNotBeEmpty)
+		// we can't check the actual addresses here as they can change
+	})
+
+	Convey("Test GetIPFromHostName with non-existent hostname", t, func() {
+		addrs, err := common.GetIPFromHostName("thisdoesnotexist")
+		So(err, ShouldNotBeNil)
+		So(err.Error(), ShouldContainSubstring, "lookup thisdoesnotexist")
+		So(addrs, ShouldBeEmpty)
+	})
+
+	Convey("Test AreSocketsEqual with equal IPv4 sockets", t, func() {
+		result, err := common.AreSocketsEqual("127.0.0.1:9000", "127.0.0.1:9000")
+		So(err, ShouldBeNil)
+		So(result, ShouldBeTrue)
+	})
+
+	Convey("Test AreSocketsEqual with equal IPv6 sockets", t, func() {
+		result, err := common.AreSocketsEqual("[::1]:9000", "[0000:0000:0000:0000:0000:0000:0000:00001]:9000")
+		So(err, ShouldBeNil)
+		So(result, ShouldBeTrue)
+	})
+
+	Convey("Test AreSocketsEqual with different IPv4 socket ports", t, func() {
+		result, err := common.AreSocketsEqual("127.0.0.1:9000", "127.0.0.1:9001")
+		So(err, ShouldBeNil)
+		So(result, ShouldBeFalse)
+	})
+
+	Convey("Test AreSocketsEqual with different IPv4 socket hosts", t, func() {
+		result, err := common.AreSocketsEqual("127.0.0.1:9000", "127.0.0.2:9000")
+		So(err, ShouldBeNil)
+		So(result, ShouldBeFalse)
+	})
+
+	Convey("Test AreSocketsEqual with 2 equal host names", t, func() {
+		result, err := common.AreSocketsEqual("localhost:9000", "localhost:9000")
+		So(err, ShouldBeNil)
+		So(result, ShouldBeTrue)
+	})
+
+	Convey("Test AreSocketsEqual with 2 different host names", t, func() {
+		result, err := common.AreSocketsEqual("localhost:9000", "notlocalhost:9000")
+		So(err, ShouldBeNil)
+		So(result, ShouldBeFalse)
+	})
+
+	Convey("Test AreSocketsEqual with hostname and IP address", t, func() {
+		result, err := common.AreSocketsEqual("localhost:9000", "127.0.0.1:9000")
+		So(err, ShouldBeNil)
+		So(result, ShouldBeFalse)
+	})
+
+	Convey("Test AreSocketsEqual with IP address and hostname", t, func() {
+		result, err := common.AreSocketsEqual("127.0.0.1:9000", "localhost:9000")
+		So(err, ShouldBeNil)
+		So(result, ShouldBeFalse)
+	})
+
+	Convey("Test AreSocketsEqual with invalid first socket", t, func() {
+		result, err := common.AreSocketsEqual("127.0.0.1", "localhost:9000")
+		So(err, ShouldNotBeNil)
+		So(result, ShouldBeFalse)
+	})
+
+	Convey("Test AreSocketsEqual with invalid second socket", t, func() {
+		result, err := common.AreSocketsEqual("localhost:9000", "127.0.0.1")
+		So(err, ShouldNotBeNil)
+		So(result, ShouldBeFalse)
+	})
 }