feat(sync): initial commit for streaming sync

initial working prototype for sync

fix: pre-load chunk readers on manifest fetch

feat: make chunkSize configurable

fix minimal build

fix: linter errors

Signed-off-by: Vishwas Rajashekar <dev@vrajashkr.com>

examples/config-sync-normal.json

@@ -0,0 +1,30 @@
+{
+	"distSpecVersion": "1.1.1",
+	"storage": {
+		"rootDirectory": "./temp/zot1"
+	},
+	"http": {
+		"address": "127.0.0.1",
+		"port": "8080"
+	},
+	"log": {
+		"level": "debug"
+	},
+	"extensions": {
+		"sync": {
+			"enable": true,
+			"registries": [
+				{
+					"urls": [
+						"http://localhost:9000"
+					],
+					"onDemand": true,
+					"tlsVerify": false,
+					"maxRetries": 5,
+					"retryDelay": "30s",
+					"syncTimeout": "10m"
+				}
+			]
+		}
+	}
+}

examples/config-sync-stream.json

@@ -0,0 +1,32 @@
+{
+	"distSpecVersion": "1.1.1",
+	"storage": {
+		"rootDirectory": "./temp/zotstream"
+	},
+	"http": {
+		"address": "127.0.0.1",
+		"port": "8080"
+	},
+	"log": {
+		"level": "debug"
+	},
+	"extensions": {
+		"sync": {
+			"enable": true,
+			"enableStreaming": true,
+			"streamChunkSizeBytes": 32768,
+			"registries": [
+				{
+					"urls": [
+						"http://localhost:9000"
+					],
+					"onDemand": true,
+					"tlsVerify": false,
+					"maxRetries": 5,
+					"retryDelay": "30s",
+					"syncTimeout": "10m"
+				}
+			]
+		}
+	}
+}