feat(sync): initial commit for streaming sync

initial working prototype for sync

fix: pre-load chunk readers on manifest fetch

feat: make chunkSize configurable

fix minimal build

fix: linter errors

Signed-off-by: Vishwas Rajashekar <dev@vrajashkr.com>

pkg/api/controller.go

@@ -17,6 +17,7 @@ import (
 
 	"github.com/gorilla/mux"
 	"github.com/gorilla/securecookie"
+	"github.com/regclient/regclient/types/manifest"
 	"github.com/zitadel/oidc/v3/pkg/client/rp"
 
 	"zotregistry.dev/zot/v2/errors"
@@ -25,6 +26,7 @@ import (
 	ext "zotregistry.dev/zot/v2/pkg/extensions"
 	events "zotregistry.dev/zot/v2/pkg/extensions/events"
 	monitoring "zotregistry.dev/zot/v2/pkg/extensions/monitoring"
+	"zotregistry.dev/zot/v2/pkg/extensions/sync"
 	log "zotregistry.dev/zot/v2/pkg/log"
 	meta "zotregistry.dev/zot/v2/pkg/meta"
 	mTypes "zotregistry.dev/zot/v2/pkg/meta/types"
@@ -41,6 +43,7 @@ const (
 type Controller struct {
 	Config          *config.Config
 	Router          *mux.Router
+	StreamManager   sync.StreamManager
 	MetaDB          mTypes.MetaDB
 	StoreController storage.StoreController
 	Log             log.Logger
@@ -374,6 +377,12 @@ func (c *Controller) Init() error {
 		}
 	}
 
+	if extensionsConfig.IsStreamingEnabled() {
+		c.Log.Info().Msg("streaming sync enabled")
+		sm := sync.NewChunkingStreamManager(c.Config, c.Log)
+		c.StreamManager = sm
+	}
+
 	return nil
 }
 
@@ -597,7 +606,8 @@ func (c *Controller) StartBackgroundTasks() {
 
 	// Always call EnableSyncExtension to ensure proper logging, even when sync is disabled
 	//nolint: contextcheck
-	syncOnDemand, err := ext.EnableSyncExtension(c.Config, c.MetaDB, c.StoreController, c.taskScheduler, c.Log)
+	syncOnDemand, err := ext.EnableSyncExtension(
+		c.Config, c.MetaDB, c.StoreController, c.taskScheduler, c.StreamManager, c.Log)
 	if err != nil {
 		c.Log.Error().Err(err).Msg("failed to start sync extension")
 	}
@@ -652,4 +662,5 @@ func RunGCTasks(conf *config.Config, storeController storage.StoreController, me
 type SyncOnDemand interface {
 	SyncImage(ctx context.Context, repo, reference string) error
 	SyncReferrers(ctx context.Context, repo string, subjectDigestStr string, referenceTypes []string) error
+	FetchManifest(ctx context.Context, repo, reference string) (manifest.Manifest, error)
 }