feat(sync): initial commit for streaming sync

initial working prototype for sync

fix: pre-load chunk readers on manifest fetch

feat: make chunkSize configurable

fix minimal build

fix: linter errors

Signed-off-by: Vishwas Rajashekar <dev@vrajashkr.com>

pkg/api/controller.go

@@ -17,6 +17,7 @@ import (
 
 	"github.com/gorilla/mux"
 	"github.com/gorilla/securecookie"
+	"github.com/regclient/regclient/types/manifest"
 	"github.com/zitadel/oidc/v3/pkg/client/rp"
 
 	"zotregistry.dev/zot/v2/errors"
@@ -25,6 +26,7 @@ import (
 	ext "zotregistry.dev/zot/v2/pkg/extensions"
 	events "zotregistry.dev/zot/v2/pkg/extensions/events"
 	monitoring "zotregistry.dev/zot/v2/pkg/extensions/monitoring"
+	"zotregistry.dev/zot/v2/pkg/extensions/sync"
 	log "zotregistry.dev/zot/v2/pkg/log"
 	meta "zotregistry.dev/zot/v2/pkg/meta"
 	mTypes "zotregistry.dev/zot/v2/pkg/meta/types"
@@ -41,6 +43,7 @@ const (
 type Controller struct {
 	Config          *config.Config
 	Router          *mux.Router
+	StreamManager   sync.StreamManager
 	MetaDB          mTypes.MetaDB
 	StoreController storage.StoreController
 	Log             log.Logger
@@ -374,6 +377,12 @@ func (c *Controller) Init() error {
 		}
 	}
 
+	if extensionsConfig.IsStreamingEnabled() {
+		c.Log.Info().Msg("streaming sync enabled")
+		sm := sync.NewChunkingStreamManager(c.Config, c.Log)
+		c.StreamManager = sm
+	}
+
 	return nil
 }
 
@@ -597,7 +606,8 @@ func (c *Controller) StartBackgroundTasks() {
 
 	// Always call EnableSyncExtension to ensure proper logging, even when sync is disabled
 	//nolint: contextcheck
-	syncOnDemand, err := ext.EnableSyncExtension(c.Config, c.MetaDB, c.StoreController, c.taskScheduler, c.Log)
+	syncOnDemand, err := ext.EnableSyncExtension(
+		c.Config, c.MetaDB, c.StoreController, c.taskScheduler, c.StreamManager, c.Log)
 	if err != nil {
 		c.Log.Error().Err(err).Msg("failed to start sync extension")
 	}
@@ -652,4 +662,5 @@ func RunGCTasks(conf *config.Config, storeController storage.StoreController, me
 type SyncOnDemand interface {
 	SyncImage(ctx context.Context, repo, reference string) error
 	SyncReferrers(ctx context.Context, repo string, subjectDigestStr string, referenceTypes []string) error
+	FetchManifest(ctx context.Context, repo, reference string) (manifest.Manifest, error)
 }

pkg/api/routes.go

@@ -1455,6 +1455,37 @@ func (rh *RouteHandler) GetBlob(response http.ResponseWriter, request *http.Requ
 
 	writeBlobError := func(err error) {
 		details := zerr.GetDetails(err)
+		extConf := rh.c.Config.CopyExtensionsConfig()
+
+		if extConf.IsStreamingEnabled() {
+			rh.c.Log.Info().Msg("streaming enabled. using stream logic")
+
+			if errors.Is(err, zerr.ErrRepoNotFound) || errors.Is(err, zerr.ErrBlobNotFound) {
+				rh.c.Log.Info().Msg("blob was not found. Connecting client to stream")
+
+				copier, err := rh.c.StreamManager.ConnectClient(digest.String(), response)
+				if err != nil {
+					rh.c.Log.Error().Err(err).Msg("failed to connect client to stream")
+					response.WriteHeader(http.StatusInternalServerError)
+
+					return
+				}
+
+				// TODO: handle partial
+				err = copier.Copy()
+				if err != nil {
+					rh.c.Log.Error().Err(err).Msg("unexpected error during stream copy")
+				}
+
+				response.Header().Set("Content-Length", strconv.FormatInt(copier.Source.InFlightReader.GetDescriptor().Size, 10))
+				response.Header().Set(constants.DistContentDigestKey, digest.String())
+				response.Header().Set("Content-Type", copier.Source.InFlightReader.GetDescriptor().MediaType)
+				response.WriteHeader(http.StatusOK)
+
+				return
+			}
+		}
+
 		if errors.Is(err, zerr.ErrBadBlobDigest) { //nolint:gocritic // errorslint conflicts with gocritic:IfElseChain
 			details["digest"] = digest.String()
 			e := apiErr.NewError(apiErr.DIGEST_INVALID).AddDetail(details)
@@ -2645,6 +2676,39 @@ func getImageManifest(ctx context.Context, routeHandler *RouteHandler, imgStore
 		routeHandler.c.Log.Info().Str("repository", name).Str("reference", reference).
 			Msg("trying to get updated image by syncing on demand")
 
+		extConf := routeHandler.c.Config.CopyExtensionsConfig()
+
+		// if streaming enabled, return manifest immediately, start sync in background
+		if extConf.IsStreamingEnabled() {
+			routeHandler.c.Log.Info().Str("repository", name).Str("reference", reference).
+				Msg("streaming is enabled. Direct fetching manifest.")
+
+			fetchedManifest, err := routeHandler.c.SyncOnDemand.FetchManifest(ctx, name, reference)
+			if err != nil {
+				routeHandler.c.Log.Err(err).Str("repository", name).Str("reference", reference).
+					Msg("failed to fetch manifest")
+
+				return imgStore.GetImageManifest(name, reference)
+			}
+
+			content, err := fetchedManifest.RawBody()
+			if err != nil {
+				routeHandler.c.Log.Err(err).Str("repository", name).Str("reference", reference).
+					Msg("failed to read manifest")
+
+				return imgStore.GetImageManifest(name, reference)
+			}
+
+			go func() {
+				if errSync := routeHandler.c.SyncOnDemand.SyncImage(ctx, name, reference); errSync != nil {
+					routeHandler.c.Log.Err(errSync).Str("repository", name).Str("reference", reference).
+						Msg("failed to sync image")
+				}
+			}()
+
+			return content, fetchedManifest.GetDescriptor().Digest, fetchedManifest.GetDescriptor().MediaType, nil
+		}
+
 		if errSync := routeHandler.c.SyncOnDemand.SyncImage(ctx, name, reference); errSync != nil {
 			routeHandler.c.Log.Err(errSync).Str("repository", name).Str("reference", reference).
 				Msg("failed to sync image")