feat(sync): move stream from global to per upstream

Signed-off-by: Vishwas Rajashekar <dev@vrajashkr.com>

pkg/extensions/config/config.go

@@ -133,13 +133,19 @@ func (e *ExtensionConfig) IsSyncEnabled() bool {
 		(e.Sync.Enable == nil && len(e.Sync.Registries) > 0))
 }
 
-// IsStreamingEnabled checks if streaming is enabled in this extensions config.
+// IsStreamingEnabled checks if streaming is enabled for any upstream registry in the sync config.
 func (e *ExtensionConfig) IsStreamingEnabled() bool {
-	if e == nil {
+	if e == nil || e.Sync == nil {
 		return false
 	}
 
-	return e.Sync != nil && e.Sync.Stream != nil && *e.Sync.Stream
+	for i := range e.Sync.Registries {
+		if e.Sync.Registries[i].IsStreamEnabled() {
+			return true
+		}
+	}
+
+	return false
 }
 
 // IsScrubEnabled checks if scrub is enabled in this extensions config.

pkg/extensions/config/sync/config.go

@@ -13,9 +13,7 @@ type Credentials struct {
 }
 
 type Config struct {
-	Enable *bool
-	// Stream is set to true when it is desired to stream blobs to clients as they are being synced to zot.
-	Stream          *bool
+	Enable          *bool
 	CredentialsFile string
 	/* DownloadDir is needed only in case of using cloud based storages
 	it uses regclient to first copy images into this dir (as oci layout)
@@ -25,11 +23,13 @@ type Config struct {
 }
 
 type RegistryConfig struct {
-	URLs                  []string
-	PollInterval          time.Duration
-	Content               []Content
-	TLSVerify             *bool
-	OnDemand              bool
+	URLs         []string
+	PollInterval time.Duration
+	Content      []Content
+	TLSVerify    *bool
+	OnDemand     bool
+	// Stream is set to true when it is desired to stream blobs to clients as they are being synced from this upstream.
+	Stream                *bool
 	CertDir               string
 	MaxRetries            *int
 	RetryDelay            *time.Duration
@@ -47,6 +47,11 @@ func (r RegistryConfig) ShouldSyncLegacyCosignTags() bool {
 	return r.SyncLegacyCosignTags == nil || *r.SyncLegacyCosignTags
 }
 
+// IsStreamEnabled returns true if streaming is enabled for this registry config.
+func (r RegistryConfig) IsStreamEnabled() bool {
+	return r.Stream != nil && *r.Stream
+}
+
 type Content struct {
 	Prefix      string
 	Tags        *Tags

pkg/extensions/extension_sync.go

@@ -65,8 +65,14 @@ func EnableSyncExtension(config *config.Config, metaDB mTypes.MetaDB,
 			// Get cluster config safely
 			clusterConfig := config.CopyClusterConfig()
 
+			// Only pass the stream manager to services that have streaming enabled on their registry config.
+			var svcStreamManager sync.StreamManager
+			if registryConfig.Stream != nil && *registryConfig.Stream {
+				svcStreamManager = streamManager
+			}
+
 			service, err := sync.New(
-				registryConfig, credsPath, clusterConfig, tmpDir, storeController, streamManager, metaDB, log)
+				registryConfig, credsPath, clusterConfig, tmpDir, storeController, svcStreamManager, metaDB, log)
 			if err != nil {
 				log.Error().Err(err).Msg("failed to initialize sync extension")
 

pkg/extensions/sync/on_demand.go

@@ -53,6 +53,17 @@ func (onDemand *BaseOnDemand) StreamManager() StreamManager {
 	return onDemand.streamManager
 }
 
+// IsStreamingEnabledForRepo returns true if any on-demand service has streaming enabled for the given repo.
+func (onDemand *BaseOnDemand) IsStreamingEnabledForRepo(repo string) bool {
+	for _, service := range onDemand.services {
+		if service.IsStreamingForRepo(repo) {
+			return true
+		}
+	}
+
+	return false
+}
+
 // FetchManifestForStream directly fetches the manifest from the upstream services and prepares the image
 // for streaming.
 // This is only intended for use with streaming sync.

pkg/extensions/sync/on_demand_disabled.go

@@ -29,3 +29,7 @@ func (onDemand *BaseOnDemand) FetchManifestForStream(
 func (onDemand *BaseOnDemand) StreamManager() StreamManager {
 	return nil
 }
+
+func (onDemand *BaseOnDemand) IsStreamingEnabledForRepo(_ string) bool {
+	return false
+}

pkg/extensions/sync/service.go

@@ -215,6 +215,21 @@ func (service *BaseService) CanRetryOnError() bool {
 	return false
 }
 
+// IsStreamingForRepo returns whether streaming is enabled for the given local repo on this service.
+// Streaming is enabled if the registry config has Stream set to true and the repo matches the content config.
+func (service *BaseService) IsStreamingForRepo(repo string) bool {
+	if !service.config.IsStreamEnabled() {
+		return false
+	}
+
+	// If no content filter is configured, all repos match.
+	if len(service.config.Content) == 0 {
+		return true
+	}
+
+	return service.contentManager.GetContentByLocalRepo(repo) != nil
+}
+
 func (service *BaseService) GetSyncTimeout() time.Duration {
 	if service.config.SyncTimeout == 0 {
 		return syncConstants.DefaultSyncTimeout
@@ -520,7 +535,7 @@ func (service *BaseService) syncRef(ctx context.Context, localRepo string, remot
 
 	copyOpts := []regclient.ImageOpts{}
 
-	if service.streamManager != nil {
+	if service.config.Stream != nil && *service.config.Stream && service.streamManager != nil {
 		service.log.Debug().Str("repo", localRepo).Str("reference", remoteImageRef.Tag).
 			Msg("streaming is enabled. Enabling reader hook")
 		copyOpts = append(copyOpts, regclient.ImageWithBlobReaderHook(service.streamManager.StreamingBlobReader))

pkg/extensions/sync/sync.go

@@ -39,6 +39,8 @@ type Service interface {
 	GetSyncTimeout() time.Duration
 
 	FetchManifest(ctx context.Context, repo, reference string) (manifest.Manifest, error)
+	// Returns whether streaming is enabled for the given local repo on this service.
+	IsStreamingForRepo(repo string) bool
 }
 
 // Registry interface must be implemented by local and remote registries.