feat: add zot subcommand to enable testing retention policy settings (#3449)

feat: add verify-feature retention subcommand with comprehensive testing and validation

Add a `verify-feature retention` subcommand that allows users to preview and
validate retention policy changes without running the actual Zot server.
The command runs GC and retention tasks in dry-run mode for immediate feedback.

- Run verify-feature retention standalone without starting the server
- Preview retention policy decisions in dry-run mode
- Configurable GC interval override via command-line flag
- Optional timeout for task completion
- Configurable log output (stdout or file)

Basic usage:
```bash
zot verify-feature retention <config-file>
```

With log file output:
```bash
zot verify-feature retention -l /var/log/zot-retention-check.log <config-file>
```

With GC interval override (runs GC tasks every 30 seconds):
```bash
zot verify-feature retention -i 30s <config-file>
```

With timeout (wait up to 5 minutes for tasks to complete):
```bash
zot verify-feature retention -t 5m <config-file>
```

Combined flags:
```bash
zot verify-feature retention -l /var/log/zot-retention-check.log -i 1m -t 10m <config-file>
```

The command supports overriding GC settings from the config:
- `-i, --gc-interval`: Override the GC interval setting (applies to all storage paths including subpaths)

- Refactored `RunGCTasks` from `controller.go` to be reusable
- Added `checkServerRunning` validation to prevent conflicts
- Implemented signal handling for graceful shutdown
- Added configuration sanitization and logging
- Set GCMaxSchedulerDelay programmatically (not user-configurable)

Added tests for coverage on main function:
- Negative test cases (no args, bad config, GC disabled, server running)
- Both BoltDB and Redis
- Retention enabled scenarios with complex image setups
- Retention disabled scenarios
- Delete referrers functionality
- Subpaths configuration
- GC interval override validation

Run the verify-feature retention tests:
```bash
go test -v ./pkg/cli/server -run TestRetentionCheck
```

Signed-off-by: Andrei Aaron <andreifdaaron@gmail.com>

pkg/api/config/config.go

@@ -36,6 +36,10 @@ type StorageConfig struct {
 	Retention     ImageRetention
 	StorageDriver map[string]interface{} `mapstructure:",omitempty"`
 	CacheDriver   map[string]interface{} `mapstructure:",omitempty"`
+
+	// GCMaxSchedulerDelay is the maximum random delay for GC task scheduling
+	// This field is not configurable by the end user
+	GCMaxSchedulerDelay time.Duration `yaml:"-"`
 }
 
 type ImageRetention struct {

pkg/api/controller.go

@@ -460,47 +460,27 @@ func (c *Controller) StartBackgroundTasks() {
 		c.HTPasswdWatcher.Run()
 	}
 
-	// Enable running garbage-collect periodically for DefaultStore
-	storageConfig := c.Config.CopyStorageConfig()
-	if storageConfig.GC {
-		gc := gc.NewGarbageCollect(c.StoreController.DefaultStore, c.MetaDB, gc.Options{
-			Delay:          storageConfig.GCDelay,
-			ImageRetention: storageConfig.Retention,
-		}, c.Audit, c.Log)
-
-		gc.CleanImageStorePeriodically(storageConfig.GCInterval, c.taskScheduler)
-	}
+	// Run GC and retention tasks
+	RunGCTasks(c.Config, c.StoreController, c.MetaDB, c.taskScheduler, c.Log, c.Audit)
 
 	// Enable running dedupe blobs both ways (dedupe or restore deduped blobs)
 	c.StoreController.DefaultStore.RunDedupeBlobs(time.Duration(0), c.taskScheduler)
 
-	// Enable extensions if extension config is provided for DefaultStore
-	extensionsConfig := c.Config.CopyExtensionsConfig()
-
 	// Always call EnableSearchExtension to ensure proper logging, even when search is disabled
 	ext.EnableSearchExtension(c.Config, c.StoreController, c.MetaDB, c.taskScheduler, c.CveScanner, c.Log)
 
 	// Always call EnableMetricsExtension to ensure proper logging, even when metrics is disabled
+	storageConfig := c.Config.CopyStorageConfig()
 	ext.EnableMetricsExtension(c.Config, c.Log, storageConfig.RootDirectory)
 
 	// runs once if metrics are enabled & imagestore is local
+	extensionsConfig := c.Config.CopyExtensionsConfig()
 	if extensionsConfig.IsMetricsEnabled() && storageConfig.StorageDriver == nil {
 		c.StoreController.DefaultStore.PopulateStorageMetrics(time.Duration(0), c.taskScheduler)
 	}
 
 	if storageConfig.SubPaths != nil {
 		for route, subStorageConfig := range storageConfig.SubPaths {
-			// Enable running garbage-collect periodically for subImageStore
-			if subStorageConfig.GC {
-				gc := gc.NewGarbageCollect(c.StoreController.SubStore[route], c.MetaDB,
-					gc.Options{
-						Delay:          subStorageConfig.GCDelay,
-						ImageRetention: subStorageConfig.Retention,
-					}, c.Audit, c.Log)
-
-				gc.CleanImageStorePeriodically(subStorageConfig.GCInterval, c.taskScheduler)
-			}
-
 			// Enable extensions if extension config is provided for subImageStore
 			ext.EnableMetricsExtension(c.Config, c.Log, subStorageConfig.RootDirectory)
 
@@ -539,6 +519,40 @@ func (c *Controller) StartBackgroundTasks() {
 	ext.EnableScheduledTasks(c.Config, c.taskScheduler, c.MetaDB, c.Log) //nolint: contextcheck
 }
 
+// RunGCTasks runs minimal GC and retention tasks without full controller.
+func RunGCTasks(conf *config.Config, storeController storage.StoreController, metaDB mTypes.MetaDB,
+	taskScheduler *scheduler.Scheduler, logger log.Logger, audit *log.Logger,
+) {
+	// Enable running garbage-collect periodically for DefaultStore
+	storageConfig := conf.CopyStorageConfig()
+	if storageConfig.GC {
+		gc := gc.NewGarbageCollect(storeController.DefaultStore, metaDB, gc.Options{
+			Delay:             storageConfig.GCDelay,
+			ImageRetention:    storageConfig.Retention,
+			MaxSchedulerDelay: storageConfig.GCMaxSchedulerDelay,
+		}, audit, logger)
+
+		gc.CleanImageStorePeriodically(storageConfig.GCInterval, taskScheduler)
+	}
+
+	// Handle subpaths
+	if storageConfig.SubPaths != nil {
+		for route, subStorageConfig := range storageConfig.SubPaths {
+			// Enable running garbage-collect periodically for subImageStore
+			if subStorageConfig.GC {
+				gc := gc.NewGarbageCollect(storeController.SubStore[route], metaDB,
+					gc.Options{
+						Delay:             subStorageConfig.GCDelay,
+						ImageRetention:    subStorageConfig.Retention,
+						MaxSchedulerDelay: subStorageConfig.GCMaxSchedulerDelay,
+					}, audit, logger)
+
+				gc.CleanImageStorePeriodically(subStorageConfig.GCInterval, taskScheduler)
+			}
+		}
+	}
+}
+
 type SyncOnDemand interface {
 	SyncImage(ctx context.Context, repo, reference string) error
 	SyncReferrers(ctx context.Context, repo string, subjectDigestStr string, referenceTypes []string) error