feat: add zot subcommand to enable testing retention policy settings (#3449)

feat: add verify-feature retention subcommand with comprehensive testing and validation

Add a `verify-feature retention` subcommand that allows users to preview and
validate retention policy changes without running the actual Zot server.
The command runs GC and retention tasks in dry-run mode for immediate feedback.

- Run verify-feature retention standalone without starting the server
- Preview retention policy decisions in dry-run mode
- Configurable GC interval override via command-line flag
- Optional timeout for task completion
- Configurable log output (stdout or file)

Basic usage:
```bash
zot verify-feature retention <config-file>
```

With log file output:
```bash
zot verify-feature retention -l /var/log/zot-retention-check.log <config-file>
```

With GC interval override (runs GC tasks every 30 seconds):
```bash
zot verify-feature retention -i 30s <config-file>
```

With timeout (wait up to 5 minutes for tasks to complete):
```bash
zot verify-feature retention -t 5m <config-file>
```

Combined flags:
```bash
zot verify-feature retention -l /var/log/zot-retention-check.log -i 1m -t 10m <config-file>
```

The command supports overriding GC settings from the config:
- `-i, --gc-interval`: Override the GC interval setting (applies to all storage paths including subpaths)

- Refactored `RunGCTasks` from `controller.go` to be reusable
- Added `checkServerRunning` validation to prevent conflicts
- Implemented signal handling for graceful shutdown
- Added configuration sanitization and logging
- Set GCMaxSchedulerDelay programmatically (not user-configurable)

Added tests for coverage on main function:
- Negative test cases (no args, bad config, GC disabled, server running)
- Both BoltDB and Redis
- Retention enabled scenarios with complex image setups
- Retention disabled scenarios
- Delete referrers functionality
- Subpaths configuration
- GC interval override validation

Run the verify-feature retention tests:
```bash
go test -v ./pkg/cli/server -run TestRetentionCheck
```

Signed-off-by: Andrei Aaron <andreifdaaron@gmail.com>

pkg/storage/gc/gc.go

@@ -37,6 +37,10 @@ type Options struct {
 	// will garbage collect blobs older than Delay
 	Delay time.Duration
 
+	// MaxSchedulerDelay is the maximum random delay for GC task scheduling
+	// Defaults to 30 seconds if not specified
+	MaxSchedulerDelay time.Duration
+
 	ImageRetention config.ImageRetention
 }
 
@@ -69,10 +73,16 @@ given an interval and a Scheduler.
 func (gc GarbageCollect) CleanImageStorePeriodically(interval time.Duration, sch *scheduler.Scheduler) {
 	processedRepos := make(map[string]struct{})
 
+	maxDelay := gc.opts.MaxSchedulerDelay
+	if maxDelay <= 0 {
+		maxDelay = 30 * time.Second // default value
+	}
+
 	generator := &GCTaskGenerator{
 		imgStore:       gc.imgStore,
 		gc:             gc,
 		processedRepos: processedRepos,
+		maxDelay:       maxDelay,
 	}
 
 	sch.SubmitGenerator(generator, interval, scheduler.MediumPriority)
@@ -808,12 +818,19 @@ type GCTaskGenerator struct {
 	nextRun        time.Time
 	done           bool
 	rand           *rand.Rand
+	maxDelay       time.Duration
 }
 
-func (gen *GCTaskGenerator) getRandomDelay() int {
-	maxDelay := 30
+func (gen *GCTaskGenerator) getRandomDelay() time.Duration {
+	maxDelay := gen.maxDelay
+	if maxDelay <= 0 {
+		maxDelay = 30 * time.Second // default fallback
+	}
 
-	return gen.rand.Intn(maxDelay)
+	// Generate random delay with nanosecond precision by working directly with
+	// time.Duration's internal representation (nanoseconds as int64).
+	// This supports sub-second delays (milliseconds, microseconds).
+	return time.Duration(gen.rand.Int63n(int64(maxDelay)))
 }
 
 func (gen *GCTaskGenerator) Name() string {
@@ -827,7 +844,7 @@ func (gen *GCTaskGenerator) Next() (scheduler.Task, error) {
 
 	delay := gen.getRandomDelay()
 
-	gen.nextRun = time.Now().Add(time.Duration(delay) * time.Second)
+	gen.nextRun = time.Now().Add(delay)
 
 	repo, err := gen.imgStore.GetNextRepository(gen.processedRepos)
 	if err != nil {