fix(scheduler): fix data race (#2085)

* fix(scheduler): data race when pushing new tasks

the problem here is that scheduler can be closed in two ways:
- canceling the context given as argument to scheduler.RunScheduler()
- running scheduler.Shutdown()

because of this shutdown can trigger a data race between calling scheduler.inShutdown()
and actually pushing tasks into the pool workers

solved that by keeping a quit channel and listening on both quit channel and ctx.Done()
and closing the worker chan and scheduler afterwards.

Signed-off-by: Petu Eusebiu <peusebiu@cisco.com>

* refactor(scheduler): refactor into a single shutdown

before this we could stop scheduler either by closing the context
provided to RunScheduler(ctx) or by running Shutdown().

simplify things by getting rid of the external context in RunScheduler().
keep an internal context in the scheduler itself and pass it down to all tasks.

Signed-off-by: Petu Eusebiu <peusebiu@cisco.com>

---------

Signed-off-by: Petu Eusebiu <peusebiu@cisco.com>

test/blackbox/sync.bats

@@ -76,7 +76,7 @@ function setup_file() {
                     ],
                     "onDemand": false,
                     "tlsVerify": false,
-                    "PollInterval": "1s",
+                    "PollInterval": "10s",
                     "content": [
                         {
                             "prefix": "**"

test/blackbox/sync_cloud.bats

@@ -30,7 +30,7 @@ function setup_file() {
         exit 1
     fi
 
-        # Download test data to folder common for the entire suite, not just this file
+    # Download test data to folder common for the entire suite, not just this file
     skopeo --insecure-policy copy --format=oci docker://ghcr.io/project-zot/golang:1.20 oci:${TEST_DATA_DIR}/golang:1.20
     # Setup zot server
     local zot_sync_per_root_dir=${BATS_FILE_TMPDIR}/zot-per
@@ -88,7 +88,7 @@ function setup_file() {
                     ],
                     "onDemand": false,
                     "tlsVerify": false,
-                    "PollInterval": "1s",
+                    "PollInterval": "10s",
                     "content": [
                         {
                             "prefix": "**"
@@ -197,7 +197,7 @@ function teardown_file() {
     [ "$status" -eq 0 ]
     [ $(echo "${lines[-1]}" | jq '.tags[]') = '"1.20"' ]
 
-    run sleep 20s
+    run sleep 30s
 
     run curl http://127.0.0.1:${zot_port1}/v2/_catalog
     [ "$status" -eq 0 ]

test/blackbox/sync_replica_cluster.bats

@@ -69,7 +69,7 @@ function setup_file() {
                     ],
                     "onDemand": false,
                     "tlsVerify": false,
-                    "PollInterval": "1s",
+                    "PollInterval": "10s",
                     "content": [
                         {
                             "prefix": "**"
@@ -105,7 +105,7 @@ EOF
                     ],
                     "onDemand": false,
                     "tlsVerify": false,
-                    "PollInterval": "1s",
+                    "PollInterval": "10s",
                     "content": [
                         {
                             "prefix": "**"