fix(scheduler): fix data race (#2085)

* fix(scheduler): data race when pushing new tasks

the problem here is that scheduler can be closed in two ways:
- canceling the context given as argument to scheduler.RunScheduler()
- running scheduler.Shutdown()

because of this shutdown can trigger a data race between calling scheduler.inShutdown()
and actually pushing tasks into the pool workers

solved that by keeping a quit channel and listening on both quit channel and ctx.Done()
and closing the worker chan and scheduler afterwards.

Signed-off-by: Petu Eusebiu <peusebiu@cisco.com>

* refactor(scheduler): refactor into a single shutdown

before this we could stop scheduler either by closing the context
provided to RunScheduler(ctx) or by running Shutdown().

simplify things by getting rid of the external context in RunScheduler().
keep an internal context in the scheduler itself and pass it down to all tasks.

Signed-off-by: Petu Eusebiu <peusebiu@cisco.com>

---------

Signed-off-by: Petu Eusebiu <peusebiu@cisco.com>

pkg/cli/client/cve_cmd_test.go

@@ -163,16 +163,14 @@ func TestNegativeServerResponse(t *testing.T) {
 		ctlr := api.NewController(conf)
 		ctlr.Log.Logger = ctlr.Log.Output(writers)
 
-		ctx := context.Background()
-
-		if err := ctlr.Init(ctx); err != nil {
+		if err := ctlr.Init(); err != nil {
 			panic(err)
 		}
 
 		ctlr.CveScanner = getMockCveScanner(ctlr.MetaDB)
 
 		go func() {
-			if err := ctlr.Run(ctx); !errors.Is(err, http.ErrServerClosed) {
+			if err := ctlr.Run(); !errors.Is(err, http.ErrServerClosed) {
 				panic(err)
 			}
 		}()
@@ -239,16 +237,14 @@ func TestServerCVEResponse(t *testing.T) {
 	ctlr := api.NewController(conf)
 	ctlr.Log.Logger = ctlr.Log.Output(writers)
 
-	ctx := context.Background()
-
-	if err := ctlr.Init(ctx); err != nil {
+	if err := ctlr.Init(); err != nil {
 		panic(err)
 	}
 
 	ctlr.CveScanner = getMockCveScanner(ctlr.MetaDB)
 
 	go func() {
-		if err := ctlr.Run(ctx); !errors.Is(err, http.ErrServerClosed) {
+		if err := ctlr.Run(); !errors.Is(err, http.ErrServerClosed) {
 			panic(err)
 		}
 	}()
@@ -578,9 +574,7 @@ func TestCVESort(t *testing.T) {
 		t.FailNow()
 	}
 
-	ctx := context.Background()
-
-	if err := ctlr.Init(ctx); err != nil {
+	if err := ctlr.Init(); err != nil {
 		panic(err)
 	}
 
@@ -617,7 +611,7 @@ func TestCVESort(t *testing.T) {
 	}
 
 	go func() {
-		if err := ctlr.Run(ctx); !errors.Is(err, http.ErrServerClosed) {
+		if err := ctlr.Run(); !errors.Is(err, http.ErrServerClosed) {
 			panic(err)
 		}
 	}()

pkg/cli/client/image_cmd_test.go

@@ -866,7 +866,7 @@ func TestServerResponseGQLWithoutPermissions(t *testing.T) {
 		}
 
 		ctlr := api.NewController(conf)
-		if err := ctlr.Init(context.Background()); err != nil {
+		if err := ctlr.Init(); err != nil {
 			So(err, ShouldNotBeNil)
 		}
 	})

pkg/cli/server/config_reloader.go

@@ -1,7 +1,6 @@
 package server
 
 import (
-	"context"
 	"os"
 	"os/signal"
 	"syscall"
@@ -35,28 +34,22 @@ func NewHotReloader(ctlr *api.Controller, filePath string) (*HotReloader, error)
 	return hotReloader, nil
 }
 
-func signalHandler(ctlr *api.Controller, sigCh chan os.Signal, ctx context.Context, cancel context.CancelFunc) {
-	select {
+func signalHandler(ctlr *api.Controller, sigCh chan os.Signal) {
 	// if signal then shutdown
-	case sig := <-sigCh:
-		defer cancel()
-
+	if sig, ok := <-sigCh; ok {
 		ctlr.Log.Info().Interface("signal", sig).Msg("received signal")
 
 		// gracefully shutdown http server
 		ctlr.Shutdown() //nolint: contextcheck
 
 		close(sigCh)
-	// if reload then return
-	case <-ctx.Done():
-		return
 	}
 }
 
-func initShutDownRoutine(ctlr *api.Controller, ctx context.Context, cancel context.CancelFunc) {
+func initShutDownRoutine(ctlr *api.Controller) {
 	sigCh := make(chan os.Signal, 1)
 
-	go signalHandler(ctlr, sigCh, ctx, cancel)
+	go signalHandler(ctlr, sigCh)
 
 	// block all async signals to this server
 	signal.Ignore()
@@ -65,12 +58,10 @@ func initShutDownRoutine(ctlr *api.Controller, ctx context.Context, cancel conte
 	signal.Notify(sigCh, syscall.SIGTERM, syscall.SIGINT, syscall.SIGHUP)
 }
 
-func (hr *HotReloader) Start() context.Context {
-	reloadCtx, cancelFunc := context.WithCancel(context.Background())
-
+func (hr *HotReloader) Start() {
 	done := make(chan bool)
 
-	initShutDownRoutine(hr.ctlr, reloadCtx, cancelFunc)
+	initShutDownRoutine(hr.ctlr)
 
 	// run watcher
 	go func() {
@@ -92,16 +83,15 @@ func (hr *HotReloader) Start() context.Context {
 
 							continue
 						}
-						// if valid config then reload
-						cancelFunc()
 
-						// create new context
-						reloadCtx, cancelFunc = context.WithCancel(context.Background())
+						// stop background tasks gracefully
+						hr.ctlr.StopBackgroundTasks()
 
-						// init shutdown routine
-						initShutDownRoutine(hr.ctlr, reloadCtx, cancelFunc)
+						// load new config
+						hr.ctlr.LoadNewConfig(newConfig)
 
-						hr.ctlr.LoadNewConfig(reloadCtx, newConfig)
+						// start background tasks based on new loaded config
+						hr.ctlr.StartBackgroundTasks()
 					}
 				// watch for errors
 				case err := <-hr.watcher.Errors:
@@ -116,6 +106,4 @@ func (hr *HotReloader) Start() context.Context {
 
 		<-done
 	}()
-
-	return reloadCtx
 }

pkg/cli/server/root.go

@@ -61,20 +61,16 @@ func newServeCmd(conf *config.Config) *cobra.Command {
 				return err
 			}
 
-			/* context used to cancel go routines so that
-			we can change their config on the fly (restart routines with different config) */
-			reloaderCtx := hotReloader.Start()
+			hotReloader.Start()
 
-			if err := ctlr.Init(reloaderCtx); err != nil {
+			if err := ctlr.Init(); err != nil {
 				ctlr.Log.Error().Err(err).Msg("failed to init controller")
 
 				return err
 			}
 
-			if err := ctlr.Run(reloaderCtx); err != nil {
+			if err := ctlr.Run(); err != nil {
 				log.Error().Err(err).Msg("failed to start controller, exiting")
-
-				return err
 			}
 
 			return nil