From c298818cc2fee88738541d2a719ba2e063d62754 Mon Sep 17 00:00:00 2001 From: Asgeir Storesund Nilsen Date: Fri, 4 Jul 2025 18:13:01 +0200 Subject: [PATCH] feat: healthz server (#3228) * feat: healthz server Signed-off-by: Asgeir Nilsen * fix: startup and readiness probe activation points Enable startup probe at end of Controller.Init and readiness probe at end of Controller.Run Signed-off-by: Asgeir Nilsen * fix: rewrote to reuse same HTTP listener Signed-off-by: Asgeir Nilsen --------- Signed-off-by: Asgeir Nilsen --- go.mod | 5 ++-- go.sum | 12 +++++--- pkg/api/controller.go | 7 +++++ pkg/api/routes.go | 5 ++++ pkg/common/healthz.go | 64 +++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 87 insertions(+), 6 deletions(-) create mode 100644 pkg/common/healthz.go diff --git a/go.mod b/go.mod index f2407fde..aed43d6d 100644 --- a/go.mod +++ b/go.mod @@ -81,6 +81,7 @@ require ( k8s.io/apimachinery v0.33.2 modernc.org/sqlite v1.38.0 oras.land/oras-go/v2 v2.6.0 + sigs.k8s.io/controller-runtime v0.21.0 ) require ( @@ -520,8 +521,8 @@ require ( gopkg.in/warnings.v0 v0.1.2 // indirect helm.sh/helm/v3 v3.17.3 // indirect k8s.io/api v0.33.1 // indirect - k8s.io/apiextensions-apiserver v0.32.2 // indirect - k8s.io/apiserver v0.32.3 // indirect + k8s.io/apiextensions-apiserver v0.33.0 // indirect + k8s.io/apiserver v0.33.0 // indirect k8s.io/cli-runtime v0.33.1 // indirect k8s.io/client-go v0.33.1 // indirect k8s.io/component-base v0.33.1 // indirect diff --git a/go.sum b/go.sum index 6bc45bda..ba29dac3 100644 --- a/go.sum +++ b/go.sum @@ -1207,6 +1207,8 @@ github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/go-logr/zapr v1.3.0 h1:XGdV8XW8zdwFiwOA2Dryh1gj2KRQyOOoNmBy4EplIcQ= +github.com/go-logr/zapr v1.3.0/go.mod h1:YKepepNBd1u/oyhd/yQmtjVXmm9uML4IXUgMOwR8/Gg= github.com/go-ole/go-ole v1.3.0 h1:Dt6ye7+vXGIKZ7Xtk4s6/xVdGDQynvom7xCFEdWr6uE= github.com/go-ole/go-ole v1.3.0/go.mod h1:5LS6F96DhAwUc7C+1HLexzMXY1xGRSryjyPPKW6zv78= github.com/go-openapi/analysis v0.23.0 h1:aGday7OWupfMs+LbmLZG4k0MYXIANxcuBTYUC03zFCU= @@ -3016,12 +3018,12 @@ honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9 honnef.co/go/tools v0.1.3/go.mod h1:NgwopIslSNH47DimFoV78dnkksY2EFtX0ajyb3K/las= k8s.io/api v0.33.1 h1:tA6Cf3bHnLIrUK4IqEgb2v++/GYUtqiu9sRVk3iBXyw= k8s.io/api v0.33.1/go.mod h1:87esjTn9DRSRTD4fWMXamiXxJhpOIREjWOSjsW1kEHw= -k8s.io/apiextensions-apiserver v0.32.2 h1:2YMk285jWMk2188V2AERy5yDwBYrjgWYggscghPCvV4= -k8s.io/apiextensions-apiserver v0.32.2/go.mod h1:GPwf8sph7YlJT3H6aKUWtd0E+oyShk/YHWQHf/OOgCA= +k8s.io/apiextensions-apiserver v0.33.0 h1:d2qpYL7Mngbsc1taA4IjJPRJ9ilnsXIrndH+r9IimOs= +k8s.io/apiextensions-apiserver v0.33.0/go.mod h1:VeJ8u9dEEN+tbETo+lFkwaaZPg6uFKLGj5vyNEwwSzc= k8s.io/apimachinery v0.33.2 h1:IHFVhqg59mb8PJWTLi8m1mAoepkUNYmptHsV+Z1m5jY= k8s.io/apimachinery v0.33.2/go.mod h1:BHW0YOu7n22fFv/JkYOEfkUYNRN0fj0BlvMFWA7b+SM= -k8s.io/apiserver v0.32.3 h1:kOw2KBuHOA+wetX1MkmrxgBr648ksz653j26ESuWNY8= -k8s.io/apiserver v0.32.3/go.mod h1:q1x9B8E/WzShF49wh3ADOh6muSfpmFL0I2t+TG0Zdgc= +k8s.io/apiserver v0.33.0 h1:QqcM6c+qEEjkOODHppFXRiw/cE2zP85704YrQ9YaBbc= +k8s.io/apiserver v0.33.0/go.mod h1:EixYOit0YTxt8zrO2kBU7ixAtxFce9gKGq367nFmqI8= k8s.io/cli-runtime v0.33.1 h1:TvpjEtF71ViFmPeYMj1baZMJR4iWUEplklsUQ7D3quA= k8s.io/cli-runtime v0.33.1/go.mod h1:9dz5Q4Uh8io4OWCLiEf/217DXwqNgiTS/IOuza99VZE= k8s.io/client-go v0.33.1 h1:ZZV/Ks2g92cyxWkRRnfUDsnhNn28eFpt26aGc8KbXF4= @@ -3104,6 +3106,8 @@ rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8 rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= +sigs.k8s.io/controller-runtime v0.21.0 h1:CYfjpEuicjUecRk+KAeyYh+ouUBn4llGyDYytIGcJS8= +sigs.k8s.io/controller-runtime v0.21.0/go.mod h1:OSg14+F65eWqIu4DceX7k/+QRAbTTvxeQSNSOQpukWM= sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3 h1:/Rv+M11QRah1itp8VhT6HoVx1Ray9eB4DBr+K+/sCJ8= sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3/go.mod h1:18nIHnGi6636UCz6m8i4DhaJ65T6EruyzmoQqI2BVDo= sigs.k8s.io/kustomize/api v0.19.0 h1:F+2HB2mU1MSiR9Hp1NEgoU2q9ItNOaBJl0I4Dlus5SQ= diff --git a/pkg/api/controller.go b/pkg/api/controller.go index 46c88842..a7348cd1 100644 --- a/pkg/api/controller.go +++ b/pkg/api/controller.go @@ -55,6 +55,7 @@ type Controller struct { HTPasswdWatcher *HTPasswdWatcher LDAPClient *LDAPClient taskScheduler *scheduler.Scheduler + Healthz *common.Healthz // runtime params chosenPort int // kernel-chosen port } @@ -63,6 +64,7 @@ func NewController(appConfig *config.Config) *Controller { var controller Controller logger := log.NewLogger(appConfig.Log.Level, appConfig.Log.Output) + controller.Healthz = common.NewHealthzServer(appConfig, logger) if appConfig.Cluster != nil { // we need the set of local sockets (IP address:port) for identifying @@ -241,9 +243,13 @@ func (c *Controller) Run() error { server.TLSConfig.ClientCAs = caCertPool } + c.Healthz.Ready() + return server.ServeTLS(listener, c.Config.HTTP.TLS.Cert, c.Config.HTTP.TLS.Key) } + c.Healthz.Ready() + return server.Serve(listener) } @@ -277,6 +283,7 @@ func (c *Controller) Init() error { } c.InitCVEInfo() + c.Healthz.Started() if c.Config.IsHtpasswdAuthEnabled() { err := c.HTPasswdWatcher.ChangeFile(c.Config.HTTP.Auth.HTPasswd.Path) diff --git a/pkg/api/routes.go b/pkg/api/routes.go index d24226b1..f115362f 100644 --- a/pkg/api/routes.go +++ b/pkg/api/routes.go @@ -65,6 +65,11 @@ func NewRouteHandler(c *Controller) *RouteHandler { } func (rh *RouteHandler) SetupRoutes() { + // health endpoints get added first + rh.c.Router.Path("/livez").Handler(rh.c.Healthz.Handler) + rh.c.Router.Path("/readyz").Handler(rh.c.Healthz.Handler) + rh.c.Router.Path("/startupz").Handler(rh.c.Healthz.Handler) + // first get Auth middleware in order to first setup openid/ldap/htpasswd, before oidc provider routes are setup authHandler := AuthHandler(rh.c) diff --git a/pkg/common/healthz.go b/pkg/common/healthz.go new file mode 100644 index 00000000..878126cd --- /dev/null +++ b/pkg/common/healthz.go @@ -0,0 +1,64 @@ +package common + +import ( + "errors" + "net/http" + + "sigs.k8s.io/controller-runtime/pkg/healthz" + + "zotregistry.dev/zot/pkg/api/config" + "zotregistry.dev/zot/pkg/log" +) + +var errNotReady = errors.New("not ready yet") + +type Healthz struct { + log log.Logger + ready bool + started bool + Handler *healthz.Handler +} + +func NewHealthzServer(appConfig *config.Config, logger log.Logger) *Healthz { + var health Healthz + health.ready = false + health.started = false + health.log = logger + health.Handler = &healthz.Handler{Checks: map[string]healthz.Checker{ + "livez": health.livez, + "readyz": health.readyz, + "startupz": health.startupz, + }} + + return &health +} + +func (h *Healthz) livez(req *http.Request) error { + return nil +} + +func (h *Healthz) readyz(req *http.Request) error { + if h.ready { + return nil + } + + return errNotReady +} + +func (h *Healthz) startupz(req *http.Request) error { + if h.started { + return nil + } + + return errNotReady +} + +func (h *Healthz) Ready() { + h.ready = true + h.log.Debug().Msg("ready") +} + +func (h *Healthz) Started() { + h.started = true + h.log.Debug().Msg("started") +}