sync: support reloading sync config when the config file changes

Signed-off-by: Petu Eusebiu <peusebiu@cisco.com>
This commit is contained in:
Petu Eusebiu
2022-02-10 16:17:49 +02:00
committed by Ramkumar Chinchani
parent 7e8cc3c71c
commit 6d04ab3cdc
11 changed files with 728 additions and 99 deletions
+72
View File
@@ -0,0 +1,72 @@
package cli
import (
"github.com/fsnotify/fsnotify"
"github.com/rs/zerolog/log"
"zotregistry.io/zot/pkg/api"
"zotregistry.io/zot/pkg/api/config"
)
type HotReloader struct {
watcher *fsnotify.Watcher
filePath string
ctlr *api.Controller
}
func NewHotReloader(ctlr *api.Controller, filePath string) (*HotReloader, error) {
// creates a new file watcher
watcher, err := fsnotify.NewWatcher()
if err != nil {
return nil, err
}
hotReloader := &HotReloader{
watcher: watcher,
filePath: filePath,
ctlr: ctlr,
}
return hotReloader, nil
}
func (hr *HotReloader) Start() {
done := make(chan bool)
// run watcher
go func() {
defer hr.watcher.Close()
go func() {
for {
select {
// watch for events
case event := <-hr.watcher.Events:
if event.Op == fsnotify.Write {
log.Info().Msg("config file changed, trying to reload config")
newConfig := config.New()
err := LoadConfiguration(newConfig, hr.filePath)
if err != nil {
log.Error().Err(err).Msg("couldn't reload config, retry writing it.")
continue
}
hr.ctlr.LoadNewConfig(newConfig)
}
// watch for errors
case err := <-hr.watcher.Errors:
log.Error().Err(err).Msgf("fsnotfy error while watching config %s", hr.filePath)
panic(err)
}
}
}()
if err := hr.watcher.Add(hr.filePath); err != nil {
log.Error().Err(err).Msgf("error adding config file %s to FsNotify watcher", hr.filePath)
panic(err)
}
<-done
}()
}
+384
View File
@@ -0,0 +1,384 @@
package cli_test
import (
"fmt"
"io"
"io/ioutil"
"os"
"testing"
"time"
. "github.com/smartystreets/goconvey/convey"
"golang.org/x/crypto/bcrypt"
"zotregistry.io/zot/pkg/cli"
"zotregistry.io/zot/pkg/test"
)
func TestConfigReloader(t *testing.T) {
oldArgs := os.Args
defer func() { os.Args = oldArgs }()
Convey("reload access control config", t, func(c C) {
port := test.GetFreePort()
baseURL := test.GetBaseURL(port)
logFile, err := ioutil.TempFile("", "zot-log*.txt")
So(err, ShouldBeNil)
username := "alice"
password := "alice"
hash, err := bcrypt.GenerateFromPassword([]byte(password), 10)
if err != nil {
panic(err)
}
usernameAndHash := fmt.Sprintf("%s:%s", username, string(hash))
htpasswdPath := test.MakeHtpasswdFileFromString(usernameAndHash)
defer os.Remove(htpasswdPath)
defer os.Remove(logFile.Name()) // clean up
content := fmt.Sprintf(`{
"distSpecVersion": "0.1.0-dev",
"storage": {
"rootDirectory": "/tmp/zot"
},
"http": {
"address": "127.0.0.1",
"port": "%s",
"realm": "zot",
"auth": {
"htpasswd": {
"path": "%s"
},
"failDelay": 1
},
"accessControl": {
"**": {
"policies": [
{
"users": ["charlie"],
"actions": ["read"]
}
],
"defaultPolicy": ["read", "create"]
},
"adminPolicy": {
"users": ["admin"],
"actions": ["read", "create", "update", "delete"]
}
}
},
"log": {
"level": "debug",
"output": "%s"
}
}`, port, htpasswdPath, logFile.Name())
cfgfile, err := ioutil.TempFile("", "zot-test*.json")
So(err, ShouldBeNil)
defer os.Remove(cfgfile.Name()) // clean up
_, err = cfgfile.Write([]byte(content))
So(err, ShouldBeNil)
// err = cfgfile.Close()
// So(err, ShouldBeNil)
os.Args = []string{"cli_test", "serve", cfgfile.Name()}
go func() {
err = cli.NewServerRootCmd().Execute()
So(err, ShouldBeNil)
}()
test.WaitTillServerReady(baseURL)
content = fmt.Sprintf(`{
"distSpecVersion": "0.1.0-dev",
"storage": {
"rootDirectory": "/tmp/zot"
},
"http": {
"address": "127.0.0.1",
"port": "%s",
"realm": "zot",
"auth": {
"htpasswd": {
"path": "%s"
},
"failDelay": 1
},
"accessControl": {
"**": {
"policies": [
{
"users": ["alice"],
"actions": ["read", "create", "update", "delete"]
}
],
"defaultPolicy": ["read"]
},
"adminPolicy": {
"users": ["admin"],
"actions": ["read", "create", "update", "delete"]
}
}
},
"log": {
"level": "debug",
"output": "%s"
}
}`, port, htpasswdPath, logFile.Name())
err = cfgfile.Truncate(0)
So(err, ShouldBeNil)
_, err = cfgfile.Seek(0, io.SeekStart)
So(err, ShouldBeNil)
_, err = cfgfile.WriteString(content)
So(err, ShouldBeNil)
err = cfgfile.Close()
So(err, ShouldBeNil)
// wait for config reload
time.Sleep(2 * time.Second)
data, err := os.ReadFile(logFile.Name())
So(err, ShouldBeNil)
So(string(data), ShouldContainSubstring, "reloaded params")
So(string(data), ShouldContainSubstring, "new configuration settings")
So(string(data), ShouldContainSubstring, "\"Users\":[\"alice\"]")
So(string(data), ShouldContainSubstring, "\"Actions\":[\"read\",\"create\",\"update\",\"delete\"]")
})
Convey("reload sync config", t, func(c C) {
port := test.GetFreePort()
baseURL := test.GetBaseURL(port)
logFile, err := ioutil.TempFile("", "zot-log*.txt")
So(err, ShouldBeNil)
defer os.Remove(logFile.Name()) // clean up
content := fmt.Sprintf(`{
"distSpecVersion": "0.1.0-dev",
"storage": {
"rootDirectory": "/tmp/zot"
},
"http": {
"address": "127.0.0.1",
"port": "%s"
},
"log": {
"level": "debug",
"output": "%s"
},
"extensions": {
"sync": {
"registries": [{
"urls": ["http://localhost:8080"],
"tlsVerify": false,
"onDemand": true,
"maxRetries": 3,
"retryDelay": "15m",
"certDir": "",
"content":[
{
"prefix": "zot-test",
"tags": {
"regex": ".*",
"semver": true
}
}
]
}]
}
}
}`, port, logFile.Name())
cfgfile, err := ioutil.TempFile("", "zot-test*.json")
So(err, ShouldBeNil)
defer os.Remove(cfgfile.Name()) // clean up
_, err = cfgfile.Write([]byte(content))
So(err, ShouldBeNil)
// err = cfgfile.Close()
// So(err, ShouldBeNil)
os.Args = []string{"cli_test", "serve", cfgfile.Name()}
go func() {
err = cli.NewServerRootCmd().Execute()
So(err, ShouldBeNil)
}()
test.WaitTillServerReady(baseURL)
content = fmt.Sprintf(`{
"distSpecVersion": "0.1.0-dev",
"storage": {
"rootDirectory": "/tmp/zot"
},
"http": {
"address": "127.0.0.1",
"port": "%s"
},
"log": {
"level": "debug",
"output": "%s"
},
"extensions": {
"sync": {
"registries": [{
"urls": ["http://localhost:9999"],
"tlsVerify": true,
"onDemand": false,
"maxRetries": 10,
"retryDelay": "5m",
"certDir": "certs",
"content":[
{
"prefix": "zot-cve-test",
"tags": {
"regex": "tag",
"semver": false
}
}
]
}]
}
}
}`, port, logFile.Name())
err = cfgfile.Truncate(0)
So(err, ShouldBeNil)
_, err = cfgfile.Seek(0, io.SeekStart)
So(err, ShouldBeNil)
_, err = cfgfile.WriteString(content)
So(err, ShouldBeNil)
err = cfgfile.Close()
So(err, ShouldBeNil)
// wait for config reload
time.Sleep(2 * time.Second)
data, err := os.ReadFile(logFile.Name())
So(err, ShouldBeNil)
So(string(data), ShouldContainSubstring, "reloaded params")
So(string(data), ShouldContainSubstring, "new configuration settings")
So(string(data), ShouldContainSubstring, "\"URLs\":[\"http://localhost:9999\"]")
So(string(data), ShouldContainSubstring, "\"TLSVerify\":true")
So(string(data), ShouldContainSubstring, "\"OnDemand\":false")
So(string(data), ShouldContainSubstring, "\"MaxRetries\":10")
So(string(data), ShouldContainSubstring, "\"RetryDelay\":300000000000")
So(string(data), ShouldContainSubstring, "\"CertDir\":\"certs\"")
So(string(data), ShouldContainSubstring, "\"Prefix\":\"zot-cve-test\"")
So(string(data), ShouldContainSubstring, "\"Regex\":\"tag\"")
So(string(data), ShouldContainSubstring, "\"Semver\":false")
})
Convey("reload bad config", t, func(c C) {
port := test.GetFreePort()
baseURL := test.GetBaseURL(port)
logFile, err := ioutil.TempFile("", "zot-log*.txt")
So(err, ShouldBeNil)
defer os.Remove(logFile.Name()) // clean up
content := fmt.Sprintf(`{
"distSpecVersion": "0.1.0-dev",
"storage": {
"rootDirectory": "/tmp/zot"
},
"http": {
"address": "127.0.0.1",
"port": "%s"
},
"log": {
"level": "debug",
"output": "%s"
},
"extensions": {
"sync": {
"registries": [{
"urls": ["http://localhost:8080"],
"tlsVerify": false,
"onDemand": true,
"maxRetries": 3,
"retryDelay": "15m",
"certDir": "",
"content":[
{
"prefix": "zot-test",
"tags": {
"regex": ".*",
"semver": true
}
}
]
}]
}
}
}`, port, logFile.Name())
cfgfile, err := ioutil.TempFile("", "zot-test*.json")
So(err, ShouldBeNil)
defer os.Remove(cfgfile.Name()) // clean up
_, err = cfgfile.Write([]byte(content))
So(err, ShouldBeNil)
// err = cfgfile.Close()
// So(err, ShouldBeNil)
os.Args = []string{"cli_test", "serve", cfgfile.Name()}
go func() {
err = cli.NewServerRootCmd().Execute()
So(err, ShouldBeNil)
}()
test.WaitTillServerReady(baseURL)
content = "[]"
err = cfgfile.Truncate(0)
So(err, ShouldBeNil)
_, err = cfgfile.Seek(0, io.SeekStart)
So(err, ShouldBeNil)
_, err = cfgfile.WriteString(content)
So(err, ShouldBeNil)
err = cfgfile.Close()
So(err, ShouldBeNil)
// wait for config reload
time.Sleep(2 * time.Second)
data, err := os.ReadFile(logFile.Name())
So(err, ShouldBeNil)
So(string(data), ShouldNotContainSubstring, "reloaded params")
So(string(data), ShouldNotContainSubstring, "new configuration settings")
So(string(data), ShouldContainSubstring, "\"URLs\":[\"http://localhost:8080\"]")
So(string(data), ShouldContainSubstring, "\"TLSVerify\":false")
So(string(data), ShouldContainSubstring, "\"OnDemand\":true")
So(string(data), ShouldContainSubstring, "\"MaxRetries\":3")
So(string(data), ShouldContainSubstring, "\"CertDir\":\"\"")
So(string(data), ShouldContainSubstring, "\"Prefix\":\"zot-test\"")
So(string(data), ShouldContainSubstring, "\"Regex\":\".*\"")
So(string(data), ShouldContainSubstring, "\"Semver\":true")
})
}
+45 -50
View File
@@ -7,7 +7,6 @@ import (
"time"
glob "github.com/bmatcuk/doublestar/v4"
"github.com/fsnotify/fsnotify"
"github.com/mitchellh/mapstructure"
distspec "github.com/opencontainers/distribution-spec/specs-go"
"github.com/rs/zerolog/log"
@@ -38,46 +37,19 @@ func newServeCmd(conf *config.Config) *cobra.Command {
Long: "`serve` stores and distributes OCI images",
Run: func(cmd *cobra.Command, args []string) {
if len(args) > 0 {
LoadConfiguration(conf, args[0])
if err := LoadConfiguration(conf, args[0]); err != nil {
panic(err)
}
}
ctlr := api.NewController(conf)
// creates a new file watcher
watcher, err := fsnotify.NewWatcher()
hotReloader, err := NewHotReloader(ctlr, args[0])
if err != nil {
panic(err)
}
defer watcher.Close()
done := make(chan bool)
// run watcher
go func() {
go func() {
for {
select {
// watch for events
case event := <-watcher.Events:
if event.Op == fsnotify.Write {
log.Info().Msg("config file changed, trying to reload accessControl config")
newConfig := config.New()
LoadConfiguration(newConfig, args[0])
ctlr.Config.AccessControl = newConfig.AccessControl
}
// watch for errors
case err := <-watcher.Errors:
log.Error().Err(err).Msgf("FsNotify error while watching config %s", args[0])
panic(err)
}
}
}()
if err := watcher.Add(args[0]); err != nil {
log.Error().Err(err).Msgf("error adding config file %s to FsNotify watcher", args[0])
panic(err)
}
<-done
}()
hotReloader.Start()
if err := ctlr.Run(); err != nil {
panic(err)
@@ -97,7 +69,9 @@ func newScrubCmd(conf *config.Config) *cobra.Command {
Long: "`scrub` checks manifest/blob integrity",
Run: func(cmd *cobra.Command, args []string) {
if len(args) > 0 {
LoadConfiguration(conf, args[0])
if err := LoadConfiguration(conf, args[0]); err != nil {
panic(err)
}
} else {
if err := cmd.Usage(); err != nil {
panic(err)
@@ -152,7 +126,10 @@ func newVerifyCmd(conf *config.Config) *cobra.Command {
Long: "`verify` validates a zot config file",
Run: func(cmd *cobra.Command, args []string) {
if len(args) > 0 {
LoadConfiguration(conf, args[0])
if err := LoadConfiguration(conf, args[0]); err != nil {
panic(err)
}
log.Info().Msgf("Config file %s is valid", args[0])
}
},
@@ -220,12 +197,13 @@ func NewCliRootCmd() *cobra.Command {
return rootCmd
}
func validateConfiguration(config *config.Config) {
func validateConfiguration(config *config.Config) error {
// enforce GC params
if config.Storage.GCDelay < 0 {
log.Error().Err(errors.ErrBadConfig).
Msgf("invalid garbage-collect delay %v specified", config.Storage.GCDelay)
panic(errors.ErrBadConfig)
return errors.ErrBadConfig
}
if !config.Storage.GC && config.Storage.GCDelay != 0 {
@@ -238,7 +216,8 @@ func validateConfiguration(config *config.Config) {
if config.HTTP.Auth == nil || (config.HTTP.Auth.HTPasswd.Path == "" && config.HTTP.Auth.LDAP == nil) {
log.Error().Err(errors.ErrBadConfig).
Msg("access control config requires httpasswd or ldap authentication to be enabled")
panic(errors.ErrBadConfig)
return errors.ErrBadConfig
}
}
@@ -246,13 +225,15 @@ func validateConfiguration(config *config.Config) {
// enforce s3 driver in case of using storage driver
if config.Storage.StorageDriver["name"] != storage.S3StorageDriverName {
log.Error().Err(errors.ErrBadConfig).Msgf("unsupported storage driver: %s", config.Storage.StorageDriver["name"])
panic(errors.ErrBadConfig)
return errors.ErrBadConfig
}
// enforce filesystem storage in case sync feature is enabled
if config.Extensions != nil && config.Extensions.Sync != nil {
log.Error().Err(errors.ErrBadConfig).Msg("sync supports only filesystem storage")
panic(errors.ErrBadConfig)
return errors.ErrBadConfig
}
}
@@ -263,7 +244,8 @@ func validateConfiguration(config *config.Config) {
if regCfg.MaxRetries != nil && regCfg.RetryDelay == nil {
log.Error().Err(errors.ErrBadConfig).Msgf("extensions.sync.registries[%d].retryDelay"+
" is required when using extensions.sync.registries[%d].maxRetries", id, id)
panic(errors.ErrBadConfig)
return errors.ErrBadConfig
}
if regCfg.Content != nil {
@@ -271,7 +253,8 @@ func validateConfiguration(config *config.Config) {
ok := glob.ValidatePattern(content.Prefix)
if !ok {
log.Error().Err(glob.ErrBadPattern).Str("pattern", content.Prefix).Msg("sync pattern could not be compiled")
panic(errors.ErrBadConfig)
return glob.ErrBadPattern
}
}
}
@@ -288,7 +271,8 @@ func validateConfiguration(config *config.Config) {
if storageConfig.StorageDriver["name"] != storage.S3StorageDriverName {
log.Error().Err(errors.ErrBadConfig).Str("subpath",
route).Msgf("unsupported storage driver: %s", storageConfig.StorageDriver["name"])
panic(errors.ErrBadConfig)
return errors.ErrBadConfig
}
}
}
@@ -301,10 +285,13 @@ func validateConfiguration(config *config.Config) {
ok := glob.ValidatePattern(pattern)
if !ok {
log.Error().Err(glob.ErrBadPattern).Str("pattern", pattern).Msg("authorization pattern could not be compiled")
panic(errors.ErrBadConfig)
return glob.ErrBadPattern
}
}
}
return nil
}
func applyDefaultValues(config *config.Config, viperInstance *viper.Viper) {
@@ -382,7 +369,7 @@ func applyDefaultValues(config *config.Config, viperInstance *viper.Viper) {
}
}
func LoadConfiguration(config *config.Config, configPath string) {
func LoadConfiguration(config *config.Config, configPath string) error {
// Default is dot (.) but because we allow glob patterns in authz
// we need another key delimiter.
viperInstance := viper.NewWithOptions(viper.KeyDelimiter("::"))
@@ -391,29 +378,37 @@ func LoadConfiguration(config *config.Config, configPath string) {
if err := viperInstance.ReadInConfig(); err != nil {
log.Error().Err(err).Msg("error while reading configuration")
panic(err)
return err
}
metaData := &mapstructure.Metadata{}
if err := viperInstance.Unmarshal(&config, metadataConfig(metaData)); err != nil {
log.Error().Err(err).Msg("error while unmarshalling new config")
panic(err)
return err
}
if len(metaData.Keys) == 0 || len(metaData.Unused) > 0 {
log.Error().Err(errors.ErrBadConfig).Msg("bad configuration, retry writing it")
panic(errors.ErrBadConfig)
return errors.ErrBadConfig
}
err := config.LoadAccessControlConfig(viperInstance)
if err != nil {
log.Error().Err(err).Msg("unable to unmarshal config's accessControl")
panic(err)
return err
}
// defaults
applyDefaultValues(config, viperInstance)
// various config checks
validateConfiguration(config)
if err := validateConfiguration(config); err != nil {
return err
}
return nil
}
+15 -8
View File
@@ -234,7 +234,7 @@ func TestVerify(t *testing.T) {
content := []byte(`{"storage":{"rootDirectory":"/tmp/zot"},
"http":{"address":"127.0.0.1","port":"8080","realm":"zot",
"auth":{"htpasswd":{"path":"test/data/htpasswd"},"failDelay":1},
"accessControl":{"\|":{"policies":[],"defaultPolicy":[]}}}}`)
"accessControl":{"[":{"policies":[],"defaultPolicy":[]}}}}`)
_, err = tmpfile.Write(content)
So(err, ShouldBeNil)
err = tmpfile.Close()
@@ -299,16 +299,19 @@ func TestVerify(t *testing.T) {
func TestLoadConfig(t *testing.T) {
Convey("Test viper load config", t, func(c C) {
config := config.New()
So(func() { cli.LoadConfiguration(config, "../../examples/config-policy.json") }, ShouldNotPanic)
err := cli.LoadConfiguration(config, "../../examples/config-policy.json")
So(err, ShouldBeNil)
})
}
func TestGC(t *testing.T) {
Convey("Test GC config", t, func(c C) {
config := config.New()
So(func() { cli.LoadConfiguration(config, "../../examples/config-multiple.json") }, ShouldNotPanic)
err := cli.LoadConfiguration(config, "../../examples/config-multiple.json")
So(err, ShouldBeNil)
So(config.Storage.GCDelay, ShouldEqual, storage.DefaultGCDelay)
So(func() { cli.LoadConfiguration(config, "../../examples/config-gc.json") }, ShouldNotPanic)
err = cli.LoadConfiguration(config, "../../examples/config-gc.json")
So(err, ShouldBeNil)
So(config.Storage.GCDelay, ShouldNotEqual, storage.DefaultGCDelay)
})
@@ -330,7 +333,8 @@ func TestGC(t *testing.T) {
err = ioutil.WriteFile(file.Name(), contents, 0o600)
So(err, ShouldBeNil)
So(func() { cli.LoadConfiguration(config, file.Name()) }, ShouldNotPanic)
err = cli.LoadConfiguration(config, file.Name())
So(err, ShouldBeNil)
})
Convey("Negative GC delay", func() {
@@ -347,7 +351,8 @@ func TestGC(t *testing.T) {
err = ioutil.WriteFile(file.Name(), contents, 0o600)
So(err, ShouldBeNil)
So(func() { cli.LoadConfiguration(config, file.Name()) }, ShouldPanic)
err = cli.LoadConfiguration(config, file.Name())
So(err, ShouldNotBeNil)
})
})
}
@@ -547,7 +552,8 @@ func TestApplyDefaultValues(t *testing.T) {
err = os.Chmod(file.Name(), 0o777)
So(err, ShouldBeNil)
cli.LoadConfiguration(oldConfig, file.Name())
err = cli.LoadConfiguration(oldConfig, file.Name())
So(err, ShouldBeNil)
configContent, err = ioutil.ReadFile(file.Name())
So(err, ShouldBeNil)
@@ -563,7 +569,8 @@ func TestApplyDefaultValues(t *testing.T) {
err = os.Chmod(file.Name(), 0o444)
So(err, ShouldBeNil)
cli.LoadConfiguration(oldConfig, file.Name())
err = cli.LoadConfiguration(oldConfig, file.Name())
So(err, ShouldBeNil)
configContent, err = ioutil.ReadFile(file.Name())
So(err, ShouldBeNil)