github/zot
1
0
Fork 0
mirror of https://github.com/project-zot/zot.git synced 2026-06-15 20:07:55 +08:00
Code Issues Packages Projects Releases Wiki Activity

Added new extension "sync"

Browse Source 
Periodically poll registries and pull images according to sync's config
Added sync on demand, syncing when clients asks for an image which
zot doesn't have.

Signed-off-by: Petu Eusebiu <peusebiu@cisco.com>
This commit is contained in:
Petu Eusebiu
2021-06-08 23:11:18 +03:00
committed by Ramkumar Chinchani
parent 1027f872ec
commit 19003e8a71
34 changed files with 3158 additions and 339 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/cli/client_test.go
+19 -18
View File
@@ -20,6 +20,7 @@ import (
"time"
"github.com/anuvu/zot/pkg/api"
"github.com/anuvu/zot/pkg/api/config"
. "github.com/smartystreets/goconvey/convey"
)
@@ -67,24 +68,24 @@ func TestTLSWithAuth(t *testing.T) {
resty.SetTLSClientConfig(&tls.Config{RootCAs: caCertPool})
defer func() { resty.SetTLSClientConfig(nil) }()
config := api.NewConfig()
config.HTTP.Port = SecurePort1
conf := config.New()
conf.HTTP.Port = SecurePort1
htpasswdPath := makeHtpasswdFile()
defer os.Remove(htpasswdPath)
config.HTTP.Auth = &api.AuthConfig{
HTPasswd: api.AuthHTPasswd{
conf.HTTP.Auth = &config.AuthConfig{
HTPasswd: config.AuthHTPasswd{
Path: htpasswdPath,
},
}
config.HTTP.TLS = &api.TLSConfig{
conf.HTTP.TLS = &config.TLSConfig{
Cert: ServerCert,
Key: ServerKey,
CACert: CACert,
}
c := api.NewController(config)
c := api.NewController(conf)
dir, err := ioutil.TempDir("", "oci-repo-test")
if err != nil {
panic(err)
@@ -173,15 +174,15 @@ func TestTLSWithoutAuth(t *testing.T) {
resty.SetTLSClientConfig(&tls.Config{RootCAs: caCertPool})
defer func() { resty.SetTLSClientConfig(nil) }()
config := api.NewConfig()
config.HTTP.Port = SecurePort1
config.HTTP.TLS = &api.TLSConfig{
conf := config.New()
conf.HTTP.Port = SecurePort1
conf.HTTP.TLS = &config.TLSConfig{
Cert: ServerCert,
Key: ServerKey,
CACert: CACert,
}
c := api.NewController(config)
c := api.NewController(conf)
dir, err := ioutil.TempDir("", "oci-repo-test")
if err != nil {
panic(err)
@@ -241,15 +242,15 @@ func TestTLSWithoutAuth(t *testing.T) {
resty.SetTLSClientConfig(&tls.Config{RootCAs: caCertPool})
defer func() { resty.SetTLSClientConfig(nil) }()
config := api.NewConfig()
config.HTTP.Port = SecurePort2
config.HTTP.TLS = &api.TLSConfig{
conf := config.New()
conf.HTTP.Port = SecurePort2
conf.HTTP.TLS = &config.TLSConfig{
Cert: ServerCert,
Key: ServerKey,
CACert: CACert,
}
c := api.NewController(config)
c := api.NewController(conf)
dir, err := ioutil.TempDir("", "oci-repo-test")
if err != nil {
panic(err)
@@ -304,15 +305,15 @@ func TestTLSBadCerts(t *testing.T) {
resty.SetTLSClientConfig(&tls.Config{RootCAs: caCertPool})
defer func() { resty.SetTLSClientConfig(nil) }()
config := api.NewConfig()
config.HTTP.Port = SecurePort3
config.HTTP.TLS = &api.TLSConfig{
conf := config.New()
conf.HTTP.Port = SecurePort3
conf.HTTP.TLS = &config.TLSConfig{
Cert: ServerCert,
Key: ServerKey,
CACert: CACert,
}
c := api.NewController(config)
c := api.NewController(conf)
dir, err := ioutil.TempDir("", "oci-repo-test")
if err != nil {
panic(err)
pkg/cli/cve_cmd_test.go
+8 -7
View File
@@ -16,7 +16,8 @@ import (
zotErrors "github.com/anuvu/zot/errors"
"github.com/anuvu/zot/pkg/api"
ext "github.com/anuvu/zot/pkg/extensions"
"github.com/anuvu/zot/pkg/api/config"
extconf "github.com/anuvu/zot/pkg/extensions/config"
"gopkg.in/resty.v1"
. "github.com/smartystreets/goconvey/convey"
@@ -286,9 +287,9 @@ func TestSearchCVECmd(t *testing.T) {
func TestServerCVEResponse(t *testing.T) {
port := getFreePort()
url := getBaseURL(port)
config := api.NewConfig()
config.HTTP.Port = port
c := api.NewController(config)
conf := config.New()
conf.HTTP.Port = port
c := api.NewController(conf)
dir, err := ioutil.TempDir("", "oci-repo-test")
if err != nil {
@@ -303,14 +304,14 @@ func TestServerCVEResponse(t *testing.T) {
defer os.RemoveAll(dir)
c.Config.Storage.RootDirectory = dir
cveConfig := &ext.CVEConfig{
cveConfig := &extconf.CVEConfig{
UpdateInterval: 2,
}
searchConfig := &ext.SearchConfig{
searchConfig := &extconf.SearchConfig{
CVE: cveConfig,
Enable: true,
}
c.Config.Extensions = &ext.ExtensionConfig{
c.Config.Extensions = &extconf.ExtensionConfig{
Search: searchConfig,
}
pkg/cli/image_cmd_test.go
+7 -6
View File
@@ -21,8 +21,9 @@ import (
zotErrors "github.com/anuvu/zot/errors"
"github.com/anuvu/zot/pkg/api"
"github.com/anuvu/zot/pkg/api/config"
"github.com/anuvu/zot/pkg/compliance/v1_0_0"
"github.com/anuvu/zot/pkg/extensions"
extconf "github.com/anuvu/zot/pkg/extensions/config"
godigest "github.com/opencontainers/go-digest"
ispec "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/phayes/freeport"
@@ -302,12 +303,12 @@ func TestServerResponse(t *testing.T) {
Convey("Test from real server", t, func() {
port := getFreePort()
url := getBaseURL(port)
config := api.NewConfig()
config.HTTP.Port = port
config.Extensions = &extensions.ExtensionConfig{
Search: &extensions.SearchConfig{Enable: true},
conf := config.New()
conf.HTTP.Port = port
conf.Extensions = &extconf.ExtensionConfig{
Search: &extconf.SearchConfig{Enable: true},
}
c := api.NewController(config)
c := api.NewController(conf)
dir, err := ioutil.TempDir("", "oci-repo-test")
if err != nil {
panic(err)
pkg/cli/root.go
+24 -12
View File
@@ -3,6 +3,7 @@ package cli
import (
"github.com/anuvu/zot/errors"
"github.com/anuvu/zot/pkg/api"
"github.com/anuvu/zot/pkg/api/config"
"github.com/anuvu/zot/pkg/storage"
"github.com/fsnotify/fsnotify"
"github.com/mitchellh/mapstructure"
@@ -22,7 +23,7 @@ func metadataConfig(md *mapstructure.Metadata) viper.DecoderConfigOption {
func NewRootCmd() *cobra.Command {
showVersion := false
config := api.NewConfig()
conf := config.New()
// "serve"
serveCmd := &cobra.Command{
@@ -32,9 +33,9 @@ func NewRootCmd() *cobra.Command {
Long: "`serve` stores and distributes OCI images",
Run: func(cmd *cobra.Command, args []string) {
if len(args) > 0 {
LoadConfiguration(config, args[0])
LoadConfiguration(conf, args[0])
}
c := api.NewController(config)
c := api.NewController(conf)
// creates a new file watcher
watcher, err := fsnotify.NewWatcher()
@@ -53,7 +54,7 @@ func NewRootCmd() *cobra.Command {
case event := <-watcher.Events:
if event.Op == fsnotify.Write {
log.Info().Msg("Config file changed, trying to reload accessControl config")
newConfig := api.NewConfig()
newConfig := config.New()
LoadConfiguration(newConfig, args[0])
c.Config.AccessControl = newConfig.AccessControl
}
@@ -85,7 +86,7 @@ func NewRootCmd() *cobra.Command {
Long: "`verify` validates a zot config file",
Run: func(cmd *cobra.Command, args []string) {
if len(args) > 0 {
config := api.NewConfig()
config := config.New()
LoadConfiguration(config, args[0])
log.Info().Msgf("Config file %s is valid", args[0])
}
@@ -102,16 +103,16 @@ func NewRootCmd() *cobra.Command {
Short: "`garbage-collect` deletes layers not referenced by any manifests",
Long: "`garbage-collect` deletes layers not referenced by any manifests",
Run: func(cmd *cobra.Command, args []string) {
log.Info().Interface("values", config).Msg("configuration settings")
if config.Storage.RootDirectory != "" {
if err := storage.Scrub(config.Storage.RootDirectory, gcDryRun); err != nil {
log.Info().Interface("values", conf).Msg("configuration settings")
if conf.Storage.RootDirectory != "" {
if err := storage.Scrub(conf.Storage.RootDirectory, gcDryRun); err != nil {
panic(err)
}
}
},
}
gcCmd.Flags().StringVarP(&config.Storage.RootDirectory, "storage-root-dir", "r", "",
gcCmd.Flags().StringVarP(&conf.Storage.RootDirectory, "storage-root-dir", "r", "",
"Use specified directory for filestore backing image data")
_ = gcCmd.MarkFlagRequired("storage-root-dir")
@@ -126,8 +127,8 @@ func NewRootCmd() *cobra.Command {
Long: "`zot`",
Run: func(cmd *cobra.Command, args []string) {
if showVersion {
log.Info().Str("distribution-spec", distspec.Version).Str("commit", api.Commit).
Str("binary-type", api.BinaryType).Msg("version")
log.Info().Str("distribution-spec", distspec.Version).Str("commit", config.Commit).
Str("binary-type", config.BinaryType).Msg("version")
}
_ = cmd.Usage()
cmd.SilenceErrors = false
@@ -145,7 +146,7 @@ func NewRootCmd() *cobra.Command {
return rootCmd
}
func LoadConfiguration(config *api.Config, configPath string) {
func LoadConfiguration(config *config.Config, configPath string) {
viper.SetConfigFile(configPath)
if err := viper.ReadInConfig(); err != nil {
@@ -178,4 +179,15 @@ func LoadConfiguration(config *api.Config, configPath string) {
log.Error().Err(errors.ErrBadConfig).Msg("Unable to unmarshal http.accessControl.key.policies")
panic(err)
}
// defaults
defualtTLSVerify := true
if config.Extensions != nil && config.Extensions.Sync != nil {
for id, regCfg := range config.Extensions.Sync.Registries {
if regCfg.TLSVerify == nil {
config.Extensions.Sync.Registries[id].TLSVerify = &defualtTLSVerify
}
}
}
}
pkg/cli/root_test.go
+2 -2
View File
@@ -6,7 +6,7 @@ import (
"path"
"testing"
"github.com/anuvu/zot/pkg/api"
"github.com/anuvu/zot/pkg/api/config"
"github.com/anuvu/zot/pkg/cli"
. "github.com/smartystreets/goconvey/convey"
"github.com/spf13/viper"
@@ -137,7 +137,7 @@ func TestVerify(t *testing.T) {
func TestLoadConfig(t *testing.T) {
Convey("Test viper load config", t, func(c C) {
config := api.NewConfig()
config := config.New()
So(func() { cli.LoadConfiguration(config, "../../examples/config-policy.json") }, ShouldNotPanic)
adminPolicy := viper.GetStringMapStringSlice("http.accessControl.adminPolicy")
So(config.AccessControl.AdminPolicy.Actions, ShouldResemble, adminPolicy["actions"])