github/zot
1
0
Fork 0
mirror of https://github.com/project-zot/zot.git synced 2026-06-16 04:17:55 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(cache): dynamodb implementation (#953)

Browse Source 
Signed-off-by: Catalin Hofnar <catalin.hofnar@gmail.com>
This commit is contained in:
Catalin-George Hofnar
2022-11-22 20:29:57 +02:00
committed by GitHub
parent 49c3d05706
commit 31b9481713
22 changed files with 1746 additions and 48 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/cli/root.go
+73
View File
@@ -237,6 +237,75 @@ func validateStorageConfig(cfg *config.Config) error {
return nil
}
func validateCacheConfig(cfg *config.Config) error {
// global
// dedupe true, remote storage, remoteCache true, but no cacheDriver (remote)
//nolint: lll
if cfg.Storage.Dedupe && cfg.Storage.StorageDriver != nil && cfg.Storage.RemoteCache && cfg.Storage.CacheDriver == nil {
log.Error().Err(errors.ErrBadConfig).Msg(
"dedupe set to true with remote storage and caching, but no remote cache configured!")
return errors.ErrBadConfig
}
if cfg.Storage.CacheDriver != nil && cfg.Storage.RemoteCache {
// local storage with remote caching
if cfg.Storage.StorageDriver == nil {
log.Error().Err(errors.ErrBadConfig).Msg("cannot have local storage driver with remote caching!")
return errors.ErrBadConfig
}
// unsupported cache driver
if cfg.Storage.CacheDriver["name"] != storageConstants.DynamoDBDriverName {
log.Error().Err(errors.ErrBadConfig).Msgf("unsupported cache driver: %s", cfg.Storage.CacheDriver["name"])
return errors.ErrBadConfig
}
}
if !cfg.Storage.RemoteCache && cfg.Storage.CacheDriver != nil {
log.Warn().Err(errors.ErrBadConfig).Msgf(
"remoteCache set to false but cacheDriver config (remote caching) provided for %s,"+
"will ignore and use local caching", cfg.Storage.RootDirectory)
}
// subpaths
for _, subPath := range cfg.Storage.SubPaths {
// dedupe true, remote storage, remoteCache true, but no cacheDriver (remote)
//nolint: lll
if subPath.Dedupe && subPath.StorageDriver != nil && subPath.RemoteCache && subPath.CacheDriver == nil {
log.Error().Err(errors.ErrBadConfig).Msg("dedupe set to true with remote storage and caching, but no remote cache configured!")
return errors.ErrBadConfig
}
if subPath.CacheDriver != nil && subPath.RemoteCache {
// local storage with remote caching
if subPath.StorageDriver == nil {
log.Error().Err(errors.ErrBadConfig).Msg("cannot have local storage driver with remote caching!")
return errors.ErrBadConfig
}
// unsupported cache driver
if subPath.CacheDriver["name"] != storageConstants.DynamoDBDriverName {
log.Error().Err(errors.ErrBadConfig).Msgf("unsupported cache driver: %s", subPath.CacheDriver["name"])
return errors.ErrBadConfig
}
}
if !subPath.RemoteCache && subPath.CacheDriver != nil {
log.Warn().Err(errors.ErrBadConfig).Msgf(
"remoteCache set to false but cacheDriver config (remote caching) provided for %s,"+
"will ignore and use local caching", subPath.RootDirectory)
}
}
return nil
}
func validateConfiguration(config *config.Config) error {
if err := validateHTTP(config); err != nil {
return err
@@ -258,6 +327,10 @@ func validateConfiguration(config *config.Config) error {
return err
}
if err := validateCacheConfig(config); err != nil {
return err
}
// check authorization config, it should have basic auth enabled or ldap
if config.HTTP.RawAccessControl != nil {
// checking for anonymous policy only authorization config: no users, no policies but anonymous policy
pkg/cli/root_test.go
+271 -22
View File
@@ -117,52 +117,301 @@ func TestVerify(t *testing.T) {
So(err, ShouldBeNil)
defer os.Remove(tmpfile.Name()) // clean up
// dedup true, can't parse database type
content := []byte(`{"storage":{"rootDirectory":"/tmp/zot", "dedupe": true,
"cache": {"type": 123}},
"http":{"address":"127.0.0.1","port":"8080","realm":"zot",
"auth":{"htpasswd":{"path":"test/data/htpasswd"},"failDelay":1}}}`)
// dedupe true, remote storage, remoteCache true, but no cacheDriver (remote)
content := []byte(`{
"storage":{
"rootDirectory":"/tmp/zot",
"dedupe":true,
"remoteCache":true,
"storageDriver":{
"name":"s3",
"rootdirectory":"/zot",
"region":"us-east-2",
"bucket":"zot-storage",
"secure":true,
"skipverify":false
}
},
"http":{
"address":"127.0.0.1",
"port":"8080",
"realm":"zot",
"auth":{
"htpasswd":{
"path":"test/data/htpasswd"
},
"failDelay":1
}
}
}`)
err = os.WriteFile(tmpfile.Name(), content, 0o0600)
So(err, ShouldBeNil)
os.Args = []string{"cli_test", "verify", tmpfile.Name()}
So(func() { _ = cli.NewServerRootCmd().Execute() }, ShouldPanic)
// dedup true, wrong database type
content = []byte(`{"storage":{"rootDirectory":"/tmp/zot", "dedupe": true,
"cache": {"type": "wrong"}},
"http":{"address":"127.0.0.1","port":"8080","realm":"zot",
"auth":{"htpasswd":{"path":"test/data/htpasswd"},"failDelay":1}}}`)
// local storage with remote caching
content = []byte(`{
"storage":{
"rootDirectory":"/tmp/zot",
"dedupe":true,
"remoteCache":true,
"cacheDriver":{
"name":"dynamodb",
"endpoint":"http://localhost:4566",
"region":"us-east-2",
"tableName":"BlobTable"
}
},
"http":{
"address":"127.0.0.1",
"port":"8080",
"realm":"zot",
"auth":{
"htpasswd":{
"path":"test/data/htpasswd"
},
"failDelay":1
}
}
}`)
err = os.WriteFile(tmpfile.Name(), content, 0o0600)
So(err, ShouldBeNil)
os.Args = []string{"cli_test", "verify", tmpfile.Name()}
So(func() { _ = cli.NewServerRootCmd().Execute() }, ShouldPanic)
// unsupported cache driver
content = []byte(`{
"storage":{
"rootDirectory":"/tmp/zot",
"dedupe":true,
"remoteCache":true,
"cacheDriver":{
"name":"unsupportedDriver"
},
"storageDriver":{
"name":"s3",
"rootdirectory":"/zot",
"region":"us-east-2",
"bucket":"zot-storage",
"secure":true,
"skipverify":false
}
},
"http":{
"address":"127.0.0.1",
"port":"8080",
"realm":"zot",
"auth":{
"htpasswd":{
"path":"test/data/htpasswd"
},
"failDelay":1
}
}
}`)
err = os.WriteFile(tmpfile.Name(), content, 0o0600)
So(err, ShouldBeNil)
os.Args = []string{"cli_test", "verify", tmpfile.Name()}
So(func() { _ = cli.NewServerRootCmd().Execute() }, ShouldPanic)
// remoteCache false but provided cacheDriver config, ignored
content = []byte(`{
"storage":{
"rootDirectory":"/tmp/zot",
"dedupe":true,
"remoteCache":false,
"cacheDriver":{
"name":"dynamodb",
"endpoint":"http://localhost:4566",
"region":"us-east-2",
"tableName":"BlobTable"
},
"storageDriver":{
"name":"s3",
"rootdirectory":"/zot",
"region":"us-east-2",
"bucket":"zot-storage",
"secure":true,
"skipverify":false
}
},
"http":{
"address":"127.0.0.1",
"port":"8080",
"realm":"zot",
"auth":{
"htpasswd":{
"path":"test/data/htpasswd"
},
"failDelay":1
}
}
}`)
err = os.WriteFile(tmpfile.Name(), content, 0o0600)
So(err, ShouldBeNil)
os.Args = []string{"cli_test", "verify", tmpfile.Name()}
So(func() { _ = cli.NewServerRootCmd().Execute() }, ShouldNotPanic)
// SubPaths
// dedup true, wrong database type
content = []byte(`{"storage":{"rootDirectory":"/tmp/zot", "dedupe": false,
"subPaths": {"/a": {"rootDirectory": "/zot-a", "dedupe": true,
"cache": {"type": "wrong"}}}},
"http":{"address":"127.0.0.1","port":"8080","realm":"zot",
"auth":{"htpasswd":{"path":"test/data/htpasswd"},"failDelay":1}}}`)
// dedupe true, remote storage, remoteCache true, but no cacheDriver (remote)
content = []byte(`{
"storage":{
"rootDirectory":"/tmp/zot",
"dedupe":false,
"subPaths":{
"/a":{
"rootDirectory":"/zot-a",
"dedupe":true,
"remoteCache":true,
"storageDriver":{
"name":"s3",
"rootdirectory":"/zot",
"region":"us-east-2",
"bucket":"zot-storage",
"secure":true,
"skipverify":false
}
}
}
},
"http":{
"address":"127.0.0.1",
"port":"8080",
"realm":"zot",
"auth":{
"htpasswd":{
"path":"test/data/htpasswd"
},
"failDelay":1
}
}
}`)
err = os.WriteFile(tmpfile.Name(), content, 0o0600)
So(err, ShouldBeNil)
os.Args = []string{"cli_test", "verify", tmpfile.Name()}
So(func() { _ = cli.NewServerRootCmd().Execute() }, ShouldPanic)
// dedup true, can't parse database type
content = []byte(`{"storage":{"rootDirectory":"/tmp/zot", "dedupe": false,
"subPaths": {"/a": {"rootDirectory": "/zot-a", "dedupe": true,
"cache": {"type": 123}}}},
"http":{"address":"127.0.0.1","port":"8080","realm":"zot",
"auth":{"htpasswd":{"path":"test/data/htpasswd"},"failDelay":1}}}`)
// local storage with remote caching
content = []byte(`{
"storage":{
"rootDirectory":"/tmp/zot",
"dedupe":false,
"subPaths":{
"/a":{
"rootDirectory":"/zot-a",
"dedupe":true,
"remoteCache":true,
"cacheDriver":{
"name":"dynamodb",
"endpoint":"http://localhost:4566",
"region":"us-east-2",
"tableName":"BlobTable"
}
}
}
},
"http":{
"address":"127.0.0.1",
"port":"8080",
"realm":"zot",
"auth":{
"htpasswd":{
"path":"test/data/htpasswd"
},
"failDelay":1
}
}
}`)
err = os.WriteFile(tmpfile.Name(), content, 0o0600)
So(err, ShouldBeNil)
os.Args = []string{"cli_test", "verify", tmpfile.Name()}
So(func() { _ = cli.NewServerRootCmd().Execute() }, ShouldPanic)
// unsupported cache driver
content = []byte(`{
"storage":{
"rootDirectory":"/tmp/zot",
"dedupe":false,
"subPaths":{
"/a":{
"rootDirectory":"/zot-a",
"dedupe":true,
"remoteCache":true,
"cacheDriver":{
"name":"badDriverName"
},
"storageDriver":{
"name":"s3",
"rootdirectory":"/zot",
"region":"us-east-2",
"bucket":"zot-storage",
"secure":true,
"skipverify":false
}
}
}
},
"http":{
"address":"127.0.0.1",
"port":"8080",
"realm":"zot",
"auth":{
"htpasswd":{
"path":"test/data/htpasswd"
},
"failDelay":1
}
}
}`)
err = os.WriteFile(tmpfile.Name(), content, 0o0600)
So(err, ShouldBeNil)
os.Args = []string{"cli_test", "verify", tmpfile.Name()}
So(func() { _ = cli.NewServerRootCmd().Execute() }, ShouldPanic)
// remoteCache false but provided cacheDriver config, ignored
content = []byte(`{
"storage":{
"rootDirectory":"/tmp/zot",
"dedupe":false,
"subPaths":{
"/a":{
"rootDirectory":"/zot-a",
"dedupe":true,
"remoteCache":false,
"cacheDriver":{
"name":"dynamodb",
"endpoint":"http://localhost:4566",
"region":"us-east-2",
"tableName":"BlobTable"
}
}
}
},
"http":{
"address":"127.0.0.1",
"port":"8080",
"realm":"zot",
"auth":{
"htpasswd":{
"path":"test/data/htpasswd"
},
"failDelay":1
}
}
}`)
err = os.WriteFile(tmpfile.Name(), content, 0o0600)
So(err, ShouldBeNil)
os.Args = []string{"cli_test", "verify", tmpfile.Name()}
So(func() { _ = cli.NewServerRootCmd().Execute() }, ShouldNotPanic)
})
Convey("Test apply defaults cache db", t, func(c C) {