mirror of
https://github.com/project-zot/zot.git
synced 2026-06-18 05:28:07 +08:00
05823cd74f
* feat: add redis cache support https://github.com/project-zot/zot/pull/2005 Fixes https://github.com/project-zot/zot/issues/2004 * feat: add redis cache support Currently, we have dynamoDB as the remote shared cache but ideal only for the cloud use case. For on-prem use case, add support for redis. Signed-off-by: Ramkumar Chinchani <rchincha@cisco.com> * feat(redis): added blackbox tests for redis Signed-off-by: Petu Eusebiu <peusebiu@cisco.com> * feat(redis): dummy implementation of MetaDB interface for redis cache Signed-off-by: Alexei Dodon <adodon@cisco.com> * feat: check validity of driver configuration on metadb instantiation Signed-off-by: Andrei Aaron <aaaron@luxoft.com> * feat: multiple fixes for redis cache driver implementation - add missing method GetAllBlobs - add redis cache tests, with and without mocking Signed-off-by: Andrei Aaron <aaaron@luxoft.com> * feat(redis): redis implementation for MetaDB Signed-off-by: Andrei Aaron <aaaron@luxoft.com> * feat(redis): use redsync to block concurrent write access to the redis DB Signed-off-by: Andrei Aaron <aaaron@luxoft.com> * feat(redis): update .github/workflows/cluster.yaml to also test redis Signed-off-by: Andrei Aaron <aaaron@luxoft.com> * feat(metadb): add keyPrefix parameter for redis and remove unneeded method meta.Crate() Signed-off-by: Andrei Aaron <aaaron@luxoft.com> * feat(redis): support RedisCluster configuration and add unit tests Signed-off-by: Andrei Aaron <aaaron@luxoft.com> * feat(redis): more tests for redis metadb implementation Signed-off-by: Andrei Aaron <aaaron@luxoft.com> * feat(redis): add more examples and update examples/README.md Signed-off-by: Andrei Aaron <aaaron@luxoft.com> * feat(redis): move option parsing and redis client initialization under pkg/api/config/redis Signed-off-by: Andrei Aaron <aaaron@luxoft.com> * chore(cachedb): move Cache interface to pkg/storage/types Signed-off-by: Andrei Aaron <aaaron@luxoft.com> * feat(redis): reorganize code in pkg/storage/cache.go Signed-off-by: Andrei Aaron <aaaron@luxoft.com> * feat(redis): call redis.SetLogger() with the zot logger as parameter Signed-off-by: Andrei Aaron <aaaron@luxoft.com> * feat(redis): rename pkg/meta/redisdb to pkg/meta/redis Signed-off-by: Andrei Aaron <aaaron@luxoft.com> --------- Signed-off-by: Ramkumar Chinchani <rchincha@cisco.com> Signed-off-by: Petu Eusebiu <peusebiu@cisco.com> Signed-off-by: Alexei Dodon <adodon@cisco.com> Signed-off-by: Andrei Aaron <aaaron@luxoft.com> Co-authored-by: a <a@tuxpa.in> Co-authored-by: Ramkumar Chinchani <rchincha@cisco.com> Co-authored-by: Petu Eusebiu <peusebiu@cisco.com> Co-authored-by: Alexei Dodon <adodon@cisco.com>
150 lines
4.7 KiB
Go
150 lines
4.7 KiB
Go
package meta
|
|
|
|
import (
|
|
"fmt"
|
|
|
|
"zotregistry.dev/zot/errors"
|
|
"zotregistry.dev/zot/pkg/api/config"
|
|
rediscfg "zotregistry.dev/zot/pkg/api/config/redis"
|
|
"zotregistry.dev/zot/pkg/log"
|
|
"zotregistry.dev/zot/pkg/meta/boltdb"
|
|
mdynamodb "zotregistry.dev/zot/pkg/meta/dynamodb"
|
|
"zotregistry.dev/zot/pkg/meta/redis"
|
|
mTypes "zotregistry.dev/zot/pkg/meta/types"
|
|
sconstants "zotregistry.dev/zot/pkg/storage/constants"
|
|
)
|
|
|
|
func New(storageConfig config.StorageConfig, log log.Logger) (mTypes.MetaDB, error) {
|
|
if storageConfig.RemoteCache {
|
|
if storageConfig.CacheDriver["name"] == sconstants.DynamoDBDriverName {
|
|
dynamoParams := getDynamoParams(storageConfig.CacheDriver, log)
|
|
|
|
client, err := mdynamodb.GetDynamoClient(dynamoParams)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
return mdynamodb.New(client, dynamoParams, log) //nolint:contextcheck
|
|
}
|
|
|
|
if storageConfig.CacheDriver["name"] == sconstants.RedisDriverName {
|
|
redisParams := getRedisParams(storageConfig.CacheDriver, log)
|
|
|
|
client, err := rediscfg.GetRedisClient(storageConfig.CacheDriver, log)
|
|
if err != nil { //nolint:wsl
|
|
return nil, err
|
|
}
|
|
|
|
return redis.New(client, redisParams, log) //nolint:contextcheck
|
|
}
|
|
|
|
// this behavior is also mentioned in the configuration validation logic inside the cli package
|
|
return nil, fmt.Errorf("%w: cachedriver %s and remotecache %t", errors.ErrBadConfig,
|
|
storageConfig.CacheDriver["name"], storageConfig.RemoteCache)
|
|
}
|
|
|
|
if driverName, ok := storageConfig.CacheDriver["name"]; ok && driverName != sconstants.BoltdbName {
|
|
// this behavior is also mentioned in the configuration validation logic inside the cli package
|
|
log.Warn().Interface("cachedriver", driverName).Bool("remotecache", storageConfig.RemoteCache).
|
|
Msg("unsupported cachedriver for remotecache disabled, will default to boltdb")
|
|
}
|
|
|
|
params := boltdb.DBParameters{}
|
|
params.RootDir = storageConfig.RootDirectory
|
|
|
|
driver, err := boltdb.GetBoltDriver(params)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
return boltdb.New(driver, log) //nolint:contextcheck
|
|
}
|
|
|
|
func getDynamoParams(cacheDriverConfig map[string]interface{}, log log.Logger) mdynamodb.DBDriverParameters {
|
|
allParametersOk := true
|
|
|
|
endpoint, ok := toStringIfOk(cacheDriverConfig, "endpoint", "", log)
|
|
allParametersOk = allParametersOk && ok
|
|
|
|
region, ok := toStringIfOk(cacheDriverConfig, "region", "", log)
|
|
allParametersOk = allParametersOk && ok
|
|
|
|
repoMetaTablename, ok := toStringIfOk(cacheDriverConfig, "repometatablename", "", log)
|
|
allParametersOk = allParametersOk && ok
|
|
|
|
repoBlobsInfoTablename, ok := toStringIfOk(cacheDriverConfig, "repoblobsinfotablename", "", log)
|
|
allParametersOk = allParametersOk && ok
|
|
|
|
imageMetaTablename, ok := toStringIfOk(cacheDriverConfig, "imagemetatablename", "", log)
|
|
allParametersOk = allParametersOk && ok
|
|
|
|
apiKeyTablename, ok := toStringIfOk(cacheDriverConfig, "apikeytablename", "", log)
|
|
allParametersOk = allParametersOk && ok
|
|
|
|
versionTablename, ok := toStringIfOk(cacheDriverConfig, "versiontablename", "", log)
|
|
allParametersOk = allParametersOk && ok
|
|
|
|
userDataTablename, ok := toStringIfOk(cacheDriverConfig, "userdatatablename", "", log)
|
|
allParametersOk = allParametersOk && ok
|
|
|
|
if !allParametersOk {
|
|
log.Panic().Msg("dynamo parameters are not specified correctly, can't proceed")
|
|
}
|
|
|
|
return mdynamodb.DBDriverParameters{
|
|
Endpoint: endpoint,
|
|
Region: region,
|
|
RepoMetaTablename: repoMetaTablename,
|
|
RepoBlobsInfoTablename: repoBlobsInfoTablename,
|
|
ImageMetaTablename: imageMetaTablename,
|
|
UserDataTablename: userDataTablename,
|
|
APIKeyTablename: apiKeyTablename,
|
|
VersionTablename: versionTablename,
|
|
}
|
|
}
|
|
|
|
func getRedisParams(cacheDriverConfig map[string]interface{}, log log.Logger) redis.DBDriverParameters {
|
|
keyPrefix, ok := toStringIfOk(cacheDriverConfig, "keyprefix", "zot", log)
|
|
if !ok {
|
|
log.Panic().Msg("redis parameters are not specified correctly, can't proceed")
|
|
}
|
|
|
|
return redis.DBDriverParameters{
|
|
KeyPrefix: keyPrefix,
|
|
}
|
|
}
|
|
|
|
func toStringIfOk(cacheDriverConfig map[string]interface{},
|
|
param string,
|
|
defaultVal string,
|
|
log log.Logger,
|
|
) (string, bool) {
|
|
val, ok := cacheDriverConfig[param]
|
|
|
|
if !ok && defaultVal != "" {
|
|
log.Info().Str("field", param).Str("default", defaultVal).
|
|
Msg("field is not present in CacheDriver config, using default value")
|
|
|
|
return defaultVal, true
|
|
} else if !ok {
|
|
log.Error().Str("field", param).Msg("failed to parse CacheDriver config, field is not present")
|
|
|
|
return "", false
|
|
}
|
|
|
|
str, ok := val.(string)
|
|
if !ok {
|
|
log.Error().Str("parameter", param).Msg("failed to parse CacheDriver config, parameter isn't a string")
|
|
|
|
return "", false
|
|
}
|
|
|
|
if str == "" {
|
|
log.Error().Str("field", param).Msg("failed to parse CacheDriver config, field is empty")
|
|
|
|
return "", false
|
|
}
|
|
|
|
return str, true
|
|
}
|