github/zot
1
0
Fork 0
mirror of https://github.com/project-zot/zot.git synced 2026-06-17 21:17:58 +08:00
Code Issues Packages Projects Releases Wiki Activity

redis driver for blob cache information and metadb (#2865)

Browse Source 
* feat: add redis cache support

https://github.com/project-zot/zot/pull/2005
Fixes https://github.com/project-zot/zot/issues/2004

* feat: add redis cache support

Currently, we have dynamoDB as the remote shared cache but ideal only
for the cloud use case.
For on-prem use case, add support for redis.

Signed-off-by: Ramkumar Chinchani <rchincha@cisco.com>

* feat(redis): added blackbox tests for redis

Signed-off-by: Petu Eusebiu <peusebiu@cisco.com>

* feat(redis): dummy implementation of MetaDB interface for redis cache

Signed-off-by: Alexei Dodon <adodon@cisco.com>

* feat: check validity of driver configuration on metadb instantiation

Signed-off-by: Andrei Aaron <aaaron@luxoft.com>

* feat: multiple fixes for redis cache driver implementation

- add missing method GetAllBlobs
- add redis cache tests, with and without mocking

Signed-off-by: Andrei Aaron <aaaron@luxoft.com>

* feat(redis): redis implementation for MetaDB

Signed-off-by: Andrei Aaron <aaaron@luxoft.com>

* feat(redis): use redsync to block concurrent write access to the redis DB

Signed-off-by: Andrei Aaron <aaaron@luxoft.com>

* feat(redis): update .github/workflows/cluster.yaml to also test redis

Signed-off-by: Andrei Aaron <aaaron@luxoft.com>

* feat(metadb): add keyPrefix parameter for redis and remove unneeded method meta.Crate()

Signed-off-by: Andrei Aaron <aaaron@luxoft.com>

* feat(redis): support RedisCluster configuration and add unit tests

Signed-off-by: Andrei Aaron <aaaron@luxoft.com>

* feat(redis): more tests for redis metadb implementation

Signed-off-by: Andrei Aaron <aaaron@luxoft.com>

* feat(redis): add more examples and update examples/README.md

Signed-off-by: Andrei Aaron <aaaron@luxoft.com>

* feat(redis): move option parsing and redis client initialization under pkg/api/config/redis

Signed-off-by: Andrei Aaron <aaaron@luxoft.com>

* chore(cachedb): move Cache interface to pkg/storage/types

Signed-off-by: Andrei Aaron <aaaron@luxoft.com>

* feat(redis): reorganize code in pkg/storage/cache.go

Signed-off-by: Andrei Aaron <aaaron@luxoft.com>

* feat(redis): call redis.SetLogger() with the zot logger as parameter

Signed-off-by: Andrei Aaron <aaaron@luxoft.com>

* feat(redis): rename pkg/meta/redisdb to pkg/meta/redis

Signed-off-by: Andrei Aaron <aaaron@luxoft.com>

---------

Signed-off-by: Ramkumar Chinchani <rchincha@cisco.com>
Signed-off-by: Petu Eusebiu <peusebiu@cisco.com>
Signed-off-by: Alexei Dodon <adodon@cisco.com>
Signed-off-by: Andrei Aaron <aaaron@luxoft.com>
Co-authored-by: a <a@tuxpa.in>
Co-authored-by: Ramkumar Chinchani <rchincha@cisco.com>
Co-authored-by: Petu Eusebiu <peusebiu@cisco.com>
Co-authored-by: Alexei Dodon <adodon@cisco.com>
This commit is contained in:
Andrei Aaron
2025-01-30 21:00:52 +02:00
committed by GitHub
parent 90e1393585
commit 05823cd74f
43 changed files with 7886 additions and 442 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/storage/cache.go
+49 -9
View File
@@ -3,12 +3,14 @@ package storage
import (
zerr "zotregistry.dev/zot/errors"
"zotregistry.dev/zot/pkg/api/config"
rediscfg "zotregistry.dev/zot/pkg/api/config/redis"
zlog "zotregistry.dev/zot/pkg/log"
"zotregistry.dev/zot/pkg/storage/cache"
"zotregistry.dev/zot/pkg/storage/constants"
storageTypes "zotregistry.dev/zot/pkg/storage/types"
)
func CreateCacheDatabaseDriver(storageConfig config.StorageConfig, log zlog.Logger) (cache.Cache, error) {
func CreateCacheDatabaseDriver(storageConfig config.StorageConfig, log zlog.Logger) (storageTypes.Cache, error) {
if !storageConfig.Dedupe && storageConfig.StorageDriver == nil {
return nil, nil //nolint:nilnil
}
@@ -32,25 +34,33 @@ func CreateCacheDatabaseDriver(storageConfig config.StorageConfig, log zlog.Logg
return nil, nil //nolint:nilnil
}
if name != constants.DynamoDBDriverName {
if name != constants.DynamoDBDriverName &&
name != constants.RedisDriverName {
log.Warn().Str("driver", name).Msg("remote cache driver unsupported!")
return nil, nil //nolint:nilnil
}
// dynamodb
dynamoParams := cache.DynamoDBDriverParameters{}
dynamoParams.Endpoint, _ = storageConfig.CacheDriver["endpoint"].(string)
dynamoParams.Region, _ = storageConfig.CacheDriver["region"].(string)
dynamoParams.TableName, _ = storageConfig.CacheDriver["cachetablename"].(string)
if name == constants.DynamoDBDriverName {
// dynamodb
return Create(name, getDynamoParams(&storageConfig), log)
}
return Create("dynamodb", dynamoParams, log)
if name == constants.RedisDriverName {
// redis
redisParams, err := getRedisParams(&storageConfig, log)
if err != nil {
return nil, err
}
return Create(name, redisParams, log)
}
}
return nil, nil //nolint:nilnil
}
func Create(dbtype string, parameters interface{}, log zlog.Logger) (cache.Cache, error) {
func Create(dbtype string, parameters interface{}, log zlog.Logger) (storageTypes.Cache, error) {
switch dbtype {
case "boltdb":
{
@@ -60,6 +70,10 @@ func Create(dbtype string, parameters interface{}, log zlog.Logger) (cache.Cache
{
return cache.NewDynamoDBCache(parameters, log)
}
case "redis":
{
return cache.NewRedisCache(parameters, log)
}
default:
{
return nil, zerr.ErrBadConfig
@@ -68,5 +82,31 @@ func Create(dbtype string, parameters interface{}, log zlog.Logger) (cache.Cache
}
func getUseRelPaths(storageConfig *config.StorageConfig) bool {
// In case of local storage we use rel paths, in case of S3 we don't
return storageConfig.StorageDriver == nil
}
func getDynamoParams(storageConfig *config.StorageConfig) cache.DynamoDBDriverParameters {
dynamoParams := cache.DynamoDBDriverParameters{}
dynamoParams.Endpoint, _ = storageConfig.CacheDriver["endpoint"].(string)
dynamoParams.Region, _ = storageConfig.CacheDriver["region"].(string)
dynamoParams.TableName, _ = storageConfig.CacheDriver["cachetablename"].(string)
return dynamoParams
}
func getRedisParams(storageConfig *config.StorageConfig, log zlog.Logger) (cache.RedisDriverParameters, error) {
redisParams := cache.RedisDriverParameters{}
client, err := rediscfg.GetRedisClient(storageConfig.CacheDriver, log)
if err != nil {
return redisParams, err
}
redisParams.RootDir = storageConfig.RootDirectory
redisParams.Client = client
redisParams.KeyPrefix, _ = storageConfig.CacheDriver["keyprefix"].(string)
redisParams.UseRelPaths = getUseRelPaths(storageConfig)
return redisParams, nil
}