diff --git a/pkg/api/controller.go b/pkg/api/controller.go index c6a55807..b0c23f21 100644 --- a/pkg/api/controller.go +++ b/pkg/api/controller.go @@ -25,8 +25,6 @@ import ( "zotregistry.io/zot/pkg/extensions/monitoring" "zotregistry.io/zot/pkg/log" "zotregistry.io/zot/pkg/meta/repodb" - bolt "zotregistry.io/zot/pkg/meta/repodb/boltdb-wrapper" - dynamoParams "zotregistry.io/zot/pkg/meta/repodb/dynamodb-wrapper/params" "zotregistry.io/zot/pkg/meta/repodb/repodbfactory" "zotregistry.io/zot/pkg/scheduler" "zotregistry.io/zot/pkg/storage" @@ -502,7 +500,7 @@ func CreateCacheDatabaseDriver(storageConfig config.StorageConfig, log log.Logge func (c *Controller) InitRepoDB(reloadCtx context.Context) error { if c.Config.Extensions != nil && c.Config.Extensions.Search != nil && *c.Config.Extensions.Search.Enable { - driver, err := CreateRepoDBDriver(c.Config.Storage.StorageConfig, c.Log) //nolint:contextcheck + driver, err := repodbfactory.New(c.Config.Storage.StorageConfig, c.Log) //nolint:contextcheck if err != nil { return err } @@ -523,84 +521,6 @@ func (c *Controller) InitRepoDB(reloadCtx context.Context) error { return nil } -func CreateRepoDBDriver(storageConfig config.StorageConfig, log log.Logger) (repodb.RepoDB, error) { - if storageConfig.RemoteCache { - dynamoParams := getDynamoParams(storageConfig.CacheDriver, log) - - return repodbfactory.Create("dynamodb", dynamoParams) //nolint:contextcheck - } - - params := bolt.DBParameters{} - params.RootDir = storageConfig.RootDirectory - - return repodbfactory.Create("boltdb", params) //nolint:contextcheck -} - -func getDynamoParams(cacheDriverConfig map[string]interface{}, log log.Logger) dynamoParams.DBDriverParameters { - allParametersOk := true - - endpoint, ok := toStringIfOk(cacheDriverConfig, "endpoint", log) - allParametersOk = allParametersOk && ok - - region, ok := toStringIfOk(cacheDriverConfig, "region", log) - allParametersOk = allParametersOk && ok - - repoMetaTablename, ok := toStringIfOk(cacheDriverConfig, "repometatablename", log) - allParametersOk = allParametersOk && ok - - manifestDataTablename, ok := toStringIfOk(cacheDriverConfig, "manifestdatatablename", log) - allParametersOk = allParametersOk && ok - - indexDataTablename, ok := toStringIfOk(cacheDriverConfig, "indexdatatablename", log) - allParametersOk = allParametersOk && ok - - artifactDataTablename, ok := toStringIfOk(cacheDriverConfig, "artifactdatatablename", log) - allParametersOk = allParametersOk && ok - - versionTablename, ok := toStringIfOk(cacheDriverConfig, "versiontablename", log) - allParametersOk = allParametersOk && ok - - if !allParametersOk { - panic("dynamo parameters are not specified correctly, can't proceede") - } - - return dynamoParams.DBDriverParameters{ - Endpoint: endpoint, - Region: region, - RepoMetaTablename: repoMetaTablename, - ManifestDataTablename: manifestDataTablename, - IndexDataTablename: indexDataTablename, - ArtifactDataTablename: artifactDataTablename, - VersionTablename: versionTablename, - } -} - -func toStringIfOk(cacheDriverConfig map[string]interface{}, param string, log log.Logger) (string, bool) { - val, ok := cacheDriverConfig[param] - - if !ok { - log.Error().Msgf("parsing CacheDriver config failed, field '%s' is not present", param) - - return "", false - } - - str, ok := val.(string) - - if !ok { - log.Error().Msgf("parsing CacheDriver config failed, parameter '%s' isn't a string", param) - - return "", false - } - - if str == "" { - log.Error().Msgf("parsing CacheDriver config failed, field '%s' is is empty", param) - - return "", false - } - - return str, ok -} - func (c *Controller) LoadNewConfig(reloadCtx context.Context, config *config.Config) { // reload access control config c.Config.HTTP.AccessControl = config.HTTP.AccessControl diff --git a/pkg/api/controller_test.go b/pkg/api/controller_test.go index 55f94e2a..2a321dae 100644 --- a/pkg/api/controller_test.go +++ b/pkg/api/controller_test.go @@ -38,6 +38,7 @@ import ( "github.com/sigstore/cosign/pkg/oci/remote" . "github.com/smartystreets/goconvey/convey" "github.com/stretchr/testify/assert" + "go.etcd.io/bbolt" "golang.org/x/crypto/bcrypt" "gopkg.in/resty.v1" @@ -48,6 +49,7 @@ import ( "zotregistry.io/zot/pkg/common" extconf "zotregistry.io/zot/pkg/extensions/config" "zotregistry.io/zot/pkg/log" + "zotregistry.io/zot/pkg/meta/repodb/repodbfactory" "zotregistry.io/zot/pkg/storage" storageConstants "zotregistry.io/zot/pkg/storage/constants" "zotregistry.io/zot/pkg/storage/local" @@ -220,7 +222,7 @@ func TestCreateRepoDBDriver(t *testing.T) { "artifactDataTablename": "ArtifactDataTable", } - testFunc := func() { _, _ = api.CreateRepoDBDriver(conf.Storage.StorageConfig, log) } + testFunc := func() { _, _ = repodbfactory.New(conf.Storage.StorageConfig, log) } So(testFunc, ShouldPanic) conf.Storage.CacheDriver = map[string]interface{}{ @@ -234,8 +236,51 @@ func TestCreateRepoDBDriver(t *testing.T) { "versiontablename": 1, } - testFunc = func() { _, _ = api.CreateRepoDBDriver(conf.Storage.StorageConfig, log) } + testFunc = func() { _, _ = repodbfactory.New(conf.Storage.StorageConfig, log) } So(testFunc, ShouldPanic) + + conf.Storage.CacheDriver = map[string]interface{}{ + "name": "dummy", + "endpoint": "http://localhost:4566", + "region": "us-east-2", + "cachetablename": "test", + "repometatablename": "RepoMetadataTable", + "manifestdatatablename": "ManifestDataTable", + "indexdatatablename": "IndexDataTable", + "artifactdatatablename": "ArtifactDataTable", + "versiontablename": "1", + } + + testFunc = func() { _, _ = repodbfactory.New(conf.Storage.StorageConfig, log) } + So(testFunc, ShouldNotPanic) + }) + + Convey("Test CreateCacheDatabaseDriver bolt", t, func() { + log := log.NewLogger("debug", "") + dir := t.TempDir() + conf := config.New() + conf.Storage.RootDirectory = dir + conf.Storage.Dedupe = true + conf.Storage.RemoteCache = false + + const perms = 0o600 + + boltDB, err := bbolt.Open(path.Join(dir, "repo.db"), perms, &bbolt.Options{Timeout: time.Second * 10}) + So(err, ShouldBeNil) + + err = boltDB.Close() + So(err, ShouldBeNil) + + err = os.Chmod(path.Join(dir, "repo.db"), 0o200) + So(err, ShouldBeNil) + + _, err = repodbfactory.New(conf.Storage.StorageConfig, log) + So(err, ShouldNotBeNil) + + err = os.Chmod(path.Join(dir, "repo.db"), 0o600) + So(err, ShouldBeNil) + + defer os.Remove(path.Join(dir, "repo.db")) }) } diff --git a/pkg/api/routes.go b/pkg/api/routes.go index dffef680..4d051897 100644 --- a/pkg/api/routes.go +++ b/pkg/api/routes.go @@ -35,7 +35,7 @@ import ( ext "zotregistry.io/zot/pkg/extensions" "zotregistry.io/zot/pkg/extensions/sync" "zotregistry.io/zot/pkg/log" - repoDBUpdate "zotregistry.io/zot/pkg/meta/repodb/update" + "zotregistry.io/zot/pkg/meta" zreg "zotregistry.io/zot/pkg/regexp" localCtx "zotregistry.io/zot/pkg/requestcontext" "zotregistry.io/zot/pkg/storage" @@ -406,7 +406,7 @@ func (rh *RouteHandler) GetManifest(response http.ResponseWriter, request *http. } if rh.c.RepoDB != nil { - err := repoDBUpdate.OnGetManifest(name, reference, digest, content, rh.c.StoreController, rh.c.RepoDB, rh.c.Log) + err := meta.OnGetManifest(name, reference, digest, content, rh.c.StoreController, rh.c.RepoDB, rh.c.Log) if errors.Is(err, zerr.ErrOrphanSignature) { rh.c.Log.Error().Err(err).Msgf("image is an orphan signature") @@ -606,7 +606,7 @@ func (rh *RouteHandler) UpdateManifest(response http.ResponseWriter, request *ht } if rh.c.RepoDB != nil { - err := repoDBUpdate.OnUpdateManifest(name, reference, mediaType, digest, body, rh.c.StoreController, rh.c.RepoDB, + err := meta.OnUpdateManifest(name, reference, mediaType, digest, body, rh.c.StoreController, rh.c.RepoDB, rh.c.Log) if errors.Is(err, zerr.ErrOrphanSignature) { rh.c.Log.Error().Err(err).Msgf("pushed image is an orphan signature") @@ -705,7 +705,7 @@ func (rh *RouteHandler) DeleteManifest(response http.ResponseWriter, request *ht } if rh.c.RepoDB != nil { - err := repoDBUpdate.OnDeleteManifest(name, reference, mediaType, manifestDigest, manifestBlob, + err := meta.OnDeleteManifest(name, reference, mediaType, manifestDigest, manifestBlob, rh.c.StoreController, rh.c.RepoDB, rh.c.Log) if errors.Is(err, zerr.ErrOrphanSignature) { rh.c.Log.Error().Err(err).Msgf("pushed image is an orphan signature") diff --git a/pkg/extensions/search/convert/convert_test.go b/pkg/extensions/search/convert/convert_test.go index b5770220..e6a59606 100644 --- a/pkg/extensions/search/convert/convert_test.go +++ b/pkg/extensions/search/convert/convert_test.go @@ -20,8 +20,9 @@ import ( "zotregistry.io/zot/pkg/extensions/search/convert" cveinfo "zotregistry.io/zot/pkg/extensions/search/cve" "zotregistry.io/zot/pkg/log" + "zotregistry.io/zot/pkg/meta/bolt" "zotregistry.io/zot/pkg/meta/repodb" - bolt "zotregistry.io/zot/pkg/meta/repodb/boltdb-wrapper" + boltdb_wrapper "zotregistry.io/zot/pkg/meta/repodb/boltdb-wrapper" . "zotregistry.io/zot/pkg/test" "zotregistry.io/zot/pkg/test/mocks" ) @@ -30,9 +31,13 @@ var ErrTestError = errors.New("TestError") func TestConvertErrors(t *testing.T) { Convey("Convert Errors", t, func() { - repoDB, err := bolt.NewBoltDBWrapper(bolt.DBParameters{ + params := bolt.DBParameters{ RootDir: t.TempDir(), - }) + } + boltDB, err := bolt.GetBoltDriver(params) + So(err, ShouldBeNil) + + repoDB, err := boltdb_wrapper.NewBoltDBWrapper(boltDB, log.NewLogger("debug", "")) So(err, ShouldBeNil) configBlob, err := json.Marshal(ispec.Image{}) diff --git a/pkg/extensions/search/cve/cve_test.go b/pkg/extensions/search/cve/cve_test.go index 2ca57651..73c28dac 100644 --- a/pkg/extensions/search/cve/cve_test.go +++ b/pkg/extensions/search/cve/cve_test.go @@ -29,8 +29,9 @@ import ( cveinfo "zotregistry.io/zot/pkg/extensions/search/cve" cvemodel "zotregistry.io/zot/pkg/extensions/search/cve/model" "zotregistry.io/zot/pkg/log" + "zotregistry.io/zot/pkg/meta/bolt" "zotregistry.io/zot/pkg/meta/repodb" - bolt "zotregistry.io/zot/pkg/meta/repodb/boltdb-wrapper" + boltdb_wrapper "zotregistry.io/zot/pkg/meta/repodb/boltdb-wrapper" "zotregistry.io/zot/pkg/storage" "zotregistry.io/zot/pkg/storage/local" . "zotregistry.io/zot/pkg/test" @@ -312,9 +313,13 @@ func TestImageFormat(t *testing.T) { false, false, log, metrics, nil, nil) storeController := storage.StoreController{DefaultStore: defaultStore} - repoDB, err := bolt.NewBoltDBWrapper(bolt.DBParameters{ + params := bolt.DBParameters{ RootDir: dbDir, - }) + } + boltDriver, err := bolt.GetBoltDriver(params) + So(err, ShouldBeNil) + + repoDB, err := boltdb_wrapper.NewBoltDBWrapper(boltDriver, log) So(err, ShouldBeNil) err = repodb.ParseStorage(repoDB, storeController, log) @@ -716,9 +721,13 @@ func TestCVESearch(t *testing.T) { func TestCVEStruct(t *testing.T) { Convey("Unit test the CVE struct", t, func() { - repoDB, err := bolt.NewBoltDBWrapper(bolt.DBParameters{ + params := bolt.DBParameters{ RootDir: t.TempDir(), - }) + } + boltDriver, err := bolt.GetBoltDriver(params) + So(err, ShouldBeNil) + + repoDB, err := boltdb_wrapper.NewBoltDBWrapper(boltDriver, log.NewLogger("debug", "")) So(err, ShouldBeNil) // Create repodb data for scannable image with vulnerabilities diff --git a/pkg/extensions/search/cve/pagination_test.go b/pkg/extensions/search/cve/pagination_test.go index 172b326f..87840743 100644 --- a/pkg/extensions/search/cve/pagination_test.go +++ b/pkg/extensions/search/cve/pagination_test.go @@ -17,16 +17,21 @@ import ( cveinfo "zotregistry.io/zot/pkg/extensions/search/cve" cvemodel "zotregistry.io/zot/pkg/extensions/search/cve/model" "zotregistry.io/zot/pkg/log" + "zotregistry.io/zot/pkg/meta/bolt" "zotregistry.io/zot/pkg/meta/repodb" - bolt "zotregistry.io/zot/pkg/meta/repodb/boltdb-wrapper" + boltdb_wrapper "zotregistry.io/zot/pkg/meta/repodb/boltdb-wrapper" "zotregistry.io/zot/pkg/test/mocks" ) func TestCVEPagination(t *testing.T) { Convey("CVE Pagination", t, func() { - repoDB, err := bolt.NewBoltDBWrapper(bolt.DBParameters{ + params := bolt.DBParameters{ RootDir: t.TempDir(), - }) + } + boltDriver, err := bolt.GetBoltDriver(params) + So(err, ShouldBeNil) + + repoDB, err := boltdb_wrapper.NewBoltDBWrapper(boltDriver, log.NewLogger("debug", "")) So(err, ShouldBeNil) // Create repodb data for scannable image with vulnerabilities diff --git a/pkg/extensions/search/cve/trivy/scanner_internal_test.go b/pkg/extensions/search/cve/trivy/scanner_internal_test.go index 961adb6b..66dfdd26 100644 --- a/pkg/extensions/search/cve/trivy/scanner_internal_test.go +++ b/pkg/extensions/search/cve/trivy/scanner_internal_test.go @@ -18,8 +18,9 @@ import ( "zotregistry.io/zot/pkg/extensions/monitoring" "zotregistry.io/zot/pkg/extensions/search/common" "zotregistry.io/zot/pkg/log" + "zotregistry.io/zot/pkg/meta/bolt" "zotregistry.io/zot/pkg/meta/repodb" - bolt "zotregistry.io/zot/pkg/meta/repodb/boltdb-wrapper" + boltdb_wrapper "zotregistry.io/zot/pkg/meta/repodb/boltdb-wrapper" "zotregistry.io/zot/pkg/storage" "zotregistry.io/zot/pkg/storage/local" "zotregistry.io/zot/pkg/test" @@ -83,9 +84,13 @@ func TestMultipleStoragePath(t *testing.T) { storeController.SubStore = subStore - repoDB, err := bolt.NewBoltDBWrapper(bolt.DBParameters{ + params := bolt.DBParameters{ RootDir: firstRootDir, - }) + } + boltDriver, err := bolt.GetBoltDriver(params) + So(err, ShouldBeNil) + + repoDB, err := boltdb_wrapper.NewBoltDBWrapper(boltDriver, log) So(err, ShouldBeNil) err = repodb.ParseStorage(repoDB, storeController, log) @@ -173,9 +178,14 @@ func TestTrivyLibraryErrors(t *testing.T) { storeController := storage.StoreController{} storeController.DefaultStore = store - repoDB, err := bolt.NewBoltDBWrapper(bolt.DBParameters{ + params := bolt.DBParameters{ RootDir: rootDir, - }) + } + + boltDriver, err := bolt.GetBoltDriver(params) + So(err, ShouldBeNil) + + repoDB, err := boltdb_wrapper.NewBoltDBWrapper(boltDriver, log) So(err, ShouldBeNil) err = repodb.ParseStorage(repoDB, storeController, log) @@ -217,9 +227,18 @@ func TestTrivyLibraryErrors(t *testing.T) { func TestImageScannable(t *testing.T) { rootDir := t.TempDir() - repoDB, err := bolt.NewBoltDBWrapper(bolt.DBParameters{ + params := bolt.DBParameters{ RootDir: rootDir, - }) + } + + boltDriver, err := bolt.GetBoltDriver(params) + if err != nil { + panic(err) + } + + log := log.NewLogger("debug", "") + + repoDB, err := boltdb_wrapper.NewBoltDBWrapper(boltDriver, log) if err != nil { panic(err) } @@ -349,7 +368,6 @@ func TestImageScannable(t *testing.T) { } // Continue with initializing the objects the scanner depends on - log := log.NewLogger("debug", "") metrics := monitoring.NewMetricsServer(false, log) store := local.NewImageStore(rootDir, false, storage.DefaultGCDelay, false, false, log, metrics, nil, nil) @@ -419,9 +437,14 @@ func TestDefaultTrivyDBUrl(t *testing.T) { storeController := storage.StoreController{} storeController.DefaultStore = store - repoDB, err := bolt.NewBoltDBWrapper(bolt.DBParameters{ + params := bolt.DBParameters{ RootDir: rootDir, - }) + } + + boltDriver, err := bolt.GetBoltDriver(params) + So(err, ShouldBeNil) + + repoDB, err := boltdb_wrapper.NewBoltDBWrapper(boltDriver, log) So(err, ShouldBeNil) err = repodb.ParseStorage(repoDB, storeController, log) diff --git a/pkg/extensions/search/resolver_test.go b/pkg/extensions/search/resolver_test.go index 3e999cfe..6c991900 100644 --- a/pkg/extensions/search/resolver_test.go +++ b/pkg/extensions/search/resolver_test.go @@ -20,8 +20,9 @@ import ( cvemodel "zotregistry.io/zot/pkg/extensions/search/cve/model" "zotregistry.io/zot/pkg/extensions/search/gql_generated" "zotregistry.io/zot/pkg/log" + "zotregistry.io/zot/pkg/meta/bolt" "zotregistry.io/zot/pkg/meta/repodb" - bolt "zotregistry.io/zot/pkg/meta/repodb/boltdb-wrapper" + boltdb_wrapper "zotregistry.io/zot/pkg/meta/repodb/boltdb-wrapper" localCtx "zotregistry.io/zot/pkg/requestcontext" "zotregistry.io/zot/pkg/storage" "zotregistry.io/zot/pkg/test/mocks" @@ -1892,9 +1893,18 @@ func TestQueryResolverErrors(t *testing.T) { } func TestCVEResolvers(t *testing.T) { //nolint:gocyclo - repoDB, err := bolt.NewBoltDBWrapper(bolt.DBParameters{ + params := bolt.DBParameters{ RootDir: t.TempDir(), - }) + } + + boltDriver, err := bolt.GetBoltDriver(params) + if err != nil { + panic(err) + } + + log := log.NewLogger("debug", "") + + repoDB, err := boltdb_wrapper.NewBoltDBWrapper(boltDriver, log) if err != nil { panic(err) } @@ -2125,8 +2135,6 @@ func TestCVEResolvers(t *testing.T) { //nolint:gocyclo }, } - log := log.NewLogger("debug", "") - cveInfo := &cveinfo.BaseCveInfo{ Log: log, Scanner: scanner, diff --git a/pkg/meta/bolt/buckets.go b/pkg/meta/bolt/buckets.go new file mode 100644 index 00000000..367e8b47 --- /dev/null +++ b/pkg/meta/bolt/buckets.go @@ -0,0 +1,11 @@ +package bolt + +// MetadataDB. +const ( + ManifestDataBucket = "ManifestData" + IndexDataBucket = "IndexData" + ArtifactDataBucket = "ArtifactData" + UserMetadataBucket = "UserMeta" + RepoMetadataBucket = "RepoMetadata" + VersionBucket = "Version" +) diff --git a/pkg/meta/bolt/parameters.go b/pkg/meta/bolt/parameters.go new file mode 100644 index 00000000..33d77568 --- /dev/null +++ b/pkg/meta/bolt/parameters.go @@ -0,0 +1,23 @@ +package bolt + +import ( + "path" + "time" + + bolt "go.etcd.io/bbolt" +) + +type DBParameters struct { + RootDir string +} + +func GetBoltDriver(params DBParameters) (*bolt.DB, error) { + const perms = 0o600 + + boltDB, err := bolt.Open(path.Join(params.RootDir, "repo.db"), perms, &bolt.Options{Timeout: time.Second * 10}) + if err != nil { + return nil, err + } + + return boltDB, nil +} diff --git a/pkg/meta/repodb/common/common.go b/pkg/meta/common/common.go similarity index 100% rename from pkg/meta/repodb/common/common.go rename to pkg/meta/common/common.go diff --git a/pkg/meta/repodb/common/common_test.go b/pkg/meta/common/common_test.go similarity index 91% rename from pkg/meta/repodb/common/common_test.go rename to pkg/meta/common/common_test.go index 3e96a663..db61bfcb 100644 --- a/pkg/meta/repodb/common/common_test.go +++ b/pkg/meta/common/common_test.go @@ -5,7 +5,7 @@ import ( . "github.com/smartystreets/goconvey/convey" - "zotregistry.io/zot/pkg/meta/repodb/common" + "zotregistry.io/zot/pkg/meta/common" ) func TestUtils(t *testing.T) { diff --git a/pkg/meta/repodb/dynamodb-wrapper/iterator/dynamo_iterator.go b/pkg/meta/dynamo/iterator.go similarity index 99% rename from pkg/meta/repodb/dynamodb-wrapper/iterator/dynamo_iterator.go rename to pkg/meta/dynamo/iterator.go index 94b7770f..695c2edf 100644 --- a/pkg/meta/repodb/dynamodb-wrapper/iterator/dynamo_iterator.go +++ b/pkg/meta/dynamo/iterator.go @@ -1,4 +1,4 @@ -package iterator +package dynamo import ( "context" diff --git a/pkg/meta/dynamo/parameters.go b/pkg/meta/dynamo/parameters.go new file mode 100644 index 00000000..bd7d7d64 --- /dev/null +++ b/pkg/meta/dynamo/parameters.go @@ -0,0 +1,36 @@ +package dynamo + +import ( + "context" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/service/dynamodb" +) + +type DBDriverParameters struct { + Endpoint, Region, RepoMetaTablename, ManifestDataTablename, IndexDataTablename, + ArtifactDataTablename, VersionTablename string +} + +func GetDynamoClient(params DBDriverParameters) (*dynamodb.Client, error) { + customResolver := aws.EndpointResolverWithOptionsFunc( + func(service, region string, options ...interface{}) (aws.Endpoint, error) { + return aws.Endpoint{ + PartitionID: "aws", + URL: params.Endpoint, + SigningRegion: region, + }, nil + }) + + // Using the SDK's default configuration, loading additional config + // and credentials values from the environment variables, shared + // credentials, and shared configuration files + cfg, err := config.LoadDefaultConfig(context.Background(), config.WithRegion(params.Region), + config.WithEndpointResolverWithOptions(customResolver)) + if err != nil { + return nil, err + } + + return dynamodb.NewFromConfig(cfg), nil +} diff --git a/pkg/meta/repodb/boltdb-wrapper/boltdb_wrapper.go b/pkg/meta/repodb/boltdb-wrapper/boltdb_wrapper.go index 83f6c0b3..c8f28382 100644 --- a/pkg/meta/repodb/boltdb-wrapper/boltdb_wrapper.go +++ b/pkg/meta/repodb/boltdb-wrapper/boltdb_wrapper.go @@ -4,44 +4,31 @@ import ( "context" "encoding/json" "fmt" - "os" - "path" "strings" "time" godigest "github.com/opencontainers/go-digest" ispec "github.com/opencontainers/image-spec/specs-go/v1" - "github.com/rs/zerolog" - bolt "go.etcd.io/bbolt" + "go.etcd.io/bbolt" zerr "zotregistry.io/zot/errors" "zotregistry.io/zot/pkg/log" + "zotregistry.io/zot/pkg/meta/bolt" + "zotregistry.io/zot/pkg/meta/common" "zotregistry.io/zot/pkg/meta/repodb" - "zotregistry.io/zot/pkg/meta/repodb/common" - "zotregistry.io/zot/pkg/meta/repodb/version" + "zotregistry.io/zot/pkg/meta/version" localCtx "zotregistry.io/zot/pkg/requestcontext" ) -type DBParameters struct { - RootDir string -} - type DBWrapper struct { - DB *bolt.DB - Patches []func(DB *bolt.DB) error + DB *bbolt.DB + Patches []func(DB *bbolt.DB) error Log log.Logger } -func NewBoltDBWrapper(params DBParameters) (*DBWrapper, error) { - const perms = 0o600 - - boltDB, err := bolt.Open(path.Join(params.RootDir, "repo.db"), perms, &bolt.Options{Timeout: time.Second * 10}) - if err != nil { - return nil, err - } - - err = boltDB.Update(func(transaction *bolt.Tx) error { - versionBuck, err := transaction.CreateBucketIfNotExists([]byte(repodb.VersionBucket)) +func NewBoltDBWrapper(boltDB *bbolt.DB, log log.Logger) (*DBWrapper, error) { + err := boltDB.Update(func(transaction *bbolt.Tx) error { + versionBuck, err := transaction.CreateBucketIfNotExists([]byte(bolt.VersionBucket)) if err != nil { return err } @@ -51,22 +38,22 @@ func NewBoltDBWrapper(params DBParameters) (*DBWrapper, error) { return err } - _, err = transaction.CreateBucketIfNotExists([]byte(repodb.ManifestDataBucket)) + _, err = transaction.CreateBucketIfNotExists([]byte(bolt.ManifestDataBucket)) if err != nil { return err } - _, err = transaction.CreateBucketIfNotExists([]byte(repodb.IndexDataBucket)) + _, err = transaction.CreateBucketIfNotExists([]byte(bolt.IndexDataBucket)) if err != nil { return err } - _, err = transaction.CreateBucketIfNotExists([]byte(repodb.ArtifactDataBucket)) + _, err = transaction.CreateBucketIfNotExists([]byte(bolt.ArtifactDataBucket)) if err != nil { return err } - _, err = transaction.CreateBucketIfNotExists([]byte(repodb.RepoMetadataBucket)) + _, err = transaction.CreateBucketIfNotExists([]byte(bolt.RepoMetadataBucket)) if err != nil { return err } @@ -80,13 +67,13 @@ func NewBoltDBWrapper(params DBParameters) (*DBWrapper, error) { return &DBWrapper{ DB: boltDB, Patches: version.GetBoltDBPatches(), - Log: log.Logger{Logger: zerolog.New(os.Stdout)}, + Log: log, }, nil } func (bdw *DBWrapper) SetManifestData(manifestDigest godigest.Digest, manifestData repodb.ManifestData) error { - err := bdw.DB.Update(func(tx *bolt.Tx) error { - buck := tx.Bucket([]byte(repodb.ManifestDataBucket)) + err := bdw.DB.Update(func(tx *bbolt.Tx) error { + buck := tx.Bucket([]byte(bolt.ManifestDataBucket)) mdBlob, err := json.Marshal(manifestData) if err != nil { @@ -107,8 +94,8 @@ func (bdw *DBWrapper) SetManifestData(manifestDigest godigest.Digest, manifestDa func (bdw *DBWrapper) GetManifestData(manifestDigest godigest.Digest) (repodb.ManifestData, error) { var manifestData repodb.ManifestData - err := bdw.DB.View(func(tx *bolt.Tx) error { - buck := tx.Bucket([]byte(repodb.ManifestDataBucket)) + err := bdw.DB.View(func(tx *bbolt.Tx) error { + buck := tx.Bucket([]byte(bolt.ManifestDataBucket)) mdBlob := buck.Get([]byte(manifestDigest)) @@ -129,9 +116,9 @@ func (bdw *DBWrapper) GetManifestData(manifestDigest godigest.Digest) (repodb.Ma func (bdw *DBWrapper) SetManifestMeta(repo string, manifestDigest godigest.Digest, manifestMeta repodb.ManifestMetadata, ) error { - err := bdw.DB.Update(func(tx *bolt.Tx) error { - dataBuck := tx.Bucket([]byte(repodb.ManifestDataBucket)) - repoBuck := tx.Bucket([]byte(repodb.RepoMetadataBucket)) + err := bdw.DB.Update(func(tx *bbolt.Tx) error { + dataBuck := tx.Bucket([]byte(bolt.ManifestDataBucket)) + repoBuck := tx.Bucket([]byte(bolt.RepoMetadataBucket)) repoMeta := repodb.RepoMetadata{ Name: repo, @@ -178,9 +165,9 @@ func (bdw *DBWrapper) SetManifestMeta(repo string, manifestDigest godigest.Diges func (bdw *DBWrapper) GetManifestMeta(repo string, manifestDigest godigest.Digest) (repodb.ManifestMetadata, error) { var manifestMetadata repodb.ManifestMetadata - err := bdw.DB.View(func(tx *bolt.Tx) error { - dataBuck := tx.Bucket([]byte(repodb.ManifestDataBucket)) - repoBuck := tx.Bucket([]byte(repodb.RepoMetadataBucket)) + err := bdw.DB.View(func(tx *bbolt.Tx) error { + dataBuck := tx.Bucket([]byte(bolt.ManifestDataBucket)) + repoBuck := tx.Bucket([]byte(bolt.RepoMetadataBucket)) mdBlob := dataBuck.Get([]byte(manifestDigest)) @@ -223,8 +210,8 @@ func (bdw *DBWrapper) GetManifestMeta(repo string, manifestDigest godigest.Diges func (bdw *DBWrapper) SetIndexData(indexDigest godigest.Digest, indexMetadata repodb.IndexData) error { // we make the assumption that the oci layout is consistent and all manifests refferenced inside the // index are present - err := bdw.DB.Update(func(tx *bolt.Tx) error { - buck := tx.Bucket([]byte(repodb.IndexDataBucket)) + err := bdw.DB.Update(func(tx *bbolt.Tx) error { + buck := tx.Bucket([]byte(bolt.IndexDataBucket)) imBlob, err := json.Marshal(indexMetadata) if err != nil { @@ -245,8 +232,8 @@ func (bdw *DBWrapper) SetIndexData(indexDigest godigest.Digest, indexMetadata re func (bdw *DBWrapper) GetIndexData(indexDigest godigest.Digest) (repodb.IndexData, error) { var indexMetadata repodb.IndexData - err := bdw.DB.View(func(tx *bolt.Tx) error { - buck := tx.Bucket([]byte(repodb.IndexDataBucket)) + err := bdw.DB.View(func(tx *bbolt.Tx) error { + buck := tx.Bucket([]byte(bolt.IndexDataBucket)) mmBlob := buck.Get([]byte(indexDigest)) @@ -266,8 +253,8 @@ func (bdw *DBWrapper) GetIndexData(indexDigest godigest.Digest) (repodb.IndexDat } func (bdw DBWrapper) SetArtifactData(artifactDigest godigest.Digest, artifactData repodb.ArtifactData) error { - err := bdw.DB.Update(func(tx *bolt.Tx) error { - buck := tx.Bucket([]byte(repodb.ArtifactDataBucket)) + err := bdw.DB.Update(func(tx *bbolt.Tx) error { + buck := tx.Bucket([]byte(bolt.ArtifactDataBucket)) imBlob, err := json.Marshal(artifactData) if err != nil { @@ -288,8 +275,8 @@ func (bdw DBWrapper) SetArtifactData(artifactDigest godigest.Digest, artifactDat func (bdw DBWrapper) GetArtifactData(artifactDigest godigest.Digest) (repodb.ArtifactData, error) { var artifactData repodb.ArtifactData - err := bdw.DB.View(func(tx *bolt.Tx) error { - buck := tx.Bucket([]byte(repodb.ArtifactDataBucket)) + err := bdw.DB.View(func(tx *bbolt.Tx) error { + buck := tx.Bucket([]byte(bolt.ArtifactDataBucket)) blob := buck.Get([]byte(artifactDigest)) @@ -309,8 +296,8 @@ func (bdw DBWrapper) GetArtifactData(artifactDigest godigest.Digest) (repodb.Art } func (bdw DBWrapper) SetReferrer(repo string, referredDigest godigest.Digest, referrer repodb.ReferrerInfo) error { - err := bdw.DB.Update(func(tx *bolt.Tx) error { - buck := tx.Bucket([]byte(repodb.RepoMetadataBucket)) + err := bdw.DB.Update(func(tx *bbolt.Tx) error { + buck := tx.Bucket([]byte(bolt.RepoMetadataBucket)) repoMetaBlob := buck.Get([]byte(repo)) @@ -375,8 +362,8 @@ func (bdw DBWrapper) SetReferrer(repo string, referredDigest godigest.Digest, re func (bdw DBWrapper) DeleteReferrer(repo string, referredDigest godigest.Digest, referrerDigest godigest.Digest, ) error { - return bdw.DB.Update(func(tx *bolt.Tx) error { - buck := tx.Bucket([]byte(repodb.RepoMetadataBucket)) + return bdw.DB.Update(func(tx *bbolt.Tx) error { + buck := tx.Bucket([]byte(bolt.RepoMetadataBucket)) repoMetaBlob := buck.Get([]byte(repo)) @@ -416,8 +403,8 @@ func (bdw DBWrapper) GetReferrersInfo(repo string, referredDigest godigest.Diges ) ([]repodb.ReferrerInfo, error) { referrersInfoResult := []repodb.ReferrerInfo{} - err := bdw.DB.View(func(tx *bolt.Tx) error { - buck := tx.Bucket([]byte(repodb.RepoMetadataBucket)) + err := bdw.DB.View(func(tx *bbolt.Tx) error { + buck := tx.Bucket([]byte(bolt.RepoMetadataBucket)) repoMetaBlob := buck.Get([]byte(repo)) if len(repoMetaBlob) == 0 { @@ -454,8 +441,8 @@ func (bdw *DBWrapper) SetRepoReference(repo string, reference string, manifestDi return err } - err := bdw.DB.Update(func(tx *bolt.Tx) error { - buck := tx.Bucket([]byte(repodb.RepoMetadataBucket)) + err := bdw.DB.Update(func(tx *bbolt.Tx) error { + buck := tx.Bucket([]byte(bolt.RepoMetadataBucket)) repoMetaBlob := buck.Get([]byte(repo)) @@ -518,8 +505,8 @@ func (bdw *DBWrapper) SetRepoReference(repo string, reference string, manifestDi func (bdw *DBWrapper) GetRepoMeta(repo string) (repodb.RepoMetadata, error) { var repoMeta repodb.RepoMetadata - err := bdw.DB.Update(func(tx *bolt.Tx) error { - buck := tx.Bucket([]byte(repodb.RepoMetadataBucket)) + err := bdw.DB.Update(func(tx *bbolt.Tx) error { + buck := tx.Bucket([]byte(bolt.RepoMetadataBucket)) repoMetaBlob := buck.Get([]byte(repo)) @@ -541,8 +528,8 @@ func (bdw *DBWrapper) GetRepoMeta(repo string) (repodb.RepoMetadata, error) { } func (bdw *DBWrapper) SetRepoMeta(repo string, repoMeta repodb.RepoMetadata) error { - err := bdw.DB.Update(func(tx *bolt.Tx) error { - buck := tx.Bucket([]byte(repodb.RepoMetadataBucket)) + err := bdw.DB.Update(func(tx *bbolt.Tx) error { + buck := tx.Bucket([]byte(bolt.RepoMetadataBucket)) repoMeta.Name = repo @@ -558,8 +545,8 @@ func (bdw *DBWrapper) SetRepoMeta(repo string, repoMeta repodb.RepoMetadata) err } func (bdw *DBWrapper) DeleteRepoTag(repo string, tag string) error { - err := bdw.DB.Update(func(tx *bolt.Tx) error { - buck := tx.Bucket([]byte(repodb.RepoMetadataBucket)) + err := bdw.DB.Update(func(tx *bbolt.Tx) error { + buck := tx.Bucket([]byte(bolt.RepoMetadataBucket)) repoMetaBlob := buck.Get([]byte(repo)) @@ -590,8 +577,8 @@ func (bdw *DBWrapper) DeleteRepoTag(repo string, tag string) error { } func (bdw *DBWrapper) IncrementRepoStars(repo string) error { - err := bdw.DB.Update(func(tx *bolt.Tx) error { - buck := tx.Bucket([]byte(repodb.RepoMetadataBucket)) + err := bdw.DB.Update(func(tx *bbolt.Tx) error { + buck := tx.Bucket([]byte(bolt.RepoMetadataBucket)) repoMetaBlob := buck.Get([]byte(repo)) if repoMetaBlob == nil { @@ -619,8 +606,8 @@ func (bdw *DBWrapper) IncrementRepoStars(repo string) error { } func (bdw *DBWrapper) DecrementRepoStars(repo string) error { - err := bdw.DB.Update(func(tx *bolt.Tx) error { - buck := tx.Bucket([]byte(repodb.RepoMetadataBucket)) + err := bdw.DB.Update(func(tx *bbolt.Tx) error { + buck := tx.Bucket([]byte(bolt.RepoMetadataBucket)) repoMetaBlob := buck.Get([]byte(repo)) if repoMetaBlob == nil { @@ -652,8 +639,8 @@ func (bdw *DBWrapper) DecrementRepoStars(repo string) error { func (bdw *DBWrapper) GetRepoStars(repo string) (int, error) { stars := 0 - err := bdw.DB.View(func(tx *bolt.Tx) error { - buck := tx.Bucket([]byte(repodb.RepoMetadataBucket)) + err := bdw.DB.View(func(tx *bbolt.Tx) error { + buck := tx.Bucket([]byte(bolt.RepoMetadataBucket)) buck.Get([]byte(repo)) repoMetaBlob := buck.Get([]byte(repo)) @@ -689,8 +676,8 @@ func (bdw *DBWrapper) GetMultipleRepoMeta(ctx context.Context, filter func(repoM return nil, err } - err = bdw.DB.View(func(tx *bolt.Tx) error { - buck := tx.Bucket([]byte(repodb.RepoMetadataBucket)) + err = bdw.DB.View(func(tx *bbolt.Tx) error { + buck := tx.Bucket([]byte(bolt.RepoMetadataBucket)) cursor := buck.Cursor() @@ -722,8 +709,8 @@ func (bdw *DBWrapper) GetMultipleRepoMeta(ctx context.Context, filter func(repoM } func (bdw *DBWrapper) IncrementImageDownloads(repo string, reference string) error { - err := bdw.DB.Update(func(tx *bolt.Tx) error { - buck := tx.Bucket([]byte(repodb.RepoMetadataBucket)) + err := bdw.DB.Update(func(tx *bbolt.Tx) error { + buck := tx.Bucket([]byte(bolt.RepoMetadataBucket)) repoMetaBlob := buck.Get([]byte(repo)) if repoMetaBlob == nil { @@ -768,8 +755,8 @@ func (bdw *DBWrapper) IncrementImageDownloads(repo string, reference string) err func (bdw *DBWrapper) AddManifestSignature(repo string, signedManifestDigest godigest.Digest, sygMeta repodb.SignatureMetadata, ) error { - err := bdw.DB.Update(func(tx *bolt.Tx) error { - buck := tx.Bucket([]byte(repodb.RepoMetadataBucket)) + err := bdw.DB.Update(func(tx *bbolt.Tx) error { + buck := tx.Bucket([]byte(bolt.RepoMetadataBucket)) repoMetaBlob := buck.Get([]byte(repo)) if repoMetaBlob == nil { @@ -825,8 +812,8 @@ func (bdw *DBWrapper) AddManifestSignature(repo string, signedManifestDigest god func (bdw *DBWrapper) DeleteSignature(repo string, signedManifestDigest godigest.Digest, sigMeta repodb.SignatureMetadata, ) error { - err := bdw.DB.Update(func(tx *bolt.Tx) error { - buck := tx.Bucket([]byte(repodb.RepoMetadataBucket)) + err := bdw.DB.Update(func(tx *bbolt.Tx) error { + buck := tx.Bucket([]byte(bolt.RepoMetadataBucket)) repoMetaBlob := buck.Get([]byte(repo)) if repoMetaBlob == nil { @@ -895,13 +882,13 @@ func (bdw *DBWrapper) SearchRepos(ctx context.Context, searchText string, filter repodb.PageInfo{}, err } - err = bdw.DB.View(func(transaction *bolt.Tx) error { + err = bdw.DB.View(func(transaction *bbolt.Tx) error { var ( manifestMetadataMap = make(map[string]repodb.ManifestMetadata) indexDataMap = make(map[string]repodb.IndexData) - repoBuck = transaction.Bucket([]byte(repodb.RepoMetadataBucket)) - indexBuck = transaction.Bucket([]byte(repodb.IndexDataBucket)) - manifestBuck = transaction.Bucket([]byte(repodb.ManifestDataBucket)) + repoBuck = transaction.Bucket([]byte(bolt.RepoMetadataBucket)) + indexBuck = transaction.Bucket([]byte(bolt.IndexDataBucket)) + manifestBuck = transaction.Bucket([]byte(bolt.ManifestDataBucket)) ) cursor := repoBuck.Cursor() @@ -1072,7 +1059,7 @@ func (bdw *DBWrapper) SearchRepos(ctx context.Context, searchText string, filter } func fetchManifestMetaWithCheck(repoMeta repodb.RepoMetadata, manifestDigest string, - manifestMetadataMap map[string]repodb.ManifestMetadata, manifestBuck *bolt.Bucket, + manifestMetadataMap map[string]repodb.ManifestMetadata, manifestBuck *bbolt.Bucket, ) (repodb.ManifestMetadata, error) { manifestMeta, manifestDownloaded := manifestMetadataMap[manifestDigest] @@ -1097,7 +1084,7 @@ func fetchManifestMetaWithCheck(repoMeta repodb.RepoMetadata, manifestDigest str } func fetchIndexDataWithCheck(indexDigest string, indexDataMap map[string]repodb.IndexData, - indexBuck *bolt.Bucket, + indexBuck *bbolt.Bucket, ) (repodb.IndexData, error) { var ( indexData repodb.IndexData @@ -1157,7 +1144,7 @@ func collectImageManifestFilterData(digest string, repoMeta repodb.RepoMetadata, func collectImageIndexFilterInfo(indexDigest string, repoMeta repodb.RepoMetadata, indexData repodb.IndexData, manifestMetadataMap map[string]repodb.ManifestMetadata, - manifestBuck *bolt.Bucket, + manifestBuck *bbolt.Bucket, ) (repodb.FilterData, error) { var indexContent ispec.Index @@ -1250,11 +1237,11 @@ func (bdw *DBWrapper) FilterTags(ctx context.Context, filter repodb.FilterFunc, repodb.PageInfo{}, err } - err = bdw.DB.View(func(tx *bolt.Tx) error { + err = bdw.DB.View(func(tx *bbolt.Tx) error { var ( - repoBuck = tx.Bucket([]byte(repodb.RepoMetadataBucket)) - indexBuck = tx.Bucket([]byte(repodb.IndexDataBucket)) - manifestBuck = tx.Bucket([]byte(repodb.ManifestDataBucket)) + repoBuck = tx.Bucket([]byte(bolt.RepoMetadataBucket)) + indexBuck = tx.Bucket([]byte(bolt.IndexDataBucket)) + manifestBuck = tx.Bucket([]byte(bolt.ManifestDataBucket)) cursor = repoBuck.Cursor() ) @@ -1417,11 +1404,11 @@ func (bdw *DBWrapper) SearchTags(ctx context.Context, searchText string, filter fmt.Errorf("repodb: error while parsing search text, invalid format %w", err) } - err = bdw.DB.View(func(tx *bolt.Tx) error { + err = bdw.DB.View(func(tx *bbolt.Tx) error { var ( - repoBuck = tx.Bucket([]byte(repodb.RepoMetadataBucket)) - indexBuck = tx.Bucket([]byte(repodb.IndexDataBucket)) - manifestBuck = tx.Bucket([]byte(repodb.ManifestDataBucket)) + repoBuck = tx.Bucket([]byte(bolt.RepoMetadataBucket)) + indexBuck = tx.Bucket([]byte(bolt.IndexDataBucket)) + manifestBuck = tx.Bucket([]byte(bolt.ManifestDataBucket)) cursor = repoBuck.Cursor() ) @@ -1583,8 +1570,8 @@ func (bdw *DBWrapper) SearchTags(ctx context.Context, searchText string, filter func (bdw *DBWrapper) PatchDB() error { var DBVersion string - err := bdw.DB.View(func(tx *bolt.Tx) error { - versionBuck := tx.Bucket([]byte(repodb.VersionBucket)) + err := bdw.DB.View(func(tx *bbolt.Tx) error { + versionBuck := tx.Bucket([]byte(bolt.VersionBucket)) DBVersion = string(versionBuck.Get([]byte(version.DBVersionKey))) return nil diff --git a/pkg/meta/repodb/boltdb-wrapper/boltdb_wrapper_test.go b/pkg/meta/repodb/boltdb-wrapper/boltdb_wrapper_test.go index 83cf9995..a0e9bfda 100644 --- a/pkg/meta/repodb/boltdb-wrapper/boltdb_wrapper_test.go +++ b/pkg/meta/repodb/boltdb-wrapper/boltdb_wrapper_test.go @@ -11,8 +11,10 @@ import ( . "github.com/smartystreets/goconvey/convey" "go.etcd.io/bbolt" + "zotregistry.io/zot/pkg/log" + "zotregistry.io/zot/pkg/meta/bolt" "zotregistry.io/zot/pkg/meta/repodb" - bolt "zotregistry.io/zot/pkg/meta/repodb/boltdb-wrapper" + boltdb_wrapper "zotregistry.io/zot/pkg/meta/repodb/boltdb-wrapper" "zotregistry.io/zot/pkg/test" ) @@ -21,7 +23,12 @@ func TestWrapperErrors(t *testing.T) { ctx := context.Background() tmpDir := t.TempDir() boltDBParams := bolt.DBParameters{RootDir: tmpDir} - boltdbWrapper, err := bolt.NewBoltDBWrapper(boltDBParams) + boltDriver, err := bolt.GetBoltDriver(boltDBParams) + So(err, ShouldBeNil) + + log := log.NewLogger("debug", "") + + boltdbWrapper, err := boltdb_wrapper.NewBoltDBWrapper(boltDriver, log) defer os.Remove("repo.db") So(boltdbWrapper, ShouldNotBeNil) So(err, ShouldBeNil) @@ -36,7 +43,7 @@ func TestWrapperErrors(t *testing.T) { Convey("GetManifestData", func() { err := boltdbWrapper.DB.Update(func(tx *bbolt.Tx) error { - dataBuck := tx.Bucket([]byte(repodb.ManifestDataBucket)) + dataBuck := tx.Bucket([]byte(bolt.ManifestDataBucket)) return dataBuck.Put([]byte("digest1"), []byte("wrong json")) }) @@ -51,8 +58,8 @@ func TestWrapperErrors(t *testing.T) { Convey("SetManifestMeta", func() { err := boltdbWrapper.DB.Update(func(tx *bbolt.Tx) error { - repoBuck := tx.Bucket([]byte(repodb.RepoMetadataBucket)) - dataBuck := tx.Bucket([]byte(repodb.ManifestDataBucket)) + repoBuck := tx.Bucket([]byte(bolt.RepoMetadataBucket)) + dataBuck := tx.Bucket([]byte(bolt.ManifestDataBucket)) err := dataBuck.Put([]byte("digest1"), repoMetaBlob) if err != nil { @@ -72,7 +79,7 @@ func TestWrapperErrors(t *testing.T) { Convey("GetArtifactData", func() { err := boltdbWrapper.DB.Update(func(tx *bbolt.Tx) error { - artifactBuck := tx.Bucket([]byte(repodb.ArtifactDataBucket)) + artifactBuck := tx.Bucket([]byte(bolt.ArtifactDataBucket)) return artifactBuck.Put([]byte("artifactDigest"), []byte("wrong json")) }) @@ -84,7 +91,7 @@ func TestWrapperErrors(t *testing.T) { Convey("SetReferrer", func() { err := boltdbWrapper.DB.Update(func(tx *bbolt.Tx) error { - repoBuck := tx.Bucket([]byte(repodb.RepoMetadataBucket)) + repoBuck := tx.Bucket([]byte(bolt.RepoMetadataBucket)) return repoBuck.Put([]byte("repo"), []byte("wrong json")) }) @@ -102,7 +109,7 @@ func TestWrapperErrors(t *testing.T) { Convey("bad repo meta blob", func() { err := boltdbWrapper.DB.Update(func(tx *bbolt.Tx) error { - repoBuck := tx.Bucket([]byte(repodb.RepoMetadataBucket)) + repoBuck := tx.Bucket([]byte(bolt.RepoMetadataBucket)) return repoBuck.Put([]byte("repo"), []byte("wrong json")) }) @@ -115,7 +122,7 @@ func TestWrapperErrors(t *testing.T) { Convey("SetRepoReference", func() { err := boltdbWrapper.DB.Update(func(tx *bbolt.Tx) error { - repoBuck := tx.Bucket([]byte(repodb.RepoMetadataBucket)) + repoBuck := tx.Bucket([]byte(bolt.RepoMetadataBucket)) return repoBuck.Put([]byte("repo1"), []byte("wrong json")) }) @@ -127,7 +134,7 @@ func TestWrapperErrors(t *testing.T) { Convey("GetRepoMeta", func() { err := boltdbWrapper.DB.Update(func(tx *bbolt.Tx) error { - repoBuck := tx.Bucket([]byte(repodb.RepoMetadataBucket)) + repoBuck := tx.Bucket([]byte(bolt.RepoMetadataBucket)) return repoBuck.Put([]byte("repo1"), []byte("wrong json")) }) @@ -139,7 +146,7 @@ func TestWrapperErrors(t *testing.T) { Convey("DeleteRepoTag", func() { err := boltdbWrapper.DB.Update(func(tx *bbolt.Tx) error { - repoBuck := tx.Bucket([]byte(repodb.RepoMetadataBucket)) + repoBuck := tx.Bucket([]byte(bolt.RepoMetadataBucket)) return repoBuck.Put([]byte("repo1"), []byte("wrong json")) }) @@ -154,7 +161,7 @@ func TestWrapperErrors(t *testing.T) { So(err, ShouldNotBeNil) err := boltdbWrapper.DB.Update(func(tx *bbolt.Tx) error { - repoBuck := tx.Bucket([]byte(repodb.RepoMetadataBucket)) + repoBuck := tx.Bucket([]byte(bolt.RepoMetadataBucket)) return repoBuck.Put([]byte("repo1"), []byte("wrong json")) }) @@ -166,7 +173,7 @@ func TestWrapperErrors(t *testing.T) { Convey("IncrementRepoStars", func() { err := boltdbWrapper.DB.Update(func(tx *bbolt.Tx) error { - repoBuck := tx.Bucket([]byte(repodb.RepoMetadataBucket)) + repoBuck := tx.Bucket([]byte(bolt.RepoMetadataBucket)) return repoBuck.Put([]byte("repo1"), []byte("wrong json")) }) @@ -181,7 +188,7 @@ func TestWrapperErrors(t *testing.T) { Convey("DecrementRepoStars", func() { err := boltdbWrapper.DB.Update(func(tx *bbolt.Tx) error { - repoBuck := tx.Bucket([]byte(repodb.RepoMetadataBucket)) + repoBuck := tx.Bucket([]byte(bolt.RepoMetadataBucket)) return repoBuck.Put([]byte("repo1"), []byte("wrong json")) }) @@ -196,7 +203,7 @@ func TestWrapperErrors(t *testing.T) { Convey("GetRepoStars", func() { err := boltdbWrapper.DB.Update(func(tx *bbolt.Tx) error { - repoBuck := tx.Bucket([]byte(repodb.RepoMetadataBucket)) + repoBuck := tx.Bucket([]byte(bolt.RepoMetadataBucket)) return repoBuck.Put([]byte("repo1"), []byte("wrong json")) }) @@ -208,7 +215,7 @@ func TestWrapperErrors(t *testing.T) { Convey("GetMultipleRepoMeta", func() { err := boltdbWrapper.DB.Update(func(tx *bbolt.Tx) error { - repoBuck := tx.Bucket([]byte(repodb.RepoMetadataBucket)) + repoBuck := tx.Bucket([]byte(bolt.RepoMetadataBucket)) return repoBuck.Put([]byte("repo1"), []byte("wrong json")) }) @@ -222,7 +229,7 @@ func TestWrapperErrors(t *testing.T) { Convey("IncrementImageDownloads", func() { err := boltdbWrapper.DB.Update(func(tx *bbolt.Tx) error { - repoBuck := tx.Bucket([]byte(repodb.RepoMetadataBucket)) + repoBuck := tx.Bucket([]byte(bolt.RepoMetadataBucket)) return repoBuck.Put([]byte("repo1"), []byte("wrong json")) }) @@ -235,7 +242,7 @@ func TestWrapperErrors(t *testing.T) { So(err, ShouldNotBeNil) err = boltdbWrapper.DB.Update(func(tx *bbolt.Tx) error { - repoBuck := tx.Bucket([]byte(repodb.RepoMetadataBucket)) + repoBuck := tx.Bucket([]byte(bolt.RepoMetadataBucket)) return repoBuck.Put([]byte("repo1"), repoMetaBlob) }) @@ -247,7 +254,7 @@ func TestWrapperErrors(t *testing.T) { Convey("AddManifestSignature", func() { err := boltdbWrapper.DB.Update(func(tx *bbolt.Tx) error { - repoBuck := tx.Bucket([]byte(repodb.RepoMetadataBucket)) + repoBuck := tx.Bucket([]byte(bolt.RepoMetadataBucket)) return repoBuck.Put([]byte("repo1"), []byte("wrong json")) }) @@ -262,7 +269,7 @@ func TestWrapperErrors(t *testing.T) { So(err, ShouldNotBeNil) err = boltdbWrapper.DB.Update(func(tx *bbolt.Tx) error { - repoBuck := tx.Bucket([]byte(repodb.RepoMetadataBucket)) + repoBuck := tx.Bucket([]byte(bolt.RepoMetadataBucket)) return repoBuck.Put([]byte("repo1"), repoMetaBlob) }) @@ -275,7 +282,7 @@ func TestWrapperErrors(t *testing.T) { // err = boltdbWrapper.DB.Update(func(tx *bbolt.Tx) error { - repoBuck := tx.Bucket([]byte(repodb.RepoMetadataBucket)) + repoBuck := tx.Bucket([]byte(bolt.RepoMetadataBucket)) repoMeta := repodb.RepoMetadata{ Tags: map[string]repodb.Descriptor{}, @@ -313,7 +320,7 @@ func TestWrapperErrors(t *testing.T) { Convey("DeleteSignature", func() { err := boltdbWrapper.DB.Update(func(tx *bbolt.Tx) error { - repoBuck := tx.Bucket([]byte(repodb.RepoMetadataBucket)) + repoBuck := tx.Bucket([]byte(bolt.RepoMetadataBucket)) return repoBuck.Put([]byte("repo1"), []byte("wrong json")) }) @@ -328,7 +335,7 @@ func TestWrapperErrors(t *testing.T) { So(err, ShouldNotBeNil) err = boltdbWrapper.DB.Update(func(tx *bbolt.Tx) error { - repoBuck := tx.Bucket([]byte(repodb.RepoMetadataBucket)) + repoBuck := tx.Bucket([]byte(bolt.RepoMetadataBucket)) repoMeta := repodb.RepoMetadata{ Tags: map[string]repodb.Descriptor{}, @@ -366,7 +373,7 @@ func TestWrapperErrors(t *testing.T) { Convey("SearchRepos", func() { err := boltdbWrapper.DB.Update(func(tx *bbolt.Tx) error { - repoBuck := tx.Bucket([]byte(repodb.RepoMetadataBucket)) + repoBuck := tx.Bucket([]byte(bolt.RepoMetadataBucket)) return repoBuck.Put([]byte("repo1"), []byte("wrong json")) }) @@ -376,8 +383,8 @@ func TestWrapperErrors(t *testing.T) { So(err, ShouldNotBeNil) err = boltdbWrapper.DB.Update(func(tx *bbolt.Tx) error { - repoBuck := tx.Bucket([]byte(repodb.RepoMetadataBucket)) - dataBuck := tx.Bucket([]byte(repodb.ManifestDataBucket)) + repoBuck := tx.Bucket([]byte(bolt.RepoMetadataBucket)) + dataBuck := tx.Bucket([]byte(bolt.ManifestDataBucket)) err := dataBuck.Put([]byte("dig1"), []byte("wrong json")) if err != nil { @@ -418,8 +425,8 @@ func TestWrapperErrors(t *testing.T) { So(err, ShouldNotBeNil) err = boltdbWrapper.DB.Update(func(tx *bbolt.Tx) error { - repoBuck := tx.Bucket([]byte(repodb.RepoMetadataBucket)) - dataBuck := tx.Bucket([]byte(repodb.ManifestDataBucket)) + repoBuck := tx.Bucket([]byte(bolt.RepoMetadataBucket)) + dataBuck := tx.Bucket([]byte(bolt.ManifestDataBucket)) manifestMeta := repodb.ManifestMetadata{ ManifestBlob: []byte("{}"), @@ -533,7 +540,7 @@ func TestWrapperErrors(t *testing.T) { ctx := context.Background() err := boltdbWrapper.DB.Update(func(tx *bbolt.Tx) error { - repoBuck := tx.Bucket([]byte(repodb.RepoMetadataBucket)) + repoBuck := tx.Bucket([]byte(bolt.RepoMetadataBucket)) return repoBuck.Put([]byte("repo1"), []byte("wrong json")) }) @@ -546,8 +553,8 @@ func TestWrapperErrors(t *testing.T) { So(err, ShouldNotBeNil) err = boltdbWrapper.DB.Update(func(tx *bbolt.Tx) error { - repoBuck := tx.Bucket([]byte(repodb.RepoMetadataBucket)) - dataBuck := tx.Bucket([]byte(repodb.ManifestDataBucket)) + repoBuck := tx.Bucket([]byte(bolt.RepoMetadataBucket)) + dataBuck := tx.Bucket([]byte(bolt.ManifestDataBucket)) manifestMeta := repodb.ManifestMetadata{ ManifestBlob: []byte("{}"), @@ -722,7 +729,7 @@ func TestWrapperErrors(t *testing.T) { func setBadIndexData(dB *bbolt.DB, digest string) error { return dB.Update(func(tx *bbolt.Tx) error { - indexDataBuck := tx.Bucket([]byte(repodb.IndexDataBucket)) + indexDataBuck := tx.Bucket([]byte(bolt.IndexDataBucket)) return indexDataBuck.Put([]byte(digest), []byte("bad json")) }) diff --git a/pkg/meta/repodb/dynamodb-wrapper/dynamo_internal_test.go b/pkg/meta/repodb/dynamodb-wrapper/dynamo_internal_test.go index 8898a4fa..521b5c1d 100644 --- a/pkg/meta/repodb/dynamodb-wrapper/dynamo_internal_test.go +++ b/pkg/meta/repodb/dynamodb-wrapper/dynamo_internal_test.go @@ -13,7 +13,7 @@ import ( . "github.com/smartystreets/goconvey/convey" "zotregistry.io/zot/pkg/log" //nolint:go-staticcheck - "zotregistry.io/zot/pkg/meta/repodb/version" + "zotregistry.io/zot/pkg/meta/version" ) func TestWrapperErrors(t *testing.T) { diff --git a/pkg/meta/repodb/dynamodb-wrapper/dynamo_test.go b/pkg/meta/repodb/dynamodb-wrapper/dynamo_test.go index b7e7f2c4..ec46964b 100644 --- a/pkg/meta/repodb/dynamodb-wrapper/dynamo_test.go +++ b/pkg/meta/repodb/dynamodb-wrapper/dynamo_test.go @@ -18,10 +18,9 @@ import ( . "github.com/smartystreets/goconvey/convey" "zotregistry.io/zot/pkg/log" + "zotregistry.io/zot/pkg/meta/dynamo" "zotregistry.io/zot/pkg/meta/repodb" - dynamo "zotregistry.io/zot/pkg/meta/repodb/dynamodb-wrapper" - "zotregistry.io/zot/pkg/meta/repodb/dynamodb-wrapper/iterator" - dynamoParams "zotregistry.io/zot/pkg/meta/repodb/dynamodb-wrapper/params" + dynamoWrapper "zotregistry.io/zot/pkg/meta/repodb/dynamodb-wrapper" "zotregistry.io/zot/pkg/test" ) @@ -44,8 +43,10 @@ func TestIterator(t *testing.T) { indexDataTablename := "IndexDataTable" + uuid.String() artifactDataTablename := "ArtifactDataTable" + uuid.String() + log := log.NewLogger("debug", "") + Convey("TestIterator", t, func() { - dynamoWrapper, err := dynamo.NewDynamoDBWrapper(dynamoParams.DBDriverParameters{ + params := dynamo.DBDriverParameters{ Endpoint: endpoint, Region: region, RepoMetaTablename: repoMetaTablename, @@ -53,7 +54,11 @@ func TestIterator(t *testing.T) { IndexDataTablename: indexDataTablename, ArtifactDataTablename: artifactDataTablename, VersionTablename: versionTablename, - }) + } + client, err := dynamo.GetDynamoClient(params) + So(err, ShouldBeNil) + + dynamoWrapper, err := dynamoWrapper.NewDynamoDBWrapper(client, params, log) So(err, ShouldBeNil) So(dynamoWrapper.ResetManifestDataTable(), ShouldBeNil) @@ -68,12 +73,12 @@ func TestIterator(t *testing.T) { err = dynamoWrapper.SetRepoReference("repo3", "tag3", "manifestType", "manifestDigest3") So(err, ShouldBeNil) - repoMetaAttributeIterator := iterator.NewBaseDynamoAttributesIterator( + repoMetaAttributeIterator := dynamo.NewBaseDynamoAttributesIterator( dynamoWrapper.Client, repoMetaTablename, "RepoMetadata", 1, - log.Logger{Logger: zerolog.New(os.Stdout)}, + log, ) attribute, err := repoMetaAttributeIterator.First(context.Background()) @@ -109,7 +114,7 @@ func TestIteratorErrors(t *testing.T) { config.WithEndpointResolverWithOptions(customResolver)) So(err, ShouldBeNil) - repoMetaAttributeIterator := iterator.NewBaseDynamoAttributesIterator( + repoMetaAttributeIterator := dynamo.NewBaseDynamoAttributesIterator( dynamodb.NewFromConfig(cfg), "RepoMetadataTable", "RepoMetadata", @@ -141,8 +146,10 @@ func TestWrapperErrors(t *testing.T) { ctx := context.Background() + log := log.NewLogger("debug", "") + Convey("Errors", t, func() { - dynamoWrapper, err := dynamo.NewDynamoDBWrapper(dynamoParams.DBDriverParameters{ //nolint:contextcheck + params := dynamo.DBDriverParameters{ //nolint:contextcheck Endpoint: endpoint, Region: region, RepoMetaTablename: repoMetaTablename, @@ -150,7 +157,11 @@ func TestWrapperErrors(t *testing.T) { IndexDataTablename: indexDataTablename, ArtifactDataTablename: artifactDataTablename, VersionTablename: versionTablename, - }) + } + client, err := dynamo.GetDynamoClient(params) //nolint:contextcheck + So(err, ShouldBeNil) + + dynamoWrapper, err := dynamoWrapper.NewDynamoDBWrapper(client, params, log) //nolint:contextcheck So(err, ShouldBeNil) So(dynamoWrapper.ResetManifestDataTable(), ShouldBeNil) //nolint:contextcheck @@ -796,7 +807,7 @@ func TestWrapperErrors(t *testing.T) { }) Convey("NewDynamoDBWrapper errors", t, func() { - _, err := dynamo.NewDynamoDBWrapper(dynamoParams.DBDriverParameters{ //nolint:contextcheck + params := dynamo.DBDriverParameters{ //nolint:contextcheck Endpoint: endpoint, Region: region, RepoMetaTablename: "", @@ -804,10 +815,14 @@ func TestWrapperErrors(t *testing.T) { IndexDataTablename: indexDataTablename, ArtifactDataTablename: artifactDataTablename, VersionTablename: versionTablename, - }) + } + client, err := dynamo.GetDynamoClient(params) + So(err, ShouldBeNil) + + _, err = dynamoWrapper.NewDynamoDBWrapper(client, params, log) So(err, ShouldNotBeNil) - _, err = dynamo.NewDynamoDBWrapper(dynamoParams.DBDriverParameters{ //nolint:contextcheck + params = dynamo.DBDriverParameters{ //nolint:contextcheck Endpoint: endpoint, Region: region, RepoMetaTablename: repoMetaTablename, @@ -815,10 +830,14 @@ func TestWrapperErrors(t *testing.T) { IndexDataTablename: indexDataTablename, ArtifactDataTablename: artifactDataTablename, VersionTablename: versionTablename, - }) + } + client, err = dynamo.GetDynamoClient(params) + So(err, ShouldBeNil) + + _, err = dynamoWrapper.NewDynamoDBWrapper(client, params, log) So(err, ShouldNotBeNil) - _, err = dynamo.NewDynamoDBWrapper(dynamoParams.DBDriverParameters{ //nolint:contextcheck + params = dynamo.DBDriverParameters{ //nolint:contextcheck Endpoint: endpoint, Region: region, RepoMetaTablename: repoMetaTablename, @@ -826,10 +845,14 @@ func TestWrapperErrors(t *testing.T) { IndexDataTablename: "", ArtifactDataTablename: artifactDataTablename, VersionTablename: versionTablename, - }) + } + client, err = dynamo.GetDynamoClient(params) + So(err, ShouldBeNil) + + _, err = dynamoWrapper.NewDynamoDBWrapper(client, params, log) So(err, ShouldNotBeNil) - _, err = dynamo.NewDynamoDBWrapper(dynamoParams.DBDriverParameters{ //nolint:contextcheck + params = dynamo.DBDriverParameters{ //nolint:contextcheck Endpoint: endpoint, Region: region, RepoMetaTablename: repoMetaTablename, @@ -837,10 +860,14 @@ func TestWrapperErrors(t *testing.T) { IndexDataTablename: indexDataTablename, ArtifactDataTablename: artifactDataTablename, VersionTablename: "", - }) + } + client, err = dynamo.GetDynamoClient(params) + So(err, ShouldBeNil) + + _, err = dynamoWrapper.NewDynamoDBWrapper(client, params, log) So(err, ShouldNotBeNil) - _, err = dynamo.NewDynamoDBWrapper(dynamoParams.DBDriverParameters{ //nolint:contextcheck + params = dynamo.DBDriverParameters{ //nolint:contextcheck Endpoint: endpoint, Region: region, RepoMetaTablename: repoMetaTablename, @@ -848,10 +875,14 @@ func TestWrapperErrors(t *testing.T) { IndexDataTablename: indexDataTablename, ArtifactDataTablename: "", VersionTablename: versionTablename, - }) + } + client, err = dynamo.GetDynamoClient(params) + So(err, ShouldBeNil) + + _, err = dynamoWrapper.NewDynamoDBWrapper(client, params, log) So(err, ShouldNotBeNil) - _, err = dynamo.NewDynamoDBWrapper(dynamoParams.DBDriverParameters{ //nolint:contextcheck + params = dynamo.DBDriverParameters{ //nolint:contextcheck Endpoint: endpoint, Region: region, RepoMetaTablename: repoMetaTablename, @@ -859,7 +890,11 @@ func TestWrapperErrors(t *testing.T) { IndexDataTablename: indexDataTablename, VersionTablename: versionTablename, ArtifactDataTablename: artifactDataTablename, - }) + } + client, err = dynamo.GetDynamoClient(params) + So(err, ShouldBeNil) + + _, err = dynamoWrapper.NewDynamoDBWrapper(client, params, log) So(err, ShouldBeNil) }) } diff --git a/pkg/meta/repodb/dynamodb-wrapper/dynamo_wrapper.go b/pkg/meta/repodb/dynamodb-wrapper/dynamo_wrapper.go index 538b23eb..8fb4ad00 100644 --- a/pkg/meta/repodb/dynamodb-wrapper/dynamo_wrapper.go +++ b/pkg/meta/repodb/dynamodb-wrapper/dynamo_wrapper.go @@ -5,26 +5,22 @@ import ( "encoding/json" "errors" "fmt" - "os" "strings" "time" "github.com/aws/aws-sdk-go-v2/aws" - "github.com/aws/aws-sdk-go-v2/config" "github.com/aws/aws-sdk-go-v2/feature/dynamodb/attributevalue" "github.com/aws/aws-sdk-go-v2/service/dynamodb" "github.com/aws/aws-sdk-go-v2/service/dynamodb/types" godigest "github.com/opencontainers/go-digest" ispec "github.com/opencontainers/image-spec/specs-go/v1" - "github.com/rs/zerolog" zerr "zotregistry.io/zot/errors" "zotregistry.io/zot/pkg/log" + "zotregistry.io/zot/pkg/meta/common" + "zotregistry.io/zot/pkg/meta/dynamo" "zotregistry.io/zot/pkg/meta/repodb" //nolint:go-staticcheck - "zotregistry.io/zot/pkg/meta/repodb/common" - "zotregistry.io/zot/pkg/meta/repodb/dynamodb-wrapper/iterator" - dynamoParams "zotregistry.io/zot/pkg/meta/repodb/dynamodb-wrapper/params" - "zotregistry.io/zot/pkg/meta/repodb/version" + "zotregistry.io/zot/pkg/meta/version" localCtx "zotregistry.io/zot/pkg/requestcontext" ) @@ -41,38 +37,19 @@ type DBWrapper struct { Log log.Logger } -func NewDynamoDBWrapper(params dynamoParams.DBDriverParameters) (*DBWrapper, error) { - // custom endpoint resolver to point to localhost - customResolver := aws.EndpointResolverWithOptionsFunc( - func(service, region string, options ...interface{}) (aws.Endpoint, error) { - return aws.Endpoint{ - PartitionID: "aws", - URL: params.Endpoint, - SigningRegion: region, - }, nil - }) - - // Using the SDK's default configuration, loading additional config - // and credentials values from the environment variables, shared - // credentials, and shared configuration files - cfg, err := config.LoadDefaultConfig(context.Background(), config.WithRegion(params.Region), - config.WithEndpointResolverWithOptions(customResolver)) - if err != nil { - return nil, err - } - +func NewDynamoDBWrapper(client *dynamodb.Client, params dynamo.DBDriverParameters, log log.Logger) (*DBWrapper, error) { dynamoWrapper := DBWrapper{ - Client: dynamodb.NewFromConfig(cfg), + Client: client, RepoMetaTablename: params.RepoMetaTablename, ManifestDataTablename: params.ManifestDataTablename, IndexDataTablename: params.IndexDataTablename, ArtifactDataTablename: params.ArtifactDataTablename, VersionTablename: params.VersionTablename, Patches: version.GetDynamoDBPatches(), - Log: log.Logger{Logger: zerolog.New(os.Stdout)}, + Log: log, } - err = dynamoWrapper.createVersionTable() + err := dynamoWrapper.createVersionTable() if err != nil { return nil, err } @@ -740,11 +717,11 @@ func (dwr *DBWrapper) GetMultipleRepoMeta(ctx context.Context, filter func(repoMeta repodb.RepoMetadata) bool, requestedPage repodb.PageInput, ) ([]repodb.RepoMetadata, error) { var ( - repoMetaAttributeIterator iterator.AttributesIterator + repoMetaAttributeIterator dynamo.AttributesIterator pageFinder repodb.PageFinder ) - repoMetaAttributeIterator = iterator.NewBaseDynamoAttributesIterator( + repoMetaAttributeIterator = dynamo.NewBaseDynamoAttributesIterator( dwr.Client, dwr.RepoMetaTablename, "RepoMetadata", 0, dwr.Log, ) @@ -790,12 +767,12 @@ func (dwr *DBWrapper) SearchRepos(ctx context.Context, searchText string, filter var ( manifestMetadataMap = make(map[string]repodb.ManifestMetadata) indexDataMap = make(map[string]repodb.IndexData) - repoMetaAttributeIterator iterator.AttributesIterator + repoMetaAttributeIterator dynamo.AttributesIterator pageFinder repodb.PageFinder pageInfo repodb.PageInfo ) - repoMetaAttributeIterator = iterator.NewBaseDynamoAttributesIterator( + repoMetaAttributeIterator = dynamo.NewBaseDynamoAttributesIterator( dwr.Client, dwr.RepoMetaTablename, "RepoMetadata", 0, dwr.Log, ) @@ -1080,11 +1057,11 @@ func (dwr *DBWrapper) FilterTags(ctx context.Context, filter repodb.FilterFunc, manifestMetadataMap = make(map[string]repodb.ManifestMetadata) indexDataMap = make(map[string]repodb.IndexData) pageFinder repodb.PageFinder - repoMetaAttributeIterator iterator.AttributesIterator + repoMetaAttributeIterator dynamo.AttributesIterator pageInfo repodb.PageInfo ) - repoMetaAttributeIterator = iterator.NewBaseDynamoAttributesIterator( + repoMetaAttributeIterator = dynamo.NewBaseDynamoAttributesIterator( dwr.Client, dwr.RepoMetaTablename, "RepoMetadata", 0, dwr.Log, ) @@ -1219,7 +1196,7 @@ func (dwr *DBWrapper) SearchTags(ctx context.Context, searchText string, filter var ( manifestMetadataMap = make(map[string]repodb.ManifestMetadata) indexDataMap = make(map[string]repodb.IndexData) - repoMetaAttributeIterator = iterator.NewBaseDynamoAttributesIterator( + repoMetaAttributeIterator = dynamo.NewBaseDynamoAttributesIterator( dwr.Client, dwr.RepoMetaTablename, "RepoMetadata", 0, dwr.Log, ) diff --git a/pkg/meta/repodb/dynamodb-wrapper/params/parameters.go b/pkg/meta/repodb/dynamodb-wrapper/params/parameters.go deleted file mode 100644 index 4e2c7314..00000000 --- a/pkg/meta/repodb/dynamodb-wrapper/params/parameters.go +++ /dev/null @@ -1,6 +0,0 @@ -package params - -type DBDriverParameters struct { - Endpoint, Region, RepoMetaTablename, ManifestDataTablename, IndexDataTablename, - ArtifactDataTablename, VersionTablename string -} diff --git a/pkg/meta/repodb/repodb.go b/pkg/meta/repodb/repodb.go index 2c5e3062..54d4ba8d 100644 --- a/pkg/meta/repodb/repodb.go +++ b/pkg/meta/repodb/repodb.go @@ -7,16 +7,6 @@ import ( godigest "github.com/opencontainers/go-digest" ) -// MetadataDB. -const ( - ManifestDataBucket = "ManifestData" - IndexDataBucket = "IndexData" - ArtifactDataBucket = "ArtifactData" - UserMetadataBucket = "UserMeta" - RepoMetadataBucket = "RepoMetadata" - VersionBucket = "Version" -) - const ( SignaturesDirPath = "/tmp/zot/signatures" SigKey = "dev.cosignproject.cosign/signature" diff --git a/pkg/meta/repodb/repodb_test.go b/pkg/meta/repodb/repodb_test.go index 685bf6fc..d50fe853 100644 --- a/pkg/meta/repodb/repodb_test.go +++ b/pkg/meta/repodb/repodb_test.go @@ -18,11 +18,13 @@ import ( ispec "github.com/opencontainers/image-spec/specs-go/v1" . "github.com/smartystreets/goconvey/convey" + "zotregistry.io/zot/pkg/log" + "zotregistry.io/zot/pkg/meta/bolt" + "zotregistry.io/zot/pkg/meta/common" + "zotregistry.io/zot/pkg/meta/dynamo" "zotregistry.io/zot/pkg/meta/repodb" - bolt "zotregistry.io/zot/pkg/meta/repodb/boltdb-wrapper" - "zotregistry.io/zot/pkg/meta/repodb/common" - dynamo "zotregistry.io/zot/pkg/meta/repodb/dynamodb-wrapper" - dynamoParams "zotregistry.io/zot/pkg/meta/repodb/dynamodb-wrapper/params" + boltdb_wrapper "zotregistry.io/zot/pkg/meta/repodb/boltdb-wrapper" + dynamodb_wrapper "zotregistry.io/zot/pkg/meta/repodb/dynamodb-wrapper" localCtx "zotregistry.io/zot/pkg/requestcontext" "zotregistry.io/zot/pkg/test" ) @@ -36,15 +38,19 @@ const ( func TestBoltDBWrapper(t *testing.T) { Convey("BoltDB Wrapper creation", t, func() { boltDBParams := bolt.DBParameters{} - searchDB, err := bolt.NewBoltDBWrapper(boltDBParams) - So(searchDB, ShouldNotBeNil) + boltDriver, err := bolt.GetBoltDriver(boltDBParams) + So(err, ShouldBeNil) + + log := log.NewLogger("debug", "") + + repoDB, err := boltdb_wrapper.NewBoltDBWrapper(boltDriver, log) + So(repoDB, ShouldNotBeNil) So(err, ShouldBeNil) err = os.Chmod("repo.db", 0o200) So(err, ShouldBeNil) - searchDB, err = bolt.NewBoltDBWrapper(boltDBParams) - So(searchDB, ShouldBeNil) + _, err = bolt.GetBoltDriver(boltDBParams) So(err, ShouldNotBeNil) err = os.Chmod("repo.db", 0o600) @@ -55,7 +61,12 @@ func TestBoltDBWrapper(t *testing.T) { Convey("BoltDB Wrapper", t, func() { boltDBParams := bolt.DBParameters{} - boltdbWrapper, err := bolt.NewBoltDBWrapper(boltDBParams) + boltDriver, err := bolt.GetBoltDriver(boltDBParams) + So(err, ShouldBeNil) + + log := log.NewLogger("debug", "") + + boltdbWrapper, err := boltdb_wrapper.NewBoltDBWrapper(boltDriver, log) defer os.Remove("repo.db") So(boltdbWrapper, ShouldNotBeNil) So(err, ShouldBeNil) @@ -79,7 +90,7 @@ func TestDynamoDBWrapper(t *testing.T) { artifactDataTablename := "ArtifactDataTable" + uuid.String() Convey("DynamoDB Wrapper", t, func() { - dynamoDBDriverParams := dynamoParams.DBDriverParameters{ + dynamoDBDriverParams := dynamo.DBDriverParameters{ Endpoint: os.Getenv("DYNAMODBMOCK_ENDPOINT"), RepoMetaTablename: repoMetaTablename, ManifestDataTablename: manifestDataTablename, @@ -89,7 +100,12 @@ func TestDynamoDBWrapper(t *testing.T) { Region: "us-east-2", } - dynamoDriver, err := dynamo.NewDynamoDBWrapper(dynamoDBDriverParams) + dynamoClient, err := dynamo.GetDynamoClient(dynamoDBDriverParams) + So(err, ShouldBeNil) + + log := log.NewLogger("debug", "") + + dynamoDriver, err := dynamodb_wrapper.NewDynamoDBWrapper(dynamoClient, dynamoDBDriverParams, log) So(dynamoDriver, ShouldNotBeNil) So(err, ShouldBeNil) @@ -1985,8 +2001,12 @@ func TestRelevanceSorting(t *testing.T) { boltDBParams := bolt.DBParameters{ RootDir: t.TempDir(), } + boltDriver, err := bolt.GetBoltDriver(boltDBParams) + So(err, ShouldBeNil) - repoDB, err := bolt.NewBoltDBWrapper(boltDBParams) + log := log.NewLogger("debug", "") + + repoDB, err := boltdb_wrapper.NewBoltDBWrapper(boltDriver, log) So(repoDB, ShouldNotBeNil) So(err, ShouldBeNil) diff --git a/pkg/meta/repodb/repodbfactory/repodb_factory.go b/pkg/meta/repodb/repodbfactory/repodb_factory.go index 747e2870..33658274 100644 --- a/pkg/meta/repodb/repodbfactory/repodb_factory.go +++ b/pkg/meta/repodb/repodbfactory/repodb_factory.go @@ -1,32 +1,68 @@ package repodbfactory import ( + "github.com/aws/aws-sdk-go-v2/service/dynamodb" + "go.etcd.io/bbolt" + "zotregistry.io/zot/errors" + "zotregistry.io/zot/pkg/api/config" + "zotregistry.io/zot/pkg/log" + "zotregistry.io/zot/pkg/meta/bolt" + "zotregistry.io/zot/pkg/meta/dynamo" "zotregistry.io/zot/pkg/meta/repodb" boltdb_wrapper "zotregistry.io/zot/pkg/meta/repodb/boltdb-wrapper" dynamodb_wrapper "zotregistry.io/zot/pkg/meta/repodb/dynamodb-wrapper" - dynamoParams "zotregistry.io/zot/pkg/meta/repodb/dynamodb-wrapper/params" ) -func Create(dbtype string, parameters interface{}) (repodb.RepoDB, error) { //nolint:contextcheck +func New(storageConfig config.StorageConfig, log log.Logger) (repodb.RepoDB, error) { + if storageConfig.RemoteCache { + dynamoParams := getDynamoParams(storageConfig.CacheDriver, log) + + client, err := dynamo.GetDynamoClient(dynamoParams) + if err != nil { + return nil, err + } + + return Create("dynamodb", client, dynamoParams, log) //nolint:contextcheck + } + + params := bolt.DBParameters{} + params.RootDir = storageConfig.RootDirectory + + driver, err := bolt.GetBoltDriver(params) + if err != nil { + return nil, err + } + + return Create("boltdb", driver, params, log) //nolint:contextcheck +} + +func Create(dbtype string, dbDriver, parameters interface{}, log log.Logger, //nolint:contextcheck +) (repodb.RepoDB, error, +) { switch dbtype { case "boltdb": { - properParameters, ok := parameters.(boltdb_wrapper.DBParameters) + properDriver, ok := dbDriver.(*bbolt.DB) if !ok { panic("failed type assertion") } - return boltdb_wrapper.NewBoltDBWrapper(properParameters) + return boltdb_wrapper.NewBoltDBWrapper(properDriver, log) } case "dynamodb": { - properParameters, ok := parameters.(dynamoParams.DBDriverParameters) + properDriver, ok := dbDriver.(*dynamodb.Client) if !ok { panic("failed type assertion") } - return dynamodb_wrapper.NewDynamoDBWrapper(properParameters) + properParameters, ok := parameters.(dynamo.DBDriverParameters) + if !ok { + panic("failed type assertion") + } + + return dynamodb_wrapper.NewDynamoDBWrapper(properDriver, properParameters, log) } default: { @@ -34,3 +70,68 @@ func Create(dbtype string, parameters interface{}) (repodb.RepoDB, error) { //no } } } + +func getDynamoParams(cacheDriverConfig map[string]interface{}, log log.Logger) dynamo.DBDriverParameters { + allParametersOk := true + + endpoint, ok := toStringIfOk(cacheDriverConfig, "endpoint", log) + allParametersOk = allParametersOk && ok + + region, ok := toStringIfOk(cacheDriverConfig, "region", log) + allParametersOk = allParametersOk && ok + + repoMetaTablename, ok := toStringIfOk(cacheDriverConfig, "repometatablename", log) + allParametersOk = allParametersOk && ok + + manifestDataTablename, ok := toStringIfOk(cacheDriverConfig, "manifestdatatablename", log) + allParametersOk = allParametersOk && ok + + indexDataTablename, ok := toStringIfOk(cacheDriverConfig, "indexdatatablename", log) + allParametersOk = allParametersOk && ok + + artifactDataTablename, ok := toStringIfOk(cacheDriverConfig, "artifactdatatablename", log) + allParametersOk = allParametersOk && ok + + versionTablename, ok := toStringIfOk(cacheDriverConfig, "versiontablename", log) + allParametersOk = allParametersOk && ok + + if !allParametersOk { + panic("dynamo parameters are not specified correctly, can't proceede") + } + + return dynamo.DBDriverParameters{ + Endpoint: endpoint, + Region: region, + RepoMetaTablename: repoMetaTablename, + ManifestDataTablename: manifestDataTablename, + IndexDataTablename: indexDataTablename, + ArtifactDataTablename: artifactDataTablename, + VersionTablename: versionTablename, + } +} + +func toStringIfOk(cacheDriverConfig map[string]interface{}, param string, log log.Logger) (string, bool) { + val, ok := cacheDriverConfig[param] + + if !ok { + log.Error().Msgf("parsing CacheDriver config failed, field '%s' is not present", param) + + return "", false + } + + str, ok := val.(string) + + if !ok { + log.Error().Msgf("parsing CacheDriver config failed, parameter '%s' isn't a string", param) + + return "", false + } + + if str == "" { + log.Error().Msgf("parsing CacheDriver config failed, field '%s' is is empty", param) + + return "", false + } + + return str, ok +} diff --git a/pkg/meta/repodb/repodbfactory/repodb_factory_test.go b/pkg/meta/repodb/repodbfactory/repodb_factory_test.go index 8b968339..01ba76e3 100644 --- a/pkg/meta/repodb/repodbfactory/repodb_factory_test.go +++ b/pkg/meta/repodb/repodbfactory/repodb_factory_test.go @@ -4,10 +4,12 @@ import ( "os" "testing" + "github.com/aws/aws-sdk-go-v2/service/dynamodb" . "github.com/smartystreets/goconvey/convey" - bolt "zotregistry.io/zot/pkg/meta/repodb/boltdb-wrapper" - dynamoParams "zotregistry.io/zot/pkg/meta/repodb/dynamodb-wrapper/params" + "zotregistry.io/zot/pkg/log" + "zotregistry.io/zot/pkg/meta/bolt" + "zotregistry.io/zot/pkg/meta/dynamo" "zotregistry.io/zot/pkg/meta/repodb/repodbfactory" ) @@ -15,7 +17,7 @@ func TestCreateDynamo(t *testing.T) { skipDynamo(t) Convey("Create", t, func() { - dynamoDBDriverParams := dynamoParams.DBDriverParameters{ + dynamoDBDriverParams := dynamo.DBDriverParameters{ Endpoint: os.Getenv("DYNAMODBMOCK_ENDPOINT"), RepoMetaTablename: "RepoMetadataTable", ManifestDataTablename: "ManifestDataTable", @@ -25,15 +27,24 @@ func TestCreateDynamo(t *testing.T) { Region: "us-east-2", } - repoDB, err := repodbfactory.Create("dynamodb", dynamoDBDriverParams) + client, err := dynamo.GetDynamoClient(dynamoDBDriverParams) + So(err, ShouldBeNil) + + log := log.NewLogger("debug", "") + + repoDB, err := repodbfactory.Create("dynamodb", client, dynamoDBDriverParams, log) So(repoDB, ShouldNotBeNil) So(err, ShouldBeNil) }) Convey("Fails", t, func() { - So(func() { _, _ = repodbfactory.Create("dynamodb", bolt.DBParameters{RootDir: "root"}) }, ShouldPanic) + log := log.NewLogger("debug", "") - repoDB, err := repodbfactory.Create("random", bolt.DBParameters{RootDir: "root"}) + So(func() { _, _ = repodbfactory.Create("dynamodb", nil, bolt.DBParameters{RootDir: "root"}, log) }, ShouldPanic) + + So(func() { _, _ = repodbfactory.Create("dynamodb", &dynamodb.Client{}, "bad", log) }, ShouldPanic) + + repoDB, err := repodbfactory.Create("random", nil, bolt.DBParameters{RootDir: "root"}, log) So(repoDB, ShouldBeNil) So(err, ShouldNotBeNil) }) @@ -42,16 +53,23 @@ func TestCreateDynamo(t *testing.T) { func TestCreateBoltDB(t *testing.T) { Convey("Create", t, func() { rootDir := t.TempDir() - - repoDB, err := repodbfactory.Create("boltdb", bolt.DBParameters{ + params := bolt.DBParameters{ RootDir: rootDir, - }) + } + boltDriver, err := bolt.GetBoltDriver(params) + So(err, ShouldBeNil) + + log := log.NewLogger("debug", "") + + repoDB, err := repodbfactory.Create("boltdb", boltDriver, params, log) So(repoDB, ShouldNotBeNil) So(err, ShouldBeNil) }) Convey("fails", t, func() { - So(func() { _, _ = repodbfactory.Create("boltdb", dynamoParams.DBDriverParameters{}) }, ShouldPanic) + log := log.NewLogger("debug", "") + + So(func() { _, _ = repodbfactory.Create("boltdb", nil, dynamo.DBDriverParameters{}, log) }, ShouldPanic) }) } diff --git a/pkg/meta/repodb/storage_parsing_test.go b/pkg/meta/repodb/storage_parsing_test.go index bc18a6c8..a6a1b21b 100644 --- a/pkg/meta/repodb/storage_parsing_test.go +++ b/pkg/meta/repodb/storage_parsing_test.go @@ -17,10 +17,11 @@ import ( zerr "zotregistry.io/zot/errors" "zotregistry.io/zot/pkg/extensions/monitoring" "zotregistry.io/zot/pkg/log" + "zotregistry.io/zot/pkg/meta/bolt" + "zotregistry.io/zot/pkg/meta/dynamo" "zotregistry.io/zot/pkg/meta/repodb" - bolt "zotregistry.io/zot/pkg/meta/repodb/boltdb-wrapper" - dynamo "zotregistry.io/zot/pkg/meta/repodb/dynamodb-wrapper" - dynamoParams "zotregistry.io/zot/pkg/meta/repodb/dynamodb-wrapper/params" + bolt_wrapper "zotregistry.io/zot/pkg/meta/repodb/boltdb-wrapper" + dynamo_wrapper "zotregistry.io/zot/pkg/meta/repodb/dynamodb-wrapper" "zotregistry.io/zot/pkg/storage" "zotregistry.io/zot/pkg/storage/local" "zotregistry.io/zot/pkg/test" @@ -32,7 +33,7 @@ const repo = "repo" var ErrTestError = errors.New("test error") func TestParseStorageErrors(t *testing.T) { - Convey("ParseStorag", t, func() { + Convey("ParseStorage", t, func() { imageStore := mocks.MockedImageStore{ GetIndexContentFn: func(repo string) ([]byte, error) { return nil, ErrTestError @@ -71,7 +72,7 @@ func TestParseStorageErrors(t *testing.T) { }) }) - Convey("LoadRepo", t, func() { + Convey("Parse Repo", t, func() { imageStore := mocks.MockedImageStore{} storeController := storage.StoreController{DefaultStore: &imageStore} repoDB := mocks.RepoDBMock{} @@ -267,11 +268,14 @@ func TestParseStorageWithStorage(t *testing.T) { Convey("Boltdb", t, func() { rootDir := t.TempDir() - repoDB, err := bolt.NewBoltDBWrapper(bolt.DBParameters{ + boltDB, err := bolt.GetBoltDriver(bolt.DBParameters{ RootDir: rootDir, }) So(err, ShouldBeNil) + repoDB, err := bolt_wrapper.NewBoltDBWrapper(boltDB, log.NewLogger("debug", "")) + So(err, ShouldBeNil) + RunParseStorageTests(rootDir, repoDB) }) } @@ -282,7 +286,7 @@ func TestParseStorageDynamoWrapper(t *testing.T) { Convey("Dynamodb", t, func() { rootDir := t.TempDir() - dynamoWrapper, err := dynamo.NewDynamoDBWrapper(dynamoParams.DBDriverParameters{ + params := dynamo.DBDriverParameters{ Endpoint: os.Getenv("DYNAMODBMOCK_ENDPOINT"), Region: "us-east-2", RepoMetaTablename: "RepoMetadataTable", @@ -290,7 +294,12 @@ func TestParseStorageDynamoWrapper(t *testing.T) { IndexDataTablename: "IndexDataTable", ArtifactDataTablename: "ArtifactDataTable", VersionTablename: "Version", - }) + } + + dynamoClient, err := dynamo.GetDynamoClient(params) + So(err, ShouldBeNil) + + dynamoWrapper, err := dynamo_wrapper.NewDynamoDBWrapper(dynamoClient, params, log.NewLogger("debug", "")) So(err, ShouldBeNil) err = dynamoWrapper.ResetManifestDataTable() diff --git a/pkg/meta/repodb/update/update.go b/pkg/meta/update.go similarity index 98% rename from pkg/meta/repodb/update/update.go rename to pkg/meta/update.go index 4b89e5fb..376e0005 100644 --- a/pkg/meta/repodb/update/update.go +++ b/pkg/meta/update.go @@ -1,4 +1,4 @@ -package update +package meta import ( "errors" @@ -7,8 +7,8 @@ import ( zerr "zotregistry.io/zot/errors" "zotregistry.io/zot/pkg/log" + "zotregistry.io/zot/pkg/meta/common" "zotregistry.io/zot/pkg/meta/repodb" - "zotregistry.io/zot/pkg/meta/repodb/common" "zotregistry.io/zot/pkg/storage" ) diff --git a/pkg/meta/repodb/update/update_test.go b/pkg/meta/update_test.go similarity index 89% rename from pkg/meta/repodb/update/update_test.go rename to pkg/meta/update_test.go index cdbe4913..9d087d10 100644 --- a/pkg/meta/repodb/update/update_test.go +++ b/pkg/meta/update_test.go @@ -1,4 +1,4 @@ -package update_test +package meta_test import ( "encoding/json" @@ -14,9 +14,10 @@ import ( zerr "zotregistry.io/zot/errors" "zotregistry.io/zot/pkg/extensions/monitoring" "zotregistry.io/zot/pkg/log" + "zotregistry.io/zot/pkg/meta" + "zotregistry.io/zot/pkg/meta/bolt" "zotregistry.io/zot/pkg/meta/repodb" bolt_wrapper "zotregistry.io/zot/pkg/meta/repodb/boltdb-wrapper" - repoDBUpdate "zotregistry.io/zot/pkg/meta/repodb/update" "zotregistry.io/zot/pkg/storage" "zotregistry.io/zot/pkg/storage/local" "zotregistry.io/zot/pkg/test" @@ -35,9 +36,13 @@ func TestOnUpdateManifest(t *testing.T) { true, true, log, metrics, nil, nil, ) - repoDB, err := bolt_wrapper.NewBoltDBWrapper(bolt_wrapper.DBParameters{ + params := bolt.DBParameters{ RootDir: rootDir, - }) + } + boltDriver, err := bolt.GetBoltDriver(params) + So(err, ShouldBeNil) + + repoDB, err := bolt_wrapper.NewBoltDBWrapper(boltDriver, log) So(err, ShouldBeNil) config, layers, manifest, err := test.GetRandomImageComponents(100) @@ -56,7 +61,7 @@ func TestOnUpdateManifest(t *testing.T) { digest := godigest.FromBytes(manifestBlob) - err = repoDBUpdate.OnUpdateManifest("repo", "tag1", "", digest, manifestBlob, storeController, repoDB, log) + err = meta.OnUpdateManifest("repo", "tag1", "", digest, manifestBlob, storeController, repoDB, log) So(err, ShouldBeNil) repoMeta, err := repoDB.GetRepoMeta("repo") @@ -80,7 +85,7 @@ func TestOnUpdateManifest(t *testing.T) { }, } - err := repoDBUpdate.OnUpdateManifest("repo", "tag1", ispec.MediaTypeImageManifest, "digest", + err := meta.OnUpdateManifest("repo", "tag1", ispec.MediaTypeImageManifest, "digest", []byte("{}"), storeController, repoDB, log) So(err, ShouldNotBeNil) }) @@ -108,7 +113,7 @@ func TestUpdateErrors(t *testing.T) { return []byte{}, "", "", zerr.ErrManifestNotFound } - err = repoDBUpdate.OnUpdateManifest("repo", "tag1", "", "digest", manifestBlob, + err = meta.OnUpdateManifest("repo", "tag1", "", "digest", manifestBlob, storeController, repoDB, log) So(err, ShouldNotBeNil) }) @@ -134,7 +139,7 @@ func TestUpdateErrors(t *testing.T) { return []byte{}, "", "", zerr.ErrManifestNotFound } - err = repoDBUpdate.OnDeleteManifest("repo", "tag1", "digest", "media", manifestBlob, + err = meta.OnDeleteManifest("repo", "tag1", "digest", "media", manifestBlob, storeController, repoDB, log) So(err, ShouldNotBeNil) @@ -142,7 +147,7 @@ func TestUpdateErrors(t *testing.T) { return []byte{}, "", "", ErrTestError } - err = repoDBUpdate.OnDeleteManifest("repo", "tag1", "digest", "media", manifestBlob, + err = meta.OnDeleteManifest("repo", "tag1", "digest", "media", manifestBlob, storeController, repoDB, log) So(err, ShouldNotBeNil) @@ -150,7 +155,7 @@ func TestUpdateErrors(t *testing.T) { return []byte{}, "", "", zerr.ErrManifestNotFound } - err = repoDBUpdate.OnDeleteManifest("repo", "tag1", "digest", "media", manifestBlob, + err = meta.OnDeleteManifest("repo", "tag1", "digest", "media", manifestBlob, storeController, repoDB, log) So(err, ShouldNotBeNil) }) @@ -160,7 +165,7 @@ func TestUpdateErrors(t *testing.T) { return ErrTestError } - err := repoDBUpdate.OnDeleteManifest("repo", "tag1", "digest", "media", + err := meta.OnDeleteManifest("repo", "tag1", "digest", "media", []byte(`{"subject": {"digest": "dig"}}`), storeController, repoDB, log) So(err, ShouldNotBeNil) @@ -187,7 +192,7 @@ func TestUpdateErrors(t *testing.T) { return []byte{}, "", "", zerr.ErrManifestNotFound } - err = repoDBUpdate.OnGetManifest("repo", "tag1", "digest", manifestBlob, + err = meta.OnGetManifest("repo", "tag1", "digest", manifestBlob, storeController, repoDB, log) So(err, ShouldNotBeNil) @@ -195,7 +200,7 @@ func TestUpdateErrors(t *testing.T) { return []byte{}, "", "", ErrTestError } - err = repoDBUpdate.OnGetManifest("repo", "tag1", "media", manifestBlob, + err = meta.OnGetManifest("repo", "tag1", "media", manifestBlob, storeController, repoDB, log) So(err, ShouldNotBeNil) }) diff --git a/pkg/meta/repodb/version/common.go b/pkg/meta/version/common.go similarity index 100% rename from pkg/meta/repodb/version/common.go rename to pkg/meta/version/common.go diff --git a/pkg/meta/repodb/version/patches.go b/pkg/meta/version/patches.go similarity index 100% rename from pkg/meta/repodb/version/patches.go rename to pkg/meta/version/patches.go diff --git a/pkg/meta/repodb/version/version_test.go b/pkg/meta/version/version_test.go similarity index 84% rename from pkg/meta/repodb/version/version_test.go rename to pkg/meta/version/version_test.go index 09bb1f5b..ad961b92 100644 --- a/pkg/meta/repodb/version/version_test.go +++ b/pkg/meta/version/version_test.go @@ -13,11 +13,12 @@ import ( . "github.com/smartystreets/goconvey/convey" "go.etcd.io/bbolt" - "zotregistry.io/zot/pkg/meta/repodb" - bolt "zotregistry.io/zot/pkg/meta/repodb/boltdb-wrapper" - dynamo "zotregistry.io/zot/pkg/meta/repodb/dynamodb-wrapper" - dynamoParams "zotregistry.io/zot/pkg/meta/repodb/dynamodb-wrapper/params" - "zotregistry.io/zot/pkg/meta/repodb/version" + "zotregistry.io/zot/pkg/log" + "zotregistry.io/zot/pkg/meta/bolt" + "zotregistry.io/zot/pkg/meta/dynamo" + boltdb_wrapper "zotregistry.io/zot/pkg/meta/repodb/boltdb-wrapper" + dynamodb_wrapper "zotregistry.io/zot/pkg/meta/repodb/dynamodb-wrapper" + "zotregistry.io/zot/pkg/meta/version" ) var ErrTestError = errors.New("test error") @@ -26,7 +27,12 @@ func TestVersioningBoltDB(t *testing.T) { Convey("Tests", t, func() { tmpDir := t.TempDir() boltDBParams := bolt.DBParameters{RootDir: tmpDir} - boltdbWrapper, err := bolt.NewBoltDBWrapper(boltDBParams) + boltDriver, err := bolt.GetBoltDriver(boltDBParams) + So(err, ShouldBeNil) + + log := log.NewLogger("debug", "") + + boltdbWrapper, err := boltdb_wrapper.NewBoltDBWrapper(boltDriver, log) defer os.Remove("repo.db") So(boltdbWrapper, ShouldNotBeNil) So(err, ShouldBeNil) @@ -53,7 +59,7 @@ func TestVersioningBoltDB(t *testing.T) { Convey("DBVersion is empty", func() { err := boltdbWrapper.DB.Update(func(tx *bbolt.Tx) error { - versionBuck := tx.Bucket([]byte(repodb.VersionBucket)) + versionBuck := tx.Bucket([]byte(bolt.VersionBucket)) return versionBuck.Put([]byte(version.DBVersionKey), []byte("")) }) @@ -99,7 +105,7 @@ func TestVersioningBoltDB(t *testing.T) { func setBoltDBVersion(db *bbolt.DB, vers string) error { err := db.Update(func(tx *bbolt.Tx) error { - versionBuck := tx.Bucket([]byte(repodb.VersionBucket)) + versionBuck := tx.Bucket([]byte(bolt.VersionBucket)) return versionBuck.Put([]byte(version.DBVersionKey), []byte(vers)) }) @@ -114,7 +120,7 @@ func TestVersioningDynamoDB(t *testing.T) { ) Convey("Tests", t, func() { - dynamoWrapper, err := dynamo.NewDynamoDBWrapper(dynamoParams.DBDriverParameters{ + params := dynamo.DBDriverParameters{ Endpoint: endpoint, Region: region, RepoMetaTablename: "RepoMetadataTable", @@ -122,13 +128,20 @@ func TestVersioningDynamoDB(t *testing.T) { ArtifactDataTablename: "ArtifactDataTable", IndexDataTablename: "IndexDataTable", VersionTablename: "Version", - }) + } + + dynamoClient, err := dynamo.GetDynamoClient(params) + So(err, ShouldBeNil) + + log := log.NewLogger("debug", "") + + dynamoWrapper, err := dynamodb_wrapper.NewDynamoDBWrapper(dynamoClient, params, log) So(err, ShouldBeNil) So(dynamoWrapper.ResetManifestDataTable(), ShouldBeNil) So(dynamoWrapper.ResetRepoMetaTable(), ShouldBeNil) - Convey("DBVersion is empty", func() { + Convey("dbVersion is empty", func() { err := setDynamoDBVersion(dynamoWrapper.Client, "") So(err, ShouldBeNil)