github/zot
1
0
Fork 0
mirror of https://github.com/project-zot/zot.git synced 2026-06-16 04:17:55 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(refator): refactoring repodb into meta (#1626)

Browse Source 
Signed-off-by: Laurentiu Niculae <niculae.laurentiu1@gmail.com>
This commit is contained in:
LaurentiuNiculae
2023-07-18 20:27:26 +03:00
committed by GitHub
parent fe9c9750b5
commit 28de980319
65 changed files with 3267 additions and 3243 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/api/authn.go
+7 -7
View File
@@ -93,7 +93,7 @@ func (amw *AuthnMiddleware) sessionAuthn(ctlr *Controller, next http.Handler, re
ctx := getReqContextWithAuthorization(identity, []string{}, request)
groups, err := ctlr.RepoDB.GetUserGroups(ctx)
groups, err := ctlr.MetaDB.GetUserGroups(ctx)
if err != nil {
if errors.Is(err, zerr.ErrUserDataNotFound) {
ctlr.Log.Err(err).Str("identity", identity).Msg("can not find user profile in DB")
@@ -168,7 +168,7 @@ func (amw *AuthnMiddleware) basicAuthn(ctlr *Controller, response http.ResponseW
return false, response, request, err
}
if err := ctlr.RepoDB.SetUserGroups(ctx, groups); err != nil {
if err := ctlr.MetaDB.SetUserGroups(ctx, groups); err != nil {
ctlr.Log.Error().Err(err).Str("identity", identity).Msg("couldn't update user profile")
return false, response, request, err
@@ -200,7 +200,7 @@ func (amw *AuthnMiddleware) basicAuthn(ctlr *Controller, response http.ResponseW
return false, response, request, err
}
if err := ctlr.RepoDB.SetUserGroups(ctx, groups); err != nil {
if err := ctlr.MetaDB.SetUserGroups(ctx, groups); err != nil {
ctlr.Log.Error().Err(err).Str("identity", identity).Msg("couldn't update user profile")
return false, response, request, err
@@ -224,7 +224,7 @@ func (amw *AuthnMiddleware) basicAuthn(ctlr *Controller, response http.ResponseW
hashedKey := hashUUID(trimmedAPIKey)
storedIdentity, err := ctlr.RepoDB.GetUserAPIKeyInfo(hashedKey)
storedIdentity, err := ctlr.MetaDB.GetUserAPIKeyInfo(hashedKey)
if err != nil {
if errors.Is(err, zerr.ErrUserAPIKeyNotFound) {
ctlr.Log.Info().Err(err).Msgf("can not find any user info for hashed key %s in DB", hashedKey)
@@ -240,14 +240,14 @@ func (amw *AuthnMiddleware) basicAuthn(ctlr *Controller, response http.ResponseW
if storedIdentity == identity {
ctx := getReqContextWithAuthorization(identity, []string{}, request)
err := ctlr.RepoDB.UpdateUserAPIKeyLastUsed(ctx, hashedKey)
err := ctlr.MetaDB.UpdateUserAPIKeyLastUsed(ctx, hashedKey)
if err != nil {
ctlr.Log.Err(err).Str("identity", identity).Msg("can not update user profile in DB")
return false, nil, nil, err
}
groups, err := ctlr.RepoDB.GetUserGroups(ctx)
groups, err := ctlr.MetaDB.GetUserGroups(ctx)
if err != nil {
ctlr.Log.Err(err).Str("identity", identity).Msg("can not get user's groups in DB")
@@ -843,7 +843,7 @@ func OAuth2Callback(ctlr *Controller, w http.ResponseWriter, r *http.Request, st
return "", err
}
if err := ctlr.RepoDB.SetUserGroups(ctx, groups); err != nil {
if err := ctlr.MetaDB.SetUserGroups(ctx, groups); err != nil {
ctlr.Log.Error().Err(err).Str("identity", email).Msg("couldn't update the user profile")
return "", err
pkg/api/controller.go
+13 -13
View File
@@ -24,8 +24,8 @@ import (
ext "zotregistry.io/zot/pkg/extensions"
"zotregistry.io/zot/pkg/extensions/monitoring"
"zotregistry.io/zot/pkg/log"
"zotregistry.io/zot/pkg/meta/repodb"
"zotregistry.io/zot/pkg/meta/repodb/repodbfactory"
"zotregistry.io/zot/pkg/meta"
mTypes "zotregistry.io/zot/pkg/meta/types"
"zotregistry.io/zot/pkg/scheduler"
"zotregistry.io/zot/pkg/storage"
)
@@ -39,7 +39,7 @@ const (
type Controller struct {
Config *config.Config
Router *mux.Router
RepoDB repodb.RepoDB
MetaDB mTypes.MetaDB
StoreController storage.StoreController
Log log.Logger
Audit *log.Logger
@@ -229,7 +229,7 @@ func (c *Controller) Init(reloadCtx context.Context) error {
return err
}
if err := c.InitRepoDB(reloadCtx); err != nil {
if err := c.InitMetaDB(reloadCtx); err != nil {
return err
}
@@ -241,7 +241,7 @@ func (c *Controller) Init(reloadCtx context.Context) error {
func (c *Controller) InitCVEInfo() {
// Enable CVE extension if extension config is provided
if c.Config != nil && c.Config.Extensions != nil {
c.CveInfo = ext.GetCVEInfo(c.Config, c.StoreController, c.RepoDB, c.Log)
c.CveInfo = ext.GetCVEInfo(c.Config, c.StoreController, c.MetaDB, c.Log)
}
}
@@ -258,11 +258,11 @@ func (c *Controller) InitImageStore() error {
return nil
}
func (c *Controller) InitRepoDB(reloadCtx context.Context) error {
// init repoDB if search is enabled or authn enabled (need to store user profiles) or apikey ext is enabled
func (c *Controller) InitMetaDB(reloadCtx context.Context) error {
// init metaDB if search is enabled or authn enabled (need to store user profiles) or apikey ext is enabled
if (c.Config.Extensions != nil && c.Config.Extensions.Search != nil && *c.Config.Extensions.Search.Enable) ||
isAuthnEnabled(c.Config) || isOpenIDAuthEnabled(c.Config) || isAPIKeyEnabled(c.Config) {
driver, err := repodbfactory.New(c.Config.Storage.StorageConfig, c.Log) //nolint:contextcheck
driver, err := meta.New(c.Config.Storage.StorageConfig, c.Log) //nolint:contextcheck
if err != nil {
return err
}
@@ -272,12 +272,12 @@ func (c *Controller) InitRepoDB(reloadCtx context.Context) error {
return err
}
err = repodb.ParseStorage(driver, c.StoreController, c.Log)
err = meta.ParseStorage(driver, c.StoreController, c.Log)
if err != nil {
return err
}
c.RepoDB = driver
c.MetaDB = driver
}
return nil
@@ -335,7 +335,7 @@ func (c *Controller) StartBackgroundTasks(reloadCtx context.Context) {
// Enable extensions if extension config is provided for DefaultStore
if c.Config != nil && c.Config.Extensions != nil {
ext.EnableMetricsExtension(c.Config, c.Log, c.Config.Storage.RootDirectory)
ext.EnableSearchExtension(c.Config, c.StoreController, c.RepoDB, taskScheduler, c.CveInfo, c.Log)
ext.EnableSearchExtension(c.Config, c.StoreController, c.MetaDB, taskScheduler, c.CveInfo, c.Log)
}
if c.Config.Storage.SubPaths != nil {
@@ -361,7 +361,7 @@ func (c *Controller) StartBackgroundTasks(reloadCtx context.Context) {
if c.Config.Extensions != nil {
ext.EnableScrubExtension(c.Config, c.Log, c.StoreController, taskScheduler)
syncOnDemand, err := ext.EnableSyncExtension(c.Config, c.RepoDB, c.StoreController, taskScheduler, c.Log)
syncOnDemand, err := ext.EnableSyncExtension(c.Config, c.MetaDB, c.StoreController, taskScheduler, c.Log)
if err != nil {
c.Log.Error().Err(err).Msg("unable to start sync extension")
}
@@ -371,7 +371,7 @@ func (c *Controller) StartBackgroundTasks(reloadCtx context.Context) {
if c.Config.Extensions != nil {
if c.Config.Extensions.Mgmt != nil && *c.Config.Extensions.Mgmt.Enable {
ext.EnablePeriodicSignaturesVerification(c.Config, taskScheduler, c.RepoDB, c.Log) //nolint: contextcheck
ext.EnablePeriodicSignaturesVerification(c.Config, taskScheduler, c.MetaDB, c.Log) //nolint: contextcheck
}
}
}
pkg/api/controller_test.go
+18 -18
View File
@@ -56,7 +56,7 @@ import (
"zotregistry.io/zot/pkg/common"
extconf "zotregistry.io/zot/pkg/extensions/config"
"zotregistry.io/zot/pkg/log"
"zotregistry.io/zot/pkg/meta/repodb/repodbfactory"
"zotregistry.io/zot/pkg/meta"
"zotregistry.io/zot/pkg/storage"
storageConstants "zotregistry.io/zot/pkg/storage/constants"
"zotregistry.io/zot/pkg/test"
@@ -203,7 +203,7 @@ func TestCreateCacheDatabaseDriver(t *testing.T) {
})
}
func TestCreateRepoDBDriver(t *testing.T) {
func TestCreateMetaDBDriver(t *testing.T) {
Convey("Test CreateCacheDatabaseDriver dynamo", t, func() {
log := log.NewLogger("debug", "")
dir := t.TempDir()
@@ -230,7 +230,7 @@ func TestCreateRepoDBDriver(t *testing.T) {
"userdatatablename": "UserDatatable",
}
testFunc := func() { _, _ = repodbfactory.New(conf.Storage.StorageConfig, log) }
testFunc := func() { _, _ = meta.New(conf.Storage.StorageConfig, log) }
So(testFunc, ShouldPanic)
conf.Storage.CacheDriver = map[string]interface{}{
@@ -244,7 +244,7 @@ func TestCreateRepoDBDriver(t *testing.T) {
"versiontablename": 1,
}
testFunc = func() { _, _ = repodbfactory.New(conf.Storage.StorageConfig, log) }
testFunc = func() { _, _ = meta.New(conf.Storage.StorageConfig, log) }
So(testFunc, ShouldPanic)
conf.Storage.CacheDriver = map[string]interface{}{
@@ -260,7 +260,7 @@ func TestCreateRepoDBDriver(t *testing.T) {
"versiontablename": "1",
}
testFunc = func() { _, _ = repodbfactory.New(conf.Storage.StorageConfig, log) }
testFunc = func() { _, _ = meta.New(conf.Storage.StorageConfig, log) }
So(testFunc, ShouldNotPanic)
})
@@ -283,7 +283,7 @@ func TestCreateRepoDBDriver(t *testing.T) {
err = os.Chmod(path.Join(dir, "repo.db"), 0o200)
So(err, ShouldBeNil)
_, err = repodbfactory.New(conf.Storage.StorageConfig, log)
_, err = meta.New(conf.Storage.StorageConfig, log)
So(err, ShouldNotBeNil)
err = os.Chmod(path.Join(dir, "repo.db"), 0o600)
@@ -3103,7 +3103,7 @@ func TestAuthnSessionErrors(t *testing.T) {
Convey("trigger basic authn middle(htpasswd) error", func() {
client := resty.New()
ctlr.RepoDB = mocks.RepoDBMock{
ctlr.MetaDB = mocks.MetaDBMock{
SetUserGroupsFn: func(ctx context.Context, groups []string) error {
return ErrUnexpectedError
},
@@ -3120,7 +3120,7 @@ func TestAuthnSessionErrors(t *testing.T) {
Convey("trigger basic authn middle(ldap) error", func() {
client := resty.New()
ctlr.RepoDB = mocks.RepoDBMock{
ctlr.MetaDB = mocks.MetaDBMock{
SetUserGroupsFn: func(ctx context.Context, groups []string) error {
return ErrUnexpectedError
},
@@ -3138,7 +3138,7 @@ func TestAuthnSessionErrors(t *testing.T) {
client := resty.New()
client.SetRedirectPolicy(test.CustomRedirectPolicy(20))
ctlr.RepoDB = mocks.RepoDBMock{
ctlr.MetaDB = mocks.MetaDBMock{
SetUserGroupsFn: func(ctx context.Context, groups []string) error {
return ErrUnexpectedError
},
@@ -3153,7 +3153,7 @@ func TestAuthnSessionErrors(t *testing.T) {
So(resp.StatusCode(), ShouldEqual, http.StatusInternalServerError)
})
Convey("trigger session middle repoDB errors", func() {
Convey("trigger session middle metaDB errors", func() {
client := resty.New()
client.SetRedirectPolicy(test.CustomRedirectPolicy(20))
@@ -3162,7 +3162,7 @@ func TestAuthnSessionErrors(t *testing.T) {
mockOIDCServer.QueueUser(user)
ctlr.RepoDB = mocks.RepoDBMock{}
ctlr.MetaDB = mocks.MetaDBMock{}
// first login user
resp, err := client.R().
@@ -3178,7 +3178,7 @@ func TestAuthnSessionErrors(t *testing.T) {
client.SetCookies(cookies)
ctlr.RepoDB = mocks.RepoDBMock{
ctlr.MetaDB = mocks.MetaDBMock{
GetUserGroupsFn: func(ctx context.Context) ([]string, error) {
return []string{}, ErrUnexpectedError
},
@@ -3198,7 +3198,7 @@ func TestAuthnSessionErrors(t *testing.T) {
client.SetCookies(cookies)
ctlr.RepoDB = mocks.RepoDBMock{
ctlr.MetaDB = mocks.MetaDBMock{
GetUserGroupsFn: func(ctx context.Context) ([]string, error) {
return []string{}, errors.ErrUserDataNotFound
},
@@ -3417,7 +3417,7 @@ func TestAuthnSessionErrors(t *testing.T) {
})
}
func TestAuthnRepoDBErrors(t *testing.T) {
func TestAuthnMetaDBErrors(t *testing.T) {
Convey("make controller", t, func() {
port := test.GetFreePort()
baseURL := test.GetBaseURL(port)
@@ -3472,7 +3472,7 @@ func TestAuthnRepoDBErrors(t *testing.T) {
Convey("trigger basic authn middle(htpasswd) error", func() {
client := resty.New()
ctlr.RepoDB = mocks.RepoDBMock{
ctlr.MetaDB = mocks.MetaDBMock{
SetUserGroupsFn: func(ctx context.Context, groups []string) error {
return ErrUnexpectedError
},
@@ -3486,7 +3486,7 @@ func TestAuthnRepoDBErrors(t *testing.T) {
So(resp.StatusCode(), ShouldEqual, http.StatusInternalServerError)
})
Convey("trigger session middle repoDB errors", func() {
Convey("trigger session middle metaDB errors", func() {
client := resty.New()
client.SetRedirectPolicy(test.CustomRedirectPolicy(20))
@@ -3509,7 +3509,7 @@ func TestAuthnRepoDBErrors(t *testing.T) {
client.SetCookies(cookies)
ctlr.RepoDB = mocks.RepoDBMock{
ctlr.MetaDB = mocks.MetaDBMock{
GetUserGroupsFn: func(ctx context.Context) ([]string, error) {
return []string{}, ErrUnexpectedError
},
@@ -4727,7 +4727,7 @@ func TestCrossRepoMount(t *testing.T) {
// in cache, now try mount blob request status and it should be 201 because now blob is present in cache
// and it should do hard link.
// make a new server with dedupe on and same rootDir (can't restart because of repodb - boltdb being open)
// make a new server with dedupe on and same rootDir (can't restart because of metadb - boltdb being open)
newDir := t.TempDir()
err = test.CopyFiles(dir, newDir)
So(err, ShouldBeNil)
pkg/api/routes.go
+9 -9
View File
@@ -157,11 +157,11 @@ func (rh *RouteHandler) SetupRoutes() {
prefixedExtensionsRouter.Use(CORSHeadersMiddleware(rh.c.Config.HTTP.AllowOrigin))
ext.SetupMgmtRoutes(rh.c.Config, prefixedExtensionsRouter, rh.c.Log)
ext.SetupSearchRoutes(rh.c.Config, prefixedExtensionsRouter, rh.c.StoreController, rh.c.RepoDB, rh.c.CveInfo,
ext.SetupSearchRoutes(rh.c.Config, prefixedExtensionsRouter, rh.c.StoreController, rh.c.MetaDB, rh.c.CveInfo,
rh.c.Log)
ext.SetupUserPreferencesRoutes(rh.c.Config, prefixedExtensionsRouter, rh.c.StoreController, rh.c.RepoDB,
ext.SetupUserPreferencesRoutes(rh.c.Config, prefixedExtensionsRouter, rh.c.StoreController, rh.c.MetaDB,
rh.c.CveInfo, rh.c.Log)
ext.SetupAPIKeyRoutes(rh.c.Config, prefixedExtensionsRouter, rh.c.RepoDB, rh.c.CookieStore, rh.c.Log)
ext.SetupAPIKeyRoutes(rh.c.Config, prefixedExtensionsRouter, rh.c.MetaDB, rh.c.CookieStore, rh.c.Log)
ext.SetupMetricsRoutes(rh.c.Config, rh.c.Router, rh.c.StoreController, authHandler, rh.c.Log)
gqlPlayground.SetupGQLPlaygroundRoutes(rh.c.Config, prefixedRouter, rh.c.StoreController, rh.c.Log)
@@ -505,8 +505,8 @@ func (rh *RouteHandler) GetManifest(response http.ResponseWriter, request *http.
return
}
if rh.c.RepoDB != nil {
err := meta.OnGetManifest(name, reference, content, rh.c.StoreController, rh.c.RepoDB, rh.c.Log)
if rh.c.MetaDB != nil {
err := meta.OnGetManifest(name, reference, content, rh.c.StoreController, rh.c.MetaDB, rh.c.Log)
if err != nil {
response.WriteHeader(http.StatusInternalServerError)
@@ -712,8 +712,8 @@ func (rh *RouteHandler) UpdateManifest(response http.ResponseWriter, request *ht
return
}
if rh.c.RepoDB != nil {
err := meta.OnUpdateManifest(name, reference, mediaType, digest, body, rh.c.StoreController, rh.c.RepoDB,
if rh.c.MetaDB != nil {
err := meta.OnUpdateManifest(name, reference, mediaType, digest, body, rh.c.StoreController, rh.c.MetaDB,
rh.c.Log)
if err != nil {
response.WriteHeader(http.StatusInternalServerError)
@@ -813,9 +813,9 @@ func (rh *RouteHandler) DeleteManifest(response http.ResponseWriter, request *ht
return
}
if rh.c.RepoDB != nil {
if rh.c.MetaDB != nil {
err := meta.OnDeleteManifest(name, reference, mediaType, manifestDigest, manifestBlob,
rh.c.StoreController, rh.c.RepoDB, rh.c.Log)
rh.c.StoreController, rh.c.MetaDB, rh.c.Log)
if err != nil {
response.WriteHeader(http.StatusInternalServerError)
pkg/api/routes_test.go
+6 -6
View File
@@ -30,7 +30,7 @@ import (
"zotregistry.io/zot/pkg/api/constants"
"zotregistry.io/zot/pkg/extensions"
extconf "zotregistry.io/zot/pkg/extensions/config"
"zotregistry.io/zot/pkg/meta/repodb"
mTypes "zotregistry.io/zot/pkg/meta/types"
localCtx "zotregistry.io/zot/pkg/requestcontext"
storageTypes "zotregistry.io/zot/pkg/storage/types"
"zotregistry.io/zot/pkg/test"
@@ -1434,7 +1434,7 @@ func TestRoutes(t *testing.T) {
request, _ := http.NewRequestWithContext(ctx, http.MethodPost, baseURL, bytes.NewReader([]byte{}))
response := httptest.NewRecorder()
extensions.CreateAPIKey(response, request, ctlr.RepoDB, ctlr.CookieStore, ctlr.Log)
extensions.CreateAPIKey(response, request, ctlr.MetaDB, ctlr.CookieStore, ctlr.Log)
resp := response.Result()
defer resp.Body.Close()
@@ -1451,7 +1451,7 @@ func TestRoutes(t *testing.T) {
request, _ = http.NewRequestWithContext(ctx, http.MethodPost, baseURL, bytes.NewReader([]byte{}))
response = httptest.NewRecorder()
extensions.CreateAPIKey(response, request, ctlr.RepoDB, ctlr.CookieStore, ctlr.Log)
extensions.CreateAPIKey(response, request, ctlr.MetaDB, ctlr.CookieStore, ctlr.Log)
resp = response.Result()
defer resp.Body.Close()
@@ -1468,8 +1468,8 @@ func TestRoutes(t *testing.T) {
request, _ = http.NewRequestWithContext(ctx, http.MethodPost, baseURL, bytes.NewReader(reqBody))
response = httptest.NewRecorder()
extensions.CreateAPIKey(response, request, mocks.RepoDBMock{
AddUserAPIKeyFn: func(ctx context.Context, hashedKey string, apiKeyDetails *repodb.APIKeyDetails) error {
extensions.CreateAPIKey(response, request, mocks.MetaDBMock{
AddUserAPIKeyFn: func(ctx context.Context, hashedKey string, apiKeyDetails *mTypes.APIKeyDetails) error {
return ErrUnexpectedError
},
}, ctlr.CookieStore, ctlr.Log)
@@ -1486,7 +1486,7 @@ func TestRoutes(t *testing.T) {
q.Add("id", "apikeyid")
request.URL.RawQuery = q.Encode()
extensions.RevokeAPIKey(response, request, mocks.RepoDBMock{
extensions.RevokeAPIKey(response, request, mocks.MetaDBMock{
DeleteUserAPIKeyFn: func(ctx context.Context, id string) error {
return ErrUnexpectedError
},