github/zot
1
0
Fork 0
mirror of https://github.com/project-zot/zot.git synced 2026-06-17 21:17:58 +08:00
Code Issues Packages Projects Releases Wiki Activity

GCS storage support (#3798)

Browse Source 
feat(storage): add a GCS driver

test(storage): add unit tests for GCS driver

test(storage): add missing unit tests for GCS driver & resolve lint issues

fix: configuration validation for GCS Storage

test(storage): resolve panic by test due to setupGCS ignoring returned error

test(storage): add dummy gcs credentials

test: add darwin support for macos to run tests

ci: update workflows to pin gcs emulator version

lint: resolve long line lengths & formatting issues

test: move error for gcs mock earlier with an error

test: stop test using local google credentials and use mock instead

test: add missing dummy creds

test(storage): use storage-testbench for GCS, isolate GCS tests, fix driver Delete

- Switch GCS emulator from fake-gcs-server to storage-testbench in CI.
  Run the GCS emulator only in the privileged-test job; remove it from
  minimal and extended test jobs.

- Consolidate GCS tests under pkg/storage/gcs (needprivileges,linux).
  Add TestMain with HTTPS proxy and /etc/hosts so tests talk to
  storage-testbench; move GCS-specific cases from storage_test.go and
  scrub_test.go into gcs_test.go. Run GCS tests via a second privileged-test
  invocation and collect coverage in coverage-needprivileges-gcs.txt.

- Make GCS driver Delete idempotent and normalize errors. Treat
  PathNotFoundError from Delete as success so that deleting an already-gone
  path (e.g. after GC under eventual consistency) does not fail. Add
  formatErr to map 404/not found to PathNotFoundError and use it for all
  driver methods so callers get consistent storage driver errors.

- Drop GCS branches and helpers from storage_test.go and scrub_test.go so
  non-privileged tests only use local/S3; GCS is tested only in
  pkg/storage/gcs with storage-testbench.

- Set GCSMOCK_ENDPOINT without /storage/v1/, as the rest of the URL is set in tests.

- Show errors in case of failure to create bucket.

- Consolidate StorageDriverMock structs inside the pkg/test/mocks package.

Signed-off-by: Andrei Aaron <andreifdaaron@gmail.com>
Co-authored-by: Steven Marks <steve.marks@qomodo.io>
This commit is contained in:
Andrei Aaron
2026-02-19 09:41:21 +02:00
committed by GitHub
parent 5b2312d538
commit 5e57656bff
19 changed files with 4634 additions and 401 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/cli/server/root.go
+57 -45
View File
@@ -299,6 +299,10 @@ func getStorageType(storageDriver map[string]any) string {
return storageConstants.S3StorageDriverName
}
if storeName == storageConstants.GCSStorageDriverName {
return storageConstants.GCSStorageDriverName
}
return storeName
}
@@ -559,6 +563,57 @@ func validateExtensionsConfig(cfg *config.Config, logger zlog.Logger) error {
return nil
}
func validateStorageConfigSection(
cfg *config.Config, logger zlog.Logger, storageConfig config.GlobalStorageConfig,
) error {
if len(storageConfig.StorageDriver) != 0 {
// enforce s3/gcs driver in case of using storage driver
if storageConfig.StorageDriver["name"] != storageConstants.S3StorageDriverName &&
storageConfig.StorageDriver["name"] != storageConstants.GCSStorageDriverName {
msg := "unsupported storage driver"
logger.Error().Err(zerr.ErrBadConfig).Interface("storageDriver", storageConfig.StorageDriver["name"]).Msg(msg)
return fmt.Errorf("%w: %s", zerr.ErrBadConfig, msg)
}
// enforce tmpDir in case sync + s3/gcs
extensionsConfig := cfg.CopyExtensionsConfig()
if extensionsConfig.IsSyncEnabled() && extensionsConfig.Sync.DownloadDir == "" {
msg := "using both sync and remote storage features needs config.Extensions.Sync.DownloadDir to be specified"
logger.Error().Err(zerr.ErrBadConfig).Msg(msg)
return fmt.Errorf("%w: %s", zerr.ErrBadConfig, msg)
}
}
// enforce s3/gcs driver on subpaths in case of using storage driver
if len(storageConfig.SubPaths) > 0 {
for route, subStorageConfig := range storageConfig.SubPaths {
if len(subStorageConfig.StorageDriver) != 0 {
if subStorageConfig.StorageDriver["name"] != storageConstants.S3StorageDriverName &&
subStorageConfig.StorageDriver["name"] != storageConstants.GCSStorageDriverName {
msg := "unsupported storage driver"
logger.Error().Err(zerr.ErrBadConfig).Str("subpath", route).Interface("storageDriver",
subStorageConfig.StorageDriver["name"]).Msg(msg)
return fmt.Errorf("%w: %s", zerr.ErrBadConfig, msg)
}
// enforce tmpDir in case sync + s3/gcs
extensionsConfig := cfg.CopyExtensionsConfig()
if extensionsConfig.IsSyncEnabled() && extensionsConfig.Sync.DownloadDir == "" {
msg := "using both sync and remote storage features needs config.Extensions.Sync.DownloadDir to be specified"
logger.Error().Err(zerr.ErrBadConfig).Msg(msg)
return fmt.Errorf("%w: %s", zerr.ErrBadConfig, msg)
}
}
}
}
return nil
}
func validateConfiguration(config *config.Config, logger zlog.Logger) error {
if err := validateHTTP(config, logger); err != nil {
return err
@@ -614,51 +669,8 @@ func validateConfiguration(config *config.Config, logger zlog.Logger) error {
}
storageConfig := config.CopyStorageConfig()
if len(storageConfig.StorageDriver) != 0 {
// enforce s3 driver in case of using storage driver
if storageConfig.StorageDriver["name"] != storageConstants.S3StorageDriverName {
msg := "unsupported storage driver"
logger.Error().Err(zerr.ErrBadConfig).Interface("cacheDriver", storageConfig.StorageDriver["name"]).Msg(msg)
return fmt.Errorf("%w: %s", zerr.ErrBadConfig, msg)
}
// enforce tmpDir in case sync + s3
extensionsConfig := config.CopyExtensionsConfig()
if extensionsConfig.IsSyncEnabled() && extensionsConfig.Sync.DownloadDir == "" {
msg := "using both sync and remote storage features needs config.Extensions.Sync.DownloadDir to be specified"
logger.Error().Err(zerr.ErrBadConfig).Msg(msg)
return fmt.Errorf("%w: %s", zerr.ErrBadConfig, msg)
}
}
// enforce s3 driver on subpaths in case of using storage driver
if storageConfig.SubPaths != nil {
if len(storageConfig.SubPaths) > 0 {
subPaths := storageConfig.SubPaths
for route, subStorageConfig := range subPaths {
if len(subStorageConfig.StorageDriver) != 0 {
if subStorageConfig.StorageDriver["name"] != storageConstants.S3StorageDriverName {
msg := "unsupported storage driver"
logger.Error().Err(zerr.ErrBadConfig).Str("subpath", route).Interface("storageDriver",
subStorageConfig.StorageDriver["name"]).Msg(msg)
return fmt.Errorf("%w: %s", zerr.ErrBadConfig, msg)
}
// enforce tmpDir in case sync + s3
extensionsConfig := config.CopyExtensionsConfig()
if extensionsConfig.IsSyncEnabled() && extensionsConfig.Sync.DownloadDir == "" {
msg := "using both sync and remote storage features needs config.Extensions.Sync.DownloadDir to be specified"
logger.Error().Err(zerr.ErrBadConfig).Msg(msg)
return fmt.Errorf("%w: %s", zerr.ErrBadConfig, msg)
}
}
}
}
if err := validateStorageConfigSection(config, logger, storageConfig); err != nil {
return err
}
// check glob patterns in authz config are compilable