mirror of
https://github.com/project-zot/zot.git
synced 2026-06-18 13:37:57 +08:00
5e57656bff
feat(storage): add a GCS driver test(storage): add unit tests for GCS driver test(storage): add missing unit tests for GCS driver & resolve lint issues fix: configuration validation for GCS Storage test(storage): resolve panic by test due to setupGCS ignoring returned error test(storage): add dummy gcs credentials test: add darwin support for macos to run tests ci: update workflows to pin gcs emulator version lint: resolve long line lengths & formatting issues test: move error for gcs mock earlier with an error test: stop test using local google credentials and use mock instead test: add missing dummy creds test(storage): use storage-testbench for GCS, isolate GCS tests, fix driver Delete - Switch GCS emulator from fake-gcs-server to storage-testbench in CI. Run the GCS emulator only in the privileged-test job; remove it from minimal and extended test jobs. - Consolidate GCS tests under pkg/storage/gcs (needprivileges,linux). Add TestMain with HTTPS proxy and /etc/hosts so tests talk to storage-testbench; move GCS-specific cases from storage_test.go and scrub_test.go into gcs_test.go. Run GCS tests via a second privileged-test invocation and collect coverage in coverage-needprivileges-gcs.txt. - Make GCS driver Delete idempotent and normalize errors. Treat PathNotFoundError from Delete as success so that deleting an already-gone path (e.g. after GC under eventual consistency) does not fail. Add formatErr to map 404/not found to PathNotFoundError and use it for all driver methods so callers get consistent storage driver errors. - Drop GCS branches and helpers from storage_test.go and scrub_test.go so non-privileged tests only use local/S3; GCS is tested only in pkg/storage/gcs with storage-testbench. - Set GCSMOCK_ENDPOINT without /storage/v1/, as the rest of the URL is set in tests. - Show errors in case of failure to create bucket. - Consolidate StorageDriverMock structs inside the pkg/test/mocks package. Signed-off-by: Andrei Aaron <andreifdaaron@gmail.com> Co-authored-by: Steven Marks <steve.marks@qomodo.io>
210 lines
5.5 KiB
Go
210 lines
5.5 KiB
Go
package gcs
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"io"
|
|
"strings"
|
|
|
|
// Add gcs support.
|
|
storagedriver "github.com/distribution/distribution/v3/registry/storage/driver"
|
|
_ "github.com/distribution/distribution/v3/registry/storage/driver/gcs"
|
|
|
|
storageConstants "zotregistry.dev/zot/v2/pkg/storage/constants"
|
|
)
|
|
|
|
type Driver struct {
|
|
store storagedriver.StorageDriver
|
|
}
|
|
|
|
func New(storeDriver storagedriver.StorageDriver) *Driver {
|
|
return &Driver{store: storeDriver}
|
|
}
|
|
|
|
func (driver *Driver) Name() string {
|
|
return storageConstants.GCSStorageDriverName
|
|
}
|
|
|
|
func (driver *Driver) EnsureDir(path string) error {
|
|
return nil
|
|
}
|
|
|
|
func (driver *Driver) DirExists(path string) bool {
|
|
if fi, err := driver.Stat(path); err == nil && fi.IsDir() {
|
|
return true
|
|
}
|
|
|
|
return false
|
|
}
|
|
|
|
func (driver *Driver) Reader(path string, offset int64) (io.ReadCloser, error) {
|
|
reader, err := driver.store.Reader(context.Background(), path, offset)
|
|
if err != nil {
|
|
return nil, driver.formatErr(err, path)
|
|
}
|
|
|
|
return reader, nil
|
|
}
|
|
|
|
func (driver *Driver) ReadFile(path string) ([]byte, error) {
|
|
content, err := driver.store.GetContent(context.Background(), path)
|
|
if err != nil {
|
|
return nil, driver.formatErr(err, path)
|
|
}
|
|
|
|
return content, nil
|
|
}
|
|
|
|
func (driver *Driver) Delete(path string) error {
|
|
err := driver.store.Delete(context.Background(), path)
|
|
if err == nil {
|
|
return nil
|
|
}
|
|
|
|
// Format the error first to convert GCS-specific 404 errors to PathNotFoundError
|
|
formattedErr := driver.formatErr(err, path)
|
|
|
|
// Check if the formatted error is PathNotFoundError
|
|
var pathNotFoundErr storagedriver.PathNotFoundError
|
|
if errors.As(formattedErr, &pathNotFoundErr) {
|
|
// For directory deletion, if the path doesn't exist, treat it as success (idempotent delete)
|
|
// In GCS, directories are just prefixes, so if all objects are deleted,
|
|
// the directory may already be gone (especially with eventual consistency in storage-testbench)
|
|
// This makes Delete idempotent: deleting a non-existent path is a no-op
|
|
return nil
|
|
}
|
|
|
|
return formattedErr
|
|
}
|
|
|
|
func (driver *Driver) Stat(path string) (storagedriver.FileInfo, error) {
|
|
fileInfo, err := driver.store.Stat(context.Background(), path)
|
|
if err != nil {
|
|
return nil, driver.formatErr(err, path)
|
|
}
|
|
|
|
return fileInfo, nil
|
|
}
|
|
|
|
func (driver *Driver) Writer(filepath string, append bool) (storagedriver.FileWriter, error) { //nolint:predeclared
|
|
writer, err := driver.store.Writer(context.Background(), filepath, append)
|
|
if err != nil {
|
|
return nil, driver.formatErr(err, filepath)
|
|
}
|
|
|
|
return writer, nil
|
|
}
|
|
|
|
func (driver *Driver) WriteFile(filepath string, content []byte) (int, error) {
|
|
var n int
|
|
|
|
stwr, err := driver.store.Writer(context.Background(), filepath, false)
|
|
if err != nil {
|
|
return -1, driver.formatErr(err, filepath)
|
|
}
|
|
defer stwr.Close()
|
|
|
|
if n, err = stwr.Write(content); err != nil {
|
|
return -1, driver.formatErr(err, filepath)
|
|
}
|
|
|
|
if err := stwr.Commit(context.Background()); err != nil {
|
|
return -1, driver.formatErr(err, filepath)
|
|
}
|
|
|
|
return n, nil
|
|
}
|
|
|
|
func (driver *Driver) Walk(path string, f storagedriver.WalkFn) error {
|
|
return driver.formatErr(driver.store.Walk(context.Background(), path, f), path)
|
|
}
|
|
|
|
func (driver *Driver) List(fullpath string) ([]string, error) {
|
|
list, err := driver.store.List(context.Background(), fullpath)
|
|
if err != nil {
|
|
return nil, driver.formatErr(err, fullpath)
|
|
}
|
|
|
|
return list, nil
|
|
}
|
|
|
|
func (driver *Driver) Move(sourcePath string, destPath string) error {
|
|
return driver.formatErr(driver.store.Move(context.Background(), sourcePath, destPath), sourcePath)
|
|
}
|
|
|
|
func (driver *Driver) SameFile(path1, path2 string) bool {
|
|
fi1, _ := driver.store.Stat(context.Background(), path1)
|
|
|
|
fi2, _ := driver.store.Stat(context.Background(), path2)
|
|
|
|
if fi1 != nil && fi2 != nil {
|
|
if fi1.IsDir() == fi2.IsDir() &&
|
|
fi1.ModTime() == fi2.ModTime() &&
|
|
fi1.Path() == fi2.Path() &&
|
|
fi1.Size() == fi2.Size() {
|
|
return true
|
|
}
|
|
}
|
|
|
|
return false
|
|
}
|
|
|
|
// Link puts an empty file that will act like a link between the original file and deduped one.
|
|
// Because gcs doesn't support symlinks, wherever the storage will encounter an empty file, it will get the original one
|
|
// from cache.
|
|
func (driver *Driver) Link(src, dest string) error {
|
|
return driver.formatErr(driver.store.PutContent(context.Background(), dest, []byte{}), dest)
|
|
}
|
|
|
|
// formatErr converts GCS-specific 404/not found errors to PathNotFoundError.
|
|
func (driver *Driver) formatErr(err error, path string) error {
|
|
switch actual := err.(type) { //nolint: errorlint
|
|
case nil:
|
|
return nil
|
|
case storagedriver.PathNotFoundError:
|
|
actual.DriverName = driver.Name()
|
|
if actual.Path == "" && path != "" {
|
|
actual.Path = path
|
|
}
|
|
|
|
return actual
|
|
case storagedriver.InvalidPathError:
|
|
actual.DriverName = driver.Name()
|
|
|
|
return actual
|
|
case storagedriver.InvalidOffsetError:
|
|
actual.DriverName = driver.Name()
|
|
|
|
return actual
|
|
default:
|
|
// Check for GCS-specific 404/not found errors by unwrapping the error chain
|
|
errToCheck := err
|
|
for errToCheck != nil {
|
|
errStr := errToCheck.Error()
|
|
isNotFound := strings.Contains(errStr, "object doesn't exist") ||
|
|
strings.Contains(errStr, "Error 404") ||
|
|
strings.Contains(errStr, "does not exist")
|
|
|
|
if isNotFound {
|
|
return storagedriver.PathNotFoundError{
|
|
DriverName: driver.Name(),
|
|
Path: path,
|
|
}
|
|
}
|
|
|
|
if unwrappable, ok := errToCheck.(interface{ Unwrap() error }); ok {
|
|
errToCheck = unwrappable.Unwrap()
|
|
} else {
|
|
break
|
|
}
|
|
}
|
|
|
|
storageError := storagedriver.Error{
|
|
DriverName: driver.Name(),
|
|
Detail: err,
|
|
}
|
|
|
|
return storageError
|
|
}
|
|
}
|