GCS storage support (#3798)

feat(storage): add a GCS driver

test(storage): add unit tests for GCS driver

test(storage): add missing unit tests for GCS driver & resolve lint issues

fix: configuration validation for GCS Storage

test(storage): resolve panic by test due to setupGCS ignoring returned error

test(storage): add dummy gcs credentials

test: add darwin support for macos to run tests

ci: update workflows to pin gcs emulator version

lint: resolve long line lengths & formatting issues

test: move error for gcs mock earlier with an error

test: stop test using local google credentials and use mock instead

test: add missing dummy creds

test(storage): use storage-testbench for GCS, isolate GCS tests, fix driver Delete

- Switch GCS emulator from fake-gcs-server to storage-testbench in CI.
  Run the GCS emulator only in the privileged-test job; remove it from
  minimal and extended test jobs.

- Consolidate GCS tests under pkg/storage/gcs (needprivileges,linux).
  Add TestMain with HTTPS proxy and /etc/hosts so tests talk to
  storage-testbench; move GCS-specific cases from storage_test.go and
  scrub_test.go into gcs_test.go. Run GCS tests via a second privileged-test
  invocation and collect coverage in coverage-needprivileges-gcs.txt.

- Make GCS driver Delete idempotent and normalize errors. Treat
  PathNotFoundError from Delete as success so that deleting an already-gone
  path (e.g. after GC under eventual consistency) does not fail. Add
  formatErr to map 404/not found to PathNotFoundError and use it for all
  driver methods so callers get consistent storage driver errors.

- Drop GCS branches and helpers from storage_test.go and scrub_test.go so
  non-privileged tests only use local/S3; GCS is tested only in
  pkg/storage/gcs with storage-testbench.

- Set GCSMOCK_ENDPOINT without /storage/v1/, as the rest of the URL is set in tests.

- Show errors in case of failure to create bucket.

- Consolidate StorageDriverMock structs inside the pkg/test/mocks package.

Signed-off-by: Andrei Aaron <andreifdaaron@gmail.com>
Co-authored-by: Steven Marks <steve.marks@qomodo.io>

pkg/test/common/rlimit_darwin.go

@@ -0,0 +1,3 @@
+package common
+
+type RlimT = uint64

pkg/test/mocks/storage_driver_mock.go

@@ -3,6 +3,7 @@ package mocks
 import (
 	"context"
 	"io"
+	"net/http"
 	"strings"
 	"time"
 
@@ -10,16 +11,17 @@ import (
 )
 
 type StorageDriverMock struct {
-	NameFn       func() string
-	GetContentFn func(ctx context.Context, path string) ([]byte, error)
-	PutContentFn func(ctx context.Context, path string, content []byte) error
-	ReaderFn     func(ctx context.Context, path string, offset int64) (io.ReadCloser, error)
-	WriterFn     func(ctx context.Context, path string, isAppend bool) (driver.FileWriter, error)
-	StatFn       func(ctx context.Context, path string) (driver.FileInfo, error)
-	ListFn       func(ctx context.Context, path string) ([]string, error)
-	MoveFn       func(ctx context.Context, sourcePath, destPath string) error
-	DeleteFn     func(ctx context.Context, path string) error
-	WalkFn       func(ctx context.Context, path string, f driver.WalkFn) error
+	NameFn        func() string
+	GetContentFn  func(ctx context.Context, path string) ([]byte, error)
+	PutContentFn  func(ctx context.Context, path string, content []byte) error
+	ReaderFn      func(ctx context.Context, path string, offset int64) (io.ReadCloser, error)
+	WriterFn      func(ctx context.Context, path string, isAppend bool) (driver.FileWriter, error)
+	StatFn        func(ctx context.Context, path string) (driver.FileInfo, error)
+	ListFn        func(ctx context.Context, path string) ([]string, error)
+	MoveFn        func(ctx context.Context, sourcePath, destPath string) error
+	DeleteFn      func(ctx context.Context, path string) error
+	WalkFn        func(ctx context.Context, path string, f driver.WalkFn, options ...func(*driver.WalkOptions)) error
+	RedirectURLFn func(r *http.Request, path string) (string, error)
 }
 
 //nolint:gochecknoglobals
@@ -100,13 +102,23 @@ func (s *StorageDriverMock) Delete(ctx context.Context, path string) error {
 	return nil
 }
 
+func (s *StorageDriverMock) RedirectURL(r *http.Request, path string) (string, error) {
+	if s != nil && s.RedirectURLFn != nil {
+		return s.RedirectURLFn(r, path)
+	}
+
+	return "", nil
+}
+
 func (s *StorageDriverMock) URLFor(ctx context.Context, path string, options map[string]any) (string, error) {
 	return "", nil
 }
 
-func (s *StorageDriverMock) Walk(ctx context.Context, path string, f driver.WalkFn) error {
+func (s *StorageDriverMock) Walk(ctx context.Context, path string, f driver.WalkFn,
+	options ...func(*driver.WalkOptions),
+) error {
 	if s != nil && s.WalkFn != nil {
-		return s.WalkFn(ctx, path, f)
+		return s.WalkFn(ctx, path, f, options...)
 	}
 
 	return nil
@@ -115,9 +127,14 @@ func (s *StorageDriverMock) Walk(ctx context.Context, path string, f driver.Walk
 type FileInfoMock struct {
 	IsDirFn func() bool
 	SizeFn  func() int64
+	PathFn  func() string
 }
 
 func (f *FileInfoMock) Path() string {
+	if f != nil && f.PathFn != nil {
+		return f.PathFn()
+	}
+
 	return ""
 }
 

pkg/test/skip/testskip.go

@@ -20,3 +20,11 @@ func SkipDynamo(t *testing.T) {
 		t.Skip("Skipping testing without AWS DynamoDB mock server")
 	}
 }
+
+func SkipGCS(t *testing.T) {
+	t.Helper()
+
+	if os.Getenv("GCSMOCK_ENDPOINT") == "" {
+		t.Skip("Skipping testing without GCS mock server")
+	}
+}