From 5e57656bffbe327bd90d08e6ece7a4df6fb51760 Mon Sep 17 00:00:00 2001 From: Andrei Aaron Date: Thu, 19 Feb 2026 09:41:21 +0200 Subject: [PATCH] GCS storage support (#3798) feat(storage): add a GCS driver test(storage): add unit tests for GCS driver test(storage): add missing unit tests for GCS driver & resolve lint issues fix: configuration validation for GCS Storage test(storage): resolve panic by test due to setupGCS ignoring returned error test(storage): add dummy gcs credentials test: add darwin support for macos to run tests ci: update workflows to pin gcs emulator version lint: resolve long line lengths & formatting issues test: move error for gcs mock earlier with an error test: stop test using local google credentials and use mock instead test: add missing dummy creds test(storage): use storage-testbench for GCS, isolate GCS tests, fix driver Delete - Switch GCS emulator from fake-gcs-server to storage-testbench in CI. Run the GCS emulator only in the privileged-test job; remove it from minimal and extended test jobs. - Consolidate GCS tests under pkg/storage/gcs (needprivileges,linux). Add TestMain with HTTPS proxy and /etc/hosts so tests talk to storage-testbench; move GCS-specific cases from storage_test.go and scrub_test.go into gcs_test.go. Run GCS tests via a second privileged-test invocation and collect coverage in coverage-needprivileges-gcs.txt. - Make GCS driver Delete idempotent and normalize errors. Treat PathNotFoundError from Delete as success so that deleting an already-gone path (e.g. after GC under eventual consistency) does not fail. Add formatErr to map 404/not found to PathNotFoundError and use it for all driver methods so callers get consistent storage driver errors. - Drop GCS branches and helpers from storage_test.go and scrub_test.go so non-privileged tests only use local/S3; GCS is tested only in pkg/storage/gcs with storage-testbench. - Set GCSMOCK_ENDPOINT without /storage/v1/, as the rest of the URL is set in tests. - Show errors in case of failure to create bucket. - Consolidate StorageDriverMock structs inside the pkg/test/mocks package. Signed-off-by: Andrei Aaron Co-authored-by: Steven Marks --- .../setup-gcs-storage-testbench/action.yaml | 10 + .../action.yaml | 8 + .github/workflows/test.yaml | 14 +- Makefile | 5 +- examples/config-gcs.json | 18 + pkg/api/config/config_elevated_test.go | 2 +- pkg/cli/server/root.go | 102 +- pkg/storage/constants/constants.go | 1 + pkg/storage/gcs/driver.go | 209 + pkg/storage/gcs/driver_test.go | 364 ++ pkg/storage/gcs/gcs.go | 38 + pkg/storage/gcs/gcs_test.go | 3682 +++++++++++++++++ pkg/storage/local/local_elevated_test.go | 2 +- pkg/storage/s3/s3_test.go | 420 +- pkg/storage/storage.go | 28 +- pkg/storage/storage_test.go | 80 +- pkg/test/common/rlimit_darwin.go | 3 + pkg/test/mocks/storage_driver_mock.go | 41 +- pkg/test/skip/testskip.go | 8 + 19 files changed, 4634 insertions(+), 401 deletions(-) create mode 100644 .github/actions/setup-gcs-storage-testbench/action.yaml create mode 100644 .github/actions/teardown-gcs-storage-testbench/action.yaml create mode 100644 examples/config-gcs.json create mode 100644 pkg/storage/gcs/driver.go create mode 100644 pkg/storage/gcs/driver_test.go create mode 100644 pkg/storage/gcs/gcs.go create mode 100644 pkg/storage/gcs/gcs_test.go create mode 100644 pkg/test/common/rlimit_darwin.go diff --git a/.github/actions/setup-gcs-storage-testbench/action.yaml b/.github/actions/setup-gcs-storage-testbench/action.yaml new file mode 100644 index 00000000..0eb5588e --- /dev/null +++ b/.github/actions/setup-gcs-storage-testbench/action.yaml @@ -0,0 +1,10 @@ +name: 'Setup storage-testbench' +description: 'Download & run Google Cloud Storage testbench container' +runs: + using: "composite" + steps: + - shell: bash + run: | + docker run -d --name storage-testbench -p 9000:9000 -p 9090:9090 ghcr.io/project-zot/ci-images/gcs-storage-testbench:v0.60.0-zot + echo "Waiting for storage-testbench..." + timeout 30s bash -c 'until curl -s http://localhost:9000/storage/v1/b; do sleep 1; done' diff --git a/.github/actions/teardown-gcs-storage-testbench/action.yaml b/.github/actions/teardown-gcs-storage-testbench/action.yaml new file mode 100644 index 00000000..3c1d8eb1 --- /dev/null +++ b/.github/actions/teardown-gcs-storage-testbench/action.yaml @@ -0,0 +1,8 @@ +name: 'Teardown storage-testbench' +description: 'Stop & remove storage-testbench container' +runs: + using: "composite" + steps: + - if: always() + shell: bash + run: docker rm -f storage-testbench || true \ No newline at end of file diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index b5fcda14..5ae10586 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -148,13 +148,23 @@ jobs: run: | cd $GITHUB_WORKSPACE go mod download + - uses: ./.github/actions/setup-gcs-storage-testbench - name: run zot privileged tests - run: sudo env "PATH=$PATH" make privileged-test + run: > + sudo env + "PATH=$PATH" + "GCSMOCK_ENDPOINT=$GCSMOCK_ENDPOINT" + "STORAGE_EMULATOR_HOST=$STORAGE_EMULATOR_HOST" + make privileged-test + env: + GCSMOCK_ENDPOINT: http://localhost:9000/ + STORAGE_EMULATOR_HOST: localhost:9000 - name: upload coverage uses: actions/upload-artifact@v6 with: name: coverage-needprivileges - path: coverage-needprivileges.txt + path: coverage-needprivileges-*.txt + - uses: ./.github/actions/teardown-gcs-storage-testbench test-coverage: name: Collect all test coverage runs-on: ubuntu-latest diff --git a/Makefile b/Makefile index ef01652d..a5c507e5 100644 --- a/Makefile +++ b/Makefile @@ -238,7 +238,8 @@ test: test-extended test-minimal test-devmode .PHONY: privileged-test privileged-test: $(if $(findstring ui,$(BUILD_LABELS)), ui) privileged-test: - env GOEXPERIMENT=jsonv2 go test -failfast -tags needprivileges,$(BUILD_LABELS) -trimpath -race -timeout 15m -cover -coverpkg ./... -coverprofile=coverage-needprivileges.txt -covermode=atomic ./pkg/storage/local/... ./pkg/cli/client/... -run ^TestElevatedPrivileges + env GOEXPERIMENT=jsonv2 go test -failfast -tags needprivileges,$(BUILD_LABELS) -trimpath -race -timeout 15m -cover -coverpkg ./... -coverprofile=coverage-needprivileges-local.txt -covermode=atomic ./pkg/storage/local/... ./pkg/cli/client/... -run ^TestElevatedPrivileges + env GOEXPERIMENT=jsonv2 go test -failfast -tags needprivileges,$(BUILD_LABELS) -trimpath -race -timeout 15m -cover -coverpkg ./... -coverprofile=coverage-needprivileges-gcs.txt -covermode=atomic ./pkg/storage/gcs/... .PHONY: testdata-certs testdata-certs: @@ -356,7 +357,7 @@ check: ./.golangci.yaml $(GOLINTER) $(GOLINTER) run --output.text.colors --build-tags debug ./pkg/debug/swagger/ ./pkg/debug/gqlplayground $(GOLINTER) run --output.text.colors --build-tags dev ./pkg/test/inject/ $(GOLINTER) run --output.text.colors --build-tags stress ./pkg/cli/server/ - $(GOLINTER) run --output.text.colors --build-tags needprivileges,$(BUILD_LABELS) ./pkg/cli/client/ ./pkg/storage/local/ ./pkg/api/config/ + $(GOLINTER) run --output.text.colors --build-tags needprivileges,$(BUILD_LABELS) ./pkg/cli/client/ ./pkg/storage/local/ ./pkg/storage/gcs/ ./pkg/api/config/ rm pkg/extensions/build/.empty .PHONY: swagger diff --git a/examples/config-gcs.json b/examples/config-gcs.json new file mode 100644 index 00000000..b114988a --- /dev/null +++ b/examples/config-gcs.json @@ -0,0 +1,18 @@ +{ + "storage": { + "rootDirectory": "/tmp/zot", + "dedupe": false, + "storageDriver": { + "name": "gcs", + "bucket": "zot-storage", + "credentialsFile": "/path/to/gcs-credentials.json" + } + }, + "http": { + "address": "127.0.0.1", + "port": "8080" + }, + "log": { + "level": "debug" + } +} diff --git a/pkg/api/config/config_elevated_test.go b/pkg/api/config/config_elevated_test.go index 134982a0..6ffccad6 100644 --- a/pkg/api/config/config_elevated_test.go +++ b/pkg/api/config/config_elevated_test.go @@ -1,4 +1,4 @@ -//go:build needprivileges +//go:build needprivileges && linux package config_test diff --git a/pkg/cli/server/root.go b/pkg/cli/server/root.go index ee61d6e3..91f02cb5 100644 --- a/pkg/cli/server/root.go +++ b/pkg/cli/server/root.go @@ -299,6 +299,10 @@ func getStorageType(storageDriver map[string]any) string { return storageConstants.S3StorageDriverName } + if storeName == storageConstants.GCSStorageDriverName { + return storageConstants.GCSStorageDriverName + } + return storeName } @@ -559,6 +563,57 @@ func validateExtensionsConfig(cfg *config.Config, logger zlog.Logger) error { return nil } +func validateStorageConfigSection( + cfg *config.Config, logger zlog.Logger, storageConfig config.GlobalStorageConfig, +) error { + if len(storageConfig.StorageDriver) != 0 { + // enforce s3/gcs driver in case of using storage driver + if storageConfig.StorageDriver["name"] != storageConstants.S3StorageDriverName && + storageConfig.StorageDriver["name"] != storageConstants.GCSStorageDriverName { + msg := "unsupported storage driver" + logger.Error().Err(zerr.ErrBadConfig).Interface("storageDriver", storageConfig.StorageDriver["name"]).Msg(msg) + + return fmt.Errorf("%w: %s", zerr.ErrBadConfig, msg) + } + + // enforce tmpDir in case sync + s3/gcs + extensionsConfig := cfg.CopyExtensionsConfig() + if extensionsConfig.IsSyncEnabled() && extensionsConfig.Sync.DownloadDir == "" { + msg := "using both sync and remote storage features needs config.Extensions.Sync.DownloadDir to be specified" + logger.Error().Err(zerr.ErrBadConfig).Msg(msg) + + return fmt.Errorf("%w: %s", zerr.ErrBadConfig, msg) + } + } + + // enforce s3/gcs driver on subpaths in case of using storage driver + if len(storageConfig.SubPaths) > 0 { + for route, subStorageConfig := range storageConfig.SubPaths { + if len(subStorageConfig.StorageDriver) != 0 { + if subStorageConfig.StorageDriver["name"] != storageConstants.S3StorageDriverName && + subStorageConfig.StorageDriver["name"] != storageConstants.GCSStorageDriverName { + msg := "unsupported storage driver" + logger.Error().Err(zerr.ErrBadConfig).Str("subpath", route).Interface("storageDriver", + subStorageConfig.StorageDriver["name"]).Msg(msg) + + return fmt.Errorf("%w: %s", zerr.ErrBadConfig, msg) + } + + // enforce tmpDir in case sync + s3/gcs + extensionsConfig := cfg.CopyExtensionsConfig() + if extensionsConfig.IsSyncEnabled() && extensionsConfig.Sync.DownloadDir == "" { + msg := "using both sync and remote storage features needs config.Extensions.Sync.DownloadDir to be specified" + logger.Error().Err(zerr.ErrBadConfig).Msg(msg) + + return fmt.Errorf("%w: %s", zerr.ErrBadConfig, msg) + } + } + } + } + + return nil +} + func validateConfiguration(config *config.Config, logger zlog.Logger) error { if err := validateHTTP(config, logger); err != nil { return err @@ -614,51 +669,8 @@ func validateConfiguration(config *config.Config, logger zlog.Logger) error { } storageConfig := config.CopyStorageConfig() - if len(storageConfig.StorageDriver) != 0 { - // enforce s3 driver in case of using storage driver - if storageConfig.StorageDriver["name"] != storageConstants.S3StorageDriverName { - msg := "unsupported storage driver" - logger.Error().Err(zerr.ErrBadConfig).Interface("cacheDriver", storageConfig.StorageDriver["name"]).Msg(msg) - - return fmt.Errorf("%w: %s", zerr.ErrBadConfig, msg) - } - - // enforce tmpDir in case sync + s3 - extensionsConfig := config.CopyExtensionsConfig() - if extensionsConfig.IsSyncEnabled() && extensionsConfig.Sync.DownloadDir == "" { - msg := "using both sync and remote storage features needs config.Extensions.Sync.DownloadDir to be specified" - logger.Error().Err(zerr.ErrBadConfig).Msg(msg) - - return fmt.Errorf("%w: %s", zerr.ErrBadConfig, msg) - } - } - - // enforce s3 driver on subpaths in case of using storage driver - if storageConfig.SubPaths != nil { - if len(storageConfig.SubPaths) > 0 { - subPaths := storageConfig.SubPaths - - for route, subStorageConfig := range subPaths { - if len(subStorageConfig.StorageDriver) != 0 { - if subStorageConfig.StorageDriver["name"] != storageConstants.S3StorageDriverName { - msg := "unsupported storage driver" - logger.Error().Err(zerr.ErrBadConfig).Str("subpath", route).Interface("storageDriver", - subStorageConfig.StorageDriver["name"]).Msg(msg) - - return fmt.Errorf("%w: %s", zerr.ErrBadConfig, msg) - } - - // enforce tmpDir in case sync + s3 - extensionsConfig := config.CopyExtensionsConfig() - if extensionsConfig.IsSyncEnabled() && extensionsConfig.Sync.DownloadDir == "" { - msg := "using both sync and remote storage features needs config.Extensions.Sync.DownloadDir to be specified" - logger.Error().Err(zerr.ErrBadConfig).Msg(msg) - - return fmt.Errorf("%w: %s", zerr.ErrBadConfig, msg) - } - } - } - } + if err := validateStorageConfigSection(config, logger, storageConfig); err != nil { + return err } // check glob patterns in authz config are compilable diff --git a/pkg/storage/constants/constants.go b/pkg/storage/constants/constants.go index 14533c28..c2b305dc 100644 --- a/pkg/storage/constants/constants.go +++ b/pkg/storage/constants/constants.go @@ -24,5 +24,6 @@ const ( DefaultGCDelay = 1 * time.Hour DefaultGCInterval = 1 * time.Hour S3StorageDriverName = "s3" + GCSStorageDriverName = "gcs" LocalStorageDriverName = "local" ) diff --git a/pkg/storage/gcs/driver.go b/pkg/storage/gcs/driver.go new file mode 100644 index 00000000..fdfd42a1 --- /dev/null +++ b/pkg/storage/gcs/driver.go @@ -0,0 +1,209 @@ +package gcs + +import ( + "context" + "errors" + "io" + "strings" + + // Add gcs support. + storagedriver "github.com/distribution/distribution/v3/registry/storage/driver" + _ "github.com/distribution/distribution/v3/registry/storage/driver/gcs" + + storageConstants "zotregistry.dev/zot/v2/pkg/storage/constants" +) + +type Driver struct { + store storagedriver.StorageDriver +} + +func New(storeDriver storagedriver.StorageDriver) *Driver { + return &Driver{store: storeDriver} +} + +func (driver *Driver) Name() string { + return storageConstants.GCSStorageDriverName +} + +func (driver *Driver) EnsureDir(path string) error { + return nil +} + +func (driver *Driver) DirExists(path string) bool { + if fi, err := driver.Stat(path); err == nil && fi.IsDir() { + return true + } + + return false +} + +func (driver *Driver) Reader(path string, offset int64) (io.ReadCloser, error) { + reader, err := driver.store.Reader(context.Background(), path, offset) + if err != nil { + return nil, driver.formatErr(err, path) + } + + return reader, nil +} + +func (driver *Driver) ReadFile(path string) ([]byte, error) { + content, err := driver.store.GetContent(context.Background(), path) + if err != nil { + return nil, driver.formatErr(err, path) + } + + return content, nil +} + +func (driver *Driver) Delete(path string) error { + err := driver.store.Delete(context.Background(), path) + if err == nil { + return nil + } + + // Format the error first to convert GCS-specific 404 errors to PathNotFoundError + formattedErr := driver.formatErr(err, path) + + // Check if the formatted error is PathNotFoundError + var pathNotFoundErr storagedriver.PathNotFoundError + if errors.As(formattedErr, &pathNotFoundErr) { + // For directory deletion, if the path doesn't exist, treat it as success (idempotent delete) + // In GCS, directories are just prefixes, so if all objects are deleted, + // the directory may already be gone (especially with eventual consistency in storage-testbench) + // This makes Delete idempotent: deleting a non-existent path is a no-op + return nil + } + + return formattedErr +} + +func (driver *Driver) Stat(path string) (storagedriver.FileInfo, error) { + fileInfo, err := driver.store.Stat(context.Background(), path) + if err != nil { + return nil, driver.formatErr(err, path) + } + + return fileInfo, nil +} + +func (driver *Driver) Writer(filepath string, append bool) (storagedriver.FileWriter, error) { //nolint:predeclared + writer, err := driver.store.Writer(context.Background(), filepath, append) + if err != nil { + return nil, driver.formatErr(err, filepath) + } + + return writer, nil +} + +func (driver *Driver) WriteFile(filepath string, content []byte) (int, error) { + var n int + + stwr, err := driver.store.Writer(context.Background(), filepath, false) + if err != nil { + return -1, driver.formatErr(err, filepath) + } + defer stwr.Close() + + if n, err = stwr.Write(content); err != nil { + return -1, driver.formatErr(err, filepath) + } + + if err := stwr.Commit(context.Background()); err != nil { + return -1, driver.formatErr(err, filepath) + } + + return n, nil +} + +func (driver *Driver) Walk(path string, f storagedriver.WalkFn) error { + return driver.formatErr(driver.store.Walk(context.Background(), path, f), path) +} + +func (driver *Driver) List(fullpath string) ([]string, error) { + list, err := driver.store.List(context.Background(), fullpath) + if err != nil { + return nil, driver.formatErr(err, fullpath) + } + + return list, nil +} + +func (driver *Driver) Move(sourcePath string, destPath string) error { + return driver.formatErr(driver.store.Move(context.Background(), sourcePath, destPath), sourcePath) +} + +func (driver *Driver) SameFile(path1, path2 string) bool { + fi1, _ := driver.store.Stat(context.Background(), path1) + + fi2, _ := driver.store.Stat(context.Background(), path2) + + if fi1 != nil && fi2 != nil { + if fi1.IsDir() == fi2.IsDir() && + fi1.ModTime() == fi2.ModTime() && + fi1.Path() == fi2.Path() && + fi1.Size() == fi2.Size() { + return true + } + } + + return false +} + +// Link puts an empty file that will act like a link between the original file and deduped one. +// Because gcs doesn't support symlinks, wherever the storage will encounter an empty file, it will get the original one +// from cache. +func (driver *Driver) Link(src, dest string) error { + return driver.formatErr(driver.store.PutContent(context.Background(), dest, []byte{}), dest) +} + +// formatErr converts GCS-specific 404/not found errors to PathNotFoundError. +func (driver *Driver) formatErr(err error, path string) error { + switch actual := err.(type) { //nolint: errorlint + case nil: + return nil + case storagedriver.PathNotFoundError: + actual.DriverName = driver.Name() + if actual.Path == "" && path != "" { + actual.Path = path + } + + return actual + case storagedriver.InvalidPathError: + actual.DriverName = driver.Name() + + return actual + case storagedriver.InvalidOffsetError: + actual.DriverName = driver.Name() + + return actual + default: + // Check for GCS-specific 404/not found errors by unwrapping the error chain + errToCheck := err + for errToCheck != nil { + errStr := errToCheck.Error() + isNotFound := strings.Contains(errStr, "object doesn't exist") || + strings.Contains(errStr, "Error 404") || + strings.Contains(errStr, "does not exist") + + if isNotFound { + return storagedriver.PathNotFoundError{ + DriverName: driver.Name(), + Path: path, + } + } + + if unwrappable, ok := errToCheck.(interface{ Unwrap() error }); ok { + errToCheck = unwrappable.Unwrap() + } else { + break + } + } + + storageError := storagedriver.Error{ + DriverName: driver.Name(), + Detail: err, + } + + return storageError + } +} diff --git a/pkg/storage/gcs/driver_test.go b/pkg/storage/gcs/driver_test.go new file mode 100644 index 00000000..80532442 --- /dev/null +++ b/pkg/storage/gcs/driver_test.go @@ -0,0 +1,364 @@ +package gcs_test + +import ( + "context" + "errors" + "fmt" + "io" + "strings" + "testing" + "time" + + "github.com/distribution/distribution/v3/registry/storage/driver" + . "github.com/smartystreets/goconvey/convey" + + "zotregistry.dev/zot/v2/pkg/extensions/monitoring" + zlog "zotregistry.dev/zot/v2/pkg/log" + "zotregistry.dev/zot/v2/pkg/storage/gcs" + "zotregistry.dev/zot/v2/pkg/test/mocks" +) + +var errTest = errors.New("error") + +type fileInfoMock struct { + isDir bool + size int64 + modTime time.Time + path string +} + +func (f *fileInfoMock) Path() string { return f.path } +func (f *fileInfoMock) Size() int64 { return f.size } +func (f *fileInfoMock) ModTime() time.Time { return f.modTime } +func (f *fileInfoMock) IsDir() bool { return f.isDir } + +func TestDriver(t *testing.T) { + Convey("GCS Driver", t, func() { + storeMock := &mocks.StorageDriverMock{} + gcsDriver := gcs.New(storeMock) + + Convey("Name", func() { + So(gcsDriver.Name(), ShouldEqual, "gcs") + }) + + Convey("EnsureDir", func() { + err := gcsDriver.EnsureDir("/test") + So(err, ShouldBeNil) + }) + + Convey("DirExists", func() { + Convey("True", func() { + storeMock.StatFn = func(ctx context.Context, path string) (driver.FileInfo, error) { + return &mocks.FileInfoMock{ + IsDirFn: func() bool { return true }, + }, nil + } + So(gcsDriver.DirExists("/test"), ShouldBeTrue) + }) + + Convey("False - Not a dir", func() { + storeMock.StatFn = func(ctx context.Context, path string) (driver.FileInfo, error) { + return &mocks.FileInfoMock{ + IsDirFn: func() bool { return false }, + }, nil + } + So(gcsDriver.DirExists("/test"), ShouldBeFalse) + }) + + Convey("False - Error", func() { + storeMock.StatFn = func(ctx context.Context, path string) (driver.FileInfo, error) { + return nil, errTest + } + So(gcsDriver.DirExists("/test"), ShouldBeFalse) + }) + }) + + Convey("Reader", func() { + Convey("Success", func() { + storeMock.ReaderFn = func(ctx context.Context, path string, offset int64) (io.ReadCloser, error) { + return io.NopCloser(strings.NewReader("")), nil + } + r, err := gcsDriver.Reader("/test", 0) + So(err, ShouldBeNil) + So(r, ShouldNotBeNil) + }) + + Convey("InvalidOffsetError", func() { + storeMock.ReaderFn = func(ctx context.Context, path string, offset int64) (io.ReadCloser, error) { + return nil, driver.InvalidOffsetError{Path: path, Offset: offset} + } + _, err := gcsDriver.Reader("/test", 100) + So(err, ShouldNotBeNil) + + var invalidOffset driver.InvalidOffsetError + So(errors.As(err, &invalidOffset), ShouldBeTrue) + So(invalidOffset.DriverName, ShouldEqual, "gcs") + }) + }) + + Convey("ReadFile", func() { + Convey("Success", func() { + storeMock.GetContentFn = func(ctx context.Context, path string) ([]byte, error) { + return []byte("content"), nil + } + content, err := gcsDriver.ReadFile("/test") + So(err, ShouldBeNil) + So(string(content), ShouldEqual, "content") + }) + + Convey("PathNotFoundError with empty Path gets path set", func() { + storeMock.GetContentFn = func(ctx context.Context, path string) ([]byte, error) { + return nil, driver.PathNotFoundError{Path: ""} // Path empty so driver sets it + } + _, err := gcsDriver.ReadFile("/requested/path") + So(err, ShouldNotBeNil) + + var pathErr driver.PathNotFoundError + So(errors.As(err, &pathErr), ShouldBeTrue) + So(pathErr.DriverName, ShouldEqual, "gcs") + So(pathErr.Path, ShouldEqual, "/requested/path") + }) + + Convey("GCS not-found string becomes PathNotFoundError", func() { + for _, msg := range []string{"object doesn't exist", "Error 404", "does not exist"} { + errMsg := msg + storeMock.GetContentFn = func(ctx context.Context, path string) ([]byte, error) { + //nolint:err113 // test needs variable not-found message + return nil, fmt.Errorf("%s", errMsg) + } + _, err := gcsDriver.ReadFile("/key") + So(err, ShouldNotBeNil) + + var pathErr driver.PathNotFoundError + So(errors.As(err, &pathErr), ShouldBeTrue) + So(pathErr.Path, ShouldEqual, "/key") + } + }) + + Convey("Generic error becomes storagedriver.Error", func() { + storeMock.GetContentFn = func(ctx context.Context, path string) ([]byte, error) { + return nil, errTest + } + _, err := gcsDriver.ReadFile("/test") + So(err, ShouldNotBeNil) + + var storageErr driver.Error + So(errors.As(err, &storageErr), ShouldBeTrue) + So(storageErr.DriverName, ShouldEqual, "gcs") + So(storageErr.Detail, ShouldEqual, errTest) + }) + }) + + Convey("Delete", func() { + Convey("Success", func() { + storeMock.DeleteFn = func(ctx context.Context, path string) error { + return nil + } + err := gcsDriver.Delete("/test") + So(err, ShouldBeNil) + }) + + Convey("PathNotFoundError is idempotent (return nil)", func() { + storeMock.DeleteFn = func(ctx context.Context, path string) error { + return driver.PathNotFoundError{Path: path} + } + err := gcsDriver.Delete("/nonexistent") + So(err, ShouldBeNil) + }) + + Convey("Other error is returned", func() { + storeMock.DeleteFn = func(ctx context.Context, path string) error { + return errTest + } + err := gcsDriver.Delete("/test") + So(err, ShouldNotBeNil) + So(errors.Is(err, errTest), ShouldBeFalse) // wrapped in storagedriver.Error + }) + }) + + Convey("Stat", func() { + Convey("Success", func() { + storeMock.StatFn = func(ctx context.Context, path string) (driver.FileInfo, error) { + return &mocks.FileInfoMock{}, nil + } + fi, err := gcsDriver.Stat("/test") + So(err, ShouldBeNil) + So(fi, ShouldNotBeNil) + }) + + Convey("InvalidPathError", func() { + storeMock.StatFn = func(ctx context.Context, path string) (driver.FileInfo, error) { + return nil, driver.InvalidPathError{Path: path} + } + _, err := gcsDriver.Stat("/bad") + So(err, ShouldNotBeNil) + + var invalidPath driver.InvalidPathError + So(errors.As(err, &invalidPath), ShouldBeTrue) + So(invalidPath.DriverName, ShouldEqual, "gcs") + }) + }) + + Convey("Writer", func() { + Convey("Success", func() { + storeMock.WriterFn = func(ctx context.Context, path string, isAppend bool) (driver.FileWriter, error) { + return &mocks.FileWriterMock{}, nil + } + w, err := gcsDriver.Writer("/test", false) + So(err, ShouldBeNil) + So(w, ShouldNotBeNil) + }) + + Convey("Error", func() { + storeMock.WriterFn = func(ctx context.Context, path string, isAppend bool) (driver.FileWriter, error) { + return nil, errTest + } + _, err := gcsDriver.Writer("/test", false) + So(err, ShouldNotBeNil) + }) + }) + + Convey("WriteFile", func() { + Convey("Success", func() { + storeMock.WriterFn = func(ctx context.Context, path string, isAppend bool) (driver.FileWriter, error) { + return &mocks.FileWriterMock{ + WriteFn: func(p []byte) (int, error) { + return len(p), nil + }, + CommitFn: func() error { + return nil + }, + CloseFn: func() error { + return nil + }, + }, nil + } + n, err := gcsDriver.WriteFile("/test", []byte("content")) + So(err, ShouldBeNil) + So(n, ShouldEqual, 7) + }) + + Convey("Writer Error", func() { + storeMock.WriterFn = func(ctx context.Context, path string, isAppend bool) (driver.FileWriter, error) { + return nil, errTest + } + _, err := gcsDriver.WriteFile("/test", []byte("content")) + So(err, ShouldNotBeNil) + }) + + Convey("Write Error", func() { + storeMock.WriterFn = func(ctx context.Context, path string, isAppend bool) (driver.FileWriter, error) { + return &mocks.FileWriterMock{ + WriteFn: func(p []byte) (int, error) { + return 0, errTest + }, + CloseFn: func() error { return nil }, + }, nil + } + _, err := gcsDriver.WriteFile("/test", []byte("content")) + So(err, ShouldNotBeNil) + }) + + Convey("Commit Error", func() { + storeMock.WriterFn = func(ctx context.Context, path string, isAppend bool) (driver.FileWriter, error) { + return &mocks.FileWriterMock{ + WriteFn: func(p []byte) (int, error) { + return len(p), nil + }, + CommitFn: func() error { + return errTest + }, + CloseFn: func() error { return nil }, + }, nil + } + _, err := gcsDriver.WriteFile("/test", []byte("content")) + So(err, ShouldNotBeNil) + }) + }) + + Convey("Walk", func() { + storeMock.WalkFn = func(ctx context.Context, path string, f driver.WalkFn, _ ...func(*driver.WalkOptions)) error { + return nil + } + err := gcsDriver.Walk("/test", nil) + So(err, ShouldBeNil) + }) + + Convey("List", func() { + Convey("Success", func() { + storeMock.ListFn = func(ctx context.Context, path string) ([]string, error) { + return []string{"a"}, nil + } + l, err := gcsDriver.List("/test") + So(err, ShouldBeNil) + So(l, ShouldResemble, []string{"a"}) + }) + + Convey("Error", func() { + storeMock.ListFn = func(ctx context.Context, path string) ([]string, error) { + return nil, errTest + } + _, err := gcsDriver.List("/test") + So(err, ShouldNotBeNil) + }) + }) + + Convey("Move", func() { + storeMock.MoveFn = func(ctx context.Context, sourcePath, destPath string) error { + return nil + } + err := gcsDriver.Move("/src", "/dst") + So(err, ShouldBeNil) + }) + + Convey("SameFile", func() { + Convey("True", func() { + now := time.Now() + storeMock.StatFn = func(ctx context.Context, path string) (driver.FileInfo, error) { + return &fileInfoMock{ + isDir: false, + size: 10, + modTime: now, + path: "/canonical/path", + }, nil + } + So(gcsDriver.SameFile("/path1", "/path2"), ShouldBeTrue) + }) + + Convey("False - Different ModTime", func() { + storeMock.StatFn = func(ctx context.Context, path string) (driver.FileInfo, error) { + modTime := time.Now() + if path == "/path2" { + modTime = modTime.Add(1 * time.Hour) + } + + return &fileInfoMock{ + isDir: false, + size: 10, + modTime: modTime, + path: path, + }, nil + } + So(gcsDriver.SameFile("/path1", "/path2"), ShouldBeFalse) + }) + }) + + Convey("Link", func() { + storeMock.PutContentFn = func(ctx context.Context, path string, content []byte) error { + return nil + } + err := gcsDriver.Link("/src", "/dst") + So(err, ShouldBeNil) + }) + }) +} + +func TestNewImageStore(t *testing.T) { + Convey("NewImageStore", t, func() { + storeMock := &mocks.StorageDriverMock{} + log := zlog.NewTestLogger() + metrics := monitoring.NewMetricsServer(false, log) + imgStore := gcs.NewImageStore("/tmp", "/tmp", true, true, log, metrics, nil, storeMock, nil, nil, nil) + So(imgStore, ShouldNotBeNil) + }) +} diff --git a/pkg/storage/gcs/gcs.go b/pkg/storage/gcs/gcs.go new file mode 100644 index 00000000..18137123 --- /dev/null +++ b/pkg/storage/gcs/gcs.go @@ -0,0 +1,38 @@ +package gcs + +import ( + // Add gcs support. + "github.com/distribution/distribution/v3/registry/storage/driver" + // Load gcs driver. + _ "github.com/distribution/distribution/v3/registry/storage/driver/gcs" + + "zotregistry.dev/zot/v2/pkg/compat" + "zotregistry.dev/zot/v2/pkg/extensions/events" + "zotregistry.dev/zot/v2/pkg/extensions/monitoring" + zlog "zotregistry.dev/zot/v2/pkg/log" + common "zotregistry.dev/zot/v2/pkg/storage/common" + "zotregistry.dev/zot/v2/pkg/storage/imagestore" + storageTypes "zotregistry.dev/zot/v2/pkg/storage/types" +) + +// NewImageStore returns a new image store backed by cloud storages. +// see https://github.com/docker/docker.github.io/tree/master/registry/storage-drivers +// Use the last argument to properly set a cache database, or it will default to boltDB local storage. +func NewImageStore(rootDir string, cacheDir string, dedupe, commit bool, log zlog.Logger, + metrics monitoring.MetricServer, linter common.Lint, store driver.StorageDriver, + cacheDriver storageTypes.Cache, compat []compat.MediaCompatibility, recorder events.Recorder, +) storageTypes.ImageStore { + return imagestore.NewImageStore( + rootDir, + cacheDir, + dedupe, + commit, + log, + metrics, + linter, + New(store), + cacheDriver, + compat, + recorder, + ) +} diff --git a/pkg/storage/gcs/gcs_test.go b/pkg/storage/gcs/gcs_test.go new file mode 100644 index 00000000..8a9b50e3 --- /dev/null +++ b/pkg/storage/gcs/gcs_test.go @@ -0,0 +1,3682 @@ +//go:build needprivileges && linux + +package gcs_test + +import ( + "bytes" + "context" + "crypto/rand" + "crypto/rsa" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "encoding/json" + "encoding/pem" + "errors" + "fmt" + "io" + "math/big" + "net" + "net/http" + "os" + "os/exec" + "path" + "regexp" + "slices" + "strings" + "sync" + "testing" + "time" + + "github.com/distribution/distribution/v3/registry/storage/driver" + "github.com/distribution/distribution/v3/registry/storage/driver/factory" + guuid "github.com/gofrs/uuid" + godigest "github.com/opencontainers/go-digest" + ispec "github.com/opencontainers/image-spec/specs-go/v1" + . "github.com/smartystreets/goconvey/convey" + + zerr "zotregistry.dev/zot/v2/errors" + "zotregistry.dev/zot/v2/pkg/api/config" + "zotregistry.dev/zot/v2/pkg/extensions/monitoring" + "zotregistry.dev/zot/v2/pkg/log" + "zotregistry.dev/zot/v2/pkg/storage" + "zotregistry.dev/zot/v2/pkg/storage/cache" + common "zotregistry.dev/zot/v2/pkg/storage/common" + storageConstants "zotregistry.dev/zot/v2/pkg/storage/constants" + "zotregistry.dev/zot/v2/pkg/storage/gc" + "zotregistry.dev/zot/v2/pkg/storage/gcs" + storageTypes "zotregistry.dev/zot/v2/pkg/storage/types" + . "zotregistry.dev/zot/v2/pkg/test/image-utils" + "zotregistry.dev/zot/v2/pkg/test/mocks" + tskip "zotregistry.dev/zot/v2/pkg/test/skip" +) + +//nolint:gochecknoglobals // test constants +const ( + repoName = "test" + tag = "0.0.1" +) + +var ( + trueVal bool = true //nolint: gochecknoglobals + errGCSMockEndpointNotSet = errors.New("GCSMOCK_ENDPOINT must be set for GCS tests") + errUnexpectedError = errors.New("unexpected err") + errBucketCreateFailed = errors.New("failed to create bucket") +) + +// httpsProxyServer manages an HTTPS proxy server on port 443. +type httpsProxyServer struct { + server *http.Server + listener net.Listener + wg sync.WaitGroup + target string + certFile string // Path to the certificate file for cleanup +} + +// newHTTPSProxyServer creates a new HTTPS proxy server that forwards requests to the target. +func newHTTPSProxyServer(target string) (*httpsProxyServer, error) { + // Generate self-signed certificate + priv, err := rsa.GenerateKey(rand.Reader, 2048) + if err != nil { + return nil, fmt.Errorf("failed to generate private key: %w", err) + } + + template := x509.Certificate{ + SerialNumber: big.NewInt(1), + Subject: pkix.Name{ + CommonName: "oauth2.googleapis.com", + }, + NotBefore: time.Now(), + NotAfter: time.Now().Add(24 * time.Hour), + KeyUsage: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + DNSNames: []string{"oauth2.googleapis.com", "www.googleapis.com", "storage.googleapis.com"}, + IPAddresses: []net.IP{net.IPv4(127, 0, 0, 1)}, + } + + certDER, err := x509.CreateCertificate(rand.Reader, &template, &template, &priv.PublicKey, priv) + if err != nil { + return nil, fmt.Errorf("failed to create certificate: %w", err) + } + + certPEM := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: certDER}) + keyPEM := pem.EncodeToMemory(&pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(priv)}) + + cert, err := tls.X509KeyPair(certPEM, keyPEM) + if err != nil { + return nil, fmt.Errorf("failed to create key pair: %w", err) + } + + // Write certificate to a temporary file so we can add it to the trusted certificates + // via SSL_CERT_FILE environment variable. This is the standard way to add custom + // trusted certificates and works with Go's crypto/x509 package, including OAuth2 clients. + certFile, err := os.CreateTemp("", "gcs-test-cert-*.pem") + if err != nil { + return nil, fmt.Errorf("failed to create temp cert file: %w", err) + } + if _, err := certFile.Write(certPEM); err != nil { + certFile.Close() + os.Remove(certFile.Name()) + + return nil, fmt.Errorf("failed to write cert to file: %w", err) + } + + if err := certFile.Close(); err != nil { + os.Remove(certFile.Name()) + + return nil, fmt.Errorf("failed to close cert file: %w", err) + } + + // Create proxy handler + handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // Build target URL + targetURL := target + r.URL.Path + if r.URL.RawQuery != "" { + targetURL += "?" + r.URL.RawQuery + } + + // Create request to target + req, err := http.NewRequestWithContext(r.Context(), r.Method, targetURL, r.Body) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + + return + } + + // Copy headers + for key, values := range r.Header { + if key != "Host" && key != "Connection" { + for _, value := range values { + req.Header.Add(key, value) + } + } + } + + // Make request + client := &http.Client{Timeout: 30 * time.Second} + resp, err := client.Do(req) + if err != nil { + http.Error(w, err.Error(), http.StatusBadGateway) + + return + } + defer resp.Body.Close() + + // Copy response headers + for key, values := range resp.Header { + if key != "Connection" && key != "Transfer-Encoding" { + for _, value := range values { + w.Header().Add(key, value) + } + } + } + + // Copy status and body + w.WriteHeader(resp.StatusCode) + _, _ = io.Copy(w, resp.Body) + }) + + // Create HTTP server with TLS config (test-only proxy). + server := &http.Server{ + Handler: handler, + ReadHeaderTimeout: 10 * time.Second, + TLSConfig: &tls.Config{ + Certificates: []tls.Certificate{cert}, + MinVersion: tls.VersionTLS12, + }, + } + + // Try to listen on port 443 (requires root or CAP_NET_BIND_SERVICE for tests). + lc := net.ListenConfig{} + listener, err := lc.Listen(context.Background(), "tcp", ":443") //nolint:gosec // G102: test proxy must listen on 443 + if err != nil { + return nil, fmt.Errorf("failed to listen on port 443: %w (may require root or CAP_NET_BIND_SERVICE)", err) + } + + tlsListener := tls.NewListener(listener, server.TLSConfig) + + return &httpsProxyServer{ + server: server, + listener: tlsListener, + target: target, + certFile: certFile.Name(), + }, nil +} + +func (p *httpsProxyServer) Start() { + p.wg.Add(1) //nolint:modernize // standard sync.WaitGroup usage + + go func() { + defer p.wg.Done() + _ = p.server.Serve(p.listener) + }() +} + +func (p *httpsProxyServer) Stop() { + _ = p.listener.Close() + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + _ = p.server.Shutdown(ctx) + p.wg.Wait() +} + +var httpsProxy *httpsProxyServer //nolint:gochecknoglobals // Test fixture shared by TestMain. + +// setupHostsEntries adds entries to /etc/hosts to redirect Google API domains to localhost. +func setupHostsEntries() error { + entries := []string{ + "127.0.0.1 www.googleapis.com", + "127.0.0.1 storage.googleapis.com", + "127.0.0.1 oauth2.googleapis.com", + } + + for _, entry := range entries { + // Check if entry already exists. + //nolint:gosec // G204: test-only, fixed entries + cmd := exec.CommandContext(context.Background(), "grep", "-q", strings.Fields(entry)[1], "/etc/hosts") + if cmd.Run() == nil { + // Entry already exists, skip + continue + } + + // Add entry (requires privileges). + //nolint:gosec // G204: test-only, controlled entry + cmd = exec.CommandContext(context.Background(), "sh", "-c", fmt.Sprintf("echo '%s' >> /etc/hosts", entry)) + if err := cmd.Run(); err != nil { + return fmt.Errorf("failed to add %s to /etc/hosts: %w", entry, err) + } + } + + return nil +} + +// teardownHostsEntries removes entries from /etc/hosts that were added for the emulator. +func teardownHostsEntries() { + domains := []string{ + "www.googleapis.com", + "storage.googleapis.com", + "oauth2.googleapis.com", + } + + for _, domain := range domains { + // Remove entry (requires privileges). + //nolint:gosec // G204: test-only, fixed domains + pattern := fmt.Sprintf("/%s/d", strings.ReplaceAll(domain, ".", "\\.")) + cmd := exec.CommandContext(context.Background(), "sed", "-i", pattern, "/etc/hosts") + _ = cmd.Run() // Ignore errors - entry might not exist + } +} + +// TestMain sets up and tears down the HTTPS proxy and /etc/hosts entries for all tests in this package. +// TestMain runs once before all tests and once after all tests complete. +// It applies to all test files in the same package (gcs_test package). +func TestMain(m *testing.M) { + // Setup /etc/hosts entries if GCSMOCK_ENDPOINT is set + if os.Getenv("GCSMOCK_ENDPOINT") != "" { + if err := setupHostsEntries(); err != nil { + fmt.Printf("Warning: Could not modify /etc/hosts: %v\n", err) + fmt.Printf("Tests may fail if /etc/hosts entries are not present\n") + } else { + fmt.Println("Added /etc/hosts entries for Google API domains") + } + } + + // Start HTTPS proxy before all tests if GCSMOCK_ENDPOINT is set + if os.Getenv("GCSMOCK_ENDPOINT") != "" { + endpoint := os.Getenv("GCSMOCK_ENDPOINT") + endpoint = strings.TrimSuffix(endpoint, "/") + target := endpoint + + var err error + httpsProxy, err = newHTTPSProxyServer(target) + if err != nil { + // Fail fast: with /etc/hosts redirecting Google domains to 127.0.0.1, + // OAuth/token calls will hit localhost:443 and fail with unclear errors + // if the proxy is not listening. Require the proxy to start. + fmt.Fprintf(os.Stderr, "Fatal: cannot start HTTPS proxy on port 443: %v\n", err) + fmt.Fprintf(os.Stderr, "This may require root or CAP_NET_BIND_SERVICE. Exiting.\n") + os.Exit(1) + } + httpsProxy.Start() + // Set SSL_CERT_FILE to trust our self-signed certificate + // This is respected by Go's crypto/x509 package when loading the system cert pool + // and will affect all TLS connections, including those made by OAuth2 clients + os.Setenv("SSL_CERT_FILE", httpsProxy.certFile) + fmt.Printf("HTTPS proxy started on port 443, certificate: %s\n", httpsProxy.certFile) + } + + // Run all tests + code := m.Run() + + // Stop proxy after all tests finish + if httpsProxy != nil { + httpsProxy.Stop() + fmt.Println("HTTPS proxy stopped") + httpsProxy = nil + } + + // Cleanup /etc/hosts entries + if os.Getenv("GCSMOCK_ENDPOINT") != "" { + teardownHostsEntries() + fmt.Println("Removed /etc/hosts entries for Google API domains") + } + + os.Exit(code) +} + +func ensureDummyGCSCreds(t *testing.T) { + t.Helper() + + if os.Getenv("GCSMOCK_ENDPOINT") != "" { + credsFile := path.Join(t.TempDir(), "dummy_creds.json") + + priv, err := rsa.GenerateKey(rand.Reader, 2048) + if err != nil { + t.Fatal(err) + } + + privBytes, err := x509.MarshalPKCS8PrivateKey(priv) + if err != nil { + t.Fatal(err) + } + + privPEM := pem.EncodeToMemory(&pem.Block{ + Type: "PRIVATE KEY", + Bytes: privBytes, + }) + + content := fmt.Sprintf(`{"type": "service_account", "project_id": "test-project", `+ + `"client_email": "test@test.com", "private_key": %q}`, string(privPEM)) + err = os.WriteFile(credsFile, []byte(content), 0o600) + if err != nil { + t.Fatal(err) + } + + t.Setenv("GOOGLE_APPLICATION_CREDENTIALS", credsFile) + } +} + +func cleanupStorage(store driver.StorageDriver, name string) { + _ = store.Delete(context.Background(), name) +} + +// createObjectsStore creates a GCS-backed store; dedupe is always true at call sites. +// +//nolint:unparam +func createObjectsStore(rootDir string, cacheDir string, dedupe bool) ( + driver.StorageDriver, + storageTypes.ImageStore, + error, +) { + bucket := "zot-storage-test" + + endpoint := os.Getenv("GCSMOCK_ENDPOINT") + if endpoint == "" { + return nil, nil, errGCSMockEndpointNotSet + } + + url := strings.TrimSuffix(endpoint, "/") + "/storage/v1/b?project=test-project" + body := fmt.Sprintf(`{"name": "%s"}`, bucket) + req, err := http.NewRequestWithContext(context.Background(), http.MethodPost, url, strings.NewReader(body)) + if err != nil { + return nil, nil, err + } + req.Header.Set("Content-Type", "application/json") + resp, err := http.DefaultClient.Do(req) //nolint:gosec // G107: Test mock + if err != nil { + return nil, nil, err + } + defer resp.Body.Close() + + // Check if bucket was created successfully or already exists. + okStatus := resp.StatusCode == http.StatusOK || + resp.StatusCode == http.StatusCreated || + resp.StatusCode == http.StatusConflict + if !okStatus { + respBody, _ := io.ReadAll(resp.Body) + + return nil, nil, fmt.Errorf("%w %s: status %d body %s", + errBucketCreateFailed, bucket, resp.StatusCode, string(respBody)) + } + + storageDriverParams := map[string]any{ + "rootDir": rootDir, + "name": "gcs", + "bucket": bucket, + } + + storeName := fmt.Sprintf("%v", storageDriverParams["name"]) + + store, err := factory.Create(context.Background(), storeName, storageDriverParams) + if err != nil { + return nil, nil, err + } + + log := log.NewTestLogger() + metrics := monitoring.NewMetricsServer(false, log) + + var cacheDriver storageTypes.Cache + + // from pkg/cli/server/root.go/applyDefaultValues, s3 magic + s3CacheDBPath := path.Join(cacheDir, storageConstants.BoltdbName+storageConstants.DBExtensionName) + + if _, err := os.Stat(s3CacheDBPath); dedupe || (!dedupe && err == nil) { + cacheDriver, _ = storage.Create("boltdb", cache.BoltDBDriverParameters{ + RootDir: cacheDir, + Name: "cache", + UseRelPaths: false, + }, log) + } + + il := gcs.NewImageStore(rootDir, cacheDir, dedupe, false, log, metrics, nil, store, cacheDriver, nil, nil) + + return store, il, nil +} + +func TestGCSDriver(t *testing.T) { + tskip.SkipGCS(t) + ensureDummyGCSCreds(t) + + uuid, err := guuid.NewV4() + if err != nil { + panic(err) + } + + testDir := path.Join("/oci-repo-test", uuid.String()) + + Convey("GCS Driver E2E", t, func() { + // Create a fresh temp dir for each run to avoid BoltDB lock issues + tdir := t.TempDir() + storeDriver, imgStore, err := createObjectsStore(testDir, tdir, true) + So(err, ShouldBeNil) + defer cleanupStorage(storeDriver, testDir) + + Convey("Init Repo", func() { + repoName := "test-repo-init" + err := imgStore.InitRepo(repoName) + So(err, ShouldBeNil) + + isValid, err := imgStore.ValidateRepo(repoName) + So(err, ShouldBeNil) + So(isValid, ShouldBeTrue) + }) + + Convey("Push and Pull Image", func() { + repoName := "test-repo-push" + image := CreateDefaultImage() + + // Upload layers + for _, content := range image.Layers { + upload, err := imgStore.NewBlobUpload(repoName) + So(err, ShouldBeNil) + + buf := bytes.NewBuffer(content) + buflen := buf.Len() + digest := godigest.FromBytes(content) + + blob, err := imgStore.PutBlobChunkStreamed(repoName, upload, buf) + So(err, ShouldBeNil) + So(blob, ShouldEqual, buflen) + + err = imgStore.FinishBlobUpload(repoName, upload, buf, digest) + So(err, ShouldBeNil) + } + + // Upload config + cblob, err := json.Marshal(image.Config) + So(err, ShouldBeNil) + cdigest := godigest.FromBytes(cblob) + _, _, err = imgStore.FullBlobUpload(repoName, bytes.NewBuffer(cblob), cdigest) + So(err, ShouldBeNil) + + // Upload manifest + mblob, err := json.Marshal(image.Manifest) + So(err, ShouldBeNil) + _, _, err = imgStore.PutImageManifest(repoName, "1.0", ispec.MediaTypeImageManifest, mblob) + So(err, ShouldBeNil) + + // Verify manifest + _, _, _, err = imgStore.GetImageManifest(repoName, "1.0") + So(err, ShouldBeNil) + + // Verify blob + blobReadCloser, _, err := imgStore.GetBlob(repoName, cdigest, ispec.MediaTypeImageConfig) + So(err, ShouldBeNil) + defer blobReadCloser.Close() + content, err := io.ReadAll(blobReadCloser) + So(err, ShouldBeNil) + So(content, ShouldResemble, cblob) + }) + + Convey("Delete Image", func() { + repoName := "test-repo-delete" + // Setup image + image := CreateDefaultImage() + + // Upload layers first (required for manifest validation) + for _, content := range image.Layers { + upload, err := imgStore.NewBlobUpload(repoName) + So(err, ShouldBeNil) + + buf := bytes.NewBuffer(content) + buflen := buf.Len() + digest := godigest.FromBytes(content) + + blob, err := imgStore.PutBlobChunkStreamed(repoName, upload, buf) + So(err, ShouldBeNil) + So(blob, ShouldEqual, buflen) + + err = imgStore.FinishBlobUpload(repoName, upload, buf, digest) + So(err, ShouldBeNil) + } + + // Upload config + cblob, err := json.Marshal(image.Config) + So(err, ShouldBeNil) + cdigest := godigest.FromBytes(cblob) + _, _, err = imgStore.FullBlobUpload(repoName, bytes.NewBuffer(cblob), cdigest) + So(err, ShouldBeNil) + + // Upload manifest + mblob, err := json.Marshal(image.Manifest) + So(err, ShouldBeNil) + _, _, err = imgStore.PutImageManifest(repoName, "1.0", ispec.MediaTypeImageManifest, mblob) + So(err, ShouldBeNil) + + err = imgStore.DeleteImageManifest(repoName, "1.0", false) + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest(repoName, "1.0") + So(err, ShouldNotBeNil) + So(errors.Is(err, zerr.ErrManifestNotFound), ShouldBeTrue) + }) + }) +} + +func TestGCSDedupe(t *testing.T) { + tskip.SkipGCS(t) + ensureDummyGCSCreds(t) + + Convey("Dedupe", t, func(c C) { + uuid, err := guuid.NewV4() + if err != nil { + panic(err) + } + + testDir := path.Join("/oci-repo-test", uuid.String()) + + tdir := t.TempDir() + + storeDriver, imgStore, err := createObjectsStore(testDir, tdir, true) + So(err, ShouldBeNil) + defer cleanupStorage(storeDriver, testDir) + + // manifest1 + upload, err := imgStore.NewBlobUpload("dedupe1") + So(err, ShouldBeNil) + So(upload, ShouldNotBeEmpty) + + content := []byte("test-data3") + buf := bytes.NewBuffer(content) + buflen := buf.Len() + digest := godigest.FromBytes(content) + blob, err := imgStore.PutBlobChunkStreamed("dedupe1", upload, buf) + So(err, ShouldBeNil) + So(blob, ShouldEqual, buflen) + + blobDigest1 := digest + So(blobDigest1, ShouldNotBeEmpty) + + err = imgStore.FinishBlobUpload("dedupe1", upload, buf, digest) + So(err, ShouldBeNil) + So(blob, ShouldEqual, buflen) + + ok, checkBlobSize1, err := imgStore.CheckBlob("dedupe1", digest) + So(ok, ShouldBeTrue) + So(checkBlobSize1, ShouldBeGreaterThan, 0) + So(err, ShouldBeNil) + + ok, checkBlobSize1, _, err = imgStore.StatBlob("dedupe1", digest) + So(ok, ShouldBeTrue) + So(checkBlobSize1, ShouldBeGreaterThan, 0) + So(err, ShouldBeNil) + + blobReadCloser, getBlobSize1, err := imgStore.GetBlob("dedupe1", digest, + "application/vnd.oci.image.layer.v1.tar+gzip") + So(getBlobSize1, ShouldBeGreaterThan, 0) + So(err, ShouldBeNil) + err = blobReadCloser.Close() + So(err, ShouldBeNil) + + cblob, cdigest := GetRandomImageConfig() + _, clen, err := imgStore.FullBlobUpload("dedupe1", bytes.NewReader(cblob), cdigest) + So(err, ShouldBeNil) + So(clen, ShouldEqual, len(cblob)) + + hasBlob, _, err := imgStore.CheckBlob("dedupe1", cdigest) + So(err, ShouldBeNil) + So(hasBlob, ShouldEqual, true) + + manifest := ispec.Manifest{ + Config: ispec.Descriptor{ + MediaType: "application/vnd.oci.image.config.v1+json", + Digest: cdigest, + Size: int64(len(cblob)), + }, + Layers: []ispec.Descriptor{ + { + MediaType: "application/vnd.oci.image.layer.v1.tar", + Digest: digest, + Size: int64(buflen), + }, + }, + } + + manifest.SchemaVersion = 2 + manifestBuf, err := json.Marshal(manifest) + So(err, ShouldBeNil) + + manifestDigest := godigest.FromBytes(manifestBuf) + _, _, err = imgStore.PutImageManifest("dedupe1", manifestDigest.String(), + ispec.MediaTypeImageManifest, manifestBuf) + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("dedupe1", manifestDigest.String()) + So(err, ShouldBeNil) + + // manifest2 + upload, err = imgStore.NewBlobUpload("dedupe2") + So(err, ShouldBeNil) + So(upload, ShouldNotBeEmpty) + + content = []byte("test-data3") + buf = bytes.NewBuffer(content) + buflen = buf.Len() + digest = godigest.FromBytes(content) + + blob, err = imgStore.PutBlobChunkStreamed("dedupe2", upload, buf) + So(err, ShouldBeNil) + So(blob, ShouldEqual, buflen) + + blobDigest2 := digest + So(blobDigest2, ShouldNotBeEmpty) + + err = imgStore.FinishBlobUpload("dedupe2", upload, buf, digest) + So(err, ShouldBeNil) + So(blob, ShouldEqual, buflen) + + ok, checkBlobSize2, err := imgStore.CheckBlob("dedupe2", digest) + So(ok, ShouldBeTrue) + So(checkBlobSize2, ShouldBeGreaterThan, 0) + So(err, ShouldBeNil) + + ok, checkBlobSize2, _, err = imgStore.StatBlob("dedupe2", digest) + So(ok, ShouldBeTrue) + So(checkBlobSize2, ShouldBeGreaterThan, 0) + So(err, ShouldBeNil) + + blobReadCloser, getBlobSize2, err := imgStore.GetBlob("dedupe2", digest, + "application/vnd.oci.image.layer.v1.tar+gzip") + So(getBlobSize2, ShouldBeGreaterThan, 0) + So(err, ShouldBeNil) + err = blobReadCloser.Close() + So(err, ShouldBeNil) + + cblob, cdigest = GetRandomImageConfig() + _, clen, err = imgStore.FullBlobUpload("dedupe2", bytes.NewReader(cblob), cdigest) + So(err, ShouldBeNil) + So(clen, ShouldEqual, len(cblob)) + + hasBlob, _, err = imgStore.CheckBlob("dedupe2", cdigest) + So(err, ShouldBeNil) + So(hasBlob, ShouldEqual, true) + + manifest = ispec.Manifest{ + Config: ispec.Descriptor{ + MediaType: "application/vnd.oci.image.config.v1+json", + Digest: cdigest, + Size: int64(len(cblob)), + }, + Layers: []ispec.Descriptor{ + { + MediaType: "application/vnd.oci.image.layer.v1.tar", + Digest: digest, + Size: int64(buflen), + }, + }, + } + + manifest.SchemaVersion = 2 + manifestBuf, err = json.Marshal(manifest) + So(err, ShouldBeNil) + + manifestDigest = godigest.FromBytes(manifestBuf) + _, _, err = imgStore.PutImageManifest("dedupe2", manifestDigest.String(), + ispec.MediaTypeImageManifest, manifestBuf) + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("dedupe2", manifestDigest.String()) + So(err, ShouldBeNil) + + So(blobDigest1, ShouldEqual, blobDigest2) + So(checkBlobSize1, ShouldEqual, checkBlobSize2) + So(getBlobSize1, ShouldEqual, getBlobSize2) + }) +} + +func TestGCSPullRange(t *testing.T) { + tskip.SkipGCS(t) + ensureDummyGCSCreds(t) + + Convey("Pull range", t, func(c C) { + uuid, err := guuid.NewV4() + if err != nil { + panic(err) + } + + testDir := path.Join("/oci-repo-test", uuid.String()) + + tdir := t.TempDir() + + storeDriver, imgStore, err := createObjectsStore(testDir, tdir, true) + So(err, ShouldBeNil) + defer cleanupStorage(storeDriver, testDir) + + upload, err := imgStore.NewBlobUpload("test") + So(err, ShouldBeNil) + So(upload, ShouldNotBeEmpty) + + content := []byte("test-data3") + buf := bytes.NewBuffer(content) + buflen := buf.Len() + digest := godigest.FromBytes(content) + blob, err := imgStore.PutBlobChunkStreamed("test", upload, buf) + So(err, ShouldBeNil) + So(blob, ShouldEqual, buflen) + + err = imgStore.FinishBlobUpload("test", upload, buf, digest) + So(err, ShouldBeNil) + + blobReadCloser, _, err := imgStore.GetBlob("test", digest, "application/vnd.oci.image.layer.v1.tar+gzip") + So(err, ShouldBeNil) + err = blobReadCloser.Close() + So(err, ShouldBeNil) + + // get range + blobReadCloser, _, _, err = imgStore.GetBlobPartial("test", digest, + "application/vnd.oci.image.layer.v1.tar+gzip", 0, 4) + So(err, ShouldBeNil) + buf.Reset() + _, err = buf.ReadFrom(blobReadCloser) + So(err, ShouldBeNil) + So(buf.String(), ShouldEqual, "test-") + err = blobReadCloser.Close() + So(err, ShouldBeNil) + + // get range - "data3" is bytes 5-9 (inclusive) of "test-data3" + blobReadCloser, _, _, err = imgStore.GetBlobPartial("test", digest, + "application/vnd.oci.image.layer.v1.tar+gzip", 5, 9) + So(err, ShouldBeNil) + buf.Reset() + _, err = buf.ReadFrom(blobReadCloser) + So(err, ShouldBeNil) + So(buf.String(), ShouldEqual, "data3") + err = blobReadCloser.Close() + So(err, ShouldBeNil) + + // get range from negative offset + blobReadCloser, _, _, err = imgStore.GetBlobPartial("test", digest, + "application/vnd.oci.image.layer.v1.tar+gzip", -4, 4) + So(err, ShouldNotBeNil) + So(blobReadCloser, ShouldBeNil) + }) +} + +func TestGCSGetAllDedupeReposCandidates(t *testing.T) { + tskip.SkipGCS(t) + ensureDummyGCSCreds(t) + + uuid, err := guuid.NewV4() + if err != nil { + panic(err) + } + + testDir := path.Join("/oci-repo-test", uuid.String()) + tdir := t.TempDir() + + storeDriver, imgStore, err := createObjectsStore(testDir, tdir, true) + if err != nil { + panic(err) + } + + defer cleanupStorage(storeDriver, testDir) + + Convey("Push repos with deduped blobs", t, func(c C) { + repoNames := []string{ + "first", + "second", + "repo/a", + "repo/a/b/c/d/e/f", + "repo/repo-b/blobs", + "foo/bar/baz", + "blobs/foo/bar/blobs", + "blobs", + "blobs/foo", + } + + storeController := storage.StoreController{DefaultStore: imgStore} + + image := CreateRandomImage() + + for _, repoName := range repoNames { + err := WriteImageToFileSystem(image, repoName, tag, storeController) + So(err, ShouldBeNil) + } + + randomBlobDigest := image.Manifest.Layers[0].Digest + + repos, err := imgStore.GetAllDedupeReposCandidates(randomBlobDigest) + So(err, ShouldBeNil) + slices.Sort(repoNames) + slices.Sort(repos) + So(repoNames, ShouldResemble, repos) + }) +} + +func TestGCSDeleteBlobsInUse(t *testing.T) { + tskip.SkipGCS(t) + ensureDummyGCSCreds(t) + + Convey("Setup manifest", t, func() { + uuid, err := guuid.NewV4() + if err != nil { + panic(err) + } + + testDir := path.Join("/oci-repo-test", uuid.String()) + tdir := t.TempDir() + + storeDriver, imgStore, err := createObjectsStore(testDir, tdir, true) + So(err, ShouldBeNil) + defer cleanupStorage(storeDriver, testDir) + // put an unused blob + content := []byte("unused blob") + buf := bytes.NewBuffer(content) + unusedDigest := godigest.FromBytes(content) + + _, _, err = imgStore.FullBlobUpload("repo", bytes.NewReader(buf.Bytes()), unusedDigest) + So(err, ShouldBeNil) + + content = []byte("test-data1") + buf = bytes.NewBuffer(content) + buflen := buf.Len() + digest := godigest.FromBytes(content) + + _, _, err = imgStore.FullBlobUpload("repo", bytes.NewReader(buf.Bytes()), digest) + So(err, ShouldBeNil) + + cblob, cdigest := GetRandomImageConfig() + + var clen int64 + _, clen, err = imgStore.FullBlobUpload("repo", bytes.NewReader(cblob), cdigest) + So(err, ShouldBeNil) + So(clen, ShouldEqual, len(cblob)) + + annotationsMap := make(map[string]string) + annotationsMap[ispec.AnnotationRefName] = tag + + manifest := ispec.Manifest{ + Config: ispec.Descriptor{ + MediaType: "application/vnd.oci.image.config.v1+json", + Digest: cdigest, + Size: int64(len(cblob)), + }, + Layers: []ispec.Descriptor{ + { + MediaType: "application/vnd.oci.image.layer.v1.tar", + Digest: digest, + Size: int64(buflen), + }, + }, + Annotations: annotationsMap, + } + + manifest.SchemaVersion = 2 + manifestBuf, err := json.Marshal(manifest) + So(err, ShouldBeNil) + + manifestDigest, _, err := imgStore.PutImageManifest("repo", tag, ispec.MediaTypeImageManifest, manifestBuf) + So(err, ShouldBeNil) + + Convey("Try to delete blob currently in use", func() { + // layer blob + err := imgStore.DeleteBlob("repo", digest) + So(err, ShouldEqual, zerr.ErrBlobReferenced) + + // manifest + err = imgStore.DeleteBlob("repo", manifestDigest) + So(err, ShouldEqual, zerr.ErrBlobReferenced) + + // config + err = imgStore.DeleteBlob("repo", cdigest) + So(err, ShouldEqual, zerr.ErrBlobReferenced) + }) + + Convey("Delete unused blob", func() { + err := imgStore.DeleteBlob("repo", unusedDigest) + So(err, ShouldBeNil) + }) + + Convey("Delete manifest first, then blob", func() { + err := imgStore.DeleteImageManifest("repo", manifestDigest.String(), false) + So(err, ShouldBeNil) + + err = imgStore.DeleteBlob("repo", digest) + So(err, ShouldBeNil) + + // config + err = imgStore.DeleteBlob("repo", cdigest) + So(err, ShouldBeNil) + }) + }) +} + +func TestGCSStorageAPIs(t *testing.T) { + tskip.SkipGCS(t) + ensureDummyGCSCreds(t) + + uuid, err := guuid.NewV4() + if err != nil { + panic(err) + } + + testDir := path.Join("/oci-repo-test", uuid.String()) + tdir := t.TempDir() + + storeDriver, imgStore, err := createObjectsStore(testDir, tdir, true) + if err != nil { + panic(err) + } + + defer cleanupStorage(storeDriver, testDir) + + Convey("Repo layout", t, func(c C) { + repoName := "test" + + Convey("Get all blobs from repo without initialization", func() { + allBlobs, err := imgStore.GetAllBlobs(repoName) + So(err, ShouldBeNil) + So(allBlobs, ShouldBeEmpty) + + ok := imgStore.DirExists(path.Join(imgStore.RootDir(), repoName)) + So(ok, ShouldBeFalse) + }) + + Convey("Validate repo without initialization", func() { + v, err := imgStore.ValidateRepo(repoName) + So(v, ShouldEqual, false) + So(err, ShouldNotBeNil) + + ok := imgStore.DirExists(path.Join(imgStore.RootDir(), repoName)) + So(ok, ShouldBeFalse) + }) + + Convey("Initialize repo", func() { + err := imgStore.InitRepo(repoName) + So(err, ShouldBeNil) + + ok := imgStore.DirExists(path.Join(imgStore.RootDir(), repoName)) + So(ok, ShouldBeTrue) + + storeController := storage.StoreController{} + storeController.DefaultStore = imgStore + So(storeController.GetImageStore("test"), ShouldResemble, imgStore) + }) + + Convey("Validate repo", func() { + repos, err := imgStore.ValidateRepo(repoName) + So(err, ShouldBeNil) + So(repos, ShouldEqual, true) + }) + + Convey("Get repos", func() { + repos, err := imgStore.GetRepositories() + So(err, ShouldBeNil) + So(repos, ShouldNotBeEmpty) + + repos, more, err := imgStore.GetNextRepositories("", -1, func(repo string) (bool, error) { + return true, nil + }) + + So(more, ShouldBeFalse) + So(err, ShouldBeNil) + So(repos, ShouldNotBeEmpty) + }) + + Convey("Get image tags", func() { + v, err := imgStore.GetImageTags("test") + So(err, ShouldBeNil) + So(v, ShouldBeEmpty) + }) + + Convey("Full blob upload unavailable algorithm", func() { + body := []byte("this blob will be hashed using an unavailable hashing algorithm") + buf := bytes.NewBuffer(body) + digest := godigest.Digest("md5:8114c3f59ef9dcf737410e0f4b00a154") + upload, n, err := imgStore.FullBlobUpload("test", buf, digest) + So(err, ShouldEqual, godigest.ErrDigestUnsupported) + So(n, ShouldEqual, -1) + So(upload, ShouldEqual, "") + + // Check no blobs are returned and there are no errors + // if other paths for different algorithms are missing + digests, err := imgStore.GetAllBlobs("test") + So(err, ShouldBeNil) + So(digests, ShouldBeEmpty) + }) + + Convey("Full blob upload", func() { + body := []byte("this is a blob") + buf := bytes.NewBuffer(body) + digest := godigest.FromBytes(body) + upload, n, err := imgStore.FullBlobUpload("test", buf, digest) + So(err, ShouldBeNil) + So(n, ShouldEqual, len(body)) + So(upload, ShouldNotBeEmpty) + + err = imgStore.VerifyBlobDigestValue("test", digest) + So(err, ShouldBeNil) + + // Check the blob is returned and there are no errors + // if other paths for different algorithms are missing + digests, err := imgStore.GetAllBlobs("test") + So(err, ShouldBeNil) + So(digests, ShouldContain, digest) + So(len(digests), ShouldEqual, 1) + }) + + Convey("Full blob upload sha512", func() { + body := []byte("this blob will be hashed using sha512") + buf := bytes.NewBuffer(body) + digest := godigest.SHA512.FromBytes(body) + upload, n, err := imgStore.FullBlobUpload("test", buf, digest) + So(err, ShouldBeNil) + So(n, ShouldEqual, len(body)) + So(upload, ShouldNotBeEmpty) + + // Check the blob is returned and there are no errors + // if other paths for different algorithms are missing + digests, err := imgStore.GetAllBlobs("test") + So(err, ShouldBeNil) + So(digests, ShouldContain, digest) + // imgStore is reused so look for this digest and + // the ones uploaded by previous tests + So(len(digests), ShouldEqual, 2) + }) + + Convey("Full blob upload sha384", func() { + body := []byte("this blob will be hashed using sha384") + buf := bytes.NewBuffer(body) + digest := godigest.SHA384.FromBytes(body) + upload, n, err := imgStore.FullBlobUpload("test", buf, digest) + So(err, ShouldBeNil) + So(n, ShouldEqual, len(body)) + So(upload, ShouldNotBeEmpty) + + // Check the blob is returned and there are no errors + // if other paths for different algorithms are missing + digests, err := imgStore.GetAllBlobs("test") + So(err, ShouldBeNil) + So(digests, ShouldContain, digest) + // imgStore is reused so look for this digest and + // the ones uploaded by previous tests + So(len(digests), ShouldEqual, 3) + }) + + Convey("New blob upload", func() { + upload, err := imgStore.NewBlobUpload("test") + So(err, ShouldBeNil) + So(upload, ShouldNotBeEmpty) + + err = imgStore.DeleteBlobUpload("test", upload) + So(err, ShouldBeNil) + + upload, err = imgStore.NewBlobUpload("test") + So(err, ShouldBeNil) + So(upload, ShouldNotBeEmpty) + + Convey("Get blob upload", func() { + bupload, err := imgStore.GetBlobUpload("test", "invalid") + So(err, ShouldNotBeNil) + So(bupload, ShouldEqual, -1) + + bupload, err = imgStore.GetBlobUpload("hi", " \255") + So(err, ShouldNotBeNil) + So(bupload, ShouldEqual, -1) + + bupload, err = imgStore.GetBlobUpload("test", upload) + So(err, ShouldBeNil) + So(bupload, ShouldBeGreaterThanOrEqualTo, 0) + + bupload, err = imgStore.BlobUploadInfo("test", upload) + So(err, ShouldBeNil) + So(bupload, ShouldBeGreaterThanOrEqualTo, 0) + + content := []byte("test-data1") + firstChunkContent := []byte("test") + firstChunkBuf := bytes.NewBuffer(firstChunkContent) + secondChunkContent := []byte("-data1") + secondChunkBuf := bytes.NewBuffer(secondChunkContent) + firstChunkLen := firstChunkBuf.Len() + secondChunkLen := secondChunkBuf.Len() + + buf := bytes.NewBuffer(content) + buflen := buf.Len() + digest := godigest.FromBytes(content) + blobDigest := digest + + // invalid chunk range + _, err = imgStore.PutBlobChunk("test", upload, 10, int64(buflen), buf) + So(err, ShouldNotBeNil) + + bupload, err = imgStore.PutBlobChunk("test", upload, 0, int64(firstChunkLen), firstChunkBuf) + So(err, ShouldBeNil) + So(bupload, ShouldEqual, firstChunkLen) + + bupload, err = imgStore.GetBlobUpload("test", upload) + So(err, ShouldBeNil) + So(bupload, ShouldEqual, int64(firstChunkLen)) + + bupload, err = imgStore.BlobUploadInfo("test", upload) + So(err, ShouldBeNil) + So(bupload, ShouldEqual, int64(firstChunkLen)) + + bupload, err = imgStore.PutBlobChunk("test", upload, int64(firstChunkLen), int64(buflen), secondChunkBuf) + So(err, ShouldBeNil) + So(bupload, ShouldEqual, int64(firstChunkLen+secondChunkLen)) + + err = imgStore.FinishBlobUpload("test", upload, buf, digest) + So(err, ShouldBeNil) + + _, _, err = imgStore.CheckBlob("test", digest) + So(err, ShouldBeNil) + + ok, _, _, err := imgStore.StatBlob("test", digest) + So(ok, ShouldBeTrue) + So(err, ShouldBeNil) + + blob, _, err := imgStore.GetBlob("test", digest, "application/vnd.oci.image.layer.v1.tar+gzip") + So(err, ShouldBeNil) + + blobBuf := new(strings.Builder) + n, err := io.Copy(blobBuf, blob) + // check errors + So(n, ShouldEqual, buflen) + So(err, ShouldBeNil) + So(blobBuf.String(), ShouldEqual, buf.String()) + + blobContent, err := imgStore.GetBlobContent("test", digest) + So(err, ShouldBeNil) + So(blobContent, ShouldResemble, content) + + err = blob.Close() + So(err, ShouldBeNil) + + manifest := ispec.Manifest{} + manifest.SchemaVersion = 2 + manifestBuf, err := json.Marshal(manifest) + So(err, ShouldBeNil) + + Convey("Bad image manifest", func() { + _, _, err = imgStore.PutImageManifest("test", digest.String(), "application/json", + manifestBuf) + So(err, ShouldNotBeNil) + + _, _, err = imgStore.PutImageManifest("test", digest.String(), ispec.MediaTypeImageManifest, + []byte{}) + So(err, ShouldNotBeNil) + + _, _, err = imgStore.PutImageManifest("test", digest.String(), ispec.MediaTypeImageManifest, + []byte(`{"test":true}`)) + So(err, ShouldNotBeNil) + + _, _, err = imgStore.PutImageManifest("test", digest.String(), ispec.MediaTypeImageManifest, + manifestBuf) + So(err, ShouldNotBeNil) + + _, _, _, err = imgStore.GetImageManifest("test", digest.String()) + So(err, ShouldNotBeNil) + + _, _, _, err = imgStore.GetImageManifest("inexistent", digest.String()) + So(err, ShouldNotBeNil) + }) + + Convey("Good image manifest", func() { + cblob, cdigest := GetRandomImageConfig() + _, clen, err := imgStore.FullBlobUpload("test", bytes.NewReader(cblob), cdigest) + So(err, ShouldBeNil) + So(clen, ShouldEqual, len(cblob)) + + hasBlob, _, err := imgStore.CheckBlob("test", cdigest) + So(err, ShouldBeNil) + So(hasBlob, ShouldEqual, true) + + annotationsMap := make(map[string]string) + annotationsMap[ispec.AnnotationRefName] = "1.0" + manifest := ispec.Manifest{ + Config: ispec.Descriptor{ + MediaType: "application/vnd.oci.image.config.v1+json", + Digest: cdigest, + Size: int64(len(cblob)), + }, + Layers: []ispec.Descriptor{ + { + MediaType: "application/vnd.oci.image.layer.v1.tar", + Digest: digest, + Size: int64(buflen), + }, + }, + Annotations: annotationsMap, + } + + manifest.SchemaVersion = 2 + manifestBuf, err = json.Marshal(manifest) + So(err, ShouldBeNil) + + digest := godigest.FromBytes(manifestBuf) + + // bad manifest + manifest.Layers[0].Digest = godigest.FromBytes([]byte("inexistent")) + badMb, err := json.Marshal(manifest) + So(err, ShouldBeNil) + + _, _, err = imgStore.PutImageManifest("test", "1.0", ispec.MediaTypeImageManifest, badMb) + So(err, ShouldNotBeNil) + + _, _, err = imgStore.PutImageManifest("test", "1.0", ispec.MediaTypeImageManifest, manifestBuf) + So(err, ShouldBeNil) + + // same manifest for coverage + _, _, err = imgStore.PutImageManifest("test", "1.0", ispec.MediaTypeImageManifest, manifestBuf) + So(err, ShouldBeNil) + + _, _, err = imgStore.PutImageManifest("test", "2.0", ispec.MediaTypeImageManifest, manifestBuf) + So(err, ShouldBeNil) + + _, _, err = imgStore.PutImageManifest("test", "3.0", ispec.MediaTypeImageManifest, manifestBuf) + So(err, ShouldBeNil) + + _, err = imgStore.GetImageTags("inexistent") + So(err, ShouldNotBeNil) + + // total tags should be 3 but they have same reference. + tags, err := imgStore.GetImageTags("test") + So(err, ShouldBeNil) + So(len(tags), ShouldEqual, 3) + + _, _, _, err = imgStore.GetImageManifest("test", digest.String()) + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("test", "3.0") + So(err, ShouldBeNil) + + err = imgStore.DeleteImageManifest("test", "1.0", false) + So(err, ShouldBeNil) + + tags, err = imgStore.GetImageTags("test") + So(err, ShouldBeNil) + So(len(tags), ShouldEqual, 2) + + repos, err := imgStore.GetRepositories() + So(err, ShouldBeNil) + So(len(repos), ShouldEqual, 1) + So(repos[0], ShouldEqual, "test") + + repos, more, err := imgStore.GetNextRepositories("", -1, func(repo string) (bool, error) { + return true, nil + }) + So(err, ShouldBeNil) + So(more, ShouldBeFalse) + So(len(repos), ShouldEqual, 1) + So(repos[0], ShouldEqual, "test") + + repos, more, err = imgStore.GetNextRepositories("", -1, func(repo string) (bool, error) { + return false, nil + }) + So(err, ShouldBeNil) + So(more, ShouldBeFalse) + So(len(repos), ShouldEqual, 0) + + // We deleted only one tag, make sure blob should not be removed. + hasBlob, _, err = imgStore.CheckBlob("test", digest) + So(err, ShouldBeNil) + So(hasBlob, ShouldEqual, true) + + // with detectManifestCollision should get error + err = imgStore.DeleteImageManifest("test", digest.String(), true) + So(err, ShouldNotBeNil) + + // If we pass reference all manifest with input reference should be deleted. + err = imgStore.DeleteImageManifest("test", digest.String(), false) + So(err, ShouldBeNil) + + tags, err = imgStore.GetImageTags("test") + So(err, ShouldBeNil) + So(len(tags), ShouldEqual, 0) + + // All tags/references are deleted, blob should not be present in disk. + hasBlob, _, err = imgStore.CheckBlob("test", digest) + So(err, ShouldNotBeNil) + So(hasBlob, ShouldEqual, false) + + hasBlob, _, _, err = imgStore.StatBlob("test", digest) + So(err, ShouldNotBeNil) + So(hasBlob, ShouldEqual, false) + + err = imgStore.DeleteBlob("test", "inexistent") + So(err, ShouldNotBeNil) + + err = imgStore.DeleteBlob("test", godigest.FromBytes([]byte("inexistent"))) + So(err, ShouldNotBeNil) + + err = imgStore.DeleteBlob("test", blobDigest) + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("test", digest.String()) + So(err, ShouldNotBeNil) + }) + }) + + err = imgStore.DeleteBlobUpload("test", upload) + So(err, ShouldNotBeNil) + }) + + Convey("New blob upload streamed", func() { + bupload, err := imgStore.NewBlobUpload("test") + So(err, ShouldBeNil) + So(bupload, ShouldNotBeEmpty) + + Convey("Get blob upload", func() { + upload, err := imgStore.GetBlobUpload("test", "invalid") + So(err, ShouldNotBeNil) + So(upload, ShouldEqual, -1) + + upload, err = imgStore.GetBlobUpload("test", bupload) + So(err, ShouldBeNil) + So(upload, ShouldBeGreaterThanOrEqualTo, 0) + + _, err = imgStore.BlobUploadInfo("test", "inexistent") + So(err, ShouldNotBeNil) + + upload, err = imgStore.BlobUploadInfo("test", bupload) + So(err, ShouldBeNil) + So(upload, ShouldBeGreaterThanOrEqualTo, 0) + + content := []byte("test-data2") + buf := bytes.NewBuffer(content) + buflen := buf.Len() + digest := godigest.FromBytes(content) + upload, err = imgStore.PutBlobChunkStreamed("test", bupload, buf) + So(err, ShouldBeNil) + So(upload, ShouldEqual, buflen) + + _, err = imgStore.PutBlobChunkStreamed("test", "inexistent", buf) + So(err, ShouldNotBeNil) + + err = imgStore.FinishBlobUpload("test", "inexistent", buf, digest) + So(err, ShouldNotBeNil) + + // invalid digest + err = imgStore.FinishBlobUpload("test", "inexistent", buf, "sha256:invalid") + So(err, ShouldNotBeNil) + + err = imgStore.FinishBlobUpload("test", bupload, buf, digest) + So(err, ShouldBeNil) + + ok, _, err := imgStore.CheckBlob("test", digest) + So(ok, ShouldBeTrue) + So(err, ShouldBeNil) + + ok, _, _, err = imgStore.StatBlob("test", digest) + So(ok, ShouldBeTrue) + So(err, ShouldBeNil) + + _, _, err = imgStore.GetBlob("test", "inexistent", "application/vnd.oci.image.layer.v1.tar+gzip") + So(err, ShouldNotBeNil) + + blob, _, err := imgStore.GetBlob("test", digest, "application/vnd.oci.image.layer.v1.tar+gzip") + So(err, ShouldBeNil) + err = blob.Close() + So(err, ShouldBeNil) + + blobContent, err := imgStore.GetBlobContent("test", digest) + So(err, ShouldBeNil) + So(content, ShouldResemble, blobContent) + + _, err = imgStore.GetBlobContent("inexistent", digest) + So(err, ShouldNotBeNil) + + manifest := ispec.Manifest{} + manifest.SchemaVersion = 2 + manifestBuf, err := json.Marshal(manifest) + So(err, ShouldBeNil) + + Convey("Bad digests", func() { + _, _, err := imgStore.FullBlobUpload("test", bytes.NewBuffer([]byte{}), "inexistent") + So(err, ShouldNotBeNil) + + _, _, err = imgStore.CheckBlob("test", "inexistent") + So(err, ShouldNotBeNil) + + _, _, _, err = imgStore.StatBlob("test", "inexistent") + So(err, ShouldNotBeNil) + }) + + Convey("Bad image manifest", func() { + _, _, err = imgStore.PutImageManifest("test", digest.String(), + ispec.MediaTypeImageManifest, manifestBuf) + So(err, ShouldNotBeNil) + + _, _, err = imgStore.PutImageManifest("test", digest.String(), + ispec.MediaTypeImageManifest, []byte("bad json")) + So(err, ShouldNotBeNil) + + _, _, _, err = imgStore.GetImageManifest("test", digest.String()) + So(err, ShouldNotBeNil) + }) + + Convey("Good image manifest", func() { + cblob, cdigest := GetRandomImageConfig() + _, clen, err := imgStore.FullBlobUpload("test", bytes.NewReader(cblob), cdigest) + So(err, ShouldBeNil) + So(clen, ShouldEqual, len(cblob)) + + hasBlob, _, err := imgStore.CheckBlob("test", cdigest) + So(err, ShouldBeNil) + So(hasBlob, ShouldEqual, true) + + manifest := ispec.Manifest{ + Config: ispec.Descriptor{ + MediaType: "application/vnd.oci.image.config.v1+json", + Digest: cdigest, + Size: int64(len(cblob)), + }, + Layers: []ispec.Descriptor{ + { + MediaType: "application/vnd.oci.image.layer.v1.tar", + Digest: digest, + Size: int64(buflen), + }, + }, + } + manifest.SchemaVersion = 2 + manifestBuf, err = json.Marshal(manifest) + So(err, ShouldBeNil) + + digest := godigest.FromBytes(manifestBuf) + _, _, err = imgStore.PutImageManifest("test", digest.String(), + ispec.MediaTypeImageManifest, manifestBuf) + So(err, ShouldBeNil) + + // same manifest for coverage + _, _, err = imgStore.PutImageManifest("test", digest.String(), + ispec.MediaTypeImageManifest, manifestBuf) + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("test", digest.String()) + So(err, ShouldBeNil) + + _, err = imgStore.GetIndexContent("inexistent") + So(err, ShouldNotBeNil) + + indexContent, err := imgStore.GetIndexContent("test") + So(err, ShouldBeNil) + + var index ispec.Index + + err = json.Unmarshal(indexContent, &index) + So(err, ShouldBeNil) + + So(len(index.Manifests), ShouldEqual, 1) + + err = imgStore.DeleteImageManifest("test", "1.0", false) + So(err, ShouldNotBeNil) + + err = imgStore.DeleteImageManifest("inexistent", "1.0", false) + So(err, ShouldNotBeNil) + + err = imgStore.DeleteImageManifest("test", digest.String(), false) + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("test", digest.String()) + So(err, ShouldNotBeNil) + }) + }) + + err = imgStore.DeleteBlobUpload("test", bupload) + So(err, ShouldNotBeNil) + }) + + Convey("Modify manifest in-place", func() { + // original blob + upload, err := imgStore.NewBlobUpload("replace") + So(err, ShouldBeNil) + So(upload, ShouldNotBeEmpty) + + content := []byte("test-data-replace-1") + buf := bytes.NewBuffer(content) + buflen := buf.Len() + digest := godigest.FromBytes(content) + blob, err := imgStore.PutBlobChunkStreamed("replace", upload, buf) + So(err, ShouldBeNil) + So(blob, ShouldEqual, buflen) + + blobDigest1 := strings.Split(digest.String(), ":")[1] + So(blobDigest1, ShouldNotBeEmpty) + + err = imgStore.FinishBlobUpload("replace", upload, buf, digest) + So(err, ShouldBeNil) + So(blob, ShouldEqual, buflen) + + cblob, cdigest := GetRandomImageConfig() + _, clen, err := imgStore.FullBlobUpload("replace", bytes.NewReader(cblob), cdigest) + So(err, ShouldBeNil) + So(clen, ShouldEqual, len(cblob)) + + hasBlob, _, err := imgStore.CheckBlob("replace", cdigest) + So(err, ShouldBeNil) + So(hasBlob, ShouldEqual, true) + + manifest := ispec.Manifest{ + Config: ispec.Descriptor{ + MediaType: "application/vnd.oci.image.config.v1+json", + Digest: cdigest, + Size: int64(len(cblob)), + }, + Layers: []ispec.Descriptor{ + { + MediaType: "application/vnd.oci.image.layer.v1.tar", + Digest: digest, + Size: int64(buflen), + }, + }, + } + manifest.SchemaVersion = 2 + manifestBuf, err := json.Marshal(manifest) + So(err, ShouldBeNil) + + digest = godigest.FromBytes(manifestBuf) + _, _, err = imgStore.PutImageManifest("replace", "1.0", ispec.MediaTypeImageManifest, manifestBuf) + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("replace", digest.String()) + So(err, ShouldBeNil) + + // new blob to replace + upload, err = imgStore.NewBlobUpload("replace") + So(err, ShouldBeNil) + So(upload, ShouldNotBeEmpty) + + content = []byte("test-data-replace-2") + buf = bytes.NewBuffer(content) + buflen = buf.Len() + digest = godigest.FromBytes(content) + blob, err = imgStore.PutBlobChunkStreamed("replace", upload, buf) + So(err, ShouldBeNil) + So(blob, ShouldEqual, buflen) + + blobDigest2 := strings.Split(digest.String(), ":")[1] + So(blobDigest2, ShouldNotBeEmpty) + + err = imgStore.FinishBlobUpload("replace", upload, buf, digest) + So(err, ShouldBeNil) + So(blob, ShouldEqual, buflen) + + cblob, cdigest = GetRandomImageConfig() + _, clen, err = imgStore.FullBlobUpload("replace", bytes.NewReader(cblob), cdigest) + So(err, ShouldBeNil) + So(clen, ShouldEqual, len(cblob)) + + hasBlob, _, err = imgStore.CheckBlob("replace", cdigest) + So(err, ShouldBeNil) + So(hasBlob, ShouldEqual, true) + + manifest = ispec.Manifest{ + Config: ispec.Descriptor{ + MediaType: "application/vnd.oci.image.config.v1+json", + Digest: cdigest, + Size: int64(len(cblob)), + }, + Layers: []ispec.Descriptor{ + { + MediaType: "application/vnd.oci.image.layer.v1.tar", + Digest: digest, + Size: int64(buflen), + }, + }, + } + manifest.SchemaVersion = 2 + manifestBuf, err = json.Marshal(manifest) + So(err, ShouldBeNil) + + _ = godigest.FromBytes(manifestBuf) + _, _, err = imgStore.PutImageManifest("replace", "1.0", ispec.MediaTypeImageManifest, manifestBuf) + So(err, ShouldBeNil) + }) + + Convey("Locks", func() { + // in parallel, a mix of read and write locks - mainly for coverage + var wg sync.WaitGroup + for range 1000 { + wg.Add(2) + + go func() { + var lockLatency time.Time + + defer wg.Done() + imgStore.Lock(&lockLatency) + func() {}() + imgStore.Unlock(&lockLatency) + }() + go func() { + var lockLatency time.Time + + defer wg.Done() + imgStore.RLock(&lockLatency) + func() {}() + imgStore.RUnlock(&lockLatency) + }() + } + + wg.Wait() + }) + }) +} + +func TestGCSReuploadCorruptedBlob(t *testing.T) { + tskip.SkipGCS(t) + ensureDummyGCSCreds(t) + + uuid, err := guuid.NewV4() + if err != nil { + panic(err) + } + + testDir := path.Join("/oci-repo-test", uuid.String()) + tdir := t.TempDir() + + rawDriver, imgStore, err := createObjectsStore(testDir, tdir, true) + if err != nil { + panic(err) + } + + defer cleanupStorage(rawDriver, testDir) + + // Wrap driver for WriteFile access + gcsDriver := gcs.New(rawDriver) + + Convey("Test errors paths", t, func() { + storeController := storage.StoreController{DefaultStore: imgStore} + + image := CreateRandomImage() + + err := WriteImageToFileSystem(image, repoName, tag, storeController) + So(err, ShouldBeNil) + }) + + Convey("Test reupload repair corrupted image", t, func() { + storeController := storage.StoreController{DefaultStore: imgStore} + + image := CreateRandomImage() + + err := WriteImageToFileSystem(image, repoName, tag, storeController) + So(err, ShouldBeNil) + + blob := image.Layers[0] + blobDigest := godigest.FromBytes(blob) + blobSize := len(blob) + blobPath := imgStore.BlobPath(repoName, blobDigest) + + ok, size, err := imgStore.CheckBlob(repoName, blobDigest) + So(ok, ShouldBeTrue) + So(size, ShouldEqual, blobSize) + So(err, ShouldBeNil) + + _, err = gcsDriver.WriteFile(blobPath, []byte("corrupted")) + So(err, ShouldBeNil) + + ok, size, err = imgStore.CheckBlob(repoName, blobDigest) + So(ok, ShouldBeFalse) + So(size, ShouldNotEqual, blobSize) + So(err, ShouldEqual, zerr.ErrBlobNotFound) + + err = WriteImageToFileSystem(image, repoName, tag, storeController) + So(err, ShouldBeNil) + + ok, size, _, err = imgStore.StatBlob(repoName, blobDigest) + So(ok, ShouldBeTrue) + So(blobSize, ShouldEqual, size) + So(err, ShouldBeNil) + + ok, size, err = imgStore.CheckBlob(repoName, blobDigest) + So(ok, ShouldBeTrue) + So(size, ShouldEqual, blobSize) + So(err, ShouldBeNil) + }) + + Convey("Test reupload repair corrupted image index", t, func() { + storeController := storage.StoreController{DefaultStore: imgStore} + + image := CreateRandomMultiarch() + + tag := "index" + + err := WriteMultiArchImageToFileSystem(image, repoName, tag, storeController) + So(err, ShouldBeNil) + + blob := image.Images[0].Layers[0] + blobDigest := godigest.FromBytes(blob) + blobSize := len(blob) + blobPath := imgStore.BlobPath(repoName, blobDigest) + + ok, size, err := imgStore.CheckBlob(repoName, blobDigest) + So(ok, ShouldBeTrue) + So(size, ShouldEqual, blobSize) + So(err, ShouldBeNil) + + _, err = gcsDriver.WriteFile(blobPath, []byte("corrupted")) + So(err, ShouldBeNil) + + ok, size, err = imgStore.CheckBlob(repoName, blobDigest) + So(ok, ShouldBeFalse) + So(size, ShouldNotEqual, blobSize) + So(err, ShouldEqual, zerr.ErrBlobNotFound) + + err = WriteMultiArchImageToFileSystem(image, repoName, tag, storeController) + So(err, ShouldBeNil) + + ok, size, _, err = imgStore.StatBlob(repoName, blobDigest) + So(ok, ShouldBeTrue) + So(blobSize, ShouldEqual, size) + So(err, ShouldBeNil) + + ok, size, err = imgStore.CheckBlob(repoName, blobDigest) + So(ok, ShouldBeTrue) + So(size, ShouldEqual, blobSize) + So(err, ShouldBeNil) + }) +} + +func TestGCSStorageHandler(t *testing.T) { + tskip.SkipGCS(t) + ensureDummyGCSCreds(t) + + Convey("Test storage handler", t, func() { + firstRootDir := "/util_test1" + firstCacheDir := t.TempDir() + + firstStorageDriver, firstStore, err := createObjectsStore(firstRootDir, firstCacheDir, true) + So(err, ShouldBeNil) + defer cleanupStorage(firstStorageDriver, firstRootDir) + + secondRootDir := "/util_test2" + secondCacheDir := t.TempDir() + + secondStorageDriver, secondStore, err := createObjectsStore(secondRootDir, secondCacheDir, true) + So(err, ShouldBeNil) + defer cleanupStorage(secondStorageDriver, secondRootDir) + + thirdRootDir := "/util_test3" + thirdCacheDir := t.TempDir() + + thirdStorageDriver, thirdStore, err := createObjectsStore(thirdRootDir, thirdCacheDir, true) + So(err, ShouldBeNil) + defer cleanupStorage(thirdStorageDriver, thirdRootDir) + storeController := storage.StoreController{} + + storeController.DefaultStore = firstStore + + subStore := make(map[string]storageTypes.ImageStore) + + subStore["/a"] = secondStore + subStore["/b"] = thirdStore + + storeController.SubStore = subStore + + imgStore := storeController.GetImageStore("zot-x-test") + So(imgStore.RootDir(), ShouldEqual, firstRootDir) + + imgStore = storeController.GetImageStore("a/zot-a-test") + So(imgStore.RootDir(), ShouldEqual, secondRootDir) + + imgStore = storeController.GetImageStore("b/zot-b-test") + So(imgStore.RootDir(), ShouldEqual, thirdRootDir) + + imgStore = storeController.GetImageStore("c/zot-c-test") + So(imgStore.RootDir(), ShouldEqual, firstRootDir) + }) +} + +func TestGCSMandatoryAnnotations(t *testing.T) { + tskip.SkipGCS(t) + ensureDummyGCSCreds(t) + + uuid, err := guuid.NewV4() + if err != nil { + panic(err) + } + + testDir := path.Join("/oci-repo-test", uuid.String()) + tdir := t.TempDir() + + testLog := log.NewTestLogger() + metrics := monitoring.NewMetricsServer(false, testLog) + + storeDriver, imgStore, err := createObjectsStore(testDir, tdir, true) + if err != nil { + panic(err) + } + + defer cleanupStorage(storeDriver, testDir) + + Convey("Setup manifest", t, func() { + content := []byte("test-data1") + buf := bytes.NewBuffer(content) + buflen := buf.Len() + digest := godigest.FromBytes(content) + + _, _, err = imgStore.FullBlobUpload("test", bytes.NewReader(buf.Bytes()), digest) + So(err, ShouldBeNil) + + cblob, cdigest := GetRandomImageConfig() + + var clen int64 + _, clen, err = imgStore.FullBlobUpload("test", bytes.NewReader(cblob), cdigest) + So(err, ShouldBeNil) + So(clen, ShouldEqual, len(cblob)) + + annotationsMap := make(map[string]string) + annotationsMap[ispec.AnnotationRefName] = "1.0" + + manifest := ispec.Manifest{ + Config: ispec.Descriptor{ + MediaType: "application/vnd.oci.image.config.v1+json", + Digest: cdigest, + Size: int64(len(cblob)), + }, + Layers: []ispec.Descriptor{ + { + MediaType: "application/vnd.oci.image.layer.v1.tar", + Digest: digest, + Size: int64(buflen), + }, + }, + Annotations: annotationsMap, + } + + manifest.SchemaVersion = 2 + manifestBuf, err := json.Marshal(manifest) + So(err, ShouldBeNil) + + Convey("Missing mandatory annotations", func() { + // Create imgStore with linter that returns false (missing annotations) + cacheDriver, _ := storage.Create("boltdb", cache.BoltDBDriverParameters{ + RootDir: tdir, + Name: "cache", + UseRelPaths: false, + }, testLog) + + imgStoreWithLinter := gcs.NewImageStore(testDir, tdir, false, false, testLog, metrics, + &mocks.MockedLint{ + LintFn: func(repo string, manifestDigest godigest.Digest, imageStore storageTypes.ImageStore) (bool, error) { + return false, nil + }, + }, storeDriver, cacheDriver, nil, nil) + + _, _, err = imgStoreWithLinter.PutImageManifest("test", "1.0.0", ispec.MediaTypeImageManifest, manifestBuf) + So(err, ShouldNotBeNil) + }) + + Convey("Error on mandatory annotations", func() { + // Create imgStore with linter that returns error + _, _ = storage.Create("boltdb", cache.BoltDBDriverParameters{ + RootDir: tdir, + Name: "cache", + UseRelPaths: false, + }, testLog) + + imgStoreWithLinter := gcs.NewImageStore(testDir, tdir, false, false, testLog, metrics, + &mocks.MockedLint{ + LintFn: func(repo string, manifestDigest godigest.Digest, imageStore storageTypes.ImageStore) (bool, error) { + //nolint: err113 + return false, errors.New("linter error") + }, + }, storeDriver, nil, nil, nil) + + _, _, err = imgStoreWithLinter.PutImageManifest("test", "1.0.0", ispec.MediaTypeImageManifest, manifestBuf) + So(err, ShouldNotBeNil) + }) + }) +} + +// pushRandomImageIndexGCS is a helper for GC tests. +func pushRandomImageIndexGCS(imgStore storageTypes.ImageStore, repoName string, +) (godigest.Digest, godigest.Digest, godigest.Digest, int64) { + content := []byte("this is a blob") + bdgst := godigest.FromBytes(content) + So(bdgst, ShouldNotBeNil) + + _, bsize, err := imgStore.FullBlobUpload(repoName, bytes.NewReader(content), bdgst) + So(err, ShouldBeNil) + So(bsize, ShouldEqual, len(content)) + + var index ispec.Index + index.SchemaVersion = 2 + index.MediaType = ispec.MediaTypeImageIndex + + var digest godigest.Digest + + for range 4 { + // upload image config blob + upload, err := imgStore.NewBlobUpload(repoName) + So(err, ShouldBeNil) + So(upload, ShouldNotBeEmpty) + + cblob, cdigest := GetRandomImageConfig() + buf := bytes.NewBuffer(cblob) + buflen := buf.Len() + blob, err := imgStore.PutBlobChunkStreamed(repoName, upload, buf) + So(err, ShouldBeNil) + So(blob, ShouldEqual, buflen) + + err = imgStore.FinishBlobUpload(repoName, upload, buf, cdigest) + So(err, ShouldBeNil) + So(blob, ShouldEqual, buflen) + + // create a manifest + manifest := ispec.Manifest{ + Config: ispec.Descriptor{ + MediaType: ispec.MediaTypeImageConfig, + Digest: cdigest, + Size: int64(len(cblob)), + }, + Layers: []ispec.Descriptor{ + { + MediaType: ispec.MediaTypeImageLayer, + Digest: bdgst, + Size: bsize, + }, + }, + } + manifest.SchemaVersion = 2 + content, err = json.Marshal(manifest) + So(err, ShouldBeNil) + + digest = godigest.FromBytes(content) + So(digest, ShouldNotBeNil) + _, _, err = imgStore.PutImageManifest(repoName, digest.String(), ispec.MediaTypeImageManifest, content) + So(err, ShouldBeNil) + + index.Manifests = append(index.Manifests, ispec.Descriptor{ + Digest: digest, + MediaType: ispec.MediaTypeImageManifest, + Size: int64(len(content)), + }) + } + + // upload index image + indexContent, err := json.Marshal(index) + So(err, ShouldBeNil) + + indexDigest := godigest.FromBytes(indexContent) + So(indexDigest, ShouldNotBeNil) + + _, _, err = imgStore.PutImageManifest(repoName, "1.0", ispec.MediaTypeImageIndex, indexContent) + So(err, ShouldBeNil) + + return bdgst, digest, indexDigest, int64(len(indexContent)) +} + +func TestGCSGarbageCollectImageManifest(t *testing.T) { + tskip.SkipGCS(t) + ensureDummyGCSCreds(t) + + testLog := log.NewTestLogger() + audit := log.NewAuditLogger("debug", "") + + ctx := context.Background() + + uuid, err := guuid.NewV4() + if err != nil { + panic(err) + } + + testDir := path.Join("/oci-repo-test", uuid.String()) + tdir := t.TempDir() + + storeDriver, imgStore, err := createObjectsStore(testDir, tdir, true) + if err != nil { + panic(err) + } + + defer cleanupStorage(storeDriver, testDir) + + Convey("Garbage collect with short delay", t, func(c C) { + gcDelay := 1 * time.Second + + garbageCollect := gc.NewGarbageCollect(imgStore, mocks.MetaDBMock{}, gc.Options{ + Delay: gcDelay, + ImageRetention: config.ImageRetention{ + Delay: gcDelay, + Policies: []config.RetentionPolicy{ + { + Repositories: []string{"**"}, + DeleteReferrers: true, + DeleteUntagged: &trueVal, + }, + }, + }, + }, audit, testLog) + + // upload orphan blob + upload, err := imgStore.NewBlobUpload(repoName) + So(err, ShouldBeNil) + So(upload, ShouldNotBeEmpty) + + content := []byte("test-data1") + buf := bytes.NewBuffer(content) + buflen := buf.Len() + odigest := godigest.FromBytes(content) + + blob, err := imgStore.PutBlobChunk(repoName, upload, 0, int64(buflen), buf) + So(err, ShouldBeNil) + So(blob, ShouldEqual, buflen) + + err = imgStore.FinishBlobUpload(repoName, upload, buf, odigest) + So(err, ShouldBeNil) + + // sleep so orphan blob can be GC'ed + time.Sleep(1 * time.Second) + + // upload blob + upload, err = imgStore.NewBlobUpload(repoName) + So(err, ShouldBeNil) + So(upload, ShouldNotBeEmpty) + + content = []byte("test-data2") + buf = bytes.NewBuffer(content) + buflen = buf.Len() + bdigest := godigest.FromBytes(content) + + blob, err = imgStore.PutBlobChunk(repoName, upload, 0, int64(buflen), buf) + So(err, ShouldBeNil) + So(blob, ShouldEqual, buflen) + + err = imgStore.FinishBlobUpload(repoName, upload, buf, bdigest) + So(err, ShouldBeNil) + + annotationsMap := make(map[string]string) + annotationsMap[ispec.AnnotationRefName] = tag + + cblob, cdigest := GetRandomImageConfig() + _, clen, err := imgStore.FullBlobUpload(repoName, bytes.NewReader(cblob), cdigest) + So(err, ShouldBeNil) + So(clen, ShouldEqual, len(cblob)) + + hasBlob, _, err := imgStore.CheckBlob(repoName, cdigest) + So(err, ShouldBeNil) + So(hasBlob, ShouldEqual, true) + + manifest := ispec.Manifest{ + Config: ispec.Descriptor{ + MediaType: "application/vnd.oci.image.config.v1+json", + Digest: cdigest, + Size: int64(len(cblob)), + }, + Layers: []ispec.Descriptor{ + { + MediaType: "application/vnd.oci.image.layer.v1.tar", + Digest: bdigest, + Size: int64(buflen), + }, + }, + Annotations: annotationsMap, + } + + manifest.SchemaVersion = 2 + manifestBuf, err := json.Marshal(manifest) + So(err, ShouldBeNil) + + digest := godigest.FromBytes(manifestBuf) + + _, _, err = imgStore.PutImageManifest(repoName, tag, ispec.MediaTypeImageManifest, manifestBuf) + So(err, ShouldBeNil) + + // put artifact referencing above image + artifactBlob := []byte("artifact") + artifactBlobDigest := godigest.FromBytes(artifactBlob) + + // push layer + _, _, err = imgStore.FullBlobUpload(repoName, bytes.NewReader(artifactBlob), artifactBlobDigest) + So(err, ShouldBeNil) + + // push config + _, _, err = imgStore.FullBlobUpload(repoName, bytes.NewReader(ispec.DescriptorEmptyJSON.Data), + ispec.DescriptorEmptyJSON.Digest) + So(err, ShouldBeNil) + + artifactManifest := ispec.Manifest{ + MediaType: ispec.MediaTypeImageManifest, + Layers: []ispec.Descriptor{ + { + MediaType: "application/octet-stream", + Digest: artifactBlobDigest, + Size: int64(len(artifactBlob)), + }, + }, + Config: ispec.DescriptorEmptyJSON, + Subject: &ispec.Descriptor{ + MediaType: ispec.MediaTypeImageManifest, + Digest: digest, + Size: int64(len(manifestBuf)), + }, + } + artifactManifest.SchemaVersion = 2 + + artifactManifestBuf, err := json.Marshal(artifactManifest) + So(err, ShouldBeNil) + + artifactDigest := godigest.FromBytes(artifactManifestBuf) + + // push artifact manifest + _, _, err = imgStore.PutImageManifest(repoName, artifactDigest.String(), + ispec.MediaTypeImageManifest, artifactManifestBuf) + So(err, ShouldBeNil) + + // push orphan artifact (missing subject) + artifactManifest.Subject = &ispec.Descriptor{ + MediaType: ispec.MediaTypeImageManifest, + Digest: godigest.FromBytes([]byte("miss")), + Size: int64(30), + } + artifactManifest.ArtifactType = "application/orphan" + + artifactManifestBuf, err = json.Marshal(artifactManifest) + So(err, ShouldBeNil) + + orphanArtifactManifestDigest := godigest.FromBytes(artifactManifestBuf) + + // push orphan artifact manifest + _, _, err = imgStore.PutImageManifest(repoName, orphanArtifactManifestDigest.String(), + ispec.MediaTypeImageManifest, artifactManifestBuf) + So(err, ShouldBeNil) + + err = garbageCollect.CleanRepo(ctx, repoName) + So(err, ShouldBeNil) + + hasBlob, _, err = imgStore.CheckBlob(repoName, odigest) + So(err, ShouldNotBeNil) + So(hasBlob, ShouldEqual, false) + + hasBlob, _, err = imgStore.CheckBlob(repoName, bdigest) + So(err, ShouldBeNil) + So(hasBlob, ShouldEqual, true) + + // sleep so orphan blob can be GC'ed + time.Sleep(1 * time.Second) + + Convey("Garbage collect blobs after manifest is removed", func() { + err = imgStore.DeleteImageManifest(repoName, digest.String(), false) + So(err, ShouldBeNil) + + err = garbageCollect.CleanRepo(ctx, repoName) + So(err, ShouldBeNil) + + hasBlob, _, err = imgStore.CheckBlob(repoName, bdigest) + So(err, ShouldNotBeNil) + So(hasBlob, ShouldEqual, false) + + hasBlob, _, err = imgStore.CheckBlob(repoName, artifactBlobDigest) + So(err, ShouldNotBeNil) + So(hasBlob, ShouldEqual, false) + + // check artifacts are gc'ed + _, _, _, err := imgStore.GetImageManifest(repoName, artifactDigest.String()) + So(err, ShouldNotBeNil) + + _, _, _, err = imgStore.GetImageManifest(repoName, orphanArtifactManifestDigest.String()) + So(err, ShouldNotBeNil) + + // check it gc'ed repo + exists := imgStore.DirExists(path.Join(imgStore.RootDir(), repoName)) + So(exists, ShouldBeFalse) + }) + }) +} + +func TestGCSGarbageCollectImageIndex(t *testing.T) { + tskip.SkipGCS(t) + ensureDummyGCSCreds(t) + + testLog := log.NewTestLogger() + audit := log.NewAuditLogger("debug", "") + + ctx := context.Background() + + uuid, err := guuid.NewV4() + if err != nil { + panic(err) + } + + testDir := path.Join("/oci-repo-test", uuid.String()) + tdir := t.TempDir() + + storeDriver, imgStore, err := createObjectsStore(testDir, tdir, true) + if err != nil { + panic(err) + } + + defer cleanupStorage(storeDriver, testDir) + + Convey("Garbage collect with short delay", t, func(c C) { + gcDelay := 2 * time.Second + imageRetentionDelay := 2 * time.Second + + garbageCollect := gc.NewGarbageCollect(imgStore, mocks.MetaDBMock{}, gc.Options{ + Delay: gcDelay, + ImageRetention: config.ImageRetention{ + Delay: imageRetentionDelay, + Policies: []config.RetentionPolicy{ + { + Repositories: []string{"**"}, + DeleteReferrers: true, + DeleteUntagged: &trueVal, + }, + }, + }, + }, audit, testLog) + + // upload orphan blob + upload, err := imgStore.NewBlobUpload(repoName) + So(err, ShouldBeNil) + So(upload, ShouldNotBeEmpty) + + content := []byte("test-data1") + buf := bytes.NewBuffer(content) + buflen := buf.Len() + odigest := godigest.FromBytes(content) + + blob, err := imgStore.PutBlobChunk(repoName, upload, 0, int64(buflen), buf) + So(err, ShouldBeNil) + So(blob, ShouldEqual, buflen) + + err = imgStore.FinishBlobUpload(repoName, upload, buf, odigest) + So(err, ShouldBeNil) + + bdgst, digest, indexDigest, indexSize := pushRandomImageIndexGCS(imgStore, repoName) + + // put artifact referencing above image + artifactBlob := []byte("artifact") + artifactBlobDigest := godigest.FromBytes(artifactBlob) + + // push layer + _, _, err = imgStore.FullBlobUpload(repoName, bytes.NewReader(artifactBlob), artifactBlobDigest) + So(err, ShouldBeNil) + + // push config + _, _, err = imgStore.FullBlobUpload(repoName, bytes.NewReader(ispec.DescriptorEmptyJSON.Data), + ispec.DescriptorEmptyJSON.Digest) + So(err, ShouldBeNil) + + // push artifact manifest pointing to index + artifactManifest := ispec.Manifest{ + MediaType: ispec.MediaTypeImageManifest, + Layers: []ispec.Descriptor{ + { + MediaType: "application/octet-stream", + Digest: artifactBlobDigest, + Size: int64(len(artifactBlob)), + }, + }, + Config: ispec.DescriptorEmptyJSON, + Subject: &ispec.Descriptor{ + MediaType: ispec.MediaTypeImageIndex, + Digest: indexDigest, + Size: indexSize, + }, + ArtifactType: "application/forIndex", + } + artifactManifest.SchemaVersion = 2 + + artifactManifestBuf, err := json.Marshal(artifactManifest) + So(err, ShouldBeNil) + + artifactDigest := godigest.FromBytes(artifactManifestBuf) + + // push artifact manifest + _, _, err = imgStore.PutImageManifest(repoName, artifactDigest.String(), + ispec.MediaTypeImageManifest, artifactManifestBuf) + So(err, ShouldBeNil) + + hasBlob, _, err := imgStore.CheckBlob(repoName, bdgst) + So(err, ShouldBeNil) + So(hasBlob, ShouldEqual, true) + + time.Sleep(2 * time.Second) + + Convey("delete index manifest, references should not be persisted", func() { + err = garbageCollect.CleanRepo(ctx, repoName) + So(err, ShouldBeNil) + + err = imgStore.DeleteImageManifest(repoName, indexDigest.String(), false) + So(err, ShouldBeNil) + + err = garbageCollect.CleanRepo(ctx, repoName) + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest(repoName, artifactDigest.String()) + So(err, ShouldNotBeNil) + + // orphan blob + hasBlob, _, err = imgStore.CheckBlob(repoName, odigest) + So(err, ShouldNotBeNil) + So(hasBlob, ShouldEqual, false) + + hasBlob, _, err = imgStore.CheckBlob(repoName, bdgst) + So(err, ShouldNotBeNil) + So(hasBlob, ShouldEqual, false) + + // check last manifest from index image + hasBlob, _, err = imgStore.CheckBlob(repoName, digest) + So(err, ShouldNotBeNil) + So(hasBlob, ShouldEqual, false) + + // check referrer is gc'ed + _, _, _, err := imgStore.GetImageManifest(repoName, artifactDigest.String()) + So(err, ShouldNotBeNil) + + hasBlob, _, err = imgStore.CheckBlob(repoName, artifactBlobDigest) + So(err, ShouldNotBeNil) + So(hasBlob, ShouldEqual, false) + + // check it gc'ed repo + exists := imgStore.DirExists(path.Join(imgStore.RootDir(), repoName)) + So(exists, ShouldBeFalse) + }) + }) +} + +func TestGCSGarbageCollectChainedImageIndexes(t *testing.T) { + tskip.SkipGCS(t) + ensureDummyGCSCreds(t) + + testLog := log.NewTestLogger() + audit := log.NewAuditLogger("debug", "") + + ctx := context.Background() + + uuid, err := guuid.NewV4() + if err != nil { + panic(err) + } + + testDir := path.Join("/oci-repo-test", uuid.String()) + tdir := t.TempDir() + + storeDriver, imgStore, err := createObjectsStore(testDir, tdir, true) + if err != nil { + panic(err) + } + + defer cleanupStorage(storeDriver, testDir) + + Convey("Garbage collect with short delay", t, func() { + gcDelay := 5 * time.Second + imageRetentionDelay := 5 * time.Second + + garbageCollect := gc.NewGarbageCollect(imgStore, mocks.MetaDBMock{}, gc.Options{ + Delay: gcDelay, + ImageRetention: config.ImageRetention{ + Delay: imageRetentionDelay, + Policies: []config.RetentionPolicy{ + { + Repositories: []string{"**"}, + DeleteReferrers: true, + DeleteUntagged: &trueVal, + }, + }, + }, + }, audit, testLog) + + // upload orphan blob + upload, err := imgStore.NewBlobUpload(repoName) + So(err, ShouldBeNil) + So(upload, ShouldNotBeEmpty) + + content := []byte("test-data1") + buf := bytes.NewBuffer(content) + buflen := buf.Len() + odigest := godigest.FromBytes(content) + + blob, err := imgStore.PutBlobChunk(repoName, upload, 0, int64(buflen), buf) + So(err, ShouldBeNil) + So(blob, ShouldEqual, buflen) + + err = imgStore.FinishBlobUpload(repoName, upload, buf, odigest) + So(err, ShouldBeNil) + + content = []byte("this is a blob") + bdgst := godigest.FromBytes(content) + So(bdgst, ShouldNotBeNil) + + _, bsize, err := imgStore.FullBlobUpload(repoName, bytes.NewReader(content), bdgst) + So(err, ShouldBeNil) + So(bsize, ShouldEqual, len(content)) + + artifactBlob := []byte("artifact") + artifactBlobDigest := godigest.FromBytes(artifactBlob) + + // push layer + _, _, err = imgStore.FullBlobUpload(repoName, bytes.NewReader(artifactBlob), artifactBlobDigest) + So(err, ShouldBeNil) + + // push config + _, _, err = imgStore.FullBlobUpload(repoName, bytes.NewReader(ispec.DescriptorEmptyJSON.Data), + ispec.DescriptorEmptyJSON.Digest) + So(err, ShouldBeNil) + + var index ispec.Index + index.SchemaVersion = 2 + index.MediaType = ispec.MediaTypeImageIndex + + var digest godigest.Digest + + for range 4 { + // upload image config blob + upload, err := imgStore.NewBlobUpload(repoName) + So(err, ShouldBeNil) + So(upload, ShouldNotBeEmpty) + + cblob, cdigest := GetRandomImageConfig() + buf := bytes.NewBuffer(cblob) + buflen := buf.Len() + blob, err := imgStore.PutBlobChunkStreamed(repoName, upload, buf) + So(err, ShouldBeNil) + So(blob, ShouldEqual, buflen) + + err = imgStore.FinishBlobUpload(repoName, upload, buf, cdigest) + So(err, ShouldBeNil) + So(blob, ShouldEqual, buflen) + + // create a manifest + manifest := ispec.Manifest{ + Config: ispec.Descriptor{ + MediaType: ispec.MediaTypeImageConfig, + Digest: cdigest, + Size: int64(len(cblob)), + }, + Layers: []ispec.Descriptor{ + { + MediaType: ispec.MediaTypeImageLayer, + Digest: bdgst, + Size: bsize, + }, + }, + } + manifest.SchemaVersion = 2 + content, err = json.Marshal(manifest) + So(err, ShouldBeNil) + + digest = godigest.FromBytes(content) + So(digest, ShouldNotBeNil) + _, _, err = imgStore.PutImageManifest(repoName, digest.String(), ispec.MediaTypeImageManifest, content) + So(err, ShouldBeNil) + + index.Manifests = append(index.Manifests, ispec.Descriptor{ + Digest: digest, + MediaType: ispec.MediaTypeImageManifest, + Size: int64(len(content)), + }) + + // for each manifest inside index, push an artifact + artifactManifest := ispec.Manifest{ + MediaType: ispec.MediaTypeImageManifest, + Layers: []ispec.Descriptor{ + { + MediaType: "application/octet-stream", + Digest: artifactBlobDigest, + Size: int64(len(artifactBlob)), + }, + }, + Config: ispec.DescriptorEmptyJSON, + Subject: &ispec.Descriptor{ + MediaType: ispec.MediaTypeImageManifest, + Digest: digest, + Size: int64(len(content)), + }, + ArtifactType: "application/forManifestInInnerIndex", + } + artifactManifest.SchemaVersion = 2 + + artifactManifestBuf, err := json.Marshal(artifactManifest) + So(err, ShouldBeNil) + + artifactDigest := godigest.FromBytes(artifactManifestBuf) + + // push artifact manifest + _, _, err = imgStore.PutImageManifest(repoName, artifactDigest.String(), + ispec.MediaTypeImageManifest, artifactManifestBuf) + So(err, ShouldBeNil) + } + + // also add a new image index inside this one + var innerIndex ispec.Index + innerIndex.SchemaVersion = 2 + innerIndex.MediaType = ispec.MediaTypeImageIndex + + for range 3 { + // upload image config blob + upload, err := imgStore.NewBlobUpload(repoName) + So(err, ShouldBeNil) + So(upload, ShouldNotBeEmpty) + + cblob, cdigest := GetRandomImageConfig() + buf := bytes.NewBuffer(cblob) + buflen := buf.Len() + blob, err := imgStore.PutBlobChunkStreamed(repoName, upload, buf) + So(err, ShouldBeNil) + So(blob, ShouldEqual, buflen) + + err = imgStore.FinishBlobUpload(repoName, upload, buf, cdigest) + So(err, ShouldBeNil) + So(blob, ShouldEqual, buflen) + + // create a manifest + manifest := ispec.Manifest{ + Config: ispec.Descriptor{ + MediaType: ispec.MediaTypeImageConfig, + Digest: cdigest, + Size: int64(len(cblob)), + }, + Layers: []ispec.Descriptor{ + { + MediaType: ispec.MediaTypeImageLayer, + Digest: bdgst, + Size: bsize, + }, + }, + } + manifest.SchemaVersion = 2 + content, err = json.Marshal(manifest) + So(err, ShouldBeNil) + + digest := godigest.FromBytes(content) + So(digest, ShouldNotBeNil) + _, _, err = imgStore.PutImageManifest(repoName, digest.String(), ispec.MediaTypeImageManifest, content) + So(err, ShouldBeNil) + + innerIndex.Manifests = append(innerIndex.Manifests, ispec.Descriptor{ + Digest: digest, + MediaType: ispec.MediaTypeImageManifest, + Size: int64(len(content)), + }) + } + + // upload inner index image + innerIndexContent, err := json.Marshal(index) + So(err, ShouldBeNil) + + innerIndexDigest := godigest.FromBytes(innerIndexContent) + So(innerIndexDigest, ShouldNotBeNil) + + _, _, err = imgStore.PutImageManifest(repoName, innerIndexDigest.String(), + ispec.MediaTypeImageIndex, innerIndexContent) + So(err, ShouldBeNil) + + // add inner index into root index + index.Manifests = append(index.Manifests, ispec.Descriptor{ + Digest: innerIndexDigest, + MediaType: ispec.MediaTypeImageIndex, + Size: int64(len(innerIndexContent)), + }) + + // push root index + // upload index image + indexContent, err := json.Marshal(index) + So(err, ShouldBeNil) + + indexDigest := godigest.FromBytes(indexContent) + So(indexDigest, ShouldNotBeNil) + + _, _, err = imgStore.PutImageManifest(repoName, "1.0", ispec.MediaTypeImageIndex, indexContent) + So(err, ShouldBeNil) + + artifactManifest := ispec.Manifest{ + MediaType: ispec.MediaTypeImageManifest, + Layers: []ispec.Descriptor{ + { + MediaType: "application/octet-stream", + Digest: artifactBlobDigest, + Size: int64(len(artifactBlob)), + }, + }, + Config: ispec.DescriptorEmptyJSON, + Subject: &ispec.Descriptor{ + MediaType: ispec.MediaTypeImageIndex, + Digest: indexDigest, + Size: int64(len(indexContent)), + }, + ArtifactType: "application/forIndex", + } + artifactManifest.SchemaVersion = 2 + + artifactManifestBuf, err := json.Marshal(artifactManifest) + So(err, ShouldBeNil) + + artifactDigest := godigest.FromBytes(artifactManifestBuf) + + // push artifact manifest + _, _, err = imgStore.PutImageManifest(repoName, artifactDigest.String(), + ispec.MediaTypeImageManifest, artifactManifestBuf) + So(err, ShouldBeNil) + + artifactManifest.Subject = &ispec.Descriptor{ + MediaType: ispec.MediaTypeImageManifest, + Digest: digest, + Size: int64(len(content)), + } + artifactManifest.ArtifactType = "application/forManifestInIndex" + + artifactManifestIndexBuf, err := json.Marshal(artifactManifest) + So(err, ShouldBeNil) + + artifactManifestIndexDigest := godigest.FromBytes(artifactManifestIndexBuf) + + // push artifact manifest referencing a manifest from index image + _, _, err = imgStore.PutImageManifest(repoName, artifactManifestIndexDigest.String(), + ispec.MediaTypeImageManifest, artifactManifestIndexBuf) + So(err, ShouldBeNil) + + artifactManifest.Subject = &ispec.Descriptor{ + MediaType: ispec.MediaTypeImageIndex, + Digest: innerIndexDigest, + Size: int64(len(innerIndexContent)), + } + artifactManifest.ArtifactType = "application/forInnerIndex" + + artifactManifestInnerIndexBuf, err := json.Marshal(artifactManifest) + So(err, ShouldBeNil) + + artifactManifestInnerIndexDigest := godigest.FromBytes(artifactManifestInnerIndexBuf) + + // push artifact manifest referencing a manifest from index image + _, _, err = imgStore.PutImageManifest(repoName, artifactManifestInnerIndexDigest.String(), + ispec.MediaTypeImageManifest, artifactManifestInnerIndexBuf) + So(err, ShouldBeNil) + + // push artifact manifest pointing to artifact above + + artifactManifest.Subject = &ispec.Descriptor{ + MediaType: ispec.MediaTypeImageManifest, + Digest: artifactDigest, + Size: int64(len(artifactManifestBuf)), + } + artifactManifest.ArtifactType = "application/forArtifact" + + artifactManifestBuf, err = json.Marshal(artifactManifest) + So(err, ShouldBeNil) + + artifactOfArtifactManifestDigest := godigest.FromBytes(artifactManifestBuf) + _, _, err = imgStore.PutImageManifest(repoName, artifactOfArtifactManifestDigest.String(), + ispec.MediaTypeImageManifest, artifactManifestBuf) + So(err, ShouldBeNil) + + // push orphan artifact (missing subject) + artifactManifest.Subject = &ispec.Descriptor{ + MediaType: ispec.MediaTypeImageManifest, + Digest: godigest.FromBytes([]byte("miss")), + Size: int64(30), + } + artifactManifest.ArtifactType = "application/orphan" + + artifactManifestBuf, err = json.Marshal(artifactManifest) + So(err, ShouldBeNil) + + orphanArtifactManifestDigest := godigest.FromBytes(artifactManifestBuf) + + // push orphan artifact manifest + _, _, err = imgStore.PutImageManifest(repoName, orphanArtifactManifestDigest.String(), + ispec.MediaTypeImageManifest, artifactManifestBuf) + So(err, ShouldBeNil) + + hasBlob, _, err := imgStore.CheckBlob(repoName, bdgst) + So(err, ShouldBeNil) + So(hasBlob, ShouldEqual, true) + + hasBlob, _, _, err = imgStore.StatBlob(repoName, bdgst) + So(err, ShouldBeNil) + So(hasBlob, ShouldEqual, true) + + hasBlob, _, err = imgStore.CheckBlob(repoName, artifactBlobDigest) + So(err, ShouldBeNil) + So(hasBlob, ShouldEqual, true) + + time.Sleep(5 * time.Second) + + Convey("delete inner referenced manifest", func() { + err = garbageCollect.CleanRepo(ctx, repoName) + So(err, ShouldBeNil) + + // check orphan artifact is gc'ed + _, _, _, err = imgStore.GetImageManifest(repoName, orphanArtifactManifestDigest.String()) + So(err, ShouldNotBeNil) + + _, _, _, err = imgStore.GetImageManifest(repoName, artifactOfArtifactManifestDigest.String()) + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest(repoName, artifactManifestIndexDigest.String()) + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest(repoName, artifactDigest.String()) + So(err, ShouldBeNil) + + err = imgStore.DeleteImageManifest(repoName, artifactDigest.String(), false) + So(err, ShouldBeNil) + + err = garbageCollect.CleanRepo(ctx, repoName) + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest(repoName, artifactOfArtifactManifestDigest.String()) + So(err, ShouldNotBeNil) + + _, _, _, err = imgStore.GetImageManifest(repoName, artifactManifestIndexDigest.String()) + So(err, ShouldBeNil) + }) + + Convey("delete index manifest, references should not be persisted", func() { + err = garbageCollect.CleanRepo(ctx, repoName) + So(err, ShouldBeNil) + + // check orphan artifact is gc'ed + _, _, _, err = imgStore.GetImageManifest(repoName, orphanArtifactManifestDigest.String()) + So(err, ShouldNotBeNil) + + _, _, _, err = imgStore.GetImageManifest(repoName, artifactOfArtifactManifestDigest.String()) + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest(repoName, artifactManifestIndexDigest.String()) + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest(repoName, artifactDigest.String()) + So(err, ShouldBeNil) + + err = imgStore.DeleteImageManifest(repoName, indexDigest.String(), false) + So(err, ShouldBeNil) + + err = garbageCollect.CleanRepo(ctx, repoName) + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest(repoName, artifactDigest.String()) + So(err, ShouldNotBeNil) + + _, _, _, err = imgStore.GetImageManifest(repoName, artifactOfArtifactManifestDigest.String()) + So(err, ShouldNotBeNil) + + // orphan blob + hasBlob, _, err = imgStore.CheckBlob(repoName, odigest) + So(err, ShouldNotBeNil) + So(hasBlob, ShouldEqual, false) + + hasBlob, _, _, err = imgStore.StatBlob(repoName, odigest) + So(err, ShouldNotBeNil) + So(hasBlob, ShouldEqual, false) + + // check artifact is gc'ed + _, _, _, err := imgStore.GetImageManifest(repoName, artifactDigest.String()) + So(err, ShouldNotBeNil) + + // check inner index artifact is gc'ed + _, _, _, err = imgStore.GetImageManifest(repoName, artifactManifestInnerIndexDigest.String()) + So(err, ShouldNotBeNil) + + // check last manifest from index image + hasBlob, _, err = imgStore.CheckBlob(repoName, digest) + So(err, ShouldNotBeNil) + So(hasBlob, ShouldEqual, false) + + _, _, _, err = imgStore.GetImageManifest(repoName, artifactManifestIndexDigest.String()) + So(err, ShouldNotBeNil) + + hasBlob, _, err = imgStore.CheckBlob(repoName, artifactBlobDigest) + So(err, ShouldNotBeNil) + So(hasBlob, ShouldEqual, false) + + hasBlob, _, err = imgStore.CheckBlob(repoName, bdgst) + So(err, ShouldNotBeNil) + So(hasBlob, ShouldEqual, false) + + // check it gc'ed repo + exists := imgStore.DirExists(path.Join(imgStore.RootDir(), repoName)) + So(exists, ShouldBeFalse) + }) + }) +} + +func TestGCSCheckAllBlobsIntegrity(t *testing.T) { + tskip.SkipGCS(t) + ensureDummyGCSCreds(t) + + Convey("test with GCS storage", t, func() { + uuid, err := guuid.NewV4() + So(err, ShouldBeNil) + + testDir := path.Join("/oci-repo-test", uuid.String()) + tdir := t.TempDir() + + storeDriver, imgStore, err := createObjectsStore(testDir, tdir, true) + So(err, ShouldBeNil) + + defer cleanupStorage(storeDriver, testDir) + + testLog := log.NewTestLogger() + + RunGCSCheckAllBlobsIntegrityTests(t, imgStore, gcs.New(storeDriver), testLog) + }) +} + +func RunGCSCheckAllBlobsIntegrityTests( //nolint: thelper + t *testing.T, imgStore storageTypes.ImageStore, driver storageTypes.Driver, testLog log.Logger, +) { + Convey("Scrub only one repo", func() { + // initialize repo + err := imgStore.InitRepo(repoName) + So(err, ShouldBeNil) + + ok := imgStore.DirExists(path.Join(imgStore.RootDir(), repoName)) + So(ok, ShouldBeTrue) + + storeCtlr := storage.StoreController{} + storeCtlr.DefaultStore = imgStore + So(storeCtlr.GetImageStore(repoName), ShouldResemble, imgStore) + + image := CreateRandomImage() + + err = WriteImageToFileSystem(image, repoName, "1.0", storeCtlr) + So(err, ShouldBeNil) + + Convey("Blobs integrity not affected", func() { + buff := bytes.NewBufferString("") + + res, err := storeCtlr.CheckAllBlobsIntegrity(context.Background()) + res.PrintScrubResults(buff) + So(err, ShouldBeNil) + + space := regexp.MustCompile(`\s+`) + str := space.ReplaceAllString(buff.String(), " ") + actual := strings.TrimSpace(str) + So(actual, ShouldContainSubstring, "REPOSITORY TAG STATUS AFFECTED BLOB ERROR") + So(actual, ShouldContainSubstring, "test 1.0 ok") + + err = WriteMultiArchImageToFileSystem(CreateMultiarchWith().RandomImages(0).Build(), repoName, "2.0", storeCtlr) + So(err, ShouldBeNil) + + buff = bytes.NewBufferString("") + + res, err = storeCtlr.CheckAllBlobsIntegrity(context.Background()) + res.PrintScrubResults(buff) + So(err, ShouldBeNil) + + str = space.ReplaceAllString(buff.String(), " ") + actual = strings.TrimSpace(str) + So(actual, ShouldContainSubstring, "REPOSITORY TAG STATUS AFFECTED BLOB ERROR") + So(actual, ShouldContainSubstring, "test 1.0 ok") + So(actual, ShouldContainSubstring, "test 2.0 ok") + }) + + Convey("Blobs integrity with context done", func() { + buff := bytes.NewBufferString("") + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + res, err := storeCtlr.CheckAllBlobsIntegrity(ctx) + res.PrintScrubResults(buff) + So(err, ShouldNotBeNil) + + space := regexp.MustCompile(`\s+`) + str := space.ReplaceAllString(buff.String(), " ") + actual := strings.TrimSpace(str) + So(actual, ShouldContainSubstring, "REPOSITORY TAG STATUS AFFECTED BLOB ERROR") + So(actual, ShouldNotContainSubstring, "test 1.0 ok") + }) + + Convey("Manifest integrity affected", func() { + // get content of manifest file + content, _, _, err := imgStore.GetImageManifest(repoName, image.ManifestDescriptor.Digest.String()) + So(err, ShouldBeNil) + + // delete content of manifest file + manifestDig := image.ManifestDescriptor.Digest.Encoded() + manifestFile := path.Join(imgStore.RootDir(), repoName, "/blobs/sha256", manifestDig) + err = driver.Delete(manifestFile) + So(err, ShouldBeNil) + + defer func() { + // put manifest content back to file + _, err = driver.WriteFile(manifestFile, content) + So(err, ShouldBeNil) + }() + + buff := bytes.NewBufferString("") + + res, err := storeCtlr.CheckAllBlobsIntegrity(context.Background()) + res.PrintScrubResults(buff) + So(err, ShouldBeNil) + + space := regexp.MustCompile(`\s+`) + str := space.ReplaceAllString(buff.String(), " ") + actual := strings.TrimSpace(str) + So(actual, ShouldContainSubstring, "REPOSITORY TAG STATUS AFFECTED BLOB ERROR") + So(actual, ShouldNotContainSubstring, "affected") + + index, err := common.GetIndex(imgStore, repoName, testLog) + So(err, ShouldBeNil) + + So(len(index.Manifests), ShouldEqual, 1) + + _, err = driver.WriteFile(manifestFile, []byte("invalid content")) + So(err, ShouldBeNil) + + buff = bytes.NewBufferString("") + + res, err = storeCtlr.CheckAllBlobsIntegrity(context.Background()) + res.PrintScrubResults(buff) + So(err, ShouldBeNil) + + str = space.ReplaceAllString(buff.String(), " ") + actual = strings.TrimSpace(str) + So(actual, ShouldContainSubstring, "REPOSITORY TAG STATUS AFFECTED BLOB ERROR") + // verify error message + So(actual, ShouldContainSubstring, fmt.Sprintf("test 1.0 affected %s invalid manifest content", manifestDig)) + + index, err = common.GetIndex(imgStore, repoName, testLog) + So(err, ShouldBeNil) + + So(len(index.Manifests), ShouldEqual, 1) + manifestDescriptor := index.Manifests[0] + + _, _, err = storage.CheckManifestAndConfig(repoName, manifestDescriptor, []byte("invalid content"), imgStore) + So(err, ShouldNotBeNil) + }) + + Convey("Config integrity affected", func() { + // get content of config file + content, err := imgStore.GetBlobContent(repoName, image.ConfigDescriptor.Digest) + So(err, ShouldBeNil) + + // delete content of config file + configDig := image.ConfigDescriptor.Digest.Encoded() + configFile := path.Join(imgStore.RootDir(), repoName, "/blobs/sha256", configDig) + err = driver.Delete(configFile) + So(err, ShouldBeNil) + + defer func() { + // put config content back to file + _, err = driver.WriteFile(configFile, content) + So(err, ShouldBeNil) + }() + + buff := bytes.NewBufferString("") + + res, err := storeCtlr.CheckAllBlobsIntegrity(context.Background()) + res.PrintScrubResults(buff) + So(err, ShouldBeNil) + + space := regexp.MustCompile(`\s+`) + str := space.ReplaceAllString(buff.String(), " ") + actual := strings.TrimSpace(str) + So(actual, ShouldContainSubstring, "REPOSITORY TAG STATUS AFFECTED BLOB ERROR") + So(actual, ShouldContainSubstring, fmt.Sprintf("test 1.0 affected %s blob not found", configDig)) + + _, err = driver.WriteFile(configFile, []byte("invalid content")) + So(err, ShouldBeNil) + + buff = bytes.NewBufferString("") + + res, err = storeCtlr.CheckAllBlobsIntegrity(context.Background()) + res.PrintScrubResults(buff) + So(err, ShouldBeNil) + + str = space.ReplaceAllString(buff.String(), " ") + actual = strings.TrimSpace(str) + So(actual, ShouldContainSubstring, "REPOSITORY TAG STATUS AFFECTED BLOB ERROR") + So(actual, ShouldContainSubstring, fmt.Sprintf("test 1.0 affected %s invalid server config", configDig)) + }) + + Convey("Layers integrity affected", func() { + // get content of layer + content, err := imgStore.GetBlobContent(repoName, image.Manifest.Layers[0].Digest) + So(err, ShouldBeNil) + + // delete content of layer file + layerDig := image.Manifest.Layers[0].Digest.Encoded() + layerFile := path.Join(imgStore.RootDir(), repoName, "/blobs/sha256", layerDig) + _, err = driver.WriteFile(layerFile, []byte(" ")) + So(err, ShouldBeNil) + + defer func() { + // put layer content back to file + _, err = driver.WriteFile(layerFile, content) + So(err, ShouldBeNil) + }() + + buff := bytes.NewBufferString("") + + res, err := storeCtlr.CheckAllBlobsIntegrity(context.Background()) + res.PrintScrubResults(buff) + So(err, ShouldBeNil) + + space := regexp.MustCompile(`\s+`) + str := space.ReplaceAllString(buff.String(), " ") + actual := strings.TrimSpace(str) + So(actual, ShouldContainSubstring, "REPOSITORY TAG STATUS AFFECTED BLOB ERROR") + So(actual, ShouldContainSubstring, fmt.Sprintf("test 1.0 affected %s bad blob digest", layerDig)) + }) + + Convey("Layer not found", func() { + // get content of layer + digest := image.Manifest.Layers[0].Digest + content, err := imgStore.GetBlobContent(repoName, digest) + So(err, ShouldBeNil) + + // change layer file permissions + layerDig := image.Manifest.Layers[0].Digest.Encoded() + repoDir := path.Join(imgStore.RootDir(), repoName) + layerFile := path.Join(repoDir, "/blobs/sha256", layerDig) + err = driver.Delete(layerFile) + So(err, ShouldBeNil) + + defer func() { + _, err := driver.WriteFile(layerFile, content) + So(err, ShouldBeNil) + }() + + index, err := common.GetIndex(imgStore, repoName, testLog) + So(err, ShouldBeNil) + + So(len(index.Manifests), ShouldEqual, 1) + + // get content of layer + imageRes := storage.CheckLayers(repoName, "1.0", []ispec.Descriptor{{Digest: digest}}, imgStore) + So(imageRes.Status, ShouldEqual, "affected") + So(imageRes.Error, ShouldEqual, "blob not found") + + buff := bytes.NewBufferString("") + + res, err := storeCtlr.CheckAllBlobsIntegrity(context.Background()) + res.PrintScrubResults(buff) + So(err, ShouldBeNil) + + space := regexp.MustCompile(`\s+`) + str := space.ReplaceAllString(buff.String(), " ") + actual := strings.TrimSpace(str) + So(actual, ShouldContainSubstring, "REPOSITORY TAG STATUS AFFECTED BLOB ERROR") + So(actual, ShouldContainSubstring, fmt.Sprintf("test 1.0 affected %s blob not found", layerDig)) + }) + + Convey("Scrub index with missing manifest blob - graceful handling", func() { + // Create a multiarch image with multiple manifests + multiarchImage := CreateMultiarchWith().RandomImages(2).Build() + err = WriteMultiArchImageToFileSystem(multiarchImage, repoName, "2.0", storeCtlr) + So(err, ShouldBeNil) + + // Get the index to find the index manifest digest + idx, err := common.GetIndex(imgStore, repoName, testLog) + So(err, ShouldBeNil) + + // Find the index manifest + var indexManifestDesc ispec.Descriptor + + for _, desc := range idx.Manifests { + if desc.MediaType == ispec.MediaTypeImageIndex { + indexManifestDesc = desc + + break + } + } + + // Get the index content to find the manifest digests within it + indexBlob, err := imgStore.GetBlobContent(repoName, indexManifestDesc.Digest) + So(err, ShouldBeNil) + + var indexContent ispec.Index + err = json.Unmarshal(indexBlob, &indexContent) + So(err, ShouldBeNil) + + // Delete one of the manifest blobs within the index (but not all) + missingManifestDig := indexContent.Manifests[0].Digest.Encoded() + missingManifestFile := path.Join(imgStore.RootDir(), repoName, "/blobs/sha256", missingManifestDig) + err = driver.Delete(missingManifestFile) + So(err, ShouldBeNil) + + buff := bytes.NewBufferString("") + + res, err := storeCtlr.CheckAllBlobsIntegrity(context.Background()) + res.PrintScrubResults(buff) + So(err, ShouldBeNil) + + space := regexp.MustCompile(`\s+`) + str := space.ReplaceAllString(buff.String(), " ") + actual := strings.TrimSpace(str) + + // Should mark the index as affected due to missing manifest + So(actual, ShouldContainSubstring, "REPOSITORY TAG STATUS AFFECTED BLOB ERROR") + So(actual, ShouldContainSubstring, "test 2.0 affected") + // Should continue processing and report the missing manifest + So(actual, ShouldContainSubstring, missingManifestDig) + }) + + Convey("Scrub index with non-missing error on manifest blob via file permissions", func() { + // Skip for non-local storage + if driver.Name() != storageConstants.LocalStorageDriverName { + return + } + + // Create a multiarch image with multiple manifests + multiarchImage := CreateMultiarchWith().RandomImages(2).Build() + err = WriteMultiArchImageToFileSystem(multiarchImage, repoName, "2.1", storeCtlr) + So(err, ShouldBeNil) + + // Get the index to find the index manifest digest + idx, err := common.GetIndex(imgStore, repoName, testLog) + So(err, ShouldBeNil) + + // Find the index manifest + var indexManifestDesc ispec.Descriptor + + for _, desc := range idx.Manifests { + if desc.MediaType == ispec.MediaTypeImageIndex { + indexManifestDesc = desc + + break + } + } + + // Get the index content to find the manifest digests within it + indexBlob, err := imgStore.GetBlobContent(repoName, indexManifestDesc.Digest) + So(err, ShouldBeNil) + + var indexContent ispec.Index + err = json.Unmarshal(indexBlob, &indexContent) + So(err, ShouldBeNil) + + // Remove read permissions on one of the manifest blobs to cause a permission denied error (non-missing error) + manifestDig := indexContent.Manifests[0].Digest.Encoded() + manifestFile := path.Join(imgStore.RootDir(), repoName, "/blobs/sha256", manifestDig) + err = os.Chmod(manifestFile, 0o000) + So(err, ShouldBeNil) + + // Restore permissions after test + defer func() { + _ = os.Chmod(manifestFile, 0o644) + }() + + buff := bytes.NewBufferString("") + + res, err := storeCtlr.CheckAllBlobsIntegrity(context.Background()) + res.PrintScrubResults(buff) + So(err, ShouldBeNil) + + space := regexp.MustCompile(`\s+`) + str := space.ReplaceAllString(buff.String(), " ") + actual := strings.TrimSpace(str) + + // Should mark the index as affected due to non-missing error on manifest + So(actual, ShouldContainSubstring, "REPOSITORY TAG STATUS AFFECTED BLOB ERROR") + So(actual, ShouldContainSubstring, "test 2.1 affected") + // Should report the manifest digest as affected blob + So(actual, ShouldContainSubstring, manifestDig) + // Should have "bad blob digest" error + So(actual, ShouldContainSubstring, "bad blob digest") + }) + + Convey("Scrub index", func() { + newImage := CreateRandomImage() + newManifestDigest := newImage.ManifestDescriptor.Digest + + err = WriteImageToFileSystem(newImage, repoName, "2.0", storeCtlr) + So(err, ShouldBeNil) + + idx, err := common.GetIndex(imgStore, repoName, testLog) + So(err, ShouldBeNil) + + manifestDescriptor, ok := common.GetManifestDescByReference(idx, image.ManifestDescriptor.Digest.String()) + So(ok, ShouldBeTrue) + + var index ispec.Index + index.SchemaVersion = 2 + index.Subject = &manifestDescriptor + index.Manifests = []ispec.Descriptor{ + { + MediaType: ispec.MediaTypeImageManifest, + Digest: newManifestDigest, + Size: newImage.ManifestDescriptor.Size, + }, + } + + indexBlob, err := json.Marshal(index) + So(err, ShouldBeNil) + indexDigest, _, err := imgStore.PutImageManifest(repoName, "", ispec.MediaTypeImageIndex, indexBlob) + So(err, ShouldBeNil) + + buff := bytes.NewBufferString("") + + res, err := storeCtlr.CheckAllBlobsIntegrity(context.Background()) + res.PrintScrubResults(buff) + So(err, ShouldBeNil) + + space := regexp.MustCompile(`\s+`) + str := space.ReplaceAllString(buff.String(), " ") + actual := strings.TrimSpace(str) + So(actual, ShouldContainSubstring, "REPOSITORY TAG STATUS AFFECTED BLOB ERROR") + So(actual, ShouldContainSubstring, "test 1.0 ok") + So(actual, ShouldContainSubstring, "test ok") + + // test scrub context done + buff = bytes.NewBufferString("") + + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + res, err = storeCtlr.CheckAllBlobsIntegrity(ctx) + res.PrintScrubResults(buff) + So(err, ShouldNotBeNil) + + str = space.ReplaceAllString(buff.String(), " ") + actual = strings.TrimSpace(str) + So(actual, ShouldContainSubstring, "REPOSITORY TAG STATUS AFFECTED BLOB ERROR") + So(actual, ShouldNotContainSubstring, "test 1.0 ok") + So(actual, ShouldNotContainSubstring, "test ok") + + // test scrub index - errors + manifestFile := path.Join(imgStore.RootDir(), repoName, "/blobs/sha256", newManifestDigest.Encoded()) + _, err = driver.WriteFile(manifestFile, []byte("invalid content")) + So(err, ShouldBeNil) + + buff = bytes.NewBufferString("") + + res, err = storeCtlr.CheckAllBlobsIntegrity(context.Background()) + res.PrintScrubResults(buff) + So(err, ShouldBeNil) + + str = space.ReplaceAllString(buff.String(), " ") + actual = strings.TrimSpace(str) + So(actual, ShouldContainSubstring, "REPOSITORY TAG STATUS AFFECTED BLOB ERROR") + So(actual, ShouldContainSubstring, "test affected") + + // delete content of manifest file + err = driver.Delete(manifestFile) + So(err, ShouldBeNil) + + buff = bytes.NewBufferString("") + + res, err = storeCtlr.CheckAllBlobsIntegrity(context.Background()) + res.PrintScrubResults(buff) + So(err, ShouldBeNil) + + str = space.ReplaceAllString(buff.String(), " ") + actual = strings.TrimSpace(str) + So(actual, ShouldContainSubstring, "REPOSITORY TAG STATUS AFFECTED BLOB ERROR") + So(actual, ShouldContainSubstring, "test affected") + + indexFile := path.Join(imgStore.RootDir(), repoName, "/blobs/sha256", indexDigest.Encoded()) + err = driver.Delete(indexFile) + So(err, ShouldBeNil) + + buff = bytes.NewBufferString("") + + res, err = storeCtlr.CheckAllBlobsIntegrity(context.Background()) + res.PrintScrubResults(buff) + So(err, ShouldBeNil) + + str = space.ReplaceAllString(buff.String(), " ") + actual = strings.TrimSpace(str) + So(actual, ShouldContainSubstring, "REPOSITORY TAG STATUS AFFECTED BLOB ERROR") + So(actual, ShouldContainSubstring, "test 1.0 ok") + So(actual, ShouldNotContainSubstring, "test affected") + + index.Manifests[0].MediaType = "invalid" + indexBlob, err = json.Marshal(index) + So(err, ShouldBeNil) + + _, err = driver.WriteFile(indexFile, indexBlob) + So(err, ShouldBeNil) + + buff = bytes.NewBufferString("") + + res, err = storeCtlr.CheckAllBlobsIntegrity(context.Background()) + res.PrintScrubResults(buff) + So(err, ShouldBeNil) + + _, _, err = storage.CheckManifestAndConfig(repoName, index.Manifests[0], []byte{}, imgStore) + So(err, ShouldNotBeNil) + So(err, ShouldEqual, zerr.ErrBadManifest) + + str = space.ReplaceAllString(buff.String(), " ") + actual = strings.TrimSpace(str) + So(actual, ShouldContainSubstring, "REPOSITORY TAG STATUS AFFECTED BLOB ERROR") + So(actual, ShouldContainSubstring, "test affected") + + _, err = driver.WriteFile(indexFile, []byte("invalid cotent")) + So(err, ShouldBeNil) + + defer func() { + err := driver.Delete(indexFile) + So(err, ShouldBeNil) + }() + + buff = bytes.NewBufferString("") + + res, err = storeCtlr.CheckAllBlobsIntegrity(context.Background()) + res.PrintScrubResults(buff) + So(err, ShouldBeNil) + + str = space.ReplaceAllString(buff.String(), " ") + actual = strings.TrimSpace(str) + So(actual, ShouldContainSubstring, "REPOSITORY TAG STATUS AFFECTED BLOB ERROR") + So(actual, ShouldContainSubstring, "test affected") + }) + + Convey("Manifest not found", func() { + // delete manifest file + manifestDig := image.ManifestDescriptor.Digest.Encoded() + manifestFile := path.Join(imgStore.RootDir(), repoName, "/blobs/sha256", manifestDig) + err = driver.Delete(manifestFile) + So(err, ShouldBeNil) + + buff := bytes.NewBufferString("") + + res, err := storeCtlr.CheckAllBlobsIntegrity(context.Background()) + res.PrintScrubResults(buff) + So(err, ShouldBeNil) + + space := regexp.MustCompile(`\s+`) + str := space.ReplaceAllString(buff.String(), " ") + actual := strings.TrimSpace(str) + So(actual, ShouldContainSubstring, "REPOSITORY TAG STATUS AFFECTED BLOB ERROR") + So(actual, ShouldNotContainSubstring, fmt.Sprintf("test 1.0 affected %s blob not found", manifestDig)) + + index, err := common.GetIndex(imgStore, repoName, testLog) + So(err, ShouldBeNil) + + So(len(index.Manifests), ShouldEqual, 1) + }) + + Convey("use the result of an already scrubed manifest which is the subject of the current manifest", func() { + index, err := common.GetIndex(imgStore, repoName, testLog) + So(err, ShouldBeNil) + + manifestDescriptor, ok := common.GetManifestDescByReference(index, image.ManifestDescriptor.Digest.String()) + So(ok, ShouldBeTrue) + + err = WriteImageToFileSystem(CreateDefaultImageWith().Subject(&manifestDescriptor).Build(), + repoName, "0.0.1", storeCtlr) + So(err, ShouldBeNil) + + buff := bytes.NewBufferString("") + + res, err := storeCtlr.CheckAllBlobsIntegrity(context.Background()) + res.PrintScrubResults(buff) + So(err, ShouldBeNil) + + space := regexp.MustCompile(`\s+`) + str := space.ReplaceAllString(buff.String(), " ") + actual := strings.TrimSpace(str) + So(actual, ShouldContainSubstring, "REPOSITORY TAG STATUS AFFECTED BLOB ERROR") + So(actual, ShouldContainSubstring, "test 1.0 ok") + So(actual, ShouldContainSubstring, "test 0.0.1 ok") + }) + + Convey("preserve affected status when CheckLayers would overwrite it", func() { + // Create an image with a subject + index, err := common.GetIndex(imgStore, repoName, testLog) + So(err, ShouldBeNil) + + manifestDescriptor, ok := common.GetManifestDescByReference(index, image.ManifestDescriptor.Digest.String()) + So(ok, ShouldBeTrue) + + subjectImage := CreateDefaultImageWith().Subject(&manifestDescriptor).Build() + err = WriteImageToFileSystem(subjectImage, repoName, "0.0.3", storeCtlr) + So(err, ShouldBeNil) + + // Delete the subject manifest to mark it as affected + subjectManifestDig := manifestDescriptor.Digest.Encoded() + subjectManifestFile := path.Join(imgStore.RootDir(), repoName, "/blobs/sha256", subjectManifestDig) + err = driver.Delete(subjectManifestFile) + So(err, ShouldBeNil) + + buff := bytes.NewBufferString("") + + res, err := storeCtlr.CheckAllBlobsIntegrity(context.Background()) + res.PrintScrubResults(buff) + So(err, ShouldBeNil) + + space := regexp.MustCompile(`\s+`) + str := space.ReplaceAllString(buff.String(), " ") + actual := strings.TrimSpace(str) + + // The manifest with the missing subject should be marked as affected + So(actual, ShouldContainSubstring, "REPOSITORY TAG STATUS AFFECTED BLOB ERROR") + So(actual, ShouldContainSubstring, "test 0.0.3 affected") + // Even if CheckLayers would pass, the affected status from the missing subject should be preserved + So(actual, ShouldContainSubstring, subjectManifestDig) + }) + + Convey("the subject of the current manifest doesn't exist", func() { + index, err := common.GetIndex(imgStore, repoName, testLog) + So(err, ShouldBeNil) + + manifestDescriptor, ok := common.GetManifestDescByReference(index, image.ManifestDescriptor.Digest.String()) + So(ok, ShouldBeTrue) + + err = WriteImageToFileSystem(CreateDefaultImageWith().Subject(&manifestDescriptor).Build(), + repoName, "0.0.2", storeCtlr) + So(err, ShouldBeNil) + + // get content of manifest file + content, _, _, err := imgStore.GetImageManifest(repoName, manifestDescriptor.Digest.String()) + So(err, ShouldBeNil) + + // delete content of manifest file + manifestDig := image.ManifestDescriptor.Digest.Encoded() + manifestFile := path.Join(imgStore.RootDir(), repoName, "/blobs/sha256", manifestDig) + err = driver.Delete(manifestFile) + So(err, ShouldBeNil) + + defer func() { + // put manifest content back to file + _, err = driver.WriteFile(manifestFile, content) + So(err, ShouldBeNil) + }() + + buff := bytes.NewBufferString("") + + res, err := storeCtlr.CheckAllBlobsIntegrity(context.Background()) + res.PrintScrubResults(buff) + So(err, ShouldBeNil) + + space := regexp.MustCompile(`\s+`) + str := space.ReplaceAllString(buff.String(), " ") + actual := strings.TrimSpace(str) + So(actual, ShouldContainSubstring, "REPOSITORY TAG STATUS AFFECTED BLOB ERROR") + So(actual, ShouldContainSubstring, "test 0.0.2 affected") + }) + + Convey("the subject of the current index doesn't exist", func() { + index, err := common.GetIndex(imgStore, repoName, testLog) + So(err, ShouldBeNil) + + manifestDescriptor, ok := common.GetManifestDescByReference(index, image.ManifestDescriptor.Digest.String()) + So(ok, ShouldBeTrue) + + err = WriteMultiArchImageToFileSystem(CreateMultiarchWith().RandomImages(1).Subject(&manifestDescriptor).Build(), + repoName, "0.0.2", storeCtlr) + So(err, ShouldBeNil) + + // get content of manifest file + content, _, _, err := imgStore.GetImageManifest(repoName, manifestDescriptor.Digest.String()) + So(err, ShouldBeNil) + + // delete content of manifest file + manifestDig := image.ManifestDescriptor.Digest.Encoded() + manifestFile := path.Join(imgStore.RootDir(), repoName, "/blobs/sha256", manifestDig) + err = driver.Delete(manifestFile) + So(err, ShouldBeNil) + + defer func() { + // put manifest content back to file + _, err = driver.WriteFile(manifestFile, content) + So(err, ShouldBeNil) + }() + + buff := bytes.NewBufferString("") + + res, err := storeCtlr.CheckAllBlobsIntegrity(context.Background()) + res.PrintScrubResults(buff) + So(err, ShouldBeNil) + + space := regexp.MustCompile(`\s+`) + str := space.ReplaceAllString(buff.String(), " ") + actual := strings.TrimSpace(str) + So(actual, ShouldContainSubstring, "REPOSITORY TAG STATUS AFFECTED BLOB ERROR") + So(actual, ShouldContainSubstring, "test 0.0.2 affected") + }) + + Convey("test errors", func() { + mockedImgStore := mocks.MockedImageStore{ + GetRepositoriesFn: func() ([]string, error) { + return []string{repoName}, nil + }, + ValidateRepoFn: func(name string) (bool, error) { + return false, nil + }, + } + + storeController := storage.StoreController{} + storeController.DefaultStore = mockedImgStore + + _, err := storeController.CheckAllBlobsIntegrity(context.Background()) + So(err, ShouldNotBeNil) + So(err, ShouldEqual, zerr.ErrRepoBadLayout) + + mockedImgStore = mocks.MockedImageStore{ + GetRepositoriesFn: func() ([]string, error) { + return []string{repoName}, nil + }, + GetIndexContentFn: func(repo string) ([]byte, error) { + return []byte{}, errUnexpectedError + }, + } + + storeController.DefaultStore = mockedImgStore + + _, err = storeController.CheckAllBlobsIntegrity(context.Background()) + So(err, ShouldNotBeNil) + So(err, ShouldEqual, errUnexpectedError) + + manifestDigest := godigest.FromString("abcd") + + mockedImgStore = mocks.MockedImageStore{ + GetRepositoriesFn: func() ([]string, error) { + return []string{repoName}, nil + }, + GetIndexContentFn: func(repo string) ([]byte, error) { + var index ispec.Index + index.SchemaVersion = 2 + index.Manifests = []ispec.Descriptor{ + { + MediaType: "InvalidMediaType", + Digest: manifestDigest, + Size: int64(100), + Annotations: map[string]string{ispec.AnnotationRefName: "1.0"}, + }, + } + + return json.Marshal(index) + }, + } + + storeController.DefaultStore = mockedImgStore + + res, err := storeController.CheckAllBlobsIntegrity(context.Background()) + So(err, ShouldBeNil) + + buff := bytes.NewBufferString("") + res.PrintScrubResults(buff) + + space := regexp.MustCompile(`\s+`) + str := space.ReplaceAllString(buff.String(), " ") + actual := strings.TrimSpace(str) + So(actual, ShouldContainSubstring, "REPOSITORY TAG STATUS AFFECTED BLOB ERROR") + So(actual, ShouldContainSubstring, fmt.Sprintf("%s 1.0 affected %s invalid manifest content", + repoName, manifestDigest.Encoded())) + }) + + Convey("scrub with non-missing error on manifest subject blob via file permissions", func() { + // Skip for non-local storage + if driver.Name() != storageConstants.LocalStorageDriverName { + return + } + + index, err := common.GetIndex(imgStore, repoName, testLog) + So(err, ShouldBeNil) + + manifestDescriptor, ok := common.GetManifestDescByReference(index, image.ManifestDescriptor.Digest.String()) + So(ok, ShouldBeTrue) + + // Create an image with a subject + subjectImage := CreateDefaultImageWith().Subject(&manifestDescriptor).Build() + err = WriteImageToFileSystem(subjectImage, repoName, "0.0.6", storeCtlr) + So(err, ShouldBeNil) + + // Get the subject manifest digest + subjectManifestDig := manifestDescriptor.Digest.Encoded() + subjectManifestFile := path.Join(imgStore.RootDir(), repoName, "/blobs/sha256", subjectManifestDig) + + // Remove read permissions to cause a permission denied error (non-missing error) + err = os.Chmod(subjectManifestFile, 0o000) + So(err, ShouldBeNil) + + // Restore permissions after test + defer func() { + _ = os.Chmod(subjectManifestFile, 0o644) + }() + + buff := bytes.NewBufferString("") + + res, err := storeCtlr.CheckAllBlobsIntegrity(context.Background()) + res.PrintScrubResults(buff) + So(err, ShouldBeNil) + + space := regexp.MustCompile(`\s+`) + str := space.ReplaceAllString(buff.String(), " ") + actual := strings.TrimSpace(str) + + // Should mark the manifest as affected due to non-missing error on subject + So(actual, ShouldContainSubstring, "REPOSITORY TAG STATUS AFFECTED BLOB ERROR") + So(actual, ShouldContainSubstring, "test 0.0.6 affected") + // Should report the subject digest as affected blob + So(actual, ShouldContainSubstring, subjectManifestDig) + // Should have "bad blob digest" error + So(actual, ShouldContainSubstring, "bad blob digest") + }) + + Convey("scrub with non-missing error on index subject blob via file permissions", func() { + // Skip for non-local storage + if driver.Name() != storageConstants.LocalStorageDriverName { + return + } + + index, err := common.GetIndex(imgStore, repoName, testLog) + So(err, ShouldBeNil) + + manifestDescriptor, ok := common.GetManifestDescByReference(index, image.ManifestDescriptor.Digest.String()) + So(ok, ShouldBeTrue) + + // Create a multiarch image with a subject + err = WriteMultiArchImageToFileSystem(CreateMultiarchWith().RandomImages(1).Subject(&manifestDescriptor).Build(), + repoName, "0.0.7", storeCtlr) + So(err, ShouldBeNil) + + // Get the subject manifest digest + subjectManifestDig := manifestDescriptor.Digest.Encoded() + subjectManifestFile := path.Join(imgStore.RootDir(), repoName, "/blobs/sha256", subjectManifestDig) + + // Remove read permissions to cause a permission denied error (non-missing error) + err = os.Chmod(subjectManifestFile, 0o000) + So(err, ShouldBeNil) + + // Restore permissions after test + defer func() { + _ = os.Chmod(subjectManifestFile, 0o644) + }() + + buff := bytes.NewBufferString("") + + res, err := storeCtlr.CheckAllBlobsIntegrity(context.Background()) + res.PrintScrubResults(buff) + So(err, ShouldBeNil) + + space := regexp.MustCompile(`\s+`) + str := space.ReplaceAllString(buff.String(), " ") + actual := strings.TrimSpace(str) + + // Should mark the index as affected due to non-missing error on subject + So(actual, ShouldContainSubstring, "REPOSITORY TAG STATUS AFFECTED BLOB ERROR") + So(actual, ShouldContainSubstring, "test 0.0.7 affected") + // Should report the subject digest as affected blob + So(actual, ShouldContainSubstring, subjectManifestDig) + // Should have "bad blob digest" error + So(actual, ShouldContainSubstring, "bad blob digest") + }) + }) +} diff --git a/pkg/storage/local/local_elevated_test.go b/pkg/storage/local/local_elevated_test.go index f3253cc5..6a70608e 100644 --- a/pkg/storage/local/local_elevated_test.go +++ b/pkg/storage/local/local_elevated_test.go @@ -1,4 +1,4 @@ -//go:build needprivileges +//go:build needprivileges && linux package local_test diff --git a/pkg/storage/s3/s3_test.go b/pkg/storage/s3/s3_test.go index f9dd22d4..534b1b74 100644 --- a/pkg/storage/s3/s3_test.go +++ b/pkg/storage/s3/s3_test.go @@ -8,7 +8,6 @@ import ( "errors" "fmt" "io" - "net/http" "os" "path" "strings" @@ -44,8 +43,6 @@ import ( //nolint:gochecknoglobals var ( testImage = "test" - fileWriterSize = 12 - fileInfoSize = 10 errorText = "new s3 error" errS3 = errors.New(errorText) errCache = errors.New("new cache error") @@ -197,191 +194,6 @@ func runAndGetScheduler() *scheduler.Scheduler { return taskScheduler } -type FileInfoMock struct { - IsDirFn func() bool - SizeFn func() int64 - PathFn func() string -} - -func (f *FileInfoMock) Path() string { - if f != nil && f.PathFn != nil { - return f.PathFn() - } - - return "" -} - -func (f *FileInfoMock) Size() int64 { - if f != nil && f.SizeFn != nil { - return f.SizeFn() - } - - return int64(fileInfoSize) -} - -func (f *FileInfoMock) ModTime() time.Time { - return time.Now() -} - -func (f *FileInfoMock) IsDir() bool { - if f != nil && f.IsDirFn != nil { - return f.IsDirFn() - } - - return true -} - -type FileWriterMock struct { - WriteFn func([]byte) (int, error) - CancelFn func() error - CommitFn func() error - CloseFn func() error -} - -func (f *FileWriterMock) Size() int64 { - return int64(fileWriterSize) -} - -func (f *FileWriterMock) Cancel(_ context.Context) error { - if f != nil && f.CancelFn != nil { - return f.CancelFn() - } - - return nil -} - -func (f *FileWriterMock) Commit(_ context.Context) error { - if f != nil && f.CommitFn != nil { - return f.CommitFn() - } - - return nil -} - -func (f *FileWriterMock) Write(p []byte) (int, error) { - if f != nil && f.WriteFn != nil { - return f.WriteFn(p) - } - - return 10, nil -} - -func (f *FileWriterMock) Close() error { - if f != nil && f.CloseFn != nil { - return f.CloseFn() - } - - return nil -} - -type StorageDriverMock struct { - NameFn func() string - GetContentFn func(ctx context.Context, path string) ([]byte, error) - PutContentFn func(ctx context.Context, path string, content []byte) error - ReaderFn func(ctx context.Context, path string, offset int64) (io.ReadCloser, error) - WriterFn func(ctx context.Context, path string, isAppend bool) (driver.FileWriter, error) - StatFn func(ctx context.Context, path string) (driver.FileInfo, error) - ListFn func(ctx context.Context, path string) ([]string, error) - MoveFn func(ctx context.Context, sourcePath, destPath string) error - DeleteFn func(ctx context.Context, path string) error - WalkFn func(ctx context.Context, path string, f driver.WalkFn, options ...func(*driver.WalkOptions)) error - RedirectURLFn func(r *http.Request, path string) (string, error) -} - -func (s *StorageDriverMock) RedirectURL(r *http.Request, path string) (string, error) { - if s != nil && s.RedirectURLFn != nil { - return s.RedirectURLFn(r, path) - } - - return "", nil -} - -func (s *StorageDriverMock) Name() string { - if s != nil && s.NameFn != nil { - return s.NameFn() - } - - return "" -} - -func (s *StorageDriverMock) GetContent(ctx context.Context, path string) ([]byte, error) { - if s != nil && s.GetContentFn != nil { - return s.GetContentFn(ctx, path) - } - - return []byte{}, nil -} - -func (s *StorageDriverMock) PutContent(ctx context.Context, path string, content []byte) error { - if s != nil && s.PutContentFn != nil { - return s.PutContentFn(ctx, path, content) - } - - return nil -} - -func (s *StorageDriverMock) Reader(ctx context.Context, path string, offset int64) (io.ReadCloser, error) { - if s != nil && s.ReaderFn != nil { - return s.ReaderFn(ctx, path, offset) - } - - return io.NopCloser(strings.NewReader("")), nil -} - -func (s *StorageDriverMock) Writer(ctx context.Context, path string, isAppend bool) (driver.FileWriter, error) { - if s != nil && s.WriterFn != nil { - return s.WriterFn(ctx, path, isAppend) - } - - return &FileWriterMock{}, nil -} - -func (s *StorageDriverMock) Stat(ctx context.Context, path string) (driver.FileInfo, error) { - if s != nil && s.StatFn != nil { - return s.StatFn(ctx, path) - } - - return &FileInfoMock{}, nil -} - -func (s *StorageDriverMock) List(ctx context.Context, path string) ([]string, error) { - if s != nil && s.ListFn != nil { - return s.ListFn(ctx, path) - } - - return []string{"a"}, nil -} - -func (s *StorageDriverMock) Move(ctx context.Context, sourcePath, destPath string) error { - if s != nil && s.MoveFn != nil { - return s.MoveFn(ctx, sourcePath, destPath) - } - - return nil -} - -func (s *StorageDriverMock) Delete(ctx context.Context, path string) error { - if s != nil && s.DeleteFn != nil { - return s.DeleteFn(ctx, path) - } - - return nil -} - -func (s *StorageDriverMock) URLFor(ctx context.Context, path string, options map[string]any) (string, error) { - return "", nil -} - -func (s *StorageDriverMock) Walk(ctx context.Context, path string, f driver.WalkFn, - options ...func(*driver.WalkOptions), -) error { - if s != nil && s.WalkFn != nil { - return s.WalkFn(ctx, path, f, options...) - } - - return nil -} - func TestStorageDriverStatFunction(t *testing.T) { tskip.SkipS3(t) @@ -795,7 +607,7 @@ func TestNegativeCasesObjectsStorage(t *testing.T) { }) Convey("Test storage driver errors", func(c C) { - imgStore = createMockStorage(testDir, tdir, false, &StorageDriverMock{ + imgStore = createMockStorage(testDir, tdir, false, &mocks.StorageDriverMock{ ListFn: func(ctx context.Context, path string) ([]string, error) { return []string{testImage}, errS3 }, @@ -809,7 +621,7 @@ func TestNegativeCasesObjectsStorage(t *testing.T) { return errS3 }, WriterFn: func(ctx context.Context, path string, isAppend bool) (driver.FileWriter, error) { - return &FileWriterMock{}, errS3 + return &mocks.FileWriterMock{}, errS3 }, ReaderFn: func(ctx context.Context, path string, offset int64) (io.ReadCloser, error) { return io.NopCloser(strings.NewReader("")), errS3 @@ -818,7 +630,7 @@ func TestNegativeCasesObjectsStorage(t *testing.T) { return errS3 }, StatFn: func(ctx context.Context, path string) (driver.FileInfo, error) { - return &FileInfoMock{}, errS3 + return &mocks.FileInfoMock{}, errS3 }, DeleteFn: func(ctx context.Context, path string) error { return errS3 @@ -872,7 +684,7 @@ func TestNegativeCasesObjectsStorage(t *testing.T) { Convey("Test ValidateRepo", func(c C) { tdir := t.TempDir() - imgStore = createMockStorage(testDir, tdir, false, &StorageDriverMock{ + imgStore = createMockStorage(testDir, tdir, false, &mocks.StorageDriverMock{ ListFn: func(ctx context.Context, path string) ([]string, error) { return []string{testImage, testImage}, errS3 }, @@ -883,9 +695,9 @@ func TestNegativeCasesObjectsStorage(t *testing.T) { }) Convey("Test GetRepositories", func(c C) { - imgStore = createMockStorage(testDir, tdir, false, &StorageDriverMock{ + imgStore = createMockStorage(testDir, tdir, false, &mocks.StorageDriverMock{ WalkFn: func(ctx context.Context, path string, f driver.WalkFn, options ...func(*driver.WalkOptions)) error { - return f(new(FileInfoMock)) + return f(new(mocks.FileInfoMock)) }, }) repos, err := imgStore.GetRepositories() @@ -894,7 +706,7 @@ func TestNegativeCasesObjectsStorage(t *testing.T) { }) Convey("Test DeleteImageManifest", func(c C) { - imgStore = createMockStorage(testDir, tdir, false, &StorageDriverMock{ + imgStore = createMockStorage(testDir, tdir, false, &mocks.StorageDriverMock{ GetContentFn: func(ctx context.Context, path string) ([]byte, error) { return []byte{}, errS3 }, @@ -904,7 +716,7 @@ func TestNegativeCasesObjectsStorage(t *testing.T) { }) Convey("Test GetIndexContent", func(c C) { - imgStore = createMockStorage(testDir, tdir, false, &StorageDriverMock{ + imgStore = createMockStorage(testDir, tdir, false, &mocks.StorageDriverMock{ GetContentFn: func(ctx context.Context, path string) ([]byte, error) { return []byte{}, driver.PathNotFoundError{} }, @@ -914,13 +726,13 @@ func TestNegativeCasesObjectsStorage(t *testing.T) { }) Convey("Test DeleteImageManifest2", func(c C) { - imgStore = createMockStorage(testDir, tdir, false, &StorageDriverMock{}) + imgStore = createMockStorage(testDir, tdir, false, &mocks.StorageDriverMock{}) err := imgStore.DeleteImageManifest(testImage, "1.0", false) So(err, ShouldNotBeNil) }) Convey("Test NewBlobUpload", func(c C) { - imgStore = createMockStorage(testDir, tdir, false, &StorageDriverMock{ + imgStore = createMockStorage(testDir, tdir, false, &mocks.StorageDriverMock{ WriterFn: func(ctx context.Context, path string, isAppend bool) (driver.FileWriter, error) { return nil, errS3 }, @@ -930,7 +742,7 @@ func TestNegativeCasesObjectsStorage(t *testing.T) { }) Convey("Test GetBlobUpload", func(c C) { - imgStore = createMockStorage(testDir, tdir, false, &StorageDriverMock{ + imgStore = createMockStorage(testDir, tdir, false, &mocks.StorageDriverMock{ WriterFn: func(ctx context.Context, path string, isAppend bool) (driver.FileWriter, error) { return nil, errS3 }, @@ -940,7 +752,7 @@ func TestNegativeCasesObjectsStorage(t *testing.T) { }) Convey("Test BlobUploadInfo", func(c C) { - imgStore = createMockStorage(testDir, tdir, false, &StorageDriverMock{ + imgStore = createMockStorage(testDir, tdir, false, &mocks.StorageDriverMock{ WriterFn: func(ctx context.Context, path string, isAppend bool) (driver.FileWriter, error) { return nil, errS3 }, @@ -950,9 +762,9 @@ func TestNegativeCasesObjectsStorage(t *testing.T) { }) Convey("Test PutBlobChunkStreamed", func(c C) { - imgStore = createMockStorage(testDir, tdir, false, &StorageDriverMock{ + imgStore = createMockStorage(testDir, tdir, false, &mocks.StorageDriverMock{ WriterFn: func(ctx context.Context, path string, isAppend bool) (driver.FileWriter, error) { - return &FileWriterMock{}, errS3 + return &mocks.FileWriterMock{}, errS3 }, }) _, err := imgStore.PutBlobChunkStreamed(testImage, "uuid", io.NopCloser(strings.NewReader(""))) @@ -960,9 +772,9 @@ func TestNegativeCasesObjectsStorage(t *testing.T) { }) Convey("Test PutBlobChunkStreamed2", func(c C) { - imgStore = createMockStorage(testDir, tdir, false, &StorageDriverMock{ + imgStore = createMockStorage(testDir, tdir, false, &mocks.StorageDriverMock{ WriterFn: func(ctx context.Context, path string, isAppend bool) (driver.FileWriter, error) { - return &FileWriterMock{WriteFn: func(b []byte) (int, error) { + return &mocks.FileWriterMock{WriteFn: func(b []byte) (int, error) { return 0, errS3 }}, errS3 }, @@ -972,9 +784,9 @@ func TestNegativeCasesObjectsStorage(t *testing.T) { }) Convey("Test PutBlobChunk", func(c C) { - imgStore = createMockStorage(testDir, tdir, false, &StorageDriverMock{ + imgStore = createMockStorage(testDir, tdir, false, &mocks.StorageDriverMock{ WriterFn: func(ctx context.Context, path string, isAppend bool) (driver.FileWriter, error) { - return &FileWriterMock{}, errS3 + return &mocks.FileWriterMock{}, errS3 }, }) _, err := imgStore.PutBlobChunk(testImage, "uuid", 0, 100, io.NopCloser(strings.NewReader(""))) @@ -982,9 +794,9 @@ func TestNegativeCasesObjectsStorage(t *testing.T) { }) Convey("Test PutBlobChunk2", func(c C) { - imgStore = createMockStorage(testDir, tdir, false, &StorageDriverMock{ + imgStore = createMockStorage(testDir, tdir, false, &mocks.StorageDriverMock{ WriterFn: func(ctx context.Context, path string, isAppend bool) (driver.FileWriter, error) { - return &FileWriterMock{ + return &mocks.FileWriterMock{ WriteFn: func(b []byte) (int, error) { return 0, errS3 }, @@ -999,9 +811,9 @@ func TestNegativeCasesObjectsStorage(t *testing.T) { }) Convey("Test PutBlobChunk3", func(c C) { - imgStore = createMockStorage(testDir, tdir, false, &StorageDriverMock{ + imgStore = createMockStorage(testDir, tdir, false, &mocks.StorageDriverMock{ WriterFn: func(ctx context.Context, path string, isAppend bool) (driver.FileWriter, error) { - return &FileWriterMock{ + return &mocks.FileWriterMock{ WriteFn: func(b []byte) (int, error) { return 0, errS3 }, @@ -1013,9 +825,9 @@ func TestNegativeCasesObjectsStorage(t *testing.T) { }) Convey("Test PutBlobChunk4", func(c C) { - imgStore = createMockStorage(testDir, tdir, false, &StorageDriverMock{ + imgStore = createMockStorage(testDir, tdir, false, &mocks.StorageDriverMock{ WriterFn: func(ctx context.Context, path string, isAppend bool) (driver.FileWriter, error) { - return &FileWriterMock{}, driver.PathNotFoundError{} + return &mocks.FileWriterMock{}, driver.PathNotFoundError{} }, }) _, err := imgStore.PutBlobChunk(testImage, "uuid", 0, 100, io.NopCloser(strings.NewReader(""))) @@ -1023,9 +835,9 @@ func TestNegativeCasesObjectsStorage(t *testing.T) { }) Convey("Test FinishBlobUpload", func(c C) { - imgStore = createMockStorage(testDir, tdir, false, &StorageDriverMock{ + imgStore = createMockStorage(testDir, tdir, false, &mocks.StorageDriverMock{ WriterFn: func(ctx context.Context, path string, isAppend bool) (driver.FileWriter, error) { - return &FileWriterMock{ + return &mocks.FileWriterMock{ CommitFn: func() error { return errS3 }, @@ -1038,9 +850,9 @@ func TestNegativeCasesObjectsStorage(t *testing.T) { }) Convey("Test FinishBlobUpload2", func(c C) { - imgStore = createMockStorage(testDir, tdir, false, &StorageDriverMock{ + imgStore = createMockStorage(testDir, tdir, false, &mocks.StorageDriverMock{ WriterFn: func(ctx context.Context, path string, isAppend bool) (driver.FileWriter, error) { - return &FileWriterMock{ + return &mocks.FileWriterMock{ CloseFn: func() error { return errS3 }, @@ -1053,7 +865,7 @@ func TestNegativeCasesObjectsStorage(t *testing.T) { }) Convey("Test FinishBlobUpload3", func(c C) { - imgStore = createMockStorage(testDir, tdir, false, &StorageDriverMock{ + imgStore = createMockStorage(testDir, tdir, false, &mocks.StorageDriverMock{ ReaderFn: func(ctx context.Context, path string, offset int64) (io.ReadCloser, error) { return nil, errS3 }, @@ -1064,7 +876,7 @@ func TestNegativeCasesObjectsStorage(t *testing.T) { }) Convey("Test FinishBlobUpload4", func(c C) { - imgStore = createMockStorage(testDir, tdir, false, &StorageDriverMock{ + imgStore = createMockStorage(testDir, tdir, false, &mocks.StorageDriverMock{ MoveFn: func(ctx context.Context, sourcePath, destPath string) error { return errS3 }, @@ -1075,9 +887,9 @@ func TestNegativeCasesObjectsStorage(t *testing.T) { }) Convey("Test FullBlobUpload", func(c C) { - imgStore = createMockStorage(testDir, tdir, false, &StorageDriverMock{ + imgStore = createMockStorage(testDir, tdir, false, &mocks.StorageDriverMock{ WriterFn: func(ctx context.Context, path string, isAppend bool) (driver.FileWriter, error) { - return &FileWriterMock{}, errS3 + return &mocks.FileWriterMock{}, errS3 }, }) d := godigest.FromBytes([]byte("")) @@ -1086,14 +898,14 @@ func TestNegativeCasesObjectsStorage(t *testing.T) { }) Convey("Test FullBlobUpload2", func(c C) { - imgStore = createMockStorage(testDir, tdir, false, &StorageDriverMock{}) + imgStore = createMockStorage(testDir, tdir, false, &mocks.StorageDriverMock{}) d := godigest.FromBytes([]byte(" ")) _, _, err := imgStore.FullBlobUpload(testImage, io.NopCloser(strings.NewReader("")), d) So(err, ShouldNotBeNil) }) Convey("Test FullBlobUpload3", func(c C) { - imgStore = createMockStorage(testDir, tdir, false, &StorageDriverMock{ + imgStore = createMockStorage(testDir, tdir, false, &mocks.StorageDriverMock{ MoveFn: func(ctx context.Context, sourcePath, destPath string) error { return errS3 }, @@ -1104,7 +916,7 @@ func TestNegativeCasesObjectsStorage(t *testing.T) { }) Convey("Test GetBlob", func(c C) { - imgStore = createMockStorage(testDir, tdir, false, &StorageDriverMock{ + imgStore = createMockStorage(testDir, tdir, false, &mocks.StorageDriverMock{ ReaderFn: func(ctx context.Context, path string, offset int64) (io.ReadCloser, error) { return io.NopCloser(strings.NewReader("")), errS3 }, @@ -1115,7 +927,7 @@ func TestNegativeCasesObjectsStorage(t *testing.T) { }) Convey("Test GetBlobContent", func(c C) { - imgStore = createMockStorage(testDir, tdir, false, &StorageDriverMock{ + imgStore = createMockStorage(testDir, tdir, false, &mocks.StorageDriverMock{ GetContentFn: func(ctx context.Context, path string) ([]byte, error) { return []byte{}, errS3 }, @@ -1127,7 +939,7 @@ func TestNegativeCasesObjectsStorage(t *testing.T) { }) Convey("Test DeleteBlob", func(c C) { - imgStore = createMockStorage(testDir, tdir, false, &StorageDriverMock{ + imgStore = createMockStorage(testDir, tdir, false, &mocks.StorageDriverMock{ DeleteFn: func(ctx context.Context, path string) error { return errS3 }, @@ -1138,7 +950,7 @@ func TestNegativeCasesObjectsStorage(t *testing.T) { }) Convey("Test GetReferrers", func(c C) { - imgStore = createMockStorage(testDir, tdir, false, &StorageDriverMock{}) + imgStore = createMockStorage(testDir, tdir, false, &mocks.StorageDriverMock{}) d := godigest.FromBytes([]byte("")) _, err := imgStore.GetReferrers(testImage, d, []string{"application/image"}) So(err, ShouldNotBeNil) @@ -2235,7 +2047,7 @@ func TestNextRepositoryMockStoreDriver(t *testing.T) { // some s3 implementations (eg, digitalocean spaces) will return pathnotfounderror for walk but not list // This code cannot be reliably covered by end to end tests Convey("Trigger PathNotFound error when Walk() is called in GetNextRepository()", t, func() { - imgStore := createMockStorage(testDir, tdir, false, &StorageDriverMock{ + imgStore := createMockStorage(testDir, tdir, false, &mocks.StorageDriverMock{ ListFn: func(ctx context.Context, path string) ([]string, error) { return []string{}, nil }, @@ -2270,12 +2082,12 @@ func TestRebuildDedupeMockStoreDriver(t *testing.T) { } Convey("Trigger Stat error in getOriginalBlobFromDisk()", t, func() { - imgStore := createMockStorage(testDir, tdir, false, &StorageDriverMock{ + imgStore := createMockStorage(testDir, tdir, false, &mocks.StorageDriverMock{ StatFn: func(ctx context.Context, path string) (driver.FileInfo, error) { - return &FileInfoMock{}, errS3 + return &mocks.FileInfoMock{}, errS3 }, WalkFn: func(ctx context.Context, path string, walkFn driver.WalkFn, options ...func(*driver.WalkOptions)) error { - return walkFn(&FileInfoMock{ + return walkFn(&mocks.FileInfoMock{ IsDirFn: func() bool { return false }, @@ -2294,24 +2106,24 @@ func TestRebuildDedupeMockStoreDriver(t *testing.T) { }) Convey("Trigger GetContent error in restoreDedupedBlobs()", t, func() { - imgStore := createMockStorage(testDir, tdir, false, &StorageDriverMock{ + imgStore := createMockStorage(testDir, tdir, false, &mocks.StorageDriverMock{ StatFn: func(ctx context.Context, path string) (driver.FileInfo, error) { if path == blobPath("path/to", validDigest) { - return &FileInfoMock{ + return &mocks.FileInfoMock{ SizeFn: func() int64 { return int64(0) }, }, nil } - return &FileInfoMock{ + return &mocks.FileInfoMock{ SizeFn: func() int64 { return int64(10) }, }, nil }, WalkFn: func(ctx context.Context, path string, walkFn driver.WalkFn, options ...func(*driver.WalkOptions)) error { - _ = walkFn(&FileInfoMock{ + _ = walkFn(&mocks.FileInfoMock{ IsDirFn: func() bool { return false }, @@ -2319,7 +2131,7 @@ func TestRebuildDedupeMockStoreDriver(t *testing.T) { return blobPath("path/to", validDigest) }, }) - _ = walkFn(&FileInfoMock{ + _ = walkFn(&mocks.FileInfoMock{ IsDirFn: func() bool { return false }, @@ -2343,24 +2155,24 @@ func TestRebuildDedupeMockStoreDriver(t *testing.T) { }) Convey("Trigger GetContent error in restoreDedupedBlobs()", t, func() { - imgStore := createMockStorage(testDir, tdir, false, &StorageDriverMock{ + imgStore := createMockStorage(testDir, tdir, false, &mocks.StorageDriverMock{ StatFn: func(ctx context.Context, path string) (driver.FileInfo, error) { if path == blobPath("path/to", validDigest) { - return &FileInfoMock{ + return &mocks.FileInfoMock{ SizeFn: func() int64 { return int64(0) }, }, nil } - return &FileInfoMock{ + return &mocks.FileInfoMock{ SizeFn: func() int64 { return int64(10) }, }, nil }, WalkFn: func(ctx context.Context, path string, walkFn driver.WalkFn, options ...func(*driver.WalkOptions)) error { - _ = walkFn(&FileInfoMock{ + _ = walkFn(&mocks.FileInfoMock{ IsDirFn: func() bool { return false }, @@ -2368,7 +2180,7 @@ func TestRebuildDedupeMockStoreDriver(t *testing.T) { return blobPath("path/to", validDigest) }, }) - _ = walkFn(&FileInfoMock{ + _ = walkFn(&mocks.FileInfoMock{ IsDirFn: func() bool { return false }, @@ -2380,7 +2192,7 @@ func TestRebuildDedupeMockStoreDriver(t *testing.T) { return nil }, WriterFn: func(ctx context.Context, path string, isAppend bool) (driver.FileWriter, error) { - return &FileWriterMock{}, errS3 + return &mocks.FileWriterMock{}, errS3 }, }) @@ -2392,24 +2204,24 @@ func TestRebuildDedupeMockStoreDriver(t *testing.T) { }) Convey("Trigger Stat() error in restoreDedupedBlobs()", t, func() { - imgStore := createMockStorage(testDir, tdir, false, &StorageDriverMock{ + imgStore := createMockStorage(testDir, tdir, false, &mocks.StorageDriverMock{ StatFn: func(ctx context.Context, path string) (driver.FileInfo, error) { if path == blobPath("path/to", validDigest) { - return &FileInfoMock{ + return &mocks.FileInfoMock{ SizeFn: func() int64 { return int64(10) }, }, nil } - return &FileInfoMock{ + return &mocks.FileInfoMock{ SizeFn: func() int64 { return int64(10) }, }, errS3 }, WalkFn: func(ctx context.Context, path string, walkFn driver.WalkFn, options ...func(*driver.WalkOptions)) error { - _ = walkFn(&FileInfoMock{ + _ = walkFn(&mocks.FileInfoMock{ IsDirFn: func() bool { return false }, @@ -2417,7 +2229,7 @@ func TestRebuildDedupeMockStoreDriver(t *testing.T) { return blobPath("path/to", validDigest) }, }) - _ = walkFn(&FileInfoMock{ + _ = walkFn(&mocks.FileInfoMock{ IsDirFn: func() bool { return false }, @@ -2437,24 +2249,24 @@ func TestRebuildDedupeMockStoreDriver(t *testing.T) { So(err, ShouldNotBeNil) Convey("Trigger Stat() error in dedupeBlobs()", func() { - imgStore := createMockStorage(testDir, t.TempDir(), true, &StorageDriverMock{ + imgStore := createMockStorage(testDir, t.TempDir(), true, &mocks.StorageDriverMock{ StatFn: func(ctx context.Context, path string) (driver.FileInfo, error) { if path == blobPath("path/to", validDigest) { - return &FileInfoMock{ + return &mocks.FileInfoMock{ SizeFn: func() int64 { return int64(10) }, }, nil } - return &FileInfoMock{ + return &mocks.FileInfoMock{ SizeFn: func() int64 { return int64(10) }, }, errS3 }, WalkFn: func(ctx context.Context, path string, walkFn driver.WalkFn, options ...func(*driver.WalkOptions)) error { - _ = walkFn(&FileInfoMock{ + _ = walkFn(&mocks.FileInfoMock{ IsDirFn: func() bool { return false }, @@ -2462,7 +2274,7 @@ func TestRebuildDedupeMockStoreDriver(t *testing.T) { return blobPath("path/to", validDigest) }, }) - _ = walkFn(&FileInfoMock{ + _ = walkFn(&mocks.FileInfoMock{ IsDirFn: func() bool { return false }, @@ -2485,24 +2297,24 @@ func TestRebuildDedupeMockStoreDriver(t *testing.T) { Convey("Trigger PutContent() error in dedupeBlobs()", t, func() { tdir := t.TempDir() - imgStore := createMockStorage(testDir, tdir, true, &StorageDriverMock{ + imgStore := createMockStorage(testDir, tdir, true, &mocks.StorageDriverMock{ StatFn: func(ctx context.Context, path string) (driver.FileInfo, error) { if path == blobPath("path/to", validDigest) { - return &FileInfoMock{ + return &mocks.FileInfoMock{ SizeFn: func() int64 { return int64(0) }, }, nil } - return &FileInfoMock{ + return &mocks.FileInfoMock{ SizeFn: func() int64 { return int64(10) }, }, nil }, WalkFn: func(ctx context.Context, path string, walkFn driver.WalkFn, options ...func(*driver.WalkOptions)) error { - _ = walkFn(&FileInfoMock{ + _ = walkFn(&mocks.FileInfoMock{ IsDirFn: func() bool { return false }, @@ -2510,7 +2322,7 @@ func TestRebuildDedupeMockStoreDriver(t *testing.T) { return blobPath("path/to", validDigest) }, }) - _ = walkFn(&FileInfoMock{ + _ = walkFn(&mocks.FileInfoMock{ IsDirFn: func() bool { return false }, @@ -2536,24 +2348,24 @@ func TestRebuildDedupeMockStoreDriver(t *testing.T) { //nolint: dupl Convey("Trigger getOriginalBlob() error in dedupeBlobs()", t, func() { tdir := t.TempDir() - imgStore := createMockStorage(testDir, tdir, true, &StorageDriverMock{ + imgStore := createMockStorage(testDir, tdir, true, &mocks.StorageDriverMock{ StatFn: func(ctx context.Context, path string) (driver.FileInfo, error) { if path == blobPath("path/to", validDigest) { - return &FileInfoMock{ + return &mocks.FileInfoMock{ SizeFn: func() int64 { return int64(0) }, }, nil } - return &FileInfoMock{ + return &mocks.FileInfoMock{ SizeFn: func() int64 { return int64(0) }, }, nil }, WalkFn: func(ctx context.Context, path string, walkFn driver.WalkFn, options ...func(*driver.WalkOptions)) error { - _ = walkFn(&FileInfoMock{ + _ = walkFn(&mocks.FileInfoMock{ IsDirFn: func() bool { return false }, @@ -2561,7 +2373,7 @@ func TestRebuildDedupeMockStoreDriver(t *testing.T) { return blobPath("path/to", validDigest) }, }) - _ = walkFn(&FileInfoMock{ + _ = walkFn(&mocks.FileInfoMock{ IsDirFn: func() bool { return false }, @@ -2584,24 +2396,24 @@ func TestRebuildDedupeMockStoreDriver(t *testing.T) { //nolint: dupl Convey("Trigger Stat() error in dedupeBlobs()", t, func() { tdir := t.TempDir() - imgStore := createMockStorage(testDir, tdir, true, &StorageDriverMock{ + imgStore := createMockStorage(testDir, tdir, true, &mocks.StorageDriverMock{ StatFn: func(ctx context.Context, path string) (driver.FileInfo, error) { if path == blobPath("path/to", validDigest) { - return &FileInfoMock{ + return &mocks.FileInfoMock{ SizeFn: func() int64 { return int64(10) }, }, nil } - return &FileInfoMock{ + return &mocks.FileInfoMock{ SizeFn: func() int64 { return int64(10) }, }, errS3 }, WalkFn: func(ctx context.Context, path string, walkFn driver.WalkFn, options ...func(*driver.WalkOptions)) error { - _ = walkFn(&FileInfoMock{ + _ = walkFn(&mocks.FileInfoMock{ IsDirFn: func() bool { return false }, @@ -2609,7 +2421,7 @@ func TestRebuildDedupeMockStoreDriver(t *testing.T) { return blobPath("path/to", validDigest) }, }) - _ = walkFn(&FileInfoMock{ + _ = walkFn(&mocks.FileInfoMock{ IsDirFn: func() bool { return false }, @@ -2631,7 +2443,7 @@ func TestRebuildDedupeMockStoreDriver(t *testing.T) { Convey("Trigger getNextDigestWithBlobPaths err", t, func() { tdir := t.TempDir() - imgStore := createMockStorage(testDir, tdir, true, &StorageDriverMock{ + imgStore := createMockStorage(testDir, tdir, true, &mocks.StorageDriverMock{ WalkFn: func(ctx context.Context, path string, f driver.WalkFn, options ...func(*driver.WalkOptions)) error { return errS3 }, @@ -2643,10 +2455,10 @@ func TestRebuildDedupeMockStoreDriver(t *testing.T) { Convey("Skip files with invalid algorithm directory", t, func() { tdir := t.TempDir() - imgStore := createMockStorage(testDir, tdir, true, &StorageDriverMock{ + imgStore := createMockStorage(testDir, tdir, true, &mocks.StorageDriverMock{ WalkFn: func(ctx context.Context, path string, walkFn driver.WalkFn, options ...func(*driver.WalkOptions)) error { // File in blobs directory but with invalid algorithm name - _ = walkFn(&FileInfoMock{ + _ = walkFn(&mocks.FileInfoMock{ IsDirFn: func() bool { return false }, @@ -2668,10 +2480,10 @@ func TestRebuildDedupeMockStoreDriver(t *testing.T) { Convey("Skip files with invalid digest hash", t, func() { tdir := t.TempDir() - imgStore := createMockStorage(testDir, tdir, true, &StorageDriverMock{ + imgStore := createMockStorage(testDir, tdir, true, &mocks.StorageDriverMock{ WalkFn: func(ctx context.Context, path string, walkFn driver.WalkFn, options ...func(*driver.WalkOptions)) error { // File with valid algorithm but invalid hash format - _ = walkFn(&FileInfoMock{ + _ = walkFn(&mocks.FileInfoMock{ IsDirFn: func() bool { return false }, @@ -2692,24 +2504,24 @@ func TestRebuildDedupeMockStoreDriver(t *testing.T) { }) Convey("Trigger cache errors", t, func() { - storageDriverMockIfBranch := &StorageDriverMock{ + storageDriverMockIfBranch := &mocks.StorageDriverMock{ StatFn: func(ctx context.Context, path string) (driver.FileInfo, error) { if path == blobPath("path/to", validDigest) { - return &FileInfoMock{ + return &mocks.FileInfoMock{ SizeFn: func() int64 { return int64(0) }, }, nil } - return &FileInfoMock{ + return &mocks.FileInfoMock{ SizeFn: func() int64 { return int64(10) }, }, nil }, WalkFn: func(ctx context.Context, path string, walkFn driver.WalkFn, options ...func(*driver.WalkOptions)) error { - _ = walkFn(&FileInfoMock{ + _ = walkFn(&mocks.FileInfoMock{ IsDirFn: func() bool { return false }, @@ -2717,7 +2529,7 @@ func TestRebuildDedupeMockStoreDriver(t *testing.T) { return blobPath("path/to", validDigest) }, }) - _ = walkFn(&FileInfoMock{ + _ = walkFn(&mocks.FileInfoMock{ IsDirFn: func() bool { return false }, @@ -2730,24 +2542,24 @@ func TestRebuildDedupeMockStoreDriver(t *testing.T) { }, } - storageDriverMockElseBranch := &StorageDriverMock{ + storageDriverMockElseBranch := &mocks.StorageDriverMock{ StatFn: func(ctx context.Context, path string) (driver.FileInfo, error) { if path == blobPath("path/to", validDigest) { - return &FileInfoMock{ + return &mocks.FileInfoMock{ SizeFn: func() int64 { return int64(10) }, }, nil } - return &FileInfoMock{ + return &mocks.FileInfoMock{ SizeFn: func() int64 { return int64(10) }, }, nil }, WalkFn: func(ctx context.Context, path string, walkFn driver.WalkFn, options ...func(*driver.WalkOptions)) error { - _ = walkFn(&FileInfoMock{ + _ = walkFn(&mocks.FileInfoMock{ IsDirFn: func() bool { return false }, @@ -2755,7 +2567,7 @@ func TestRebuildDedupeMockStoreDriver(t *testing.T) { return blobPath("path/to", validDigest) }, }) - _ = walkFn(&FileInfoMock{ + _ = walkFn(&mocks.FileInfoMock{ IsDirFn: func() bool { return false }, @@ -3629,7 +3441,7 @@ func TestS3DedupeErr(t *testing.T) { Convey("Test DedupeBlob", t, func(c C) { tdir := t.TempDir() - imgStore = createMockStorage(testDir, tdir, true, &StorageDriverMock{}) + imgStore = createMockStorage(testDir, tdir, true, &mocks.StorageDriverMock{}) err = os.Remove(path.Join(tdir, storageConstants.BoltdbName+storageConstants.DBExtensionName)) digest := godigest.NewDigestFromEncoded(godigest.SHA256, "digest") @@ -3638,7 +3450,7 @@ func TestS3DedupeErr(t *testing.T) { err := imgStore.DedupeBlob("", digest, "", "") So(err, ShouldNotBeNil) - imgStore = createMockStorage(testDir, tdir, true, &StorageDriverMock{ + imgStore = createMockStorage(testDir, tdir, true, &mocks.StorageDriverMock{ MoveFn: func(ctx context.Context, sourcePath string, destPath string) error { return errS3 }, @@ -3658,7 +3470,7 @@ func TestS3DedupeErr(t *testing.T) { Convey("Test DedupeBlob - error on second store.Stat()", t, func(c C) { tdir := t.TempDir() - imgStore = createMockStorage(testDir, tdir, true, &StorageDriverMock{ + imgStore = createMockStorage(testDir, tdir, true, &mocks.StorageDriverMock{ StatFn: func(ctx context.Context, path string) (driver.FileInfo, error) { if path == "dst2" { return driver.FileInfoInternal{}, errS3 @@ -3679,7 +3491,7 @@ func TestS3DedupeErr(t *testing.T) { Convey("Test DedupeBlob - error on store.PutContent()", t, func(c C) { tdir := t.TempDir() - imgStore = createMockStorage(testDir, tdir, true, &StorageDriverMock{ + imgStore = createMockStorage(testDir, tdir, true, &mocks.StorageDriverMock{ PutContentFn: func(ctx context.Context, path string, content []byte) error { return errS3 }, @@ -3698,7 +3510,7 @@ func TestS3DedupeErr(t *testing.T) { Convey("Test DedupeBlob - error on cache.PutBlob()", t, func(c C) { tdir := t.TempDir() - imgStore = createMockStorage(testDir, tdir, true, &StorageDriverMock{ + imgStore = createMockStorage(testDir, tdir, true, &mocks.StorageDriverMock{ StatFn: func(ctx context.Context, path string) (driver.FileInfo, error) { return nil, nil //nolint:nilnil }, @@ -3714,7 +3526,7 @@ func TestS3DedupeErr(t *testing.T) { Convey("Test DedupeBlob - error on store.Delete()", t, func(c C) { tdir := t.TempDir() - imgStore = createMockStorage(testDir, tdir, true, &StorageDriverMock{ + imgStore = createMockStorage(testDir, tdir, true, &mocks.StorageDriverMock{ DeleteFn: func(ctx context.Context, path string) error { return errS3 }, @@ -3733,7 +3545,7 @@ func TestS3DedupeErr(t *testing.T) { Convey("Test copyBlob() - error on initRepo()", t, func(c C) { tdir := t.TempDir() - imgStore = createMockStorage(testDir, tdir, true, &StorageDriverMock{ + imgStore = createMockStorage(testDir, tdir, true, &mocks.StorageDriverMock{ PutContentFn: func(ctx context.Context, path string, content []byte) error { return errS3 }, @@ -3741,7 +3553,7 @@ func TestS3DedupeErr(t *testing.T) { return driver.FileInfoInternal{}, errS3 }, WriterFn: func(ctx context.Context, path string, isAppend bool) (driver.FileWriter, error) { - return &FileWriterMock{}, errS3 + return &mocks.FileWriterMock{}, errS3 }, }) @@ -3757,7 +3569,7 @@ func TestS3DedupeErr(t *testing.T) { Convey("Test copyBlob() - error on store.PutContent()", t, func(c C) { tdir := t.TempDir() - imgStore = createMockStorage(testDir, tdir, true, &StorageDriverMock{ + imgStore = createMockStorage(testDir, tdir, true, &mocks.StorageDriverMock{ PutContentFn: func(ctx context.Context, path string, content []byte) error { return errS3 }, @@ -3778,7 +3590,7 @@ func TestS3DedupeErr(t *testing.T) { Convey("Test copyBlob() - error on store.Stat()", t, func(c C) { tdir := t.TempDir() - imgStore = createMockStorage(testDir, tdir, true, &StorageDriverMock{ + imgStore = createMockStorage(testDir, tdir, true, &mocks.StorageDriverMock{ StatFn: func(ctx context.Context, path string) (driver.FileInfo, error) { return driver.FileInfoInternal{}, errS3 }, @@ -3797,7 +3609,7 @@ func TestS3DedupeErr(t *testing.T) { Convey("Test GetBlob() - error on second store.Stat()", t, func(c C) { tdir := t.TempDir() - imgStore = createMockStorage(testDir, tdir, true, &StorageDriverMock{}) + imgStore = createMockStorage(testDir, tdir, true, &mocks.StorageDriverMock{}) digest := godigest.NewDigestFromEncoded(godigest.SHA256, "7173b809ca12ec5dee4506cd86be934c4596dd234ee82c0662eac04a8c2c71dc") @@ -3817,7 +3629,7 @@ func TestS3DedupeErr(t *testing.T) { err = os.WriteFile(path.Join(tdir, storageConstants.BoltdbName+storageConstants.DBExtensionName), input, 0o600) So(err, ShouldBeNil) - imgStore = createMockStorage(testDir, tdir, true, &StorageDriverMock{ + imgStore = createMockStorage(testDir, tdir, true, &mocks.StorageDriverMock{ StatFn: func(ctx context.Context, path string) (driver.FileInfo, error) { if strings.Contains(path, "repo1/dst1") { return driver.FileInfoInternal{}, driver.PathNotFoundError{} @@ -3845,7 +3657,7 @@ func TestS3DedupeErr(t *testing.T) { Convey("Test GetBlob() - error on store.Reader()", t, func(c C) { tdir := t.TempDir() - imgStore = createMockStorage(testDir, tdir, true, &StorageDriverMock{}) + imgStore = createMockStorage(testDir, tdir, true, &mocks.StorageDriverMock{}) digest := godigest.NewDigestFromEncoded(godigest.SHA256, "7173b809ca12ec5dee4506cd86be934c4596dd234ee82c0662eac04a8c2c71dc") @@ -3865,9 +3677,9 @@ func TestS3DedupeErr(t *testing.T) { err = os.WriteFile(path.Join(tdir, storageConstants.BoltdbName+storageConstants.DBExtensionName), input, 0o600) So(err, ShouldBeNil) - imgStore = createMockStorage(testDir, tdir, true, &StorageDriverMock{ + imgStore = createMockStorage(testDir, tdir, true, &mocks.StorageDriverMock{ StatFn: func(ctx context.Context, path string) (driver.FileInfo, error) { - return &FileInfoMock{ + return &mocks.FileInfoMock{ SizeFn: func() int64 { return 0 }, @@ -3909,9 +3721,9 @@ func TestS3DedupeErr(t *testing.T) { digest := godigest.NewDigestFromEncoded(godigest.SHA256, "7173b809ca12ec5dee4506cd86be934c4596dd234ee82c0662eac04a8c2c71dc") - imgStore = createMockStorage(testDir, tdir, true, &StorageDriverMock{ + imgStore = createMockStorage(testDir, tdir, true, &mocks.StorageDriverMock{ StatFn: func(ctx context.Context, path string) (driver.FileInfo, error) { - return &FileInfoMock{ + return &mocks.FileInfoMock{ SizeFn: func() int64 { return 0 }, @@ -3940,7 +3752,7 @@ func TestS3DedupeErr(t *testing.T) { blobPath := path.Join(testDir, "repo/blobs/sha256", hash) - imgStore = createMockStorage(testDir, tdir, true, &StorageDriverMock{ + imgStore = createMockStorage(testDir, tdir, true, &mocks.StorageDriverMock{ MoveFn: func(ctx context.Context, sourcePath, destPath string) error { if destPath == blobPath { return nil @@ -3953,7 +3765,7 @@ func TestS3DedupeErr(t *testing.T) { return nil, errS3 } - return &FileInfoMock{}, nil + return &mocks.FileInfoMock{}, nil }, }) @@ -3969,7 +3781,7 @@ func TestS3DedupeErr(t *testing.T) { Convey("Test FullBlobUpload", t, func(c C) { tdir := t.TempDir() - imgStore = createMockStorage(testDir, tdir, true, &StorageDriverMock{ + imgStore = createMockStorage(testDir, tdir, true, &mocks.StorageDriverMock{ MoveFn: func(ctx context.Context, sourcePath, destPath string) error { return errS3 }, @@ -3981,7 +3793,7 @@ func TestS3DedupeErr(t *testing.T) { Convey("Test FinishBlobUpload", t, func(c C) { tdir := t.TempDir() - imgStore = createMockStorage(testDir, tdir, true, &StorageDriverMock{ + imgStore = createMockStorage(testDir, tdir, true, &mocks.StorageDriverMock{ MoveFn: func(ctx context.Context, sourcePath, destPath string) error { return errS3 }, @@ -4003,9 +3815,9 @@ func TestInjectDedupe(t *testing.T) { testDir := path.Join("/oci-repo-test", uuid.String()) Convey("Inject errors in DedupeBlob function", t, func() { - imgStore := createMockStorage(testDir, tdir, true, &StorageDriverMock{ + imgStore := createMockStorage(testDir, tdir, true, &mocks.StorageDriverMock{ StatFn: func(ctx context.Context, path string) (driver.FileInfo, error) { - return &FileInfoMock{}, errS3 + return &mocks.FileInfoMock{}, errS3 }, }) err := imgStore.DedupeBlob("blob", "digest", "", "newblob") diff --git a/pkg/storage/storage.go b/pkg/storage/storage.go index de7cf558..52a333b5 100644 --- a/pkg/storage/storage.go +++ b/pkg/storage/storage.go @@ -18,6 +18,7 @@ import ( "zotregistry.dev/zot/v2/pkg/log" common "zotregistry.dev/zot/v2/pkg/storage/common" "zotregistry.dev/zot/v2/pkg/storage/constants" + "zotregistry.dev/zot/v2/pkg/storage/gcs" "zotregistry.dev/zot/v2/pkg/storage/local" "zotregistry.dev/zot/v2/pkg/storage/s3" storageTypes "zotregistry.dev/zot/v2/pkg/storage/types" @@ -63,7 +64,7 @@ func New(config *config.Config, linter common.Lint, metrics monitoring.MetricSer ) } else { storeName := fmt.Sprintf("%v", config.Storage.StorageDriver["name"]) - if storeName != constants.S3StorageDriverName { + if storeName != constants.S3StorageDriverName && storeName != constants.GCSStorageDriverName { log.Error().Err(zerr.ErrBadConfig).Str("storageDriver", storeName). Msg("unsupported storage driver") @@ -92,8 +93,15 @@ func New(config *config.Config, linter common.Lint, metrics monitoring.MetricSer // false positive lint - linter does not implement Lint method //nolint: typecheck,contextcheck - defaultStore = s3.NewImageStore(rootDir, config.Storage.RootDirectory, - config.Storage.Dedupe, config.Storage.Commit, log, metrics, linter, store, cacheDriver, config.HTTP.Compat, recorder) + if storeName == constants.S3StorageDriverName { + defaultStore = s3.NewImageStore(rootDir, config.Storage.RootDirectory, + config.Storage.Dedupe, config.Storage.Commit, log, metrics, linter, store, cacheDriver, + config.HTTP.Compat, recorder) + } else { + defaultStore = gcs.NewImageStore(rootDir, config.Storage.RootDirectory, + config.Storage.Dedupe, config.Storage.Commit, log, metrics, linter, store, cacheDriver, + config.HTTP.Compat, recorder) + } } storeController.DefaultStore = defaultStore @@ -178,7 +186,7 @@ func getSubStore(cfg *config.Config, subPaths map[string]config.StorageConfig, } } else { storeName := fmt.Sprintf("%v", storageConfig.StorageDriver["name"]) - if storeName != constants.S3StorageDriverName { + if storeName != constants.S3StorageDriverName && storeName != constants.GCSStorageDriverName { log.Error().Err(zerr.ErrBadConfig).Str("storageDriver", storeName). Msg("unsupported storage driver") @@ -210,9 +218,15 @@ func getSubStore(cfg *config.Config, subPaths map[string]config.StorageConfig, // false positive lint - linter does not implement Lint method //nolint: typecheck - subImageStore[route] = s3.NewImageStore(rootDir, storageConfig.RootDirectory, - storageConfig.Dedupe, storageConfig.Commit, log, metrics, linter, store, cacheDriver, cfg.HTTP.Compat, recorder, - ) + if storeName == constants.S3StorageDriverName { + subImageStore[route] = s3.NewImageStore(rootDir, storageConfig.RootDirectory, + storageConfig.Dedupe, storageConfig.Commit, log, metrics, linter, store, cacheDriver, cfg.HTTP.Compat, recorder, + ) + } else { + subImageStore[route] = gcs.NewImageStore(rootDir, storageConfig.RootDirectory, + storageConfig.Dedupe, storageConfig.Commit, log, metrics, linter, store, cacheDriver, cfg.HTTP.Compat, recorder, + ) + } } } diff --git a/pkg/storage/storage_test.go b/pkg/storage/storage_test.go index 1c20e186..735243d1 100644 --- a/pkg/storage/storage_test.go +++ b/pkg/storage/storage_test.go @@ -59,7 +59,9 @@ var DeleteReferrers = config.ImageRetention{ //nolint: gochecknoglobals } func cleanupStorage(store storageTypes.Driver, name string) { - _ = store.Delete(name) + if store != nil { + _ = store.Delete(name) + } } type createObjectStoreOpts struct { @@ -207,7 +209,8 @@ func TestGetAllDedupeReposCandidates(t *testing.T) { defer DumpKeys(t, opts.miniRedisAddr) } - if testcase.storageType == storageConstants.S3StorageDriverName { + switch testcase.storageType { + case storageConstants.S3StorageDriverName: tskip.SkipS3(t) uuid, err := guuid.NewV4() @@ -219,9 +222,10 @@ func TestGetAllDedupeReposCandidates(t *testing.T) { opts.rootDir = testDir var store storageTypes.Driver + store, imgStore, _, _ = createObjectsStore(opts) defer cleanupStorage(store, testDir) - } else { + default: _, imgStore, _, _ = createObjectsStore(opts) } @@ -279,7 +283,8 @@ func TestStorageAPIs(t *testing.T) { defer DumpKeys(t, opts.miniRedisAddr) } - if testcase.storageType == storageConstants.S3StorageDriverName { + switch testcase.storageType { + case storageConstants.S3StorageDriverName: tskip.SkipS3(t) uuid, err := guuid.NewV4() @@ -291,9 +296,10 @@ func TestStorageAPIs(t *testing.T) { opts.rootDir = testDir var store storageTypes.Driver + store, imgStore, _, _ = createObjectsStore(opts) defer cleanupStorage(store, testDir) - } else { + default: _, imgStore, _, _ = createObjectsStore(opts) } @@ -1028,7 +1034,8 @@ func TestMandatoryAnnotations(t *testing.T) { defer DumpKeys(t, opts.miniRedisAddr) } - if testcase.storageType == storageConstants.S3StorageDriverName { + switch testcase.storageType { + case storageConstants.S3StorageDriverName: tskip.SkipS3(t) uuid, err := guuid.NewV4() @@ -1040,6 +1047,7 @@ func TestMandatoryAnnotations(t *testing.T) { opts.rootDir = testDir var cacheDriver storageTypes.Cache + store, _, cacheDriver, _ = createObjectsStore(opts) imgStore = imagestore.NewImageStore(testDir, cacheDir, false, false, log, metrics, @@ -1050,8 +1058,9 @@ func TestMandatoryAnnotations(t *testing.T) { }, store, cacheDriver, nil, nil) defer cleanupStorage(store, testDir) - } else { + default: var cacheDriver storageTypes.Cache + store, _, cacheDriver, _ = createObjectsStore(opts) imgStore = imagestore.NewImageStore(cacheDir, cacheDir, true, true, log, metrics, @@ -1115,8 +1124,10 @@ func TestMandatoryAnnotations(t *testing.T) { }, store, nil, nil, nil) } else { var cacheDriver storageTypes.Cache - store, _, cacheDriver, _ = createObjectsStore(opts) - + store, _, cacheDriver, err := createObjectsStore(opts) + if err != nil { + t.Fatal(err) + } imgStore = imagestore.NewImageStore(cacheDir, cacheDir, true, true, log, metrics, &mocks.MockedLint{ LintFn: func(repo string, manifestDigest godigest.Digest, imageStore storageTypes.ImageStore) (bool, error) { @@ -1223,7 +1234,8 @@ func TestDeleteBlobsInUse(t *testing.T) { defer DumpKeys(t, opts.miniRedisAddr) } - if testcase.storageType == storageConstants.S3StorageDriverName { + switch testcase.storageType { + case storageConstants.S3StorageDriverName: tskip.SkipS3(t) uuid, err := guuid.NewV4() @@ -1238,7 +1250,7 @@ func TestDeleteBlobsInUse(t *testing.T) { store, imgStore, _, _ = createObjectsStore(opts) defer cleanupStorage(store, testDir) - } else { + default: _, imgStore, _, _ = createObjectsStore(opts) } @@ -1532,7 +1544,8 @@ func TestReuploadCorruptedBlob(t *testing.T) { defer DumpKeys(t, opts.miniRedisAddr) } - if testcase.storageType == storageConstants.S3StorageDriverName { + switch testcase.storageType { + case storageConstants.S3StorageDriverName: tskip.SkipS3(t) uuid, err := guuid.NewV4() @@ -1545,7 +1558,7 @@ func TestReuploadCorruptedBlob(t *testing.T) { driver, imgStore, _, _ = createObjectsStore(opts) defer cleanupStorage(driver, testDir) - } else { + default: driver, imgStore, _, _ = createObjectsStore(opts) } @@ -1666,7 +1679,8 @@ func TestStorageHandler(t *testing.T) { defer DumpKeys(t, opts.miniRedisAddr) } - if testcase.storageType == storageConstants.S3StorageDriverName { + switch testcase.storageType { + case storageConstants.S3StorageDriverName: tskip.SkipS3(t) var ( @@ -1695,7 +1709,7 @@ func TestStorageHandler(t *testing.T) { thirdStorageDriver, thirdStore, _, _ = createObjectsStore(opts) defer cleanupStorage(thirdStorageDriver, thirdRootDir) - } else { + default: firstRootDir = t.TempDir() opts.rootDir = firstRootDir opts.cacheDir = firstRootDir @@ -1784,7 +1798,8 @@ func TestGarbageCollectImageManifest(t *testing.T) { Convey("Garbage collect with default/long delay", func() { var imgStore storageTypes.ImageStore - if testcase.storageType == storageConstants.S3StorageDriverName { + switch testcase.storageType { + case storageConstants.S3StorageDriverName: tskip.SkipS3(t) uuid, err := guuid.NewV4() @@ -1796,9 +1811,10 @@ func TestGarbageCollectImageManifest(t *testing.T) { opts.rootDir = testDir var store storageTypes.Driver + store, imgStore, _, _ = createObjectsStore(opts) defer cleanupStorage(store, testDir) - } else { + default: _, imgStore, _, _ = createObjectsStore(opts) } @@ -1945,7 +1961,8 @@ func TestGarbageCollectImageManifest(t *testing.T) { gcDelay := 1 * time.Second - if testcase.storageType == storageConstants.S3StorageDriverName { + switch testcase.storageType { + case storageConstants.S3StorageDriverName: tskip.SkipS3(t) uuid, err := guuid.NewV4() @@ -1957,9 +1974,10 @@ func TestGarbageCollectImageManifest(t *testing.T) { opts.rootDir = testDir var store storageTypes.Driver + store, imgStore, _, _ = createObjectsStore(opts) defer cleanupStorage(store, testDir) - } else { + default: _, imgStore, _, _ = createObjectsStore(opts) } @@ -2220,7 +2238,8 @@ func TestGarbageCollectImageManifest(t *testing.T) { gcDelay := 3 * time.Second - if testcase.storageType == storageConstants.S3StorageDriverName { + switch testcase.storageType { + case storageConstants.S3StorageDriverName: tskip.SkipS3(t) uuid, err := guuid.NewV4() @@ -2232,9 +2251,10 @@ func TestGarbageCollectImageManifest(t *testing.T) { opts.rootDir = testDir var store storageTypes.Driver + store, imgStore, _, _ = createObjectsStore(opts) defer cleanupStorage(store, testDir) - } else { + default: _, imgStore, _, _ = createObjectsStore(opts) } @@ -2467,7 +2487,8 @@ func TestGarbageCollectImageIndex(t *testing.T) { Convey("Garbage collect with default/long delay", func() { var imgStore storageTypes.ImageStore - if testcase.storageType == storageConstants.S3StorageDriverName { + switch testcase.storageType { + case storageConstants.S3StorageDriverName: tskip.SkipS3(t) uuid, err := guuid.NewV4() @@ -2479,9 +2500,10 @@ func TestGarbageCollectImageIndex(t *testing.T) { opts.rootDir = testDir var store storageTypes.Driver + store, imgStore, _, _ = createObjectsStore(opts) defer cleanupStorage(store, testDir) - } else { + default: _, imgStore, _, _ = createObjectsStore(opts) } @@ -2586,7 +2608,8 @@ func TestGarbageCollectImageIndex(t *testing.T) { gcDelay := 2 * time.Second imageRetentionDelay := 2 * time.Second - if testcase.storageType == storageConstants.S3StorageDriverName { + switch testcase.storageType { + case storageConstants.S3StorageDriverName: tskip.SkipS3(t) uuid, err := guuid.NewV4() @@ -2598,9 +2621,10 @@ func TestGarbageCollectImageIndex(t *testing.T) { opts.rootDir = testDir var store storageTypes.Driver + store, imgStore, _, _ = createObjectsStore(opts) defer cleanupStorage(store, testDir) - } else { + default: _, imgStore, _, _ = createObjectsStore(opts) } @@ -2876,7 +2900,8 @@ func TestGarbageCollectChainedImageIndexes(t *testing.T) { defer DumpKeys(t, opts.miniRedisAddr) } - if testcase.storageType == storageConstants.S3StorageDriverName { + switch testcase.storageType { + case storageConstants.S3StorageDriverName: tskip.SkipS3(t) uuid, err := guuid.NewV4() @@ -2888,9 +2913,10 @@ func TestGarbageCollectChainedImageIndexes(t *testing.T) { opts.rootDir = testDir var store storageTypes.Driver + store, imgStore, _, _ = createObjectsStore(opts) defer cleanupStorage(store, testDir) - } else { + default: _, imgStore, _, _ = createObjectsStore(opts) } diff --git a/pkg/test/common/rlimit_darwin.go b/pkg/test/common/rlimit_darwin.go new file mode 100644 index 00000000..5f1d87b0 --- /dev/null +++ b/pkg/test/common/rlimit_darwin.go @@ -0,0 +1,3 @@ +package common + +type RlimT = uint64 diff --git a/pkg/test/mocks/storage_driver_mock.go b/pkg/test/mocks/storage_driver_mock.go index 36344220..5df3531f 100644 --- a/pkg/test/mocks/storage_driver_mock.go +++ b/pkg/test/mocks/storage_driver_mock.go @@ -3,6 +3,7 @@ package mocks import ( "context" "io" + "net/http" "strings" "time" @@ -10,16 +11,17 @@ import ( ) type StorageDriverMock struct { - NameFn func() string - GetContentFn func(ctx context.Context, path string) ([]byte, error) - PutContentFn func(ctx context.Context, path string, content []byte) error - ReaderFn func(ctx context.Context, path string, offset int64) (io.ReadCloser, error) - WriterFn func(ctx context.Context, path string, isAppend bool) (driver.FileWriter, error) - StatFn func(ctx context.Context, path string) (driver.FileInfo, error) - ListFn func(ctx context.Context, path string) ([]string, error) - MoveFn func(ctx context.Context, sourcePath, destPath string) error - DeleteFn func(ctx context.Context, path string) error - WalkFn func(ctx context.Context, path string, f driver.WalkFn) error + NameFn func() string + GetContentFn func(ctx context.Context, path string) ([]byte, error) + PutContentFn func(ctx context.Context, path string, content []byte) error + ReaderFn func(ctx context.Context, path string, offset int64) (io.ReadCloser, error) + WriterFn func(ctx context.Context, path string, isAppend bool) (driver.FileWriter, error) + StatFn func(ctx context.Context, path string) (driver.FileInfo, error) + ListFn func(ctx context.Context, path string) ([]string, error) + MoveFn func(ctx context.Context, sourcePath, destPath string) error + DeleteFn func(ctx context.Context, path string) error + WalkFn func(ctx context.Context, path string, f driver.WalkFn, options ...func(*driver.WalkOptions)) error + RedirectURLFn func(r *http.Request, path string) (string, error) } //nolint:gochecknoglobals @@ -100,13 +102,23 @@ func (s *StorageDriverMock) Delete(ctx context.Context, path string) error { return nil } +func (s *StorageDriverMock) RedirectURL(r *http.Request, path string) (string, error) { + if s != nil && s.RedirectURLFn != nil { + return s.RedirectURLFn(r, path) + } + + return "", nil +} + func (s *StorageDriverMock) URLFor(ctx context.Context, path string, options map[string]any) (string, error) { return "", nil } -func (s *StorageDriverMock) Walk(ctx context.Context, path string, f driver.WalkFn) error { +func (s *StorageDriverMock) Walk(ctx context.Context, path string, f driver.WalkFn, + options ...func(*driver.WalkOptions), +) error { if s != nil && s.WalkFn != nil { - return s.WalkFn(ctx, path, f) + return s.WalkFn(ctx, path, f, options...) } return nil @@ -115,9 +127,14 @@ func (s *StorageDriverMock) Walk(ctx context.Context, path string, f driver.Walk type FileInfoMock struct { IsDirFn func() bool SizeFn func() int64 + PathFn func() string } func (f *FileInfoMock) Path() string { + if f != nil && f.PathFn != nil { + return f.PathFn() + } + return "" } diff --git a/pkg/test/skip/testskip.go b/pkg/test/skip/testskip.go index 53794a8d..4df2fa9b 100644 --- a/pkg/test/skip/testskip.go +++ b/pkg/test/skip/testskip.go @@ -20,3 +20,11 @@ func SkipDynamo(t *testing.T) { t.Skip("Skipping testing without AWS DynamoDB mock server") } } + +func SkipGCS(t *testing.T) { + t.Helper() + + if os.Getenv("GCSMOCK_ENDPOINT") == "" { + t.Skip("Skipping testing without GCS mock server") + } +}