mirror of
https://github.com/project-zot/zot.git
synced 2026-06-17 12:58:02 +08:00
55b68228da
* feat(storage): redirect blob pulls to backend URLs * fix: rebase conflicts Signed-off-by: Ramkumar Chinchani <rchincha.dev@gmail.com> * refactor: rename redirect field Signed-off-by: Ramkumar Chinchani <rchincha.dev@gmail.com> * test: relax brittle TestPeriodicGC substore log assertion Signed-off-by: Ramkumar Chinchani <rchincha.dev@gmail.com> * feat(storage): improve blob redirect config handling and validation Signed-off-by: Ramkumar Chinchani <rchincha.dev@gmail.com> * fix(storage): address PR review feedback for blob redirect Signed-off-by: Ramkumar Chinchani <rchincha.dev@gmail.com> * feat(storage): apply latest PR review fixes for blob redirect Signed-off-by: Ramkumar Chinchani <rchincha.dev@gmail.com> * test: fix blob redirect and verify test regressions Signed-off-by: Ramkumar Chinchani <rchincha.dev@gmail.com> * fix(storage): enforce redirectBlobURL validation and add redirect tests Signed-off-by: Ramkumar Chinchani <rchincha.dev@gmail.com> * fix(storage): fix err113/noctx lint errors in storage driver tests - Replace httptest.NewRequest with httptest.NewRequestWithContext in s3, gcs, and imagestore driver tests (noctx) - Replace dynamic errors.New in s3 driver test with a package-level static sentinel error (err113) Signed-off-by: Ramkumar Chinchani <rchincha.dev@gmail.com> * test(storage): use temp dirs in imagestore redirect tests Signed-off-by: Ramkumar Chinchani <rchincha.dev@gmail.com> * fix: handle ranged blob redirects and add regression tests Signed-off-by: Ramkumar Chinchani <rchincha.dev@gmail.com> * fix: validate blob digest consistently in GetBlob Signed-off-by: Ramkumar Chinchani <rchincha.dev@gmail.com> * test: fix GetBlobPartialFn mock return values for range requests The test 'does not redirect ranged blob requests' was failing because the mock was returning incorrect length values. For a range request 'bytes=0-0' (1 byte), it was returning 4 bytes, which caused a length mismatch check in GetBlob to return HTTP 500. Fix the mock to dynamically calculate the correct length: to - from + 1 Signed-off-by: Ramkumar Chinchani <rchincha.dev@gmail.com> * fix(storage): preserve signed URL bytes in normalizeBlobRedirectURL Preserve the original URL bytes from backend storage drivers (important for signed/presigned URLs) while only lowercasing the scheme prefix. URL re-serialization via net/url can invalidate signatures through path escaping or canonicalization. Add regression tests covering signed URL query parameters and mixed-case scheme handling. Signed-off-by: Ramkumar Chinchani <rchincha.dev@gmail.com> * fix(storage): address PR review comments for blob redirect - Return signed redirect URLs unchanged; validate scheme/CRLF/host only, no URL normalization that would corrupt signed URL bytes - Add inline comments for all non-obvious decisions: range bypass, soft fallback on invalid URL, local driver empty return, subpath resolution, redirectBlobURL config constraint on local/empty driver - Expand TestNormalizeBlobRedirectURL to cover allowed schemes (http/https), parse failure, missing host, and CRLF injection cases - Add TestIsBlobRedirectEnabled covering subpath-only enablement with default store disabled Signed-off-by: Ramkumar Chinchani <rchincha.dev@gmail.com> * test(storage): address remaining blob redirect review comments Signed-off-by: Ramkumar Chinchani <rchincha.dev@gmail.com> * fix: gofumpt formatting in routes_test.go Signed-off-by: Ramkumar Chinchani <rchincha.dev@gmail.com> --------- Signed-off-by: Ramkumar Chinchani <rchincha.dev@gmail.com> Co-authored-by: Akash Kumar <meakash7902@gmail.com>
432 lines
12 KiB
Go
432 lines
12 KiB
Go
package gcs_test
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/distribution/distribution/v3/registry/storage/driver"
|
|
. "github.com/smartystreets/goconvey/convey"
|
|
|
|
"zotregistry.dev/zot/v2/pkg/extensions/monitoring"
|
|
zlog "zotregistry.dev/zot/v2/pkg/log"
|
|
"zotregistry.dev/zot/v2/pkg/storage/gcs"
|
|
"zotregistry.dev/zot/v2/pkg/test/mocks"
|
|
)
|
|
|
|
var errTest = errors.New("error")
|
|
|
|
type fileInfoMock struct {
|
|
isDir bool
|
|
size int64
|
|
modTime time.Time
|
|
path string
|
|
}
|
|
|
|
func (f *fileInfoMock) Path() string { return f.path }
|
|
func (f *fileInfoMock) Size() int64 { return f.size }
|
|
func (f *fileInfoMock) ModTime() time.Time { return f.modTime }
|
|
func (f *fileInfoMock) IsDir() bool { return f.isDir }
|
|
|
|
func TestDriver(t *testing.T) {
|
|
Convey("GCS Driver", t, func() {
|
|
storeMock := &mocks.StorageDriverMock{}
|
|
gcsDriver := gcs.New(storeMock)
|
|
|
|
Convey("Name", func() {
|
|
So(gcsDriver.Name(), ShouldEqual, "gcs")
|
|
})
|
|
|
|
Convey("EnsureDir", func() {
|
|
err := gcsDriver.EnsureDir("/test")
|
|
So(err, ShouldBeNil)
|
|
})
|
|
|
|
Convey("DirExists", func() {
|
|
Convey("True", func() {
|
|
storeMock.StatFn = func(ctx context.Context, path string) (driver.FileInfo, error) {
|
|
return &mocks.FileInfoMock{
|
|
IsDirFn: func() bool { return true },
|
|
}, nil
|
|
}
|
|
So(gcsDriver.DirExists("/test"), ShouldBeTrue)
|
|
})
|
|
|
|
Convey("False - Not a dir", func() {
|
|
storeMock.StatFn = func(ctx context.Context, path string) (driver.FileInfo, error) {
|
|
return &mocks.FileInfoMock{
|
|
IsDirFn: func() bool { return false },
|
|
}, nil
|
|
}
|
|
So(gcsDriver.DirExists("/test"), ShouldBeFalse)
|
|
})
|
|
|
|
Convey("False - Error", func() {
|
|
storeMock.StatFn = func(ctx context.Context, path string) (driver.FileInfo, error) {
|
|
return nil, errTest
|
|
}
|
|
So(gcsDriver.DirExists("/test"), ShouldBeFalse)
|
|
})
|
|
})
|
|
|
|
Convey("Reader", func() {
|
|
Convey("Success", func() {
|
|
storeMock.ReaderFn = func(ctx context.Context, path string, offset int64) (io.ReadCloser, error) {
|
|
return io.NopCloser(strings.NewReader("")), nil
|
|
}
|
|
r, err := gcsDriver.Reader("/test", 0)
|
|
So(err, ShouldBeNil)
|
|
So(r, ShouldNotBeNil)
|
|
})
|
|
|
|
Convey("InvalidOffsetError", func() {
|
|
storeMock.ReaderFn = func(ctx context.Context, path string, offset int64) (io.ReadCloser, error) {
|
|
return nil, driver.InvalidOffsetError{Path: path, Offset: offset}
|
|
}
|
|
_, err := gcsDriver.Reader("/test", 100)
|
|
So(err, ShouldNotBeNil)
|
|
|
|
var invalidOffset driver.InvalidOffsetError
|
|
So(errors.As(err, &invalidOffset), ShouldBeTrue)
|
|
So(invalidOffset.DriverName, ShouldEqual, "gcs")
|
|
})
|
|
})
|
|
|
|
Convey("ReadFile", func() {
|
|
Convey("Success", func() {
|
|
storeMock.GetContentFn = func(ctx context.Context, path string) ([]byte, error) {
|
|
return []byte("content"), nil
|
|
}
|
|
content, err := gcsDriver.ReadFile("/test")
|
|
So(err, ShouldBeNil)
|
|
So(string(content), ShouldEqual, "content")
|
|
})
|
|
|
|
Convey("PathNotFoundError with empty Path gets path set", func() {
|
|
storeMock.GetContentFn = func(ctx context.Context, path string) ([]byte, error) {
|
|
return nil, driver.PathNotFoundError{Path: ""} // Path empty so driver sets it
|
|
}
|
|
_, err := gcsDriver.ReadFile("/requested/path")
|
|
So(err, ShouldNotBeNil)
|
|
|
|
var pathErr driver.PathNotFoundError
|
|
So(errors.As(err, &pathErr), ShouldBeTrue)
|
|
So(pathErr.DriverName, ShouldEqual, "gcs")
|
|
So(pathErr.Path, ShouldEqual, "/requested/path")
|
|
})
|
|
|
|
Convey("GCS not-found string becomes PathNotFoundError", func() {
|
|
for _, msg := range []string{"object doesn't exist", "Error 404", "does not exist"} {
|
|
errMsg := msg
|
|
storeMock.GetContentFn = func(ctx context.Context, path string) ([]byte, error) {
|
|
//nolint:err113 // test needs variable not-found message
|
|
return nil, fmt.Errorf("%s", errMsg)
|
|
}
|
|
_, err := gcsDriver.ReadFile("/key")
|
|
So(err, ShouldNotBeNil)
|
|
|
|
var pathErr driver.PathNotFoundError
|
|
So(errors.As(err, &pathErr), ShouldBeTrue)
|
|
So(pathErr.Path, ShouldEqual, "/key")
|
|
}
|
|
})
|
|
|
|
Convey("Generic error becomes storagedriver.Error", func() {
|
|
storeMock.GetContentFn = func(ctx context.Context, path string) ([]byte, error) {
|
|
return nil, errTest
|
|
}
|
|
_, err := gcsDriver.ReadFile("/test")
|
|
So(err, ShouldNotBeNil)
|
|
|
|
var storageErr driver.Error
|
|
So(errors.As(err, &storageErr), ShouldBeTrue)
|
|
So(storageErr.DriverName, ShouldEqual, "gcs")
|
|
So(storageErr.Detail, ShouldEqual, errTest)
|
|
})
|
|
})
|
|
|
|
Convey("Delete", func() {
|
|
Convey("Success", func() {
|
|
storeMock.DeleteFn = func(ctx context.Context, path string) error {
|
|
return nil
|
|
}
|
|
err := gcsDriver.Delete("/test")
|
|
So(err, ShouldBeNil)
|
|
})
|
|
|
|
Convey("PathNotFoundError is idempotent (return nil)", func() {
|
|
storeMock.DeleteFn = func(ctx context.Context, path string) error {
|
|
return driver.PathNotFoundError{Path: path}
|
|
}
|
|
err := gcsDriver.Delete("/nonexistent")
|
|
So(err, ShouldBeNil)
|
|
})
|
|
|
|
Convey("Other error is returned", func() {
|
|
storeMock.DeleteFn = func(ctx context.Context, path string) error {
|
|
return errTest
|
|
}
|
|
err := gcsDriver.Delete("/test")
|
|
So(err, ShouldNotBeNil)
|
|
So(errors.Is(err, errTest), ShouldBeFalse) // wrapped in storagedriver.Error
|
|
})
|
|
})
|
|
|
|
Convey("Stat", func() {
|
|
Convey("Success", func() {
|
|
storeMock.StatFn = func(ctx context.Context, path string) (driver.FileInfo, error) {
|
|
return &mocks.FileInfoMock{}, nil
|
|
}
|
|
fi, err := gcsDriver.Stat("/test")
|
|
So(err, ShouldBeNil)
|
|
So(fi, ShouldNotBeNil)
|
|
})
|
|
|
|
Convey("InvalidPathError", func() {
|
|
storeMock.StatFn = func(ctx context.Context, path string) (driver.FileInfo, error) {
|
|
return nil, driver.InvalidPathError{Path: path}
|
|
}
|
|
_, err := gcsDriver.Stat("/bad")
|
|
So(err, ShouldNotBeNil)
|
|
|
|
var invalidPath driver.InvalidPathError
|
|
So(errors.As(err, &invalidPath), ShouldBeTrue)
|
|
So(invalidPath.DriverName, ShouldEqual, "gcs")
|
|
})
|
|
})
|
|
|
|
Convey("Writer", func() {
|
|
Convey("Success", func() {
|
|
storeMock.WriterFn = func(ctx context.Context, path string, isAppend bool) (driver.FileWriter, error) {
|
|
return &mocks.FileWriterMock{}, nil
|
|
}
|
|
w, err := gcsDriver.Writer("/test", false)
|
|
So(err, ShouldBeNil)
|
|
So(w, ShouldNotBeNil)
|
|
})
|
|
|
|
Convey("Error", func() {
|
|
storeMock.WriterFn = func(ctx context.Context, path string, isAppend bool) (driver.FileWriter, error) {
|
|
return nil, errTest
|
|
}
|
|
_, err := gcsDriver.Writer("/test", false)
|
|
So(err, ShouldNotBeNil)
|
|
})
|
|
})
|
|
|
|
Convey("WriteFile", func() {
|
|
Convey("Success", func() {
|
|
storeMock.WriterFn = func(ctx context.Context, path string, isAppend bool) (driver.FileWriter, error) {
|
|
return &mocks.FileWriterMock{
|
|
WriteFn: func(p []byte) (int, error) {
|
|
return len(p), nil
|
|
},
|
|
CommitFn: func() error {
|
|
return nil
|
|
},
|
|
CloseFn: func() error {
|
|
return nil
|
|
},
|
|
}, nil
|
|
}
|
|
n, err := gcsDriver.WriteFile("/test", []byte("content"))
|
|
So(err, ShouldBeNil)
|
|
So(n, ShouldEqual, 7)
|
|
})
|
|
|
|
Convey("Writer Error", func() {
|
|
storeMock.WriterFn = func(ctx context.Context, path string, isAppend bool) (driver.FileWriter, error) {
|
|
return nil, errTest
|
|
}
|
|
_, err := gcsDriver.WriteFile("/test", []byte("content"))
|
|
So(err, ShouldNotBeNil)
|
|
})
|
|
|
|
Convey("Write Error", func() {
|
|
storeMock.WriterFn = func(ctx context.Context, path string, isAppend bool) (driver.FileWriter, error) {
|
|
return &mocks.FileWriterMock{
|
|
WriteFn: func(p []byte) (int, error) {
|
|
return 0, errTest
|
|
},
|
|
CloseFn: func() error { return nil },
|
|
}, nil
|
|
}
|
|
_, err := gcsDriver.WriteFile("/test", []byte("content"))
|
|
So(err, ShouldNotBeNil)
|
|
})
|
|
|
|
Convey("Commit Error", func() {
|
|
storeMock.WriterFn = func(ctx context.Context, path string, isAppend bool) (driver.FileWriter, error) {
|
|
return &mocks.FileWriterMock{
|
|
WriteFn: func(p []byte) (int, error) {
|
|
return len(p), nil
|
|
},
|
|
CommitFn: func() error {
|
|
return errTest
|
|
},
|
|
CloseFn: func() error { return nil },
|
|
}, nil
|
|
}
|
|
_, err := gcsDriver.WriteFile("/test", []byte("content"))
|
|
So(err, ShouldNotBeNil)
|
|
})
|
|
})
|
|
|
|
Convey("Walk", func() {
|
|
Convey("Success", func() {
|
|
storeMock.WalkFn = func(ctx context.Context, path string, f driver.WalkFn, _ ...func(*driver.WalkOptions)) error {
|
|
return nil
|
|
}
|
|
err := gcsDriver.Walk("/test", nil)
|
|
So(err, ShouldBeNil)
|
|
})
|
|
|
|
Convey("Direct io.EOF is returned as io.EOF", func() {
|
|
storeMock.WalkFn = func(ctx context.Context, path string, f driver.WalkFn, _ ...func(*driver.WalkOptions)) error {
|
|
return io.EOF
|
|
}
|
|
err := gcsDriver.Walk("/test", nil)
|
|
So(errors.Is(err, io.EOF), ShouldBeTrue)
|
|
})
|
|
|
|
Convey("io.EOF wrapped in storagedriver.Error is returned as io.EOF", func() {
|
|
storeMock.WalkFn = func(ctx context.Context, path string, f driver.WalkFn, _ ...func(*driver.WalkOptions)) error {
|
|
return driver.Error{
|
|
DriverName: "gcs",
|
|
Detail: io.EOF,
|
|
}
|
|
}
|
|
err := gcsDriver.Walk("/test", nil)
|
|
So(errors.Is(err, io.EOF), ShouldBeTrue)
|
|
})
|
|
|
|
Convey("Non-EOF error is formatted normally", func() {
|
|
storeMock.WalkFn = func(ctx context.Context, path string, f driver.WalkFn, _ ...func(*driver.WalkOptions)) error {
|
|
return errTest
|
|
}
|
|
err := gcsDriver.Walk("/test", nil)
|
|
So(err, ShouldNotBeNil)
|
|
So(errors.Is(err, io.EOF), ShouldBeFalse)
|
|
|
|
var storageErr driver.Error
|
|
So(errors.As(err, &storageErr), ShouldBeTrue)
|
|
})
|
|
})
|
|
|
|
Convey("List", func() {
|
|
Convey("Success", func() {
|
|
storeMock.ListFn = func(ctx context.Context, path string) ([]string, error) {
|
|
return []string{"a"}, nil
|
|
}
|
|
l, err := gcsDriver.List("/test")
|
|
So(err, ShouldBeNil)
|
|
So(l, ShouldResemble, []string{"a"})
|
|
})
|
|
|
|
Convey("Error", func() {
|
|
storeMock.ListFn = func(ctx context.Context, path string) ([]string, error) {
|
|
return nil, errTest
|
|
}
|
|
_, err := gcsDriver.List("/test")
|
|
So(err, ShouldNotBeNil)
|
|
})
|
|
})
|
|
|
|
Convey("Move", func() {
|
|
storeMock.MoveFn = func(ctx context.Context, sourcePath, destPath string) error {
|
|
return nil
|
|
}
|
|
err := gcsDriver.Move("/src", "/dst")
|
|
So(err, ShouldBeNil)
|
|
})
|
|
|
|
Convey("SameFile", func() {
|
|
Convey("True", func() {
|
|
now := time.Now()
|
|
storeMock.StatFn = func(ctx context.Context, path string) (driver.FileInfo, error) {
|
|
return &fileInfoMock{
|
|
isDir: false,
|
|
size: 10,
|
|
modTime: now,
|
|
path: "/canonical/path",
|
|
}, nil
|
|
}
|
|
So(gcsDriver.SameFile("/path1", "/path2"), ShouldBeTrue)
|
|
})
|
|
|
|
Convey("False - Different ModTime", func() {
|
|
storeMock.StatFn = func(ctx context.Context, path string) (driver.FileInfo, error) {
|
|
modTime := time.Now()
|
|
if path == "/path2" {
|
|
modTime = modTime.Add(1 * time.Hour)
|
|
}
|
|
|
|
return &fileInfoMock{
|
|
isDir: false,
|
|
size: 10,
|
|
modTime: modTime,
|
|
path: path,
|
|
}, nil
|
|
}
|
|
So(gcsDriver.SameFile("/path1", "/path2"), ShouldBeFalse)
|
|
})
|
|
})
|
|
|
|
Convey("Link", func() {
|
|
storeMock.PutContentFn = func(ctx context.Context, path string, content []byte) error {
|
|
return nil
|
|
}
|
|
err := gcsDriver.Link("/src", "/dst")
|
|
So(err, ShouldBeNil)
|
|
})
|
|
|
|
Convey("RedirectURL", func() {
|
|
req := httptest.NewRequestWithContext(context.Background(), http.MethodGet,
|
|
"http://localhost/v2/repo/blobs/sha256:abc", nil)
|
|
|
|
Convey("Success", func() {
|
|
storeMock.RedirectURLFn = func(r *http.Request, path string) (string, error) {
|
|
So(r, ShouldEqual, req)
|
|
So(path, ShouldEqual, "/blob/path")
|
|
|
|
return "https://example.com/signed", nil
|
|
}
|
|
|
|
url, err := gcsDriver.RedirectURL(req, "/blob/path")
|
|
So(err, ShouldBeNil)
|
|
So(url, ShouldEqual, "https://example.com/signed")
|
|
})
|
|
|
|
Convey("Error", func() {
|
|
storeMock.RedirectURLFn = func(_ *http.Request, _ string) (string, error) {
|
|
return "", errTest
|
|
}
|
|
|
|
url, err := gcsDriver.RedirectURL(req, "/blob/path")
|
|
So(url, ShouldEqual, "")
|
|
So(err, ShouldNotBeNil)
|
|
|
|
var storageErr driver.Error
|
|
So(errors.As(err, &storageErr), ShouldBeTrue)
|
|
So(storageErr.DriverName, ShouldEqual, "gcs")
|
|
})
|
|
})
|
|
})
|
|
}
|
|
|
|
func TestNewImageStore(t *testing.T) {
|
|
Convey("NewImageStore", t, func() {
|
|
storeMock := &mocks.StorageDriverMock{}
|
|
log := zlog.NewTestLogger()
|
|
metrics := monitoring.NewMetricsServer(false, log)
|
|
imgStore := gcs.NewImageStore("/tmp", "/tmp", true, true, log, metrics, nil, storeMock, nil, nil, nil)
|
|
So(imgStore, ShouldNotBeNil)
|
|
})
|
|
}
|