lint: upgrade golangci-lint

Signed-off-by: Ramkumar Chinchani <rchincha@cisco.com>
This commit is contained in:
Ramkumar Chinchani
2021-12-13 19:23:31 +00:00
committed by Ravi Chamarthy
parent 5f04092e71
commit ac3801ea2d
71 changed files with 3038 additions and 2575 deletions
+151 -155
View File
@@ -11,28 +11,22 @@ import (
"os"
"path"
"strings"
"testing"
"time"
godigest "github.com/opencontainers/go-digest"
//"strings"
"testing"
"github.com/docker/distribution/registry/storage/driver"
"github.com/docker/distribution/registry/storage/driver/factory"
_ "github.com/docker/distribution/registry/storage/driver/s3-aws"
guuid "github.com/gofrs/uuid"
godigest "github.com/opencontainers/go-digest"
"github.com/rs/zerolog"
. "github.com/smartystreets/goconvey/convey"
"gopkg.in/resty.v1"
zerr "zotregistry.io/zot/errors"
"zotregistry.io/zot/pkg/extensions/monitoring"
"zotregistry.io/zot/pkg/log"
"zotregistry.io/zot/pkg/storage"
"zotregistry.io/zot/pkg/storage/s3"
// Add s3 support
storageDriver "github.com/docker/distribution/registry/storage/driver"
"github.com/docker/distribution/registry/storage/driver/factory"
_ "github.com/docker/distribution/registry/storage/driver/s3-aws"
"gopkg.in/resty.v1"
)
// nolint: gochecknoglobals
@@ -44,17 +38,19 @@ var (
errS3 = errors.New(errorText)
)
func cleanupStorage(store storageDriver.StorageDriver, name string) {
func cleanupStorage(store driver.StorageDriver, name string) {
_ = store.Delete(context.Background(), name)
}
func skipIt(t *testing.T) {
t.Helper()
if os.Getenv("S3MOCK_ENDPOINT") == "" {
t.Skip("Skipping testing without AWS S3 mock server")
}
}
func createMockStorage(rootDir string, store storageDriver.StorageDriver) storage.ImageStore {
func createMockStorage(rootDir string, store driver.StorageDriver) storage.ImageStore {
log := log.Logger{Logger: zerolog.New(os.Stdout)}
metrics := monitoring.NewMetricsServer(false, log)
il := s3.NewImageStore(rootDir, false, false, log, metrics, store)
@@ -62,7 +58,7 @@ func createMockStorage(rootDir string, store storageDriver.StorageDriver) storag
return il
}
func createObjectsStore(rootDir string) (storageDriver.StorageDriver, storage.ImageStore, error) {
func createObjectsStore(rootDir string) (driver.StorageDriver, storage.ImageStore, error) {
bucket := "zot-storage-test"
endpoint := os.Getenv("S3MOCK_ENDPOINT")
storageDriverParams := map[string]interface{}{
@@ -167,12 +163,12 @@ type StorageDriverMock struct {
getContentFn func(ctx context.Context, path string) ([]byte, error)
putContentFn func(ctx context.Context, path string, content []byte) error
readerFn func(ctx context.Context, path string, offset int64) (io.ReadCloser, error)
writerFn func(ctx context.Context, path string, append bool) (storageDriver.FileWriter, error)
statFn func(ctx context.Context, path string) (storageDriver.FileInfo, error)
writerFn func(ctx context.Context, path string, isAppend bool) (driver.FileWriter, error)
statFn func(ctx context.Context, path string) (driver.FileInfo, error)
listFn func(ctx context.Context, path string) ([]string, error)
moveFn func(ctx context.Context, sourcePath string, destPath string) error
deleteFn func(ctx context.Context, path string) error
walkFn func(ctx context.Context, path string, f storageDriver.WalkFn) error
walkFn func(ctx context.Context, path string, f driver.WalkFn) error
}
func (s *StorageDriverMock) Name() string {
@@ -207,15 +203,15 @@ func (s *StorageDriverMock) Reader(ctx context.Context, path string, offset int6
return ioutil.NopCloser(strings.NewReader("")), nil
}
func (s *StorageDriverMock) Writer(ctx context.Context, path string, append bool) (storageDriver.FileWriter, error) {
func (s *StorageDriverMock) Writer(ctx context.Context, path string, isAppend bool) (driver.FileWriter, error) {
if s != nil && s.writerFn != nil {
return s.writerFn(ctx, path, append)
return s.writerFn(ctx, path, isAppend)
}
return &FileWriterMock{}, nil
}
func (s *StorageDriverMock) Stat(ctx context.Context, path string) (storageDriver.FileInfo, error) {
func (s *StorageDriverMock) Stat(ctx context.Context, path string) (driver.FileInfo, error) {
if s != nil && s.statFn != nil {
return s.statFn(ctx, path)
}
@@ -251,7 +247,7 @@ func (s *StorageDriverMock) URLFor(ctx context.Context, path string, options map
return "", nil
}
func (s *StorageDriverMock) Walk(ctx context.Context, path string, f storageDriver.WalkFn) error {
func (s *StorageDriverMock) Walk(ctx context.Context, path string, f driver.WalkFn) error {
if s != nil && s.walkFn != nil {
return s.walkFn(ctx, path, f)
}
@@ -269,117 +265,117 @@ func TestNegativeCasesObjectsStorage(t *testing.T) {
testDir := path.Join("/oci-repo-test", uuid.String())
store, il, _ := createObjectsStore(testDir)
defer cleanupStorage(store, testDir)
storeDriver, imgStore, _ := createObjectsStore(testDir)
defer cleanupStorage(storeDriver, testDir)
Convey("Invalid validate repo", t, func(c C) {
So(il, ShouldNotBeNil)
So(il.InitRepo(testImage), ShouldBeNil)
objects, err := store.List(context.Background(), path.Join(il.RootDir(), testImage))
So(imgStore, ShouldNotBeNil)
So(imgStore.InitRepo(testImage), ShouldBeNil)
objects, err := storeDriver.List(context.Background(), path.Join(imgStore.RootDir(), testImage))
So(err, ShouldBeNil)
for _, object := range objects {
t.Logf("Removing object: %s", object)
err := store.Delete(context.Background(), object)
err := storeDriver.Delete(context.Background(), object)
So(err, ShouldBeNil)
}
_, err = il.ValidateRepo(testImage)
_, err = imgStore.ValidateRepo(testImage)
So(err, ShouldNotBeNil)
_, err = il.GetRepositories()
_, err = imgStore.GetRepositories()
So(err, ShouldBeNil)
})
Convey("Invalid get image tags", t, func(c C) {
store, il, err := createObjectsStore(testDir)
defer cleanupStorage(store, testDir)
storeDriver, imgStore, err := createObjectsStore(testDir)
defer cleanupStorage(storeDriver, testDir)
So(err, ShouldBeNil)
So(il.InitRepo(testImage), ShouldBeNil)
So(imgStore.InitRepo(testImage), ShouldBeNil)
So(store.Move(context.Background(), path.Join(testDir, testImage, "index.json"),
So(storeDriver.Move(context.Background(), path.Join(testDir, testImage, "index.json"),
path.Join(testDir, testImage, "blobs")), ShouldBeNil)
ok, _ := il.ValidateRepo(testImage)
ok, _ := imgStore.ValidateRepo(testImage)
So(ok, ShouldBeFalse)
_, err = il.GetImageTags(testImage)
_, err = imgStore.GetImageTags(testImage)
So(err, ShouldNotBeNil)
So(store.Delete(context.Background(), path.Join(testDir, testImage)), ShouldBeNil)
So(storeDriver.Delete(context.Background(), path.Join(testDir, testImage)), ShouldBeNil)
So(il.InitRepo(testImage), ShouldBeNil)
So(store.PutContent(context.Background(), path.Join(testDir, testImage, "index.json"), []byte{}), ShouldBeNil)
_, err = il.GetImageTags(testImage)
So(imgStore.InitRepo(testImage), ShouldBeNil)
So(storeDriver.PutContent(context.Background(), path.Join(testDir, testImage, "index.json"), []byte{}), ShouldBeNil)
_, err = imgStore.GetImageTags(testImage)
So(err, ShouldNotBeNil)
})
Convey("Invalid get image manifest", t, func(c C) {
store, il, err := createObjectsStore(testDir)
defer cleanupStorage(store, testDir)
storeDriver, imgStore, err := createObjectsStore(testDir)
defer cleanupStorage(storeDriver, testDir)
So(err, ShouldBeNil)
So(il, ShouldNotBeNil)
So(il.InitRepo(testImage), ShouldBeNil)
So(store.Delete(context.Background(), path.Join(testDir, testImage, "index.json")), ShouldBeNil)
_, _, _, err = il.GetImageManifest(testImage, "")
So(imgStore, ShouldNotBeNil)
So(imgStore.InitRepo(testImage), ShouldBeNil)
So(storeDriver.Delete(context.Background(), path.Join(testDir, testImage, "index.json")), ShouldBeNil)
_, _, _, err = imgStore.GetImageManifest(testImage, "")
So(err, ShouldNotBeNil)
So(store.Delete(context.Background(), path.Join(testDir, testImage)), ShouldBeNil)
So(il.InitRepo(testImage), ShouldBeNil)
So(store.PutContent(context.Background(), path.Join(testDir, testImage, "index.json"), []byte{}), ShouldBeNil)
_, _, _, err = il.GetImageManifest(testImage, "")
So(storeDriver.Delete(context.Background(), path.Join(testDir, testImage)), ShouldBeNil)
So(imgStore.InitRepo(testImage), ShouldBeNil)
So(storeDriver.PutContent(context.Background(), path.Join(testDir, testImage, "index.json"), []byte{}), ShouldBeNil)
_, _, _, err = imgStore.GetImageManifest(testImage, "")
So(err, ShouldNotBeNil)
})
Convey("Invalid validate repo", t, func(c C) {
store, il, err := createObjectsStore(testDir)
defer cleanupStorage(store, testDir)
storeDriver, imgStore, err := createObjectsStore(testDir)
defer cleanupStorage(storeDriver, testDir)
So(err, ShouldBeNil)
So(il, ShouldNotBeNil)
So(imgStore, ShouldNotBeNil)
So(il.InitRepo(testImage), ShouldBeNil)
So(store.Delete(context.Background(), path.Join(testDir, testImage, "index.json")), ShouldBeNil)
_, err = il.ValidateRepo(testImage)
So(imgStore.InitRepo(testImage), ShouldBeNil)
So(storeDriver.Delete(context.Background(), path.Join(testDir, testImage, "index.json")), ShouldBeNil)
_, err = imgStore.ValidateRepo(testImage)
So(err, ShouldNotBeNil)
So(store.Delete(context.Background(), path.Join(testDir, testImage)), ShouldBeNil)
So(il.InitRepo(testImage), ShouldBeNil)
So(store.Move(context.Background(), path.Join(testDir, testImage, "index.json"),
So(storeDriver.Delete(context.Background(), path.Join(testDir, testImage)), ShouldBeNil)
So(imgStore.InitRepo(testImage), ShouldBeNil)
So(storeDriver.Move(context.Background(), path.Join(testDir, testImage, "index.json"),
path.Join(testDir, testImage, "_index.json")), ShouldBeNil)
ok, err := il.ValidateRepo(testImage)
ok, err := imgStore.ValidateRepo(testImage)
So(err, ShouldBeNil)
So(ok, ShouldBeFalse)
})
Convey("Invalid finish blob upload", t, func(c C) {
store, il, err := createObjectsStore(testDir)
defer cleanupStorage(store, testDir)
storeDriver, imgStore, err := createObjectsStore(testDir)
defer cleanupStorage(storeDriver, testDir)
So(err, ShouldBeNil)
So(il, ShouldNotBeNil)
So(imgStore, ShouldNotBeNil)
So(il.InitRepo(testImage), ShouldBeNil)
v, err := il.NewBlobUpload(testImage)
So(imgStore.InitRepo(testImage), ShouldBeNil)
upload, err := imgStore.NewBlobUpload(testImage)
So(err, ShouldBeNil)
So(v, ShouldNotBeEmpty)
So(upload, ShouldNotBeEmpty)
content := []byte("test-data1")
buf := bytes.NewBuffer(content)
l := buf.Len()
d := godigest.FromBytes(content)
buflen := buf.Len()
digest := godigest.FromBytes(content)
b, err := il.PutBlobChunk(testImage, v, 0, int64(l), buf)
blob, err := imgStore.PutBlobChunk(testImage, upload, 0, int64(buflen), buf)
So(err, ShouldBeNil)
So(b, ShouldEqual, l)
So(blob, ShouldEqual, buflen)
src := il.BlobUploadPath(testImage, v)
fw, err := store.Writer(context.Background(), src, true)
src := imgStore.BlobUploadPath(testImage, upload)
stwr, err := storeDriver.Writer(context.Background(), src, true)
So(err, ShouldBeNil)
_, err = fw.Write([]byte("another-chunk-of-data"))
_, err = stwr.Write([]byte("another-chunk-of-data"))
So(err, ShouldBeNil)
err = fw.Close()
err = stwr.Close()
So(err, ShouldBeNil)
err = il.FinishBlobUpload(testImage, v, buf, d.String())
err = imgStore.FinishBlobUpload(testImage, upload, buf, digest.String())
So(err, ShouldNotBeNil)
})
Convey("Test storage driver errors", t, func(c C) {
il = createMockStorage(testDir, &StorageDriverMock{
imgStore = createMockStorage(testDir, &StorageDriverMock{
listFn: func(ctx context.Context, path string) ([]string, error) {
return []string{testImage}, errS3
},
@@ -392,202 +388,202 @@ func TestNegativeCasesObjectsStorage(t *testing.T) {
putContentFn: func(ctx context.Context, path string, content []byte) error {
return errS3
},
writerFn: func(ctx context.Context, path string, append bool) (storageDriver.FileWriter, error) {
writerFn: func(ctx context.Context, path string, isAppend bool) (driver.FileWriter, error) {
return &FileWriterMock{}, errS3
},
readerFn: func(ctx context.Context, path string, offset int64) (io.ReadCloser, error) {
return ioutil.NopCloser(strings.NewReader("")), errS3
},
walkFn: func(ctx context.Context, path string, f storageDriver.WalkFn) error {
walkFn: func(ctx context.Context, path string, f driver.WalkFn) error {
return errS3
},
statFn: func(ctx context.Context, path string) (storageDriver.FileInfo, error) {
statFn: func(ctx context.Context, path string) (driver.FileInfo, error) {
return &FileInfoMock{}, errS3
},
deleteFn: func(ctx context.Context, path string) error {
return errS3
},
})
So(il, ShouldNotBeNil)
So(imgStore, ShouldNotBeNil)
So(il.InitRepo(testImage), ShouldNotBeNil)
_, err := il.ValidateRepo(testImage)
So(imgStore.InitRepo(testImage), ShouldNotBeNil)
_, err := imgStore.ValidateRepo(testImage)
So(err, ShouldNotBeNil)
v, err := il.NewBlobUpload(testImage)
upload, err := imgStore.NewBlobUpload(testImage)
So(err, ShouldNotBeNil)
content := []byte("test-data1")
buf := bytes.NewBuffer(content)
l := buf.Len()
d := godigest.FromBytes(content)
buflen := buf.Len()
digest := godigest.FromBytes(content)
_, err = il.PutBlobChunk(testImage, v, 0, int64(l), buf)
_, err = imgStore.PutBlobChunk(testImage, upload, 0, int64(buflen), buf)
So(err, ShouldNotBeNil)
err = il.FinishBlobUpload(testImage, v, buf, d.String())
err = imgStore.FinishBlobUpload(testImage, upload, buf, digest.String())
So(err, ShouldNotBeNil)
err = il.DeleteBlob(testImage, d.String())
err = imgStore.DeleteBlob(testImage, digest.String())
So(err, ShouldNotBeNil)
err = il.DeleteBlobUpload(testImage, v)
err = imgStore.DeleteBlobUpload(testImage, upload)
So(err, ShouldNotBeNil)
err = il.DeleteImageManifest(testImage, "1.0")
err = imgStore.DeleteImageManifest(testImage, "1.0")
So(err, ShouldNotBeNil)
_, err = il.PutImageManifest(testImage, "1.0", "application/json", []byte{})
_, err = imgStore.PutImageManifest(testImage, "1.0", "application/json", []byte{})
So(err, ShouldNotBeNil)
_, err = il.PutBlobChunkStreamed(testImage, v, bytes.NewBuffer([]byte(testImage)))
_, err = imgStore.PutBlobChunkStreamed(testImage, upload, bytes.NewBuffer([]byte(testImage)))
So(err, ShouldNotBeNil)
_, _, err = il.FullBlobUpload(testImage, bytes.NewBuffer([]byte{}), "inexistent")
_, _, err = imgStore.FullBlobUpload(testImage, bytes.NewBuffer([]byte{}), "inexistent")
So(err, ShouldNotBeNil)
_, _, err = il.CheckBlob(testImage, d.String())
_, _, err = imgStore.CheckBlob(testImage, digest.String())
So(err, ShouldNotBeNil)
})
Convey("Test ValidateRepo", t, func(c C) {
il = createMockStorage(testDir, &StorageDriverMock{
imgStore = createMockStorage(testDir, &StorageDriverMock{
listFn: func(ctx context.Context, path string) ([]string, error) {
return []string{testImage, testImage}, errS3
},
})
_, err := il.ValidateRepo(testImage)
_, err := imgStore.ValidateRepo(testImage)
So(err, ShouldNotBeNil)
})
Convey("Test ValidateRepo2", t, func(c C) {
il = createMockStorage(testDir, &StorageDriverMock{
imgStore = createMockStorage(testDir, &StorageDriverMock{
listFn: func(ctx context.Context, path string) ([]string, error) {
return []string{"test/test/oci-layout", "test/test/index.json"}, nil
},
statFn: func(ctx context.Context, path string) (storageDriver.FileInfo, error) {
statFn: func(ctx context.Context, path string) (driver.FileInfo, error) {
return &FileInfoMock{}, nil
},
})
_, err := il.ValidateRepo(testImage)
_, err := imgStore.ValidateRepo(testImage)
So(err, ShouldNotBeNil)
})
Convey("Test ValidateRepo3", t, func(c C) {
il = createMockStorage(testDir, &StorageDriverMock{
imgStore = createMockStorage(testDir, &StorageDriverMock{
listFn: func(ctx context.Context, path string) ([]string, error) {
return []string{"test/test/oci-layout", "test/test/index.json"}, nil
},
statFn: func(ctx context.Context, path string) (storageDriver.FileInfo, error) {
statFn: func(ctx context.Context, path string) (driver.FileInfo, error) {
return &FileInfoMock{}, nil
},
getContentFn: func(ctx context.Context, path string) ([]byte, error) {
return []byte{}, errS3
},
})
_, err := il.ValidateRepo(testImage)
_, err := imgStore.ValidateRepo(testImage)
So(err, ShouldNotBeNil)
})
Convey("Test ValidateRepo4", t, func(c C) {
ociLayout := []byte(`{"imageLayoutVersion": "9.9.9"}`)
il = createMockStorage(testDir, &StorageDriverMock{
imgStore = createMockStorage(testDir, &StorageDriverMock{
listFn: func(ctx context.Context, path string) ([]string, error) {
return []string{"test/test/oci-layout", "test/test/index.json"}, nil
},
statFn: func(ctx context.Context, path string) (storageDriver.FileInfo, error) {
statFn: func(ctx context.Context, path string) (driver.FileInfo, error) {
return &FileInfoMock{}, nil
},
getContentFn: func(ctx context.Context, path string) ([]byte, error) {
return ociLayout, nil
},
})
_, err := il.ValidateRepo(testImage)
_, err := imgStore.ValidateRepo(testImage)
So(err, ShouldNotBeNil)
})
Convey("Test GetRepositories", t, func(c C) {
il = createMockStorage(testDir, &StorageDriverMock{
walkFn: func(ctx context.Context, path string, f storageDriver.WalkFn) error {
imgStore = createMockStorage(testDir, &StorageDriverMock{
walkFn: func(ctx context.Context, path string, f driver.WalkFn) error {
return f(new(FileInfoMock))
},
})
repos, err := il.GetRepositories()
repos, err := imgStore.GetRepositories()
So(repos, ShouldBeEmpty)
So(err, ShouldBeNil)
})
Convey("Test DeleteImageManifest", t, func(c C) {
il = createMockStorage(testDir, &StorageDriverMock{
imgStore = createMockStorage(testDir, &StorageDriverMock{
getContentFn: func(ctx context.Context, path string) ([]byte, error) {
return []byte{}, errS3
},
})
err := il.DeleteImageManifest(testImage, "1.0")
err := imgStore.DeleteImageManifest(testImage, "1.0")
So(err, ShouldNotBeNil)
})
Convey("Test DeleteImageManifest2", t, func(c C) {
il = createMockStorage(testDir, &StorageDriverMock{})
err := il.DeleteImageManifest(testImage, "1.0")
imgStore = createMockStorage(testDir, &StorageDriverMock{})
err := imgStore.DeleteImageManifest(testImage, "1.0")
So(err, ShouldNotBeNil)
})
Convey("Test NewBlobUpload", t, func(c C) {
il = createMockStorage(testDir, &StorageDriverMock{
imgStore = createMockStorage(testDir, &StorageDriverMock{
putContentFn: func(ctx context.Context, path string, content []byte) error {
return errS3
},
})
_, err := il.NewBlobUpload(testImage)
_, err := imgStore.NewBlobUpload(testImage)
So(err, ShouldNotBeNil)
})
Convey("Test GetBlobUpload", t, func(c C) {
il = createMockStorage(testDir, &StorageDriverMock{
statFn: func(ctx context.Context, path string) (storageDriver.FileInfo, error) {
imgStore = createMockStorage(testDir, &StorageDriverMock{
statFn: func(ctx context.Context, path string) (driver.FileInfo, error) {
return &FileInfoMock{}, errS3
},
})
_, err := il.GetBlobUpload(testImage, "uuid")
_, err := imgStore.GetBlobUpload(testImage, "uuid")
So(err, ShouldNotBeNil)
})
Convey("Test PutBlobChunkStreamed", t, func(c C) {
il = createMockStorage(testDir, &StorageDriverMock{
writerFn: func(ctx context.Context, path string, append bool) (storageDriver.FileWriter, error) {
imgStore = createMockStorage(testDir, &StorageDriverMock{
writerFn: func(ctx context.Context, path string, isAppend bool) (driver.FileWriter, error) {
return &FileWriterMock{}, errS3
},
})
_, err := il.PutBlobChunkStreamed(testImage, "uuid", ioutil.NopCloser(strings.NewReader("")))
_, err := imgStore.PutBlobChunkStreamed(testImage, "uuid", ioutil.NopCloser(strings.NewReader("")))
So(err, ShouldNotBeNil)
})
Convey("Test PutBlobChunkStreamed2", t, func(c C) {
il = createMockStorage(testDir, &StorageDriverMock{
writerFn: func(ctx context.Context, path string, append bool) (storageDriver.FileWriter, error) {
imgStore = createMockStorage(testDir, &StorageDriverMock{
writerFn: func(ctx context.Context, path string, isAppend bool) (driver.FileWriter, error) {
return &FileWriterMock{writeFn: func(b []byte) (int, error) {
return 0, errS3
}}, nil
},
})
_, err := il.PutBlobChunkStreamed(testImage, "uuid", ioutil.NopCloser(strings.NewReader("")))
_, err := imgStore.PutBlobChunkStreamed(testImage, "uuid", ioutil.NopCloser(strings.NewReader("")))
So(err, ShouldNotBeNil)
})
Convey("Test PutBlobChunk", t, func(c C) {
il = createMockStorage(testDir, &StorageDriverMock{
writerFn: func(ctx context.Context, path string, append bool) (storageDriver.FileWriter, error) {
imgStore = createMockStorage(testDir, &StorageDriverMock{
writerFn: func(ctx context.Context, path string, isAppend bool) (driver.FileWriter, error) {
return &FileWriterMock{}, errS3
},
})
_, err := il.PutBlobChunk(testImage, "uuid", 0, 100, ioutil.NopCloser(strings.NewReader("")))
_, err := imgStore.PutBlobChunk(testImage, "uuid", 0, 100, ioutil.NopCloser(strings.NewReader("")))
So(err, ShouldNotBeNil)
})
Convey("Test PutBlobChunk2", t, func(c C) {
il = createMockStorage(testDir, &StorageDriverMock{
writerFn: func(ctx context.Context, path string, append bool) (storageDriver.FileWriter, error) {
imgStore = createMockStorage(testDir, &StorageDriverMock{
writerFn: func(ctx context.Context, path string, isAppend bool) (driver.FileWriter, error) {
return &FileWriterMock{
writeFn: func(b []byte) (int, error) {
return 0, errS3
@@ -598,13 +594,13 @@ func TestNegativeCasesObjectsStorage(t *testing.T) {
}, nil
},
})
_, err := il.PutBlobChunk(testImage, "uuid", 0, 100, ioutil.NopCloser(strings.NewReader("")))
_, err := imgStore.PutBlobChunk(testImage, "uuid", 0, 100, ioutil.NopCloser(strings.NewReader("")))
So(err, ShouldNotBeNil)
})
Convey("Test PutBlobChunk3", t, func(c C) {
il = createMockStorage(testDir, &StorageDriverMock{
writerFn: func(ctx context.Context, path string, append bool) (storageDriver.FileWriter, error) {
imgStore = createMockStorage(testDir, &StorageDriverMock{
writerFn: func(ctx context.Context, path string, isAppend bool) (driver.FileWriter, error) {
return &FileWriterMock{
writeFn: func(b []byte) (int, error) {
return 0, errS3
@@ -612,13 +608,13 @@ func TestNegativeCasesObjectsStorage(t *testing.T) {
}, nil
},
})
_, err := il.PutBlobChunk(testImage, "uuid", 12, 100, ioutil.NopCloser(strings.NewReader("")))
_, err := imgStore.PutBlobChunk(testImage, "uuid", 12, 100, ioutil.NopCloser(strings.NewReader("")))
So(err, ShouldNotBeNil)
})
Convey("Test FinishBlobUpload", t, func(c C) {
il = createMockStorage(testDir, &StorageDriverMock{
writerFn: func(ctx context.Context, path string, append bool) (storageDriver.FileWriter, error) {
imgStore = createMockStorage(testDir, &StorageDriverMock{
writerFn: func(ctx context.Context, path string, isAppend bool) (driver.FileWriter, error) {
return &FileWriterMock{
commitFn: func() error {
return errS3
@@ -627,13 +623,13 @@ func TestNegativeCasesObjectsStorage(t *testing.T) {
},
})
d := godigest.FromBytes([]byte("test"))
err := il.FinishBlobUpload(testImage, "uuid", ioutil.NopCloser(strings.NewReader("")), d.String())
err := imgStore.FinishBlobUpload(testImage, "uuid", ioutil.NopCloser(strings.NewReader("")), d.String())
So(err, ShouldNotBeNil)
})
Convey("Test FinishBlobUpload2", t, func(c C) {
il = createMockStorage(testDir, &StorageDriverMock{
writerFn: func(ctx context.Context, path string, append bool) (storageDriver.FileWriter, error) {
imgStore = createMockStorage(testDir, &StorageDriverMock{
writerFn: func(ctx context.Context, path string, isAppend bool) (driver.FileWriter, error) {
return &FileWriterMock{
closeFn: func() error {
return errS3
@@ -642,91 +638,91 @@ func TestNegativeCasesObjectsStorage(t *testing.T) {
},
})
d := godigest.FromBytes([]byte("test"))
err := il.FinishBlobUpload(testImage, "uuid", ioutil.NopCloser(strings.NewReader("")), d.String())
err := imgStore.FinishBlobUpload(testImage, "uuid", ioutil.NopCloser(strings.NewReader("")), d.String())
So(err, ShouldNotBeNil)
})
Convey("Test FinishBlobUpload3", t, func(c C) {
il = createMockStorage(testDir, &StorageDriverMock{
imgStore = createMockStorage(testDir, &StorageDriverMock{
readerFn: func(ctx context.Context, path string, offset int64) (io.ReadCloser, error) {
return nil, errS3
},
})
d := godigest.FromBytes([]byte("test"))
err := il.FinishBlobUpload(testImage, "uuid", ioutil.NopCloser(strings.NewReader("")), d.String())
err := imgStore.FinishBlobUpload(testImage, "uuid", ioutil.NopCloser(strings.NewReader("")), d.String())
So(err, ShouldNotBeNil)
})
Convey("Test FinishBlobUpload4", t, func(c C) {
il = createMockStorage(testDir, &StorageDriverMock{
imgStore = createMockStorage(testDir, &StorageDriverMock{
moveFn: func(ctx context.Context, sourcePath, destPath string) error {
return errS3
},
})
d := godigest.FromBytes([]byte(""))
err := il.FinishBlobUpload(testImage, "uuid", ioutil.NopCloser(strings.NewReader("")), d.String())
err := imgStore.FinishBlobUpload(testImage, "uuid", ioutil.NopCloser(strings.NewReader("")), d.String())
So(err, ShouldNotBeNil)
})
Convey("Test FullBlobUpload", t, func(c C) {
il = createMockStorage(testDir, &StorageDriverMock{
writerFn: func(ctx context.Context, path string, append bool) (storageDriver.FileWriter, error) {
imgStore = createMockStorage(testDir, &StorageDriverMock{
writerFn: func(ctx context.Context, path string, isAppend bool) (driver.FileWriter, error) {
return &FileWriterMock{}, errS3
},
})
d := godigest.FromBytes([]byte(""))
_, _, err := il.FullBlobUpload(testImage, ioutil.NopCloser(strings.NewReader("")), d.String())
_, _, err := imgStore.FullBlobUpload(testImage, ioutil.NopCloser(strings.NewReader("")), d.String())
So(err, ShouldNotBeNil)
})
Convey("Test FullBlobUpload2", t, func(c C) {
il = createMockStorage(testDir, &StorageDriverMock{})
imgStore = createMockStorage(testDir, &StorageDriverMock{})
d := godigest.FromBytes([]byte(" "))
_, _, err := il.FullBlobUpload(testImage, ioutil.NopCloser(strings.NewReader("")), d.String())
_, _, err := imgStore.FullBlobUpload(testImage, ioutil.NopCloser(strings.NewReader("")), d.String())
So(err, ShouldNotBeNil)
})
Convey("Test FullBlobUpload3", t, func(c C) {
il = createMockStorage(testDir, &StorageDriverMock{
imgStore = createMockStorage(testDir, &StorageDriverMock{
moveFn: func(ctx context.Context, sourcePath, destPath string) error {
return errS3
},
})
d := godigest.FromBytes([]byte(""))
_, _, err := il.FullBlobUpload(testImage, ioutil.NopCloser(strings.NewReader("")), d.String())
_, _, err := imgStore.FullBlobUpload(testImage, ioutil.NopCloser(strings.NewReader("")), d.String())
So(err, ShouldNotBeNil)
})
Convey("Test GetBlob", t, func(c C) {
il = createMockStorage(testDir, &StorageDriverMock{
imgStore = createMockStorage(testDir, &StorageDriverMock{
readerFn: func(ctx context.Context, path string, offset int64) (io.ReadCloser, error) {
return ioutil.NopCloser(strings.NewReader("")), errS3
},
})
d := godigest.FromBytes([]byte(""))
_, _, err := il.GetBlob(testImage, d.String(), "")
_, _, err := imgStore.GetBlob(testImage, d.String(), "")
So(err, ShouldNotBeNil)
})
Convey("Test DeleteBlob", t, func(c C) {
il = createMockStorage(testDir, &StorageDriverMock{
imgStore = createMockStorage(testDir, &StorageDriverMock{
deleteFn: func(ctx context.Context, path string) error {
return errS3
},
})
d := godigest.FromBytes([]byte(""))
err := il.DeleteBlob(testImage, d.String())
err := imgStore.DeleteBlob(testImage, d.String())
So(err, ShouldNotBeNil)
})
Convey("Test GetReferrers", t, func(c C) {
il = createMockStorage(testDir, &StorageDriverMock{
imgStore = createMockStorage(testDir, &StorageDriverMock{
deleteFn: func(ctx context.Context, path string) error {
return errS3
},
})
d := godigest.FromBytes([]byte(""))
_, err := il.GetReferrers(testImage, d.String(), "application/image")
_, err := imgStore.GetReferrers(testImage, d.String(), "application/image")
So(err, ShouldNotBeNil)
So(err, ShouldEqual, zerr.ErrMethodNotSupported)
})
+184 -125
View File
@@ -5,6 +5,7 @@ import (
"context"
"crypto/sha256"
"encoding/json"
"errors"
"fmt"
"io"
"path"
@@ -12,25 +13,26 @@ import (
"strings"
"sync"
// Add s3 support.
"github.com/docker/distribution/registry/storage/driver"
// Load s3 driver.
_ "github.com/docker/distribution/registry/storage/driver/s3-aws"
guuid "github.com/gofrs/uuid"
"github.com/notaryproject/notation-go-lib"
godigest "github.com/opencontainers/go-digest"
ispec "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/rs/zerolog"
"zotregistry.io/zot/errors"
zerr "zotregistry.io/zot/errors"
"zotregistry.io/zot/pkg/extensions/monitoring"
zlog "zotregistry.io/zot/pkg/log"
"zotregistry.io/zot/pkg/storage"
// Add s3 support
storageDriver "github.com/docker/distribution/registry/storage/driver"
_ "github.com/docker/distribution/registry/storage/driver/s3-aws" // Load s3 driver
)
// ObjectStorage provides the image storage operations.
type ObjectStorage struct {
rootDir string
store storageDriver.StorageDriver
store driver.StorageDriver
lock *sync.RWMutex
blobUploads map[string]storage.BlobUpload
log zerolog.Logger
@@ -55,19 +57,19 @@ func (is *ObjectStorage) DirExists(d string) bool {
// NewObjectStorage returns a new image store backed by cloud storages.
// see https://github.com/docker/docker.github.io/tree/master/registry/storage-drivers
func NewImageStore(rootDir string, gc bool, dedupe bool, log zlog.Logger, m monitoring.MetricServer,
store storageDriver.StorageDriver) storage.ImageStore {
is := &ObjectStorage{
func NewImageStore(rootDir string, gc bool, dedupe bool, log zlog.Logger, metrics monitoring.MetricServer,
store driver.StorageDriver) storage.ImageStore {
imgStore := &ObjectStorage{
rootDir: rootDir,
store: store,
lock: &sync.RWMutex{},
blobUploads: make(map[string]storage.BlobUpload),
log: log.With().Caller().Logger(),
isMultiPartUpload: make(map[string]bool),
metrics: m,
metrics: metrics,
}
return is
return imgStore
}
// RLock read-lock.
@@ -101,15 +103,17 @@ func (is *ObjectStorage) initRepo(name string) error {
ilPath := path.Join(repoDir, ispec.ImageLayoutFile)
if _, err := is.store.Stat(context.Background(), ilPath); err != nil {
il := ispec.ImageLayout{Version: ispec.ImageLayoutVersion}
buf, err := json.Marshal(il)
buf, err := json.Marshal(il)
if err != nil {
is.log.Error().Err(err).Msg("unable to marshal JSON")
return err
}
if _, err := writeFile(is.store, ilPath, buf); err != nil {
is.log.Error().Err(err).Str("file", ilPath).Msg("unable to write file")
return err
}
}
@@ -119,15 +123,17 @@ func (is *ObjectStorage) initRepo(name string) error {
if _, err := is.store.Stat(context.Background(), indexPath); err != nil {
index := ispec.Index{}
index.SchemaVersion = 2
buf, err := json.Marshal(index)
buf, err := json.Marshal(index)
if err != nil {
is.log.Error().Err(err).Msg("unable to marshal JSON")
return err
}
if _, err := writeFile(is.store, indexPath, buf); err != nil {
is.log.Error().Err(err).Str("file", ilPath).Msg("unable to write file")
return err
}
}
@@ -151,18 +157,19 @@ func (is *ObjectStorage) ValidateRepo(name string) (bool, error) {
// for objects storage we can not create empty dirs, so we check only against index.json and oci-layout
dir := path.Join(is.rootDir, name)
if fi, err := is.store.Stat(context.Background(), dir); err != nil || !fi.IsDir() {
return false, errors.ErrRepoNotFound
return false, zerr.ErrRepoNotFound
}
files, err := is.store.List(context.Background(), dir)
if err != nil {
is.log.Error().Err(err).Str("dir", dir).Msg("unable to read directory")
return false, errors.ErrRepoNotFound
return false, zerr.ErrRepoNotFound
}
// nolint:gomnd
if len(files) < 2 {
return false, errors.ErrRepoBadVersion
return false, zerr.ErrRepoBadVersion
}
found := map[string]bool{
@@ -205,7 +212,7 @@ func (is *ObjectStorage) ValidateRepo(name string) (bool, error) {
}
if il.Version != ispec.ImageLayoutVersion {
return false, errors.ErrRepoBadVersion
return false, zerr.ErrRepoBadVersion
}
return true, nil
@@ -219,18 +226,18 @@ func (is *ObjectStorage) GetRepositories() ([]string, error) {
defer is.RUnlock()
stores := make([]string, 0)
err := is.store.Walk(context.Background(), dir, func(fileInfo storageDriver.FileInfo) error {
err := is.store.Walk(context.Background(), dir, func(fileInfo driver.FileInfo) error {
if !fileInfo.IsDir() {
return nil
}
rel, err := filepath.Rel(is.rootDir, fileInfo.Path())
if err != nil {
return nil
return nil //nolint:nilerr // ignore paths that are not under root dir
}
if ok, err := is.ValidateRepo(rel); !ok || err != nil {
return nil
return nil //nolint:nilerr // ignore invalid repos
}
stores = append(stores, rel)
@@ -239,8 +246,8 @@ func (is *ObjectStorage) GetRepositories() ([]string, error) {
})
// if the root directory is not yet created then return an empty slice of repositories
_, ok := err.(storageDriver.PathNotFoundError)
if ok {
var perr driver.PathNotFoundError
if errors.As(err, &perr) {
return stores, nil
}
@@ -251,7 +258,7 @@ func (is *ObjectStorage) GetRepositories() ([]string, error) {
func (is *ObjectStorage) GetImageTags(repo string) ([]string, error) {
dir := path.Join(is.rootDir, repo)
if fi, err := is.store.Stat(context.Background(), dir); err != nil || !fi.IsDir() {
return nil, errors.ErrRepoNotFound
return nil, zerr.ErrRepoNotFound
}
is.RLock()
@@ -265,7 +272,8 @@ func (is *ObjectStorage) GetImageTags(repo string) ([]string, error) {
var index ispec.Index
if err := json.Unmarshal(buf, &index); err != nil {
is.log.Error().Err(err).Str("dir", dir).Msg("invalid JSON")
return nil, errors.ErrRepoNotFound
return nil, zerr.ErrRepoNotFound
}
tags := make([]string, 0)
@@ -284,7 +292,7 @@ func (is *ObjectStorage) GetImageTags(repo string) ([]string, error) {
func (is *ObjectStorage) GetImageManifest(repo string, reference string) ([]byte, string, string, error) {
dir := path.Join(is.rootDir, repo)
if fi, err := is.store.Stat(context.Background(), dir); err != nil || !fi.IsDir() {
return nil, "", "", errors.ErrRepoNotFound
return nil, "", "", zerr.ErrRepoNotFound
}
is.RLock()
@@ -298,6 +306,7 @@ func (is *ObjectStorage) GetImageManifest(repo string, reference string) ([]byte
var index ispec.Index
if err := json.Unmarshal(buf, &index); err != nil {
is.log.Error().Err(err).Str("dir", dir).Msg("invalid JSON")
return nil, "", "", err
}
@@ -307,19 +316,19 @@ func (is *ObjectStorage) GetImageManifest(repo string, reference string) ([]byte
mediaType := ""
for _, m := range index.Manifests {
if reference == m.Digest.String() {
digest = m.Digest
mediaType = m.MediaType
for _, manifest := range index.Manifests {
if reference == manifest.Digest.String() {
digest = manifest.Digest
mediaType = manifest.MediaType
found = true
break
}
v, ok := m.Annotations[ispec.AnnotationRefName]
v, ok := manifest.Annotations[ispec.AnnotationRefName]
if ok && v == reference {
digest = m.Digest
mediaType = m.MediaType
digest = manifest.Digest
mediaType = manifest.MediaType
found = true
break
@@ -327,7 +336,7 @@ func (is *ObjectStorage) GetImageManifest(repo string, reference string) ([]byte
}
if !found {
return nil, "", "", errors.ErrManifestNotFound
return nil, "", "", zerr.ErrManifestNotFound
}
p := path.Join(dir, "blobs", digest.Algorithm().String(), digest.Encoded())
@@ -335,12 +344,14 @@ func (is *ObjectStorage) GetImageManifest(repo string, reference string) ([]byte
buf, err = is.store.GetContent(context.Background(), p)
if err != nil {
is.log.Error().Err(err).Str("blob", p).Msg("failed to read manifest")
return nil, "", "", err
}
var manifest ispec.Manifest
if err := json.Unmarshal(buf, &manifest); err != nil {
is.log.Error().Err(err).Str("dir", dir).Msg("invalid JSON")
return nil, "", "", err
}
@@ -354,29 +365,34 @@ func (is *ObjectStorage) PutImageManifest(repo string, reference string, mediaTy
body []byte) (string, error) {
if err := is.InitRepo(repo); err != nil {
is.log.Debug().Err(err).Msg("init repo")
return "", err
}
if mediaType != ispec.MediaTypeImageManifest {
is.log.Debug().Interface("actual", mediaType).
Interface("expected", ispec.MediaTypeImageManifest).Msg("bad manifest media type")
return "", errors.ErrBadManifest
return "", zerr.ErrBadManifest
}
if len(body) == 0 {
is.log.Debug().Int("len", len(body)).Msg("invalid body length")
return "", errors.ErrBadManifest
return "", zerr.ErrBadManifest
}
var m ispec.Manifest
if err := json.Unmarshal(body, &m); err != nil {
is.log.Error().Err(err).Msg("unable to unmarshal JSON")
return "", errors.ErrBadManifest
return "", zerr.ErrBadManifest
}
if m.SchemaVersion != storage.SchemaVersion {
is.log.Error().Int("SchemaVersion", m.SchemaVersion).Msg("invalid manifest")
return "", errors.ErrBadManifest
return "", zerr.ErrBadManifest
}
for _, l := range m.Layers {
@@ -386,19 +402,21 @@ func (is *ObjectStorage) PutImageManifest(repo string, reference string, mediaTy
if _, err := is.store.Stat(context.Background(), blobPath); err != nil {
is.log.Error().Err(err).Str("blobPath", blobPath).Msg("unable to find blob")
return digest.String(), errors.ErrBlobNotFound
return digest.String(), zerr.ErrBlobNotFound
}
}
mDigest := godigest.FromBytes(body)
refIsDigest := false
d, err := godigest.Parse(reference)
dgst, err := godigest.Parse(reference)
if err == nil {
if d.String() != mDigest.String() {
is.log.Error().Str("actual", mDigest.String()).Str("expected", d.String()).
if dgst.String() != mDigest.String() {
is.log.Error().Str("actual", mDigest.String()).Str("expected", dgst.String()).
Msg("manifest digest is not valid")
return "", errors.ErrBadManifest
return "", zerr.ErrBadManifest
}
refIsDigest = true
@@ -417,31 +435,34 @@ func (is *ObjectStorage) PutImageManifest(repo string, reference string, mediaTy
var index ispec.Index
if err := json.Unmarshal(buf, &index); err != nil {
is.log.Error().Err(err).Str("dir", dir).Msg("invalid JSON")
return "", errors.ErrRepoBadVersion
return "", zerr.ErrRepoBadVersion
}
updateIndex := true
// create a new descriptor
desc := ispec.Descriptor{MediaType: mediaType, Size: int64(len(body)), Digest: mDigest,
Platform: &ispec.Platform{Architecture: "amd64", OS: "linux"}}
desc := ispec.Descriptor{
MediaType: mediaType, Size: int64(len(body)), Digest: mDigest,
Platform: &ispec.Platform{Architecture: "amd64", OS: "linux"},
}
if !refIsDigest {
desc.Annotations = map[string]string{ispec.AnnotationRefName: reference}
}
for i, m := range index.Manifests {
if reference == m.Digest.String() {
for midx, manifest := range index.Manifests {
if reference == manifest.Digest.String() {
// nothing changed, so don't update
desc = m
desc = manifest
updateIndex = false
break
}
v, ok := m.Annotations[ispec.AnnotationRefName]
v, ok := manifest.Annotations[ispec.AnnotationRefName]
if ok && v == reference {
if m.Digest.String() == mDigest.String() {
if manifest.Digest.String() == mDigest.String() {
// nothing changed, so don't update
desc = m
desc = manifest
updateIndex = false
break
@@ -455,11 +476,11 @@ func (is *ObjectStorage) PutImageManifest(repo string, reference string, mediaTy
Str("new digest", mDigest.String()).
Msg("updating existing tag with new manifest contents")
desc = m
desc = manifest
desc.Size = int64(len(body))
desc.Digest = mDigest
index.Manifests = append(index.Manifests[:i], index.Manifests[i+1:]...)
index.Manifests = append(index.Manifests[:midx], index.Manifests[midx+1:]...)
break
}
@@ -475,6 +496,7 @@ func (is *ObjectStorage) PutImageManifest(repo string, reference string, mediaTy
if err = is.store.PutContent(context.Background(), manifestPath, body); err != nil {
is.log.Error().Err(err).Str("file", manifestPath).Msg("unable to write")
return "", err
}
@@ -486,11 +508,13 @@ func (is *ObjectStorage) PutImageManifest(repo string, reference string, mediaTy
if err != nil {
is.log.Error().Err(err).Str("file", indexPath).Msg("unable to marshal JSON")
return "", err
}
if err = is.store.PutContent(context.Background(), indexPath, buf); err != nil {
is.log.Error().Err(err).Str("file", manifestPath).Msg("unable to write")
return "", err
}
@@ -504,13 +528,13 @@ func (is *ObjectStorage) PutImageManifest(repo string, reference string, mediaTy
func (is *ObjectStorage) DeleteImageManifest(repo string, reference string) error {
dir := path.Join(is.rootDir, repo)
if fi, err := is.store.Stat(context.Background(), dir); err != nil || !fi.IsDir() {
return errors.ErrRepoNotFound
return zerr.ErrRepoNotFound
}
isTag := false
// as per spec "reference" can only be a digest and not a tag
digest, err := godigest.Parse(reference)
dgst, err := godigest.Parse(reference)
if err != nil {
is.log.Debug().Str("invalid digest: ", reference).Msg("storage: assuming tag")
@@ -528,40 +552,42 @@ func (is *ObjectStorage) DeleteImageManifest(repo string, reference string) erro
var index ispec.Index
if err := json.Unmarshal(buf, &index); err != nil {
is.log.Error().Err(err).Str("dir", dir).Msg("invalid JSON")
return err
}
found := false
var m ispec.Descriptor
var manifest ispec.Descriptor
// we are deleting, so keep only those manifests that don't match
outIndex := index
outIndex.Manifests = []ispec.Descriptor{}
for _, m = range index.Manifests {
for _, manifest = range index.Manifests {
if isTag {
tag, ok := m.Annotations[ispec.AnnotationRefName]
tag, ok := manifest.Annotations[ispec.AnnotationRefName]
if ok && tag == reference {
is.log.Debug().Str("deleting tag", tag).Msg("")
digest = m.Digest
dgst = manifest.Digest
found = true
continue
}
} else if reference == m.Digest.String() {
} else if reference == manifest.Digest.String() {
is.log.Debug().Str("deleting reference", reference).Msg("")
found = true
continue
}
outIndex.Manifests = append(outIndex.Manifests, m)
outIndex.Manifests = append(outIndex.Manifests, manifest)
}
if !found {
return errors.ErrManifestNotFound
return zerr.ErrManifestNotFound
}
// now update "index.json"
@@ -575,6 +601,7 @@ func (is *ObjectStorage) DeleteImageManifest(repo string, reference string) erro
if _, err := writeFile(is.store, file, buf); err != nil {
is.log.Debug().Str("deleting reference", reference).Msg("")
return err
}
@@ -582,15 +609,16 @@ func (is *ObjectStorage) DeleteImageManifest(repo string, reference string) erro
// e.g. 1.0.1 & 1.0.2 have same blob digest so if we delete 1.0.1, blob should not be removed.
toDelete := true
for _, m = range outIndex.Manifests {
if digest.String() == m.Digest.String() {
for _, manifest = range outIndex.Manifests {
if dgst.String() == manifest.Digest.String() {
toDelete = false
break
}
}
if toDelete {
p := path.Join(dir, "blobs", digest.Algorithm().String(), digest.Encoded())
p := path.Join(dir, "blobs", dgst.Algorithm().String(), dgst.Encoded())
err = is.store.Delete(context.Background(), p)
if err != nil {
@@ -624,18 +652,18 @@ func (is *ObjectStorage) NewBlobUpload(repo string) (string, error) {
return "", err
}
u := uuid.String()
uid := uuid.String()
blobUploadPath := is.BlobUploadPath(repo, u)
blobUploadPath := is.BlobUploadPath(repo, uid)
// here we should create an empty multi part upload, but that's not possible
// so we just create a regular empty file which will be overwritten by FinishBlobUpload
err = is.store.PutContent(context.Background(), blobUploadPath, []byte{})
if err != nil {
return "", errors.ErrRepoNotFound
return "", zerr.ErrRepoNotFound
}
return u, nil
return uid, nil
}
// GetBlobUpload returns the current size of a blob upload.
@@ -648,17 +676,17 @@ func (is *ObjectStorage) GetBlobUpload(repo string, uuid string) (int64, error)
// created by NewBlobUpload, it should have 0 size every time
isMultiPartStarted, ok := is.isMultiPartUpload[blobUploadPath]
if !isMultiPartStarted || !ok {
fi, err := is.store.Stat(context.Background(), blobUploadPath)
binfo, err := is.store.Stat(context.Background(), blobUploadPath)
if err != nil {
_, ok := err.(storageDriver.PathNotFoundError)
if ok {
return -1, errors.ErrUploadNotFound
var perr driver.PathNotFoundError
if errors.As(err, &perr) {
return -1, zerr.ErrUploadNotFound
}
return -1, err
}
fileSize = fi.Size()
fileSize = binfo.Size()
} else {
// otherwise get the size of multi parts upload
fi, err := getMultipartFileWriter(is, blobUploadPath)
@@ -683,12 +711,13 @@ func (is *ObjectStorage) PutBlobChunkStreamed(repo string, uuid string, body io.
_, err := is.store.Stat(context.Background(), blobUploadPath)
if err != nil {
return -1, errors.ErrUploadNotFound
return -1, zerr.ErrUploadNotFound
}
file, err := getMultipartFileWriter(is, blobUploadPath)
if err != nil {
is.log.Error().Err(err).Msg("failed to create multipart upload")
return -1, err
}
@@ -699,16 +728,18 @@ func (is *ObjectStorage) PutBlobChunkStreamed(repo string, uuid string, body io.
_, err = buf.ReadFrom(body)
if err != nil {
is.log.Error().Err(err).Msg("failed to read blob")
return -1, err
}
n, err := file.Write(buf.Bytes())
nbytes, err := file.Write(buf.Bytes())
if err != nil {
is.log.Error().Err(err).Msg("failed to append to file")
return -1, err
}
return int64(n), err
return int64(nbytes), err
}
// PutBlobChunk writes another chunk of data to the specified blob. It returns
@@ -723,12 +754,13 @@ func (is *ObjectStorage) PutBlobChunk(repo string, uuid string, from int64, to i
_, err := is.store.Stat(context.Background(), blobUploadPath)
if err != nil {
return -1, errors.ErrUploadNotFound
return -1, zerr.ErrUploadNotFound
}
file, err := getMultipartFileWriter(is, blobUploadPath)
if err != nil {
is.log.Error().Err(err).Msg("failed to create multipart upload")
return -1, err
}
@@ -739,13 +771,14 @@ func (is *ObjectStorage) PutBlobChunk(repo string, uuid string, from int64, to i
err := file.Cancel()
if err != nil {
is.log.Error().Err(err).Msg("failed to cancel multipart upload")
return -1, err
}
is.log.Error().Int64("expected", from).Int64("actual", file.Size()).
Msg("invalid range start for blob upload")
return -1, errors.ErrBadUploadRange
return -1, zerr.ErrBadUploadRange
}
buf := new(bytes.Buffer)
@@ -753,18 +786,20 @@ func (is *ObjectStorage) PutBlobChunk(repo string, uuid string, from int64, to i
_, err = buf.ReadFrom(body)
if err != nil {
is.log.Error().Err(err).Msg("failed to read blob")
return -1, err
}
n, err := file.Write(buf.Bytes())
nbytes, err := file.Write(buf.Bytes())
if err != nil {
is.log.Error().Err(err).Msg("failed to append to file")
return -1, err
}
is.isMultiPartUpload[blobUploadPath] = true
return int64(n), err
return int64(nbytes), err
}
// BlobUploadInfo returns the current blob size in bytes.
@@ -777,22 +812,24 @@ func (is *ObjectStorage) BlobUploadInfo(repo string, uuid string) (int64, error)
// created by NewBlobUpload, it should have 0 size every time
isMultiPartStarted, ok := is.isMultiPartUpload[blobUploadPath]
if !isMultiPartStarted || !ok {
fi, err := is.store.Stat(context.Background(), blobUploadPath)
uploadInfo, err := is.store.Stat(context.Background(), blobUploadPath)
if err != nil {
is.log.Error().Err(err).Str("blob", blobUploadPath).Msg("failed to stat blob")
return -1, err
}
fileSize = fi.Size()
fileSize = uploadInfo.Size()
} else {
// otherwise get the size of multi parts upload
fi, err := getMultipartFileWriter(is, blobUploadPath)
binfo, err := getMultipartFileWriter(is, blobUploadPath)
if err != nil {
is.log.Error().Err(err).Str("blob", blobUploadPath).Msg("failed to stat blob")
return -1, err
}
fileSize = fi.Size()
fileSize = binfo.Size()
}
return fileSize, nil
@@ -803,7 +840,8 @@ func (is *ObjectStorage) FinishBlobUpload(repo string, uuid string, body io.Read
dstDigest, err := godigest.Parse(digest)
if err != nil {
is.log.Error().Err(err).Str("digest", digest).Msg("failed to parse digest")
return errors.ErrBadBlobDigest
return zerr.ErrBadBlobDigest
}
src := is.BlobUploadPath(repo, uuid)
@@ -812,11 +850,13 @@ func (is *ObjectStorage) FinishBlobUpload(repo string, uuid string, body io.Read
fileWriter, err := is.store.Writer(context.Background(), src, true)
if err != nil {
is.log.Error().Err(err).Str("blob", src).Msg("failed to open blob")
return errors.ErrBadBlobDigest
return zerr.ErrBadBlobDigest
}
if err := fileWriter.Commit(); err != nil {
is.log.Error().Err(err).Msg("failed to commit file")
return err
}
@@ -827,19 +867,22 @@ func (is *ObjectStorage) FinishBlobUpload(repo string, uuid string, body io.Read
fileReader, err := is.store.Reader(context.Background(), src, 0)
if err != nil {
is.log.Error().Err(err).Str("blob", src).Msg("failed to open file")
return errors.ErrUploadNotFound
return zerr.ErrUploadNotFound
}
srcDigest, err := godigest.FromReader(fileReader)
if err != nil {
is.log.Error().Err(err).Str("blob", src).Msg("failed to open blob")
return errors.ErrBadBlobDigest
return zerr.ErrBadBlobDigest
}
if srcDigest != dstDigest {
is.log.Error().Str("srcDigest", srcDigest.String()).
Str("dstDigest", dstDigest.String()).Msg("actual digest not equal to expected digest")
return errors.ErrBadBlobDigest
return zerr.ErrBadBlobDigest
}
fileReader.Close()
@@ -849,6 +892,7 @@ func (is *ObjectStorage) FinishBlobUpload(repo string, uuid string, body io.Read
if err := is.store.Move(context.Background(), src, dst); err != nil {
is.log.Error().Err(err).Str("src", src).Str("dstDigest", dstDigest.String()).
Str("dst", dst).Msg("unable to finish blob")
return err
}
@@ -867,7 +911,8 @@ func (is *ObjectStorage) FullBlobUpload(repo string, body io.Reader, digest stri
dstDigest, err := godigest.Parse(digest)
if err != nil {
is.log.Error().Err(err).Str("digest", digest).Msg("failed to parse digest")
return "", -1, errors.ErrBadBlobDigest
return "", -1, zerr.ErrBadBlobDigest
}
u, err := guuid.NewV4()
@@ -886,18 +931,21 @@ func (is *ObjectStorage) FullBlobUpload(repo string, body io.Reader, digest stri
_, err = buf.ReadFrom(body)
if err != nil {
is.log.Error().Err(err).Msg("failed to read blob")
return "", -1, err
}
n, err := writeFile(is.store, src, buf.Bytes())
nbytes, err := writeFile(is.store, src, buf.Bytes())
if err != nil {
is.log.Error().Err(err).Msg("failed to write blob")
return "", -1, err
}
_, err = digester.Write(buf.Bytes())
if err != nil {
is.log.Error().Err(err).Msg("digester failed to write")
return "", -1, err
}
@@ -905,7 +953,8 @@ func (is *ObjectStorage) FullBlobUpload(repo string, body io.Reader, digest stri
if srcDigest != dstDigest {
is.log.Error().Str("srcDigest", srcDigest.String()).
Str("dstDigest", dstDigest.String()).Msg("actual digest not equal to expected digest")
return "", -1, errors.ErrBadBlobDigest
return "", -1, zerr.ErrBadBlobDigest
}
is.Lock()
@@ -916,10 +965,11 @@ func (is *ObjectStorage) FullBlobUpload(repo string, body io.Reader, digest stri
if err := is.store.Move(context.Background(), src, dst); err != nil {
is.log.Error().Err(err).Str("src", src).Str("dstDigest", dstDigest.String()).
Str("dst", dst).Msg("unable to finish blob")
return "", -1, err
}
return uuid, int64(n), nil
return uuid, int64(nbytes), nil
}
func (is *ObjectStorage) DedupeBlob(src string, dstDigest godigest.Digest, dst string) error {
@@ -931,6 +981,7 @@ func (is *ObjectStorage) DeleteBlobUpload(repo string, uuid string) error {
blobUploadPath := is.BlobUploadPath(repo, uuid)
if err := is.store.Delete(context.Background(), blobUploadPath); err != nil {
is.log.Error().Err(err).Str("blobUploadPath", blobUploadPath).Msg("error deleting blob upload")
return err
}
@@ -944,22 +995,23 @@ func (is *ObjectStorage) BlobPath(repo string, digest godigest.Digest) string {
// CheckBlob verifies a blob and returns true if the blob is correct.
func (is *ObjectStorage) CheckBlob(repo string, digest string) (bool, int64, error) {
d, err := godigest.Parse(digest)
dgst, err := godigest.Parse(digest)
if err != nil {
is.log.Error().Err(err).Str("digest", digest).Msg("failed to parse digest")
return false, -1, errors.ErrBadBlobDigest
return false, -1, zerr.ErrBadBlobDigest
}
blobPath := is.BlobPath(repo, d)
blobPath := is.BlobPath(repo, dgst)
is.RLock()
defer is.RUnlock()
blobInfo, err := is.store.Stat(context.Background(), blobPath)
binfo, err := is.store.Stat(context.Background(), blobPath)
if err != nil {
_, ok := err.(storageDriver.PathNotFoundError)
if ok {
return false, -1, errors.ErrBlobNotFound
var perr driver.PathNotFoundError
if errors.As(err, &perr) {
return false, -1, zerr.ErrBlobNotFound
}
is.log.Error().Err(err).Str("blob", blobPath).Msg("failed to stat blob")
@@ -969,37 +1021,39 @@ func (is *ObjectStorage) CheckBlob(repo string, digest string) (bool, int64, err
is.log.Debug().Str("blob path", blobPath).Msg("blob path found")
return true, blobInfo.Size(), nil
return true, binfo.Size(), nil
}
// GetBlob returns a stream to read the blob.
// FIXME: we should probably parse the manifest and use (digest, mediaType) as a
// blob selector instead of directly downloading the blob.
func (is *ObjectStorage) GetBlob(repo string, digest string, mediaType string) (io.Reader, int64, error) {
d, err := godigest.Parse(digest)
dgst, err := godigest.Parse(digest)
if err != nil {
is.log.Error().Err(err).Str("digest", digest).Msg("failed to parse digest")
return nil, -1, errors.ErrBadBlobDigest
return nil, -1, zerr.ErrBadBlobDigest
}
blobPath := is.BlobPath(repo, d)
blobPath := is.BlobPath(repo, dgst)
is.RLock()
defer is.RUnlock()
blobInfo, err := is.store.Stat(context.Background(), blobPath)
binfo, err := is.store.Stat(context.Background(), blobPath)
if err != nil {
is.log.Error().Err(err).Str("blob", blobPath).Msg("failed to stat blob")
return nil, -1, errors.ErrBlobNotFound
return nil, -1, zerr.ErrBlobNotFound
}
blobReader, err := is.store.Reader(context.Background(), blobPath, 0)
if err != nil {
is.log.Error().Err(err).Str("blob", blobPath).Msg("failed to open blob")
return nil, -1, err
}
return blobReader, blobInfo.Size(), nil
return blobReader, binfo.Size(), nil
}
func (is *ObjectStorage) GetBlobContent(repo string, digest string) ([]byte, error) {
@@ -1013,6 +1067,7 @@ func (is *ObjectStorage) GetBlobContent(repo string, digest string) ([]byte, err
_, err = buf.ReadFrom(blob)
if err != nil {
is.log.Error().Err(err).Msg("failed to read blob")
return []byte{}, err
}
@@ -1020,7 +1075,7 @@ func (is *ObjectStorage) GetBlobContent(repo string, digest string) ([]byte, err
}
func (is *ObjectStorage) GetReferrers(repo, digest string, mediaType string) ([]notation.Descriptor, error) {
return nil, errors.ErrMethodNotSupported
return nil, zerr.ErrMethodNotSupported
}
func (is *ObjectStorage) GetIndexContent(repo string) ([]byte, error) {
@@ -1029,7 +1084,8 @@ func (is *ObjectStorage) GetIndexContent(repo string) ([]byte, error) {
buf, err := is.store.GetContent(context.Background(), path.Join(dir, "index.json"))
if err != nil {
is.log.Error().Err(err).Str("dir", dir).Msg("failed to read index.json")
return []byte{}, errors.ErrRepoNotFound
return []byte{}, zerr.ErrRepoNotFound
}
return buf, nil
@@ -1037,13 +1093,14 @@ func (is *ObjectStorage) GetIndexContent(repo string) ([]byte, error) {
// DeleteBlob removes the blob from the repository.
func (is *ObjectStorage) DeleteBlob(repo string, digest string) error {
d, err := godigest.Parse(digest)
dgst, err := godigest.Parse(digest)
if err != nil {
is.log.Error().Err(err).Str("digest", digest).Msg("failed to parse digest")
return errors.ErrBlobNotFound
return zerr.ErrBlobNotFound
}
blobPath := is.BlobPath(repo, d)
blobPath := is.BlobPath(repo, dgst)
is.Lock()
defer is.Unlock()
@@ -1051,11 +1108,13 @@ func (is *ObjectStorage) DeleteBlob(repo string, digest string) error {
_, err = is.store.Stat(context.Background(), blobPath)
if err != nil {
is.log.Error().Err(err).Str("blob", blobPath).Msg("failed to stat blob")
return errors.ErrBlobNotFound
return zerr.ErrBlobNotFound
}
if err := is.store.Delete(context.Background(), blobPath); err != nil {
is.log.Error().Err(err).Str("blobPath", blobPath).Msg("unable to remove blob path")
return err
}
@@ -1064,17 +1123,17 @@ func (is *ObjectStorage) DeleteBlob(repo string, digest string) error {
// Do not use for multipart upload, buf must not be empty.
// If you want to create an empty file use is.store.PutContent().
func writeFile(store storageDriver.StorageDriver, filepath string, buf []byte) (int, error) {
func writeFile(store driver.StorageDriver, filepath string, buf []byte) (int, error) {
var n int
if fw, err := store.Writer(context.Background(), filepath, false); err == nil {
defer fw.Close()
if stwr, err := store.Writer(context.Background(), filepath, false); err == nil {
defer stwr.Close()
if n, err = fw.Write(buf); err != nil {
if n, err = stwr.Write(buf); err != nil {
return -1, err
}
if err := fw.Commit(); err != nil {
if err := stwr.Commit(); err != nil {
return -1, err
}
} else {
@@ -1087,19 +1146,19 @@ func writeFile(store storageDriver.StorageDriver, filepath string, buf []byte) (
// Because we can not create an empty multipart upload, we store multi part uploads
// so that we know when to create a fileWriter with append=true or with append=false
// Trying and handling errors results in weird s3 api errors.
func getMultipartFileWriter(is *ObjectStorage, filepath string) (storageDriver.FileWriter, error) {
var file storageDriver.FileWriter
func getMultipartFileWriter(imgStore *ObjectStorage, filepath string) (driver.FileWriter, error) {
var file driver.FileWriter
var err error
isMultiPartStarted, ok := is.isMultiPartUpload[filepath]
isMultiPartStarted, ok := imgStore.isMultiPartUpload[filepath]
if !isMultiPartStarted || !ok {
file, err = is.store.Writer(context.Background(), filepath, false)
file, err = imgStore.store.Writer(context.Background(), filepath, false)
if err != nil {
return file, err
}
} else {
file, err = is.store.Writer(context.Background(), filepath, true)
file, err = imgStore.store.Writer(context.Background(), filepath, true)
if err != nil {
return file, err
}