github/zot
1
0
Fork 0
mirror of https://github.com/project-zot/zot.git synced 2026-06-16 20:38:08 +08:00
Code Issues Packages Projects Releases Wiki Activity

Fix file handlers not being closed after calls to ImageStore.GetBlob

Browse Source 
This is to fixes hitting the FD limit when reading blobs from the disk in the graphql API

Signed-off-by: Andrei Aaron <andaaron@cisco.com>
This commit is contained in:
Andrei Aaron
2022-08-19 10:38:59 +00:00
committed by Ramkumar Chinchani
parent 74630ed3a0
commit bd9ad998cd
10 changed files with 83 additions and 54 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/storage/local.go
+5 -3
View File
@@ -1351,7 +1351,7 @@ func (is *ImageStoreLocal) copyBlob(repo, blobPath, dstRecord string) (int64, er
// GetBlob returns a stream to read the blob.
// blob selector instead of directly downloading the blob.
func (is *ImageStoreLocal) GetBlob(repo, digest, mediaType string) (io.Reader, int64, error) {
func (is *ImageStoreLocal) GetBlob(repo, digest, mediaType string) (io.ReadCloser, int64, error) {
var lockLatency time.Time
parsedDigest, err := godigest.Parse(digest)
@@ -1373,14 +1373,15 @@ func (is *ImageStoreLocal) GetBlob(repo, digest, mediaType string) (io.Reader, i
return nil, -1, zerr.ErrBlobNotFound
}
blobReader, err := os.Open(blobPath)
blobReadCloser, err := os.Open(blobPath)
if err != nil {
is.log.Error().Err(err).Str("blob", blobPath).Msg("failed to open blob")
return nil, -1, err
}
return blobReader, binfo.Size(), nil
// The caller function is responsible for calling Close()
return blobReadCloser, binfo.Size(), nil
}
func (is *ImageStoreLocal) GetBlobContent(repo, digest string) ([]byte, error) {
@@ -1388,6 +1389,7 @@ func (is *ImageStoreLocal) GetBlobContent(repo, digest string) ([]byte, error) {
if err != nil {
return []byte{}, err
}
defer blob.Close()
buf := new(bytes.Buffer)
pkg/storage/local_test.go
+10 -3
View File
@@ -751,13 +751,16 @@ func FuzzGetBlob(f *testing.F) {
t.Error(err)
}
_, _, err = imgStore.GetBlob(repoName, digest.String(), "application/vnd.oci.image.layer.v1.tar+gzip")
blobReadCloser, _, err := imgStore.GetBlob(repoName, digest.String(), "application/vnd.oci.image.layer.v1.tar+gzip")
if err != nil {
if isKnownErr(err) {
return
}
t.Error(err)
}
if err = blobReadCloser.Close(); err != nil {
t.Error(err)
}
})
}
@@ -951,7 +954,9 @@ func TestDedupeLinks(t *testing.T) {
_, _, err = imgStore.CheckBlob("dedupe1", digest.String())
So(err, ShouldBeNil)
_, _, err = imgStore.GetBlob("dedupe1", digest.String(), "application/vnd.oci.image.layer.v1.tar+gzip")
blobrc, _, err := imgStore.GetBlob("dedupe1", digest.String(), "application/vnd.oci.image.layer.v1.tar+gzip")
So(err, ShouldBeNil)
err = blobrc.Close()
So(err, ShouldBeNil)
cblob, cdigest := test.GetRandomImageConfig()
@@ -1009,7 +1014,9 @@ func TestDedupeLinks(t *testing.T) {
_, _, err = imgStore.CheckBlob("dedupe2", digest.String())
So(err, ShouldBeNil)
_, _, err = imgStore.GetBlob("dedupe2", digest.String(), "application/vnd.oci.image.layer.v1.tar+gzip")
blobrc, _, err = imgStore.GetBlob("dedupe2", digest.String(), "application/vnd.oci.image.layer.v1.tar+gzip")
So(err, ShouldBeNil)
err = blobrc.Close()
So(err, ShouldBeNil)
cblob, cdigest = test.GetRandomImageConfig()
pkg/storage/s3/s3.go
+7 -5
View File
@@ -1253,7 +1253,7 @@ func (is *ObjectStorage) copyBlob(repo string, blobPath string, dstRecord string
// GetBlob returns a stream to read the blob.
// blob selector instead of directly downloading the blob.
func (is *ObjectStorage) GetBlob(repo, digest, mediaType string) (io.Reader, int64, error) {
func (is *ObjectStorage) GetBlob(repo, digest, mediaType string) (io.ReadCloser, int64, error) {
var lockLatency time.Time
dgst, err := godigest.Parse(digest)
@@ -1275,7 +1275,7 @@ func (is *ObjectStorage) GetBlob(repo, digest, mediaType string) (io.Reader, int
return nil, -1, zerr.ErrBlobNotFound
}
blobReader, err := is.store.Reader(context.Background(), blobPath, 0)
blobReadCloser, err := is.store.Reader(context.Background(), blobPath, 0)
if err != nil {
is.log.Error().Err(err).Str("blob", blobPath).Msg("failed to open blob")
@@ -1301,18 +1301,19 @@ func (is *ObjectStorage) GetBlob(repo, digest, mediaType string) (io.Reader, int
return nil, -1, zerr.ErrBlobNotFound
}
blobReader, err := is.store.Reader(context.Background(), dstRecord, 0)
blobReadCloser, err := is.store.Reader(context.Background(), dstRecord, 0)
if err != nil {
is.log.Error().Err(err).Str("blob", dstRecord).Msg("failed to open blob")
return nil, -1, err
}
return blobReader, binfo.Size(), nil
return blobReadCloser, binfo.Size(), nil
}
}
return blobReader, binfo.Size(), nil
// The caller function is responsible for calling Close()
return blobReadCloser, binfo.Size(), nil
}
func (is *ObjectStorage) GetBlobContent(repo, digest string) ([]byte, error) {
@@ -1320,6 +1321,7 @@ func (is *ObjectStorage) GetBlobContent(repo, digest string) ([]byte, error) {
if err != nil {
return []byte{}, err
}
defer blob.Close()
buf := new(bytes.Buffer)
pkg/storage/s3/s3_test.go
+12 -3
View File
@@ -866,9 +866,12 @@ func TestS3Dedupe(t *testing.T) {
So(checkBlobSize1, ShouldBeGreaterThan, 0)
So(err, ShouldBeNil)
_, getBlobSize1, err := imgStore.GetBlob("dedupe1", digest.String(), "application/vnd.oci.image.layer.v1.tar+gzip")
blobReadCloser, getBlobSize1, err := imgStore.GetBlob("dedupe1", digest.String(),
"application/vnd.oci.image.layer.v1.tar+gzip")
So(getBlobSize1, ShouldBeGreaterThan, 0)
So(err, ShouldBeNil)
err = blobReadCloser.Close()
So(err, ShouldBeNil)
cblob, cdigest := test.GetRandomImageConfig()
_, clen, err := imgStore.FullBlobUpload("dedupe1", bytes.NewReader(cblob), cdigest.String())
@@ -928,11 +931,14 @@ func TestS3Dedupe(t *testing.T) {
So(err, ShouldBeNil)
So(checkBlobSize2, ShouldBeGreaterThan, 0)
_, getBlobSize2, err := imgStore.GetBlob("dedupe2", digest.String(), "application/vnd.oci.image.layer.v1.tar+gzip")
blobReadCloser, getBlobSize2, err := imgStore.GetBlob("dedupe2", digest.String(),
"application/vnd.oci.image.layer.v1.tar+gzip")
So(err, ShouldBeNil)
So(getBlobSize2, ShouldBeGreaterThan, 0)
So(checkBlobSize1, ShouldEqual, checkBlobSize2)
So(getBlobSize1, ShouldEqual, getBlobSize2)
err = blobReadCloser.Close()
So(err, ShouldBeNil)
cblob, cdigest = test.GetRandomImageConfig()
_, clen, err = imgStore.FullBlobUpload("dedupe2", bytes.NewReader(cblob), cdigest.String())
@@ -1039,9 +1045,12 @@ func TestS3Dedupe(t *testing.T) {
So(err, ShouldBeNil)
// check that we retrieve the real dedupe2/blob (which is deduped earlier - 0 size) when switching to dedupe false
_, getBlobSize2, err = imgStore.GetBlob("dedupe2", digest.String(), "application/vnd.oci.image.layer.v1.tar+gzip")
blobReadCloser, getBlobSize2, err = imgStore.GetBlob("dedupe2", digest.String(),
"application/vnd.oci.image.layer.v1.tar+gzip")
So(err, ShouldBeNil)
So(getBlobSize1, ShouldEqual, getBlobSize2)
err = blobReadCloser.Close()
So(err, ShouldBeNil)
_, checkBlobSize2, err := imgStore.CheckBlob("dedupe2", digest.String())
So(err, ShouldBeNil)
pkg/storage/storage.go
+1 -1
View File
@@ -39,7 +39,7 @@ type ImageStore interface {
DeleteBlobUpload(repo, uuid string) error
BlobPath(repo string, digest digest.Digest) string
CheckBlob(repo, digest string) (bool, int64, error)
GetBlob(repo, digest, mediaType string) (io.Reader, int64, error)
GetBlob(repo, digest, mediaType string) (io.ReadCloser, int64, error)
DeleteBlob(repo, digest string) error
GetIndexContent(repo string) ([]byte, error)
GetBlobContent(repo, digest string) ([]byte, error)
pkg/storage/storage_test.go
+6 -2
View File
@@ -242,7 +242,9 @@ func TestStorageAPIs(t *testing.T) {
_, _, err = imgStore.CheckBlob("test", digest.String())
So(err, ShouldBeNil)
_, _, err = imgStore.GetBlob("test", digest.String(), "application/vnd.oci.image.layer.v1.tar+gzip")
blob, _, err := imgStore.GetBlob("test", digest.String(), "application/vnd.oci.image.layer.v1.tar+gzip")
So(err, ShouldBeNil)
err = blob.Close()
So(err, ShouldBeNil)
manifest := ispec.Manifest{}
@@ -431,7 +433,9 @@ func TestStorageAPIs(t *testing.T) {
_, _, err = imgStore.GetBlob("test", "inexistent", "application/vnd.oci.image.layer.v1.tar+gzip")
So(err, ShouldNotBeNil)
_, _, err = imgStore.GetBlob("test", digest.String(), "application/vnd.oci.image.layer.v1.tar+gzip")
blob, _, err := imgStore.GetBlob("test", digest.String(), "application/vnd.oci.image.layer.v1.tar+gzip")
So(err, ShouldBeNil)
err = blob.Close()
So(err, ShouldBeNil)
blobContent, err := imgStore.GetBlobContent("test", digest.String())