github/zot
1
0
Fork 0
mirror of https://github.com/project-zot/zot.git synced 2026-06-15 20:07:55 +08:00
Code Issues Packages Projects Releases Wiki Activity

dedupe: stat blob path before creating link

Browse Source
This commit is contained in:
Shivam Mishra
2021-07-20 14:04:10 -07:00
committed by Ramkumar Chinchani
parent f10c174c0e
commit 53b5fa6493
7 changed files with 146 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/storage/cache.go
+6 -4
View File
@@ -51,6 +51,12 @@ func NewCache(rootDir string, name string, log zlog.Logger) *Cache {
}
func (c *Cache) PutBlob(digest string, path string) error {
if path == "" {
c.log.Error().Err(errors.ErrEmptyValue).Str("digest", digest).Msg("empty path provided")
return errors.ErrEmptyValue
}
// use only relative (to rootDir) paths on blobs
relp, err := filepath.Rel(c.rootDir, path)
if err != nil {
@@ -109,10 +115,6 @@ func (c *Cache) GetBlob(digest string) (string, error) {
return "", err
}
if len(blobPath.String()) == 0 {
return "", nil
}
return blobPath.String(), nil
}
pkg/storage/cache_test.go
+5
View File
@@ -49,5 +49,10 @@ func TestCache(t *testing.T) {
err = c.DeleteBlob("key", "bogusValue")
So(err, ShouldBeNil)
// try to insert empty path
err = c.PutBlob("key", "")
So(err, ShouldNotBeNil)
So(err, ShouldEqual, errors.ErrEmptyValue)
})
}
pkg/storage/storage.go
+14
View File
@@ -956,6 +956,7 @@ retry:
}
if dstRecord == "" {
// cache record doesn't exist, so first disk and cache entry for this digest
if err := is.cache.PutBlob(dstDigest.String(), dst); err != nil {
is.log.Error().Err(err).Str("blobPath", dst).Msg("dedupe: unable to insert blob record")
@@ -971,6 +972,8 @@ retry:
is.log.Debug().Str("src", src).Str("dst", dst).Msg("dedupe: rename")
} else {
// cache record exists, but due to GC and upgrades from older versions,
// disk content and cache records may go out of sync
dstRecord = path.Join(is.rootDir, dstRecord)
dstRecordFi, err := os.Stat(dstRecord)
@@ -985,19 +988,30 @@ retry:
}
goto retry
}
dstFi, err := os.Stat(dst)
if err != nil && !os.IsNotExist(err) {
is.log.Error().Err(err).Str("blobPath", dstRecord).Msg("dedupe: unable to stat")
return err
}
if !os.SameFile(dstFi, dstRecordFi) {
// blob lookup cache out of sync with actual disk contents
if err := os.Remove(dst); err != nil && !os.IsNotExist(err) {
is.log.Error().Err(err).Str("dst", dst).Msg("dedupe: unable to remove blob")
return err
}
is.log.Debug().Str("blobPath", dst).Msg("dedupe: creating hard link")
if err := os.Link(dstRecord, dst); err != nil {
is.log.Error().Err(err).Str("blobPath", dst).Str("link", dstRecord).Msg("dedupe: unable to hard link")
return err
}
}
if err := os.Remove(src); err != nil {
is.log.Error().Err(err).Str("src", src).Msg("dedupe: uname to remove blob")
return err
pkg/storage/storage_test.go
+117
View File
@@ -6,11 +6,13 @@ import (
"encoding/json"
"io/ioutil"
"os"
"os/exec"
"path"
"strings"
"sync"
"testing"
"github.com/anuvu/zot/errors"
"github.com/anuvu/zot/pkg/log"
"github.com/anuvu/zot/pkg/storage"
godigest "github.com/opencontainers/go-digest"
@@ -532,6 +534,51 @@ func TestNegativeCases(t *testing.T) {
il := storage.NewImageStore(dir, true, true, log.Logger{Logger: zerolog.New(os.Stdout)})
So(il, ShouldNotBeNil)
So(il.InitRepo("test"), ShouldBeNil)
err = os.MkdirAll(path.Join(dir, "invalid-test"), 0755)
So(err, ShouldBeNil)
err = os.Chmod(path.Join(dir, "invalid-test"), 0000) // remove all perms
So(err, ShouldBeNil)
_, err = il.ValidateRepo("invalid-test")
So(err, ShouldNotBeNil)
So(err, ShouldEqual, errors.ErrRepoNotFound)
err = os.Chmod(path.Join(dir, "invalid-test"), 0755) // remove all perms
So(err, ShouldBeNil)
err = ioutil.WriteFile(path.Join(dir, "invalid-test", "blobs"), []byte{}, 0755) // nolint: gosec
So(err, ShouldBeNil)
err = ioutil.WriteFile(path.Join(dir, "invalid-test", "index.json"), []byte{}, 0755) // nolint: gosec
So(err, ShouldBeNil)
err = ioutil.WriteFile(path.Join(dir, "invalid-test", ispec.ImageLayoutFile), []byte{}, 0755) // nolint: gosec
So(err, ShouldBeNil)
isValid, err := il.ValidateRepo("invalid-test")
So(err, ShouldBeNil)
So(isValid, ShouldEqual, false)
err = os.Remove(path.Join(dir, "invalid-test", "blobs"))
So(err, ShouldBeNil)
err = os.Mkdir(path.Join(dir, "invalid-test", "blobs"), 0755)
So(err, ShouldBeNil)
isValid, err = il.ValidateRepo("invalid-test")
So(err, ShouldNotBeNil)
So(isValid, ShouldEqual, false)
err = ioutil.WriteFile(path.Join(dir, "invalid-test", ispec.ImageLayoutFile), []byte("{}"), 0755) // nolint: gosec
So(err, ShouldBeNil)
isValid, err = il.ValidateRepo("invalid-test")
So(err, ShouldNotBeNil)
So(err, ShouldEqual, errors.ErrRepoBadVersion)
So(isValid, ShouldEqual, false)
files, err := ioutil.ReadDir(path.Join(dir, "test"))
So(err, ShouldBeNil)
for _, f := range files {
@@ -597,6 +644,76 @@ func TestNegativeCases(t *testing.T) {
_, _, _, err = il.GetImageManifest("test", "")
So(err, ShouldNotBeNil)
})
Convey("Invalid dedupe sceanrios", t, func() {
dir, err := ioutil.TempDir("", "oci-repo-test")
if err != nil {
panic(err)
}
defer os.RemoveAll(dir)
il := storage.NewImageStore(dir, true, true, log.Logger{Logger: zerolog.New(os.Stdout)})
v, err := il.NewBlobUpload("dedupe1")
So(err, ShouldBeNil)
So(v, ShouldNotBeEmpty)
content := []byte("test-data3")
buf := bytes.NewBuffer(content)
l := buf.Len()
d := godigest.FromBytes(content)
b, err := il.PutBlobChunkStreamed("dedupe1", v, buf)
So(err, ShouldBeNil)
So(b, ShouldEqual, l)
blobDigest1 := strings.Split(d.String(), ":")[1]
So(blobDigest1, ShouldNotBeEmpty)
err = il.FinishBlobUpload("dedupe1", v, buf, d.String())
So(err, ShouldBeNil)
So(b, ShouldEqual, l)
// Create a file at the same place where FinishBlobUpload will create
err = il.InitRepo("dedupe2")
So(err, ShouldBeNil)
err = os.MkdirAll(path.Join(dir, "dedupe2", "blobs/sha256"), 0755)
So(err, ShouldBeNil)
err = ioutil.WriteFile(path.Join(dir, "dedupe2", "blobs/sha256", blobDigest1), content, 0755) // nolint: gosec
So(err, ShouldBeNil)
v, err = il.NewBlobUpload("dedupe2")
So(err, ShouldBeNil)
So(v, ShouldNotBeEmpty)
content = []byte("test-data3")
buf = bytes.NewBuffer(content)
l = buf.Len()
d = godigest.FromBytes(content)
b, err = il.PutBlobChunkStreamed("dedupe2", v, buf)
So(err, ShouldBeNil)
So(b, ShouldEqual, l)
cmd := exec.Command("sudo", "chattr", "+i", path.Join(dir, "dedupe2", "blobs/sha256", blobDigest1)) // nolint: gosec
_, err = cmd.Output()
if err != nil {
panic(err)
}
err = il.FinishBlobUpload("dedupe2", v, buf, d.String())
So(err, ShouldNotBeNil)
So(b, ShouldEqual, l)
cmd = exec.Command("sudo", "chattr", "-i", path.Join(dir, "dedupe2", "blobs/sha256", blobDigest1)) // nolint: gosec
_, err = cmd.Output()
if err != nil {
panic(err)
}
err = il.FinishBlobUpload("dedupe2", v, buf, d.String())
So(err, ShouldBeNil)
So(b, ShouldEqual, l)
})
}
func TestHardLink(t *testing.T) {