mirror of
https://github.com/project-zot/zot.git
synced 2026-06-18 13:37:57 +08:00
Ramkumar Chinchani
10dcd182f6
* feat(storage): add a common blobstore to store all blobs (#3906) Currently, zot uses one of the existing repos as the master copy for a blob to achieve dedupe, which complicates dedupe tracking logic. Furthermore, we have a global storage lock which is becoming a bottleneck. In order to move to a per-repo lock, we first need to simplify this logic. Now use a single hidden global repo (_blobstore/) as a blob store instead. Signed-off-by: Ramkumar Chinchani <rchincha.dev@gmail.com> * fix(storage): address blobstore cache correctness issues Signed-off-by: Ramkumar Chinchani <rchincha.dev@gmail.com> --------- Signed-off-by: Ramkumar Chinchani <rchincha.dev@gmail.com>
182 lines
4.9 KiB
Go
182 lines
4.9 KiB
Go
package cache_test
|
|
|
|
import (
|
|
"path"
|
|
"testing"
|
|
|
|
. "github.com/smartystreets/goconvey/convey"
|
|
|
|
"zotregistry.dev/zot/v2/errors"
|
|
"zotregistry.dev/zot/v2/pkg/log"
|
|
"zotregistry.dev/zot/v2/pkg/storage"
|
|
"zotregistry.dev/zot/v2/pkg/storage/cache"
|
|
)
|
|
|
|
func TestBoltDBCache(t *testing.T) {
|
|
Convey("Make a new cache", t, func() {
|
|
dir := t.TempDir()
|
|
|
|
log := log.NewTestLogger()
|
|
So(log, ShouldNotBeNil)
|
|
|
|
_, err := storage.Create("boltdb", "failTypeAssertion", log)
|
|
So(err, ShouldNotBeNil)
|
|
|
|
cacheDriver, _ := storage.Create("boltdb", cache.BoltDBDriverParameters{"/deadBEEF", "cache_test", true}, log)
|
|
So(cacheDriver, ShouldBeNil)
|
|
|
|
cacheDriver, _ = storage.Create("boltdb", cache.BoltDBDriverParameters{dir, "cache_test", true}, log)
|
|
So(cacheDriver, ShouldNotBeNil)
|
|
|
|
name := cacheDriver.Name()
|
|
So(name, ShouldEqual, "boltdb")
|
|
|
|
val, err := cacheDriver.GetBlob("key")
|
|
So(err, ShouldEqual, errors.ErrCacheMiss)
|
|
So(val, ShouldBeEmpty)
|
|
|
|
exists := cacheDriver.HasBlob("key", "value")
|
|
So(exists, ShouldBeFalse)
|
|
|
|
err = cacheDriver.PutBlob("key", path.Join(dir, "value"))
|
|
So(err, ShouldBeNil)
|
|
|
|
err = cacheDriver.PutBlob("key", "value")
|
|
So(err, ShouldNotBeNil)
|
|
|
|
exists = cacheDriver.HasBlob("key", "value")
|
|
So(exists, ShouldBeTrue)
|
|
|
|
val, err = cacheDriver.GetBlob("key")
|
|
So(err, ShouldBeNil)
|
|
So(val, ShouldNotBeEmpty)
|
|
|
|
err = cacheDriver.DeleteBlob("bogusKey", "bogusValue")
|
|
So(err, ShouldEqual, errors.ErrCacheMiss)
|
|
|
|
err = cacheDriver.DeleteBlob("key", "bogusValue")
|
|
So(err, ShouldEqual, errors.ErrCacheMiss)
|
|
|
|
// try to insert empty path
|
|
err = cacheDriver.PutBlob("key", "")
|
|
So(err, ShouldNotBeNil)
|
|
So(err, ShouldEqual, errors.ErrEmptyValue)
|
|
|
|
cacheDriver, _ = storage.Create("boltdb", cache.BoltDBDriverParameters{t.TempDir(), "cache_test", false}, log)
|
|
So(cacheDriver, ShouldNotBeNil)
|
|
|
|
err = cacheDriver.PutBlob("key1", "originalBlobPath")
|
|
So(err, ShouldBeNil)
|
|
|
|
err = cacheDriver.PutBlob("key1", "duplicateBlobPath")
|
|
So(err, ShouldBeNil)
|
|
|
|
val, err = cacheDriver.GetBlob("key1")
|
|
So(val, ShouldEqual, "originalBlobPath")
|
|
So(err, ShouldBeNil)
|
|
|
|
err = cacheDriver.DeleteBlob("key1", "duplicateBlobPath")
|
|
So(err, ShouldBeNil)
|
|
|
|
val, err = cacheDriver.GetBlob("key1")
|
|
So(val, ShouldEqual, "originalBlobPath")
|
|
So(err, ShouldBeNil)
|
|
|
|
err = cacheDriver.PutBlob("key1", "duplicateBlobPath")
|
|
So(err, ShouldBeNil)
|
|
|
|
// deleting original when duplicates exist should keep the original (no promotion)
|
|
err = cacheDriver.DeleteBlob("key1", "originalBlobPath")
|
|
So(err, ShouldBeNil)
|
|
|
|
// original should still be "originalBlobPath" since duplicates exist
|
|
val, err = cacheDriver.GetBlob("key1")
|
|
So(val, ShouldEqual, "originalBlobPath")
|
|
So(err, ShouldBeNil)
|
|
|
|
// now delete the duplicate
|
|
err = cacheDriver.DeleteBlob("key1", "duplicateBlobPath")
|
|
So(err, ShouldBeNil)
|
|
|
|
// now delete the original (no more duplicates, should clean up)
|
|
err = cacheDriver.DeleteBlob("key1", "originalBlobPath")
|
|
So(err, ShouldBeNil)
|
|
|
|
// should be empty
|
|
val, err = cacheDriver.GetBlob("key1")
|
|
So(err, ShouldNotBeNil)
|
|
So(val, ShouldBeEmpty)
|
|
|
|
// try to add three same values
|
|
err = cacheDriver.PutBlob("key2", "duplicate")
|
|
So(err, ShouldBeNil)
|
|
|
|
err = cacheDriver.PutBlob("key2", "duplicate")
|
|
So(err, ShouldBeNil)
|
|
|
|
err = cacheDriver.PutBlob("key2", "duplicate")
|
|
So(err, ShouldBeNil)
|
|
|
|
val, err = cacheDriver.GetBlob("key2")
|
|
So(val, ShouldEqual, "duplicate")
|
|
So(err, ShouldBeNil)
|
|
|
|
err = cacheDriver.DeleteBlob("key2", "duplicate")
|
|
So(err, ShouldBeNil)
|
|
|
|
// should be empty
|
|
val, err = cacheDriver.GetBlob("key2")
|
|
So(err, ShouldNotBeNil)
|
|
So(val, ShouldBeEmpty)
|
|
})
|
|
|
|
Convey("Test cache.GetAllBlos()", t, func() {
|
|
dir := t.TempDir()
|
|
|
|
log := log.NewTestLogger()
|
|
So(log, ShouldNotBeNil)
|
|
|
|
_, err := storage.Create("boltdb", "failTypeAssertion", log)
|
|
So(err, ShouldNotBeNil)
|
|
|
|
cacheDriver, _ := storage.Create("boltdb", cache.BoltDBDriverParameters{dir, "cache_test", false}, log)
|
|
So(cacheDriver, ShouldNotBeNil)
|
|
|
|
err = cacheDriver.PutBlob("digest", "first")
|
|
So(err, ShouldBeNil)
|
|
|
|
blobs, err := cacheDriver.GetAllBlobs("digest")
|
|
So(err, ShouldBeNil)
|
|
So(blobs, ShouldResemble, []string{"first"})
|
|
|
|
err = cacheDriver.PutBlob("digest", "second")
|
|
So(err, ShouldBeNil)
|
|
|
|
err = cacheDriver.PutBlob("digest", "third")
|
|
So(err, ShouldBeNil)
|
|
|
|
blobs, err = cacheDriver.GetAllBlobs("digest")
|
|
So(err, ShouldBeNil)
|
|
|
|
// "first" is the original, "second" and "third" are duplicates
|
|
So(blobs, ShouldResemble, []string{"first", "second", "third"})
|
|
|
|
// deleting "first" (original) should keep it since duplicates exist
|
|
err = cacheDriver.DeleteBlob("digest", "first")
|
|
So(err, ShouldBeNil)
|
|
|
|
blobs, err = cacheDriver.GetAllBlobs("digest")
|
|
So(err, ShouldBeNil)
|
|
|
|
So(blobs, ShouldResemble, []string{"first", "second", "third"})
|
|
|
|
err = cacheDriver.DeleteBlob("digest", "third")
|
|
So(err, ShouldBeNil)
|
|
|
|
blobs, err = cacheDriver.GetAllBlobs("digest")
|
|
So(err, ShouldBeNil)
|
|
|
|
So(blobs, ShouldResemble, []string{"first", "second"})
|
|
})
|
|
}
|