feat(storage): add a common blobstore to store all blobs (#3906)

Currently, zot uses one of the existing repos as the master copy for a blob to
achieve dedupe, which complicates dedupe tracking logic. Furthermore, we
have a global storage lock which is becoming a bottleneck. In order to
move to a per-repo lock, we first need to simplify this logic.

Now use a single hidden global repo (_blobstore/) as a blob store instead.

Signed-off-by: Ramkumar Chinchani <rchincha.dev@gmail.com>

pkg/storage/cache/boltdb.go

@@ -114,6 +114,10 @@ func (d *BoltDBDriver) PutBlob(digest godigest.Digest, path string) error {
 		}
 	}
 
+	if len(path) == 0 {
+		return zerr.ErrEmptyValue
+	}
+
 	if err := d.db.Update(func(tx *bbolt.Tx) error {
 		root := tx.Bucket([]byte(constants.BlobsCache))
 		if root == nil {
@@ -132,21 +136,6 @@ func (d *BoltDBDriver) PutBlob(digest godigest.Digest, path string) error {
 			return err
 		}
 
-		// create nested deduped bucket where we store all the deduped blobs + original blob
-		deduped, err := bucket.CreateBucketIfNotExists([]byte(constants.DuplicatesBucket))
-		if err != nil {
-			// this is a serious failure
-			d.log.Error().Err(err).Str("bucket", constants.DuplicatesBucket).Msg("failed to create a bucket")
-
-			return err
-		}
-
-		if err := deduped.Put([]byte(path), nil); err != nil {
-			d.log.Error().Err(err).Str("bucket", constants.DuplicatesBucket).Str("value", path).Msg("failed to put record")
-
-			return err
-		}
-
 		// create origin bucket and insert only the original blob
 		origin := bucket.Bucket([]byte(constants.OriginalBucket))
 		if origin == nil {
@@ -164,6 +153,28 @@ func (d *BoltDBDriver) PutBlob(digest godigest.Digest, path string) error {
 
 				return err
 			}
+
+			return nil
+		}
+
+		// original bucket exists; if path is the same as the original, this is idempotent
+		if origin.Get([]byte(path)) != nil {
+			return nil
+		}
+
+		// otherwise, this is a duplicate blob - add to the duplicates bucket
+		deduped, err := bucket.CreateBucketIfNotExists([]byte(constants.DuplicatesBucket))
+		if err != nil {
+			// this is a serious failure
+			d.log.Error().Err(err).Str("bucket", constants.DuplicatesBucket).Msg("failed to create a bucket")
+
+			return err
+		}
+
+		if err := deduped.Put([]byte(path), nil); err != nil {
+			d.log.Error().Err(err).Str("bucket", constants.DuplicatesBucket).Str("value", path).Msg("failed to put record")
+
+			return err
 		}
 
 		return nil
@@ -275,18 +286,20 @@ func (d *BoltDBDriver) HasBlob(digest godigest.Digest, blob string) bool {
 			return zerr.ErrCacheMiss
 		}
 
-		deduped := bucket.Bucket([]byte(constants.DuplicatesBucket))
-		if deduped == nil {
-			return zerr.ErrCacheMiss
+		// check original bucket first
+		if origin.Get([]byte(blob)) != nil {
+			return nil
 		}
 
-		if origin.Get([]byte(blob)) == nil {
-			if deduped.Get([]byte(blob)) == nil {
-				return zerr.ErrCacheMiss
+		// check duplicates bucket
+		deduped := bucket.Bucket([]byte(constants.DuplicatesBucket))
+		if deduped != nil {
+			if deduped.Get([]byte(blob)) != nil {
+				return nil
 			}
 		}
 
-		return nil
+		return zerr.ErrCacheMiss
 	}); err != nil {
 		return false
 	}
@@ -330,22 +343,33 @@ func (d *BoltDBDriver) DeleteBlob(digest godigest.Digest, path string) error {
 			return zerr.ErrCacheMiss
 		}
 
+		// check duplicates bucket first
 		deduped := bucket.Bucket([]byte(constants.DuplicatesBucket))
-		if deduped == nil {
-			return zerr.ErrCacheMiss
-		}
-
-		if err := deduped.Delete([]byte(path)); err != nil {
-			d.log.Error().Err(err).Str("digest", digest.String()).Str("bucket", constants.DuplicatesBucket).
-				Str("path", path).Msg("failed to delete")
-
-			return err
+		if deduped != nil {
+			if deduped.Get([]byte(path)) != nil {
+				if err := deduped.Delete([]byte(path)); err != nil {
+					d.log.Error().Err(err).Str("digest", digest.String()).Str("bucket", constants.DuplicatesBucket).
+						Str("path", path).Msg("failed to delete")
+
+					return err
+				}
+
+				return nil
+			}
 		}
 
+		// check original bucket
 		origin := bucket.Bucket([]byte(constants.OriginalBucket))
+		deleted := false
+
 		if origin != nil {
-			originBlob := d.getOne(origin)
-			if originBlob != nil {
+			if origin.Get([]byte(path)) != nil {
+				// if duplicates still exist, keep the original (global blobstore file stays)
+				if deduped != nil && d.getOne(deduped) != nil {
+					return nil
+				}
+
+				// no more duplicates, safe to remove the original
 				if err := origin.Delete([]byte(path)); err != nil {
 					d.log.Error().Err(err).Str("digest", digest.String()).Str("bucket", constants.OriginalBucket).
 						Str("path", path).Msg("failed to delete")
@@ -353,19 +377,14 @@ func (d *BoltDBDriver) DeleteBlob(digest godigest.Digest, path string) error {
 					return err
 				}
 
-				// move next candidate to origin bucket, next GetKey will return this one and storage will move the content here
-				dedupedBlob := d.getOne(deduped)
-				if dedupedBlob != nil {
-					if err := origin.Put(dedupedBlob, nil); err != nil {
-						d.log.Error().Err(err).Str("digest", digest.String()).Str("bucket", constants.OriginalBucket).Str("path", path).
-							Msg("failed to put")
-
-						return err
-					}
-				}
+				deleted = true
 			}
 		}
 
+		if !deleted {
+			return zerr.ErrCacheMiss
+		}
+
 		// if no key in origin bucket then digest bucket is empty, remove it
 		k := d.getOne(origin)
 		if k == nil {
@@ -376,9 +395,11 @@ func (d *BoltDBDriver) DeleteBlob(digest godigest.Digest, path string) error {
 
 				return err
 			}
+
+			return nil
 		}
 
-		return nil
+		return zerr.ErrCacheMiss
 	}); err != nil {
 		return err
 	}

pkg/storage/cache/boltdb_test.go

@@ -55,7 +55,7 @@ func TestBoltDBCache(t *testing.T) {
 		So(err, ShouldEqual, errors.ErrCacheMiss)
 
 		err = cacheDriver.DeleteBlob("key", "bogusValue")
-		So(err, ShouldBeNil)
+		So(err, ShouldEqual, errors.ErrCacheMiss)
 
 		// try to insert empty path
 		err = cacheDriver.PutBlob("key", "")
@@ -85,16 +85,23 @@ func TestBoltDBCache(t *testing.T) {
 		err = cacheDriver.PutBlob("key1", "duplicateBlobPath")
 		So(err, ShouldBeNil)
 
+		// deleting original when duplicates exist should keep the original (no promotion)
 		err = cacheDriver.DeleteBlob("key1", "originalBlobPath")
 		So(err, ShouldBeNil)
 
+		// original should still be "originalBlobPath" since duplicates exist
 		val, err = cacheDriver.GetBlob("key1")
-		So(val, ShouldEqual, "duplicateBlobPath")
+		So(val, ShouldEqual, "originalBlobPath")
 		So(err, ShouldBeNil)
 
+		// now delete the duplicate
 		err = cacheDriver.DeleteBlob("key1", "duplicateBlobPath")
 		So(err, ShouldBeNil)
 
+		// now delete the original (no more duplicates, should clean up)
+		err = cacheDriver.DeleteBlob("key1", "originalBlobPath")
+		So(err, ShouldBeNil)
+
 		// should be empty
 		val, err = cacheDriver.GetBlob("key1")
 		So(err, ShouldNotBeNil)
@@ -147,15 +154,17 @@ func TestBoltDBCache(t *testing.T) {
 		blobs, err := cacheDriver.GetAllBlobs("digest")
 		So(err, ShouldBeNil)
 
+		// "first" is the original, "second" and "third" are duplicates
 		So(blobs, ShouldResemble, []string{"first", "second", "third"})
 
+		// deleting "first" (original) should keep it since duplicates exist
 		err = cacheDriver.DeleteBlob("digest", "first")
 		So(err, ShouldBeNil)
 
 		blobs, err = cacheDriver.GetAllBlobs("digest")
 		So(err, ShouldBeNil)
 
-		So(blobs, ShouldResemble, []string{"second", "third"})
+		So(blobs, ShouldResemble, []string{"first", "second", "third"})
 
 		err = cacheDriver.DeleteBlob("digest", "third")
 		So(err, ShouldBeNil)
@@ -163,6 +172,6 @@ func TestBoltDBCache(t *testing.T) {
 		blobs, err = cacheDriver.GetAllBlobs("digest")
 		So(err, ShouldBeNil)
 
-		So(blobs, ShouldResemble, []string{"second"})
+		So(blobs, ShouldResemble, []string{"first", "second"})
 	})
 }

pkg/storage/cache/dynamodb.go

@@ -187,13 +187,22 @@ func (d *DynamoDBDriver) PutBlob(digest godigest.Digest, path string) error {
 		return zerr.ErrEmptyValue
 	}
 
-	if originBlob, _ := d.GetBlob(digest); originBlob == "" {
-		// first entry, so add original blob
+	originBlob, _ := d.GetBlob(digest)
+	if originBlob == "" {
+		// first entry, so add original blob only
 		if err := d.putOriginBlob(digest, path); err != nil {
 			return err
 		}
+
+		return nil
 	}
 
+	// if same as original, this is idempotent
+	if originBlob == path {
+		return nil
+	}
+
+	// add as duplicate
 	expression := "ADD DuplicateBlobPath :i"
 	attrPath := types.AttributeValueMemberSS{Value: []string{path}}
 
@@ -245,27 +254,69 @@ func (d *DynamoDBDriver) HasBlob(digest godigest.Digest, path string) bool {
 func (d *DynamoDBDriver) DeleteBlob(digest godigest.Digest, path string) error {
 	marshaledKey, _ := attributevalue.MarshalMap(map[string]any{"Digest": digest.String()})
 
-	expression := "DELETE DuplicateBlobPath :i"
-	attrPath := types.AttributeValueMemberSS{Value: []string{path}}
+	// check if path is a duplicate first
+	duplicateBlob, _ := d.GetDuplicateBlob(digest)
+	if duplicateBlob != "" {
+		// check if path is in the duplicates set
+		resp, err := d.client.GetItem(context.TODO(), &dynamodb.GetItemInput{
+			TableName: aws.String(d.tableName),
+			Key: map[string]types.AttributeValue{
+				"Digest": &types.AttributeValueMemberS{Value: digest.String()},
+			},
+		})
+		if err != nil {
+			return err
+		}
 
-	if err := d.updateItem(digest, expression, map[string]types.AttributeValue{":i": &attrPath}); err != nil {
-		d.log.Error().Err(err).Str("digest", digest.String()).Str("path", path).Msg("failed to delete")
+		out := Blob{}
+		if resp.Item != nil {
+			_ = attributevalue.UnmarshalMap(resp.Item, &out)
 
-		return err
-	}
+			if slices.Contains(out.DuplicateBlobPath, path) {
+				expression := "DELETE DuplicateBlobPath :i"
+				attrPath := types.AttributeValueMemberSS{Value: []string{path}}
 
-	originBlob, _ := d.GetBlob(digest)
-	// if original blob is the one deleted
-	if originBlob == path {
-		// move duplicate blob to original, storage will move content here
-		originBlob, _ = d.GetDuplicateBlob(digest)
-		if originBlob != "" {
-			if err := d.putOriginBlob(digest, originBlob); err != nil {
-				return err
+				if err := d.updateItem(digest, expression, map[string]types.AttributeValue{":i": &attrPath}); err != nil {
+					d.log.Error().Err(err).Str("digest", digest.String()).Str("path", path).Msg("failed to delete")
+
+					return err
+				}
+
+				return nil
 			}
 		}
 	}
 
+	originBlob, err := d.GetBlob(digest)
+	if err != nil {
+		// ErrCacheMiss means the digest doesn't exist at all — path not found
+		return err
+	}
+
+	// if original blob is the one being deleted
+	if originBlob == path {
+		// check if duplicates still exist
+		remainingDuplicate, _ := d.GetDuplicateBlob(digest)
+		if remainingDuplicate != "" {
+			// duplicates still exist, keep the original (global blobstore file stays)
+			return nil
+		}
+
+		// no more duplicates, remove the original
+		_, err = d.client.DeleteItem(context.TODO(), &dynamodb.DeleteItemInput{
+			Key:       marshaledKey,
+			TableName: &d.tableName,
+		})
+		if err != nil {
+			d.log.Error().Err(err).Str("digest", digest.String()).Str("path", path).Msg("failed to delete")
+
+			return err
+		}
+
+		return nil
+	}
+
+	// originBlob is empty but record exists (orphaned entry) — clean up
 	if originBlob == "" {
 		d.log.Debug().Str("digest", digest.String()).Str("path", path).Msg("deleting empty bucket")
 
@@ -273,9 +324,12 @@ func (d *DynamoDBDriver) DeleteBlob(digest godigest.Digest, path string) error {
 			Key:       marshaledKey,
 			TableName: &d.tableName,
 		})
+
+		return zerr.ErrCacheMiss
 	}
 
-	return nil
+	// path not found in duplicates or original
+	return zerr.ErrCacheMiss
 }
 
 func (d *DynamoDBDriver) GetDuplicateBlob(digest godigest.Digest) (string, error) {

pkg/storage/cache/dynamodb_test.go

@@ -82,7 +82,7 @@ func TestDynamoDB(t *testing.T) {
 		So(exists, ShouldBeTrue)
 
 		exists = cacheDriver.HasBlob(keyDigest, path.Join(dir, "value1"))
-		So(exists, ShouldBeFalse)
+		So(exists, ShouldBeTrue)
 
 		err = cacheDriver.DeleteBlob(keyDigest, path.Join(dir, "value2"))
 		So(err, ShouldBeNil)
@@ -111,16 +111,16 @@ func TestDynamoDB(t *testing.T) {
 		So(err, ShouldBeNil)
 
 		val, err = cacheDriver.GetBlob("key1")
-		So(val, ShouldEqual, "duplicateBlobPath")
+		So(val, ShouldEqual, "originalBlobPath")
 		So(err, ShouldBeNil)
 
 		err = cacheDriver.DeleteBlob("key1", "duplicateBlobPath")
 		So(err, ShouldBeNil)
 
-		// should be empty
+		// original remains while duplicates are removed
 		val, err = cacheDriver.GetBlob("key1")
-		So(err, ShouldNotBeNil)
-		So(val, ShouldBeEmpty)
+		So(err, ShouldBeNil)
+		So(val, ShouldEqual, "originalBlobPath")
 
 		// try to add three same values
 		err = cacheDriver.PutBlob("key2", "duplicate")
@@ -176,7 +176,7 @@ func TestDynamoDB(t *testing.T) {
 		blobs, err = cacheDriver.GetAllBlobs("digest")
 		So(err, ShouldBeNil)
 
-		So(blobs, ShouldResemble, []string{"second", "third"})
+		So(blobs, ShouldResemble, []string{"first", "second", "third"})
 
 		err = cacheDriver.DeleteBlob("digest", "third")
 		So(err, ShouldBeNil)
@@ -184,7 +184,7 @@ func TestDynamoDB(t *testing.T) {
 		blobs, err = cacheDriver.GetAllBlobs("digest")
 		So(err, ShouldBeNil)
 
-		So(blobs, ShouldResemble, []string{"second"})
+		So(blobs, ShouldResemble, []string{"first", "second"})
 	})
 }
 

pkg/storage/cache/redis.go

@@ -145,7 +145,18 @@ func (d *RedisDriver) PutBlob(digest godigest.Digest, path string) error {
 
 				return err
 			}
+
+			// first entry is only stored as the original, not as a duplicate
+			return nil
 		}
+
+		// check if this is the same as the original (idempotent)
+		currentPath, err := d.db.HGet(ctx, d.join(constants.BlobsCache, constants.OriginalBucket),
+			digest.String()).Result()
+		if err == nil && currentPath == path {
+			return nil
+		}
+
 		// add path to the set of paths which the digest represents
 		if err := txrp.SAdd(ctx, d.join(constants.BlobsCache, constants.DuplicatesBucket,
 			digest.String()), path).Err(); err != nil {
@@ -234,7 +245,23 @@ func (d *RedisDriver) HasBlob(digest godigest.Digest, path string) bool {
 	}
 
 	ctx := context.TODO()
-	// see if we are in the set
+
+	// check if path is the original
+	currentPath, err := d.db.HGet(ctx, d.join(constants.BlobsCache, constants.OriginalBucket), digest.String()).Result()
+	if err != nil {
+		if !goerrors.Is(err, redis.Nil) {
+			d.log.Error().Err(err).Str("hget", d.join(constants.BlobsCache, constants.OriginalBucket)).
+				Str("digest", digest.String()).Msg("unable to get record")
+		}
+
+		return false
+	}
+
+	if currentPath == path {
+		return true
+	}
+
+	// check if path is in the duplicates set
 	exists, err := d.db.SIsMember(ctx, d.join(constants.BlobsCache, constants.DuplicatesBucket,
 		digest.String()), path).Result()
 	if err != nil {
@@ -244,25 +271,7 @@ func (d *RedisDriver) HasBlob(digest godigest.Digest, path string) bool {
 		return false
 	}
 
-	if !exists {
-		return false
-	}
-
-	// see if the path entry exists. is this actually needed? i guess it doesn't really hurt (it is fast)
-	exists, err = d.db.HExists(ctx, d.join(constants.BlobsCache, constants.OriginalBucket), digest.String()).Result()
-
-	d.log.Error().Err(err).Str("hexists", d.join(constants.BlobsCache, constants.OriginalBucket)).
-		Str("digest", digest.String()).Msg("unable to get record")
-
-	if err != nil {
-		return false
-	}
-
-	if !exists {
-		return false
-	}
-
-	return true
+	return exists
 }
 
 func (d *RedisDriver) DeleteBlob(digest godigest.Digest, path string) error {
@@ -291,47 +300,57 @@ func (d *RedisDriver) DeleteBlob(digest godigest.Digest, path string) error {
 		}
 	}()
 
+	// check duplicates first
 	pathSet := d.join(constants.BlobsCache, constants.DuplicatesBucket, digest.String())
 
-	// delete path from the set of paths which the digest represents
-	_, err = d.db.SRem(ctx, pathSet, path).Result()
+	exists, err := d.db.SIsMember(ctx, pathSet, path).Result()
 	if err != nil {
-		d.log.Error().Err(err).Str("srem", pathSet).Str("value", path).Msg("failed to delete record")
+		d.log.Error().Err(err).Str("sismember", pathSet).Str("value", path).Msg("failed to lookup record")
 
 		return err
 	}
 
+	if exists {
+		// delete path from the set of paths which the digest represents
+		_, err = d.db.SRem(ctx, pathSet, path).Result()
+		if err != nil {
+			d.log.Error().Err(err).Str("srem", pathSet).Str("value", path).Msg("failed to delete record")
+
+			return err
+		}
+
+		return nil
+	}
+
+	// check if path is the original
 	currentPath, err := d.GetBlob(digest)
 	if err != nil {
 		return err
 	}
 
 	if currentPath != path {
-		// nothing we need to do, return nil yay
-		return nil
+		// path not found in duplicates or original
+		return zerr.ErrCacheMiss
 	}
 
-	// we need to set a new path
-	newPath, err := d.db.SRandMember(ctx, pathSet).Result()
+	// path is the original - check if there are still duplicates
+	dupes, err := d.db.SCard(ctx, pathSet).Result()
 	if err != nil {
-		if goerrors.Is(err, redis.Nil) {
-			_, err := d.db.HDel(ctx, d.join(constants.BlobsCache, constants.OriginalBucket), digest.String()).Result()
-			if err != nil {
-				return err
-			}
-
-			return nil
-		}
-
-		d.log.Error().Err(err).Str("srandmember", pathSet).Msg("failed to get new path")
+		d.log.Error().Err(err).Str("scard", pathSet).Msg("failed to count duplicates")
 
 		return err
 	}
 
-	if _, err := d.db.HSet(ctx, d.join(constants.BlobsCache, constants.OriginalBucket),
-		digest.String(), newPath).Result(); err != nil {
-		d.log.Error().Err(err).Str("hset", d.join(constants.BlobsCache, constants.OriginalBucket)).Str("value", newPath).
-			Msg("unable to put record")
+	if dupes > 0 {
+		// duplicates still exist, keep the original (global blobstore file stays)
+		return nil
+	}
+
+	// no more duplicates, remove the original
+	if _, err := d.db.HDel(ctx, d.join(constants.BlobsCache, constants.OriginalBucket),
+		digest.String()).Result(); err != nil {
+		d.log.Error().Err(err).Str("hdel", d.join(constants.BlobsCache, constants.OriginalBucket)).Str("value", path).
+			Msg("failed to delete record")
 
 		return err
 	}

pkg/storage/cache/redis_test.go

@@ -73,7 +73,7 @@ func TestRedisCache(t *testing.T) {
 		So(err, ShouldEqual, zerr.ErrCacheMiss)
 
 		err = cacheDriver.DeleteBlob("key", "bogusValue")
-		So(err, ShouldBeNil)
+		So(err, ShouldEqual, zerr.ErrCacheMiss)
 
 		// try to insert empty path
 		err = cacheDriver.PutBlob("key", "")
@@ -112,16 +112,16 @@ func TestRedisCache(t *testing.T) {
 		So(err, ShouldBeNil)
 
 		val, err = cacheDriver.GetBlob("key1")
-		So(val, ShouldEqual, "duplicateBlobPath")
+		So(val, ShouldEqual, "originalBlobPath")
 		So(err, ShouldBeNil)
 
 		err = cacheDriver.DeleteBlob("key1", "duplicateBlobPath")
 		So(err, ShouldBeNil)
 
-		// should be empty
+		// original remains while duplicates are removed
 		val, err = cacheDriver.GetBlob("key1")
-		So(err, ShouldNotBeNil)
-		So(val, ShouldBeEmpty)
+		So(err, ShouldBeNil)
+		So(val, ShouldEqual, "originalBlobPath")
 
 		// try to add three same values
 		err = cacheDriver.PutBlob("key2", "duplicate")
@@ -186,7 +186,8 @@ func TestRedisCache(t *testing.T) {
 
 		blobs, err = cacheDriver.GetAllBlobs("digest")
 		So(err, ShouldBeNil)
-		So(len(blobs), ShouldEqual, 2)
+		So(len(blobs), ShouldEqual, 3)
+		So(blobs, ShouldContain, "first")
 		So(blobs, ShouldContain, "second")
 		So(blobs, ShouldContain, "third")
 
@@ -196,7 +197,7 @@ func TestRedisCache(t *testing.T) {
 		blobs, err = cacheDriver.GetAllBlobs("digest")
 		So(err, ShouldBeNil)
 
-		So(blobs, ShouldResemble, []string{"second"})
+		So(blobs, ShouldResemble, []string{"first", "second"})
 	})
 }
 
@@ -355,10 +356,10 @@ func TestRedisMocked(t *testing.T) {
 
 				mock.Regexp().ExpectSetNX(keyPrefix+"locks:key", `.*`, 8*time.Second).SetVal(true)
 				mock.ExpectHExists(keyPrefix+constants.BlobsCache+":"+constants.OriginalBucket, "key").
-					SetVal(false)
+					SetVal(true)
+				mock.ExpectHGet(keyPrefix+constants.BlobsCache+":"+constants.OriginalBucket, "key").
+					SetVal(path.Join(pathPrefix, "original"))
 				mock.ExpectTxPipeline()
-				mock.ExpectHSet(keyPrefix+constants.BlobsCache+":"+constants.OriginalBucket, "key",
-					path.Join(pathPrefix, "val")).SetVal(1)
 				mock.ExpectSAdd(keyPrefix+constants.BlobsCache+":"+constants.DuplicatesBucket+":key",
 					path.Join(pathPrefix, "val")).SetErr(ErrTestError)
 
@@ -385,8 +386,6 @@ func TestRedisMocked(t *testing.T) {
 				mock.ExpectTxPipeline()
 				mock.ExpectHSet(keyPrefix+constants.BlobsCache+":"+constants.OriginalBucket, "key",
 					path.Join(pathPrefix, "val")).SetVal(1)
-				mock.ExpectSAdd(keyPrefix+constants.BlobsCache+":"+constants.DuplicatesBucket+":key",
-					path.Join(pathPrefix, "val")).SetVal(1)
 				mock.ExpectTxPipelineExec()
 
 				err = cacheDriver.PutBlob("key", path.Join(dir, "val"))
@@ -459,8 +458,6 @@ func TestRedisMocked(t *testing.T) {
 				mock.ExpectTxPipeline()
 				mock.ExpectHSet(keyPrefix+constants.BlobsCache+":"+constants.OriginalBucket, "key",
 					path.Join(pathPrefix, "val1")).SetVal(1)
-				mock.ExpectSAdd(keyPrefix+constants.BlobsCache+":"+constants.DuplicatesBucket+":key",
-					path.Join(pathPrefix, "val1")).SetVal(1)
 				mock.ExpectTxPipelineExec()
 
 				err = cacheDriver.PutBlob("key", path.Join(dir, "val1"))
@@ -468,10 +465,10 @@ func TestRedisMocked(t *testing.T) {
 
 				mock.Regexp().ExpectSetNX(keyPrefix+"locks:key", `.*`, 8*time.Second).SetVal(true)
 				mock.ExpectHExists(keyPrefix+constants.BlobsCache+":"+constants.OriginalBucket, "key").
-					SetVal(false)
+					SetVal(true)
+				mock.ExpectHGet(keyPrefix+constants.BlobsCache+":"+constants.OriginalBucket, "key").
+					SetVal(path.Join(pathPrefix, "val1"))
 				mock.ExpectTxPipeline()
-				mock.ExpectHSet(keyPrefix+constants.BlobsCache+":"+constants.OriginalBucket, "key",
-					path.Join(pathPrefix, "val2")).SetVal(1)
 				mock.ExpectSAdd(keyPrefix+constants.BlobsCache+":"+constants.DuplicatesBucket+":key",
 					path.Join(pathPrefix, "val2")).SetVal(1)
 				mock.ExpectTxPipelineExec()
@@ -492,7 +489,7 @@ func TestRedisMocked(t *testing.T) {
 				So(err, ShouldBeNil)
 			})
 
-			Convey("HasBlob HExists returns error"+testID, func() {
+			Convey("HasBlob HGet returns error"+testID, func() {
 				// initialize mock client
 				cacheDB, mock := redismock.NewClientMock()
 				redisDriverParams.Client = cacheDB
@@ -502,9 +499,7 @@ func TestRedisMocked(t *testing.T) {
 				So(cacheDriver, ShouldNotBeNil)
 				So(err, ShouldBeNil)
 
-				mock.ExpectSIsMember(keyPrefix+constants.BlobsCache+":"+constants.DuplicatesBucket+":key",
-					path.Join(pathPrefix, "val")).SetVal(true)
-				mock.ExpectHExists(keyPrefix+constants.BlobsCache+":"+constants.OriginalBucket, "key").
+				mock.ExpectHGet(keyPrefix+constants.BlobsCache+":"+constants.OriginalBucket, "key").
 					SetErr(ErrTestError)
 
 				ok := cacheDriver.HasBlob("key", path.Join(dir, "val"))
@@ -524,6 +519,8 @@ func TestRedisMocked(t *testing.T) {
 				So(cacheDriver, ShouldNotBeNil)
 				So(err, ShouldBeNil)
 
+				mock.ExpectHGet(keyPrefix+constants.BlobsCache+":"+constants.OriginalBucket, "key").
+					SetVal(path.Join(pathPrefix, "other"))
 				mock.ExpectSIsMember(keyPrefix+constants.BlobsCache+":"+constants.DuplicatesBucket+":key",
 					path.Join(pathPrefix, "val")).SetErr(ErrTestError)
 
@@ -534,7 +531,7 @@ func TestRedisMocked(t *testing.T) {
 				So(err, ShouldBeNil)
 			})
 
-			Convey("HasBlob HExists returns false"+testID, func() {
+			Convey("HasBlob HGet returns redis nil"+testID, func() {
 				// initialize mock client
 				cacheDB, mock := redismock.NewClientMock()
 				redisDriverParams.Client = cacheDB
@@ -544,10 +541,8 @@ func TestRedisMocked(t *testing.T) {
 				So(cacheDriver, ShouldNotBeNil)
 				So(err, ShouldBeNil)
 
-				mock.ExpectSIsMember(keyPrefix+constants.BlobsCache+":"+constants.DuplicatesBucket+":key",
-					path.Join(pathPrefix, "val")).SetVal(true)
-				mock.ExpectHExists(keyPrefix+constants.BlobsCache+":"+constants.OriginalBucket, "key").
-					SetVal(false)
+				mock.ExpectHGet(keyPrefix+constants.BlobsCache+":"+constants.OriginalBucket, "key").
+					RedisNil()
 
 				ok := cacheDriver.HasBlob("key", path.Join(dir, "val"))
 				So(ok, ShouldBeFalse)
@@ -566,31 +561,26 @@ func TestRedisMocked(t *testing.T) {
 				So(cacheDriver, ShouldNotBeNil)
 				So(err, ShouldBeNil)
 
-				// Create entry for 1st path
+				// Create origin entry for val1
 				mock.Regexp().ExpectSetNX(keyPrefix+"locks:key", `.*`, 8*time.Second).SetVal(true)
 				mock.ExpectHExists(keyPrefix+constants.BlobsCache+":"+constants.OriginalBucket, "key").
 					SetVal(false)
 				mock.ExpectTxPipeline()
 				mock.ExpectHSet(keyPrefix+constants.BlobsCache+":"+constants.OriginalBucket, "key",
 					path.Join(pathPrefix, "val1")).SetVal(1)
-				mock.ExpectSAdd(keyPrefix+constants.BlobsCache+":"+constants.DuplicatesBucket+":key",
-					path.Join(pathPrefix, "val1")).SetVal(1)
 				mock.ExpectTxPipelineExec()
 
 				err = cacheDriver.PutBlob("key", path.Join(dir, "val1"))
 				So(err, ShouldBeNil)
 
 				Convey("DeleteBlob error in HDel"+testID, func() {
-					// If the 2nd path does not exist, HDel is callled
-					// Error switching to new path
 					mock.Regexp().ExpectSetNX(keyPrefix+"locks:key", `.*`, 8*time.Second).SetVal(true)
-					mock.ExpectSRem(keyPrefix+constants.BlobsCache+":"+constants.DuplicatesBucket+":key",
-						path.Join(pathPrefix, "val1")).SetVal(1)
+					mock.ExpectSIsMember(keyPrefix+constants.BlobsCache+":"+constants.DuplicatesBucket+":key",
+						path.Join(pathPrefix, "val1")).SetVal(false)
 					mock.ExpectHGet(keyPrefix+constants.BlobsCache+":"+constants.OriginalBucket, "key").
 						SetVal(path.Join(pathPrefix, "val1"))
-					// failed to get new path
-					mock.ExpectSRandMember(keyPrefix + constants.BlobsCache + ":" + constants.DuplicatesBucket + ":key").
-						RedisNil()
+					mock.ExpectSCard(keyPrefix + constants.BlobsCache + ":" + constants.DuplicatesBucket + ":key").
+						SetVal(0)
 					mock.ExpectHDel(keyPrefix+constants.BlobsCache+":"+constants.OriginalBucket, "key").
 						SetErr(ErrTestError)
 
@@ -598,17 +588,14 @@ func TestRedisMocked(t *testing.T) {
 					So(err, ShouldEqual, ErrTestError)
 				})
 
-				Convey("DeleteBlob succeeds in deleting all data for original blob"+testID, func() {
-					// If the 2nd path does not exist, HDel is callled
-					// Error switching to new path
+				Convey("DeleteBlob succeeds in deleting original when no duplicates"+testID, func() {
 					mock.Regexp().ExpectSetNX(keyPrefix+"locks:key", `.*`, 8*time.Second).SetVal(true)
-					mock.ExpectSRem(keyPrefix+constants.BlobsCache+":"+constants.DuplicatesBucket+":key",
-						path.Join(pathPrefix, "val1")).SetVal(1)
+					mock.ExpectSIsMember(keyPrefix+constants.BlobsCache+":"+constants.DuplicatesBucket+":key",
+						path.Join(pathPrefix, "val1")).SetVal(false)
 					mock.ExpectHGet(keyPrefix+constants.BlobsCache+":"+constants.OriginalBucket, "key").
 						SetVal(path.Join(pathPrefix, "val1"))
-					// failed to get new path
-					mock.ExpectSRandMember(keyPrefix + constants.BlobsCache + ":" + constants.DuplicatesBucket + ":key").
-						RedisNil()
+					mock.ExpectSCard(keyPrefix + constants.BlobsCache + ":" + constants.DuplicatesBucket + ":key").
+						SetVal(0)
 					mock.ExpectHDel(keyPrefix+constants.BlobsCache+":"+constants.OriginalBucket, "key").
 						SetVal(1)
 
@@ -616,43 +603,27 @@ func TestRedisMocked(t *testing.T) {
 					So(err, ShouldBeNil)
 				})
 
-				Convey("DeleteBlob error in SRandMember"+testID, func() {
-					// Create entry for 2nd path
+				Convey("DeleteBlob error in SCard"+testID, func() {
 					mock.Regexp().ExpectSetNX(keyPrefix+"locks:key", `.*`, 8*time.Second).SetVal(true)
-					mock.ExpectHExists(keyPrefix+constants.BlobsCache+":"+constants.OriginalBucket, "key").
-						SetVal(false)
-					mock.ExpectTxPipeline()
-					mock.ExpectHSet(keyPrefix+constants.BlobsCache+":"+constants.OriginalBucket, "key",
-						path.Join(pathPrefix, "val2")).SetVal(1)
-					mock.ExpectSAdd(keyPrefix+constants.BlobsCache+":"+constants.DuplicatesBucket+":key",
-						path.Join(pathPrefix, "val2")).SetVal(1)
-					mock.ExpectTxPipelineExec()
-
-					err = cacheDriver.PutBlob("key", path.Join(dir, "val2"))
-					So(err, ShouldBeNil)
-
-					// Error switching to new path
-					mock.Regexp().ExpectSetNX(keyPrefix+"locks:key", `.*`, 8*time.Second).SetVal(true)
-					mock.ExpectSRem(keyPrefix+constants.BlobsCache+":"+constants.DuplicatesBucket+":key",
-						path.Join(pathPrefix, "val1")).SetVal(1)
+					mock.ExpectSIsMember(keyPrefix+constants.BlobsCache+":"+constants.DuplicatesBucket+":key",
+						path.Join(pathPrefix, "val1")).SetVal(false)
 					mock.ExpectHGet(keyPrefix+constants.BlobsCache+":"+constants.OriginalBucket, "key").
 						SetVal(path.Join(pathPrefix, "val1"))
-					// failed to get new path
-					mock.ExpectSRandMember(keyPrefix + constants.BlobsCache + ":" + constants.DuplicatesBucket + ":key").
+					mock.ExpectSCard(keyPrefix + constants.BlobsCache + ":" + constants.DuplicatesBucket + ":key").
 						SetErr(ErrTestError)
 
 					err = cacheDriver.DeleteBlob("key", path.Join(dir, "val1"))
 					So(err, ShouldEqual, ErrTestError)
 				})
 
-				Convey("DeleteBlob error in HSet"+testID, func() {
-					// Create entry for 2nd path
+				Convey("DeleteBlob keeps original when duplicates exist"+testID, func() {
+					// Add duplicate val2
 					mock.Regexp().ExpectSetNX(keyPrefix+"locks:key", `.*`, 8*time.Second).SetVal(true)
 					mock.ExpectHExists(keyPrefix+constants.BlobsCache+":"+constants.OriginalBucket, "key").
-						SetVal(false)
+						SetVal(true)
+					mock.ExpectHGet(keyPrefix+constants.BlobsCache+":"+constants.OriginalBucket, "key").
+						SetVal(path.Join(pathPrefix, "val1"))
 					mock.ExpectTxPipeline()
-					mock.ExpectHSet(keyPrefix+constants.BlobsCache+":"+constants.OriginalBucket, "key",
-						path.Join(pathPrefix, "val2")).SetVal(1)
 					mock.ExpectSAdd(keyPrefix+constants.BlobsCache+":"+constants.DuplicatesBucket+":key",
 						path.Join(pathPrefix, "val2")).SetVal(1)
 					mock.ExpectTxPipelineExec()
@@ -660,45 +631,14 @@ func TestRedisMocked(t *testing.T) {
 					err = cacheDriver.PutBlob("key", path.Join(dir, "val2"))
 					So(err, ShouldBeNil)
 
-					// Error switching to new path
+					// delete original val1, keep as long as duplicates exist
 					mock.Regexp().ExpectSetNX(keyPrefix+"locks:key", `.*`, 8*time.Second).SetVal(true)
-					mock.ExpectSRem(keyPrefix+constants.BlobsCache+":"+constants.DuplicatesBucket+":key",
-						path.Join(pathPrefix, "val1")).SetVal(1)
+					mock.ExpectSIsMember(keyPrefix+constants.BlobsCache+":"+constants.DuplicatesBucket+":key",
+						path.Join(pathPrefix, "val1")).SetVal(false)
 					mock.ExpectHGet(keyPrefix+constants.BlobsCache+":"+constants.OriginalBucket, "key").
 						SetVal(path.Join(pathPrefix, "val1"))
-					mock.ExpectSRandMember(keyPrefix + constants.BlobsCache + ":" + constants.DuplicatesBucket + ":key").
-						SetVal(path.Join(pathPrefix, "val2"))
-					mock.ExpectHSet(keyPrefix+constants.BlobsCache+":"+constants.OriginalBucket, "key",
-						path.Join(pathPrefix, "val2")).SetErr(ErrTestError)
-
-					err = cacheDriver.DeleteBlob("key", path.Join(dir, "val1"))
-					So(err, ShouldEqual, ErrTestError)
-				})
-
-				Convey("DeleteBlob succeeds in switching original blob path"+testID, func() {
-					// Create entry for 2nd path
-					mock.Regexp().ExpectSetNX(keyPrefix+"locks:key", `.*`, 8*time.Second).SetVal(true)
-					mock.ExpectHExists(keyPrefix+constants.BlobsCache+":"+constants.OriginalBucket, "key").
-						SetVal(false)
-					mock.ExpectTxPipeline()
-					mock.ExpectHSet(keyPrefix+constants.BlobsCache+":"+constants.OriginalBucket, "key",
-						path.Join(pathPrefix, "val2")).SetVal(1)
-					mock.ExpectSAdd(keyPrefix+constants.BlobsCache+":"+constants.DuplicatesBucket+":key",
-						path.Join(pathPrefix, "val2")).SetVal(1)
-					mock.ExpectTxPipelineExec()
-
-					err = cacheDriver.PutBlob("key", path.Join(dir, "val2"))
-					So(err, ShouldBeNil)
-
-					mock.Regexp().ExpectSetNX(keyPrefix+"locks:key", `.*`, 8*time.Second).SetVal(true)
-					mock.ExpectSRem(keyPrefix+constants.BlobsCache+":"+constants.DuplicatesBucket+":key",
-						path.Join(pathPrefix, "val1")).SetVal(1)
-					mock.ExpectHGet(keyPrefix+constants.BlobsCache+":"+constants.OriginalBucket, "key").
-						SetVal(path.Join(pathPrefix, "val1"))
-					mock.ExpectSRandMember(keyPrefix + constants.BlobsCache + ":" + constants.DuplicatesBucket + ":key").
-						SetVal(path.Join(pathPrefix, "val2"))
-					mock.ExpectHSet(keyPrefix+constants.BlobsCache+":"+constants.OriginalBucket, "key",
-						path.Join(pathPrefix, "val2")).SetVal(1)
+					mock.ExpectSCard(keyPrefix + constants.BlobsCache + ":" + constants.DuplicatesBucket + ":key").
+						SetVal(1)
 
 					err = cacheDriver.DeleteBlob("key", path.Join(dir, "val1"))
 					So(err, ShouldBeNil)