refactor(storage): refactor storage into a single ImageStore (#1656)

unified both local and s3 ImageStore logic into a single ImageStore
added a new driver interface for common file/dirs manipulations
to be implemented by different storage types

refactor(gc): drop umoci dependency, implemented internal gc

added retentionDelay config option that specifies
the garbage collect delay for images without tags

this will also clean manifests which are part of an index image
(multiarch) that no longer exist.

fix(dedupe): skip blobs under .sync/ directory

if startup dedupe is running while also syncing is running
ignore blobs under sync's temporary storage

fix(storage): do not allow image indexes modifications

when deleting a manifest verify that it is not part of a multiarch image
and throw a MethodNotAllowed error to the client if it is.
we don't want to modify multiarch images

Signed-off-by: Petu Eusebiu <peusebiu@cisco.com>

pkg/test/common.go

@@ -2068,7 +2068,7 @@ func GetDefaultLayersBlobs() [][]byte {
 }
 
 func GetDefaultImageStore(rootDir string, log zLog.Logger) stypes.ImageStore {
-	return local.NewImageStore(rootDir, false, time.Hour, false, false, log,
+	return local.NewImageStore(rootDir, false, false, time.Hour, time.Hour, false, false, log,
 		monitoring.NewMetricsServer(false, log),
 		mocks.MockedLint{
 			LintFn: func(repo string, manifestDigest godigest.Digest, imageStore stypes.ImageStore) (bool, error) {

pkg/test/mocks/cache_mock.go

@@ -17,6 +17,16 @@ type CacheMock struct {
 
 	// Delete a blob from the cachedb.
 	DeleteBlobFn func(digest godigest.Digest, path string) error
+
+	UsesRelativePathsFn func() bool
+}
+
+func (cacheMock CacheMock) UsesRelativePaths() bool {
+	if cacheMock.UsesRelativePathsFn != nil {
+		return cacheMock.UsesRelativePaths()
+	}
+
+	return false
 }
 
 func (cacheMock CacheMock) Name() string {

pkg/test/mocks/image_store_mock.go

@@ -35,7 +35,7 @@ type MockedImageStore struct {
 	DeleteBlobUploadFn     func(repo string, uuid string) error
 	BlobPathFn             func(repo string, digest godigest.Digest) string
 	CheckBlobFn            func(repo string, digest godigest.Digest) (bool, int64, error)
-	StatBlobFn             func(repo string, digest godigest.Digest) (bool, int64, error)
+	StatBlobFn             func(repo string, digest godigest.Digest) (bool, int64, time.Time, error)
 	GetBlobPartialFn       func(repo string, digest godigest.Digest, mediaType string, from, to int64,
 	) (io.ReadCloser, int64, int64, error)
 	GetBlobFn          func(repo string, digest godigest.Digest, mediaType string) (io.ReadCloser, int64, error)
@@ -51,6 +51,7 @@ type MockedImageStore struct {
 	RunDedupeBlobsFn             func(interval time.Duration, sch *scheduler.Scheduler)
 	RunDedupeForDigestFn         func(digest godigest.Digest, dedupe bool, duplicateBlobs []string) error
 	GetNextDigestWithBlobPathsFn func(lastDigests []godigest.Digest) (godigest.Digest, []string, error)
+	GetAllBlobsFn                func(repo string) ([]string, error)
 }
 
 func (is MockedImageStore) Lock(t *time.Time) {
@@ -142,6 +143,14 @@ func (is MockedImageStore) GetImageTags(name string) ([]string, error) {
 	return []string{}, nil
 }
 
+func (is MockedImageStore) GetAllBlobs(repo string) ([]string, error) {
+	if is.GetAllBlobsFn != nil {
+		return is.GetAllBlobsFn(repo)
+	}
+
+	return []string{}, nil
+}
+
 func (is MockedImageStore) DeleteImageManifest(name string, reference string, detectCollision bool) error {
 	if is.DeleteImageManifestFn != nil {
 		return is.DeleteImageManifestFn(name, reference, detectCollision)
@@ -252,12 +261,12 @@ func (is MockedImageStore) CheckBlob(repo string, digest godigest.Digest) (bool,
 	return true, 0, nil
 }
 
-func (is MockedImageStore) StatBlob(repo string, digest godigest.Digest) (bool, int64, error) {
+func (is MockedImageStore) StatBlob(repo string, digest godigest.Digest) (bool, int64, time.Time, error) {
 	if is.StatBlobFn != nil {
 		return is.StatBlobFn(repo, digest)
 	}
 
-	return true, 0, nil
+	return true, 0, time.Time{}, nil
 }
 
 func (is MockedImageStore) GetBlobPartial(repo string, digest godigest.Digest, mediaType string, from, to int64,

pkg/test/oci-layout/oci_layout_test.go

@@ -346,7 +346,7 @@ func TestExtractImageDetails(t *testing.T) {
 	Convey("extractImageDetails good workflow", t, func() {
 		dir := t.TempDir()
 		testLogger := log.NewLogger("debug", "")
-		imageStore := local.NewImageStore(dir, false, 0, false, false,
+		imageStore := local.NewImageStore(dir, false, false, 0, 0, false, false,
 			testLogger, monitoring.NewMetricsServer(false, testLogger), nil, nil)
 
 		storeController := storage.StoreController{
@@ -382,7 +382,7 @@ func TestExtractImageDetails(t *testing.T) {
 	Convey("extractImageDetails bad ispec.ImageManifest", t, func() {
 		dir := t.TempDir()
 		testLogger := log.NewLogger("debug", "")
-		imageStore := local.NewImageStore(dir, false, 0, false, false,
+		imageStore := local.NewImageStore(dir, false, false, 0, 0, false, false,
 			testLogger, monitoring.NewMetricsServer(false, testLogger), nil, nil)
 
 		storeController := storage.StoreController{
@@ -402,7 +402,7 @@ func TestExtractImageDetails(t *testing.T) {
 	Convey("extractImageDetails bad imageConfig", t, func() {
 		dir := t.TempDir()
 		testLogger := log.NewLogger("debug", "")
-		imageStore := local.NewImageStore(dir, false, 0, false, false,
+		imageStore := local.NewImageStore(dir, false, false, 0, 0, false, false,
 			testLogger, monitoring.NewMetricsServer(false, testLogger), nil, nil)
 
 		storeController := storage.StoreController{