dedupe: use hard links to dedupe blobs

As the number of repos and layers increases, the greater the probability
that layers are duplicated. We dedupe using hard links when content is
the same. This is intended to be purely a storage layer optimization.
Access control when available is orthogonal this optimization.

Add a durable cache to help speed up layer lookups.

Update README.

Add more unit tests.

pkg/api/controller.go

@@ -7,6 +7,7 @@ import (
 	"io/ioutil"
 	"net"
 	"net/http"
+	"os"
 
 	"github.com/anuvu/zot/errors"
 	"github.com/anuvu/zot/pkg/log"
@@ -41,12 +42,16 @@ func (c *Controller) Run() error {
 	engine.Use(log.SessionLogger(c.Log), handlers.RecoveryHandler(handlers.RecoveryLogger(c.Log),
 		handlers.PrintRecoveryStack(false)))
 
+	c.ImageStore = storage.NewImageStore(c.Config.Storage.RootDirectory, c.Log)
+	if c.ImageStore == nil {
+		// we can't proceed without at least a image store
+		os.Exit(1)
+	}
+
 	c.Router = engine
 	c.Router.UseEncodedPath()
 	_ = NewRouteHandler(c)
 
-	c.ImageStore = storage.NewImageStore(c.Config.Storage.RootDirectory, c.Log)
-
 	addr := fmt.Sprintf("%s:%s", c.Config.HTTP.Address, c.Config.HTTP.Port)
 	server := &http.Server{Addr: addr, Handler: c.Router}
 	c.Server = server

pkg/api/routes.go

@@ -20,7 +20,6 @@ import (
 	"sort"
 	"strconv"
 	"strings"
-	"sync"
 
 	_ "github.com/anuvu/zot/docs" // nolint (golint) - as required by swaggo
 	"github.com/anuvu/zot/errors"
@@ -41,12 +40,11 @@ const (
 )
 
 type RouteHandler struct {
-	c        *Controller
-	blobLock sync.RWMutex
+	c *Controller
 }
 
 func NewRouteHandler(c *Controller) *RouteHandler {
-	rh := &RouteHandler{c: c, blobLock: sync.RWMutex{}}
+	rh := &RouteHandler{c: c}
 	rh.SetupRoutes()
 
 	return rh
@@ -56,9 +54,9 @@ func NewRouteHandler(c *Controller) *RouteHandler {
 func (rh *RouteHandler) blobRLockWrapper(f func(w http.ResponseWriter,
 	r *http.Request)) func(w http.ResponseWriter, r *http.Request) {
 	return func(w http.ResponseWriter, r *http.Request) {
-		rh.blobLock.RLock()
+		rh.c.ImageStore.RLock()
 		f(w, r)
-		rh.blobLock.RUnlock()
+		rh.c.ImageStore.RUnlock()
 	}
 }
 
@@ -66,9 +64,9 @@ func (rh *RouteHandler) blobRLockWrapper(f func(w http.ResponseWriter,
 func (rh *RouteHandler) blobLockWrapper(f func(w http.ResponseWriter,
 	r *http.Request)) func(w http.ResponseWriter, r *http.Request) {
 	return func(w http.ResponseWriter, r *http.Request) {
-		rh.blobLock.Lock()
+		rh.c.ImageStore.Lock()
 		f(w, r)
-		rh.blobLock.Unlock()
+		rh.c.ImageStore.Unlock()
 	}
 }