Catalog content discovery (#2782)

fix(sync): use pagination when querying remote catalog

feat(api): added /v2/_catalog pagination, fixes #2715

Signed-off-by: Eusebiu Petu <petu.eusebiu@gmail.com>

pkg/storage/imagestore/imagestore.go

@@ -265,6 +265,89 @@ func (is *ImageStore) ValidateRepo(name string) (bool, error) {
 	return true, nil
 }
 
+func (is *ImageStore) GetNextRepositories(lastRepo string, maxEntries int, filterFn storageTypes.FilterRepoFunc,
+) ([]string, bool, error) {
+	var lockLatency time.Time
+
+	dir := is.rootDir
+
+	is.RLock(&lockLatency)
+	defer is.RUnlock(&lockLatency)
+
+	stores := make([]string, 0)
+
+	moreEntries := false
+	entries := 0
+	found := false
+	err := is.storeDriver.Walk(dir, func(fileInfo driver.FileInfo) error {
+		if entries == maxEntries {
+			moreEntries = true
+
+			return io.EOF
+		}
+
+		if !fileInfo.IsDir() {
+			return nil
+		}
+
+		// skip .sync and .uploads dirs no need to try to validate them
+		if strings.HasSuffix(fileInfo.Path(), syncConstants.SyncBlobUploadDir) ||
+			strings.HasSuffix(fileInfo.Path(), ispec.ImageBlobsDir) ||
+			strings.HasSuffix(fileInfo.Path(), storageConstants.BlobUploadDir) {
+			return driver.ErrSkipDir
+		}
+
+		rel, err := filepath.Rel(is.rootDir, fileInfo.Path())
+		if err != nil {
+			return nil //nolint:nilerr // ignore paths that are not under root dir
+		}
+
+		if ok, err := is.ValidateRepo(rel); !ok || err != nil {
+			return nil //nolint:nilerr // ignore invalid repos
+		}
+
+		if lastRepo == rel {
+			found = true
+
+			return nil
+		}
+
+		if lastRepo == "" {
+			found = true
+		}
+
+		ok, err := filterFn(rel)
+		if err != nil {
+			return err
+		}
+
+		if found && ok {
+			entries++
+
+			stores = append(stores, rel)
+		}
+
+		return nil
+	})
+
+	// if the root directory is not yet created then return an empty slice of repositories
+
+	driverErr := &driver.Error{}
+
+	if errors.As(err, &driver.PathNotFoundError{}) {
+		is.log.Debug().Msg("empty rootDir")
+
+		return stores, false, nil
+	}
+
+	if errors.Is(err, io.EOF) ||
+		(errors.As(err, driverErr) && errors.Is(driverErr.Detail, io.EOF)) {
+		return stores, moreEntries, nil
+	}
+
+	return stores, moreEntries, err
+}
+
 // GetRepositories returns a list of all the repositories under this store.
 func (is *ImageStore) GetRepositories() ([]string, error) {
 	var lockLatency time.Time

pkg/storage/storage_controller.go

@@ -7,8 +7,9 @@ import (
 )
 
 const (
-	CosignType   = "cosign"
-	NotationType = "notation"
+	CosignType       = "cosign"
+	NotationType     = "notation"
+	DefaultStorePath = "/"
 )
 
 type StoreController struct {
@@ -29,6 +30,21 @@ func GetRoutePrefix(name string) string {
 	return "/" + names[0]
 }
 
+func (sc StoreController) GetStorePath(name string) string {
+	if sc.SubStore != nil && name != "" {
+		subStorePath := GetRoutePrefix(name)
+
+		_, ok := sc.SubStore[subStorePath]
+		if !ok {
+			return DefaultStorePath
+		}
+
+		return subStorePath
+	}
+
+	return DefaultStorePath
+}
+
 func (sc StoreController) GetImageStore(name string) storageTypes.ImageStore {
 	if sc.SubStore != nil {
 		// SubStore is being provided, now we need to find equivalent image store and this will be found by splitting name

pkg/storage/storage_test.go

@@ -283,6 +283,14 @@ func TestStorageAPIs(t *testing.T) {
 					repos, err := imgStore.GetRepositories()
 					So(err, ShouldBeNil)
 					So(repos, ShouldNotBeEmpty)
+
+					repos, more, err := imgStore.GetNextRepositories("", -1, func(repo string) (bool, error) {
+						return true, nil
+					})
+
+					So(more, ShouldBeFalse)
+					So(err, ShouldBeNil)
+					So(repos, ShouldNotBeEmpty)
 				})
 
 				Convey("Get image tags", func() {
@@ -564,6 +572,21 @@ func TestStorageAPIs(t *testing.T) {
 							So(len(repos), ShouldEqual, 1)
 							So(repos[0], ShouldEqual, "test")
 
+							repos, more, err := imgStore.GetNextRepositories("", -1, func(repo string) (bool, error) {
+								return true, nil
+							})
+							So(err, ShouldBeNil)
+							So(more, ShouldBeFalse)
+							So(len(repos), ShouldEqual, 1)
+							So(repos[0], ShouldEqual, "test")
+
+							repos, more, err = imgStore.GetNextRepositories("", -1, func(repo string) (bool, error) {
+								return false, nil
+							})
+							So(err, ShouldBeNil)
+							So(more, ShouldBeFalse)
+							So(len(repos), ShouldEqual, 0)
+
 							// We deleted only one tag, make sure blob should not be removed.
 							hasBlob, _, err = imgStore.CheckBlob("test", digest)
 							So(err, ShouldBeNil)

pkg/storage/types/types.go

@@ -12,6 +12,8 @@ import (
 	"zotregistry.dev/zot/pkg/scheduler"
 )
 
+type FilterRepoFunc func(repo string) (bool, error)
+
 type StoreController interface {
 	GetImageStore(name string) ImageStore
 	GetDefaultImageStore() ImageStore
@@ -30,6 +32,7 @@ type ImageStore interface { //nolint:interfacebloat
 	ValidateRepo(name string) (bool, error)
 	GetRepositories() ([]string, error)
 	GetNextRepository(repo string) (string, error)
+	GetNextRepositories(repo string, maxEntries int, fn FilterRepoFunc) ([]string, bool, error)
 	GetImageTags(repo string) ([]string, error)
 	GetImageManifest(repo, reference string) ([]byte, godigest.Digest, string, error)
 	PutImageManifest(repo, reference, mediaType string, body []byte) (godigest.Digest, godigest.Digest, error)