chore(trivy): update trivy version and enforce OCI compliant repo names in local image storage (#1068)

1. chore(trivy): update trivy library version

The trivy team switched github.com/urfave/cli for viper so
there are some other code changes as well.

Since we don't use github.com/urfave/cli directly in our software
we needed to add a tools.go in order for "go mod tidy" to not delete it.
See this pattern explained in:
- https://github.com/99designs/gqlgen#quick-start
- https://github.com/golang/go/wiki/Modules#how-can-i-track-tool-dependencies-for-a-module
- https://github.com/go-modules-by-example/index/blob/master/010_tools/README.md#walk-through

The jobs using "go get -u" have been updated to use "go install", since go get
modifies the go.mod by upgrading some of the packages, but downgrading trivy to an older
version with broken dependencies

2. fix(storage) Update local storage to ignore folder names not compliant with dist spec
Also updated trivy to download the DB and cache results under the rootDir/_trivy folder

3. fix(s3): one of the s3 tests was missing the skipIt call
This caused a failure when running locally without s3 being available

4. make sure the offline scanning is enabled, and zot only downloads the trivy DB
on the regular schedule, and doesn't download the DB on every image scan

ci: increase build and test timeout as tests are reaching the limit more often

Signed-off-by: Andrei Aaron <aaaron@luxoft.com>

pkg/api/controller_test.go

@@ -6117,7 +6117,7 @@ func TestInjectTooManyOpenFiles(t *testing.T) {
 
 func TestGCSignaturesAndUntaggedManifests(t *testing.T) {
 	Convey("Make controller", t, func() {
-		repoName := "testrepo"
+		repoName := "testrepo" //nolint:goconst
 		tag := "0.0.1"
 
 		port := test.GetFreePort()
@@ -6369,7 +6369,7 @@ func TestGCSignaturesAndUntaggedManifests(t *testing.T) {
 
 func TestPeriodicGC(t *testing.T) {
 	Convey("Periodic gc enabled for default store", t, func() {
-		repoName := "testRepo"
+		repoName := "testrepo" //nolint:goconst
 
 		port := test.GetFreePort()
 		conf := config.New()
@@ -6445,7 +6445,7 @@ func TestPeriodicGC(t *testing.T) {
 	})
 
 	Convey("Periodic gc error", t, func() {
-		repoName := "testRepo"
+		repoName := "testrepo" //nolint:goconst
 
 		port := test.GetFreePort()
 		conf := config.New()
@@ -6505,7 +6505,7 @@ func TestSearchRoutes(t *testing.T) {
 		cm.StartAndWait(port)
 		defer cm.StopServer()
 
-		repoName := "testrepo"
+		repoName := "testrepo" //nolint:goconst
 		inaccessibleRepo := "inaccessible"
 
 		cfg, layers, manifest, err := test.GetImageComponents(10000)

pkg/api/regexp.go

@@ -1,74 +0,0 @@
-package api
-
-import "regexp"
-
-//nolint:gochecknoglobals
-var (
-	// alphaNumericRegexp defines the alpha numeric atom, typically a
-	// component of names. This only allows lower case characters and digits.
-	alphaNumericRegexp = match(`[a-z0-9]+`)
-
-	// separatorRegexp defines the separators allowed to be embedded in name
-	// components. This allow one period, one or two underscore and multiple
-	// dashes.
-	separatorRegexp = match(`(?:[._]|__|[-]*)`)
-
-	// nameComponentRegexp restricts registry path component names to start
-	// with at least one letter or number, with following parts able to be
-	// separated by one period, one or two underscore and multiple dashes.
-	nameComponentRegexp = expression(
-		alphaNumericRegexp,
-		optional(repeated(separatorRegexp, alphaNumericRegexp)))
-
-	// NameRegexp is the format for the name component of references. The
-	// regexp has capturing groups for the domain and name part omitting
-	// the separating forward slash from either.
-	NameRegexp = expression(
-		nameComponentRegexp,
-		optional(repeated(literal(`/`), nameComponentRegexp)))
-)
-
-// match compiles the string to a regular expression.
-//
-//nolint:gochecknoglobals
-var match = regexp.MustCompile
-
-// literal compiles s into a literal regular expression, escaping any regexp
-// reserved characters.
-func literal(s string) *regexp.Regexp {
-	regx := match(regexp.QuoteMeta(s))
-
-	if _, complete := regx.LiteralPrefix(); !complete {
-		panic("must be a literal")
-	}
-
-	return regx
-}
-
-// expression defines a full expression, where each regular expression must
-// follow the previous.
-func expression(res ...*regexp.Regexp) *regexp.Regexp {
-	var s string
-	for _, re := range res {
-		s += re.String()
-	}
-
-	return match(s)
-}
-
-// optional wraps the expression in a non-capturing group and makes the
-// production optional.
-func optional(res ...*regexp.Regexp) *regexp.Regexp {
-	return match(group(expression(res...)).String() + `?`)
-}
-
-// repeated wraps the regexp in a non-capturing group to get one or more
-// matches.
-func repeated(res ...*regexp.Regexp) *regexp.Regexp {
-	return match(group(expression(res...)).String() + `+`)
-}
-
-// group wraps the regexp in a non-capturing group.
-func group(res ...*regexp.Regexp) *regexp.Regexp {
-	return match(`(?:` + expression(res...).String() + `)`)
-}

pkg/api/routes.go

@@ -36,6 +36,7 @@ import (
 	"zotregistry.io/zot/pkg/extensions/sync"
 	"zotregistry.io/zot/pkg/log"
 	repoDBUpdate "zotregistry.io/zot/pkg/meta/repodb/update"
+	zreg "zotregistry.io/zot/pkg/regexp"
 	localCtx "zotregistry.io/zot/pkg/requestcontext"
 	"zotregistry.io/zot/pkg/storage"
 	"zotregistry.io/zot/pkg/test" //nolint:goimports
@@ -73,34 +74,34 @@ func (rh *RouteHandler) SetupRoutes() {
 	// https://github.com/opencontainers/distribution-spec/blob/main/spec.md#endpoints
 	prefixedRouter := rh.c.Router.PathPrefix(constants.RoutePrefix).Subrouter()
 	{
-		prefixedRouter.HandleFunc(fmt.Sprintf("/{name:%s}/tags/list", NameRegexp.String()),
+		prefixedRouter.HandleFunc(fmt.Sprintf("/{name:%s}/tags/list", zreg.NameRegexp.String()),
 			rh.ListTags).Methods(allowedMethods("GET")...)
-		prefixedRouter.HandleFunc(fmt.Sprintf("/{name:%s}/manifests/{reference}", NameRegexp.String()),
+		prefixedRouter.HandleFunc(fmt.Sprintf("/{name:%s}/manifests/{reference}", zreg.NameRegexp.String()),
 			rh.CheckManifest).Methods(allowedMethods("HEAD")...)
-		prefixedRouter.HandleFunc(fmt.Sprintf("/{name:%s}/manifests/{reference}", NameRegexp.String()),
+		prefixedRouter.HandleFunc(fmt.Sprintf("/{name:%s}/manifests/{reference}", zreg.NameRegexp.String()),
 			rh.GetManifest).Methods(allowedMethods("GET")...)
-		prefixedRouter.HandleFunc(fmt.Sprintf("/{name:%s}/manifests/{reference}", NameRegexp.String()),
+		prefixedRouter.HandleFunc(fmt.Sprintf("/{name:%s}/manifests/{reference}", zreg.NameRegexp.String()),
 			rh.UpdateManifest).Methods("PUT")
-		prefixedRouter.HandleFunc(fmt.Sprintf("/{name:%s}/manifests/{reference}", NameRegexp.String()),
+		prefixedRouter.HandleFunc(fmt.Sprintf("/{name:%s}/manifests/{reference}", zreg.NameRegexp.String()),
 			rh.DeleteManifest).Methods("DELETE")
-		prefixedRouter.HandleFunc(fmt.Sprintf("/{name:%s}/blobs/{digest}", NameRegexp.String()),
+		prefixedRouter.HandleFunc(fmt.Sprintf("/{name:%s}/blobs/{digest}", zreg.NameRegexp.String()),
 			rh.CheckBlob).Methods("HEAD")
-		prefixedRouter.HandleFunc(fmt.Sprintf("/{name:%s}/blobs/{digest}", NameRegexp.String()),
+		prefixedRouter.HandleFunc(fmt.Sprintf("/{name:%s}/blobs/{digest}", zreg.NameRegexp.String()),
 			rh.GetBlob).Methods("GET")
-		prefixedRouter.HandleFunc(fmt.Sprintf("/{name:%s}/blobs/{digest}", NameRegexp.String()),
+		prefixedRouter.HandleFunc(fmt.Sprintf("/{name:%s}/blobs/{digest}", zreg.NameRegexp.String()),
 			rh.DeleteBlob).Methods("DELETE")
-		prefixedRouter.HandleFunc(fmt.Sprintf("/{name:%s}/blobs/uploads/", NameRegexp.String()),
+		prefixedRouter.HandleFunc(fmt.Sprintf("/{name:%s}/blobs/uploads/", zreg.NameRegexp.String()),
 			rh.CreateBlobUpload).Methods("POST")
-		prefixedRouter.HandleFunc(fmt.Sprintf("/{name:%s}/blobs/uploads/{session_id}", NameRegexp.String()),
+		prefixedRouter.HandleFunc(fmt.Sprintf("/{name:%s}/blobs/uploads/{session_id}", zreg.NameRegexp.String()),
 			rh.GetBlobUpload).Methods("GET")
-		prefixedRouter.HandleFunc(fmt.Sprintf("/{name:%s}/blobs/uploads/{session_id}", NameRegexp.String()),
+		prefixedRouter.HandleFunc(fmt.Sprintf("/{name:%s}/blobs/uploads/{session_id}", zreg.NameRegexp.String()),
 			rh.PatchBlobUpload).Methods("PATCH")
-		prefixedRouter.HandleFunc(fmt.Sprintf("/{name:%s}/blobs/uploads/{session_id}", NameRegexp.String()),
+		prefixedRouter.HandleFunc(fmt.Sprintf("/{name:%s}/blobs/uploads/{session_id}", zreg.NameRegexp.String()),
 			rh.UpdateBlobUpload).Methods("PUT")
-		prefixedRouter.HandleFunc(fmt.Sprintf("/{name:%s}/blobs/uploads/{session_id}", NameRegexp.String()),
+		prefixedRouter.HandleFunc(fmt.Sprintf("/{name:%s}/blobs/uploads/{session_id}", zreg.NameRegexp.String()),
 			rh.DeleteBlobUpload).Methods("DELETE")
 		// support for OCI artifact references
-		prefixedRouter.HandleFunc(fmt.Sprintf("/{name:%s}/referrers/{digest}", NameRegexp.String()),
+		prefixedRouter.HandleFunc(fmt.Sprintf("/{name:%s}/referrers/{digest}", zreg.NameRegexp.String()),
 			rh.GetReferrers).Methods(allowedMethods("GET")...)
 		prefixedRouter.HandleFunc(constants.ExtCatalogPrefix,
 			rh.ListRepositories).Methods(allowedMethods("GET")...)
@@ -112,7 +113,7 @@ func (rh *RouteHandler) SetupRoutes() {
 
 	// support for ORAS artifact reference types (alpha 1) - image signature use case
 	rh.c.Router.HandleFunc(fmt.Sprintf("%s/{name:%s}/manifests/{digest}/referrers",
-		constants.ArtifactSpecRoutePrefix, NameRegexp.String()), rh.GetOrasReferrers).Methods("GET")
+		constants.ArtifactSpecRoutePrefix, zreg.NameRegexp.String()), rh.GetOrasReferrers).Methods("GET")
 
 	// swagger
 	debug.SetupSwaggerRoutes(rh.c.Config, rh.c.Router, rh.c.Log)