chore(trivy): update trivy version and enforce OCI compliant repo names in local image storage (#1068)

1. chore(trivy): update trivy library version

The trivy team switched github.com/urfave/cli for viper so
there are some other code changes as well.

Since we don't use github.com/urfave/cli directly in our software
we needed to add a tools.go in order for "go mod tidy" to not delete it.
See this pattern explained in:
- https://github.com/99designs/gqlgen#quick-start
- https://github.com/golang/go/wiki/Modules#how-can-i-track-tool-dependencies-for-a-module
- https://github.com/go-modules-by-example/index/blob/master/010_tools/README.md#walk-through

The jobs using "go get -u" have been updated to use "go install", since go get
modifies the go.mod by upgrading some of the packages, but downgrading trivy to an older
version with broken dependencies

2. fix(storage) Update local storage to ignore folder names not compliant with dist spec
Also updated trivy to download the DB and cache results under the rootDir/_trivy folder

3. fix(s3): one of the s3 tests was missing the skipIt call
This caused a failure when running locally without s3 being available

4. make sure the offline scanning is enabled, and zot only downloads the trivy DB
on the regular schedule, and doesn't download the DB on every image scan

ci: increase build and test timeout as tests are reaching the limit more often

Signed-off-by: Andrei Aaron <aaaron@luxoft.com>
This commit is contained in:
Andrei Aaron
2023-01-18 18:24:44 +02:00
committed by GitHub
parent 9294ebb0ec
commit fac1d1d05d
20 changed files with 968 additions and 796 deletions
+4 -4
View File
@@ -6117,7 +6117,7 @@ func TestInjectTooManyOpenFiles(t *testing.T) {
func TestGCSignaturesAndUntaggedManifests(t *testing.T) {
Convey("Make controller", t, func() {
repoName := "testrepo"
repoName := "testrepo" //nolint:goconst
tag := "0.0.1"
port := test.GetFreePort()
@@ -6369,7 +6369,7 @@ func TestGCSignaturesAndUntaggedManifests(t *testing.T) {
func TestPeriodicGC(t *testing.T) {
Convey("Periodic gc enabled for default store", t, func() {
repoName := "testRepo"
repoName := "testrepo" //nolint:goconst
port := test.GetFreePort()
conf := config.New()
@@ -6445,7 +6445,7 @@ func TestPeriodicGC(t *testing.T) {
})
Convey("Periodic gc error", t, func() {
repoName := "testRepo"
repoName := "testrepo" //nolint:goconst
port := test.GetFreePort()
conf := config.New()
@@ -6505,7 +6505,7 @@ func TestSearchRoutes(t *testing.T) {
cm.StartAndWait(port)
defer cm.StopServer()
repoName := "testrepo"
repoName := "testrepo" //nolint:goconst
inaccessibleRepo := "inaccessible"
cfg, layers, manifest, err := test.GetImageComponents(10000)
+16 -15
View File
@@ -36,6 +36,7 @@ import (
"zotregistry.io/zot/pkg/extensions/sync"
"zotregistry.io/zot/pkg/log"
repoDBUpdate "zotregistry.io/zot/pkg/meta/repodb/update"
zreg "zotregistry.io/zot/pkg/regexp"
localCtx "zotregistry.io/zot/pkg/requestcontext"
"zotregistry.io/zot/pkg/storage"
"zotregistry.io/zot/pkg/test" //nolint:goimports
@@ -73,34 +74,34 @@ func (rh *RouteHandler) SetupRoutes() {
// https://github.com/opencontainers/distribution-spec/blob/main/spec.md#endpoints
prefixedRouter := rh.c.Router.PathPrefix(constants.RoutePrefix).Subrouter()
{
prefixedRouter.HandleFunc(fmt.Sprintf("/{name:%s}/tags/list", NameRegexp.String()),
prefixedRouter.HandleFunc(fmt.Sprintf("/{name:%s}/tags/list", zreg.NameRegexp.String()),
rh.ListTags).Methods(allowedMethods("GET")...)
prefixedRouter.HandleFunc(fmt.Sprintf("/{name:%s}/manifests/{reference}", NameRegexp.String()),
prefixedRouter.HandleFunc(fmt.Sprintf("/{name:%s}/manifests/{reference}", zreg.NameRegexp.String()),
rh.CheckManifest).Methods(allowedMethods("HEAD")...)
prefixedRouter.HandleFunc(fmt.Sprintf("/{name:%s}/manifests/{reference}", NameRegexp.String()),
prefixedRouter.HandleFunc(fmt.Sprintf("/{name:%s}/manifests/{reference}", zreg.NameRegexp.String()),
rh.GetManifest).Methods(allowedMethods("GET")...)
prefixedRouter.HandleFunc(fmt.Sprintf("/{name:%s}/manifests/{reference}", NameRegexp.String()),
prefixedRouter.HandleFunc(fmt.Sprintf("/{name:%s}/manifests/{reference}", zreg.NameRegexp.String()),
rh.UpdateManifest).Methods("PUT")
prefixedRouter.HandleFunc(fmt.Sprintf("/{name:%s}/manifests/{reference}", NameRegexp.String()),
prefixedRouter.HandleFunc(fmt.Sprintf("/{name:%s}/manifests/{reference}", zreg.NameRegexp.String()),
rh.DeleteManifest).Methods("DELETE")
prefixedRouter.HandleFunc(fmt.Sprintf("/{name:%s}/blobs/{digest}", NameRegexp.String()),
prefixedRouter.HandleFunc(fmt.Sprintf("/{name:%s}/blobs/{digest}", zreg.NameRegexp.String()),
rh.CheckBlob).Methods("HEAD")
prefixedRouter.HandleFunc(fmt.Sprintf("/{name:%s}/blobs/{digest}", NameRegexp.String()),
prefixedRouter.HandleFunc(fmt.Sprintf("/{name:%s}/blobs/{digest}", zreg.NameRegexp.String()),
rh.GetBlob).Methods("GET")
prefixedRouter.HandleFunc(fmt.Sprintf("/{name:%s}/blobs/{digest}", NameRegexp.String()),
prefixedRouter.HandleFunc(fmt.Sprintf("/{name:%s}/blobs/{digest}", zreg.NameRegexp.String()),
rh.DeleteBlob).Methods("DELETE")
prefixedRouter.HandleFunc(fmt.Sprintf("/{name:%s}/blobs/uploads/", NameRegexp.String()),
prefixedRouter.HandleFunc(fmt.Sprintf("/{name:%s}/blobs/uploads/", zreg.NameRegexp.String()),
rh.CreateBlobUpload).Methods("POST")
prefixedRouter.HandleFunc(fmt.Sprintf("/{name:%s}/blobs/uploads/{session_id}", NameRegexp.String()),
prefixedRouter.HandleFunc(fmt.Sprintf("/{name:%s}/blobs/uploads/{session_id}", zreg.NameRegexp.String()),
rh.GetBlobUpload).Methods("GET")
prefixedRouter.HandleFunc(fmt.Sprintf("/{name:%s}/blobs/uploads/{session_id}", NameRegexp.String()),
prefixedRouter.HandleFunc(fmt.Sprintf("/{name:%s}/blobs/uploads/{session_id}", zreg.NameRegexp.String()),
rh.PatchBlobUpload).Methods("PATCH")
prefixedRouter.HandleFunc(fmt.Sprintf("/{name:%s}/blobs/uploads/{session_id}", NameRegexp.String()),
prefixedRouter.HandleFunc(fmt.Sprintf("/{name:%s}/blobs/uploads/{session_id}", zreg.NameRegexp.String()),
rh.UpdateBlobUpload).Methods("PUT")
prefixedRouter.HandleFunc(fmt.Sprintf("/{name:%s}/blobs/uploads/{session_id}", NameRegexp.String()),
prefixedRouter.HandleFunc(fmt.Sprintf("/{name:%s}/blobs/uploads/{session_id}", zreg.NameRegexp.String()),
rh.DeleteBlobUpload).Methods("DELETE")
// support for OCI artifact references
prefixedRouter.HandleFunc(fmt.Sprintf("/{name:%s}/referrers/{digest}", NameRegexp.String()),
prefixedRouter.HandleFunc(fmt.Sprintf("/{name:%s}/referrers/{digest}", zreg.NameRegexp.String()),
rh.GetReferrers).Methods(allowedMethods("GET")...)
prefixedRouter.HandleFunc(constants.ExtCatalogPrefix,
rh.ListRepositories).Methods(allowedMethods("GET")...)
@@ -112,7 +113,7 @@ func (rh *RouteHandler) SetupRoutes() {
// support for ORAS artifact reference types (alpha 1) - image signature use case
rh.c.Router.HandleFunc(fmt.Sprintf("%s/{name:%s}/manifests/{digest}/referrers",
constants.ArtifactSpecRoutePrefix, NameRegexp.String()), rh.GetOrasReferrers).Methods("GET")
constants.ArtifactSpecRoutePrefix, zreg.NameRegexp.String()), rh.GetOrasReferrers).Methods("GET")
// swagger
debug.SetupSwaggerRoutes(rh.c.Config, rh.c.Router, rh.c.Log)
+1 -1
View File
@@ -1298,7 +1298,7 @@ func TestScrub(t *testing.T) {
dir := t.TempDir()
repoName := "badIndex"
repoName := "badindex"
repo, err := os.MkdirTemp(dir, repoName)
if err != nil {
+7 -2
View File
@@ -555,8 +555,13 @@ func TestRepoListWithNewestImage(t *testing.T) {
ShouldBeGreaterThan,
0,
)
// This really depends on the test data, but with the current test images it's CRITICAL
So(vulnerabilities.MaxSeverity, ShouldEqual, "CRITICAL")
if repo.Name == "zot-cve-test" {
// This really depends on the test data, but with the current test image it's HIGH
So(vulnerabilities.MaxSeverity, ShouldEqual, "HIGH")
} else if repo.Name == "zot-test" {
// This really depends on the test data, but with the current test image it's CRITICAL
So(vulnerabilities.MaxSeverity, ShouldEqual, "CRITICAL")
}
}
})
}
+80 -60
View File
@@ -1,20 +1,19 @@
package trivy
import (
"context"
"encoding/json"
"flag"
"path"
"strings"
"sync"
dbTypes "github.com/aquasecurity/trivy-db/pkg/types"
"github.com/aquasecurity/trivy/pkg/commands/artifact"
"github.com/aquasecurity/trivy/pkg/commands/operation"
"github.com/aquasecurity/trivy/pkg/flag"
"github.com/aquasecurity/trivy/pkg/types"
regTypes "github.com/google/go-containerregistry/pkg/v1/types"
godigest "github.com/opencontainers/go-digest"
ispec "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/urfave/cli/v2"
zerr "zotregistry.io/zot/errors"
"zotregistry.io/zot/pkg/extensions/search/common"
@@ -24,51 +23,44 @@ import (
"zotregistry.io/zot/pkg/storage"
)
type trivyCtx struct {
Input string
Ctx *cli.Context
}
const dbRepository = "ghcr.io/aquasecurity/trivy-db"
// newTrivyContext set some trivy configuration value and return a context.
func newTrivyContext(dir string) *trivyCtx {
tCtx := &trivyCtx{}
// getNewScanOptions sets trivy configuration values for our scans and returns them as
// a trivy Options structure.
func getNewScanOptions(dir string) *flag.Options {
scanOptions := flag.Options{
GlobalOptions: flag.GlobalOptions{
CacheDir: dir,
},
ScanOptions: flag.ScanOptions{
SecurityChecks: []string{types.SecurityCheckVulnerability},
OfflineScan: true,
},
VulnerabilityOptions: flag.VulnerabilityOptions{
VulnType: []string{types.VulnTypeOS, types.VulnTypeLibrary},
},
DBOptions: flag.DBOptions{
DBRepository: dbRepository,
SkipDBUpdate: true,
},
ReportOptions: flag.ReportOptions{
Format: "table",
Severities: []dbTypes.Severity{
dbTypes.SeverityUnknown,
dbTypes.SeverityLow,
dbTypes.SeverityMedium,
dbTypes.SeverityHigh,
dbTypes.SeverityCritical,
},
},
}
app := &cli.App{}
flagSet := &flag.FlagSet{}
var cacheDir string
flagSet.StringVar(&cacheDir, "cache-dir", dir, "")
var vuln string
flagSet.StringVar(&vuln, "vuln-type", strings.Join([]string{types.VulnTypeOS, types.VulnTypeLibrary}, ","), "")
var severity string
flagSet.StringVar(&severity, "severity", strings.Join(dbTypes.SeverityNames, ","), "")
flagSet.StringVar(&tCtx.Input, "input", "", "")
var securityCheck string
flagSet.StringVar(&securityCheck, "security-checks", types.SecurityCheckVulnerability, "")
var reportFormat string
flagSet.StringVar(&reportFormat, "format", "table", "")
ctx := cli.NewContext(app, flagSet, nil)
tCtx.Ctx = ctx
return tCtx
return &scanOptions
}
type cveTrivyController struct {
DefaultCveConfig *trivyCtx
SubCveConfig map[string]*trivyCtx
DefaultCveConfig *flag.Options
SubCveConfig map[string]*flag.Options
}
type Scanner struct {
@@ -85,25 +77,27 @@ func NewScanner(storeController storage.StoreController,
) *Scanner {
cveController := cveTrivyController{}
subCveConfig := make(map[string]*trivyCtx)
subCveConfig := make(map[string]*flag.Options)
if storeController.DefaultStore != nil {
imageStore := storeController.DefaultStore
rootDir := imageStore.RootDir()
ctx := newTrivyContext(rootDir)
cacheDir := path.Join(rootDir, "_trivy")
opts := getNewScanOptions(cacheDir)
cveController.DefaultCveConfig = ctx
cveController.DefaultCveConfig = opts
}
if storeController.SubStore != nil {
for route, storage := range storeController.SubStore {
rootDir := storage.RootDir()
ctx := newTrivyContext(rootDir)
cacheDir := path.Join(rootDir, "_trivy")
opts := getNewScanOptions(cacheDir)
subCveConfig[route] = ctx
subCveConfig[route] = opts
}
}
@@ -119,33 +113,58 @@ func NewScanner(storeController storage.StoreController,
}
}
func (scanner Scanner) getTrivyContext(image string) *trivyCtx {
func (scanner Scanner) getTrivyOptions(image string) flag.Options {
// Split image to get route prefix
prefixName := common.GetRoutePrefix(image)
var tCtx *trivyCtx
var opts flag.Options
var ok bool
var rootDir string
// Get corresponding CVE trivy config, if no sub cve config present that means its default
tCtx, ok = scanner.cveController.SubCveConfig[prefixName]
_, ok = scanner.cveController.SubCveConfig[prefixName]
if ok {
opts = *scanner.cveController.SubCveConfig[prefixName]
imgStore := scanner.storeController.SubStore[prefixName]
rootDir = imgStore.RootDir()
} else {
tCtx = scanner.cveController.DefaultCveConfig
opts = *scanner.cveController.DefaultCveConfig
imgStore := scanner.storeController.DefaultStore
rootDir = imgStore.RootDir()
}
tCtx.Input = path.Join(rootDir, image)
opts.ScanOptions.Target = path.Join(rootDir, image)
opts.ImageOptions.Input = path.Join(rootDir, image)
return tCtx
return opts
}
func (scanner Scanner) runTrivy(opts flag.Options) (types.Report, error) {
ctx := context.Background()
runner, err := artifact.NewRunner(ctx, opts)
if err != nil {
return types.Report{}, err
}
defer runner.Close(ctx)
report, err := runner.ScanImage(ctx, opts)
if err != nil {
return types.Report{}, err
}
report, err = runner.Filter(ctx, opts, report)
if err != nil {
return types.Report{}, err
}
return report, nil
}
func (scanner Scanner) IsImageFormatScannable(image string) (bool, error) {
@@ -208,10 +227,9 @@ func (scanner Scanner) ScanImage(image string) (map[string]cvemodel.CVE, error)
scanner.log.Debug().Str("image", image).Msg("scanning image")
tCtx := scanner.getTrivyContext(image)
scanner.dbLock.Lock()
report, err := artifact.TrivyImageRun(tCtx.Ctx)
opts := scanner.getTrivyOptions(image)
report, err := scanner.runTrivy(opts)
scanner.dbLock.Unlock()
if err != nil { //nolint: wsl
@@ -288,7 +306,7 @@ func (scanner Scanner) UpdateDB() error {
defer scanner.dbLock.Unlock()
if scanner.storeController.DefaultStore != nil {
dbDir := scanner.storeController.DefaultStore.RootDir()
dbDir := path.Join(scanner.storeController.DefaultStore.RootDir(), "_trivy")
err := scanner.updateDB(dbDir)
if err != nil {
@@ -298,7 +316,9 @@ func (scanner Scanner) UpdateDB() error {
if scanner.storeController.SubStore != nil {
for _, storage := range scanner.storeController.SubStore {
err := scanner.updateDB(storage.RootDir())
dbDir := path.Join(storage.RootDir(), "_trivy")
err := scanner.updateDB(dbDir)
if err != nil {
return err
}
@@ -313,7 +333,7 @@ func (scanner Scanner) UpdateDB() error {
func (scanner Scanner) updateDB(dbDir string) error {
scanner.log.Debug().Msgf("Download Trivy DB to destination dir: %s", dbDir)
err := operation.DownloadDB("dev", dbDir, false, false, false)
err := operation.DownloadDB("dev", dbDir, dbRepository, false, false, false)
if err != nil {
scanner.log.Error().Err(err).Msgf("Error downloading Trivy DB to destination dir: %s", dbDir)
@@ -102,19 +102,23 @@ func TestMultipleStoragePath(t *testing.T) {
img1 := "a/test/image1:tag1"
img2 := "b/test/image2:tag2"
ctx := scanner.getTrivyContext(img0)
So(ctx.Input, ShouldEqual, path.Join(firstStore.RootDir(), img0))
opts := scanner.getTrivyOptions(img0)
So(opts.ScanOptions.Target, ShouldEqual, path.Join(firstStore.RootDir(), img0))
ctx = scanner.getTrivyContext(img1)
So(ctx.Input, ShouldEqual, path.Join(secondStore.RootDir(), img1))
opts = scanner.getTrivyOptions(img1)
So(opts.ScanOptions.Target, ShouldEqual, path.Join(secondStore.RootDir(), img1))
ctx = scanner.getTrivyContext(img2)
So(ctx.Input, ShouldEqual, path.Join(thirdStore.RootDir(), img2))
opts = scanner.getTrivyOptions(img2)
So(opts.ScanOptions.Target, ShouldEqual, path.Join(thirdStore.RootDir(), img2))
generateTestImage(storeController, img0)
generateTestImage(storeController, img1)
generateTestImage(storeController, img2)
// Download DB since DB download on scan is disabled
err = scanner.UpdateDB()
So(err, ShouldBeNil)
// Scanning image in default store
cveMap, err := scanner.ScanImage(img0)
So(err, ShouldBeNil)
@@ -152,3 +156,65 @@ func TestMultipleStoragePath(t *testing.T) {
So(err, ShouldBeNil)
})
}
func TestTrivyLibraryErrors(t *testing.T) {
Convey("Test trivy API errors", t, func() {
// Create temporary directory
rootDir := t.TempDir()
err := test.CopyFiles("../../../../../test/data/zot-test", path.Join(rootDir, "zot-test"))
So(err, ShouldBeNil)
log := log.NewLogger("debug", "")
metrics := monitoring.NewMetricsServer(false, log)
conf := config.New()
conf.Extensions = &extconf.ExtensionConfig{}
conf.Extensions.Lint = &extconf.LintConfig{}
// Create ImageStore
store := local.NewImageStore(rootDir, false, storage.DefaultGCDelay, false, false, log, metrics, nil, nil)
storeController := storage.StoreController{}
storeController.DefaultStore = store
repoDB, err := bolt.NewBoltDBWrapper(bolt.DBParameters{
RootDir: rootDir,
})
So(err, ShouldBeNil)
err = repodb.SyncRepoDB(repoDB, storeController, log)
So(err, ShouldBeNil)
scanner := NewScanner(storeController, repoDB, log)
// Download DB since DB download on scan is disabled
err = scanner.UpdateDB()
So(err, ShouldBeNil)
img := "zot-test:0.0.1"
// Scanning image with correct options
opts := scanner.getTrivyOptions(img)
_, err = scanner.runTrivy(opts)
So(err, ShouldBeNil)
// Scanning image with incorrect cache options
// to trigger runner initialization errors
opts.CacheOptions.CacheBackend = "redis://asdf!$%&!*)("
_, err = scanner.runTrivy(opts)
So(err, ShouldNotBeNil)
// Scanning image with invalid input to trigger a scanner error
opts = scanner.getTrivyOptions("nonexisting_image:0.0.1")
_, err = scanner.runTrivy(opts)
So(err, ShouldNotBeNil)
// Scanning image with incorrect report options
// to trigger report filtering errors
opts = scanner.getTrivyOptions(img)
opts.ReportOptions.IgnorePolicy = "invalid file path"
_, err = scanner.runTrivy(opts)
So(err, ShouldNotBeNil)
})
}
+5 -1
View File
@@ -1,4 +1,4 @@
package api
package regexp
import "regexp"
@@ -26,6 +26,10 @@ var (
NameRegexp = expression(
nameComponentRegexp,
optional(repeated(literal(`/`), nameComponentRegexp)))
// FullNameRegexp is the format which matches the full string of the
// name component of reference.
FullNameRegexp = expression(match("^"), NameRegexp, match("$"))
)
// match compiles the string to a regular expression.
+11
View File
@@ -31,6 +31,7 @@ import (
"zotregistry.io/zot/pkg/common"
"zotregistry.io/zot/pkg/extensions/monitoring"
zlog "zotregistry.io/zot/pkg/log"
zreg "zotregistry.io/zot/pkg/regexp"
"zotregistry.io/zot/pkg/scheduler"
"zotregistry.io/zot/pkg/storage"
"zotregistry.io/zot/pkg/storage/cache"
@@ -154,6 +155,12 @@ func (is *ImageStoreLocal) initRepo(name string) error {
return zerr.ErrInvalidRepositoryName
}
if !zreg.FullNameRegexp.MatchString(name) {
is.log.Error().Str("repo", name).Msg("invalid repository name")
return zerr.ErrInvalidRepositoryName
}
// create "blobs" subdir
err := ensureDir(path.Join(repoDir, "blobs"), is.log)
if err != nil {
@@ -221,6 +228,10 @@ func (is *ImageStoreLocal) ValidateRepo(name string) (bool, error) {
// https://github.com/opencontainers/image-spec/blob/master/image-layout.md#content
// at least, expect at least 3 entries - ["blobs", "oci-layout", "index.json"]
// and an additional/optional BlobUploadDir in each image store
if !zreg.FullNameRegexp.MatchString(name) {
return false, zerr.ErrInvalidRepositoryName
}
dir := path.Join(is.rootDir, name)
if !is.DirExists(dir) {
return false, zerr.ErrRepoNotFound
+219 -7
View File
@@ -30,6 +30,7 @@ import (
"zotregistry.io/zot/pkg/log"
"zotregistry.io/zot/pkg/storage"
"zotregistry.io/zot/pkg/storage/cache"
storageConstants "zotregistry.io/zot/pkg/storage/constants"
"zotregistry.io/zot/pkg/storage/local"
"zotregistry.io/zot/pkg/test"
)
@@ -1326,6 +1327,11 @@ func TestNegativeCases(t *testing.T) {
// Init repo should fail if repo is invalid UTF-8
err = imgStore.InitRepo("hi \255")
So(err, ShouldNotBeNil)
// Init repo should fail if repo name does not match spec
err = imgStore.InitRepo("_trivy")
So(err, ShouldNotBeNil)
So(errors.Is(err, zerr.ErrInvalidRepositoryName), ShouldBeTrue)
})
Convey("Invalid validate repo", t, func(c C) {
@@ -1430,7 +1436,7 @@ func TestNegativeCases(t *testing.T) {
So(func() { _, _ = imgStore.ValidateRepo("test") }, ShouldPanic)
}
err = os.Chmod(dir, 0o755) // remove all perms
err = os.Chmod(dir, 0o755) // add perms
if err != nil {
panic(err)
}
@@ -2483,10 +2489,53 @@ func TestValidateRepo(t *testing.T) {
_, err = imgStore.ValidateRepo("test-dir")
So(err, ShouldNotBeNil)
})
Convey("Get error when repo name is not compliant with repo spec", t, func() {
dir := t.TempDir()
log := log.Logger{Logger: zerolog.New(os.Stdout)}
metrics := monitoring.NewMetricsServer(false, log)
cacheDriver, _ := storage.Create("boltdb", cache.BoltDBDriverParameters{
RootDir: dir,
Name: "cache",
UseRelPaths: true,
}, log)
imgStore := local.NewImageStore(dir, true, storage.DefaultGCDelay,
true, true, log, metrics, nil, cacheDriver)
_, err := imgStore.ValidateRepo(".")
So(err, ShouldNotBeNil)
So(errors.Is(err, zerr.ErrInvalidRepositoryName), ShouldBeTrue)
_, err = imgStore.ValidateRepo("..")
So(err, ShouldNotBeNil)
So(errors.Is(err, zerr.ErrInvalidRepositoryName), ShouldBeTrue)
err = os.Mkdir(path.Join(dir, "_test-dir"), 0o755)
So(err, ShouldBeNil)
_, err = imgStore.ValidateRepo("_test-dir")
So(err, ShouldNotBeNil)
So(errors.Is(err, zerr.ErrInvalidRepositoryName), ShouldBeTrue)
err = os.Mkdir(path.Join(dir, ".test-dir"), 0o755)
So(err, ShouldBeNil)
_, err = imgStore.ValidateRepo(".test-dir")
So(err, ShouldNotBeNil)
So(errors.Is(err, zerr.ErrInvalidRepositoryName), ShouldBeTrue)
err = os.Mkdir(path.Join(dir, "-test-dir"), 0o755)
So(err, ShouldBeNil)
_, err = imgStore.ValidateRepo("-test-dir")
So(err, ShouldNotBeNil)
So(errors.Is(err, zerr.ErrInvalidRepositoryName), ShouldBeTrue)
})
}
func TestGetRepositoriesError(t *testing.T) {
Convey("Get error when returning relative path", t, func() {
func TestGetRepositories(t *testing.T) {
Convey("Verify errors and repos returned by GetRepositories()", t, func() {
dir := t.TempDir()
log := log.Logger{Logger: zerolog.New(os.Stdout)}
@@ -2500,15 +2549,178 @@ func TestGetRepositoriesError(t *testing.T) {
true, true, log, metrics, nil, cacheDriver,
)
// create valid directory with permissions
err := os.Mkdir(path.Join(dir, "test-dir"), 0o755)
// Create valid directory with permissions
err := os.Mkdir(path.Join(dir, "test-dir"), 0o755) //nolint: gosec
So(err, ShouldBeNil)
err = os.WriteFile(path.Join(dir, "test-dir/test-file"), []byte("this is test file"), 0o000)
err = os.WriteFile(path.Join(dir, "test-dir/test-file"), []byte("this is test file"), 0o755) //nolint: gosec
So(err, ShouldBeNil)
_, err = imgStore.GetRepositories()
// Folder is not a repo as it is missing the requires files/subfolder
repos, err := imgStore.GetRepositories()
So(err, ShouldBeNil)
So(len(repos), ShouldEqual, 0)
il := ispec.ImageLayout{Version: ispec.ImageLayoutVersion}
layoutFileContent, err := json.Marshal(il)
So(err, ShouldBeNil)
// Folder becomes a repo after the missing content is added
err = os.Mkdir(path.Join(dir, "test-dir", "blobs"), 0o755) //nolint: gosec
So(err, ShouldBeNil)
err = os.Mkdir(path.Join(dir, "test-dir", storageConstants.BlobUploadDir), 0o755) //nolint: gosec
So(err, ShouldBeNil)
err = os.WriteFile(path.Join(dir, "test-dir", "index.json"), []byte{}, 0o755) //nolint: gosec
So(err, ShouldBeNil)
err = os.WriteFile(path.Join(dir, "test-dir", ispec.ImageLayoutFile), layoutFileContent, 0o755) //nolint: gosec
So(err, ShouldBeNil)
// Verify the new repo is turned
repos, err = imgStore.GetRepositories()
So(err, ShouldBeNil)
So(len(repos), ShouldEqual, 1)
So(repos[0], ShouldEqual, "test-dir")
// create directory starting with underscore, which is not OCI a dist spec compliant repo name
// [a-z0-9]+([._-][a-z0-9]+)*(/[a-z0-9]+([._-][a-z0-9]+)*)*
err = os.MkdirAll(path.Join(dir, "_trivy", "db"), 0o755)
So(err, ShouldBeNil)
err = os.WriteFile(path.Join(dir, "_trivy", "db", "trivy.db"), []byte("this is test file"), 0o755) //nolint: gosec
So(err, ShouldBeNil)
// Folder with invalid name is not a repo as it is missing the requires files/subfolder
repos, err = imgStore.GetRepositories()
So(err, ShouldBeNil)
So(len(repos), ShouldEqual, 1)
So(repos[0], ShouldEqual, "test-dir")
// Add missing content to folder with invalid name
err = os.Mkdir(path.Join(dir, "_trivy", "blobs"), 0o755) //nolint: gosec
So(err, ShouldBeNil)
err = os.Mkdir(path.Join(dir, "_trivy", storageConstants.BlobUploadDir), 0o755) //nolint: gosec
So(err, ShouldBeNil)
err = os.WriteFile(path.Join(dir, "_trivy", "index.json"), []byte{}, 0o755) //nolint: gosec
So(err, ShouldBeNil)
err = os.WriteFile(path.Join(dir, "_trivy", ispec.ImageLayoutFile), layoutFileContent, 0o755) //nolint: gosec
So(err, ShouldBeNil)
// Folder with invalid name doesn't become a repo after the missing content is added
repos, err = imgStore.GetRepositories()
So(err, ShouldBeNil)
t.Logf("repos %v", repos)
So(len(repos), ShouldEqual, 1)
So(repos[0], ShouldEqual, "test-dir")
// Rename folder with invalid name to a valid one
err = os.Rename(path.Join(dir, "_trivy"), path.Join(dir, "test-dir-2"))
So(err, ShouldBeNil)
// Verify both repos are now visible
repos, err = imgStore.GetRepositories()
So(err, ShouldBeNil)
t.Logf("repos %v", repos)
So(len(repos), ShouldEqual, 2)
So(repos, ShouldContain, "test-dir")
So(repos, ShouldContain, "test-dir-2")
})
Convey("Verify GetRepositories() doesn't return '.' when having an oci layout as root directory ", t, func() {
dir := t.TempDir()
log := log.Logger{Logger: zerolog.New(os.Stdout)}
metrics := monitoring.NewMetricsServer(false, log)
cacheDriver, _ := storage.Create("boltdb", cache.BoltDBDriverParameters{
RootDir: dir,
Name: "cache",
UseRelPaths: true,
}, log)
imgStore := local.NewImageStore(dir, true, storage.DefaultGCDelay,
true, true, log, metrics, nil, cacheDriver,
)
// Root dir does not contain repos
repos, err := imgStore.GetRepositories()
So(err, ShouldBeNil)
So(len(repos), ShouldEqual, 0)
// Configure root directory as an oci layout
err = os.Mkdir(path.Join(dir, "blobs"), 0o755) //nolint: gosec
So(err, ShouldBeNil)
err = os.Mkdir(path.Join(dir, storageConstants.BlobUploadDir), 0o755) //nolint: gosec
So(err, ShouldBeNil)
err = os.WriteFile(path.Join(dir, "index.json"), []byte{}, 0o755) //nolint: gosec
So(err, ShouldBeNil)
il := ispec.ImageLayout{Version: ispec.ImageLayoutVersion}
layoutFileContent, err := json.Marshal(il)
So(err, ShouldBeNil)
err = os.WriteFile(path.Join(dir, ispec.ImageLayoutFile), layoutFileContent, 0o755) //nolint: gosec
So(err, ShouldBeNil)
// Verify root directory is not returned as a repo
repos, err = imgStore.GetRepositories()
So(err, ShouldBeNil)
t.Logf("repos %v", repos)
So(len(repos), ShouldEqual, 0)
})
Convey("Verify GetRepositories() doesn't return '..'", t, func() {
dir := t.TempDir()
rootDir := path.Join(dir, "rootDir")
err := os.Mkdir(rootDir, 0o755)
So(err, ShouldBeNil)
log := log.Logger{Logger: zerolog.New(os.Stdout)}
metrics := monitoring.NewMetricsServer(false, log)
cacheDriver, _ := storage.Create("boltdb", cache.BoltDBDriverParameters{
RootDir: rootDir,
Name: "cache",
UseRelPaths: true,
}, log)
imgStore := local.NewImageStore(rootDir, true, storage.DefaultGCDelay,
true, true, log, metrics, nil, cacheDriver,
)
// Root dir does not contain repos
repos, err := imgStore.GetRepositories()
So(err, ShouldBeNil)
So(len(repos), ShouldEqual, 0)
// Configure parent of root directory as an oci layout
err = os.Mkdir(path.Join(dir, "blobs"), 0o755) //nolint: gosec
So(err, ShouldBeNil)
err = os.Mkdir(path.Join(dir, storageConstants.BlobUploadDir), 0o755) //nolint: gosec
So(err, ShouldBeNil)
err = os.WriteFile(path.Join(dir, "index.json"), []byte{}, 0o755) //nolint: gosec
So(err, ShouldBeNil)
il := ispec.ImageLayout{Version: ispec.ImageLayoutVersion}
layoutFileContent, err := json.Marshal(il)
So(err, ShouldBeNil)
err = os.WriteFile(path.Join(dir, ispec.ImageLayoutFile), layoutFileContent, 0o755) //nolint: gosec
So(err, ShouldBeNil)
// Verify root directory is not returned as a repo
repos, err = imgStore.GetRepositories()
So(err, ShouldBeNil)
t.Logf("repos %v", repos)
So(len(repos), ShouldEqual, 0)
})
}
+11
View File
@@ -26,6 +26,7 @@ import (
zerr "zotregistry.io/zot/errors"
"zotregistry.io/zot/pkg/extensions/monitoring"
zlog "zotregistry.io/zot/pkg/log"
zreg "zotregistry.io/zot/pkg/regexp"
"zotregistry.io/zot/pkg/scheduler"
"zotregistry.io/zot/pkg/storage"
"zotregistry.io/zot/pkg/storage/cache"
@@ -122,6 +123,12 @@ func (is *ObjectStorage) Unlock(lockStart *time.Time) {
func (is *ObjectStorage) initRepo(name string) error {
repoDir := path.Join(is.rootDir, name)
if !zreg.FullNameRegexp.MatchString(name) {
is.log.Error().Str("repo", name).Msg("invalid repository name")
return zerr.ErrInvalidRepositoryName
}
// "oci-layout" file - create if it doesn't exist
ilPath := path.Join(repoDir, ispec.ImageLayoutFile)
if _, err := is.store.Stat(context.Background(), ilPath); err != nil {
@@ -176,6 +183,10 @@ func (is *ObjectStorage) InitRepo(name string) error {
// ValidateRepo validates that the repository layout is complaint with the OCI repo layout.
func (is *ObjectStorage) ValidateRepo(name string) (bool, error) {
if !zreg.FullNameRegexp.MatchString(name) {
return false, zerr.ErrInvalidRepositoryName
}
// https://github.com/opencontainers/image-spec/blob/master/image-layout.md#content
// at least, expect at least 3 entries - ["blobs", "oci-layout", "index.json"]
// and an additional/optional BlobUploadDir in each image store
+48 -2
View File
@@ -386,8 +386,8 @@ func TestStorageDriverStatFunction(t *testing.T) {
3) the returned storageDriver.FileInfo will report that isDir() is true.
*/
Convey("Validate storageDriver.Stat() and isDir() functions with zot storage API", t, func(c C) {
repo1 := "repo/testImageA"
repo2 := "repo/testImage"
repo1 := "repo/testimagea"
repo2 := "repo/testimage"
So(imgStore, ShouldNotBeNil)
@@ -446,6 +446,8 @@ func TestStorageDriverStatFunction(t *testing.T) {
}
func TestGetOrasAndOCIReferrers(t *testing.T) {
skipIt(t)
repo := "zot-test"
uuid, err := guuid.NewV4()
@@ -655,6 +657,50 @@ func TestNegativeCasesObjectsStorage(t *testing.T) {
storeDriver, imgStore, _ := createObjectsStore(testDir, tdir, true)
defer cleanupStorage(storeDriver, testDir)
Convey("Invalid repo name", func(c C) {
// Validate repo should fail if repo name does not match spec
_, err := imgStore.ValidateRepo(".")
So(err, ShouldNotBeNil)
So(errors.Is(err, zerr.ErrInvalidRepositoryName), ShouldBeTrue)
_, err = imgStore.ValidateRepo("..")
So(err, ShouldNotBeNil)
So(errors.Is(err, zerr.ErrInvalidRepositoryName), ShouldBeTrue)
_, err = imgStore.ValidateRepo("_test-dir")
So(err, ShouldNotBeNil)
So(errors.Is(err, zerr.ErrInvalidRepositoryName), ShouldBeTrue)
_, err = imgStore.ValidateRepo(".test-dir")
So(err, ShouldNotBeNil)
So(errors.Is(err, zerr.ErrInvalidRepositoryName), ShouldBeTrue)
_, err = imgStore.ValidateRepo("-test-dir")
So(err, ShouldNotBeNil)
So(errors.Is(err, zerr.ErrInvalidRepositoryName), ShouldBeTrue)
// Init repo should fail if repo name does not match spec
err = imgStore.InitRepo(".")
So(err, ShouldNotBeNil)
So(errors.Is(err, zerr.ErrInvalidRepositoryName), ShouldBeTrue)
err = imgStore.InitRepo("..")
So(err, ShouldNotBeNil)
So(errors.Is(err, zerr.ErrInvalidRepositoryName), ShouldBeTrue)
err = imgStore.InitRepo("_test-dir")
So(err, ShouldNotBeNil)
So(errors.Is(err, zerr.ErrInvalidRepositoryName), ShouldBeTrue)
err = imgStore.InitRepo(".test-dir")
So(err, ShouldNotBeNil)
So(errors.Is(err, zerr.ErrInvalidRepositoryName), ShouldBeTrue)
err = imgStore.InitRepo("-test-dir")
So(err, ShouldNotBeNil)
So(errors.Is(err, zerr.ErrInvalidRepositoryName), ShouldBeTrue)
})
Convey("Invalid validate repo", func(c C) {
So(imgStore.InitRepo(testImage), ShouldBeNil)
objects, err := storeDriver.List(context.Background(), path.Join(imgStore.RootDir(), testImage))
+6 -1
View File
@@ -149,6 +149,11 @@ func CopyFiles(sourceDir, destDir string) error {
destFilePath := path.Join(destDir, file.Name())
if file.IsDir() {
if strings.HasPrefix(file.Name(), "_") {
// Some tests create the trivy related folders under test/_trivy
continue
}
if err = CopyFiles(sourceFilePath, destFilePath); err != nil {
return err
}
@@ -283,7 +288,7 @@ func WaitTillServerReady(url string) {
func WaitTillTrivyDBDownloadStarted(rootDir string) {
for {
if _, err := os.Stat(path.Join(rootDir, "trivy.db")); err == nil {
if _, err := os.Stat(path.Join(rootDir, "_trivy", "db", "trivy.db")); err == nil {
break
}
+41
View File
@@ -72,6 +72,47 @@ func TestCopyFiles(t *testing.T) {
err = test.CopyFiles(dir, os.TempDir())
So(err, ShouldNotBeNil)
})
Convey("sourceDir contains a folder starting with invalid characters", t, func() {
srcDir := t.TempDir()
dstDir := t.TempDir()
err := os.MkdirAll(path.Join(srcDir, "_trivy", "db"), 0o755)
if err != nil {
panic(err)
}
err = os.MkdirAll(path.Join(srcDir, "test-index"), 0o755)
if err != nil {
panic(err)
}
filePathTrivy := path.Join(srcDir, "_trivy", "db", "trivy.db")
err = os.WriteFile(filePathTrivy, []byte("some dummy file content"), 0o644) //nolint: gosec
if err != nil {
panic(err)
}
var index ispec.Index
content, err := json.Marshal(index)
if err != nil {
panic(err)
}
err = os.WriteFile(path.Join(srcDir, "test-index", "index.json"), content, 0o644) //nolint: gosec
if err != nil {
panic(err)
}
err = test.CopyFiles(srcDir, dstDir)
So(err, ShouldBeNil)
_, err = os.Stat(path.Join(dstDir, "_trivy", "db", "trivy.db"))
So(err, ShouldNotBeNil)
So(os.IsNotExist(err), ShouldBeTrue)
_, err = os.Stat(path.Join(dstDir, "test-index", "index.json"))
So(err, ShouldBeNil)
})
}
func TestGetOciLayoutDigests(t *testing.T) {