fix(cve): Fix CVE scanning in images containing Jar files (#1475)

pkg/cli/extensions_test.go

@@ -641,6 +641,13 @@ func TestServeSearchEnabled(t *testing.T) {
 		substring := `"Extensions":{"Search":{"Enable":true,"CVE":null}`
 
 		found, err := readLogFileAndSearchString(logPath, substring, readLogFileTimeout)
+
+		if !found {
+			data, err := os.ReadFile(logPath)
+			So(err, ShouldBeNil)
+			t.Log(string(data))
+		}
+
 		So(found, ShouldBeTrue)
 		So(err, ShouldBeNil)
 	})
@@ -680,20 +687,26 @@ func TestServeSearchEnabledCVE(t *testing.T) {
 		// to avoid data race when multiple go routines write to trivy DB instance.
 		WaitTillTrivyDBDownloadStarted(tempDir)
 
-		substring := "\"Search\":{\"Enable\":true,\"CVE\":{\"UpdateInterval\":3600000000000,\"Trivy\":null}}"
+		// The default config handling logic will convert the 1h interval to a 2h interval
+		substring := "\"Search\":{\"Enable\":true,\"CVE\":{\"UpdateInterval\":7200000000000,\"Trivy\":" +
+			"{\"DBRepository\":\"ghcr.io/aquasecurity/trivy-db\",\"JavaDBRepository\":\"ghcr.io/aquasecurity/trivy-java-db\"}}}"
 
 		found, err := readLogFileAndSearchString(logPath, substring, readLogFileTimeout)
+
+		defer func() {
+			if !found {
+				data, err := os.ReadFile(logPath)
+				So(err, ShouldBeNil)
+				t.Log(string(data))
+			}
+		}()
+
 		So(found, ShouldBeTrue)
 		So(err, ShouldBeNil)
 
 		found, err = readLogFileAndSearchString(logPath, "updating the CVE database", readLogFileTimeout)
 		So(found, ShouldBeTrue)
 		So(err, ShouldBeNil)
-
-		substring = "CVE update interval set to too-short interval < 2h, changing update duration to 2 hours and continuing." //nolint:lll // gofumpt conflicts with lll
-		found, err = readLogFileAndSearchString(logPath, substring, readLogFileTimeout)
-		So(found, ShouldBeTrue)
-		So(err, ShouldBeNil)
 	})
 }
 
@@ -729,6 +742,13 @@ func TestServeSearchEnabledNoCVE(t *testing.T) {
 
 		substring := `"Extensions":{"Search":{"Enable":true,"CVE":null}` //nolint:lll // gofumpt conflicts with lll
 		found, err := readLogFileAndSearchString(logPath, substring, readLogFileTimeout)
+
+		if !found {
+			data, err := os.ReadFile(logPath)
+			So(err, ShouldBeNil)
+			t.Log(string(data))
+		}
+
 		So(found, ShouldBeTrue)
 		So(err, ShouldBeNil)
 	})

pkg/cli/root.go

@@ -448,7 +448,7 @@ func validateAuthzPolicies(config *config.Config) error {
 	return nil
 }
 
-//nolint:gocyclo
+//nolint:gocyclo,cyclop,nestif
 func applyDefaultValues(config *config.Config, viperInstance *viper.Viper) {
 	defaultVal := true
 
@@ -503,6 +503,35 @@ func applyDefaultValues(config *config.Config, viperInstance *viper.Viper) {
 			if config.Extensions.Search.Enable == nil {
 				config.Extensions.Search.Enable = &defaultVal
 			}
+
+			if *config.Extensions.Search.Enable && config.Extensions.Search.CVE != nil {
+				defaultUpdateInterval, _ := time.ParseDuration("2h")
+
+				if config.Extensions.Search.CVE.UpdateInterval < defaultUpdateInterval {
+					config.Extensions.Search.CVE.UpdateInterval = defaultUpdateInterval
+
+					log.Warn().Msg("CVE update interval set to too-short interval < 2h, " +
+						"changing update duration to 2 hours and continuing.")
+				}
+
+				if config.Extensions.Search.CVE.Trivy == nil {
+					config.Extensions.Search.CVE.Trivy = &extconf.TrivyConfig{}
+				}
+
+				if config.Extensions.Search.CVE.Trivy.DBRepository == "" {
+					defaultDBDownloadURL := "ghcr.io/aquasecurity/trivy-db"
+					log.Info().Str("trivyDownloadURL", defaultDBDownloadURL).
+						Msg("Config: using default Trivy DB download URL.")
+					config.Extensions.Search.CVE.Trivy.DBRepository = defaultDBDownloadURL
+				}
+
+				if config.Extensions.Search.CVE.Trivy.JavaDBRepository == "" {
+					defaultJavaDBDownloadURL := "ghcr.io/aquasecurity/trivy-java-db"
+					log.Info().Str("trivyJavaDownloadURL", defaultJavaDBDownloadURL).
+						Msg("Config: using default Trivy Java DB download URL.")
+					config.Extensions.Search.CVE.Trivy.JavaDBRepository = defaultJavaDBDownloadURL
+				}
+			}
 		}
 
 		if config.Extensions.Metrics != nil {

pkg/extensions/config/config.go

@@ -42,7 +42,8 @@ type CVEConfig struct {
 }
 
 type TrivyConfig struct {
-	DBRepository string // default is "ghcr.io/aquasecurity/trivy-db"
+	DBRepository     string // default is "ghcr.io/aquasecurity/trivy-db"
+	JavaDBRepository string // default is "ghcr.io/aquasecurity/trivy-java-db"
 }
 
 type MetricsConfig struct {

pkg/extensions/extension_search.go

@@ -45,27 +45,16 @@ func GetCVEInfo(config *config.Config, storeController storage.StoreController,
 		return nil
 	}
 
-	dbRepository := ""
+	dbRepository := config.Extensions.Search.CVE.Trivy.DBRepository
+	javaDBRepository := config.Extensions.Search.CVE.Trivy.JavaDBRepository
 
-	if config.Extensions.Search.CVE.Trivy != nil {
-		dbRepository = config.Extensions.Search.CVE.Trivy.DBRepository
-	}
-
-	return cveinfo.NewCVEInfo(storeController, repoDB, dbRepository, log)
+	return cveinfo.NewCVEInfo(storeController, repoDB, dbRepository, javaDBRepository, log)
 }
 
 func EnableSearchExtension(config *config.Config, storeController storage.StoreController,
 	repoDB repodb.RepoDB, taskScheduler *scheduler.Scheduler, cveInfo CveInfo, log log.Logger,
 ) {
 	if config.Extensions.Search != nil && *config.Extensions.Search.Enable && config.Extensions.Search.CVE != nil {
-		defaultUpdateInterval, _ := time.ParseDuration("2h")
-
-		if config.Extensions.Search.CVE.UpdateInterval < defaultUpdateInterval {
-			config.Extensions.Search.CVE.UpdateInterval = defaultUpdateInterval
-
-			log.Warn().Msg("CVE update interval set to too-short interval < 2h, changing update duration to 2 hours and continuing.") //nolint:lll // gofumpt conflicts with lll
-		}
-
 		updateInterval := config.Extensions.Search.CVE.UpdateInterval
 
 		downloadTrivyDB(updateInterval, taskScheduler, cveInfo, log)
@@ -77,6 +66,7 @@ func EnableSearchExtension(config *config.Config, storeController storage.StoreC
 func downloadTrivyDB(interval time.Duration, sch *scheduler.Scheduler, cveInfo CveInfo, log log.Logger) {
 	generator := NewTrivyTaskGenerator(interval, cveInfo, log)
 
+	log.Info().Msg("Submitting CVE DB update scheduler")
 	sch.SubmitGenerator(generator, interval, scheduler.HighPriority)
 }
 

pkg/extensions/extension_search_test.go

@@ -46,7 +46,7 @@ func TestTrivyDBGenerator(t *testing.T) {
 			DefaultStore: mocks.MockedImageStore{},
 		}
 
-		cveInfo := cveinfo.NewCVEInfo(storeController, repoDB, "", logger)
+		cveInfo := cveinfo.NewCVEInfo(storeController, repoDB, "ghcr.io/project-zot/trivy-db", "", logger)
 		generator := NewTrivyTaskGenerator(time.Minute, cveInfo, logger)
 
 		sch.SubmitGenerator(generator, 12000*time.Millisecond, scheduler.HighPriority)

pkg/extensions/search/cve/cve.go

@@ -46,9 +46,9 @@ type BaseCveInfo struct {
 }
 
 func NewCVEInfo(storeController storage.StoreController, repoDB repodb.RepoDB,
-	dbRepository string, log log.Logger,
+	dbRepository, javaDBRepository string, log log.Logger,
 ) *BaseCveInfo {
-	scanner := trivy.NewScanner(storeController, repoDB, dbRepository, log)
+	scanner := trivy.NewScanner(storeController, repoDB, dbRepository, javaDBRepository, log)
 
 	return &BaseCveInfo{
 		Log:     log,

pkg/extensions/search/cve/cve_test.go

@@ -327,7 +327,7 @@ func TestImageFormat(t *testing.T) {
 		err = repodb.ParseStorage(repoDB, storeController, log)
 		So(err, ShouldBeNil)
 
-		cveInfo := cveinfo.NewCVEInfo(storeController, repoDB, "", log)
+		cveInfo := cveinfo.NewCVEInfo(storeController, repoDB, "ghcr.io/project-zot/trivy-db", "", log)
 
 		isValidImage, err := cveInfo.Scanner.IsImageFormatScannable("zot-test", "")
 		So(err, ShouldNotBeNil)
@@ -390,7 +390,7 @@ func TestImageFormat(t *testing.T) {
 			DefaultStore: mocks.MockedImageStore{},
 		}
 
-		cveInfo := cveinfo.NewCVEInfo(storeController, repoDB, "", log)
+		cveInfo := cveinfo.NewCVEInfo(storeController, repoDB, "ghcr.io/project-zot/trivy-db", "", log)
 
 		isScanable, err := cveInfo.Scanner.IsImageFormatScannable("repo", "tag")
 		So(err, ShouldBeNil)

pkg/extensions/search/cve/trivy/scanner.go

@@ -12,10 +12,12 @@ import (
 	"github.com/aquasecurity/trivy/pkg/commands/operation"
 	fanalTypes "github.com/aquasecurity/trivy/pkg/fanal/types"
 	"github.com/aquasecurity/trivy/pkg/flag"
+	"github.com/aquasecurity/trivy/pkg/javadb"
 	"github.com/aquasecurity/trivy/pkg/types"
 	regTypes "github.com/google/go-containerregistry/pkg/v1/types"
 	godigest "github.com/opencontainers/go-digest"
 	ispec "github.com/opencontainers/image-spec/specs-go/v1"
+	_ "modernc.org/sqlite"
 
 	zerr "zotregistry.io/zot/errors"
 	cvemodel "zotregistry.io/zot/pkg/extensions/search/cve/model"
@@ -24,11 +26,9 @@ import (
 	"zotregistry.io/zot/pkg/storage"
 )
 
-const defaultDBRepository = "ghcr.io/aquasecurity/trivy-db"
-
 // getNewScanOptions sets trivy configuration values for our scans and returns them as
 // a trivy Options structure.
-func getNewScanOptions(dir, dbRepository string) *flag.Options {
+func getNewScanOptions(dir, dbRepository, javaDBRepository string) *flag.Options {
 	scanOptions := flag.Options{
 		GlobalOptions: flag.GlobalOptions{
 			CacheDir: dir,
@@ -41,8 +41,10 @@ func getNewScanOptions(dir, dbRepository string) *flag.Options {
 			VulnType: []string{types.VulnTypeOS, types.VulnTypeLibrary},
 		},
 		DBOptions: flag.DBOptions{
-			DBRepository: dbRepository,
-			SkipDBUpdate: true,
+			DBRepository:     dbRepository,
+			JavaDBRepository: javaDBRepository,
+			SkipDBUpdate:     true,
+			SkipJavaDBUpdate: true,
 		},
 		ReportOptions: flag.ReportOptions{
 			Format: "table",
@@ -65,33 +67,30 @@ type cveTrivyController struct {
 }
 
 type Scanner struct {
-	repoDB          repodb.RepoDB
-	cveController   cveTrivyController
-	storeController storage.StoreController
-	log             log.Logger
-	dbLock          *sync.Mutex
-	cache           *CveCache
-	dbRepository    string
+	repoDB           repodb.RepoDB
+	cveController    cveTrivyController
+	storeController  storage.StoreController
+	log              log.Logger
+	dbLock           *sync.Mutex
+	cache            *CveCache
+	dbRepository     string
+	javaDBRepository string
 }
 
 func NewScanner(storeController storage.StoreController,
-	repoDB repodb.RepoDB, dbRepository string, log log.Logger,
+	repoDB repodb.RepoDB, dbRepository, javaDBRepository string, log log.Logger,
 ) *Scanner {
 	cveController := cveTrivyController{}
 
 	subCveConfig := make(map[string]*flag.Options)
 
-	if dbRepository == "" {
-		dbRepository = defaultDBRepository
-	}
-
 	if storeController.DefaultStore != nil {
 		imageStore := storeController.DefaultStore
 
 		rootDir := imageStore.RootDir()
 
 		cacheDir := path.Join(rootDir, "_trivy")
-		opts := getNewScanOptions(cacheDir, dbRepository)
+		opts := getNewScanOptions(cacheDir, dbRepository, javaDBRepository)
 
 		cveController.DefaultCveConfig = opts
 	}
@@ -101,7 +100,7 @@ func NewScanner(storeController storage.StoreController,
 			rootDir := storage.RootDir()
 
 			cacheDir := path.Join(rootDir, "_trivy")
-			opts := getNewScanOptions(cacheDir, dbRepository)
+			opts := getNewScanOptions(cacheDir, dbRepository, javaDBRepository)
 
 			subCveConfig[route] = opts
 		}
@@ -110,13 +109,14 @@ func NewScanner(storeController storage.StoreController,
 	cveController.SubCveConfig = subCveConfig
 
 	return &Scanner{
-		log:             log,
-		repoDB:          repoDB,
-		cveController:   cveController,
-		storeController: storeController,
-		dbLock:          &sync.Mutex{},
-		cache:           NewCveCache(10000, log), //nolint:gomnd
-		dbRepository:    dbRepository,
+		log:              log,
+		repoDB:           repoDB,
+		cveController:    cveController,
+		storeController:  storeController,
+		dbLock:           &sync.Mutex{},
+		cache:            NewCveCache(10000, log), //nolint:gomnd
+		dbRepository:     dbRepository,
+		javaDBRepository: javaDBRepository,
 	}
 }
 
@@ -371,14 +371,25 @@ func (scanner Scanner) updateDB(dbDir string) error {
 
 	ctx := context.Background()
 
-	err := operation.DownloadDB(ctx, "dev", dbDir, scanner.dbRepository, false, false,
-		fanalTypes.RegistryOptions{Insecure: false})
+	registryOpts := fanalTypes.RegistryOptions{Insecure: false}
+
+	err := operation.DownloadDB(ctx, "dev", dbDir, scanner.dbRepository, false, false, registryOpts)
 	if err != nil {
 		scanner.log.Error().Err(err).Str("dbDir", dbDir).Msg("Error downloading Trivy DB to destination dir")
 
 		return err
 	}
 
+	if scanner.javaDBRepository != "" {
+		javadb.Init(dbDir, scanner.javaDBRepository, false, false, registryOpts.Insecure)
+
+		if err := javadb.Update(); err != nil {
+			scanner.log.Error().Err(err).Str("dbDir", dbDir).Msg("Error downloading Trivy Java DB to destination dir")
+
+			return err
+		}
+	}
+
 	scanner.log.Debug().Str("dbDir", dbDir).Msg("Finished downloading Trivy DB to destination dir")
 
 	return nil

pkg/extensions/search/cve/trivy/scanner_internal_test.go

@@ -102,7 +102,7 @@ func TestMultipleStoragePath(t *testing.T) {
 		err = repodb.ParseStorage(repoDB, storeController, log)
 		So(err, ShouldBeNil)
 
-		scanner := NewScanner(storeController, repoDB, "ghcr.io/project-zot/trivy-db", log)
+		scanner := NewScanner(storeController, repoDB, "ghcr.io/project-zot/trivy-db", "", log)
 
 		So(scanner.storeController.DefaultStore, ShouldNotBeNil)
 		So(scanner.storeController.SubStore, ShouldNotBeNil)
@@ -197,9 +197,23 @@ func TestTrivyLibraryErrors(t *testing.T) {
 		err = repodb.ParseStorage(repoDB, storeController, log)
 		So(err, ShouldBeNil)
 
-		scanner := NewScanner(storeController, repoDB, "ghcr.io/project-zot/trivy-db", log)
+		// Download DB fails for missing DB url
+		scanner := NewScanner(storeController, repoDB, "", "", log)
+
+		err = scanner.UpdateDB()
+		So(err, ShouldNotBeNil)
+
+		// Download DB fails for invalid Java DB
+		scanner = NewScanner(storeController, repoDB, "ghcr.io/project-zot/trivy-db",
+			"ghcr.io/project-zot/trivy-not-db", log)
+
+		err = scanner.UpdateDB()
+		So(err, ShouldNotBeNil)
+
+		// Download DB passes for valid Trivy DB url, and missing Trivy Java DB url
+		// Download DB is necessary since DB download on scan is disabled
+		scanner = NewScanner(storeController, repoDB, "ghcr.io/project-zot/trivy-db", "", log)
 
-		// Download DB since DB download on scan is disabled
 		err = scanner.UpdateDB()
 		So(err, ShouldBeNil)
 
@@ -381,7 +395,8 @@ func TestImageScannable(t *testing.T) {
 	storeController := storage.StoreController{}
 	storeController.DefaultStore = store
 
-	scanner := NewScanner(storeController, repoDB, "ghcr.io/project-zot/trivy-db", log)
+	scanner := NewScanner(storeController, repoDB, "ghcr.io/project-zot/trivy-db",
+		"ghcr.io/aquasecurity/trivy-java-db", log)
 
 	Convey("Valid image should be scannable", t, func() {
 		result, err := scanner.IsImageFormatScannable("repo1", "valid")
@@ -434,6 +449,9 @@ func TestDefaultTrivyDBUrl(t *testing.T) {
 		err := test.CopyFiles("../../../../../test/data/zot-test", path.Join(rootDir, "zot-test"))
 		So(err, ShouldBeNil)
 
+		err = test.CopyFiles("../../../../../test/data/zot-cve-java-test", path.Join(rootDir, "zot-cve-java-test"))
+		So(err, ShouldBeNil)
+
 		log := log.NewLogger("debug", "")
 		metrics := monitoring.NewMetricsServer(false, log)
 
@@ -456,18 +474,25 @@ func TestDefaultTrivyDBUrl(t *testing.T) {
 		err = repodb.ParseStorage(repoDB, storeController, log)
 		So(err, ShouldBeNil)
 
-		// Use empty string for DB repository, the default url would be used internally
-		scanner := NewScanner(storeController, repoDB, "", log)
+		scanner := NewScanner(storeController, repoDB, "ghcr.io/aquasecurity/trivy-db",
+			"ghcr.io/aquasecurity/trivy-java-db", log)
 
 		// Download DB since DB download on scan is disabled
 		err = scanner.UpdateDB()
 		So(err, ShouldBeNil)
 
+		// Scanning image
 		img := "zot-test:0.0.1"
 
-		// Scanning image
 		opts := scanner.getTrivyOptions(img)
 		_, err = scanner.runTrivy(opts)
 		So(err, ShouldBeNil)
+
+		// Scanning image containing a jar file
+		img = "zot-cve-java-test:0.0.1"
+
+		opts = scanner.getTrivyOptions(img)
+		_, err = scanner.runTrivy(opts)
+		So(err, ShouldBeNil)
 	})
 }

pkg/extensions/search/search_test.go

@@ -712,7 +712,8 @@ func TestRepoListWithNewestImage(t *testing.T) {
 
 		defer ctlr.Shutdown()
 
-		substring := "{\"Search\":{\"Enable\":true,\"CVE\":{\"UpdateInterval\":3600000000000,\"Trivy\":{\"DBRepository\":\"ghcr.io/project-zot/trivy-db\"}}}" //nolint: lll
+		substring := "{\"Search\":{\"Enable\":true,\"CVE\":{\"UpdateInterval\":3600000000000," +
+			"\"Trivy\":{\"DBRepository\":\"ghcr.io/project-zot/trivy-db\",\"JavaDBRepository\":\"\"}}}"
 		found, err := readFileAndSearchString(logPath, substring, 2*time.Minute)
 		So(found, ShouldBeTrue)
 		So(err, ShouldBeNil)
@@ -3384,7 +3385,8 @@ func TestGlobalSearch(t *testing.T) {
 		defer ctlr.Shutdown()
 
 		// Wait for trivy db to download
-		substring := "{\"Search\":{\"Enable\":true,\"CVE\":{\"UpdateInterval\":3600000000000,\"Trivy\":{\"DBRepository\":\"ghcr.io/project-zot/trivy-db\"}}}" //nolint: lll
+		substring := "{\"Search\":{\"Enable\":true,\"CVE\":{\"UpdateInterval\":3600000000000," +
+			"\"Trivy\":{\"DBRepository\":\"ghcr.io/project-zot/trivy-db\",\"JavaDBRepository\":\"\"}}}"
 		found, err := readFileAndSearchString(logPath, substring, 2*time.Minute)
 		So(found, ShouldBeTrue)
 		So(err, ShouldBeNil)

pkg/storage/local/local.go

@@ -1968,6 +1968,6 @@ func (is *ImageStoreLocal) RunDedupeBlobs(interval time.Duration, sch *scheduler
 			Log:      is.log,
 		}
 
-		sch.SubmitGenerator(generator, interval, scheduler.HighPriority)
+		sch.SubmitGenerator(generator, interval, scheduler.MediumPriority)
 	}
 }

pkg/storage/s3/s3.go

@@ -1644,5 +1644,5 @@ func (is *ObjectStorage) RunDedupeBlobs(interval time.Duration, sch *scheduler.S
 		Log:      is.log,
 	}
 
-	sch.SubmitGenerator(generator, interval, scheduler.HighPriority)
+	sch.SubmitGenerator(generator, interval, scheduler.MediumPriority)
 }

pkg/test/common_test.go

@@ -723,7 +723,7 @@ func TestReadLogFileAndSearchString(t *testing.T) {
 	defer os.Remove(logPath)
 
 	Convey("Invalid path", t, func() {
-		_, err = test.ReadLogFileAndSearchString("invalidPath", "DB update completed, next update scheduled", 90*time.Second)
+		_, err = test.ReadLogFileAndSearchString("invalidPath", "DB update completed, next update scheduled", 1*time.Second)
 		So(err, ShouldNotBeNil)
 	})
 
@@ -750,7 +750,7 @@ func TestReadLogFileAndCountStringOccurence(t *testing.T) {
 
 	Convey("Invalid path", t, func() {
 		_, err = test.ReadLogFileAndCountStringOccurence("invalidPath",
-			"DB update completed, next update scheduled", 90*time.Second, 1)
+			"DB update completed, next update scheduled", 1*time.Second, 1)
 		So(err, ShouldNotBeNil)
 	})