From 7c78f80a961e2a75fe47dce4a6bde2c307278803 Mon Sep 17 00:00:00 2001 From: Andrei Aaron Date: Fri, 22 Sep 2023 21:49:17 +0300 Subject: [PATCH] feat(cve): implement CVE scanning as background tasks (#1833) 1. Move existing CVE DB download generator/task login under the cve package 2. Add a new CVE scanner task generator and task type to run in the background, as well as tests for it 3. Move the CVE cache in its own package 4. Add a CVE scanner methods to check if an entry is present in the cache, and to retreive the results 5. Modify the FilterTags MetaDB method to not exit on first error This is needed in order to pass all tags to the generator, instead of the generator stopping at the first set of invalid data 6. Integrate the new scanning task generator with the existing zot code. 7. Fix an issue where the CVE scan results for multiarch images was not cached 8. Rewrite some of the older CVE tests to use the new image-utils test package 9. Use the CVE scanner as attribute of the controller instead of CveInfo. Remove functionality of CVE DB update from CveInfo, it is now responsible, as the name states, only for providing CVE information. 10. The logic to get maximum severity and cve count for image sumaries now uses only the scanner cache. 11. Removed the GetCVESummaryForImage method from CveInfo as it was only used in tests Signed-off-by: Andrei Aaron --- pkg/api/config/config.go | 4 + pkg/api/controller.go | 6 +- pkg/api/routes.go | 2 +- pkg/cli/client/cve_cmd_internal_test.go | 74 +- pkg/extensions/extension_search.go | 142 +-- pkg/extensions/extension_search_disabled.go | 10 +- .../search/cve/{trivy => cache}/cache.go | 6 +- pkg/extensions/search/cve/cve.go | 90 +- pkg/extensions/search/cve/cve_test.go | 806 ++++++++++-------- pkg/extensions/search/cve/scan.go | 207 +++++ pkg/extensions/search/cve/scan_test.go | 670 +++++++++++++++ pkg/extensions/search/cve/trivy/scanner.go | 25 +- .../search/cve/trivy/scanner_test.go | 10 +- pkg/extensions/search/cve/update.go | 120 +++ .../cve/update_test.go} | 11 +- pkg/extensions/search/resolver_test.go | 128 +-- pkg/extensions/search/search_test.go | 189 ++-- pkg/meta/boltdb/boltdb.go | 34 +- pkg/meta/dynamodb/dynamodb.go | 45 +- pkg/meta/types/types.go | 2 +- pkg/test/mocks/cve_mock.go | 38 +- 21 files changed, 1812 insertions(+), 807 deletions(-) rename pkg/extensions/search/cve/{trivy => cache}/cache.go (86%) create mode 100644 pkg/extensions/search/cve/scan.go create mode 100644 pkg/extensions/search/cve/scan_test.go create mode 100644 pkg/extensions/search/cve/update.go rename pkg/extensions/{extension_search_test.go => search/cve/update_test.go} (84%) diff --git a/pkg/api/config/config.go b/pkg/api/config/config.go index b0142f2f..8ab156e3 100644 --- a/pkg/api/config/config.go +++ b/pkg/api/config/config.go @@ -348,6 +348,10 @@ func (c *Config) IsSearchEnabled() bool { return c.Extensions != nil && c.Extensions.Search != nil && *c.Extensions.Search.Enable } +func (c *Config) IsCveScanningEnabled() bool { + return c.IsSearchEnabled() && c.Extensions.Search.CVE != nil +} + func (c *Config) IsUIEnabled() bool { return c.Extensions != nil && c.Extensions.UI != nil && *c.Extensions.UI.Enable } diff --git a/pkg/api/controller.go b/pkg/api/controller.go index cfd37279..73639d88 100644 --- a/pkg/api/controller.go +++ b/pkg/api/controller.go @@ -46,7 +46,7 @@ type Controller struct { Audit *log.Logger Server *http.Server Metrics monitoring.MetricServer - CveInfo ext.CveInfo + CveScanner ext.CveScanner SyncOnDemand SyncOnDemand RelyingParties map[string]rp.RelyingParty CookieStore sessions.Store @@ -241,7 +241,7 @@ func (c *Controller) Init(reloadCtx context.Context) error { func (c *Controller) InitCVEInfo() { // Enable CVE extension if extension config is provided if c.Config != nil && c.Config.Extensions != nil { - c.CveInfo = ext.GetCVEInfo(c.Config, c.StoreController, c.MetaDB, c.Log) + c.CveScanner = ext.GetCveScanner(c.Config, c.StoreController, c.MetaDB, c.Log) } } @@ -347,7 +347,7 @@ func (c *Controller) StartBackgroundTasks(reloadCtx context.Context) { // Enable extensions if extension config is provided for DefaultStore if c.Config != nil && c.Config.Extensions != nil { ext.EnableMetricsExtension(c.Config, c.Log, c.Config.Storage.RootDirectory) - ext.EnableSearchExtension(c.Config, c.StoreController, c.MetaDB, taskScheduler, c.CveInfo, c.Log) + ext.EnableSearchExtension(c.Config, c.StoreController, c.MetaDB, taskScheduler, c.CveScanner, c.Log) } if c.Config.Storage.SubPaths != nil { diff --git a/pkg/api/routes.go b/pkg/api/routes.go index aecceb8c..97cbe7ad 100644 --- a/pkg/api/routes.go +++ b/pkg/api/routes.go @@ -184,7 +184,7 @@ func (rh *RouteHandler) SetupRoutes() { // Preconditions for enabling the actual extension routes are part of extensions themselves ext.SetupMetricsRoutes(rh.c.Config, rh.c.Router, authHandler, rh.c.Log, rh.c.Metrics) - ext.SetupSearchRoutes(rh.c.Config, prefixedRouter, rh.c.StoreController, rh.c.MetaDB, rh.c.CveInfo, + ext.SetupSearchRoutes(rh.c.Config, prefixedRouter, rh.c.StoreController, rh.c.MetaDB, rh.c.CveScanner, rh.c.Log) ext.SetupImageTrustRoutes(rh.c.Config, prefixedRouter, rh.c.MetaDB, rh.c.Log) ext.SetupMgmtRoutes(rh.c.Config, prefixedRouter, rh.c.Log) diff --git a/pkg/cli/client/cve_cmd_internal_test.go b/pkg/cli/client/cve_cmd_internal_test.go index 94e15ca8..5883465f 100644 --- a/pkg/cli/client/cve_cmd_internal_test.go +++ b/pkg/cli/client/cve_cmd_internal_test.go @@ -531,7 +531,7 @@ func TestNegativeServerResponse(t *testing.T) { panic(err) } - ctlr.CveInfo = getMockCveInfo(ctlr.MetaDB, ctlr.Log) + ctlr.CveScanner = getMockCveScanner(ctlr.MetaDB) go func() { if err := ctlr.Run(ctx); !errors.Is(err, http.ErrServerClosed) { @@ -606,7 +606,7 @@ func TestServerCVEResponse(t *testing.T) { panic(err) } - ctlr.CveInfo = getMockCveInfo(ctlr.MetaDB, ctlr.Log) + ctlr.CveScanner = getMockCveScanner(ctlr.MetaDB) go func() { if err := ctlr.Run(ctx); !errors.Is(err, http.ErrServerClosed) { @@ -947,39 +947,35 @@ func TestCVESort(t *testing.T) { panic(err) } - ctlr.CveInfo = cveinfo.BaseCveInfo{ - Log: ctlr.Log, - MetaDB: mocks.MetaDBMock{}, - Scanner: mocks.CveScannerMock{ - ScanImageFn: func(image string) (map[string]cvemodel.CVE, error) { - return map[string]cvemodel.CVE{ - "CVE-2023-1255": { - ID: "CVE-2023-1255", - Severity: "LOW", - Title: "Input buffer over-read in AES-XTS implementation and testing", - }, - "CVE-2023-2650": { - ID: "CVE-2023-2650", - Severity: "MEDIUM", - Title: "Possible DoS translating ASN.1 object identifier and executer", - }, - "CVE-2023-2975": { - ID: "CVE-2023-2975", - Severity: "HIGH", - Title: "AES-SIV cipher implementation contains a bug that can break", - }, - "CVE-2023-3446": { - ID: "CVE-2023-3446", - Severity: "CRITICAL", - Title: "Excessive time spent checking DH keys and parenthesis", - }, - "CVE-2023-3817": { - ID: "CVE-2023-3817", - Severity: "MEDIUM", - Title: "Excessive time spent checking DH q parameter and arguments", - }, - }, nil - }, + ctlr.CveScanner = mocks.CveScannerMock{ + ScanImageFn: func(image string) (map[string]cvemodel.CVE, error) { + return map[string]cvemodel.CVE{ + "CVE-2023-1255": { + ID: "CVE-2023-1255", + Severity: "LOW", + Title: "Input buffer over-read in AES-XTS implementation and testing", + }, + "CVE-2023-2650": { + ID: "CVE-2023-2650", + Severity: "MEDIUM", + Title: "Possible DoS translating ASN.1 object identifier and executer", + }, + "CVE-2023-2975": { + ID: "CVE-2023-2975", + Severity: "HIGH", + Title: "AES-SIV cipher implementation contains a bug that can break", + }, + "CVE-2023-3446": { + ID: "CVE-2023-3446", + Severity: "CRITICAL", + Title: "Excessive time spent checking DH keys and parenthesis", + }, + "CVE-2023-3817": { + ID: "CVE-2023-3817", + Severity: "MEDIUM", + Title: "Excessive time spent checking DH q parameter and arguments", + }, + }, nil }, } @@ -1373,7 +1369,7 @@ func TestCVECommandErrors(t *testing.T) { }) } -func getMockCveInfo(metaDB mTypes.MetaDB, log log.Logger) cveinfo.CveInfo { +func getMockCveScanner(metaDB mTypes.MetaDB) cveinfo.Scanner { // MetaDB loaded with initial data now mock the scanner // Setup test CVE data in mock scanner scanner := mocks.CveScannerMock{ @@ -1472,11 +1468,7 @@ func getMockCveInfo(metaDB mTypes.MetaDB, log log.Logger) cveinfo.CveInfo { }, } - return &cveinfo.BaseCveInfo{ - Log: log, - Scanner: scanner, - MetaDB: metaDB, - } + return &scanner } type mockServiceForRetry struct { diff --git a/pkg/extensions/extension_search.go b/pkg/extensions/extension_search.go index 83024a51..ecfa0461 100644 --- a/pkg/extensions/extension_search.go +++ b/pkg/extensions/extension_search.go @@ -4,9 +4,7 @@ package extensions import ( - "context" "net/http" - "sync" "time" gqlHandler "github.com/99designs/gqlgen/graphql/handler" @@ -24,145 +22,58 @@ import ( "zotregistry.io/zot/pkg/storage" ) -type ( - CveInfo cveinfo.CveInfo - state int -) +const scanInterval = 15 * time.Minute -const ( - pending state = iota - running - done -) +type CveScanner cveinfo.Scanner func IsBuiltWithSearchExtension() bool { return true } -func GetCVEInfo(config *config.Config, storeController storage.StoreController, +func GetCveScanner(conf *config.Config, storeController storage.StoreController, metaDB mTypes.MetaDB, log log.Logger, -) CveInfo { - if config.Extensions.Search == nil || !*config.Extensions.Search.Enable || config.Extensions.Search.CVE == nil { +) CveScanner { + if !conf.IsCveScanningEnabled() { return nil } - dbRepository := config.Extensions.Search.CVE.Trivy.DBRepository - javaDBRepository := config.Extensions.Search.CVE.Trivy.JavaDBRepository + dbRepository := conf.Extensions.Search.CVE.Trivy.DBRepository + javaDBRepository := conf.Extensions.Search.CVE.Trivy.JavaDBRepository - return cveinfo.NewCVEInfo(storeController, metaDB, dbRepository, javaDBRepository, log) + return cveinfo.NewScanner(storeController, metaDB, dbRepository, javaDBRepository, log) } -func EnableSearchExtension(config *config.Config, storeController storage.StoreController, - metaDB mTypes.MetaDB, taskScheduler *scheduler.Scheduler, cveInfo CveInfo, log log.Logger, +func EnableSearchExtension(conf *config.Config, storeController storage.StoreController, + metaDB mTypes.MetaDB, taskScheduler *scheduler.Scheduler, cveScanner CveScanner, log log.Logger, ) { - if config.Extensions.Search != nil && *config.Extensions.Search.Enable && config.Extensions.Search.CVE != nil { - updateInterval := config.Extensions.Search.CVE.UpdateInterval + if conf.IsCveScanningEnabled() { + updateInterval := conf.Extensions.Search.CVE.UpdateInterval - downloadTrivyDB(updateInterval, taskScheduler, cveInfo, log) + downloadTrivyDB(updateInterval, taskScheduler, cveScanner, log) + startScanner(scanInterval, metaDB, taskScheduler, cveScanner, log) } else { log.Info().Msg("CVE config not provided, skipping CVE update") } } -func downloadTrivyDB(interval time.Duration, sch *scheduler.Scheduler, cveInfo CveInfo, log log.Logger) { - generator := NewTrivyTaskGenerator(interval, cveInfo, log) +func downloadTrivyDB(interval time.Duration, sch *scheduler.Scheduler, cveScanner CveScanner, log log.Logger) { + generator := cveinfo.NewDBUpdateTaskGenerator(interval, cveScanner, log) log.Info().Msg("Submitting CVE DB update scheduler") sch.SubmitGenerator(generator, interval, scheduler.HighPriority) } -func NewTrivyTaskGenerator(interval time.Duration, cveInfo CveInfo, log log.Logger) *TrivyTaskGenerator { - generator := &TrivyTaskGenerator{interval, cveInfo, log, pending, 0, time.Now(), &sync.Mutex{}} +func startScanner(interval time.Duration, metaDB mTypes.MetaDB, sch *scheduler.Scheduler, + cveScanner CveScanner, log log.Logger, +) { + generator := cveinfo.NewScanTaskGenerator(metaDB, cveScanner, log) - return generator -} - -type TrivyTaskGenerator struct { - interval time.Duration - cveInfo CveInfo - log log.Logger - status state - waitTime time.Duration - lastTaskTime time.Time - lock *sync.Mutex -} - -func (gen *TrivyTaskGenerator) Next() (scheduler.Task, error) { - var newTask scheduler.Task - - gen.lock.Lock() - - if gen.status == pending && time.Since(gen.lastTaskTime) >= gen.waitTime { - newTask = newTrivyTask(gen.interval, gen.cveInfo, gen, gen.log) - gen.status = running - } - gen.lock.Unlock() - - return newTask, nil -} - -func (gen *TrivyTaskGenerator) IsDone() bool { - gen.lock.Lock() - status := gen.status - gen.lock.Unlock() - - return status == done -} - -func (gen *TrivyTaskGenerator) IsReady() bool { - return true -} - -func (gen *TrivyTaskGenerator) Reset() { - gen.lock.Lock() - gen.status = pending - gen.waitTime = 0 - gen.lock.Unlock() -} - -type trivyTask struct { - interval time.Duration - cveInfo cveinfo.CveInfo - generator *TrivyTaskGenerator - log log.Logger -} - -func newTrivyTask(interval time.Duration, cveInfo cveinfo.CveInfo, - generator *TrivyTaskGenerator, log log.Logger, -) *trivyTask { - return &trivyTask{interval, cveInfo, generator, log} -} - -func (trivyT *trivyTask) DoWork(ctx context.Context) error { - trivyT.log.Info().Msg("updating the CVE database") - - err := trivyT.cveInfo.UpdateDB() - if err != nil { - trivyT.generator.lock.Lock() - trivyT.generator.status = pending - - if trivyT.generator.waitTime == 0 { - trivyT.generator.waitTime = time.Second - } - - trivyT.generator.waitTime *= 2 - trivyT.generator.lastTaskTime = time.Now() - trivyT.generator.lock.Unlock() - - return err - } - - trivyT.generator.lock.Lock() - trivyT.generator.lastTaskTime = time.Now() - trivyT.generator.status = done - trivyT.generator.lock.Unlock() - trivyT.log.Info().Str("DB update completed, next update scheduled after", trivyT.interval.String()).Msg("") - - return nil + log.Info().Msg("Submitting CVE scan scheduler") + sch.SubmitGenerator(generator, interval, scheduler.MediumPriority) } func SetupSearchRoutes(conf *config.Config, router *mux.Router, storeController storage.StoreController, - metaDB mTypes.MetaDB, cveInfo CveInfo, log log.Logger, + metaDB mTypes.MetaDB, cveScanner CveScanner, log log.Logger, ) { if !conf.IsSearchEnabled() { log.Info().Msg("skip enabling the search route as the config prerequisites are not met") @@ -172,6 +83,13 @@ func SetupSearchRoutes(conf *config.Config, router *mux.Router, storeController log.Info().Msg("setting up search routes") + var cveInfo cveinfo.CveInfo + if conf.IsCveScanningEnabled() { + cveInfo = cveinfo.NewCVEInfo(cveScanner, metaDB, log) + } else { + cveInfo = nil + } + resConfig := search.GetResolverConfig(log, storeController, metaDB, cveInfo) allowedMethods := zcommon.AllowedMethods(http.MethodGet, http.MethodPost) diff --git a/pkg/extensions/extension_search_disabled.go b/pkg/extensions/extension_search_disabled.go index a898d747..0001ba40 100644 --- a/pkg/extensions/extension_search_disabled.go +++ b/pkg/extensions/extension_search_disabled.go @@ -13,11 +13,11 @@ import ( "zotregistry.io/zot/pkg/storage" ) -type CveInfo interface{} +type CveScanner interface{} -func GetCVEInfo(config *config.Config, storeController storage.StoreController, +func GetCveScanner(config *config.Config, storeController storage.StoreController, metaDB mTypes.MetaDB, log log.Logger, -) CveInfo { +) CveScanner { return nil } @@ -27,7 +27,7 @@ func IsBuiltWithSearchExtension() bool { // EnableSearchExtension ... func EnableSearchExtension(config *config.Config, storeController storage.StoreController, - metaDB mTypes.MetaDB, scheduler *scheduler.Scheduler, cveInfo CveInfo, log log.Logger, + metaDB mTypes.MetaDB, scheduler *scheduler.Scheduler, cveScanner CveScanner, log log.Logger, ) { log.Warn().Msg("skipping enabling search extension because given zot binary doesn't include this feature," + "please build a binary that does so") @@ -35,7 +35,7 @@ func EnableSearchExtension(config *config.Config, storeController storage.StoreC // SetupSearchRoutes ... func SetupSearchRoutes(config *config.Config, router *mux.Router, storeController storage.StoreController, - metaDB mTypes.MetaDB, cveInfo CveInfo, log log.Logger, + metaDB mTypes.MetaDB, cveScanner CveScanner, log log.Logger, ) { log.Warn().Msg("skipping setting up search routes because given zot binary doesn't include this feature," + "please build a binary that does so") diff --git a/pkg/extensions/search/cve/trivy/cache.go b/pkg/extensions/search/cve/cache/cache.go similarity index 86% rename from pkg/extensions/search/cve/trivy/cache.go rename to pkg/extensions/search/cve/cache/cache.go index e384d4d7..20d3e2c5 100644 --- a/pkg/extensions/search/cve/trivy/cache.go +++ b/pkg/extensions/search/cve/cache/cache.go @@ -1,4 +1,4 @@ -package trivy +package cache import ( lru "github.com/hashicorp/golang-lru/v2" @@ -22,6 +22,10 @@ func (cveCache *CveCache) Add(image string, cveMap map[string]cvemodel.CVE) { cveCache.cache.Add(image, cveMap) } +func (cveCache *CveCache) Contains(image string) bool { + return cveCache.cache.Contains(image) +} + func (cveCache *CveCache) Get(image string) map[string]cvemodel.CVE { cveMap, ok := cveCache.cache.Get(image) if !ok { diff --git a/pkg/extensions/search/cve/cve.go b/pkg/extensions/search/cve/cve.go index ae9df83d..f4747cff 100644 --- a/pkg/extensions/search/cve/cve.go +++ b/pkg/extensions/search/cve/cve.go @@ -22,15 +22,15 @@ type CveInfo interface { GetImageListWithCVEFixed(repo, cveID string) ([]cvemodel.TagInfo, error) GetCVEListForImage(repo, tag string, searchedCVE string, pageinput cvemodel.PageInput, ) ([]cvemodel.CVE, zcommon.PageInfo, error) - GetCVESummaryForImage(repo, ref string) (cvemodel.ImageCVESummary, error) GetCVESummaryForImageMedia(repo, digest, mediaType string) (cvemodel.ImageCVESummary, error) - UpdateDB() error } type Scanner interface { ScanImage(image string) (map[string]cvemodel.CVE, error) IsImageFormatScannable(repo, ref string) (bool, error) IsImageMediaScannable(repo, digestStr, mediaType string) (bool, error) + IsResultCached(digestStr string) bool + GetCachedResult(digestStr string) map[string]cvemodel.CVE UpdateDB() error } @@ -40,11 +40,13 @@ type BaseCveInfo struct { MetaDB mTypes.MetaDB } -func NewCVEInfo(storeController storage.StoreController, metaDB mTypes.MetaDB, +func NewScanner(storeController storage.StoreController, metaDB mTypes.MetaDB, dbRepository, javaDBRepository string, log log.Logger, -) *BaseCveInfo { - scanner := trivy.NewScanner(storeController, metaDB, dbRepository, javaDBRepository, log) +) Scanner { + return trivy.NewScanner(storeController, metaDB, dbRepository, javaDBRepository, log) +} +func NewCVEInfo(scanner Scanner, metaDB mTypes.MetaDB, log log.Logger) *BaseCveInfo { return &BaseCveInfo{ Log: log, Scanner: scanner, @@ -72,7 +74,7 @@ func (cveinfo BaseCveInfo) GetImageListForCVE(repo, cveID string) ([]cvemodel.Ta isScanableImage, err := cveinfo.Scanner.IsImageFormatScannable(repo, manifestDigestStr) if !isScanableImage || err != nil { - cveinfo.Log.Info().Str("image", repo+":"+tag).Err(err).Msg("image is not scanable") + cveinfo.Log.Debug().Str("image", repo+":"+tag).Err(err).Msg("image is not scanable") continue } @@ -94,7 +96,8 @@ func (cveinfo BaseCveInfo) GetImageListForCVE(repo, cveID string) ([]cvemodel.Ta }) } default: - cveinfo.Log.Error().Str("mediaType", descriptor.MediaType).Msg("media type not supported for scanning") + cveinfo.Log.Debug().Str("image", repo+":"+tag).Str("mediaType", descriptor.MediaType). + Msg("image media type not supported for scanning") } } @@ -187,7 +190,8 @@ func (cveinfo BaseCveInfo) GetImageListWithCVEFixed(repo, cveID string) ([]cvemo }) } default: - cveinfo.Log.Error().Str("mediaType", descriptor.MediaType).Msg("media type not supported") + cveinfo.Log.Debug().Str("mediaType", descriptor.MediaType). + Msg("image media type not supported for scanning") } } @@ -250,7 +254,7 @@ func (cveinfo *BaseCveInfo) isManifestVulnerable(repo, tag, manifestDigestStr, c isValidImage, err := cveinfo.Scanner.IsImageMediaScannable(repo, manifestDigestStr, ispec.MediaTypeImageManifest) if !isValidImage || err != nil { - cveinfo.Log.Debug().Str("image", image).Str("cve-id", cveID). + cveinfo.Log.Debug().Str("image", image).Str("cve-id", cveID).Err(err). Msg("image media type not supported for scanning, adding as a vulnerable image") return true @@ -335,6 +339,8 @@ func (cveinfo BaseCveInfo) GetCVEListForImage(repo, ref string, searchedCVE stri ) { isValidImage, err := cveinfo.Scanner.IsImageFormatScannable(repo, ref) if !isValidImage { + cveinfo.Log.Debug().Str("image", repo+":"+ref).Err(err).Msg("image is not scanable") + return []cvemodel.CVE{}, zcommon.PageInfo{}, err } @@ -357,50 +363,11 @@ func (cveinfo BaseCveInfo) GetCVEListForImage(repo, ref string, searchedCVE stri return cveList, pageInfo, nil } -func (cveinfo BaseCveInfo) GetCVESummaryForImage(repo, ref string) (cvemodel.ImageCVESummary, error) { - // There are several cases, expected returned values below: - // not scannable / error during scan - max severity "" - cve count 0 - Errors - // scannable no issues found - max severity "NONE" - cve count 0 - no Errors - // scannable issues found - max severity from Scanner - cve count >0 - no Errors - imageCVESummary := cvemodel.ImageCVESummary{ - Count: 0, - MaxSeverity: cvemodel.SeverityNotScanned, - } - - isValidImage, err := cveinfo.Scanner.IsImageFormatScannable(repo, ref) - if !isValidImage { - return imageCVESummary, err - } - - image := zcommon.GetFullImageName(repo, ref) - - cveMap, err := cveinfo.Scanner.ScanImage(image) - if err != nil { - return imageCVESummary, err - } - - imageCVESummary.Count = len(cveMap) - - if imageCVESummary.Count == 0 { - imageCVESummary.MaxSeverity = cvemodel.SeverityNone - - return imageCVESummary, nil - } - - imageCVESummary.MaxSeverity = cvemodel.SeverityUnknown - for _, cve := range cveMap { - if cvemodel.CompareSeverities(imageCVESummary.MaxSeverity, cve.Severity) > 0 { - imageCVESummary.MaxSeverity = cve.Severity - } - } - - return imageCVESummary, nil -} - func (cveinfo BaseCveInfo) GetCVESummaryForImageMedia(repo, digest, mediaType string, ) (cvemodel.ImageCVESummary, error) { // There are several cases, expected returned values below: - // not scannable / error during scan - max severity "" - cve count 0 - Errors + // not scanned yet - max severity "" - cve count 0 - no Errors + // not scannable - max severity "" - cve count 0 - has Errors // scannable no issues found - max severity "NONE" - cve count 0 - no Errors // scannable issues found - max severity from Scanner - cve count >0 - no Errors imageCVESummary := cvemodel.ImageCVESummary{ @@ -408,20 +375,21 @@ func (cveinfo BaseCveInfo) GetCVESummaryForImageMedia(repo, digest, mediaType st MaxSeverity: cvemodel.SeverityNotScanned, } - isValidImage, err := cveinfo.Scanner.IsImageMediaScannable(repo, digest, mediaType) - if !isValidImage { + // For this call we only look at the scanner cache, we skip the actual scanning to save time + if !cveinfo.Scanner.IsResultCached(digest) { + isValidImage, err := cveinfo.Scanner.IsImageMediaScannable(repo, digest, mediaType) + if !isValidImage { + cveinfo.Log.Debug().Str("digest", digest).Str("mediaType", mediaType). + Err(err).Msg("image is not scannable") + } + return imageCVESummary, err } - image := repo + "@" + digest - - cveMap, err := cveinfo.Scanner.ScanImage(image) - if err != nil { - return imageCVESummary, err - } + // We will make due with cached results + cveMap := cveinfo.Scanner.GetCachedResult(digest) imageCVESummary.Count = len(cveMap) - if imageCVESummary.Count == 0 { imageCVESummary.MaxSeverity = cvemodel.SeverityNone @@ -438,10 +406,6 @@ func (cveinfo BaseCveInfo) GetCVESummaryForImageMedia(repo, digest, mediaType st return imageCVESummary, nil } -func (cveinfo BaseCveInfo) UpdateDB() error { - return cveinfo.Scanner.UpdateDB() -} - func GetFixedTags(allTags, vulnerableTags []cvemodel.TagInfo) []cvemodel.TagInfo { sort.Slice(allTags, func(i, j int) bool { return allTags[i].Timestamp.Before(allTags[j].Timestamp) diff --git a/pkg/extensions/search/cve/cve_test.go b/pkg/extensions/search/cve/cve_test.go index 1637df5d..6743608d 100644 --- a/pkg/extensions/search/cve/cve_test.go +++ b/pkg/extensions/search/cve/cve_test.go @@ -30,8 +30,8 @@ import ( extconf "zotregistry.io/zot/pkg/extensions/config" "zotregistry.io/zot/pkg/extensions/monitoring" cveinfo "zotregistry.io/zot/pkg/extensions/search/cve" + cvecache "zotregistry.io/zot/pkg/extensions/search/cve/cache" cvemodel "zotregistry.io/zot/pkg/extensions/search/cve/model" - "zotregistry.io/zot/pkg/extensions/search/cve/trivy" "zotregistry.io/zot/pkg/log" "zotregistry.io/zot/pkg/meta" "zotregistry.io/zot/pkg/meta/boltdb" @@ -339,49 +339,49 @@ func TestImageFormat(t *testing.T) { err = meta.ParseStorage(metaDB, storeController, log) So(err, ShouldBeNil) - cveInfo := cveinfo.NewCVEInfo(storeController, metaDB, "ghcr.io/project-zot/trivy-db", "", log) + scanner := cveinfo.NewScanner(storeController, metaDB, "ghcr.io/project-zot/trivy-db", "", log) - isValidImage, err := cveInfo.Scanner.IsImageFormatScannable("zot-test", "") + isValidImage, err := scanner.IsImageFormatScannable("zot-test", "") So(err, ShouldNotBeNil) So(isValidImage, ShouldEqual, false) - isValidImage, err = cveInfo.Scanner.IsImageFormatScannable("zot-test", "0.0.1") + isValidImage, err = scanner.IsImageFormatScannable("zot-test", "0.0.1") So(err, ShouldBeNil) So(isValidImage, ShouldEqual, true) - isValidImage, err = cveInfo.Scanner.IsImageFormatScannable("zot-test", "0.0.") + isValidImage, err = scanner.IsImageFormatScannable("zot-test", "0.0.") So(err, ShouldNotBeNil) So(isValidImage, ShouldEqual, false) - isValidImage, err = cveInfo.Scanner.IsImageFormatScannable("zot-noindex-test", "") + isValidImage, err = scanner.IsImageFormatScannable("zot-noindex-test", "") So(err, ShouldNotBeNil) So(isValidImage, ShouldEqual, false) - isValidImage, err = cveInfo.Scanner.IsImageFormatScannable("zot--tet", "") + isValidImage, err = scanner.IsImageFormatScannable("zot--tet", "") So(err, ShouldNotBeNil) So(isValidImage, ShouldEqual, false) - isValidImage, err = cveInfo.Scanner.IsImageFormatScannable("zot-noindex-test", "") + isValidImage, err = scanner.IsImageFormatScannable("zot-noindex-test", "") So(err, ShouldNotBeNil) So(isValidImage, ShouldEqual, false) - isValidImage, err = cveInfo.Scanner.IsImageFormatScannable("zot-squashfs-noblobs", "") + isValidImage, err = scanner.IsImageFormatScannable("zot-squashfs-noblobs", "") So(err, ShouldNotBeNil) So(isValidImage, ShouldEqual, false) - isValidImage, err = cveInfo.Scanner.IsImageFormatScannable("zot-squashfs-invalid-index", "") + isValidImage, err = scanner.IsImageFormatScannable("zot-squashfs-invalid-index", "") So(err, ShouldNotBeNil) So(isValidImage, ShouldEqual, false) - isValidImage, err = cveInfo.Scanner.IsImageFormatScannable("zot-squashfs-invalid-blob", "") + isValidImage, err = scanner.IsImageFormatScannable("zot-squashfs-invalid-blob", "") So(err, ShouldNotBeNil) So(isValidImage, ShouldEqual, false) - isValidImage, err = cveInfo.Scanner.IsImageFormatScannable("zot-squashfs-test:0.3.22-squashfs", "") + isValidImage, err = scanner.IsImageFormatScannable("zot-squashfs-test:0.3.22-squashfs", "") So(err, ShouldNotBeNil) So(isValidImage, ShouldEqual, false) - isValidImage, err = cveInfo.Scanner.IsImageFormatScannable("zot-nonreadable-test", "") + isValidImage, err = scanner.IsImageFormatScannable("zot-nonreadable-test", "") So(err, ShouldNotBeNil) So(isValidImage, ShouldEqual, false) }) @@ -408,9 +408,9 @@ func TestImageFormat(t *testing.T) { DefaultStore: mocks.MockedImageStore{}, } - cveInfo := cveinfo.NewCVEInfo(storeController, metaDB, "ghcr.io/project-zot/trivy-db", "", log) + scanner := cveinfo.NewScanner(storeController, metaDB, "ghcr.io/project-zot/trivy-db", "", log) - isScanable, err := cveInfo.Scanner.IsImageFormatScannable("repo", "tag") + isScanable, err := scanner.IsImageFormatScannable("repo", "tag") So(err, ShouldBeNil) So(isScanable, ShouldBeTrue) }) @@ -739,8 +739,18 @@ func TestCVESearch(t *testing.T) { }) } -func TestCVEStruct(t *testing.T) { +func TestCVEStruct(t *testing.T) { //nolint:gocyclo Convey("Unit test the CVE struct", t, func() { + const repo1 = "repo1" + const repo2 = "repo2" + const repo3 = "repo3" + const repo4 = "repo4" + const repo5 = "repo5" + const repo6 = "repo6" + const repo7 = "repo7" + const repo100 = "repo100" + const repoMultiarch = "repoIndex" + params := boltdb.DBParameters{ RootDir: t.TempDir(), } @@ -751,214 +761,93 @@ func TestCVEStruct(t *testing.T) { So(err, ShouldBeNil) // Create metadb data for scannable image with vulnerabilities - timeStamp11 := time.Date(2008, 1, 1, 12, 0, 0, 0, time.UTC) - - configBlob11, err := json.Marshal(ispec.Image{ - Created: &timeStamp11, - }) - So(err, ShouldBeNil) - - manifestBlob11, err := json.Marshal(ispec.Manifest{ - Config: ispec.Descriptor{ - MediaType: ispec.MediaTypeImageConfig, - Size: 0, - Digest: godigest.FromBytes(configBlob11), - }, - Layers: []ispec.Descriptor{ - { - MediaType: ispec.MediaTypeImageLayerGzip, - Size: 0, - Digest: godigest.NewDigestFromEncoded(godigest.SHA256, "digest"), - }, - }, - }) - So(err, ShouldBeNil) + image11 := CreateImageWith().DefaultLayers(). + ImageConfig(ispec.Image{Created: DateRef(2008, 1, 1, 12, 0, 0, 0, time.UTC)}).Build() repoMeta11 := mTypes.ManifestMetadata{ - ManifestBlob: manifestBlob11, - ConfigBlob: configBlob11, + ManifestBlob: image11.ManifestDescriptor.Data, + ConfigBlob: image11.ConfigDescriptor.Data, DownloadCount: 0, Signatures: mTypes.ManifestSignatures{}, } - digest11 := godigest.FromBytes(manifestBlob11) - err = metaDB.SetManifestMeta("repo1", digest11, repoMeta11) + err = metaDB.SetManifestMeta(repo1, image11.ManifestDescriptor.Digest, repoMeta11) So(err, ShouldBeNil) - err = metaDB.SetRepoReference("repo1", "0.1.0", digest11, ispec.MediaTypeImageManifest) + err = metaDB.SetRepoReference(repo1, "0.1.0", image11.ManifestDescriptor.Digest, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) - timeStamp12 := time.Date(2009, 1, 1, 12, 0, 0, 0, time.UTC) - - configBlob12, err := json.Marshal(ispec.Image{ - Created: &timeStamp12, - }) - So(err, ShouldBeNil) - - manifestBlob12, err := json.Marshal(ispec.Manifest{ - Config: ispec.Descriptor{ - MediaType: ispec.MediaTypeImageConfig, - Size: 0, - Digest: godigest.FromBytes(configBlob12), - }, - Layers: []ispec.Descriptor{ - { - MediaType: ispec.MediaTypeImageLayerGzip, - Size: 0, - Digest: godigest.NewDigestFromEncoded(godigest.SHA256, "digest"), - }, - }, - }) - So(err, ShouldBeNil) + image12 := CreateImageWith().DefaultLayers(). + ImageConfig(ispec.Image{Created: DateRef(2009, 1, 1, 12, 0, 0, 0, time.UTC)}).Build() repoMeta12 := mTypes.ManifestMetadata{ - ManifestBlob: manifestBlob12, - ConfigBlob: configBlob12, + ManifestBlob: image12.ManifestDescriptor.Data, + ConfigBlob: image12.ConfigDescriptor.Data, DownloadCount: 0, Signatures: mTypes.ManifestSignatures{}, } - digest12 := godigest.FromBytes(manifestBlob12) - err = metaDB.SetManifestMeta("repo1", digest12, repoMeta12) + err = metaDB.SetManifestMeta(repo1, image12.ManifestDescriptor.Digest, repoMeta12) So(err, ShouldBeNil) - err = metaDB.SetRepoReference("repo1", "1.0.0", digest12, ispec.MediaTypeImageManifest) + err = metaDB.SetRepoReference(repo1, "1.0.0", image12.ManifestDescriptor.Digest, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) - timeStamp13 := time.Date(2010, 1, 1, 12, 0, 0, 0, time.UTC) - - configBlob13, err := json.Marshal(ispec.Image{ - Created: &timeStamp13, - }) - So(err, ShouldBeNil) - - manifestBlob13, err := json.Marshal(ispec.Manifest{ - Config: ispec.Descriptor{ - MediaType: ispec.MediaTypeImageConfig, - Size: 0, - Digest: godigest.FromBytes(configBlob13), - }, - Layers: []ispec.Descriptor{ - { - MediaType: ispec.MediaTypeImageLayerGzip, - Size: 0, - Digest: godigest.NewDigestFromEncoded(godigest.SHA256, "digest"), - }, - }, - }) - So(err, ShouldBeNil) + image13 := CreateImageWith().DefaultLayers(). + ImageConfig(ispec.Image{Created: DateRef(2010, 1, 1, 12, 0, 0, 0, time.UTC)}).Build() repoMeta13 := mTypes.ManifestMetadata{ - ManifestBlob: manifestBlob13, - ConfigBlob: configBlob13, + ManifestBlob: image13.ManifestDescriptor.Data, + ConfigBlob: image13.ConfigDescriptor.Data, + DownloadCount: 0, + Signatures: mTypes.ManifestSignatures{}, } - digest13 := godigest.FromBytes(manifestBlob13) - err = metaDB.SetManifestMeta("repo1", digest13, repoMeta13) + err = metaDB.SetManifestMeta(repo1, image13.ManifestDescriptor.Digest, repoMeta13) So(err, ShouldBeNil) - err = metaDB.SetRepoReference("repo1", "1.1.0", digest13, ispec.MediaTypeImageManifest) + err = metaDB.SetRepoReference(repo1, "1.1.0", image13.ManifestDescriptor.Digest, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) - timeStamp14 := time.Date(2011, 1, 1, 12, 0, 0, 0, time.UTC) - - configBlob14, err := json.Marshal(ispec.Image{ - Created: &timeStamp14, - }) - So(err, ShouldBeNil) - - manifestBlob14, err := json.Marshal(ispec.Manifest{ - Config: ispec.Descriptor{ - MediaType: ispec.MediaTypeImageConfig, - Size: 0, - Digest: godigest.FromBytes(configBlob14), - }, - Layers: []ispec.Descriptor{ - { - MediaType: ispec.MediaTypeImageLayerGzip, - Size: 0, - Digest: godigest.NewDigestFromEncoded(godigest.SHA256, "digest"), - }, - }, - }) - So(err, ShouldBeNil) + image14 := CreateImageWith().DefaultLayers(). + ImageConfig(ispec.Image{Created: DateRef(2011, 1, 1, 12, 0, 0, 0, time.UTC)}).Build() repoMeta14 := mTypes.ManifestMetadata{ - ManifestBlob: manifestBlob14, - ConfigBlob: configBlob14, + ManifestBlob: image14.ManifestDescriptor.Data, + ConfigBlob: image14.ConfigDescriptor.Data, } - digest14 := godigest.FromBytes(manifestBlob14) - err = metaDB.SetManifestMeta("repo1", digest14, repoMeta14) + err = metaDB.SetManifestMeta(repo1, image14.ManifestDescriptor.Digest, repoMeta14) So(err, ShouldBeNil) - err = metaDB.SetRepoReference("repo1", "1.0.1", digest14, ispec.MediaTypeImageManifest) + err = metaDB.SetRepoReference(repo1, "1.0.1", image14.ManifestDescriptor.Digest, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) // Create metadb data for scannable image with no vulnerabilities - timeStamp61 := time.Date(2011, 1, 1, 12, 0, 0, 0, time.UTC) - - configBlob61, err := json.Marshal(ispec.Image{ - Created: &timeStamp61, - }) - So(err, ShouldBeNil) - - manifestBlob61, err := json.Marshal(ispec.Manifest{ - Config: ispec.Descriptor{ - MediaType: ispec.MediaTypeImageConfig, - Size: 0, - Digest: godigest.FromBytes(configBlob61), - }, - Layers: []ispec.Descriptor{ - { - MediaType: ispec.MediaTypeImageLayerGzip, - Size: 0, - Digest: godigest.NewDigestFromEncoded(godigest.SHA256, "digest"), - }, - }, - }) - So(err, ShouldBeNil) + image61 := CreateImageWith().DefaultLayers(). + ImageConfig(ispec.Image{Created: DateRef(2016, 1, 1, 12, 0, 0, 0, time.UTC)}).Build() repoMeta61 := mTypes.ManifestMetadata{ - ManifestBlob: manifestBlob61, - ConfigBlob: configBlob61, + ManifestBlob: image61.ManifestDescriptor.Data, + ConfigBlob: image61.ConfigDescriptor.Data, } - digest61 := godigest.FromBytes(manifestBlob61) - err = metaDB.SetManifestMeta("repo6", digest61, repoMeta61) + err = metaDB.SetManifestMeta(repo6, image61.ManifestDescriptor.Digest, repoMeta61) So(err, ShouldBeNil) - err = metaDB.SetRepoReference("repo6", "1.0.0", digest61, ispec.MediaTypeImageManifest) + err = metaDB.SetRepoReference(repo6, "1.0.0", image61.ManifestDescriptor.Digest, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) // Create metadb data for image not supporting scanning - timeStamp21 := time.Date(2009, 1, 1, 12, 0, 0, 0, time.UTC) - - configBlob21, err := json.Marshal(ispec.Image{ - Created: &timeStamp21, - }) - So(err, ShouldBeNil) - - manifestBlob21, err := json.Marshal(ispec.Manifest{ - Config: ispec.Descriptor{ - MediaType: ispec.MediaTypeImageConfig, - Size: 0, - Digest: godigest.FromBytes(configBlob21), - }, - Layers: []ispec.Descriptor{ - { - MediaType: ispec.MediaTypeImageLayerNonDistributableGzip, //nolint:staticcheck - Size: 0, - Digest: godigest.NewDigestFromEncoded(godigest.SHA256, "digest"), - }, - }, - }) - So(err, ShouldBeNil) + image21 := CreateImageWith().Layers([]Layer{{ + MediaType: ispec.MediaTypeImageLayerNonDistributableGzip, //nolint:staticcheck + Blob: []byte{10, 10, 10}, + Digest: godigest.FromBytes([]byte{10, 10, 10}), + }}).ImageConfig(ispec.Image{Created: DateRef(2009, 1, 1, 12, 0, 0, 0, time.UTC)}).Build() repoMeta21 := mTypes.ManifestMetadata{ - ManifestBlob: manifestBlob21, - ConfigBlob: configBlob21, + ManifestBlob: image21.ManifestDescriptor.Data, + ConfigBlob: image21.ConfigDescriptor.Data, } - digest21 := godigest.FromBytes(manifestBlob21) - err = metaDB.SetManifestMeta("repo2", digest21, repoMeta21) + err = metaDB.SetManifestMeta(repo2, image21.ManifestDescriptor.Digest, repoMeta21) So(err, ShouldBeNil) - err = metaDB.SetRepoReference("repo2", "1.0.0", digest21, ispec.MediaTypeImageManifest) + err = metaDB.SetRepoReference(repo2, "1.0.0", image21.ManifestDescriptor.Digest, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) // Create metadb data for invalid images/negative tests @@ -970,87 +859,192 @@ func TestCVEStruct(t *testing.T) { } digest31 := godigest.FromBytes(manifestBlob31) - err = metaDB.SetManifestMeta("repo3", digest31, repoMeta31) + err = metaDB.SetManifestMeta(repo3, digest31, repoMeta31) So(err, ShouldBeNil) - err = metaDB.SetRepoReference("repo3", "invalid-manifest", digest31, ispec.MediaTypeImageManifest) + err = metaDB.SetRepoReference(repo3, "invalid-manifest", digest31, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) - configBlob41 := []byte("invalid config blob") - So(err, ShouldBeNil) + image41 := CreateImageWith().DefaultLayers(). + CustomConfigBlob([]byte("invalid config blob"), ispec.MediaTypeImageConfig).Build() repoMeta41 := mTypes.ManifestMetadata{ - ConfigBlob: configBlob41, + ManifestBlob: image41.ManifestDescriptor.Data, + ConfigBlob: image41.ConfigDescriptor.Data, } - digest41 := godigest.FromString("abc7") - err = metaDB.SetManifestMeta("repo4", digest41, repoMeta41) + err = metaDB.SetManifestMeta(repo4, image41.ManifestDescriptor.Digest, repoMeta41) So(err, ShouldBeNil) - err = metaDB.SetRepoReference("repo4", "invalid-config", digest41, ispec.MediaTypeImageManifest) + err = metaDB.SetRepoReference(repo4, "invalid-config", image41.ManifestDescriptor.Digest, + ispec.MediaTypeImageManifest) So(err, ShouldBeNil) digest51 := godigest.FromString("abc8") - err = metaDB.SetRepoReference("repo5", "nonexitent-manifest", digest51, ispec.MediaTypeImageManifest) + err = metaDB.SetRepoReference(repo5, "nonexitent-manifest", digest51, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) - // ------ Multiarch image - _, _, manifestContent1, err := GetRandomImageComponents(100) + // Create metadb data for scannable image which errors during scan + image71 := CreateImageWith().DefaultLayers(). + ImageConfig(ispec.Image{Created: DateRef(2000, 1, 1, 12, 0, 0, 0, time.UTC)}).Build() + + repoMeta71 := mTypes.ManifestMetadata{ + ManifestBlob: image71.ManifestDescriptor.Data, + ConfigBlob: image71.ConfigDescriptor.Data, + } + + err = metaDB.SetManifestMeta(repo7, image71.ManifestDescriptor.Digest, repoMeta71) So(err, ShouldBeNil) - manifestContent1Blob, err := json.Marshal(manifestContent1) - So(err, ShouldBeNil) - diestManifestFromIndex1 := godigest.FromBytes(manifestContent1Blob) - err = metaDB.SetManifestData(diestManifestFromIndex1, mTypes.ManifestData{ - ManifestBlob: manifestContent1Blob, - ConfigBlob: []byte("{}"), - }) + err = metaDB.SetRepoReference(repo7, "1.0.0", image71.ManifestDescriptor.Digest, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) - _, _, manifestContent2, err := GetRandomImageComponents(100) - So(err, ShouldBeNil) - manifestContent2Blob, err := json.Marshal(manifestContent2) - So(err, ShouldBeNil) - diestManifestFromIndex2 := godigest.FromBytes(manifestContent2Blob) - err = metaDB.SetManifestData(diestManifestFromIndex1, mTypes.ManifestData{ - ManifestBlob: manifestContent2Blob, - ConfigBlob: []byte("{}"), - }) - So(err, ShouldBeNil) + // create multiarch image with vulnerabilities + multiarchImage := CreateRandomMultiarch() - indexBlob, err := GetIndexBlobWithManifests( - []godigest.Digest{diestManifestFromIndex1, diestManifestFromIndex2}, + err = metaDB.SetIndexData( + multiarchImage.IndexDescriptor.Digest, + mTypes.IndexData{IndexBlob: multiarchImage.IndexDescriptor.Data}, ) So(err, ShouldBeNil) - indexDigest := godigest.FromBytes(indexBlob) - err = metaDB.SetIndexData(indexDigest, mTypes.IndexData{ - IndexBlob: indexBlob, - }) + err = metaDB.SetManifestData( + multiarchImage.Images[0].ManifestDescriptor.Digest, + mTypes.ManifestData{ + ManifestBlob: multiarchImage.Images[0].ManifestDescriptor.Data, + ConfigBlob: multiarchImage.Images[0].ConfigDescriptor.Data, + }, + ) So(err, ShouldBeNil) - err = metaDB.SetRepoReference("repoIndex", "tagIndex", indexDigest, ispec.MediaTypeImageIndex) + err = metaDB.SetManifestData( + multiarchImage.Images[1].ManifestDescriptor.Digest, + mTypes.ManifestData{ + ManifestBlob: multiarchImage.Images[1].ManifestDescriptor.Data, + ConfigBlob: multiarchImage.Images[1].ConfigDescriptor.Data, + }, + ) So(err, ShouldBeNil) + err = metaDB.SetManifestData( + multiarchImage.Images[2].ManifestDescriptor.Digest, + mTypes.ManifestData{ + ManifestBlob: multiarchImage.Images[2].ManifestDescriptor.Data, + ConfigBlob: multiarchImage.Images[2].ConfigDescriptor.Data, + }, + ) + So(err, ShouldBeNil) + + err = metaDB.SetRepoReference( + repoMultiarch, + "tagIndex", + multiarchImage.IndexDescriptor.Digest, + ispec.MediaTypeImageIndex, + ) + So(err, ShouldBeNil) + + // Keep a record of all the image references / digest pairings + // This is normally done in MetaDB, but we want to verify + // the whole flow, including MetaDB + imageMap := map[string]string{} + + image11Digest := image11.ManifestDescriptor.Digest.String() + image11Media := image11.ManifestDescriptor.MediaType + image11Name := repo1 + ":0.1.0" + imageMap[image11Name] = image11Digest + image12Digest := image12.ManifestDescriptor.Digest.String() + image12Media := image12.ManifestDescriptor.MediaType + image12Name := repo1 + ":1.0.0" + imageMap[image12Name] = image12Digest + image13Digest := image13.ManifestDescriptor.Digest.String() + image13Media := image13.ManifestDescriptor.MediaType + image13Name := repo1 + ":1.1.0" + imageMap[image13Name] = image13Digest + image14Digest := image14.ManifestDescriptor.Digest.String() + image14Media := image14.ManifestDescriptor.MediaType + image14Name := repo1 + ":1.0.1" + imageMap[image14Name] = image14Digest + image21Digest := image21.ManifestDescriptor.Digest.String() + image21Media := image21.ManifestDescriptor.MediaType + image21Name := repo2 + ":1.0.0" + imageMap[image21Name] = image21Digest + image31Digest := digest31.String() + image31Media := ispec.MediaTypeImageManifest + image31Name := repo3 + ":invalid-manifest" + imageMap[image31Name] = image31Digest + image41Digest := image41.ManifestDescriptor.Digest.String() + image41Media := image41.ManifestDescriptor.MediaType + image41Name := repo4 + ":invalid-config" + imageMap[image41Name] = image41Digest + image51Digest := digest51.String() + image51Media := ispec.MediaTypeImageManifest + image51Name := repo5 + ":nonexitent-manifest" + imageMap[image51Name] = digest51.String() + image61Digest := image61.ManifestDescriptor.Digest.String() + image61Media := image61.ManifestDescriptor.MediaType + image61Name := repo6 + ":1.0.0" + imageMap[image61Name] = image61Digest + image71Digest := image71.ManifestDescriptor.Digest.String() + image71Media := image71.ManifestDescriptor.MediaType + image71Name := repo7 + ":1.0.0" + imageMap[image71Name] = image71Digest + indexDigest := multiarchImage.IndexDescriptor.Digest.String() + indexMedia := multiarchImage.IndexDescriptor.MediaType + indexName := repoMultiarch + ":tagIndex" + imageMap[indexName] = indexDigest + indexM1Digest := multiarchImage.Images[0].ManifestDescriptor.Digest.String() + indexM1Name := "repoIndex@" + indexM1Digest + imageMap[indexM1Name] = indexM1Digest + indexM2Digest := multiarchImage.Images[1].ManifestDescriptor.Digest.String() + indexM2Name := "repoIndex@" + indexM2Digest + imageMap[indexM2Name] = indexM2Digest + indexM3Digest := multiarchImage.Images[2].ManifestDescriptor.Digest.String() + indexM3Name := "repoIndex@" + indexM3Digest + imageMap[indexM3Name] = indexM3Digest + + log := log.NewLogger("debug", "") + + // Initialize a test CVE cache + cache := cvecache.NewCveCache(100, log) + // MetaDB loaded with initial data, now mock the scanner // Setup test CVE data in mock scanner scanner := mocks.CveScannerMock{ ScanImageFn: func(image string) (map[string]cvemodel.CVE, error) { - repo1 := "repo1" + result := cache.Get(image) + // Will not match sending the repo:tag as a parameter, but we don't care + if result != nil { + return result, nil + } + + repo, ref, isTag := zcommon.GetImageDirAndReference(image) + if isTag { + foundRef, ok := imageMap[image] + if !ok { + return nil, ErrBadTest + } + ref = foundRef + } + + defer func() { + t.Logf("ScanImageFn cached for image %s digest %s: %v", image, ref, cache.Get(ref)) + }() - repo, ref, _ := zcommon.GetImageDirAndReference(image) // Images in chronological order - if image == "repo1:0.1.0" || ref == digest11.String() { - return map[string]cvemodel.CVE{ + if repo == repo1 && ref == image11Digest { + result := map[string]cvemodel.CVE{ "CVE1": { ID: "CVE1", Severity: "MEDIUM", Title: "Title CVE1", Description: "Description CVE1", }, - }, nil + } + + cache.Add(ref, result) + + return result, nil } - if image == "repo1:1.0.0" || (repo == repo1 && - zcommon.Contains([]string{digest12.String(), digest21.String()}, ref)) { - return map[string]cvemodel.CVE{ + if repo == repo1 && zcommon.Contains([]string{image12Digest, image21Digest}, ref) { + result := map[string]cvemodel.CVE{ "CVE1": { ID: "CVE1", Severity: "MEDIUM", @@ -1069,24 +1063,32 @@ func TestCVEStruct(t *testing.T) { Title: "Title CVE3", Description: "Description CVE3", }, - }, nil + } + + cache.Add(ref, result) + + return result, nil } - if image == "repo1:1.1.0" || (repo == repo1 && ref == digest13.String()) { - return map[string]cvemodel.CVE{ + if repo == repo1 && ref == image13Digest { + result := map[string]cvemodel.CVE{ "CVE3": { ID: "CVE3", Severity: "LOW", Title: "Title CVE3", Description: "Description CVE3", }, - }, nil + } + + cache.Add(ref, result) + + return result, nil } // As a minor release on 1.0.0 banch // does not include all fixes published in 1.1.0 - if image == "repo1:1.0.1" || (repo == repo1 && ref == digest14.String()) { - return map[string]cvemodel.CVE{ + if repo == repo1 && ref == image14Digest { + result := map[string]cvemodel.CVE{ "CVE1": { ID: "CVE1", Severity: "MEDIUM", @@ -1099,25 +1101,49 @@ func TestCVEStruct(t *testing.T) { Title: "Title CVE3", Description: "Description CVE3", }, - }, nil + } + + cache.Add(ref, result) + + return result, nil } - if image == "repoIndex:tagIndex" || (repo == "repoIndex" && ref == indexDigest.String()) { - return map[string]cvemodel.CVE{ + // Unexpected error while scanning + if repo == repo7 { + return map[string]cvemodel.CVE{}, ErrFailedScan + } + + if (repo == repoMultiarch && ref == indexDigest) || + (repo == repoMultiarch && ref == indexM1Digest) { + result := map[string]cvemodel.CVE{ "CVE1": { ID: "CVE1", Severity: "MEDIUM", Title: "Title CVE1", Description: "Description CVE1", }, - }, nil + } + + // Simulate scanning an index results in scanning its manifests + if ref == indexDigest { + cache.Add(indexM1Digest, result) + cache.Add(indexM2Digest, map[string]cvemodel.CVE{}) + cache.Add(indexM3Digest, map[string]cvemodel.CVE{}) + } + + cache.Add(ref, result) + + return result, nil } // By default the image has no vulnerabilities - return map[string]cvemodel.CVE{}, nil + result = map[string]cvemodel.CVE{} + cache.Add(ref, result) + + return result, nil }, IsImageFormatScannableFn: func(repo string, reference string) (bool, error) { - if repo == "repoIndex" { + if repo == repoMultiarch { return true, nil } @@ -1173,86 +1199,50 @@ func TestCVEStruct(t *testing.T) { return false, nil }, IsImageMediaScannableFn: func(repo, digest, mediaType string) (bool, error) { - if repo == "repo2" { - if digest == digest21.String() { - return false, nil - } + if repo == repo2 && digest == image21Digest { + return false, zerr.ErrScanNotSupported + } + if repo == repo3 && digest == image31Digest { + return false, zerr.ErrTagMetaNotFound + } + if repo == repo5 && digest == image51Digest { + return false, zerr.ErrManifestDataNotFound + } + if repo == repo100 { + return false, zerr.ErrRepoMetaNotFound } return true, nil }, + IsResultCachedFn: func(digest string) bool { + t.Logf("IsResultCachedFn found in cache for digest %s: %v", digest, cache.Get(digest)) + + return cache.Contains(digest) + }, + GetCachedResultFn: func(digest string) map[string]cvemodel.CVE { + t.Logf("GetCachedResultFn found in cache for digest %s: %v", digest, cache.Get(digest)) + + return cache.Get(digest) + }, } - log := log.NewLogger("debug", "") cveInfo := cveinfo.BaseCveInfo{Log: log, Scanner: scanner, MetaDB: metaDB} - t.Log("Test GetCVESummaryForImage") - - // Image is found - cveSummary, err := cveInfo.GetCVESummaryForImage("repo1", "0.1.0") - So(err, ShouldBeNil) - So(cveSummary.Count, ShouldEqual, 1) - So(cveSummary.MaxSeverity, ShouldEqual, "MEDIUM") - - cveSummary, err = cveInfo.GetCVESummaryForImage("repo1", "1.0.0") - So(err, ShouldBeNil) - So(cveSummary.Count, ShouldEqual, 3) - So(cveSummary.MaxSeverity, ShouldEqual, "HIGH") - - cveSummary, err = cveInfo.GetCVESummaryForImage("repo1", "1.0.1") - So(err, ShouldBeNil) - So(cveSummary.Count, ShouldEqual, 2) - So(cveSummary.MaxSeverity, ShouldEqual, "MEDIUM") - - cveSummary, err = cveInfo.GetCVESummaryForImage("repo1", "1.1.0") - So(err, ShouldBeNil) - So(cveSummary.Count, ShouldEqual, 1) - So(cveSummary.MaxSeverity, ShouldEqual, "LOW") - - cveSummary, err = cveInfo.GetCVESummaryForImage("repo6", "1.0.0") - So(err, ShouldBeNil) - So(cveSummary.Count, ShouldEqual, 0) - So(cveSummary.MaxSeverity, ShouldEqual, "NONE") - - // Image is not scannable - cveSummary, err = cveInfo.GetCVESummaryForImage("repo2", "1.0.0") - So(err, ShouldEqual, zerr.ErrScanNotSupported) - So(cveSummary.Count, ShouldEqual, 0) - So(cveSummary.MaxSeverity, ShouldEqual, "") - - // Tag is not found - cveSummary, err = cveInfo.GetCVESummaryForImage("repo3", "1.0.0") - So(err, ShouldEqual, zerr.ErrTagMetaNotFound) - So(cveSummary.Count, ShouldEqual, 0) - So(cveSummary.MaxSeverity, ShouldEqual, "") - - // Manifest is not found - cveSummary, err = cveInfo.GetCVESummaryForImage("repo5", "nonexitent-manifest") - So(err, ShouldEqual, zerr.ErrManifestDataNotFound) - So(cveSummary.Count, ShouldEqual, 0) - So(cveSummary.MaxSeverity, ShouldEqual, "") - - // Repo is not found - cveSummary, err = cveInfo.GetCVESummaryForImage("repo100", "1.0.0") - So(err, ShouldEqual, zerr.ErrRepoMetaNotFound) - So(cveSummary.Count, ShouldEqual, 0) - So(cveSummary.MaxSeverity, ShouldEqual, "") - - t.Log("Test GetCVEListForImage") + t.Log("\nTest GetCVEListForImage\n") pageInput := cvemodel.PageInput{ SortBy: cveinfo.SeverityDsc, } // Image is found - cveList, pageInfo, err := cveInfo.GetCVEListForImage("repo1", "0.1.0", "", pageInput) + cveList, pageInfo, err := cveInfo.GetCVEListForImage(repo1, "0.1.0", "", pageInput) So(err, ShouldBeNil) So(len(cveList), ShouldEqual, 1) So(cveList[0].ID, ShouldEqual, "CVE1") So(pageInfo.ItemCount, ShouldEqual, 1) So(pageInfo.TotalCount, ShouldEqual, 1) - cveList, pageInfo, err = cveInfo.GetCVEListForImage("repo1", "1.0.0", "", pageInput) + cveList, pageInfo, err = cveInfo.GetCVEListForImage(repo1, "1.0.0", "", pageInput) So(err, ShouldBeNil) So(len(cveList), ShouldEqual, 3) So(cveList[0].ID, ShouldEqual, "CVE2") @@ -1261,7 +1251,7 @@ func TestCVEStruct(t *testing.T) { So(pageInfo.ItemCount, ShouldEqual, 3) So(pageInfo.TotalCount, ShouldEqual, 3) - cveList, pageInfo, err = cveInfo.GetCVEListForImage("repo1", "1.0.1", "", pageInput) + cveList, pageInfo, err = cveInfo.GetCVEListForImage(repo1, "1.0.1", "", pageInput) So(err, ShouldBeNil) So(len(cveList), ShouldEqual, 2) So(cveList[0].ID, ShouldEqual, "CVE1") @@ -1269,63 +1259,157 @@ func TestCVEStruct(t *testing.T) { So(pageInfo.ItemCount, ShouldEqual, 2) So(pageInfo.TotalCount, ShouldEqual, 2) - cveList, pageInfo, err = cveInfo.GetCVEListForImage("repo1", "1.1.0", "", pageInput) + cveList, pageInfo, err = cveInfo.GetCVEListForImage(repo1, "1.1.0", "", pageInput) So(err, ShouldBeNil) So(len(cveList), ShouldEqual, 1) So(cveList[0].ID, ShouldEqual, "CVE3") So(pageInfo.ItemCount, ShouldEqual, 1) So(pageInfo.TotalCount, ShouldEqual, 1) - cveList, pageInfo, err = cveInfo.GetCVEListForImage("repo6", "1.0.0", "", pageInput) + cveList, pageInfo, err = cveInfo.GetCVEListForImage(repo6, "1.0.0", "", pageInput) So(err, ShouldBeNil) So(len(cveList), ShouldEqual, 0) So(pageInfo.ItemCount, ShouldEqual, 0) So(pageInfo.TotalCount, ShouldEqual, 0) + // Image is multiarch + cveList, pageInfo, err = cveInfo.GetCVEListForImage(repoMultiarch, "tagIndex", "", pageInput) + So(err, ShouldBeNil) + So(len(cveList), ShouldEqual, 1) + So(cveList[0].ID, ShouldEqual, "CVE1") + So(pageInfo.ItemCount, ShouldEqual, 1) + So(pageInfo.TotalCount, ShouldEqual, 1) + // Image is not scannable - cveList, pageInfo, err = cveInfo.GetCVEListForImage("repo2", "1.0.0", "", pageInput) + cveList, pageInfo, err = cveInfo.GetCVEListForImage(repo2, "1.0.0", "", pageInput) So(err, ShouldEqual, zerr.ErrScanNotSupported) So(len(cveList), ShouldEqual, 0) So(pageInfo.ItemCount, ShouldEqual, 0) So(pageInfo.TotalCount, ShouldEqual, 0) // Tag is not found - cveList, pageInfo, err = cveInfo.GetCVEListForImage("repo3", "1.0.0", "", pageInput) + cveList, pageInfo, err = cveInfo.GetCVEListForImage(repo3, "1.0.0", "", pageInput) So(err, ShouldEqual, zerr.ErrTagMetaNotFound) So(len(cveList), ShouldEqual, 0) So(pageInfo.ItemCount, ShouldEqual, 0) So(pageInfo.TotalCount, ShouldEqual, 0) + // Config not valid + cveList, pageInfo, err = cveInfo.GetCVEListForImage(repo4, "invalid-config", "", pageInput) + So(err, ShouldBeNil) + So(len(cveList), ShouldEqual, 0) + So(pageInfo.ItemCount, ShouldEqual, 0) + So(pageInfo.TotalCount, ShouldEqual, 0) + // Manifest is not found - cveList, pageInfo, err = cveInfo.GetCVEListForImage("repo5", "nonexitent-manifest", "", pageInput) + cveList, pageInfo, err = cveInfo.GetCVEListForImage(repo5, "nonexitent-manifest", "", pageInput) So(err, ShouldEqual, zerr.ErrManifestDataNotFound) So(len(cveList), ShouldEqual, 0) So(pageInfo.ItemCount, ShouldEqual, 0) So(pageInfo.TotalCount, ShouldEqual, 0) + // Scan failed + cveList, pageInfo, err = cveInfo.GetCVEListForImage(repo7, "1.0.0", "", pageInput) + So(err, ShouldEqual, ErrFailedScan) + So(len(cveList), ShouldEqual, 0) + So(pageInfo.ItemCount, ShouldEqual, 0) + So(pageInfo.TotalCount, ShouldEqual, 0) + // Repo is not found - cveList, pageInfo, err = cveInfo.GetCVEListForImage("repo100", "1.0.0", "", pageInput) + cveList, pageInfo, err = cveInfo.GetCVEListForImage(repo100, "1.0.0", "", pageInput) So(err, ShouldEqual, zerr.ErrRepoMetaNotFound) So(len(cveList), ShouldEqual, 0) So(pageInfo.ItemCount, ShouldEqual, 0) So(pageInfo.TotalCount, ShouldEqual, 0) - t.Log("Test GetImageListWithCVEFixed") + // By this point the cache should already be pupulated by previous function calls + t.Log("\nTest GetCVESummaryForImage\n") // Image is found - tagList, err := cveInfo.GetImageListWithCVEFixed("repo1", "CVE1") + cveSummary, err := cveInfo.GetCVESummaryForImageMedia(repo1, image11Digest, image11Media) + So(err, ShouldBeNil) + So(cveSummary.Count, ShouldEqual, 1) + So(cveSummary.MaxSeverity, ShouldEqual, "MEDIUM") + + cveSummary, err = cveInfo.GetCVESummaryForImageMedia(repo1, image12Digest, image12Media) + So(err, ShouldBeNil) + So(cveSummary.Count, ShouldEqual, 3) + So(cveSummary.MaxSeverity, ShouldEqual, "HIGH") + + cveSummary, err = cveInfo.GetCVESummaryForImageMedia(repo1, image14Digest, image14Media) + So(err, ShouldBeNil) + So(cveSummary.Count, ShouldEqual, 2) + So(cveSummary.MaxSeverity, ShouldEqual, "MEDIUM") + + cveSummary, err = cveInfo.GetCVESummaryForImageMedia(repo1, image13Digest, image13Media) + So(err, ShouldBeNil) + So(cveSummary.Count, ShouldEqual, 1) + So(cveSummary.MaxSeverity, ShouldEqual, "LOW") + + cveSummary, err = cveInfo.GetCVESummaryForImageMedia(repo6, image61Digest, image61Media) + So(err, ShouldBeNil) + So(cveSummary.Count, ShouldEqual, 0) + So(cveSummary.MaxSeverity, ShouldEqual, "NONE") + + // Image is multiarch + cveSummary, err = cveInfo.GetCVESummaryForImageMedia(repoMultiarch, indexDigest, indexMedia) + So(err, ShouldBeNil) + So(cveSummary.Count, ShouldEqual, 1) + So(cveSummary.MaxSeverity, ShouldEqual, "MEDIUM") + + // Image is not scannable + cveSummary, err = cveInfo.GetCVESummaryForImageMedia(repo2, image21Digest, image21Media) + So(err, ShouldEqual, zerr.ErrScanNotSupported) + So(cveSummary.Count, ShouldEqual, 0) + So(cveSummary.MaxSeverity, ShouldEqual, "") + + // Tag is not found + cveSummary, err = cveInfo.GetCVESummaryForImageMedia(repo3, image31Digest, image31Media) + So(err, ShouldEqual, zerr.ErrTagMetaNotFound) + So(cveSummary.Count, ShouldEqual, 0) + So(cveSummary.MaxSeverity, ShouldEqual, "") + + // Config not valid + cveSummary, err = cveInfo.GetCVESummaryForImageMedia(repo4, image41Digest, image41Media) + So(err, ShouldBeNil) + So(cveSummary.Count, ShouldEqual, 0) + So(cveSummary.MaxSeverity, ShouldEqual, "NONE") + + // Manifest is not found + cveSummary, err = cveInfo.GetCVESummaryForImageMedia(repo5, image51Digest, image51Media) + So(err, ShouldEqual, zerr.ErrManifestDataNotFound) + So(cveSummary.Count, ShouldEqual, 0) + So(cveSummary.MaxSeverity, ShouldEqual, "") + + // Scan failed + cveSummary, err = cveInfo.GetCVESummaryForImageMedia(repo5, image71Digest, image71Media) + So(err, ShouldBeNil) + So(cveSummary.Count, ShouldEqual, 0) + So(cveSummary.MaxSeverity, ShouldEqual, "") + + // Repo is not found + cveSummary, err = cveInfo.GetCVESummaryForImageMedia(repo100, + godigest.FromString("missing_digest").String(), ispec.MediaTypeImageManifest) + So(err, ShouldEqual, zerr.ErrRepoMetaNotFound) + So(cveSummary.Count, ShouldEqual, 0) + So(cveSummary.MaxSeverity, ShouldEqual, "") + + t.Log("\nTest GetImageListWithCVEFixed\n") + + // Image is found + tagList, err := cveInfo.GetImageListWithCVEFixed(repo1, "CVE1") So(err, ShouldBeNil) So(len(tagList), ShouldEqual, 1) So(tagList[0].Tag, ShouldEqual, "1.1.0") - tagList, err = cveInfo.GetImageListWithCVEFixed("repo1", "CVE2") + tagList, err = cveInfo.GetImageListWithCVEFixed(repo1, "CVE2") So(err, ShouldBeNil) So(len(tagList), ShouldEqual, 2) expectedTags := []string{"1.0.1", "1.1.0"} So(expectedTags, ShouldContain, tagList[0].Tag) So(expectedTags, ShouldContain, tagList[1].Tag) - tagList, err = cveInfo.GetImageListWithCVEFixed("repo1", "CVE3") + tagList, err = cveInfo.GetImageListWithCVEFixed(repo1, "CVE3") So(err, ShouldBeNil) // CVE3 is not present in 0.1.0, but that is older than all other // images where it is present. The rest of the images explicitly have it. @@ -1333,38 +1417,38 @@ func TestCVEStruct(t *testing.T) { So(len(tagList), ShouldEqual, 0) // Image doesn't have any CVEs in the first place - tagList, err = cveInfo.GetImageListWithCVEFixed("repo6", "CVE1") + tagList, err = cveInfo.GetImageListWithCVEFixed(repo6, "CVE1") So(err, ShouldBeNil) So(len(tagList), ShouldEqual, 1) So(tagList[0].Tag, ShouldEqual, "1.0.0") // Image is not scannable - tagList, err = cveInfo.GetImageListWithCVEFixed("repo2", "CVE100") + tagList, err = cveInfo.GetImageListWithCVEFixed(repo2, "CVE100") // CVE is not considered fixed as scan is not possible // but do not return an error So(err, ShouldBeNil) So(len(tagList), ShouldEqual, 0) // Tag is not found, but we should not error - tagList, err = cveInfo.GetImageListWithCVEFixed("repo3", "CVE101") + tagList, err = cveInfo.GetImageListWithCVEFixed(repo3, "CVE101") So(err, ShouldBeNil) So(len(tagList), ShouldEqual, 0) // Manifest is not found, we just consider exclude it from the fixed list - tagList, err = cveInfo.GetImageListWithCVEFixed("repo5", "CVE101") + tagList, err = cveInfo.GetImageListWithCVEFixed(repo5, "CVE101") So(err, ShouldBeNil) So(len(tagList), ShouldEqual, 0) // Repo is not found, there could potentially be unaffected tags in the repo // but we can't access their data - tagList, err = cveInfo.GetImageListWithCVEFixed("repo100", "CVE100") + tagList, err = cveInfo.GetImageListWithCVEFixed(repo100, "CVE100") So(err, ShouldEqual, zerr.ErrRepoMetaNotFound) So(len(tagList), ShouldEqual, 0) - t.Log("Test GetImageListForCVE") + t.Log("\nTest GetImageListForCVE\n") // Image is found - tagList, err = cveInfo.GetImageListForCVE("repo1", "CVE1") + tagList, err = cveInfo.GetImageListForCVE(repo1, "CVE1") So(err, ShouldBeNil) So(len(tagList), ShouldEqual, 3) expectedTags = []string{"0.1.0", "1.0.0", "1.0.1"} @@ -1372,12 +1456,12 @@ func TestCVEStruct(t *testing.T) { So(expectedTags, ShouldContain, tagList[1].Tag) So(expectedTags, ShouldContain, tagList[2].Tag) - tagList, err = cveInfo.GetImageListForCVE("repo1", "CVE2") + tagList, err = cveInfo.GetImageListForCVE(repo1, "CVE2") So(err, ShouldBeNil) So(len(tagList), ShouldEqual, 1) So(tagList[0].Tag, ShouldEqual, "1.0.0") - tagList, err = cveInfo.GetImageListForCVE("repo1", "CVE3") + tagList, err = cveInfo.GetImageListForCVE(repo1, "CVE3") So(err, ShouldBeNil) So(len(tagList), ShouldEqual, 3) expectedTags = []string{"1.0.0", "1.0.1", "1.1.0"} @@ -1386,29 +1470,29 @@ func TestCVEStruct(t *testing.T) { So(expectedTags, ShouldContain, tagList[2].Tag) // Image/repo doesn't have the CVE at all - tagList, err = cveInfo.GetImageListForCVE("repo6", "CVE1") + tagList, err = cveInfo.GetImageListForCVE(repo6, "CVE1") So(err, ShouldBeNil) So(len(tagList), ShouldEqual, 0) // Image is not scannable - tagList, err = cveInfo.GetImageListForCVE("repo2", "CVE100") + tagList, err = cveInfo.GetImageListForCVE(repo2, "CVE100") // Image is not considered affected with CVE as scan is not possible // but do not return an error So(err, ShouldBeNil) So(len(tagList), ShouldEqual, 0) // Tag is not found, but we should not error - tagList, err = cveInfo.GetImageListForCVE("repo3", "CVE101") + tagList, err = cveInfo.GetImageListForCVE(repo3, "CVE101") So(err, ShouldBeNil) So(len(tagList), ShouldEqual, 0) // Repo is not found, assume it is affected by the CVE - // But we don't have enough of it's data to actually return it - tagList, err = cveInfo.GetImageListForCVE("repo100", "CVE100") + // But we don't have enough of its data to actually return it + tagList, err = cveInfo.GetImageListForCVE(repo100, "CVE100") So(err, ShouldEqual, zerr.ErrRepoMetaNotFound) So(len(tagList), ShouldEqual, 0) - t.Log("Test errors while scanning") + t.Log("\nTest errors while scanning\n") faultyScanner := mocks.CveScannerMock{ ScanImageFn: func(image string) (map[string]cvemodel.CVE, error) { @@ -1419,42 +1503,36 @@ func TestCVEStruct(t *testing.T) { cveInfo = cveinfo.BaseCveInfo{Log: log, Scanner: faultyScanner, MetaDB: metaDB} - cveSummary, err = cveInfo.GetCVESummaryForImage("repo1", "0.1.0") - So(err, ShouldNotBeNil) + cveSummary, err = cveInfo.GetCVESummaryForImageMedia(repo1, image11Digest, image11Media) + So(err, ShouldBeNil) So(cveSummary.Count, ShouldEqual, 0) So(cveSummary.MaxSeverity, ShouldEqual, "") - cveList, pageInfo, err = cveInfo.GetCVEListForImage("repo1", "0.1.0", "", pageInput) + cveList, pageInfo, err = cveInfo.GetCVEListForImage(repo1, "0.1.0", "", pageInput) So(err, ShouldNotBeNil) So(cveList, ShouldBeEmpty) So(pageInfo.ItemCount, ShouldEqual, 0) So(pageInfo.TotalCount, ShouldEqual, 0) - tagList, err = cveInfo.GetImageListWithCVEFixed("repo1", "CVE1") + tagList, err = cveInfo.GetImageListWithCVEFixed(repo1, "CVE1") // CVE is not considered fixed as scan is not possible // but do not return an error So(err, ShouldBeNil) So(len(tagList), ShouldEqual, 0) - tagList, err = cveInfo.GetImageListForCVE("repo1", "CVE1") + tagList, err = cveInfo.GetImageListForCVE(repo1, "CVE1") // Image is not considered affected with CVE as scan is not possible // but do not return an error So(err, ShouldBeNil) So(len(tagList), ShouldEqual, 0) - cveInfo = cveinfo.BaseCveInfo{Log: log, Scanner: scanner, MetaDB: metaDB} - - tagList, err = cveInfo.GetImageListForCVE("repoIndex", "CVE1") - So(err, ShouldBeNil) - So(len(tagList), ShouldEqual, 1) - cveInfo = cveinfo.BaseCveInfo{Log: log, Scanner: mocks.CveScannerMock{ IsImageFormatScannableFn: func(repo, reference string) (bool, error) { return false, nil }, }, MetaDB: metaDB} - _, err = cveInfo.GetImageListForCVE("repoIndex", "CVE1") + _, err = cveInfo.GetImageListForCVE(repoMultiarch, "CVE1") So(err, ShouldBeNil) cveInfo = cveinfo.BaseCveInfo{Log: log, Scanner: mocks.CveScannerMock{ @@ -1466,7 +1544,7 @@ func TestCVEStruct(t *testing.T) { }, }, MetaDB: metaDB} - _, err = cveInfo.GetImageListForCVE("repoIndex", "CVE1") + _, err = cveInfo.GetImageListForCVE(repoMultiarch, "CVE1") So(err, ShouldBeNil) }) } @@ -1548,12 +1626,23 @@ func TestFixedTagsWithIndex(t *testing.T) { BaseConfig: extconf.BaseConfig{Enable: &defaultVal}, CVE: &extconf.CVEConfig{ UpdateInterval: 24 * time.Hour, - Trivy: &extconf.TrivyConfig{}, + Trivy: &extconf.TrivyConfig{ + DBRepository: "ghcr.io/project-zot/trivy-db", + }, }, }, } + + logFile, err := os.CreateTemp(t.TempDir(), "zot-log*.txt") + So(err, ShouldBeNil) + + logPath := logFile.Name() + defer os.Remove(logPath) + + writers := io.MultiWriter(os.Stdout, logFile) + ctlr := api.NewController(conf) - So(ctlr, ShouldNotBeNil) + ctlr.Log.Logger = ctlr.Log.Output(writers) cm := NewControllerManager(ctlr) cm.StartAndWait(port) @@ -1565,7 +1654,6 @@ func TestFixedTagsWithIndex(t *testing.T) { Platform: ispec.Platform{OS: "linux", Architecture: "amd64"}, }) So(err, ShouldBeNil) - vulnDigest := vulnManifest.Digest() fixedManifestCreated := time.Date(2010, 1, 1, 1, 1, 1, 1, time.UTC) fixedManifest, err := GetImageWithConfig(ispec.Image{ @@ -1576,7 +1664,6 @@ func TestFixedTagsWithIndex(t *testing.T) { fixedDigest := fixedManifest.Digest() multiArch := GetMultiarchImageForImages([]Image{fixedManifest, vulnManifest}) - multiArchDigest := multiArch.Digest() err = UploadMultiarchImage(multiArch, baseURL, "repo", "multi-arch-tag") So(err, ShouldBeNil) @@ -1592,21 +1679,18 @@ func TestFixedTagsWithIndex(t *testing.T) { err = UploadImage(simpleVulnImg, baseURL, "repo", "vuln-img") So(err, ShouldBeNil) - scanner := trivy.NewScanner(ctlr.StoreController, ctlr.MetaDB, "ghcr.io/project-zot/trivy-db", "", ctlr.Log) - - err = scanner.UpdateDB() + // Wait for trivy db to download + found, err := ReadLogFileAndSearchString(logPath, "DB update completed, next update scheduled", 180*time.Second) So(err, ShouldBeNil) + So(found, ShouldBeTrue) - cveInfo := cveinfo.NewCVEInfo(ctlr.StoreController, ctlr.MetaDB, "ghcr.io/project-zot/trivy-db", "", ctlr.Log) + cveInfo := cveinfo.NewCVEInfo(ctlr.CveScanner, ctlr.MetaDB, ctlr.Log) tagsInfo, err := cveInfo.GetImageListWithCVEFixed("repo", Vulnerability1ID) So(err, ShouldBeNil) So(len(tagsInfo), ShouldEqual, 1) So(len(tagsInfo[0].Manifests), ShouldEqual, 1) So(tagsInfo[0].Manifests[0].Digest, ShouldResemble, fixedDigest) - _ = tagsInfo - _ = vulnDigest - _ = multiArchDigest const query = ` { @@ -1659,7 +1743,8 @@ func TestImageListWithCVEFixedErrors(t *testing.T) { return mTypes.IndexData{}, zerr.ErrIndexDataNotFount } - cveInfo := cveinfo.NewCVEInfo(storeController, metaDB, "", "", log) + scanner := cveinfo.NewScanner(storeController, metaDB, "", "", log) + cveInfo := cveinfo.NewCVEInfo(scanner, metaDB, log) _, err := cveInfo.GetImageListWithCVEFixed("repo", Vulnerability1ID) So(err, ShouldBeNil) @@ -1680,7 +1765,8 @@ func TestImageListWithCVEFixedErrors(t *testing.T) { return mTypes.IndexData{}, zerr.ErrIndexDataNotFount } - cveInfo := cveinfo.NewCVEInfo(storeController, metaDB, "", "", log) + scanner := cveinfo.NewScanner(storeController, metaDB, "", "", log) + cveInfo := cveinfo.NewCVEInfo(scanner, metaDB, log) _, err := cveInfo.GetImageListWithCVEFixed("repo", Vulnerability1ID) So(err, ShouldBeNil) @@ -1701,7 +1787,8 @@ func TestImageListWithCVEFixedErrors(t *testing.T) { return mTypes.IndexData{IndexBlob: []byte(`bad index`)}, nil } - cveInfo := cveinfo.NewCVEInfo(storeController, metaDB, "", "", log) + scanner := cveinfo.NewScanner(storeController, metaDB, "", "", log) + cveInfo := cveinfo.NewCVEInfo(scanner, metaDB, log) _, err := cveInfo.GetImageListWithCVEFixed("repo", Vulnerability1ID) So(err, ShouldBeNil) @@ -1719,7 +1806,8 @@ func TestImageListWithCVEFixedErrors(t *testing.T) { }, nil } - cveInfo := cveinfo.NewCVEInfo(storeController, metaDB, "", "", log) + scanner := cveinfo.NewScanner(storeController, metaDB, "", "", log) + cveInfo := cveinfo.NewCVEInfo(scanner, metaDB, log) _, err := cveInfo.GetImageListWithCVEFixed("repo", Vulnerability1ID) So(err, ShouldBeNil) @@ -1751,7 +1839,8 @@ func TestImageListWithCVEFixedErrors(t *testing.T) { return mTypes.ManifestData{}, zerr.ErrManifestDataNotFound } - cveInfo := cveinfo.NewCVEInfo(storeController, metaDB, "", "", log) + scanner := cveinfo.NewScanner(storeController, metaDB, "", "", log) + cveInfo := cveinfo.NewCVEInfo(scanner, metaDB, log) tagsInfo, err := cveInfo.GetImageListWithCVEFixed("repo", Vulnerability1ID) So(err, ShouldBeNil) @@ -1770,7 +1859,8 @@ func TestImageListWithCVEFixedErrors(t *testing.T) { }, nil } - cveInfo := cveinfo.NewCVEInfo(storeController, metaDB, "", "", log) + scanner := cveinfo.NewScanner(storeController, metaDB, "", "", log) + cveInfo := cveinfo.NewCVEInfo(scanner, metaDB, log) tagsInfo, err := cveInfo.GetImageListWithCVEFixed("repo", Vulnerability1ID) So(err, ShouldBeNil) @@ -1788,27 +1878,13 @@ func TestGetCVESummaryForImageMediaErrors(t *testing.T) { log := log.NewLogger("debug", "") Convey("IsImageMediaScannable returns false", func() { - cveInfo := cveinfo.NewCVEInfo(storeController, metaDB, "", "", log) - cveInfo.Scanner = mocks.CveScannerMock{ + scanner := mocks.CveScannerMock{ IsImageMediaScannableFn: func(repo, digest, mediaType string) (bool, error) { return false, zerr.ErrScanNotSupported }, } - _, err := cveInfo.GetCVESummaryForImageMedia("repo", "digest", ispec.MediaTypeImageManifest) - So(err, ShouldNotBeNil) - }) - - Convey("Scan fails", func() { - cveInfo := cveinfo.NewCVEInfo(storeController, metaDB, "", "", log) - cveInfo.Scanner = mocks.CveScannerMock{ - IsImageMediaScannableFn: func(repo, digest, mediaType string) (bool, error) { - return true, nil - }, - ScanImageFn: func(image string) (map[string]cvemodel.CVE, error) { - return nil, zerr.ErrScanNotSupported - }, - } + cveInfo := cveinfo.NewCVEInfo(scanner, metaDB, log) _, err := cveInfo.GetCVESummaryForImageMedia("repo", "digest", ispec.MediaTypeImageManifest) So(err, ShouldNotBeNil) diff --git a/pkg/extensions/search/cve/scan.go b/pkg/extensions/search/cve/scan.go new file mode 100644 index 00000000..cacf084a --- /dev/null +++ b/pkg/extensions/search/cve/scan.go @@ -0,0 +1,207 @@ +package cveinfo + +import ( + "context" + "sync" + + godigest "github.com/opencontainers/go-digest" + + "zotregistry.io/zot/pkg/log" + mTypes "zotregistry.io/zot/pkg/meta/types" + reqCtx "zotregistry.io/zot/pkg/requestcontext" + "zotregistry.io/zot/pkg/scheduler" +) + +func NewScanTaskGenerator( + metaDB mTypes.MetaDB, + scanner Scanner, + log log.Logger, +) scheduler.TaskGenerator { + return &scanTaskGenerator{ + log: log, + metaDB: metaDB, + scanner: scanner, + lock: &sync.Mutex{}, + scanErrors: map[string]error{}, + scheduled: map[string]bool{}, + done: false, + } +} + +// scanTaskGenerator takes all manifests from repodb and runs the CVE scanner on them. +// If the scanner already has results cached for a specific manifests, or it cannot be +// scanned, the manifest will be skipped. +// If there are no manifests missing from the cache, the generator finishes. +type scanTaskGenerator struct { + log log.Logger + metaDB mTypes.MetaDB + scanner Scanner + lock *sync.Mutex + scanErrors map[string]error + scheduled map[string]bool + done bool +} + +func (gen *scanTaskGenerator) getMatcherFunc() mTypes.FilterFunc { + return func(repoMeta mTypes.RepoMetadata, manifestMeta mTypes.ManifestMetadata) bool { + // Note this matcher will return information based on scan status of manifests + // An index scan aggregates results of manifest scans + // If at least one of its manifests can be scanned, + // the index and its tag will be returned by the caller function too + repoName := repoMeta.Name + manifestDigest := godigest.FromBytes(manifestMeta.ManifestBlob).String() + + if gen.isScheduled(manifestDigest) { + // We skip this manifest as it has already scheduled + return false + } + + if gen.hasError(manifestDigest) { + // We skip this manifest as it has already been scanned and errored + // This is to prevent the generator attempting to run a scan + // in a loop of the same image which would consistently fail + return false + } + + if gen.scanner.IsResultCached(manifestDigest) { + // We skip this manifest, it was already scanned + return false + } + + ok, err := gen.scanner.IsImageFormatScannable(repoName, manifestDigest) + if !ok || err != nil { + // We skip this manifest, we cannot scan it + return false + } + + return true + } +} + +func (gen *scanTaskGenerator) addError(digest string, err error) { + gen.lock.Lock() + defer gen.lock.Unlock() + + gen.scanErrors[digest] = err +} + +func (gen *scanTaskGenerator) hasError(digest string) bool { + gen.lock.Lock() + defer gen.lock.Unlock() + + _, ok := gen.scanErrors[digest] + + return ok +} + +func (gen *scanTaskGenerator) setScheduled(digest string, isScheduled bool) { + gen.lock.Lock() + defer gen.lock.Unlock() + + if _, ok := gen.scheduled[digest]; ok && !isScheduled { + delete(gen.scheduled, digest) + } else if isScheduled { + gen.scheduled[digest] = true + } +} + +func (gen *scanTaskGenerator) isScheduled(digest string) bool { + gen.lock.Lock() + defer gen.lock.Unlock() + + _, ok := gen.scheduled[digest] + + return ok +} + +func (gen *scanTaskGenerator) Next() (scheduler.Task, error) { + // metaRB requires us to use a context for authorization + userAc := reqCtx.NewUserAccessControl() + userAc.SetUsername("scheduler") + userAc.SetIsAdmin(true) + ctx := userAc.DeriveContext(context.Background()) + + // Obtain a list of repos with unscanned scannable manifests + // We may implement a method to return just 1 match at some point + reposMeta, _, _, err := gen.metaDB.FilterTags(ctx, gen.getMatcherFunc()) + if err != nil { + // Do not crash the generator for potential repodb inconistencies + // as there may be scannable images not yet scanned + gen.log.Warn().Err(err).Msg("Scheduled CVE scan: error while obtaining repo metadata") + } + + // no reposMeta are returned, all results are in already in cache + // or manifests cannot be scanned + if len(reposMeta) == 0 { + gen.log.Info().Msg("Scheduled CVE scan: finished for available images") + + gen.done = true + + return nil, nil + } + + // Since reposMeta will always contain just unscanned images we can pick + // any repo and any tag out of the resulting matches + repoMeta := reposMeta[0] + + var digest string + + // Pick any tag + for _, descriptor := range repoMeta.Tags { + digest = descriptor.Digest + + break + } + + // Mark the digest as scheduled so it is skipped on next generator run + gen.setScheduled(digest, true) + + return newScanTask(gen, repoMeta.Name, digest), nil +} + +func (gen *scanTaskGenerator) IsDone() bool { + return gen.done +} + +func (gen *scanTaskGenerator) IsReady() bool { + return true +} + +func (gen *scanTaskGenerator) Reset() { + gen.lock.Lock() + defer gen.lock.Unlock() + + gen.scheduled = map[string]bool{} + gen.scanErrors = map[string]error{} + gen.done = false +} + +type scanTask struct { + generator *scanTaskGenerator + repo string + digest string +} + +func newScanTask(generator *scanTaskGenerator, repo string, digest string) *scanTask { + return &scanTask{generator, repo, digest} +} + +func (st *scanTask) DoWork(ctx context.Context) error { + // When work finished clean this entry from the generator + defer st.generator.setScheduled(st.digest, false) + + image := st.repo + "@" + st.digest + + // We cache the results internally in the scanner + // so we can discard the actual results for now + if _, err := st.generator.scanner.ScanImage(image); err != nil { + st.generator.log.Error().Err(err).Str("image", image).Msg("Scheduled CVE scan errored for image") + st.generator.addError(st.digest, err) + + return err + } + + st.generator.log.Debug().Str("image", image).Msg("Scheduled CVE scan completed successfully for image") + + return nil +} diff --git a/pkg/extensions/search/cve/scan_test.go b/pkg/extensions/search/cve/scan_test.go new file mode 100644 index 00000000..cf4227d3 --- /dev/null +++ b/pkg/extensions/search/cve/scan_test.go @@ -0,0 +1,670 @@ +//go:build search +// +build search + +package cveinfo_test + +import ( + "context" + "encoding/json" + "errors" + "io" + "os" + "testing" + "time" + + regTypes "github.com/google/go-containerregistry/pkg/v1/types" + godigest "github.com/opencontainers/go-digest" + ispec "github.com/opencontainers/image-spec/specs-go/v1" + . "github.com/smartystreets/goconvey/convey" + + zerr "zotregistry.io/zot/errors" + "zotregistry.io/zot/pkg/api/config" + zcommon "zotregistry.io/zot/pkg/common" + "zotregistry.io/zot/pkg/extensions/monitoring" + cveinfo "zotregistry.io/zot/pkg/extensions/search/cve" + cvecache "zotregistry.io/zot/pkg/extensions/search/cve/cache" + cvemodel "zotregistry.io/zot/pkg/extensions/search/cve/model" + "zotregistry.io/zot/pkg/log" + "zotregistry.io/zot/pkg/meta" + "zotregistry.io/zot/pkg/meta/boltdb" + mTypes "zotregistry.io/zot/pkg/meta/types" + "zotregistry.io/zot/pkg/scheduler" + "zotregistry.io/zot/pkg/storage" + "zotregistry.io/zot/pkg/storage/local" + . "zotregistry.io/zot/pkg/test" + . "zotregistry.io/zot/pkg/test/image-utils" + "zotregistry.io/zot/pkg/test/mocks" +) + +var ( + ErrBadTest = errors.New("there is a bug in the test") + ErrFailedScan = errors.New("scan has failed intentionally") +) + +func TestScanGeneratorWithMockedData(t *testing.T) { //nolint: gocyclo + Convey("Test CVE scanning task scheduler with diverse mocked data", t, func() { + repo1 := "repo1" + repoIndex := "repoIndex" + + logFile, err := os.CreateTemp(t.TempDir(), "zot-log*.txt") + logPath := logFile.Name() + So(err, ShouldBeNil) + + defer os.Remove(logFile.Name()) // clean up + + logger := log.NewLogger("debug", logPath) + writers := io.MultiWriter(os.Stdout, logFile) + logger.Logger = logger.Output(writers) + + cfg := config.New() + cfg.Scheduler = &config.SchedulerConfig{NumWorkers: 3} + sch := scheduler.NewScheduler(cfg, logger) + + params := boltdb.DBParameters{ + RootDir: t.TempDir(), + } + boltDriver, err := boltdb.GetBoltDriver(params) + So(err, ShouldBeNil) + + metaDB, err := boltdb.New(boltDriver, log.NewLogger("debug", "")) + So(err, ShouldBeNil) + + // Create metadb data for scannable image with vulnerabilities + image11 := CreateImageWith().DefaultLayers(). + ImageConfig(ispec.Image{Created: DateRef(2008, 1, 1, 12, 0, 0, 0, time.UTC)}).Build() + + repoMeta11 := mTypes.ManifestMetadata{ + ManifestBlob: image11.ManifestDescriptor.Data, + ConfigBlob: image11.ConfigDescriptor.Data, + DownloadCount: 0, + Signatures: mTypes.ManifestSignatures{}, + } + + err = metaDB.SetManifestMeta("repo1", image11.ManifestDescriptor.Digest, repoMeta11) + So(err, ShouldBeNil) + err = metaDB.SetRepoReference("repo1", "0.1.0", image11.ManifestDescriptor.Digest, ispec.MediaTypeImageManifest) + So(err, ShouldBeNil) + + image12 := CreateImageWith().DefaultLayers(). + ImageConfig(ispec.Image{Created: DateRef(2009, 1, 1, 12, 0, 0, 0, time.UTC)}).Build() + + repoMeta12 := mTypes.ManifestMetadata{ + ManifestBlob: image12.ManifestDescriptor.Data, + ConfigBlob: image12.ConfigDescriptor.Data, + DownloadCount: 0, + Signatures: mTypes.ManifestSignatures{}, + } + + err = metaDB.SetManifestMeta("repo1", image12.ManifestDescriptor.Digest, repoMeta12) + So(err, ShouldBeNil) + err = metaDB.SetRepoReference("repo1", "1.0.0", image12.ManifestDescriptor.Digest, ispec.MediaTypeImageManifest) + So(err, ShouldBeNil) + + image13 := CreateImageWith().DefaultLayers(). + ImageConfig(ispec.Image{Created: DateRef(2010, 1, 1, 12, 0, 0, 0, time.UTC)}).Build() + + repoMeta13 := mTypes.ManifestMetadata{ + ManifestBlob: image13.ManifestDescriptor.Data, + ConfigBlob: image13.ConfigDescriptor.Data, + DownloadCount: 0, + Signatures: mTypes.ManifestSignatures{}, + } + + err = metaDB.SetManifestMeta("repo1", image13.ManifestDescriptor.Digest, repoMeta13) + So(err, ShouldBeNil) + err = metaDB.SetRepoReference("repo1", "1.1.0", image13.ManifestDescriptor.Digest, ispec.MediaTypeImageManifest) + So(err, ShouldBeNil) + + image14 := CreateImageWith().DefaultLayers(). + ImageConfig(ispec.Image{Created: DateRef(2011, 1, 1, 12, 0, 0, 0, time.UTC)}).Build() + + repoMeta14 := mTypes.ManifestMetadata{ + ManifestBlob: image14.ManifestDescriptor.Data, + ConfigBlob: image14.ConfigDescriptor.Data, + } + + err = metaDB.SetManifestMeta("repo1", image14.ManifestDescriptor.Digest, repoMeta14) + So(err, ShouldBeNil) + err = metaDB.SetRepoReference("repo1", "1.0.1", image14.ManifestDescriptor.Digest, ispec.MediaTypeImageManifest) + So(err, ShouldBeNil) + + // Create metadb data for scannable image with no vulnerabilities + image61 := CreateImageWith().DefaultLayers(). + ImageConfig(ispec.Image{Created: DateRef(2016, 1, 1, 12, 0, 0, 0, time.UTC)}).Build() + + repoMeta61 := mTypes.ManifestMetadata{ + ManifestBlob: image61.ManifestDescriptor.Data, + ConfigBlob: image61.ConfigDescriptor.Data, + } + + err = metaDB.SetManifestMeta("repo6", image61.ManifestDescriptor.Digest, repoMeta61) + So(err, ShouldBeNil) + err = metaDB.SetRepoReference("repo6", "1.0.0", image61.ManifestDescriptor.Digest, ispec.MediaTypeImageManifest) + So(err, ShouldBeNil) + + // Create metadb data for image not supporting scanning + image21 := CreateImageWith().Layers([]Layer{{ + MediaType: ispec.MediaTypeImageLayerNonDistributableGzip, //nolint:staticcheck + Blob: []byte{10, 10, 10}, + Digest: godigest.FromBytes([]byte{10, 10, 10}), + }}).ImageConfig(ispec.Image{Created: DateRef(2009, 1, 1, 12, 0, 0, 0, time.UTC)}).Build() + + repoMeta21 := mTypes.ManifestMetadata{ + ManifestBlob: image21.ManifestDescriptor.Data, + ConfigBlob: image21.ConfigDescriptor.Data, + } + + err = metaDB.SetManifestMeta("repo2", image21.ManifestDescriptor.Digest, repoMeta21) + So(err, ShouldBeNil) + err = metaDB.SetRepoReference("repo2", "1.0.0", image21.ManifestDescriptor.Digest, ispec.MediaTypeImageManifest) + So(err, ShouldBeNil) + + // Create metadb data for invalid images/negative tests + manifestBlob31 := []byte("invalid manifest blob") + So(err, ShouldBeNil) + + repoMeta31 := mTypes.ManifestMetadata{ + ManifestBlob: manifestBlob31, + } + + digest31 := godigest.FromBytes(manifestBlob31) + err = metaDB.SetManifestMeta("repo3", digest31, repoMeta31) + So(err, ShouldBeNil) + err = metaDB.SetRepoReference("repo3", "invalid-manifest", digest31, ispec.MediaTypeImageManifest) + So(err, ShouldBeNil) + + image41 := CreateImageWith().DefaultLayers(). + CustomConfigBlob([]byte("invalid config blob"), ispec.MediaTypeImageConfig).Build() + + repoMeta41 := mTypes.ManifestMetadata{ + ManifestBlob: image41.ManifestDescriptor.Data, + ConfigBlob: image41.ConfigDescriptor.Data, + } + + err = metaDB.SetManifestMeta("repo4", image41.ManifestDescriptor.Digest, repoMeta41) + So(err, ShouldBeNil) + err = metaDB.SetRepoReference("repo4", "invalid-config", image41.ManifestDescriptor.Digest, + ispec.MediaTypeImageManifest) + So(err, ShouldBeNil) + + digest51 := godigest.FromString("abc8") + err = metaDB.SetRepoReference("repo5", "nonexitent-manifest", digest51, ispec.MediaTypeImageManifest) + So(err, ShouldBeNil) + + // Create metadb data for scannable image which errors during scan + image71 := CreateImageWith().DefaultLayers(). + ImageConfig(ispec.Image{Created: DateRef(2000, 1, 1, 12, 0, 0, 0, time.UTC)}).Build() + + repoMeta71 := mTypes.ManifestMetadata{ + ManifestBlob: image71.ManifestDescriptor.Data, + ConfigBlob: image71.ConfigDescriptor.Data, + } + + err = metaDB.SetManifestMeta("repo7", image71.ManifestDescriptor.Digest, repoMeta71) + So(err, ShouldBeNil) + err = metaDB.SetRepoReference("repo7", "1.0.0", image71.ManifestDescriptor.Digest, ispec.MediaTypeImageManifest) + So(err, ShouldBeNil) + + // Create multiarch image with vulnerabilities + multiarchImage := CreateRandomMultiarch() + + err = metaDB.SetIndexData( + multiarchImage.IndexDescriptor.Digest, + mTypes.IndexData{IndexBlob: multiarchImage.IndexDescriptor.Data}, + ) + So(err, ShouldBeNil) + + err = metaDB.SetManifestData( + multiarchImage.Images[0].ManifestDescriptor.Digest, + mTypes.ManifestData{ + ManifestBlob: multiarchImage.Images[0].ManifestDescriptor.Data, + ConfigBlob: multiarchImage.Images[0].ConfigDescriptor.Data, + }, + ) + So(err, ShouldBeNil) + + err = metaDB.SetManifestData( + multiarchImage.Images[1].ManifestDescriptor.Digest, + mTypes.ManifestData{ + ManifestBlob: multiarchImage.Images[1].ManifestDescriptor.Data, + ConfigBlob: multiarchImage.Images[1].ConfigDescriptor.Data, + }, + ) + So(err, ShouldBeNil) + + err = metaDB.SetManifestData( + multiarchImage.Images[2].ManifestDescriptor.Digest, + mTypes.ManifestData{ + ManifestBlob: multiarchImage.Images[2].ManifestDescriptor.Data, + ConfigBlob: multiarchImage.Images[2].ConfigDescriptor.Data, + }, + ) + So(err, ShouldBeNil) + + err = metaDB.SetRepoReference( + repoIndex, + "tagIndex", + multiarchImage.IndexDescriptor.Digest, + ispec.MediaTypeImageIndex, + ) + So(err, ShouldBeNil) + + // Keep a record of all the image references / digest pairings + // This is normally done in MetaDB, but we want to verify + // the whole flow, including MetaDB + imageMap := map[string]string{} + + image11Digest := image11.ManifestDescriptor.Digest.String() + image11Name := "repo1:0.1.0" + imageMap[image11Name] = image11Digest + image12Digest := image12.ManifestDescriptor.Digest.String() + image12Name := "repo1:1.0.0" + imageMap[image12Name] = image12Digest + image13Digest := image13.ManifestDescriptor.Digest.String() + image13Name := "repo1:1.1.0" + imageMap[image13Name] = image13Digest + image14Digest := image14.ManifestDescriptor.Digest.String() + image14Name := "repo1:1.0.1" + imageMap[image14Name] = image14Digest + image21Digest := image21.ManifestDescriptor.Digest.String() + image21Name := "repo2:1.0.0" + imageMap[image21Name] = image21Digest + image31Name := "repo3:invalid-manifest" + imageMap[image31Name] = digest31.String() + image41Digest := image41.ManifestDescriptor.Digest.String() + image41Name := "repo4:invalid-config" + imageMap[image41Name] = image41Digest + image51Name := "repo5:nonexitent-manifest" + imageMap[image51Name] = digest51.String() + image61Digest := image61.ManifestDescriptor.Digest.String() + image61Name := "repo6:1.0.0" + imageMap[image61Name] = image61Digest + image71Digest := image71.ManifestDescriptor.Digest.String() + image71Name := "repo7:1.0.0" + imageMap[image71Name] = image71Digest + indexDigest := multiarchImage.IndexDescriptor.Digest.String() + indexName := "repoIndex:tagIndex" + imageMap[indexName] = indexDigest + indexM1Digest := multiarchImage.Images[0].ManifestDescriptor.Digest.String() + indexM1Name := "repoIndex@" + indexM1Digest + imageMap[indexM1Name] = indexM1Digest + indexM2Digest := multiarchImage.Images[1].ManifestDescriptor.Digest.String() + indexM2Name := "repoIndex@" + indexM2Digest + imageMap[indexM2Name] = indexM2Digest + indexM3Digest := multiarchImage.Images[2].ManifestDescriptor.Digest.String() + indexM3Name := "repoIndex@" + indexM3Digest + imageMap[indexM3Name] = indexM3Digest + + // Initialize a test CVE cache + cache := cvecache.NewCveCache(10, logger) + + // MetaDB loaded with initial data, now mock the scanner + // Setup test CVE data in mock scanner + scanner := mocks.CveScannerMock{ + ScanImageFn: func(image string) (map[string]cvemodel.CVE, error) { + result := cache.Get(image) + // Will not match sending the repo:tag as a parameter, but we don't care + if result != nil { + return result, nil + } + + repo, ref, isTag := zcommon.GetImageDirAndReference(image) + if isTag { + foundRef, ok := imageMap[image] + if !ok { + return nil, ErrBadTest + } + ref = foundRef + } + + // Images in chronological order + if repo == repo1 && ref == image11Digest { + result := map[string]cvemodel.CVE{ + "CVE1": { + ID: "CVE1", + Severity: "MEDIUM", + Title: "Title CVE1", + Description: "Description CVE1", + }, + } + + cache.Add(ref, result) + + return result, nil + } + + if repo == repo1 && zcommon.Contains([]string{image12Digest, image21Digest}, ref) { + result := map[string]cvemodel.CVE{ + "CVE1": { + ID: "CVE1", + Severity: "MEDIUM", + Title: "Title CVE1", + Description: "Description CVE1", + }, + "CVE2": { + ID: "CVE2", + Severity: "HIGH", + Title: "Title CVE2", + Description: "Description CVE2", + }, + "CVE3": { + ID: "CVE3", + Severity: "LOW", + Title: "Title CVE3", + Description: "Description CVE3", + }, + } + + cache.Add(ref, result) + + return result, nil + } + + if repo == repo1 && ref == image13Digest { + result := map[string]cvemodel.CVE{ + "CVE3": { + ID: "CVE3", + Severity: "LOW", + Title: "Title CVE3", + Description: "Description CVE3", + }, + } + + cache.Add(ref, result) + + return result, nil + } + + // As a minor release on 1.0.0 banch + // does not include all fixes published in 1.1.0 + if repo == repo1 && ref == image14Digest { + result := map[string]cvemodel.CVE{ + "CVE1": { + ID: "CVE1", + Severity: "MEDIUM", + Title: "Title CVE1", + Description: "Description CVE1", + }, + "CVE3": { + ID: "CVE3", + Severity: "LOW", + Title: "Title CVE3", + Description: "Description CVE3", + }, + } + + cache.Add(ref, result) + + return result, nil + } + + // Unexpected error while scanning + if repo == "repo7" { + return map[string]cvemodel.CVE{}, ErrFailedScan + } + + if (repo == repoIndex && ref == indexDigest) || + (repo == repoIndex && ref == indexM1Digest) { + result := map[string]cvemodel.CVE{ + "CVE1": { + ID: "CVE1", + Severity: "MEDIUM", + Title: "Title CVE1", + Description: "Description CVE1", + }, + } + + // Simulate scanning an index results in scanning its manifests + if ref == indexDigest { + cache.Add(indexM1Digest, result) + cache.Add(indexM2Digest, map[string]cvemodel.CVE{}) + cache.Add(indexM3Digest, map[string]cvemodel.CVE{}) + } + + cache.Add(ref, result) + + return result, nil + } + + // By default the image has no vulnerabilities + result = map[string]cvemodel.CVE{} + cache.Add(ref, result) + + return result, nil + }, + IsImageFormatScannableFn: func(repo string, reference string) (bool, error) { + if repo == repoIndex { + return true, nil + } + + // Almost same logic compared to actual Trivy specific implementation + imageDir, inputTag := repo, reference + + repoMeta, err := metaDB.GetRepoMeta(imageDir) + if err != nil { + return false, err + } + + manifestDigestStr := reference + + if zcommon.IsTag(reference) { + var ok bool + + descriptor, ok := repoMeta.Tags[inputTag] + if !ok { + return false, zerr.ErrTagMetaNotFound + } + + manifestDigestStr = descriptor.Digest + } + + manifestDigest, err := godigest.Parse(manifestDigestStr) + if err != nil { + return false, err + } + + manifestData, err := metaDB.GetManifestData(manifestDigest) + if err != nil { + return false, err + } + + var manifestContent ispec.Manifest + + err = json.Unmarshal(manifestData.ManifestBlob, &manifestContent) + if err != nil { + return false, zerr.ErrScanNotSupported + } + + for _, imageLayer := range manifestContent.Layers { + switch imageLayer.MediaType { + case ispec.MediaTypeImageLayerGzip, ispec.MediaTypeImageLayer, string(regTypes.DockerLayer): + + return true, nil + default: + + return false, zerr.ErrScanNotSupported + } + } + + return false, nil + }, + IsImageMediaScannableFn: func(repo, digest, mediaType string) (bool, error) { + if repo == "repo2" { + if digest == image21Digest { + return false, nil + } + } + + return true, nil + }, + IsResultCachedFn: func(digest string) bool { + return cache.Contains(digest) + }, + UpdateDBFn: func() error { + cache.Purge() + + return nil + }, + } + + // Purge scan, it should not be needed + So(scanner.UpdateDB(), ShouldBeNil) + + // Verify none of the entries are cached to begin with + t.Log("verify cache is initially empty") + + for image, digestStr := range imageMap { + t.Log("expecting " + image + " " + digestStr + " to be absent from cache") + So(scanner.IsResultCached(digestStr), ShouldBeFalse) + } + + // Start the generator + generator := cveinfo.NewScanTaskGenerator(metaDB, scanner, logger) + + sch.SubmitGenerator(generator, 10*time.Second, scheduler.MediumPriority) + + ctx, cancel := context.WithCancel(context.Background()) + + sch.RunScheduler(ctx) + + defer cancel() + + // Make sure the scanner generator has completed despite errors + found, err := ReadLogFileAndSearchString(logPath, + "Scheduled CVE scan: finished for available images", 20*time.Second) + So(err, ShouldBeNil) + So(found, ShouldBeTrue) + + t.Log("verify cache is up to date after scanner generator ran") + + // Verify all of the entries are cached + for image, digestStr := range imageMap { + repo, _, _ := zcommon.GetImageDirAndReference(image) + + ok, err := scanner.IsImageFormatScannable(repo, digestStr) + if ok && err == nil && repo != "repo7" { + t.Log("expecting " + image + " " + digestStr + " to be present in cache") + So(scanner.IsResultCached(digestStr), ShouldBeTrue) + } else { + // We don't cache results for unscannable manifests + t.Log("expecting " + image + " " + digestStr + " to be absent from cache") + So(scanner.IsResultCached(digestStr), ShouldBeFalse) + } + } + + // Make sure the scanner generator is catching the metadb error for repo5:nonexitent-manifest + found, err = ReadLogFileAndSearchString(logPath, + "Scheduled CVE scan: error while obtaining repo metadata", 20*time.Second) + So(err, ShouldBeNil) + So(found, ShouldBeTrue) + + // Make sure the scanner generator is catching the scanning error for repo7 + found, err = ReadLogFileAndSearchString(logPath, + "Scheduled CVE scan errored for image", 20*time.Second) + So(err, ShouldBeNil) + So(found, ShouldBeTrue) + + // Make sure the scanner generator is triggered at least twice + found, err = ReadLogFileAndCountStringOccurence(logPath, + "Scheduled CVE scan: finished for available images", 30*time.Second, 2) + So(err, ShouldBeNil) + So(found, ShouldBeTrue) + }) +} + +func TestScanGeneratorWithRealData(t *testing.T) { + Convey("Test CVE scanning task scheduler real data", t, func() { + rootDir := t.TempDir() + + logFile, err := os.CreateTemp(t.TempDir(), "zot-log*.txt") + logPath := logFile.Name() + So(err, ShouldBeNil) + + defer os.Remove(logFile.Name()) // clean up + + logger := log.NewLogger("debug", logPath) + writers := io.MultiWriter(os.Stdout, logFile) + logger.Logger = logger.Output(writers) + + cfg := config.New() + cfg.Scheduler = &config.SchedulerConfig{NumWorkers: 3} + + boltDriver, err := boltdb.GetBoltDriver(boltdb.DBParameters{RootDir: rootDir}) + So(err, ShouldBeNil) + + metaDB, err := boltdb.New(boltDriver, logger) + So(err, ShouldBeNil) + + imageStore := local.NewImageStore(rootDir, false, false, 0, 0, false, false, + logger, monitoring.NewMetricsServer(false, logger), nil, nil) + storeController := storage.StoreController{DefaultStore: imageStore} + + image := CreateRandomVulnerableImage() + + err = WriteImageToFileSystem(image, "zot-test", "0.0.1", storeController) + So(err, ShouldBeNil) + + err = meta.ParseStorage(metaDB, storeController, logger) + So(err, ShouldBeNil) + + scanner := cveinfo.NewScanner(storeController, metaDB, "ghcr.io/project-zot/trivy-db", "", logger) + err = scanner.UpdateDB() + So(err, ShouldBeNil) + + So(scanner.IsResultCached(image.DigestStr()), ShouldBeFalse) + + sch := scheduler.NewScheduler(cfg, logger) + + generator := cveinfo.NewScanTaskGenerator(metaDB, scanner, logger) + + // Start the generator + sch.SubmitGenerator(generator, 120*time.Second, scheduler.MediumPriority) + + ctx, cancel := context.WithCancel(context.Background()) + + sch.RunScheduler(ctx) + + defer cancel() + + // Make sure the scanner generator has completed + found, err := ReadLogFileAndSearchString(logPath, + "Scheduled CVE scan: finished for available images", 120*time.Second) + So(err, ShouldBeNil) + So(found, ShouldBeTrue) + + found, err = ReadLogFileAndSearchString(logPath, + image.ManifestDescriptor.Digest.String(), 120*time.Second) + So(err, ShouldBeNil) + So(found, ShouldBeTrue) + + found, err = ReadLogFileAndSearchString(logPath, + "Scheduled CVE scan completed successfully for image", 120*time.Second) + So(err, ShouldBeNil) + So(found, ShouldBeTrue) + + So(scanner.IsResultCached(image.DigestStr()), ShouldBeTrue) + + cveMap, err := scanner.ScanImage("zot-test:0.0.1") + So(err, ShouldBeNil) + t.Logf("cveMap: %v", cveMap) + // As of September 22 2023 there are 5 CVEs: + // CVE-2023-1255, CVE-2023-2650, CVE-2023-2975, CVE-2023-3817, CVE-2023-3446 + // There may be more discovered in the future + So(len(cveMap), ShouldBeGreaterThanOrEqualTo, 5) + So(cveMap, ShouldContainKey, "CVE-2023-1255") + So(cveMap, ShouldContainKey, "CVE-2023-2650") + So(cveMap, ShouldContainKey, "CVE-2023-2975") + So(cveMap, ShouldContainKey, "CVE-2023-3817") + So(cveMap, ShouldContainKey, "CVE-2023-3446") + + cveInfo := cveinfo.NewCVEInfo(scanner, metaDB, logger) + + // Based on cache population only, no extra scanning + cveSummary, err := cveInfo.GetCVESummaryForImageMedia("zot-test", image.DigestStr(), + image.ManifestDescriptor.MediaType) + So(err, ShouldBeNil) + So(cveSummary.Count, ShouldBeGreaterThanOrEqualTo, 5) + // As of September 22 the max severity is MEDIUM, but new CVEs could appear in the future + So([]string{"MEDIUM", "HIGH", "CRITICAL"}, ShouldContain, cveSummary.MaxSeverity) + }) +} diff --git a/pkg/extensions/search/cve/trivy/scanner.go b/pkg/extensions/search/cve/trivy/scanner.go index e952d631..86e0a2f1 100644 --- a/pkg/extensions/search/cve/trivy/scanner.go +++ b/pkg/extensions/search/cve/trivy/scanner.go @@ -23,6 +23,7 @@ import ( zerr "zotregistry.io/zot/errors" zcommon "zotregistry.io/zot/pkg/common" + cvecache "zotregistry.io/zot/pkg/extensions/search/cve/cache" cvemodel "zotregistry.io/zot/pkg/extensions/search/cve/model" "zotregistry.io/zot/pkg/log" mcommon "zotregistry.io/zot/pkg/meta/common" @@ -78,7 +79,7 @@ type Scanner struct { storeController storage.StoreController log log.Logger dbLock *sync.Mutex - cache *CveCache + cache *cvecache.CveCache dbRepository string javaDBRepository string } @@ -120,7 +121,7 @@ func NewScanner(storeController storage.StoreController, cveController: cveController, storeController: storeController, dbLock: &sync.Mutex{}, - cache: NewCveCache(cacheSize, log), + cache: cvecache.NewCveCache(cacheSize, log), dbRepository: dbRepository, javaDBRepository: javaDBRepository, } @@ -258,9 +259,6 @@ func (scanner Scanner) isManifestScanable(digestStr string) (bool, error) { case ispec.MediaTypeImageLayerGzip, ispec.MediaTypeImageLayer, string(regTypes.DockerLayer): continue default: - scanner.log.Debug().Str("mediaType", imageLayer.MediaType). - Msg("image media type not supported for scanning") - return false, zerr.ErrScanNotSupported } } @@ -304,6 +302,15 @@ func (scanner Scanner) isIndexScanable(digestStr string) (bool, error) { return false, nil } +func (scanner Scanner) IsResultCached(digest string) bool { + // Check if the entry exists in cache without updating the recent-ness + return scanner.cache.Contains(digest) +} + +func (scanner Scanner) GetCachedResult(digest string) map[string]cvemodel.CVE { + return scanner.cache.Get(digest) +} + func (scanner Scanner) ScanImage(image string) (map[string]cvemodel.CVE, error) { var ( originalImageInput = image @@ -430,6 +437,10 @@ func (scanner Scanner) scanManifest(repo, digest string) (map[string]cvemodel.CV } func (scanner Scanner) scanIndex(repo, digest string) (map[string]cvemodel.CVE, error) { + if cachedMap := scanner.cache.Get(digest); cachedMap != nil { + return cachedMap, nil + } + indexData, err := scanner.metaDB.GetIndexData(godigest.Digest(digest)) if err != nil { return map[string]cvemodel.CVE{}, err @@ -457,12 +468,14 @@ func (scanner Scanner) scanIndex(repo, digest string) (map[string]cvemodel.CVE, } } + scanner.cache.Add(digest, indexCveIDMap) + return indexCveIDMap, nil } // UpdateDB downloads the Trivy DB / Cache under the store root directory. func (scanner Scanner) UpdateDB() error { - // We need a lock as using multiple substores each with it's own DB + // We need a lock as using multiple substores each with its own DB // can result in a DATARACE because some varibles in trivy-db are global // https://github.com/project-zot/trivy-db/blob/main/pkg/db/db.go#L23 scanner.dbLock.Lock() diff --git a/pkg/extensions/search/cve/trivy/scanner_test.go b/pkg/extensions/search/cve/trivy/scanner_test.go index bb336c41..c9a9aa21 100644 --- a/pkg/extensions/search/cve/trivy/scanner_test.go +++ b/pkg/extensions/search/cve/trivy/scanner_test.go @@ -216,8 +216,14 @@ func TestVulnerableLayer(t *testing.T) { cveMap, err := scanner.ScanImage("repo@" + img.DigestStr()) So(err, ShouldBeNil) t.Logf("cveMap: %v", cveMap) - // As of July 15 2023 there are 3 CVEs: CVE-2023-1255, CVE-2023-2650, CVE-2023-2975 + // As of September 17 2023 there are 5 CVEs: + // CVE-2023-1255, CVE-2023-2650, CVE-2023-2975, CVE-2023-3817, CVE-2023-3446 // There may be more discovered in the future - So(len(cveMap), ShouldBeGreaterThanOrEqualTo, 3) + So(len(cveMap), ShouldBeGreaterThanOrEqualTo, 5) + So(cveMap, ShouldContainKey, "CVE-2023-1255") + So(cveMap, ShouldContainKey, "CVE-2023-2650") + So(cveMap, ShouldContainKey, "CVE-2023-2975") + So(cveMap, ShouldContainKey, "CVE-2023-3817") + So(cveMap, ShouldContainKey, "CVE-2023-3446") }) } diff --git a/pkg/extensions/search/cve/update.go b/pkg/extensions/search/cve/update.go new file mode 100644 index 00000000..871bb974 --- /dev/null +++ b/pkg/extensions/search/cve/update.go @@ -0,0 +1,120 @@ +package cveinfo + +import ( + "context" + "sync" + "time" + + "zotregistry.io/zot/pkg/log" + "zotregistry.io/zot/pkg/scheduler" +) + +type state int + +const ( + pending state = iota + running + done +) + +func NewDBUpdateTaskGenerator( + interval time.Duration, + scanner Scanner, + log log.Logger, +) scheduler.TaskGenerator { + generator := &DBUpdateTaskGenerator{ + interval, + scanner, + log, + pending, + 0, + time.Now(), + &sync.Mutex{}, + } + + return generator +} + +type DBUpdateTaskGenerator struct { + interval time.Duration + scanner Scanner + log log.Logger + status state + waitTime time.Duration + lastTaskTime time.Time + lock *sync.Mutex +} + +func (gen *DBUpdateTaskGenerator) Next() (scheduler.Task, error) { + var newTask scheduler.Task + + gen.lock.Lock() + + if gen.status == pending && time.Since(gen.lastTaskTime) >= gen.waitTime { + newTask = newDBUpdadeTask(gen.interval, gen.scanner, gen, gen.log) + gen.status = running + } + gen.lock.Unlock() + + return newTask, nil +} + +func (gen *DBUpdateTaskGenerator) IsDone() bool { + gen.lock.Lock() + status := gen.status + gen.lock.Unlock() + + return status == done +} + +func (gen *DBUpdateTaskGenerator) IsReady() bool { + return true +} + +func (gen *DBUpdateTaskGenerator) Reset() { + gen.lock.Lock() + gen.status = pending + gen.waitTime = 0 + gen.lock.Unlock() +} + +type dbUpdateTask struct { + interval time.Duration + scanner Scanner + generator *DBUpdateTaskGenerator + log log.Logger +} + +func newDBUpdadeTask(interval time.Duration, scanner Scanner, + generator *DBUpdateTaskGenerator, log log.Logger, +) *dbUpdateTask { + return &dbUpdateTask{interval, scanner, generator, log} +} + +func (dbt *dbUpdateTask) DoWork(ctx context.Context) error { + dbt.log.Info().Msg("updating the CVE database") + + err := dbt.scanner.UpdateDB() + if err != nil { + dbt.generator.lock.Lock() + dbt.generator.status = pending + + if dbt.generator.waitTime == 0 { + dbt.generator.waitTime = time.Second + } + + dbt.generator.waitTime *= 2 + dbt.generator.lastTaskTime = time.Now() + dbt.generator.lock.Unlock() + + return err + } + + dbt.generator.lock.Lock() + dbt.generator.lastTaskTime = time.Now() + dbt.generator.status = done + dbt.generator.lock.Unlock() + dbt.log.Info().Str("DB update completed, next update scheduled after", dbt.interval.String()).Msg("") + + return nil +} diff --git a/pkg/extensions/extension_search_test.go b/pkg/extensions/search/cve/update_test.go similarity index 84% rename from pkg/extensions/extension_search_test.go rename to pkg/extensions/search/cve/update_test.go index d1f94cd0..bb050285 100644 --- a/pkg/extensions/extension_search_test.go +++ b/pkg/extensions/search/cve/update_test.go @@ -1,7 +1,7 @@ //go:build search // +build search -package extensions_test +package cveinfo_test import ( "context" @@ -14,7 +14,6 @@ import ( . "github.com/smartystreets/goconvey/convey" "zotregistry.io/zot/pkg/api/config" - . "zotregistry.io/zot/pkg/extensions" cveinfo "zotregistry.io/zot/pkg/extensions/search/cve" "zotregistry.io/zot/pkg/log" mTypes "zotregistry.io/zot/pkg/meta/types" @@ -24,8 +23,8 @@ import ( "zotregistry.io/zot/pkg/test/mocks" ) -func TestTrivyDBGenerator(t *testing.T) { - Convey("Test trivy task scheduler reset", t, func() { +func TestCVEDBGenerator(t *testing.T) { + Convey("Test CVE DB task scheduler reset", t, func() { logFile, err := os.CreateTemp(t.TempDir(), "zot-log*.txt") logPath := logFile.Name() So(err, ShouldBeNil) @@ -57,8 +56,8 @@ func TestTrivyDBGenerator(t *testing.T) { }, } - cveInfo := cveinfo.NewCVEInfo(storeController, metaDB, "ghcr.io/project-zot/trivy-db", "", logger) - generator := NewTrivyTaskGenerator(time.Minute, cveInfo, logger) + cveScanner := cveinfo.NewScanner(storeController, metaDB, "ghcr.io/project-zot/trivy-db", "", logger) + generator := cveinfo.NewDBUpdateTaskGenerator(time.Minute, cveScanner, logger) sch.SubmitGenerator(generator, 12000*time.Millisecond, scheduler.HighPriority) diff --git a/pkg/extensions/search/resolver_test.go b/pkg/extensions/search/resolver_test.go index 29a108c8..eedcce38 100644 --- a/pkg/extensions/search/resolver_test.go +++ b/pkg/extensions/search/resolver_test.go @@ -2078,6 +2078,68 @@ func TestCVEResolvers(t *testing.T) { //nolint:gocyclo } } + getCveResults := func(digestStr string) map[string]cvemodel.CVE { + if digestStr == digest1.String() { + return map[string]cvemodel.CVE{ + "CVE1": { + ID: "CVE1", + Severity: "HIGH", + Title: "Title CVE1", + Description: "Description CVE1", + }, + "CVE2": { + ID: "CVE2", + Severity: "MEDIUM", + Title: "Title CVE2", + Description: "Description CVE2", + }, + "CVE3": { + ID: "CVE3", + Severity: "LOW", + Title: "Title CVE3", + Description: "Description CVE3", + }, + "CVE34": { + ID: "CVE34", + Severity: "LOW", + Title: "Title for CVE34", + Description: "Description CVE34", + }, + } + } + + if digestStr == digest2.String() { + return map[string]cvemodel.CVE{ + "CVE2": { + ID: "CVE2", + Severity: "MEDIUM", + Title: "Title CVE2", + Description: "Description CVE2", + }, + "CVE3": { + ID: "CVE3", + Severity: "LOW", + Title: "Title CVE3", + Description: "Description CVE3", + }, + } + } + + if digestStr == digest3.String() { + return map[string]cvemodel.CVE{ + "CVE3": { + ID: "CVE3", + Severity: "LOW", + Title: "Title CVE3", + Description: "Description CVE3", + }, + } + } + + // By default the image has no vulnerabilities + return map[string]cvemodel.CVE{} + } + // MetaDB loaded with initial data, now mock the scanner // Setup test CVE data in mock scanner scanner := mocks.CveScannerMock{ @@ -2092,65 +2154,13 @@ func TestCVEResolvers(t *testing.T) { //nolint:gocyclo digest = godigest.Digest(digestStr) } - if digest.String() == digest1.String() { - return map[string]cvemodel.CVE{ - "CVE1": { - ID: "CVE1", - Severity: "HIGH", - Title: "Title CVE1", - Description: "Description CVE1", - }, - "CVE2": { - ID: "CVE2", - Severity: "MEDIUM", - Title: "Title CVE2", - Description: "Description CVE2", - }, - "CVE3": { - ID: "CVE3", - Severity: "LOW", - Title: "Title CVE3", - Description: "Description CVE3", - }, - "CVE34": { - ID: "CVE34", - Severity: "LOW", - Title: "Title for CVE34", - Description: "Description CVE34", - }, - }, nil - } - - if digest.String() == digest2.String() { - return map[string]cvemodel.CVE{ - "CVE2": { - ID: "CVE2", - Severity: "MEDIUM", - Title: "Title CVE2", - Description: "Description CVE2", - }, - "CVE3": { - ID: "CVE3", - Severity: "LOW", - Title: "Title CVE3", - Description: "Description CVE3", - }, - }, nil - } - - if digest.String() == digest3.String() { - return map[string]cvemodel.CVE{ - "CVE3": { - ID: "CVE3", - Severity: "LOW", - Title: "Title CVE3", - Description: "Description CVE3", - }, - }, nil - } - - // By default the image has no vulnerabilities - return map[string]cvemodel.CVE{}, nil + return getCveResults(digest.String()), nil + }, + GetCachedResultFn: func(digestStr string) map[string]cvemodel.CVE { + return getCveResults(digestStr) + }, + IsResultCachedFn: func(digestStr string) bool { + return true }, } diff --git a/pkg/extensions/search/search_test.go b/pkg/extensions/search/search_test.go index 33a8d858..90322865 100644 --- a/pkg/extensions/search/search_test.go +++ b/pkg/extensions/search/search_test.go @@ -220,87 +220,84 @@ func uploadNewRepoTag(tag string, repoName string, baseURL string, layers [][]by return err } -func getMockCveInfo(metaDB mTypes.MetaDB, log log.Logger) cveinfo.CveInfo { +func getMockCveScanner(metaDB mTypes.MetaDB) cveinfo.Scanner { // MetaDB loaded with initial data, mock the scanner // Setup test CVE data in mock scanner + getCveResults := func(image string) map[string]cvemodel.CVE { + if image == "zot-cve-test:0.0.1" || image == "a/zot-cve-test:0.0.1" || + image == "zot-test:0.0.1" || image == "a/zot-test:0.0.1" || + strings.Contains(image, "sha256:40d1f74918aefed733c590f798d7eafde8fc0a7ec63bb8bc52eaae133cf92495") { + return map[string]cvemodel.CVE{ + "CVE1": { + ID: "CVE1", + Severity: "MEDIUM", + Title: "Title CVE1", + Description: "Description CVE1", + }, + "CVE2": { + ID: "CVE2", + Severity: "HIGH", + Title: "Title CVE2", + Description: "Description CVE2", + }, + "CVE3": { + ID: "CVE3", + Severity: "LOW", + Title: "Title CVE3", + Description: "Description CVE3", + }, + "CVE4": { + ID: "CVE4", + Severity: "CRITICAL", + Title: "Title CVE4", + Description: "Description CVE4", + }, + } + } + + if image == "test-repo:latest" || + strings.Contains(image, "sha256:9f8e1a125c4fb03a0f157d75999b73284ccc5cba18eb772e4643e3499343607e") { + return map[string]cvemodel.CVE{ + "CVE1": { + ID: "CVE1", + Severity: "MEDIUM", + Title: "Title CVE1", + Description: "Description CVE1", + }, + "CVE2": { + ID: "CVE2", + Severity: "HIGH", + Title: "Title CVE2", + Description: "Description CVE2", + }, + "CVE3": { + ID: "CVE3", + Severity: "LOW", + Title: "Title CVE3", + Description: "Description CVE3", + }, + "CVE4": { + ID: "CVE4", + Severity: "CRITICAL", + Title: "Title CVE4", + Description: "Description CVE4", + }, + } + } + + // By default the image has no vulnerabilities + return map[string]cvemodel.CVE{} + } + scanner := mocks.CveScannerMock{ ScanImageFn: func(image string) (map[string]cvemodel.CVE, error) { - if image == "zot-cve-test:0.0.1" || image == "a/zot-cve-test:0.0.1" || - strings.Contains(image, "zot-cve-test@sha256:40d1f74918aefed733c590f798d7eafde8fc0a7ec63bb8bc52eaae133cf92495") || - strings.Contains(image, "a/zot-cve-test@sha256:40d1f74918aefed733c590f798d7eafde8fc0a7ec63bb8bc52eaae133cf92495") { - return map[string]cvemodel.CVE{ - "CVE1": { - ID: "CVE1", - Severity: "MEDIUM", - Title: "Title CVE1", - Description: "Description CVE1", - }, - "CVE2": { - ID: "CVE2", - Severity: "HIGH", - Title: "Title CVE2", - Description: "Description CVE2", - }, - "CVE3": { - ID: "CVE3", - Severity: "LOW", - Title: "Title CVE3", - Description: "Description CVE3", - }, - }, nil - } - - if image == "zot-test:0.0.1" || image == "a/zot-test:0.0.1" || - strings.Contains(image, "a/zot-test@sha256:40d1f74918aefed733c590f798d7eafde8fc0a7ec63bb8bc52eaae133cf92495") || - strings.Contains(image, "zot-test@sha256:40d1f74918aefed733c590f798d7eafde8fc0a7ec63bb8bc52eaae133cf92495") { - return map[string]cvemodel.CVE{ - "CVE3": { - ID: "CVE3", - Severity: "LOW", - Title: "Title CVE3", - Description: "Description CVE3", - }, - "CVE4": { - ID: "CVE4", - Severity: "CRITICAL", - Title: "Title CVE4", - Description: "Description CVE4", - }, - }, nil - } - - if image == "test-repo:latest" || - image == "test-repo@sha256:9f8e1a125c4fb03a0f157d75999b73284ccc5cba18eb772e4643e3499343607e" { - return map[string]cvemodel.CVE{ - "CVE1": { - ID: "CVE1", - Severity: "MEDIUM", - Title: "Title CVE1", - Description: "Description CVE1", - }, - "CVE2": { - ID: "CVE2", - Severity: "HIGH", - Title: "Title CVE2", - Description: "Description CVE2", - }, - "CVE3": { - ID: "CVE3", - Severity: "LOW", - Title: "Title CVE3", - Description: "Description CVE3", - }, - "CVE4": { - ID: "CVE4", - Severity: "CRITICAL", - Title: "Title CVE4", - Description: "Description CVE4", - }, - }, nil - } - - // By default the image has no vulnerabilities - return map[string]cvemodel.CVE{}, nil + return getCveResults(image), nil + }, + GetCachedResultFn: func(digestStr string) map[string]cvemodel.CVE { + return getCveResults(digestStr) + }, + IsResultCachedFn: func(digestStr string) bool { + return true }, IsImageFormatScannableFn: func(repo string, reference string) (bool, error) { // Almost same logic compared to actual Trivy specific implementation @@ -357,11 +354,7 @@ func getMockCveInfo(metaDB mTypes.MetaDB, log log.Logger) cveinfo.CveInfo { }, } - return &cveinfo.BaseCveInfo{ - Log: log, - Scanner: scanner, - MetaDB: metaDB, - } + return &scanner } func TestRepoListWithNewestImage(t *testing.T) { @@ -698,7 +691,7 @@ func TestRepoListWithNewestImage(t *testing.T) { panic(err) } - ctlr.CveInfo = getMockCveInfo(ctlr.MetaDB, ctlr.Log) + ctlr.CveScanner = getMockCveScanner(ctlr.MetaDB) go func() { if err := ctlr.Run(ctx); !errors.Is(err, http.ErrServerClosed) { @@ -783,13 +776,7 @@ func TestRepoListWithNewestImage(t *testing.T) { ShouldBeGreaterThan, 0, ) - if repo.Name == "zot-cve-test" { - // This really depends on the test data, but with the current test image it's HIGH - So(vulnerabilities.MaxSeverity, ShouldEqual, "HIGH") - } else if repo.Name == "zot-test" { - // This really depends on the test data, but with the current test image it's CRITICAL - So(vulnerabilities.MaxSeverity, ShouldEqual, "CRITICAL") - } + So(vulnerabilities.MaxSeverity, ShouldEqual, "CRITICAL") } }) } @@ -3396,7 +3383,7 @@ func TestGlobalSearch(t *testing.T) { panic(err) } - ctlr.CveInfo = getMockCveInfo(ctlr.MetaDB, ctlr.Log) + ctlr.CveScanner = getMockCveScanner(ctlr.MetaDB) go func() { if err := ctlr.Run(ctx); !errors.Is(err, http.ErrServerClosed) { @@ -3592,9 +3579,15 @@ func TestGlobalSearch(t *testing.T) { // RepoInfo object does not provide vulnerability information so we need to check differently t.Logf("Found vulnerability summary %v", repoSummary.NewestImage.Vulnerabilities) - So(repoSummary.NewestImage.Vulnerabilities.Count, ShouldEqual, 0) - // There are 0 vulnerabilities this data used in tests - So(repoSummary.NewestImage.Vulnerabilities.MaxSeverity, ShouldEqual, "NONE") + if repoName == "repo1" { //nolint:goconst + So(repoSummary.NewestImage.Vulnerabilities.Count, ShouldEqual, 4) + // There are 4 vulnerabilities in the data used in tests + So(repoSummary.NewestImage.Vulnerabilities.MaxSeverity, ShouldEqual, "CRITICAL") + } else { + So(repoSummary.NewestImage.Vulnerabilities.Count, ShouldEqual, 0) + // There are 0 vulnerabilities this data used in tests + So(repoSummary.NewestImage.Vulnerabilities.MaxSeverity, ShouldEqual, "NONE") + } } query = ` @@ -3659,9 +3652,9 @@ func TestGlobalSearch(t *testing.T) { // RepoInfo object does not provide vulnerability information so we need to check differently t.Logf("Found vulnerability summary %v", actualImageSummary.Vulnerabilities) - // There are 0 vulnerabilities this data used in tests - So(actualImageSummary.Vulnerabilities.Count, ShouldEqual, 0) - So(actualImageSummary.Vulnerabilities.MaxSeverity, ShouldEqual, "NONE") + // There are 4 vulnerabilities in the data used in tests + So(actualImageSummary.Vulnerabilities.Count, ShouldEqual, 4) + So(actualImageSummary.Vulnerabilities.MaxSeverity, ShouldEqual, "CRITICAL") }) } @@ -6204,7 +6197,7 @@ func TestImageSummary(t *testing.T) { panic(err) } - ctlr.CveInfo = getMockCveInfo(ctlr.MetaDB, ctlr.Log) + ctlr.CveScanner = getMockCveScanner(ctlr.MetaDB) go func() { if err := ctlr.Run(ctx); !errors.Is(err, http.ErrServerClosed) { diff --git a/pkg/meta/boltdb/boltdb.go b/pkg/meta/boltdb/boltdb.go index fc4e850f..7ba432ce 100644 --- a/pkg/meta/boltdb/boltdb.go +++ b/pkg/meta/boltdb/boltdb.go @@ -1152,6 +1152,7 @@ func (bdw *BoltDB) FilterTags(ctx context.Context, filterFunc mTypes.FilterFunc, cursor = repoBuck.Cursor() userBookmarks = getUserBookmarks(ctx, transaction) userStars = getUserStars(ctx, transaction) + viewError error ) repoName, repoMetaBlob := cursor.First() @@ -1163,9 +1164,10 @@ func (bdw *BoltDB) FilterTags(ctx context.Context, filterFunc mTypes.FilterFunc, repoMeta := mTypes.RepoMetadata{} - err := json.Unmarshal(repoMetaBlob, &repoMeta) - if err != nil { - return err + if err := json.Unmarshal(repoMetaBlob, &repoMeta); err != nil { + viewError = errors.Join(viewError, err) + + continue } repoMeta.IsBookmarked = zcommon.Contains(userBookmarks, repoMeta.Name) @@ -1180,7 +1182,10 @@ func (bdw *BoltDB) FilterTags(ctx context.Context, filterFunc mTypes.FilterFunc, manifestMeta, err := fetchManifestMetaWithCheck(repoMeta, manifestDigest, manifestMetadataMap, manifestBuck) if err != nil { - return fmt.Errorf("metadb: error while unmashaling manifest metadata for digest %s %w", manifestDigest, err) + err = fmt.Errorf("metadb: error while unmashaling manifest metadata for digest %s %w", manifestDigest, err) + viewError = errors.Join(viewError, err) + + continue } if filterFunc(repoMeta, manifestMeta) { @@ -1192,14 +1197,20 @@ func (bdw *BoltDB) FilterTags(ctx context.Context, filterFunc mTypes.FilterFunc, indexData, err := fetchIndexDataWithCheck(indexDigest, indexDataMap, indexBuck) if err != nil { - return fmt.Errorf("metadb: error while getting index data for digest %s %w", indexDigest, err) + err = fmt.Errorf("metadb: error while getting index data for digest %s %w", indexDigest, err) + viewError = errors.Join(viewError, err) + + continue } var indexContent ispec.Index err = json.Unmarshal(indexData.IndexBlob, &indexContent) if err != nil { - return fmt.Errorf("metadb: error while unmashaling index content for digest %s %w", indexDigest, err) + err = fmt.Errorf("metadb: error while unmashaling index content for digest %s %w", indexDigest, err) + viewError = errors.Join(viewError, err) + + continue } matchedManifests := []ispec.Descriptor{} @@ -1209,7 +1220,10 @@ func (bdw *BoltDB) FilterTags(ctx context.Context, filterFunc mTypes.FilterFunc, manifestMeta, err := fetchManifestMetaWithCheck(repoMeta, manifestDigest, manifestMetadataMap, manifestBuck) if err != nil { - return fmt.Errorf("metadb: error while getting manifest data for digest %s %w", manifestDigest, err) + err = fmt.Errorf("metadb: error while getting manifest data for digest %s %w", manifestDigest, err) + viewError = errors.Join(viewError, err) + + continue } if filterFunc(repoMeta, manifestMeta) { @@ -1223,7 +1237,9 @@ func (bdw *BoltDB) FilterTags(ctx context.Context, filterFunc mTypes.FilterFunc, indexBlob, err := json.Marshal(indexContent) if err != nil { - return err + viewError = errors.Join(viewError, err) + + continue } indexData.IndexBlob = indexBlob @@ -1247,7 +1263,7 @@ func (bdw *BoltDB) FilterTags(ctx context.Context, filterFunc mTypes.FilterFunc, foundRepos = append(foundRepos, repoMeta) } - return nil + return viewError }) return foundRepos, manifestMetadataMap, indexDataMap, err diff --git a/pkg/meta/dynamodb/dynamodb.go b/pkg/meta/dynamodb/dynamodb.go index 48d2c5ae..e5b32021 100644 --- a/pkg/meta/dynamodb/dynamodb.go +++ b/pkg/meta/dynamodb/dynamodb.go @@ -1007,6 +1007,7 @@ func (dwr *DynamoDB) FilterTags(ctx context.Context, filterFunc mTypes.FilterFun repoMetaAttributeIterator AttributesIterator userBookmarks = getUserBookmarks(ctx, dwr) userStars = getUserStars(ctx, dwr) + aggregateError error ) repoMetaAttributeIterator = NewBaseDynamoAttributesIterator( @@ -1014,19 +1015,24 @@ func (dwr *DynamoDB) FilterTags(ctx context.Context, filterFunc mTypes.FilterFun ) repoMetaAttribute, err := repoMetaAttributeIterator.First(ctx) + if err != nil { + return foundRepos, manifestMetadataMap, indexDataMap, err + } for ; repoMetaAttribute != nil; repoMetaAttribute, err = repoMetaAttributeIterator.Next(ctx) { if err != nil { - return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, map[string]mTypes.IndexData{}, - err + aggregateError = errors.Join(aggregateError, err) + + continue } var repoMeta mTypes.RepoMetadata err := attributevalue.Unmarshal(repoMetaAttribute, &repoMeta) if err != nil { - return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, map[string]mTypes.IndexData{}, - err + aggregateError = errors.Join(aggregateError, err) + + continue } if ok, err := reqCtx.RepoIsUserAvailable(ctx, repoMeta.Name); !ok || err != nil { @@ -1046,8 +1052,10 @@ func (dwr *DynamoDB) FilterTags(ctx context.Context, filterFunc mTypes.FilterFun manifestMeta, err := dwr.fetchManifestMetaWithCheck(repoMeta.Name, manifestDigest, //nolint:contextcheck manifestMetadataMap) if err != nil { - return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, map[string]mTypes.IndexData{}, - fmt.Errorf("metadb: error while unmashaling manifest metadata for digest %s \n%w", manifestDigest, err) + err = fmt.Errorf("metadb: error while unmashaling manifest metadata for digest %s \n%w", manifestDigest, err) + aggregateError = errors.Join(aggregateError, err) + + continue } if filterFunc(repoMeta, manifestMeta) { @@ -1059,16 +1067,20 @@ func (dwr *DynamoDB) FilterTags(ctx context.Context, filterFunc mTypes.FilterFun indexData, err := dwr.fetchIndexDataWithCheck(indexDigest, indexDataMap) //nolint:contextcheck if err != nil { - return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, map[string]mTypes.IndexData{}, - fmt.Errorf("metadb: error while getting index data for digest %s %w", indexDigest, err) + err = fmt.Errorf("metadb: error while getting index data for digest %s %w", indexDigest, err) + aggregateError = errors.Join(aggregateError, err) + + continue } var indexContent ispec.Index err = json.Unmarshal(indexData.IndexBlob, &indexContent) if err != nil { - return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, map[string]mTypes.IndexData{}, - fmt.Errorf("metadb: error while unmashaling index content for digest %s %w", indexDigest, err) + err = fmt.Errorf("metadb: error while unmashaling index content for digest %s %w", indexDigest, err) + aggregateError = errors.Join(aggregateError, err) + + continue } matchedManifests := []ispec.Descriptor{} @@ -1079,8 +1091,10 @@ func (dwr *DynamoDB) FilterTags(ctx context.Context, filterFunc mTypes.FilterFun manifestMeta, err := dwr.fetchManifestMetaWithCheck(repoMeta.Name, manifestDigest, //nolint:contextcheck manifestMetadataMap) if err != nil { - return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, map[string]mTypes.IndexData{}, - fmt.Errorf("%w metadb: error while getting manifest data for digest %s", err, manifestDigest) + err = fmt.Errorf("%w metadb: error while getting manifest data for digest %s", err, manifestDigest) + aggregateError = errors.Join(aggregateError, err) + + continue } if filterFunc(repoMeta, manifestMeta) { @@ -1094,8 +1108,9 @@ func (dwr *DynamoDB) FilterTags(ctx context.Context, filterFunc mTypes.FilterFun indexBlob, err := json.Marshal(indexContent) if err != nil { - return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, map[string]mTypes.IndexData{}, - err + aggregateError = errors.Join(aggregateError, err) + + continue } indexData.IndexBlob = indexBlob @@ -1119,7 +1134,7 @@ func (dwr *DynamoDB) FilterTags(ctx context.Context, filterFunc mTypes.FilterFun foundRepos = append(foundRepos, repoMeta) } - return foundRepos, manifestMetadataMap, indexDataMap, err + return foundRepos, manifestMetadataMap, indexDataMap, aggregateError } func (dwr *DynamoDB) FilterRepos(ctx context.Context, filter mTypes.FilterRepoFunc, diff --git a/pkg/meta/types/types.go b/pkg/meta/types/types.go index cb9a74aa..b439eaba 100644 --- a/pkg/meta/types/types.go +++ b/pkg/meta/types/types.go @@ -66,7 +66,7 @@ type MetaDB interface { //nolint:interfacebloat // SetManifestData sets ManifestData for a given manifest in the database SetManifestData(manifestDigest godigest.Digest, md ManifestData) error - // GetManifestData return the manifest and it's related config + // GetManifestData return the manifest and its related config GetManifestData(manifestDigest godigest.Digest) (ManifestData, error) // GetManifestMeta returns ManifestMetadata for a given manifest from the database diff --git a/pkg/test/mocks/cve_mock.go b/pkg/test/mocks/cve_mock.go index 1cd460f8..9c1e950f 100644 --- a/pkg/test/mocks/cve_mock.go +++ b/pkg/test/mocks/cve_mock.go @@ -10,11 +10,8 @@ type CveInfoMock struct { GetImageListWithCVEFixedFn func(repo, cveID string) ([]cvemodel.TagInfo, error) GetCVEListForImageFn func(repo string, reference string, searchedCVE string, pageInput cvemodel.PageInput, ) ([]cvemodel.CVE, common.PageInfo, error) - GetCVESummaryForImageFn func(repo string, reference string, - ) (cvemodel.ImageCVESummary, error) GetCVESummaryForImageMediaFn func(repo string, digest, mediaType string, ) (cvemodel.ImageCVESummary, error) - UpdateDBFn func() error } func (cveInfo CveInfoMock) GetImageListForCVE(repo, cveID string) ([]cvemodel.TagInfo, error) { @@ -47,15 +44,6 @@ func (cveInfo CveInfoMock) GetCVEListForImage(repo string, reference string, return []cvemodel.CVE{}, common.PageInfo{}, nil } -func (cveInfo CveInfoMock) GetCVESummaryForImage(repo string, reference string, -) (cvemodel.ImageCVESummary, error) { - if cveInfo.GetCVESummaryForImageFn != nil { - return cveInfo.GetCVESummaryForImageFn(repo, reference) - } - - return cvemodel.ImageCVESummary{}, nil -} - func (cveInfo CveInfoMock) GetCVESummaryForImageMedia(repo, digest, mediaType string, ) (cvemodel.ImageCVESummary, error) { if cveInfo.GetCVESummaryForImageMediaFn != nil { @@ -65,17 +53,11 @@ func (cveInfo CveInfoMock) GetCVESummaryForImageMedia(repo, digest, mediaType st return cvemodel.ImageCVESummary{}, nil } -func (cveInfo CveInfoMock) UpdateDB() error { - if cveInfo.UpdateDBFn != nil { - return cveInfo.UpdateDBFn() - } - - return nil -} - type CveScannerMock struct { IsImageFormatScannableFn func(repo string, reference string) (bool, error) IsImageMediaScannableFn func(repo string, digest, mediaType string) (bool, error) + IsResultCachedFn func(digest string) bool + GetCachedResultFn func(digest string) map[string]cvemodel.CVE ScanImageFn func(image string) (map[string]cvemodel.CVE, error) UpdateDBFn func() error } @@ -96,6 +78,22 @@ func (scanner CveScannerMock) IsImageMediaScannable(repo string, digest, mediaTy return true, nil } +func (scanner CveScannerMock) IsResultCached(digest string) bool { + if scanner.IsResultCachedFn != nil { + return scanner.IsResultCachedFn(digest) + } + + return false +} + +func (scanner CveScannerMock) GetCachedResult(digest string) map[string]cvemodel.CVE { + if scanner.GetCachedResultFn != nil { + return scanner.GetCachedResultFn(digest) + } + + return map[string]cvemodel.CVE{} +} + func (scanner CveScannerMock) ScanImage(image string) (map[string]cvemodel.CVE, error) { if scanner.ScanImageFn != nil { return scanner.ScanImageFn(image)