github/zot
1
0
Fork 0
mirror of https://github.com/project-zot/zot.git synced 2026-06-16 04:17:55 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(cve): implement CVE scanning as background tasks (#1833)

Browse Source 
1. Move existing CVE DB download generator/task login under the cve package
2. Add a new CVE scanner task generator and task type to run in the background, as well as tests for it
3. Move the CVE cache in its own package
4. Add a CVE scanner methods to check if an entry is present in the cache, and to retreive the results
5. Modify the FilterTags MetaDB method to not exit on first error
This is needed in order to pass all tags to the generator,
instead of the generator stopping at the first set of invalid data
6. Integrate the new scanning task generator with the existing zot code.
7. Fix an issue where the CVE scan results for multiarch images was not cached
8. Rewrite some of the older CVE tests to use the new image-utils test package
9. Use the CVE scanner as attribute of the controller instead of CveInfo.
Remove functionality of CVE DB update from CveInfo, it is now responsible,
as the name states, only for providing CVE information.
10. The logic to get maximum severity and cve count for image sumaries now uses only the scanner cache.
11. Removed the GetCVESummaryForImage method from CveInfo as it was only used in tests

Signed-off-by: Andrei Aaron <aaaron@luxoft.com>
This commit is contained in:
Andrei Aaron
2023-09-22 21:49:17 +03:00
committed by GitHub
parent 4e04be420e
commit 7c78f80a96
21 changed files with 1812 additions and 807 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/meta/boltdb/boltdb.go
+25 -9
View File
@@ -1152,6 +1152,7 @@ func (bdw *BoltDB) FilterTags(ctx context.Context, filterFunc mTypes.FilterFunc,
cursor = repoBuck.Cursor()
userBookmarks = getUserBookmarks(ctx, transaction)
userStars = getUserStars(ctx, transaction)
viewError error
)
repoName, repoMetaBlob := cursor.First()
@@ -1163,9 +1164,10 @@ func (bdw *BoltDB) FilterTags(ctx context.Context, filterFunc mTypes.FilterFunc,
repoMeta := mTypes.RepoMetadata{}
err := json.Unmarshal(repoMetaBlob, &repoMeta)
if err != nil {
return err
if err := json.Unmarshal(repoMetaBlob, &repoMeta); err != nil {
viewError = errors.Join(viewError, err)
continue
}
repoMeta.IsBookmarked = zcommon.Contains(userBookmarks, repoMeta.Name)
@@ -1180,7 +1182,10 @@ func (bdw *BoltDB) FilterTags(ctx context.Context, filterFunc mTypes.FilterFunc,
manifestMeta, err := fetchManifestMetaWithCheck(repoMeta, manifestDigest, manifestMetadataMap, manifestBuck)
if err != nil {
return fmt.Errorf("metadb: error while unmashaling manifest metadata for digest %s %w", manifestDigest, err)
err = fmt.Errorf("metadb: error while unmashaling manifest metadata for digest %s %w", manifestDigest, err)
viewError = errors.Join(viewError, err)
continue
}
if filterFunc(repoMeta, manifestMeta) {
@@ -1192,14 +1197,20 @@ func (bdw *BoltDB) FilterTags(ctx context.Context, filterFunc mTypes.FilterFunc,
indexData, err := fetchIndexDataWithCheck(indexDigest, indexDataMap, indexBuck)
if err != nil {
return fmt.Errorf("metadb: error while getting index data for digest %s %w", indexDigest, err)
err = fmt.Errorf("metadb: error while getting index data for digest %s %w", indexDigest, err)
viewError = errors.Join(viewError, err)
continue
}
var indexContent ispec.Index
err = json.Unmarshal(indexData.IndexBlob, &indexContent)
if err != nil {
return fmt.Errorf("metadb: error while unmashaling index content for digest %s %w", indexDigest, err)
err = fmt.Errorf("metadb: error while unmashaling index content for digest %s %w", indexDigest, err)
viewError = errors.Join(viewError, err)
continue
}
matchedManifests := []ispec.Descriptor{}
@@ -1209,7 +1220,10 @@ func (bdw *BoltDB) FilterTags(ctx context.Context, filterFunc mTypes.FilterFunc,
manifestMeta, err := fetchManifestMetaWithCheck(repoMeta, manifestDigest, manifestMetadataMap, manifestBuck)
if err != nil {
return fmt.Errorf("metadb: error while getting manifest data for digest %s %w", manifestDigest, err)
err = fmt.Errorf("metadb: error while getting manifest data for digest %s %w", manifestDigest, err)
viewError = errors.Join(viewError, err)
continue
}
if filterFunc(repoMeta, manifestMeta) {
@@ -1223,7 +1237,9 @@ func (bdw *BoltDB) FilterTags(ctx context.Context, filterFunc mTypes.FilterFunc,
indexBlob, err := json.Marshal(indexContent)
if err != nil {
return err
viewError = errors.Join(viewError, err)
continue
}
indexData.IndexBlob = indexBlob
@@ -1247,7 +1263,7 @@ func (bdw *BoltDB) FilterTags(ctx context.Context, filterFunc mTypes.FilterFunc,
foundRepos = append(foundRepos, repoMeta)
}
return nil
return viewError
})
return foundRepos, manifestMetadataMap, indexDataMap, err
pkg/meta/dynamodb/dynamodb.go
+30 -15
View File
@@ -1007,6 +1007,7 @@ func (dwr *DynamoDB) FilterTags(ctx context.Context, filterFunc mTypes.FilterFun
repoMetaAttributeIterator AttributesIterator
userBookmarks = getUserBookmarks(ctx, dwr)
userStars = getUserStars(ctx, dwr)
aggregateError error
)
repoMetaAttributeIterator = NewBaseDynamoAttributesIterator(
@@ -1014,19 +1015,24 @@ func (dwr *DynamoDB) FilterTags(ctx context.Context, filterFunc mTypes.FilterFun
)
repoMetaAttribute, err := repoMetaAttributeIterator.First(ctx)
if err != nil {
return foundRepos, manifestMetadataMap, indexDataMap, err
}
for ; repoMetaAttribute != nil; repoMetaAttribute, err = repoMetaAttributeIterator.Next(ctx) {
if err != nil {
return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, map[string]mTypes.IndexData{},
err
aggregateError = errors.Join(aggregateError, err)
continue
}
var repoMeta mTypes.RepoMetadata
err := attributevalue.Unmarshal(repoMetaAttribute, &repoMeta)
if err != nil {
return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, map[string]mTypes.IndexData{},
err
aggregateError = errors.Join(aggregateError, err)
continue
}
if ok, err := reqCtx.RepoIsUserAvailable(ctx, repoMeta.Name); !ok || err != nil {
@@ -1046,8 +1052,10 @@ func (dwr *DynamoDB) FilterTags(ctx context.Context, filterFunc mTypes.FilterFun
manifestMeta, err := dwr.fetchManifestMetaWithCheck(repoMeta.Name, manifestDigest, //nolint:contextcheck
manifestMetadataMap)
if err != nil {
return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, map[string]mTypes.IndexData{},
fmt.Errorf("metadb: error while unmashaling manifest metadata for digest %s \n%w", manifestDigest, err)
err = fmt.Errorf("metadb: error while unmashaling manifest metadata for digest %s \n%w", manifestDigest, err)
aggregateError = errors.Join(aggregateError, err)
continue
}
if filterFunc(repoMeta, manifestMeta) {
@@ -1059,16 +1067,20 @@ func (dwr *DynamoDB) FilterTags(ctx context.Context, filterFunc mTypes.FilterFun
indexData, err := dwr.fetchIndexDataWithCheck(indexDigest, indexDataMap) //nolint:contextcheck
if err != nil {
return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, map[string]mTypes.IndexData{},
fmt.Errorf("metadb: error while getting index data for digest %s %w", indexDigest, err)
err = fmt.Errorf("metadb: error while getting index data for digest %s %w", indexDigest, err)
aggregateError = errors.Join(aggregateError, err)
continue
}
var indexContent ispec.Index
err = json.Unmarshal(indexData.IndexBlob, &indexContent)
if err != nil {
return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, map[string]mTypes.IndexData{},
fmt.Errorf("metadb: error while unmashaling index content for digest %s %w", indexDigest, err)
err = fmt.Errorf("metadb: error while unmashaling index content for digest %s %w", indexDigest, err)
aggregateError = errors.Join(aggregateError, err)
continue
}
matchedManifests := []ispec.Descriptor{}
@@ -1079,8 +1091,10 @@ func (dwr *DynamoDB) FilterTags(ctx context.Context, filterFunc mTypes.FilterFun
manifestMeta, err := dwr.fetchManifestMetaWithCheck(repoMeta.Name, manifestDigest, //nolint:contextcheck
manifestMetadataMap)
if err != nil {
return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, map[string]mTypes.IndexData{},
fmt.Errorf("%w metadb: error while getting manifest data for digest %s", err, manifestDigest)
err = fmt.Errorf("%w metadb: error while getting manifest data for digest %s", err, manifestDigest)
aggregateError = errors.Join(aggregateError, err)
continue
}
if filterFunc(repoMeta, manifestMeta) {
@@ -1094,8 +1108,9 @@ func (dwr *DynamoDB) FilterTags(ctx context.Context, filterFunc mTypes.FilterFun
indexBlob, err := json.Marshal(indexContent)
if err != nil {
return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, map[string]mTypes.IndexData{},
err
aggregateError = errors.Join(aggregateError, err)
continue
}
indexData.IndexBlob = indexBlob
@@ -1119,7 +1134,7 @@ func (dwr *DynamoDB) FilterTags(ctx context.Context, filterFunc mTypes.FilterFun
foundRepos = append(foundRepos, repoMeta)
}
return foundRepos, manifestMetadataMap, indexDataMap, err
return foundRepos, manifestMetadataMap, indexDataMap, aggregateError
}
func (dwr *DynamoDB) FilterRepos(ctx context.Context, filter mTypes.FilterRepoFunc,
pkg/meta/types/types.go
+1 -1
View File
@@ -66,7 +66,7 @@ type MetaDB interface { //nolint:interfacebloat
// SetManifestData sets ManifestData for a given manifest in the database
SetManifestData(manifestDigest godigest.Digest, md ManifestData) error
// GetManifestData return the manifest and it's related config
// GetManifestData return the manifest and its related config
GetManifestData(manifestDigest godigest.Digest) (ManifestData, error)
// GetManifestMeta returns ManifestMetadata for a given manifest from the database