mirror of
https://github.com/project-zot/zot.git
synced 2026-06-17 04:48:26 +08:00
implement scrub to check manifest/blob integrity
Signed-off-by: Andreea-Lupu <andreealupu1470@yahoo.com>
This commit is contained in:
Andreea-Lupu
committed by
Ramkumar Chinchani
Ramkumar Chinchani
parent
914cf5c356
commit
c61c3836db
@@ -0,0 +1,258 @@
|
||||
package storage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"path"
|
||||
"strings"
|
||||
|
||||
"github.com/olekukonko/tablewriter"
|
||||
godigest "github.com/opencontainers/go-digest"
|
||||
ispec "github.com/opencontainers/image-spec/specs-go/v1"
|
||||
"github.com/opencontainers/umoci"
|
||||
"github.com/opencontainers/umoci/oci/casext"
|
||||
"zotregistry.io/zot/errors"
|
||||
)
|
||||
|
||||
const (
|
||||
colImageNameIndex = iota
|
||||
colTagIndex
|
||||
colStatusIndex
|
||||
colErrorIndex
|
||||
|
||||
imageNameWidth = 32
|
||||
tagWidth = 24
|
||||
statusWidth = 8
|
||||
errorWidth = 8
|
||||
)
|
||||
|
||||
type ScrubImageResult struct {
|
||||
ImageName string `json:"image_name"`
|
||||
Tag string `json:"tag"`
|
||||
Status string `json:"status"`
|
||||
Error string `json:"error"`
|
||||
}
|
||||
|
||||
type ScrubResults struct {
|
||||
ScrubResults []ScrubImageResult `json:"scrub_results"`
|
||||
}
|
||||
|
||||
func (sc StoreController) CheckAllBlobsIntegrity() (ScrubResults, error) {
|
||||
results := ScrubResults{}
|
||||
|
||||
imageStoreList := make(map[string]ImageStore)
|
||||
if sc.SubStore != nil {
|
||||
imageStoreList = sc.SubStore
|
||||
}
|
||||
|
||||
imageStoreList[""] = sc.DefaultStore
|
||||
|
||||
for _, is := range imageStoreList {
|
||||
images, err := is.GetRepositories()
|
||||
|
||||
if err != nil {
|
||||
return results, err
|
||||
}
|
||||
|
||||
for _, repo := range images {
|
||||
imageResults, err := checkImage(repo, is)
|
||||
|
||||
if err != nil {
|
||||
return results, err
|
||||
}
|
||||
|
||||
results.ScrubResults = append(results.ScrubResults, imageResults...)
|
||||
}
|
||||
}
|
||||
|
||||
return results, nil
|
||||
}
|
||||
|
||||
func checkImage(imageName string, is ImageStore) ([]ScrubImageResult, error) {
|
||||
results := []ScrubImageResult{}
|
||||
|
||||
dir := path.Join(is.RootDir(), imageName)
|
||||
if !is.DirExists(dir) {
|
||||
return results, errors.ErrRepoNotFound
|
||||
}
|
||||
|
||||
ctxUmoci := context.Background()
|
||||
|
||||
oci, err := umoci.OpenLayout(dir)
|
||||
if err != nil {
|
||||
return results, err
|
||||
}
|
||||
|
||||
defer oci.Close()
|
||||
|
||||
is.RLock()
|
||||
defer is.RUnlock()
|
||||
|
||||
buf, err := ioutil.ReadFile(path.Join(dir, "index.json"))
|
||||
|
||||
if err != nil {
|
||||
return results, err
|
||||
}
|
||||
|
||||
var index ispec.Index
|
||||
if err := json.Unmarshal(buf, &index); err != nil {
|
||||
return results, errors.ErrRepoNotFound
|
||||
}
|
||||
|
||||
for _, m := range index.Manifests {
|
||||
tag, ok := m.Annotations[ispec.AnnotationRefName]
|
||||
if ok {
|
||||
imageResult := checkIntegrity(ctxUmoci, imageName, tag, oci, m, dir)
|
||||
results = append(results, imageResult)
|
||||
}
|
||||
}
|
||||
|
||||
return results, nil
|
||||
}
|
||||
|
||||
func checkIntegrity(ctx context.Context, imageName, tagName string, oci casext.Engine, manifest ispec.Descriptor,
|
||||
dir string) ScrubImageResult {
|
||||
// check manifest and config
|
||||
stat, err := umoci.Stat(ctx, oci, manifest)
|
||||
|
||||
imageRes := ScrubImageResult{}
|
||||
|
||||
if err != nil {
|
||||
imageRes = getResult(imageName, tagName, err)
|
||||
} else {
|
||||
// check layers
|
||||
for _, s := range stat.History {
|
||||
layer := s.Layer
|
||||
if layer == nil {
|
||||
continue
|
||||
}
|
||||
|
||||
// check layer
|
||||
layerPath := path.Join(dir, "blobs", layer.Digest.Algorithm().String(), layer.Digest.Hex())
|
||||
|
||||
_, err = os.Stat(layerPath)
|
||||
if err != nil {
|
||||
imageRes = getResult(imageName, tagName, errors.ErrBlobNotFound)
|
||||
break
|
||||
}
|
||||
|
||||
f, err := os.Open(layerPath)
|
||||
if err != nil {
|
||||
imageRes = getResult(imageName, tagName, errors.ErrBlobNotFound)
|
||||
break
|
||||
}
|
||||
|
||||
computedDigest, err := godigest.FromReader(f)
|
||||
f.Close()
|
||||
|
||||
if err != nil {
|
||||
imageRes = getResult(imageName, tagName, errors.ErrBadBlobDigest)
|
||||
break
|
||||
}
|
||||
|
||||
if computedDigest != layer.Digest {
|
||||
imageRes = getResult(imageName, tagName, errors.ErrBadBlobDigest)
|
||||
break
|
||||
}
|
||||
|
||||
imageRes = getResult(imageName, tagName, nil)
|
||||
}
|
||||
}
|
||||
|
||||
return imageRes
|
||||
}
|
||||
|
||||
func getResult(imageName, tag string, err error) ScrubImageResult {
|
||||
var status string
|
||||
|
||||
var errField string
|
||||
|
||||
if err != nil {
|
||||
status = "affected"
|
||||
errField = err.Error()
|
||||
} else {
|
||||
status = "ok"
|
||||
errField = ""
|
||||
}
|
||||
|
||||
return ScrubImageResult{
|
||||
ImageName: imageName,
|
||||
Tag: tag,
|
||||
Status: status,
|
||||
Error: errField,
|
||||
}
|
||||
}
|
||||
|
||||
func getScrubTableWriter(writer io.Writer) *tablewriter.Table {
|
||||
table := tablewriter.NewWriter(writer)
|
||||
|
||||
table.SetAutoWrapText(false)
|
||||
table.SetAutoFormatHeaders(true)
|
||||
table.SetHeaderAlignment(tablewriter.ALIGN_LEFT)
|
||||
table.SetAlignment(tablewriter.ALIGN_LEFT)
|
||||
table.SetCenterSeparator("")
|
||||
table.SetColumnSeparator("")
|
||||
table.SetRowSeparator("")
|
||||
table.SetHeaderLine(false)
|
||||
table.SetBorder(false)
|
||||
table.SetTablePadding(" ")
|
||||
table.SetNoWhiteSpace(true)
|
||||
table.SetColMinWidth(colImageNameIndex, imageNameWidth)
|
||||
table.SetColMinWidth(colTagIndex, tagWidth)
|
||||
table.SetColMinWidth(colStatusIndex, statusWidth)
|
||||
table.SetColMinWidth(colErrorIndex, errorWidth)
|
||||
|
||||
return table
|
||||
}
|
||||
|
||||
func printScrubTableHeader(writer io.Writer) {
|
||||
table := getScrubTableWriter(writer)
|
||||
|
||||
row := make([]string, 4)
|
||||
|
||||
row[colImageNameIndex] = "IMAGE NAME"
|
||||
row[colTagIndex] = "TAG"
|
||||
row[colStatusIndex] = "STATUS"
|
||||
row[colErrorIndex] = "ERROR"
|
||||
|
||||
table.Append(row)
|
||||
table.Render()
|
||||
}
|
||||
|
||||
func printImageResult(imageResult ScrubImageResult) string {
|
||||
var builder strings.Builder
|
||||
|
||||
table := getScrubTableWriter(&builder)
|
||||
table.SetColMinWidth(colImageNameIndex, imageNameWidth)
|
||||
table.SetColMinWidth(colTagIndex, tagWidth)
|
||||
table.SetColMinWidth(colStatusIndex, statusWidth)
|
||||
table.SetColMinWidth(colErrorIndex, errorWidth)
|
||||
|
||||
row := make([]string, 4)
|
||||
|
||||
row[colImageNameIndex] = imageResult.ImageName
|
||||
row[colTagIndex] = imageResult.Tag
|
||||
row[colStatusIndex] = imageResult.Status
|
||||
row[colErrorIndex] = imageResult.Error
|
||||
|
||||
table.Append(row)
|
||||
table.Render()
|
||||
|
||||
return builder.String()
|
||||
}
|
||||
|
||||
func (results ScrubResults) PrintScrubResults(resultWriter io.Writer) {
|
||||
var builder strings.Builder
|
||||
|
||||
printScrubTableHeader(&builder)
|
||||
fmt.Fprint(resultWriter, builder.String())
|
||||
|
||||
for _, res := range results.ScrubResults {
|
||||
imageResult := printImageResult(res)
|
||||
fmt.Fprint(resultWriter, imageResult)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,249 @@
|
||||
package storage_test
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"path"
|
||||
"regexp"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
godigest "github.com/opencontainers/go-digest"
|
||||
ispec "github.com/opencontainers/image-spec/specs-go/v1"
|
||||
. "github.com/smartystreets/goconvey/convey"
|
||||
"zotregistry.io/zot/pkg/extensions/monitoring"
|
||||
"zotregistry.io/zot/pkg/log"
|
||||
"zotregistry.io/zot/pkg/storage"
|
||||
)
|
||||
|
||||
const (
|
||||
repoName = "test"
|
||||
)
|
||||
|
||||
func TestCheckAllBlobsIntegrity(t *testing.T) {
|
||||
dir, err := ioutil.TempDir("", "scrub-test")
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
defer os.RemoveAll(dir)
|
||||
|
||||
log := log.NewLogger("debug", "")
|
||||
|
||||
metrics := monitoring.NewMetricsServer(false, log)
|
||||
|
||||
il := storage.NewImageStore(dir, true, true, log, metrics)
|
||||
|
||||
Convey("Scrub only one repo", t, func(c C) {
|
||||
// initialize repo
|
||||
err = il.InitRepo(repoName)
|
||||
So(err, ShouldBeNil)
|
||||
ok := il.DirExists(path.Join(il.RootDir(), repoName))
|
||||
So(ok, ShouldBeTrue)
|
||||
storeController := storage.StoreController{}
|
||||
storeController.DefaultStore = il
|
||||
So(storeController.GetImageStore(repoName), ShouldResemble, il)
|
||||
|
||||
sc := storage.StoreController{}
|
||||
sc.DefaultStore = il
|
||||
|
||||
const tag = "1.0"
|
||||
|
||||
var manifest string
|
||||
var config string
|
||||
var layer string
|
||||
|
||||
// create layer digest
|
||||
body := []byte("this is a blob")
|
||||
buf := bytes.NewBuffer(body)
|
||||
l := buf.Len()
|
||||
d := godigest.FromBytes(body)
|
||||
u, n, err := il.FullBlobUpload(repoName, buf, d.String())
|
||||
So(err, ShouldBeNil)
|
||||
So(n, ShouldEqual, len(body))
|
||||
So(u, ShouldNotBeEmpty)
|
||||
layer = d.String()
|
||||
|
||||
//create config digest
|
||||
created := time.Now().Format("2006-01-02T15:04:05Z")
|
||||
configBody := []byte(fmt.Sprintf(`{
|
||||
"created": "%v",
|
||||
"architecture": "amd64",
|
||||
"os": "linux",
|
||||
"rootfs": {
|
||||
"type": "layers",
|
||||
"diff_ids": [
|
||||
"",
|
||||
""
|
||||
]
|
||||
},
|
||||
"history": [
|
||||
{
|
||||
"created": "%v",
|
||||
"created_by": ""
|
||||
},
|
||||
{
|
||||
"created": "%v",
|
||||
"created_by": "",
|
||||
"empty_layer": true
|
||||
}
|
||||
]
|
||||
}`, created, created, created))
|
||||
configBuf := bytes.NewBuffer(configBody)
|
||||
configLen := configBuf.Len()
|
||||
configDigest := godigest.FromBytes(configBody)
|
||||
uConfig, nConfig, err := il.FullBlobUpload(repoName, configBuf, configDigest.String())
|
||||
So(err, ShouldBeNil)
|
||||
So(nConfig, ShouldEqual, len(configBody))
|
||||
So(uConfig, ShouldNotBeEmpty)
|
||||
config = configDigest.String()
|
||||
|
||||
// create manifest and add it to the repository
|
||||
annotationsMap := make(map[string]string)
|
||||
annotationsMap[ispec.AnnotationRefName] = tag
|
||||
m := ispec.Manifest{
|
||||
Config: ispec.Descriptor{
|
||||
MediaType: "application/vnd.oci.image.config.v1+json",
|
||||
Digest: configDigest,
|
||||
Size: int64(configLen),
|
||||
},
|
||||
Layers: []ispec.Descriptor{
|
||||
{
|
||||
MediaType: "application/vnd.oci.image.layer.v1.tar",
|
||||
Digest: d,
|
||||
Size: int64(l),
|
||||
},
|
||||
},
|
||||
Annotations: annotationsMap,
|
||||
}
|
||||
|
||||
m.SchemaVersion = 2
|
||||
mb, _ := json.Marshal(m)
|
||||
|
||||
manifest, err = il.PutImageManifest(repoName, tag, ispec.MediaTypeImageManifest, mb)
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
Convey("Blobs integrity not affected", func() {
|
||||
buff := bytes.NewBufferString("")
|
||||
|
||||
res, err := sc.CheckAllBlobsIntegrity()
|
||||
res.PrintScrubResults(buff)
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
space := regexp.MustCompile(`\s+`)
|
||||
str := space.ReplaceAllString(buff.String(), " ")
|
||||
actual := strings.TrimSpace(str)
|
||||
So(actual, ShouldContainSubstring, "IMAGE NAME TAG STATUS ERROR")
|
||||
So(actual, ShouldContainSubstring, "test 1.0 ok")
|
||||
})
|
||||
|
||||
Convey("Manifest integrity affected", func() {
|
||||
// get content of manifest file
|
||||
content, _, _, err := il.GetImageManifest(repoName, manifest)
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
// delete content of manifest file
|
||||
manifest = strings.ReplaceAll(manifest, "sha256:", "")
|
||||
manifestFile := path.Join(il.RootDir(), repoName, "/blobs/sha256", manifest)
|
||||
err = os.Truncate(manifestFile, 0)
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
buff := bytes.NewBufferString("")
|
||||
|
||||
res, err := sc.CheckAllBlobsIntegrity()
|
||||
res.PrintScrubResults(buff)
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
space := regexp.MustCompile(`\s+`)
|
||||
str := space.ReplaceAllString(buff.String(), " ")
|
||||
actual := strings.TrimSpace(str)
|
||||
So(actual, ShouldContainSubstring, "IMAGE NAME TAG STATUS ERROR")
|
||||
// verify error message
|
||||
So(actual, ShouldContainSubstring, "test 1.0 affected parse application/vnd.oci.image.manifest.v1+json")
|
||||
|
||||
// put manifest content back to file
|
||||
err = ioutil.WriteFile(manifestFile, content, 0600)
|
||||
So(err, ShouldBeNil)
|
||||
})
|
||||
|
||||
Convey("Config integrity affected", func() {
|
||||
// get content of config file
|
||||
content, err := il.GetBlobContent(repoName, config)
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
// delete content of config file
|
||||
config = strings.ReplaceAll(config, "sha256:", "")
|
||||
configFile := path.Join(il.RootDir(), repoName, "/blobs/sha256", config)
|
||||
err = os.Truncate(configFile, 0)
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
buff := bytes.NewBufferString("")
|
||||
|
||||
res, err := sc.CheckAllBlobsIntegrity()
|
||||
res.PrintScrubResults(buff)
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
space := regexp.MustCompile(`\s+`)
|
||||
str := space.ReplaceAllString(buff.String(), " ")
|
||||
actual := strings.TrimSpace(str)
|
||||
So(actual, ShouldContainSubstring, "IMAGE NAME TAG STATUS ERROR")
|
||||
So(actual, ShouldContainSubstring, "test 1.0 affected stat: parse application/vnd.oci.image.config.v1+json")
|
||||
|
||||
// put config content back to file
|
||||
err = ioutil.WriteFile(configFile, content, 0600)
|
||||
So(err, ShouldBeNil)
|
||||
})
|
||||
|
||||
Convey("Layers integrity affected", func() {
|
||||
// get content of layer
|
||||
content, err := il.GetBlobContent(repoName, layer)
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
// delete content of layer file
|
||||
layer = strings.ReplaceAll(layer, "sha256:", "")
|
||||
layerFile := path.Join(il.RootDir(), repoName, "/blobs/sha256", layer)
|
||||
err = os.Truncate(layerFile, 0)
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
buff := bytes.NewBufferString("")
|
||||
|
||||
res, err := sc.CheckAllBlobsIntegrity()
|
||||
res.PrintScrubResults(buff)
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
space := regexp.MustCompile(`\s+`)
|
||||
str := space.ReplaceAllString(buff.String(), " ")
|
||||
actual := strings.TrimSpace(str)
|
||||
So(actual, ShouldContainSubstring, "IMAGE NAME TAG STATUS ERROR")
|
||||
So(actual, ShouldContainSubstring, "test 1.0 affected blob: bad blob digest")
|
||||
|
||||
// put layer content back to file
|
||||
err = ioutil.WriteFile(layerFile, content, 0600)
|
||||
So(err, ShouldBeNil)
|
||||
})
|
||||
|
||||
Convey("Layer not found", func() {
|
||||
// delete layer file
|
||||
layer = strings.ReplaceAll(layer, "sha256:", "")
|
||||
layerFile := path.Join(il.RootDir(), repoName, "/blobs/sha256", layer)
|
||||
err = os.Remove(layerFile)
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
buff := bytes.NewBufferString("")
|
||||
|
||||
res, err := sc.CheckAllBlobsIntegrity()
|
||||
res.PrintScrubResults(buff)
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
space := regexp.MustCompile(`\s+`)
|
||||
str := space.ReplaceAllString(buff.String(), " ")
|
||||
actual := strings.TrimSpace(str)
|
||||
So(actual, ShouldContainSubstring, "IMAGE NAME TAG STATUS ERROR")
|
||||
So(actual, ShouldContainSubstring, "test 1.0 affected blob: not found")
|
||||
})
|
||||
})
|
||||
}
|
||||
Reference in New Issue
Block a user