github/zot
1
0
Fork 0
mirror of https://github.com/project-zot/zot.git synced 2026-06-16 04:17:55 +08:00
Code Issues Packages Projects Releases Wiki Activity

Leave zot repositories in a consistent state after zot hits fd limit closes #359

Browse Source 
Signed-off-by: Alexei Dodon <adodon@cisco.com>
This commit is contained in:
Alexei Dodon
2022-01-23 22:26:02 +02:00
committed by Ramkumar Chinchani
parent ba41368469
commit ad519e2d3e
7 changed files with 633 additions and 33 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/api/controller_test.go
+348
View File
@@ -5,6 +5,7 @@ package api_test
import (
"bufio"
"bytes"
"context"
"crypto/tls"
"crypto/x509"
@@ -4124,6 +4125,353 @@ func TestStorageCommit(t *testing.T) {
})
}
func TestInjectInterruptedImageManifest(t *testing.T) {
Convey("Make a new controller", t, func() {
port := test.GetFreePort()
baseURL := test.GetBaseURL(port)
conf := config.New()
conf.HTTP.Port = port
ctlr := api.NewController(conf)
dir := t.TempDir()
ctlr.Config.Storage.RootDirectory = dir
go startServer(ctlr)
defer stopServer(ctlr)
test.WaitTillServerReady(baseURL)
rthdlr := api.NewRouteHandler(ctlr)
Convey("Upload a blob & a config blob; Create an image manifest", func() {
// create a blob/layer
resp, err := resty.R().Post(baseURL + "/v2/repotest/blobs/uploads/")
So(err, ShouldBeNil)
So(resp.StatusCode(), ShouldEqual, http.StatusAccepted)
loc := test.Location(baseURL, resp)
So(loc, ShouldNotBeEmpty)
// since we are not specifying any prefix i.e provided in config while starting server,
// so it should store repotest to global root dir
_, err = os.Stat(path.Join(dir, "repotest"))
So(err, ShouldBeNil)
resp, err = resty.R().Get(loc)
So(err, ShouldBeNil)
So(resp.StatusCode(), ShouldEqual, http.StatusNoContent)
content := []byte("this is a dummy blob")
digest := godigest.FromBytes(content)
So(digest, ShouldNotBeNil)
// monolithic blob upload: success
resp, err = resty.R().SetQueryParam("digest", digest.String()).
SetHeader("Content-Type", "application/octet-stream").SetBody(content).Put(loc)
So(err, ShouldBeNil)
So(resp.StatusCode(), ShouldEqual, http.StatusCreated)
blobLoc := resp.Header().Get("Location")
So(blobLoc, ShouldNotBeEmpty)
So(resp.Header().Get("Content-Length"), ShouldEqual, "0")
So(resp.Header().Get(constants.DistContentDigestKey), ShouldNotBeEmpty)
// upload image config blob
resp, err = resty.R().Post(baseURL + "/v2/repotest/blobs/uploads/")
So(err, ShouldBeNil)
So(resp.StatusCode(), ShouldEqual, http.StatusAccepted)
loc = test.Location(baseURL, resp)
cblob, cdigest := test.GetRandomImageConfig()
resp, err = resty.R().
SetContentLength(true).
SetHeader("Content-Length", fmt.Sprintf("%d", len(cblob))).
SetHeader("Content-Type", "application/octet-stream").
SetQueryParam("digest", cdigest.String()).
SetBody(cblob).
Put(loc)
So(err, ShouldBeNil)
So(resp.StatusCode(), ShouldEqual, http.StatusCreated)
// create a manifest
manifest := ispec.Manifest{
Config: ispec.Descriptor{
MediaType: "application/vnd.oci.image.config.v1+json",
Digest: cdigest,
Size: int64(len(cblob)),
},
Layers: []ispec.Descriptor{
{
MediaType: "application/vnd.oci.image.layer.v1.tar",
Digest: digest,
Size: int64(len(content)),
},
},
}
manifest.SchemaVersion = 2
content, err = json.Marshal(manifest)
So(err, ShouldBeNil)
digest = godigest.FromBytes(content)
So(digest, ShouldNotBeNil)
// Testing router path: @Router /v2/{name}/manifests/{reference} [put]
Convey("Uploading an image manifest blob (when injected simulates an interrupted image manifest upload)", func() {
injected := test.InjectFailure(0)
request, _ := http.NewRequestWithContext(context.TODO(), "PUT", baseURL, bytes.NewReader(content))
request = mux.SetURLVars(request, map[string]string{"name": "repotest", "reference": "1.0"})
request.Header.Set("Content-Type", "application/vnd.oci.image.manifest.v1+json")
response := httptest.NewRecorder()
rthdlr.UpdateManifest(response, request)
resp := response.Result()
defer resp.Body.Close()
So(resp, ShouldNotBeNil)
if injected {
So(resp.StatusCode, ShouldEqual, http.StatusInternalServerError)
} else {
So(resp.StatusCode, ShouldEqual, http.StatusCreated)
}
})
})
})
}
func TestInjectTooManyOpenFiles(t *testing.T) {
Convey("Make a new controller", t, func() {
port := test.GetFreePort()
baseURL := test.GetBaseURL(port)
conf := config.New()
conf.HTTP.Port = port
ctlr := api.NewController(conf)
dir := t.TempDir()
ctlr.Config.Storage.RootDirectory = dir
go startServer(ctlr)
defer stopServer(ctlr)
test.WaitTillServerReady(baseURL)
rthdlr := api.NewRouteHandler(ctlr)
// create a blob/layer
resp, err := resty.R().Post(baseURL + "/v2/repotest/blobs/uploads/")
So(err, ShouldBeNil)
So(resp.StatusCode(), ShouldEqual, http.StatusAccepted)
loc := test.Location(baseURL, resp)
So(loc, ShouldNotBeEmpty)
// since we are not specifying any prefix i.e provided in config while starting server,
// so it should store repotest to global root dir
_, err = os.Stat(path.Join(dir, "repotest"))
So(err, ShouldBeNil)
resp, err = resty.R().Get(loc)
So(err, ShouldBeNil)
So(resp.StatusCode(), ShouldEqual, http.StatusNoContent)
content := []byte("this is a dummy blob")
digest := godigest.FromBytes(content)
So(digest, ShouldNotBeNil)
// monolithic blob upload
injected := test.InjectFailure(0)
if injected {
request, _ := http.NewRequestWithContext(context.TODO(), "PUT", loc, bytes.NewReader(content))
tokens := strings.Split(loc, "/")
request = mux.SetURLVars(request, map[string]string{"name": "repotest", "session_id": tokens[len(tokens)-1]})
q := request.URL.Query()
q.Add("digest", digest.String())
request.URL.RawQuery = q.Encode()
request.Header.Set("Content-Type", "application/octet-stream")
request.Header.Set("Content-Length", fmt.Sprintf("%d", len(content)))
response := httptest.NewRecorder()
rthdlr.UpdateBlobUpload(response, request)
resp := response.Result()
defer resp.Body.Close()
So(resp, ShouldNotBeNil)
So(resp.StatusCode, ShouldEqual, http.StatusInternalServerError)
} else {
resp, err = resty.R().SetQueryParam("digest", digest.String()).
SetHeader("Content-Type", "application/octet-stream").
SetBody(content).Put(loc)
So(err, ShouldBeNil)
So(resp.StatusCode(), ShouldEqual, http.StatusCreated)
blobLoc := resp.Header().Get("Location")
So(blobLoc, ShouldNotBeEmpty)
So(resp.Header().Get("Content-Length"), ShouldEqual, "0")
So(resp.Header().Get(constants.DistContentDigestKey), ShouldNotBeEmpty)
}
// upload image config blob
resp, err = resty.R().Post(baseURL + "/v2/repotest/blobs/uploads/")
So(err, ShouldBeNil)
So(resp.StatusCode(), ShouldEqual, http.StatusAccepted)
loc = test.Location(baseURL, resp)
cblob, cdigest := test.GetRandomImageConfig()
resp, err = resty.R().
SetContentLength(true).
SetHeader("Content-Length", fmt.Sprintf("%d", len(cblob))).
SetHeader("Content-Type", "application/octet-stream").
SetQueryParam("digest", cdigest.String()).
SetBody(cblob).
Put(loc)
So(err, ShouldBeNil)
So(resp.StatusCode(), ShouldEqual, http.StatusCreated)
// create a manifest
manifest := ispec.Manifest{
Config: ispec.Descriptor{
MediaType: "application/vnd.oci.image.config.v1+json",
Digest: cdigest,
Size: int64(len(cblob)),
},
Layers: []ispec.Descriptor{
{
MediaType: "application/vnd.oci.image.layer.v1.tar",
Digest: digest,
Size: int64(len(content)),
},
},
}
manifest.SchemaVersion = 2
content, err = json.Marshal(manifest)
So(err, ShouldBeNil)
digest = godigest.FromBytes(content)
So(digest, ShouldNotBeNil)
// Testing router path: @Router /v2/{name}/manifests/{reference} [put]
// nolint: lll
Convey("Uploading an image manifest blob (when injected simulates that PutImageManifest failed due to 'too many open files' error)", func() {
injected := test.InjectFailure(1)
request, _ := http.NewRequestWithContext(context.TODO(), "PUT", baseURL, bytes.NewReader(content))
request = mux.SetURLVars(request, map[string]string{"name": "repotest", "reference": "1.0"})
request.Header.Set("Content-Type", "application/vnd.oci.image.manifest.v1+json")
response := httptest.NewRecorder()
rthdlr.UpdateManifest(response, request)
resp := response.Result()
So(resp, ShouldNotBeNil)
defer resp.Body.Close()
if injected {
So(resp.StatusCode, ShouldEqual, http.StatusInternalServerError)
} else {
So(resp.StatusCode, ShouldEqual, http.StatusCreated)
}
})
Convey("when injected simulates a `too many open files` error inside PutImageManifest method of img store", func() {
injected := test.InjectFailure(2)
request, _ := http.NewRequestWithContext(context.TODO(), "PUT", baseURL, bytes.NewReader(content))
request = mux.SetURLVars(request, map[string]string{"name": "repotest", "reference": "1.0"})
request.Header.Set("Content-Type", "application/vnd.oci.image.manifest.v1+json")
response := httptest.NewRecorder()
rthdlr.UpdateManifest(response, request)
resp := response.Result()
defer resp.Body.Close()
So(resp, ShouldNotBeNil)
if injected {
So(resp.StatusCode, ShouldEqual, http.StatusInternalServerError)
} else {
So(resp.StatusCode, ShouldEqual, http.StatusCreated)
}
})
Convey("code coverage: error inside PutImageManifest method of img store (unable to marshal JSON)", func() {
injected := test.InjectFailure(1)
request, _ := http.NewRequestWithContext(context.TODO(), "PUT", baseURL, bytes.NewReader(content))
request = mux.SetURLVars(request, map[string]string{"name": "repotest", "reference": "1.0"})
request.Header.Set("Content-Type", "application/vnd.oci.image.manifest.v1+json")
response := httptest.NewRecorder()
rthdlr.UpdateManifest(response, request)
resp := response.Result()
defer resp.Body.Close()
So(resp, ShouldNotBeNil)
if injected {
So(resp.StatusCode, ShouldEqual, http.StatusInternalServerError)
} else {
So(resp.StatusCode, ShouldEqual, http.StatusCreated)
}
})
Convey("code coverage: error inside PutImageManifest method of img store (umoci.OpenLayout error)", func() {
injected := test.InjectFailure(3)
request, _ := http.NewRequestWithContext(context.TODO(), "PUT", baseURL, bytes.NewReader(content))
request = mux.SetURLVars(request, map[string]string{"name": "repotest", "reference": "1.0"})
request.Header.Set("Content-Type", "application/vnd.oci.image.manifest.v1+json")
response := httptest.NewRecorder()
rthdlr.UpdateManifest(response, request)
resp := response.Result()
defer resp.Body.Close()
So(resp, ShouldNotBeNil)
if injected {
So(resp.StatusCode, ShouldEqual, http.StatusInternalServerError)
} else {
So(resp.StatusCode, ShouldEqual, http.StatusCreated)
}
})
Convey("code coverage: error inside PutImageManifest method of img store (oci.GC)", func() {
injected := test.InjectFailure(4)
request, _ := http.NewRequestWithContext(context.TODO(), "PUT", baseURL, bytes.NewReader(content))
request = mux.SetURLVars(request, map[string]string{"name": "repotest", "reference": "1.0"})
request.Header.Set("Content-Type", "application/vnd.oci.image.manifest.v1+json")
response := httptest.NewRecorder()
rthdlr.UpdateManifest(response, request)
resp := response.Result()
defer resp.Body.Close()
So(resp, ShouldNotBeNil)
if injected {
So(resp.StatusCode, ShouldEqual, http.StatusInternalServerError)
} else {
So(resp.StatusCode, ShouldEqual, http.StatusCreated)
}
})
Convey("when index.json is not in json format", func() {
resp, err = resty.R().SetHeader("Content-Type", "application/vnd.oci.image.manifest.v1+json").
SetBody(content).Put(baseURL + "/v2/repotest/manifests/v1.0")
So(err, ShouldBeNil)
So(resp.StatusCode(), ShouldEqual, http.StatusCreated)
digestHdr := resp.Header().Get(constants.DistContentDigestKey)
So(digestHdr, ShouldNotBeEmpty)
So(digestHdr, ShouldEqual, digest.String())
indexFile := path.Join(dir, "repotest", "index.json")
_, err = os.Stat(indexFile)
So(err, ShouldBeNil)
indexContent := []byte(`not a JSON content`)
err = ioutil.WriteFile(indexFile, indexContent, 0o600)
So(err, ShouldBeNil)
resp, err = resty.R().SetHeader("Content-Type", "application/vnd.oci.image.manifest.v1+json").
SetBody(content).Put(baseURL + "/v2/repotest/manifests/v1.1")
So(err, ShouldBeNil)
So(resp.StatusCode(), ShouldEqual, http.StatusInternalServerError)
})
})
}
func getAllBlobs(imagePath string) []string {
blobList := make([]string, 0)
pkg/api/errors.go
+7
View File
@@ -34,6 +34,7 @@ const (
UNAUTHORIZED
DENIED
UNSUPPORTED
INVALID_INDEX
)
func (e ErrorCode) String() string {
@@ -53,6 +54,7 @@ func (e ErrorCode) String() string {
UNAUTHORIZED: "UNAUTHORIZED",
DENIED: "DENIED",
UNSUPPORTED: "UNSUPPORTED",
INVALID_INDEX: "INVALID_INDEX",
}
return errMap[e]
@@ -152,6 +154,11 @@ func NewError(code ErrorCode, detail ...interface{}) Error { //nolint: interface
Description: `The operation was unsupported due to a missing
implementation or invalid set of parameters.`,
},
INVALID_INDEX: {
Message: "Invalid format of index.json file of the repo",
Description: "index.json file does not contain data in json format",
},
}
err, ok := errMap[code]
pkg/api/routes.go
+37 -23
View File
@@ -33,6 +33,7 @@ import (
ext "zotregistry.io/zot/pkg/extensions"
"zotregistry.io/zot/pkg/log"
"zotregistry.io/zot/pkg/storage"
"zotregistry.io/zot/pkg/test"
// as required by swaggo.
_ "zotregistry.io/zot/swagger"
@@ -428,7 +429,9 @@ func (rh *RouteHandler) UpdateManifest(response http.ResponseWriter, request *ht
}
body, err := ioutil.ReadAll(request.Body)
if err != nil {
// hard to reach test case, injected error (simulates an interrupted image manifest upload)
// err could be io.ErrUnexpectedEOF
if err := test.Error(err); err != nil {
rh.c.Log.Error().Err(err).Msg("unexpected error")
response.WriteHeader(http.StatusInternalServerError)
@@ -449,8 +452,20 @@ func (rh *RouteHandler) UpdateManifest(response http.ResponseWriter, request *ht
} else if errors.Is(err, zerr.ErrBlobNotFound) {
WriteJSON(response, http.StatusBadRequest,
NewErrorList(NewError(BLOB_UNKNOWN, map[string]string{"blob": digest})))
} else if errors.Is(err, zerr.ErrRepoBadVersion) {
WriteJSON(response, http.StatusInternalServerError,
NewErrorList(NewError(INVALID_INDEX, map[string]string{"name": name})))
} else {
rh.c.Log.Error().Err(err).Msg("unexpected error")
// could be syscall.EMFILE (Err:0x18 too many opened files), etc
rh.c.Log.Error().Err(err).Msg("unexpected error: performing cleanup")
if err = imgStore.DeleteImageManifest(name, reference); err != nil {
// deletion of image manifest is important, but not critical for image repo consistancy
// in the worst scenario a partial manifest file written to disk will not affect the repo because
// the new manifest was not added to "index.json" file (it is possible that GC will take care of it)
rh.c.Log.Error().Err(err).Msgf("couldn't remove image manifest %s in repo %s", reference, name)
}
response.WriteHeader(http.StatusInternalServerError)
}
@@ -954,15 +969,7 @@ func (rh *RouteHandler) PatchBlobUpload(response http.ResponseWriter, request *h
}
if err != nil {
if errors.Is(err, io.ErrUnexpectedEOF) { //nolint:gocritic // errorslint conflicts with gocritic:IfElseChain
rh.c.Log.Warn().Msg("received unexpected EOF, removing .uploads/ files")
if err = imgStore.DeleteBlobUpload(name, sessionID); err != nil {
rh.c.Log.Error().Err(err).Msgf("couldn't remove blobUpload %s in repo %s", sessionID, name)
}
response.WriteHeader(http.StatusInternalServerError)
} else if errors.Is(err, zerr.ErrBadUploadRange) {
if errors.Is(err, zerr.ErrBadUploadRange) { //nolint:gocritic // errorslint conflicts with gocritic:IfElseChain
WriteJSON(response, http.StatusRequestedRangeNotSatisfiable,
NewErrorList(NewError(BLOB_UPLOAD_INVALID, map[string]string{"session_id": sessionID})))
} else if errors.Is(err, zerr.ErrRepoNotFound) {
@@ -972,7 +979,12 @@ func (rh *RouteHandler) PatchBlobUpload(response http.ResponseWriter, request *h
WriteJSON(response, http.StatusNotFound,
NewErrorList(NewError(BLOB_UPLOAD_UNKNOWN, map[string]string{"session_id": sessionID})))
} else {
rh.c.Log.Error().Err(err).Msg("unexpected error")
// could be io.ErrUnexpectedEOF, syscall.EMFILE (Err:0x18 too many opened files), etc
rh.c.Log.Error().Err(err).Msg("unexpected error: removing .uploads/ files")
if err = imgStore.DeleteBlobUpload(name, sessionID); err != nil {
rh.c.Log.Error().Err(err).Msgf("couldn't remove blobUpload %s in repo %s", sessionID, name)
}
response.WriteHeader(http.StatusInternalServerError)
}
@@ -1071,15 +1083,7 @@ func (rh *RouteHandler) UpdateBlobUpload(response http.ResponseWriter, request *
_, err = imgStore.PutBlobChunk(name, sessionID, from, to, request.Body)
if err != nil {
if errors.Is(err, io.ErrUnexpectedEOF) { //nolint:gocritic // errorslint conflicts with gocritic:IfElseChain
rh.c.Log.Warn().Msg("received unexpected EOF, removing .uploads/ files")
if err = imgStore.DeleteBlobUpload(name, sessionID); err != nil {
rh.c.Log.Error().Err(err).Msgf("couldn't remove blobUpload %s in repo %s", sessionID, name)
}
response.WriteHeader(http.StatusInternalServerError)
} else if errors.Is(err, zerr.ErrBadUploadRange) {
if errors.Is(err, zerr.ErrBadUploadRange) { //nolint:gocritic // errorslint conflicts with gocritic:IfElseChain
WriteJSON(response, http.StatusBadRequest,
NewErrorList(NewError(BLOB_UPLOAD_INVALID, map[string]string{"session_id": sessionID})))
} else if errors.Is(err, zerr.ErrRepoNotFound) {
@@ -1089,7 +1093,12 @@ func (rh *RouteHandler) UpdateBlobUpload(response http.ResponseWriter, request *
WriteJSON(response, http.StatusNotFound,
NewErrorList(NewError(BLOB_UPLOAD_UNKNOWN, map[string]string{"session_id": sessionID})))
} else {
rh.c.Log.Error().Err(err).Msg("unexpected error")
// could be io.ErrUnexpectedEOF, syscall.EMFILE (Err:0x18 too many opened files), etc
rh.c.Log.Error().Err(err).Msg("unexpected error: removing .uploads/ files")
if err = imgStore.DeleteBlobUpload(name, sessionID); err != nil {
rh.c.Log.Error().Err(err).Msgf("couldn't remove blobUpload %s in repo %s", sessionID, name)
}
response.WriteHeader(http.StatusInternalServerError)
}
@@ -1113,7 +1122,12 @@ finish:
WriteJSON(response, http.StatusNotFound,
NewErrorList(NewError(BLOB_UPLOAD_UNKNOWN, map[string]string{"session_id": sessionID})))
} else {
rh.c.Log.Error().Err(err).Msg("unexpected error")
// could be io.ErrUnexpectedEOF, syscall.EMFILE (Err:0x18 too many opened files), etc
rh.c.Log.Error().Err(err).Msg("unexpected error: removing .uploads/ files")
if err = imgStore.DeleteBlobUpload(name, sessionID); err != nil {
rh.c.Log.Error().Err(err).Msgf("couldn't remove blobUpload %s in repo %s", sessionID, name)
}
response.WriteHeader(http.StatusInternalServerError)
}