github/zot
1
0
Fork 0
mirror of https://github.com/project-zot/zot.git synced 2026-06-16 20:38:08 +08:00
Code Issues Packages Projects Releases Wiki Activity

routes: refactor locks to handle large file uploads

Browse Source 
The storage layer is protected with read-write locks.
However, we may be holding the locks over unnecessarily large critical
sections.

The typical workflow is that a blob is first uploaded via a per-client
private session-id meaning the blob is not publicly visible yet. When
the blob being uploaded is very large, the transfer takes a long time
while holding the lock.

Private session-id based uploads don't really need locks, and hold locks
only when blobs are published after the upload is complete.
This commit is contained in:
Ramkumar Chinchani
2020-09-30 17:16:34 -07:00
committed by Shivam Mishra
parent 25ad71787a
commit 386c72d332
2 changed files with 47 additions and 32 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/storage/storage.go
+35
View File
@@ -111,6 +111,9 @@ func (is *ImageStore) Unlock() {
func (is *ImageStore) InitRepo(name string) error {
repoDir := path.Join(is.rootDir, name)
is.Lock()
defer is.Unlock()
if fi, err := os.Stat(repoDir); err == nil && fi.IsDir() {
return nil
}
@@ -217,6 +220,9 @@ func (is *ImageStore) ValidateRepo(name string) (bool, error) {
func (is *ImageStore) GetRepositories() ([]string, error) {
dir := is.rootDir
is.RLock()
defer is.RUnlock()
_, err := ioutil.ReadDir(dir)
if err != nil {
is.log.Error().Err(err).Msg("failure walking storage root-dir")
@@ -258,6 +264,9 @@ func (is *ImageStore) GetImageTags(repo string) ([]string, error) {
return nil, errors.ErrRepoNotFound
}
is.RLock()
defer is.RUnlock()
buf, err := ioutil.ReadFile(path.Join(dir, "index.json"))
if err != nil {
is.log.Error().Err(err).Str("dir", dir).Msg("failed to read index.json")
@@ -289,6 +298,9 @@ func (is *ImageStore) GetImageManifest(repo string, reference string) ([]byte, s
return nil, "", "", errors.ErrRepoNotFound
}
is.RLock()
defer is.RUnlock()
buf, err := ioutil.ReadFile(path.Join(dir, "index.json"))
if err != nil {
@@ -414,6 +426,9 @@ func (is *ImageStore) PutImageManifest(repo string, reference string, mediaType
refIsDigest = true
}
is.Lock()
defer is.Unlock()
dir := path.Join(is.rootDir, repo)
buf, err := ioutil.ReadFile(path.Join(dir, "index.json"))
@@ -532,6 +547,9 @@ func (is *ImageStore) DeleteImageManifest(repo string, reference string) error {
return errors.ErrBadManifest
}
is.Lock()
defer is.Unlock()
buf, err := ioutil.ReadFile(path.Join(dir, "index.json"))
if err != nil {
@@ -770,6 +788,10 @@ func (is *ImageStore) FinishBlobUpload(repo string, uuid string, body io.Reader,
}
dir := path.Join(is.rootDir, repo, "blobs", dstDigest.Algorithm().String())
is.Lock()
defer is.Unlock()
ensureDir(dir, is.log)
dst := is.BlobPath(repo, dstDigest)
@@ -835,6 +857,10 @@ func (is *ImageStore) FullBlobUpload(repo string, body io.Reader, digest string)
}
dir := path.Join(is.rootDir, repo, "blobs", dstDigest.Algorithm().String())
is.Lock()
defer is.Unlock()
ensureDir(dir, is.log)
dst := is.BlobPath(repo, dstDigest)
@@ -948,6 +974,9 @@ func (is *ImageStore) CheckBlob(repo string, digest string,
blobPath := is.BlobPath(repo, d)
is.RLock()
defer is.RUnlock()
blobInfo, err := os.Stat(blobPath)
if err != nil {
is.log.Error().Err(err).Str("blob", blobPath).Msg("failed to stat blob")
@@ -969,6 +998,9 @@ func (is *ImageStore) GetBlob(repo string, digest string, mediaType string) (io.
blobPath := is.BlobPath(repo, d)
is.RLock()
defer is.RUnlock()
blobInfo, err := os.Stat(blobPath)
if err != nil {
is.log.Error().Err(err).Str("blob", blobPath).Msg("failed to stat blob")
@@ -994,6 +1026,9 @@ func (is *ImageStore) DeleteBlob(repo string, digest string) error {
blobPath := is.BlobPath(repo, d)
is.Lock()
defer is.Unlock()
_, err = os.Stat(blobPath)
if err != nil {
is.log.Error().Err(err).Str("blob", blobPath).Msg("failed to stat blob")