github/zot
1
0
Fork 0
mirror of https://github.com/project-zot/zot.git synced 2026-06-17 21:17:58 +08:00
Code Issues Packages Projects Releases Wiki Activity

make gc periodic

Browse Source 
Signed-off-by: Andreea-Lupu <andreealupu1470@yahoo.com>
This commit is contained in:
Andreea-Lupu
2022-03-21 20:40:37 +02:00
committed by Ramkumar Chinchani
parent 89c5f4f604
commit 5e35dfa28f
10 changed files with 327 additions and 36 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/storage/s3/storage.go
+3
View File
@@ -1006,6 +1006,9 @@ func (is *ObjectStorage) DedupeBlob(src string, dstDigest godigest.Digest, dst s
return nil
}
func (is *ObjectStorage) RunGCPeriodically(gcInterval time.Duration) {
}
// DeleteBlobUpload deletes an existing blob upload that is currently in progress.
func (is *ObjectStorage) DeleteBlobUpload(repo string, uuid string) error {
blobUploadPath := is.BlobUploadPath(repo, uuid)
pkg/storage/storage.go
+1
View File
@@ -44,4 +44,5 @@ type ImageStore interface {
GetIndexContent(repo string) ([]byte, error)
GetBlobContent(repo, digest string) ([]byte, error)
GetReferrers(repo, digest string, mediaType string) ([]artifactspec.Descriptor, error)
RunGCPeriodically(gcInterval time.Duration)
}
pkg/storage/storage_fs.go
+62 -15
View File
@@ -691,14 +691,7 @@ func (is *ImageStoreFS) PutImageManifest(repo string, reference string, mediaTyp
}
if is.gc {
oci, err := umoci.OpenLayout(dir)
if err := test.Error(err); err != nil {
return "", err
}
defer oci.Close()
err = oci.GC(context.Background(), ifOlderThan(is, repo, is.gcDelay))
if err := test.Error(err); err != nil {
if err := is.garbageCollect(dir, repo); err != nil {
return "", err
}
}
@@ -793,13 +786,7 @@ func (is *ImageStoreFS) DeleteImageManifest(repo string, reference string) error
}
if is.gc {
oci, err := umoci.OpenLayout(dir)
if err != nil {
return err
}
defer oci.Close()
if err := oci.GC(context.Background(), ifOlderThan(is, repo, is.gcDelay)); err != nil {
if err := is.garbageCollect(dir, repo); err != nil {
return err
}
}
@@ -1610,6 +1597,21 @@ func ensureDir(dir string, log zerolog.Logger) error {
return nil
}
func (is *ImageStoreFS) garbageCollect(dir string, repo string) error {
oci, err := umoci.OpenLayout(dir)
if err := test.Error(err); err != nil {
return err
}
defer oci.Close()
err = oci.GC(context.Background(), ifOlderThan(is, repo, is.gcDelay))
if err := test.Error(err); err != nil {
return err
}
return nil
}
func ifOlderThan(imgStore *ImageStoreFS, repo string, delay time.Duration) casext.GCPolicy {
return func(ctx context.Context, digest godigest.Digest) (bool, error) {
blobPath := imgStore.BlobPath(repo, digest)
@@ -1641,3 +1643,48 @@ func DirExists(d string) bool {
return true
}
func gcAllRepos(imgStore *ImageStoreFS) error {
repos, err := imgStore.GetRepositories()
if err != nil {
return err
}
for _, repo := range repos {
dir := path.Join(imgStore.RootDir(), repo)
var lockLatency time.Time
imgStore.Lock(&lockLatency)
err := imgStore.garbageCollect(dir, repo)
imgStore.Unlock(&lockLatency)
if err != nil {
return err
}
}
return nil
}
func (is *ImageStoreFS) RunGCPeriodically(gcInterval time.Duration) {
go func() {
for {
execMessage := fmt.Sprintf("executing GC of orphaned blobs for %s", is.RootDir())
is.log.Info().Msg(execMessage)
err := gcAllRepos(is)
if err != nil {
errMessage := fmt.Sprintf("error while running GC for %s", is.RootDir())
is.log.Error().Err(err).Msg(errMessage)
}
completedMessage := fmt.Sprintf("GC completed for %s, next GC scheduled after", is.RootDir())
is.log.Info().Str(completedMessage, gcInterval.String()).Msg("")
time.Sleep(gcInterval)
}
}()
}
pkg/storage/storage_fs_test.go
+64
View File
@@ -5,6 +5,7 @@ import (
"crypto/rand"
_ "crypto/sha256"
"encoding/json"
"fmt"
"io/ioutil"
"math/big"
"os"
@@ -1079,6 +1080,69 @@ func TestGarbageCollect(t *testing.T) {
})
}
func TestGarbageCollectForImageStore(t *testing.T) {
Convey("Garbage collect for all repos from an ImageStore", t, func(c C) {
dir := t.TempDir()
Convey("Garbage collect error for repo with config removed", func() {
logFile, _ := ioutil.TempFile("", "zot-log*.txt")
defer os.Remove(logFile.Name()) // clean up
log := log.NewLogger("debug", logFile.Name())
metrics := monitoring.NewMetricsServer(false, log)
imgStore := storage.NewImageStore(dir, true, 1*time.Second, true, true, log, metrics)
repoName := "gc-all-repos-short"
err := test.CopyFiles("../../test/data/zot-test", path.Join(dir, repoName))
if err != nil {
panic(err)
}
err = os.Remove(path.Join(dir, repoName, "blobs/sha256",
"2bacca16b9df395fc855c14ccf50b12b58d35d468b8e7f25758aff90f89bf396"))
if err != nil {
panic(err)
}
imgStore.RunGCPeriodically(24 * time.Hour)
time.Sleep(500 * time.Millisecond)
data, err := os.ReadFile(logFile.Name())
So(err, ShouldBeNil)
So(string(data), ShouldContainSubstring, fmt.Sprintf("error while running GC for %s", imgStore.RootDir()))
})
Convey("Garbage collect error - not enough permissions to access index.json", func() {
logFile, _ := ioutil.TempFile("", "zot-log*.txt")
defer os.Remove(logFile.Name()) // clean up
log := log.NewLogger("debug", logFile.Name())
metrics := monitoring.NewMetricsServer(false, log)
imgStore := storage.NewImageStore(dir, true, 1*time.Second, true, true, log, metrics)
repoName := "gc-all-repos-short"
err := test.CopyFiles("../../test/data/zot-test", path.Join(dir, repoName))
if err != nil {
panic(err)
}
So(os.Chmod(path.Join(dir, repoName, "index.json"), 0o000), ShouldBeNil)
imgStore.RunGCPeriodically(24 * time.Hour)
time.Sleep(500 * time.Millisecond)
data, err := os.ReadFile(logFile.Name())
So(err, ShouldBeNil)
So(string(data), ShouldContainSubstring, fmt.Sprintf("error while running GC for %s", imgStore.RootDir()))
So(os.Chmod(path.Join(dir, repoName, "index.json"), 0o755), ShouldBeNil)
})
})
}
func randSeq(n int) string {
letters := []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ")