From e7a5e092145b3ecd22bf59fd1bfdbcc1340092ba Mon Sep 17 00:00:00 2001 From: Andrei Aaron Date: Sun, 15 Jun 2025 07:38:52 +0300 Subject: [PATCH] fix: image retention policy to handle patterns even if metadb is not instantiated (#3200) It is to fix #3185. This fixes the case where MetaDB is not instantiated (none of the conditions match), and we want to retain tags only by pattern (which should not need to use MetaBD). Without this fix you could only use retention to delete untagged manifests. If you specified only the key "patterns" under "keepTags", zot would crash. It was possible to not specify "keepTags" all, which would retain all tags, but it was not possible to retains specific tags. Basically the case quoted below, from the documentation, was broken:: https://zotregistry.dev/v2.1.4/articles/retention/#configuration-example ``` When you specify a regex pattern with no rules other than the default, all tags matching the pattern are retained. ``` This would only work if MetaDb was instantiated by an unrelated configured feature. Signed-off-by: Andrei Aaron --- pkg/retention/candidate.go | 24 + pkg/retention/retention.go | 46 +- pkg/retention/types/types.go | 3 +- pkg/storage/gc/gc.go | 20 +- pkg/storage/gc/gc_test.go | 875 ++++++++++++++++++++++++++++++++++- 5 files changed, 958 insertions(+), 10 deletions(-) diff --git a/pkg/retention/candidate.go b/pkg/retention/candidate.go index 176e5bb4..73abc172 100644 --- a/pkg/retention/candidate.go +++ b/pkg/retention/candidate.go @@ -1,6 +1,8 @@ package retention import ( + ispec "github.com/opencontainers/image-spec/specs-go/v1" + mTypes "zotregistry.dev/zot/pkg/meta/types" "zotregistry.dev/zot/pkg/retention/types" ) @@ -27,3 +29,25 @@ func GetCandidates(repoMeta mTypes.RepoMeta) []*types.Candidate { return candidates } + +func GetCandidatesFromIndex(index ispec.Index) []*types.Candidate { + candidates := make([]*types.Candidate, 0) + + // collect all manifests in the repo + for _, manifest := range index.Manifests { + tag, ok := manifest.Annotations[ispec.AnnotationRefName] + if !ok { + continue + } + + candidate := &types.Candidate{ + MediaType: manifest.MediaType, + DigestStr: string(manifest.Digest), + Tag: tag, + } + + candidates = append(candidates, candidate) + } + + return candidates +} diff --git a/pkg/retention/retention.go b/pkg/retention/retention.go index efeaa508..fe9e0e24 100644 --- a/pkg/retention/retention.go +++ b/pkg/retention/retention.go @@ -98,7 +98,51 @@ func (p policyManager) getRules(tagPolicy config.KeepTagsPolicy) []types.Rule { return rules } -func (p policyManager) GetRetainedTags(ctx context.Context, repoMeta mTypes.RepoMeta, index ispec.Index) []string { +// GetRetainedTagsFromIndex uses only index information to match tags against patterns and determine +// a list of tags to be retained. This function is to be used only in case MetaDB information is not available, +// if the DB is not instantiated. +func (p policyManager) GetRetainedTagsFromIndex(ctx context.Context, repo string, index ispec.Index) []string { + candidates := GetCandidatesFromIndex(index) + retainTags := make([]string, 0) + + // group all tags by tag policy + grouped := p.groupCandidatesByTagPolicy(repo, candidates) + + for _, candidates := range grouped { + if zcommon.IsContextDone(ctx) { + return nil + } + + for _, retainCandidate := range candidates.candidates { + // there may be duplicates + if !zcommon.Contains(retainTags, retainCandidate.Tag) { + reason := fmt.Sprintf(retainedStrFormat, retainCandidate.RetainedBy) + + logAction(repo, "keep", reason, retainCandidate, p.config.DryRun, &p.log) + + retainTags = append(retainTags, retainCandidate.Tag) + } + } + } + + // log tags which will be removed + for _, candidate := range candidates { + if !zcommon.Contains(retainTags, candidate.Tag) { + logAction(repo, "delete", filteredByTagNames, candidate, p.config.DryRun, &p.log) + + if p.auditLog != nil { + logAction(repo, "delete", filteredByTagNames, candidate, p.config.DryRun, p.auditLog) + } + } + } + + return retainTags +} + +// GetRetainedTagsFromMetaDB uses MetaDB information to apply retention rules and obtain a list of tags to be retained. +func (p policyManager) GetRetainedTagsFromMetaDB(ctx context.Context, repoMeta mTypes.RepoMeta, + index ispec.Index, +) []string { repo := repoMeta.Name matchedByName := make([]string, 0) diff --git a/pkg/retention/types/types.go b/pkg/retention/types/types.go index 172770b2..6686d38f 100644 --- a/pkg/retention/types/types.go +++ b/pkg/retention/types/types.go @@ -22,7 +22,8 @@ type PolicyManager interface { HasDeleteReferrer(repo string) bool HasDeleteUntagged(repo string) bool HasTagRetention(repo string) bool - GetRetainedTags(ctx context.Context, repoMeta mTypes.RepoMeta, index ispec.Index) []string + GetRetainedTagsFromIndex(ctx context.Context, repo string, index ispec.Index) []string + GetRetainedTagsFromMetaDB(ctx context.Context, repoMeta mTypes.RepoMeta, index ispec.Index) []string } type Rule interface { diff --git a/pkg/storage/gc/gc.go b/pkg/storage/gc/gc.go index e3d0762f..c9638640 100644 --- a/pkg/storage/gc/gc.go +++ b/pkg/storage/gc/gc.go @@ -363,16 +363,22 @@ func (gc GarbageCollect) removeTagsPerRetentionPolicy(ctx context.Context, repo return nil } - repoMeta, err := gc.metaDB.GetRepoMeta(ctx, repo) - if err != nil { - gc.log.Error().Err(err).Str("module", "gc").Str("repository", repo). - Msg("failed to get repoMeta") + var retainTags []string - return err + if gc.metaDB != nil { + repoMeta, err := gc.metaDB.GetRepoMeta(ctx, repo) + if err != nil { + gc.log.Error().Err(err).Str("module", "gc").Str("repository", repo). + Msg("failed to get repoMeta") + + return err + } + + retainTags = gc.policyMgr.GetRetainedTagsFromMetaDB(ctx, repoMeta, *index) + } else { + retainTags = gc.policyMgr.GetRetainedTagsFromIndex(ctx, repo, *index) } - retainTags := gc.policyMgr.GetRetainedTags(ctx, repoMeta, *index) - // remove for _, desc := range index.Manifests { if zcommon.IsContextDone(ctx) { diff --git a/pkg/storage/gc/gc_test.go b/pkg/storage/gc/gc_test.go index c0c2661d..e02e7d31 100644 --- a/pkg/storage/gc/gc_test.go +++ b/pkg/storage/gc/gc_test.go @@ -57,7 +57,7 @@ var testCases = []struct { }, } -func TestGarbageCollectAndRetention(t *testing.T) { +func TestGarbageCollectAndRetentionMetaDB(t *testing.T) { log := zlog.NewLogger("debug", "") audit := zlog.NewAuditLogger("debug", "/dev/null") @@ -671,6 +671,61 @@ func TestGarbageCollectAndRetention(t *testing.T) { So(err, ShouldBeNil) }) + Convey("retain all tags if keeptags is not specified", func() { + gc := gc.NewGarbageCollect(imgStore, metaDB, gc.Options{ + Delay: storageConstants.DefaultGCDelay, + ImageRetention: config.ImageRetention{ + Delay: storageConstants.DefaultRetentionDelay, + Policies: []config.RetentionPolicy{ + { + Repositories: []string{"**"}, + DeleteReferrers: true, + DeleteUntagged: &trueVal, + }, + }, + }, + }, audit, log) + + err = gc.CleanRepo(ctx, "retention") + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("gc-test1", "0.0.1") + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("gc-test1", "0.0.2") + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("gc-test2", "0.0.1") + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("gc-test3", "0.0.1") + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("retention", "0.0.1") + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("retention", "0.0.2") + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("retention", "0.0.3") + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("retention", "0.0.4") + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("retention", "0.0.5") + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("retention", "0.0.6") + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("retention", "0.0.7") + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("retention", "0.0.8") + So(err, ShouldBeNil) + }) + Convey("retain new tags", func() { sevenDays := 7 * 24 * time.Hour @@ -1455,3 +1510,821 @@ func readTagsFromStorage(rootDir, repoName string, digest godigest.Digest) ([]st return result, nil } + +func TestGarbageCollectAndRetentionNoMetaDB(t *testing.T) { + log := zlog.NewLogger("debug", "") + audit := zlog.NewAuditLogger("debug", "/dev/null") + + metrics := monitoring.NewMetricsServer(false, log) + + trueVal := true + + for _, testcase := range testCases { + testcase := testcase + t.Run(testcase.testCaseName, func(t *testing.T) { + var imgStore storageTypes.ImageStore + + var metaDB mTypes.MetaDB + metaDB = nil + + if testcase.storageType == storageConstants.S3StorageDriverName { + tskip.SkipDynamo(t) + tskip.SkipS3(t) + + uuid, err := guuid.NewV4() + if err != nil { + panic(err) + } + + rootDir := path.Join("/oci-repo-test", uuid.String()) + cacheDir := t.TempDir() + + bucket := "zot-storage-test" + + storageDriverParams := map[string]interface{}{ + "rootDir": rootDir, + "name": "s3", + "region": region, + "bucket": bucket, + "regionendpoint": os.Getenv("S3MOCK_ENDPOINT"), + "accesskey": "minioadmin", + "secretkey": "minioadmin", + "secure": false, + "skipverify": false, + "forcepathstyle": true, + } + + storeName := fmt.Sprintf("%v", storageDriverParams["name"]) + + store, err := factory.Create(context.Background(), storeName, storageDriverParams) + if err != nil { + panic(err) + } + + defer store.Delete(context.Background(), rootDir) //nolint: errcheck + + // create bucket if it doesn't exists + _, err = resty.R().Put("http://" + os.Getenv("S3MOCK_ENDPOINT") + "/" + bucket) + if err != nil { + panic(err) + } + + imgStore = s3.NewImageStore(rootDir, cacheDir, true, false, log, metrics, nil, store, nil, nil, nil) + } else { + // Create temporary directory + rootDir := t.TempDir() + + // Create ImageStore + imgStore = local.NewImageStore(rootDir, false, false, log, metrics, nil, nil, nil, nil) + } + + storeController := storage.StoreController{} + storeController.DefaultStore = imgStore + + ctx := context.Background() + + Convey("setup gc images", t, func() { + // for gc testing + // basic images + gcTest1 := CreateRandomImage() + err := WriteImageToFileSystem(gcTest1, "gc-test1", "0.0.1", storeController) + So(err, ShouldBeNil) + + // also add same image(same digest) with another tag + err = WriteImageToFileSystem(gcTest1, "gc-test1", "0.0.2", storeController) + So(err, ShouldBeNil) + + gcTest2 := CreateRandomImage() + err = WriteImageToFileSystem(gcTest2, "gc-test2", "0.0.1", storeController) + So(err, ShouldBeNil) + + gcTest3 := CreateRandomImage() + err = WriteImageToFileSystem(gcTest3, "gc-test3", "0.0.1", storeController) + So(err, ShouldBeNil) + + gcTest4 := CreateRandomMultiarch() + err = WriteMultiArchImageToFileSystem(gcTest4, "gc-test4", "0.0.1", storeController) + So(err, ShouldBeNil) + + // referrers + ref1 := CreateRandomImageWith().Subject(gcTest1.DescriptorRef()).Build() + err = WriteImageToFileSystem(ref1, "gc-test1", ref1.DigestStr(), storeController) + So(err, ShouldBeNil) + + ref2 := CreateRandomImageWith().Subject(gcTest2.DescriptorRef()).Build() + err = WriteImageToFileSystem(ref2, "gc-test2", ref2.DigestStr(), storeController) + So(err, ShouldBeNil) + + ref3 := CreateRandomImageWith().Subject(gcTest3.DescriptorRef()).Build() + err = WriteImageToFileSystem(ref3, "gc-test3", ref3.DigestStr(), storeController) + So(err, ShouldBeNil) + + ref4 := CreateMultiarchWith().RandomImages(3).Subject(gcTest4.DescriptorRef()).Build() + err = WriteMultiArchImageToFileSystem(ref4, "gc-test4", ref4.DigestStr(), storeController) + So(err, ShouldBeNil) + + // referrers pointing to referrers + refOfRef1 := CreateRandomImageWith().Subject(ref1.DescriptorRef()).Build() + err = WriteImageToFileSystem(refOfRef1, "gc-test1", refOfRef1.DigestStr(), storeController) + So(err, ShouldBeNil) + + refOfRef2 := CreateRandomImageWith().Subject(ref2.DescriptorRef()).Build() + err = WriteImageToFileSystem(refOfRef2, "gc-test2", refOfRef2.DigestStr(), storeController) + So(err, ShouldBeNil) + + refOfRef3 := CreateRandomImageWith().Subject(ref3.DescriptorRef()).Build() + err = WriteImageToFileSystem(refOfRef3, "gc-test3", refOfRef3.DigestStr(), storeController) + So(err, ShouldBeNil) + + refOfRef4 := CreateMultiarchWith().RandomImages(3).Subject(ref4.DescriptorRef()).Build() + err = WriteMultiArchImageToFileSystem(refOfRef4, "gc-test4", refOfRef4.DigestStr(), storeController) + So(err, ShouldBeNil) + + // untagged images + gcUntagged1 := CreateRandomImage() + err = WriteImageToFileSystem(gcUntagged1, "gc-test1", gcUntagged1.DigestStr(), storeController) + So(err, ShouldBeNil) + + gcUntagged2 := CreateRandomImage() + err = WriteImageToFileSystem(gcUntagged2, "gc-test2", gcUntagged2.DigestStr(), storeController) + So(err, ShouldBeNil) + + gcUntagged3 := CreateRandomImage() + err = WriteImageToFileSystem(gcUntagged3, "gc-test3", gcUntagged3.DigestStr(), storeController) + So(err, ShouldBeNil) + + gcUntagged4 := CreateRandomMultiarch() + err = WriteMultiArchImageToFileSystem(gcUntagged4, "gc-test4", gcUntagged4.DigestStr(), storeController) + So(err, ShouldBeNil) + + // for image retention testing + // old images + gcOld1 := CreateRandomImage() + err = WriteImageToFileSystem(gcOld1, "retention", "0.0.1", storeController) + So(err, ShouldBeNil) + + gcOld2 := CreateRandomImage() + err = WriteImageToFileSystem(gcOld2, "retention", "0.0.2", storeController) + So(err, ShouldBeNil) + + gcOld3 := CreateRandomImage() + err = WriteImageToFileSystem(gcOld3, "retention", "0.0.3", storeController) + So(err, ShouldBeNil) + + gcOld4 := CreateRandomMultiarch() + err = WriteMultiArchImageToFileSystem(gcOld4, "retention", "0.0.7", storeController) + So(err, ShouldBeNil) + + // new images + gcNew1 := CreateRandomImage() + err = WriteImageToFileSystem(gcNew1, "retention", "0.0.4", storeController) + So(err, ShouldBeNil) + + gcNew2 := CreateRandomImage() + err = WriteImageToFileSystem(gcNew2, "retention", "0.0.5", storeController) + So(err, ShouldBeNil) + + gcNew3 := CreateRandomImage() + err = WriteImageToFileSystem(gcNew3, "retention", "0.0.6", storeController) + So(err, ShouldBeNil) + + gcNew4 := CreateRandomMultiarch() + err = WriteMultiArchImageToFileSystem(gcNew4, "retention", "0.0.8", storeController) + So(err, ShouldBeNil) + + Convey("should not gc anything", func() { + gc := gc.NewGarbageCollect(imgStore, metaDB, gc.Options{ + Delay: storageConstants.DefaultGCDelay, + ImageRetention: config.ImageRetention{ + Delay: storageConstants.DefaultRetentionDelay, + Policies: []config.RetentionPolicy{ + { + Repositories: []string{"**"}, + DeleteReferrers: true, + DeleteUntagged: &trueVal, + KeepTags: []config.KeepTagsPolicy{ + {}, + }, + }, + }, + }, + }, audit, log) + + err := gc.CleanRepo(ctx, "gc-test1") + So(err, ShouldBeNil) + + err = gc.CleanRepo(ctx, "gc-test2") + So(err, ShouldBeNil) + + err = gc.CleanRepo(ctx, "gc-test3") + So(err, ShouldBeNil) + + err = gc.CleanRepo(ctx, "gc-test4") + So(err, ShouldBeNil) + + err = gc.CleanRepo(ctx, "retention") + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("gc-test1", gcTest1.DigestStr()) + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("gc-test1", gcUntagged1.DigestStr()) + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("gc-test1", ref1.DigestStr()) + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("gc-test1", refOfRef1.DigestStr()) + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("gc-test2", gcTest2.DigestStr()) + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("gc-test2", gcUntagged2.DigestStr()) + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("gc-test2", ref2.DigestStr()) + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("gc-test2", refOfRef2.DigestStr()) + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("gc-test3", gcTest3.DigestStr()) + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("gc-test3", gcUntagged3.DigestStr()) + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("gc-test3", ref3.DigestStr()) + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("gc-test3", refOfRef3.DigestStr()) + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("gc-test4", gcTest4.DigestStr()) + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("gc-test4", gcUntagged4.DigestStr()) + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("gc-test4", ref4.DigestStr()) + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("gc-test4", refOfRef4.DigestStr()) + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("retention", "0.0.1") + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("retention", "0.0.2") + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("retention", "0.0.3") + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("retention", "0.0.4") + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("retention", "0.0.5") + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("retention", "0.0.6") + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("retention", "0.0.7") + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("retention", "0.0.8") + So(err, ShouldBeNil) + }) + + Convey("gc untagged manifests", func() { + gc := gc.NewGarbageCollect(imgStore, metaDB, gc.Options{ + Delay: storageConstants.DefaultGCDelay, + ImageRetention: config.ImageRetention{ + Delay: 1 * time.Millisecond, + Policies: []config.RetentionPolicy{ + { + Repositories: []string{"**"}, + DeleteReferrers: true, + DeleteUntagged: &trueVal, + KeepTags: []config.KeepTagsPolicy{}, + }, + }, + }, + }, audit, log) + + err := gc.CleanRepo(ctx, "gc-test1") + So(err, ShouldBeNil) + + err = gc.CleanRepo(ctx, "gc-test2") + So(err, ShouldBeNil) + + err = gc.CleanRepo(ctx, "gc-test3") + So(err, ShouldBeNil) + + err = gc.CleanRepo(ctx, "gc-test4") + So(err, ShouldBeNil) + + err = gc.CleanRepo(ctx, "retention") + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("gc-test1", gcTest1.DigestStr()) + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("gc-test1", gcUntagged1.DigestStr()) + So(err, ShouldNotBeNil) + + _, _, _, err = imgStore.GetImageManifest("gc-test1", ref1.DigestStr()) + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("gc-test1", refOfRef1.DigestStr()) + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("gc-test2", gcTest2.DigestStr()) + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("gc-test2", gcUntagged2.DigestStr()) + So(err, ShouldNotBeNil) + + _, _, _, err = imgStore.GetImageManifest("gc-test2", ref2.DigestStr()) + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("gc-test2", refOfRef2.DigestStr()) + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("gc-test3", gcTest3.DigestStr()) + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("gc-test3", gcUntagged3.DigestStr()) + So(err, ShouldNotBeNil) + + _, _, _, err = imgStore.GetImageManifest("gc-test3", ref3.DigestStr()) + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("gc-test3", refOfRef3.DigestStr()) + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("gc-test4", gcTest4.DigestStr()) + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("gc-test4", gcUntagged4.DigestStr()) + So(err, ShouldNotBeNil) + + _, _, _, err = imgStore.GetImageManifest("gc-test4", ref4.DigestStr()) + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("gc-test4", refOfRef4.DigestStr()) + So(err, ShouldBeNil) + }) + + Convey("gc all tags, untagged, and afterwards referrers", func() { + gc := gc.NewGarbageCollect(imgStore, metaDB, gc.Options{ + Delay: 1 * time.Millisecond, + ImageRetention: config.ImageRetention{ + Delay: 1 * time.Millisecond, + Policies: []config.RetentionPolicy{ + { + Repositories: []string{"gc-test1"}, + DeleteReferrers: true, + DeleteUntagged: &trueVal, + KeepTags: []config.KeepTagsPolicy{ + { + Patterns: []string{"v1"}, // should not match any tag + }, + }, + }, + }, + }, + }, audit, log) + + err := gc.CleanRepo(ctx, "gc-test1") + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("gc-test1", gcUntagged1.DigestStr()) + So(err, ShouldNotBeNil) + + // although we have two tags both should be deleted + _, _, _, err = imgStore.GetImageManifest("gc-test1", gcTest1.DigestStr()) + So(err, ShouldNotBeNil) + + _, _, _, err = imgStore.GetImageManifest("gc-test1", ref1.DigestStr()) + So(err, ShouldNotBeNil) + + _, _, _, err = imgStore.GetImageManifest("gc-test1", refOfRef1.DigestStr()) + So(err, ShouldNotBeNil) + + // now repo should get gc'ed + repos, err := imgStore.GetRepositories() + So(err, ShouldBeNil) + So(repos, ShouldNotContain, "gc-test1") + So(repos, ShouldContain, "gc-test2") + So(repos, ShouldContain, "gc-test3") + So(repos, ShouldContain, "gc-test4") + So(repos, ShouldContain, "retention") + }) + + Convey("gc with dry-run all tags, untagged, and afterwards referrers", func() { + gc := gc.NewGarbageCollect(imgStore, metaDB, gc.Options{ + Delay: 1 * time.Millisecond, + ImageRetention: config.ImageRetention{ + Delay: 1 * time.Millisecond, + DryRun: true, + Policies: []config.RetentionPolicy{ + { + Repositories: []string{"gc-test1"}, + DeleteReferrers: true, + DeleteUntagged: &trueVal, + KeepTags: []config.KeepTagsPolicy{ + { + Patterns: []string{"v1"}, // should not match any tag + }, + }, + }, + }, + }, + }, audit, log) + + err := gc.CleanRepo(ctx, "gc-test1") + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("gc-test1", gcUntagged1.DigestStr()) + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("gc-test1", ref1.DigestStr()) + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("gc-test1", refOfRef1.DigestStr()) + So(err, ShouldBeNil) + + // now repo should not be gc'ed + repos, err := imgStore.GetRepositories() + So(err, ShouldBeNil) + So(repos, ShouldContain, "gc-test1") + So(repos, ShouldContain, "gc-test2") + So(repos, ShouldContain, "gc-test3") + So(repos, ShouldContain, "gc-test4") + So(repos, ShouldContain, "retention") + + tags, err := imgStore.GetImageTags("gc-test1") + So(err, ShouldBeNil) + So(tags, ShouldContain, "0.0.1") + So(tags, ShouldContain, "0.0.2") + }) + + Convey("all tags matches for retention", func() { + gc := gc.NewGarbageCollect(imgStore, metaDB, gc.Options{ + Delay: storageConstants.DefaultGCDelay, + ImageRetention: config.ImageRetention{ + Delay: storageConstants.DefaultRetentionDelay, + Policies: []config.RetentionPolicy{ + { + Repositories: []string{"**"}, + DeleteReferrers: true, + DeleteUntagged: &trueVal, + KeepTags: []config.KeepTagsPolicy{ + { + Patterns: []string{"0.0.*"}, + }, + }, + }, + }, + }, + }, audit, log) + + err = gc.CleanRepo(ctx, "retention") + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("gc-test1", "0.0.1") + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("gc-test1", "0.0.2") + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("gc-test2", "0.0.1") + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("gc-test3", "0.0.1") + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("retention", "0.0.1") + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("retention", "0.0.2") + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("retention", "0.0.3") + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("retention", "0.0.4") + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("retention", "0.0.5") + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("retention", "0.0.6") + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("retention", "0.0.7") + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("retention", "0.0.8") + So(err, ShouldBeNil) + }) + + Convey("retain all tags if keeptags is not specified", func() { + gc := gc.NewGarbageCollect(imgStore, metaDB, gc.Options{ + Delay: storageConstants.DefaultGCDelay, + ImageRetention: config.ImageRetention{ + Delay: storageConstants.DefaultRetentionDelay, + Policies: []config.RetentionPolicy{ + { + Repositories: []string{"**"}, + DeleteReferrers: true, + DeleteUntagged: &trueVal, + }, + }, + }, + }, audit, log) + + err = gc.CleanRepo(ctx, "retention") + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("gc-test1", "0.0.1") + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("gc-test1", "0.0.2") + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("gc-test2", "0.0.1") + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("gc-test3", "0.0.1") + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("retention", "0.0.1") + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("retention", "0.0.2") + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("retention", "0.0.3") + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("retention", "0.0.4") + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("retention", "0.0.5") + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("retention", "0.0.6") + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("retention", "0.0.7") + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("retention", "0.0.8") + So(err, ShouldBeNil) + }) + + Convey("retain a subset of all tags based on patterns only", func() { + gc := gc.NewGarbageCollect(imgStore, metaDB, gc.Options{ + Delay: storageConstants.DefaultGCDelay, + ImageRetention: config.ImageRetention{ + Delay: storageConstants.DefaultRetentionDelay, + Policies: []config.RetentionPolicy{ + { + Repositories: []string{"**"}, + DeleteReferrers: true, + DeleteUntagged: &trueVal, + KeepTags: []config.KeepTagsPolicy{ + { + Patterns: []string{"0.0.1"}, + }, + }, + }, + }, + }, + }, audit, log) + + err = gc.CleanRepo(ctx, "retention") + So(err, ShouldBeNil) + + tags, err := imgStore.GetImageTags("retention") + So(err, ShouldBeNil) + t.Log(tags) + + So(tags, ShouldContain, "0.0.1") + So(tags, ShouldNotContain, "0.0.2") + So(tags, ShouldNotContain, "0.0.3") + So(tags, ShouldNotContain, "0.0.4") + So(tags, ShouldNotContain, "0.0.5") + So(tags, ShouldNotContain, "0.0.6") + So(tags, ShouldNotContain, "0.0.7") + So(tags, ShouldNotContain, "0.0.8") + }) + + Convey("retain a subset of all tags based on patterns only using multiple rules", func() { + gc := gc.NewGarbageCollect(imgStore, metaDB, gc.Options{ + Delay: storageConstants.DefaultGCDelay, + ImageRetention: config.ImageRetention{ + Delay: storageConstants.DefaultRetentionDelay, + Policies: []config.RetentionPolicy{ + { + Repositories: []string{"**"}, + DeleteReferrers: true, + DeleteUntagged: &trueVal, + KeepTags: []config.KeepTagsPolicy{ + { + Patterns: []string{"0.0.1"}, + }, + { + Patterns: []string{"0.0.2"}, + }, + }, + }, + }, + }, + }, audit, log) + + err = gc.CleanRepo(ctx, "retention") + So(err, ShouldBeNil) + + tags, err := imgStore.GetImageTags("retention") + So(err, ShouldBeNil) + t.Log(tags) + + So(tags, ShouldContain, "0.0.1") + So(tags, ShouldContain, "0.0.2") + So(tags, ShouldNotContain, "0.0.3") + So(tags, ShouldNotContain, "0.0.4") + So(tags, ShouldNotContain, "0.0.5") + So(tags, ShouldNotContain, "0.0.6") + So(tags, ShouldNotContain, "0.0.7") + So(tags, ShouldNotContain, "0.0.8") + }) + + Convey("gc - do not match any repo", func() { + gc := gc.NewGarbageCollect(imgStore, metaDB, gc.Options{ + Delay: 1 * time.Millisecond, + ImageRetention: config.ImageRetention{ + Delay: 1 * time.Millisecond, + Policies: []config.RetentionPolicy{ + { + Repositories: []string{"no-match"}, + DeleteReferrers: true, + DeleteUntagged: &trueVal, + }, + }, + }, + }, audit, log) + + err := gc.CleanRepo(ctx, "gc-test1") + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("gc-test1", gcUntagged1.DigestStr()) + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("gc-test1", ref1.DigestStr()) + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("gc-test1", refOfRef1.DigestStr()) + So(err, ShouldBeNil) + + repos, err := imgStore.GetRepositories() + So(err, ShouldBeNil) + So(repos, ShouldContain, "gc-test1") + So(repos, ShouldContain, "gc-test2") + So(repos, ShouldContain, "gc-test3") + So(repos, ShouldContain, "retention") + }) + + Convey("gc with context done", func() { + gc := gc.NewGarbageCollect(imgStore, metaDB, gc.Options{ + Delay: storageConstants.DefaultGCDelay, + ImageRetention: config.ImageRetention{ + Delay: 1 * time.Millisecond, + Policies: []config.RetentionPolicy{ + { + Repositories: []string{"**"}, + DeleteReferrers: true, + DeleteUntagged: &trueVal, + KeepTags: []config.KeepTagsPolicy{ + { + Patterns: []string{"0.0.*"}, + }, + }, + }, + }, + }, + }, audit, log) + + ctx, cancel := context.WithCancel(ctx) + cancel() + + err := gc.CleanRepo(ctx, "gc-test1") + So(err, ShouldNotBeNil) + }) + + Convey("should gc only stale blob uploads", func() { + gcDelay := 1 * time.Second + repoName := "gc-test1" + + gc := gc.NewGarbageCollect(imgStore, metaDB, gc.Options{ + Delay: gcDelay, + ImageRetention: config.ImageRetention{ + Delay: storageConstants.DefaultRetentionDelay, + Policies: []config.RetentionPolicy{ + { + Repositories: []string{"**"}, + DeleteReferrers: true, + DeleteUntagged: &trueVal, + KeepTags: []config.KeepTagsPolicy{ + {}, + }, + }, + }, + }, + }, audit, log) + + blobUploadID, err := imgStore.NewBlobUpload(repoName) + So(err, ShouldBeNil) + + content := []byte("test-data3") + buf := bytes.NewBuffer(content) + _, err = imgStore.PutBlobChunkStreamed(repoName, blobUploadID, buf) + So(err, ShouldBeNil) + + // Blob upload should be there + uploads, err := imgStore.ListBlobUploads(repoName) + So(err, ShouldBeNil) + + if testcase.testCaseName == s3TestName { + // Remote sorage is written to only after the blob upload is finished, + // there should be no space used by blob uploads + So(uploads, ShouldEqual, []string{}) + } else { + // Local storage is used right away + So(uploads, ShouldEqual, []string{blobUploadID}) + } + + isPresent, _, _, err := imgStore.StatBlobUpload(repoName, blobUploadID) + + if testcase.testCaseName == s3TestName { + // Remote sorage is written to only after the blob upload is finished, + // there should be no space used by blob uploads + So(err, ShouldNotBeNil) + So(isPresent, ShouldBeFalse) + } else { + // Local storage is used right away + So(err, ShouldBeNil) + So(isPresent, ShouldBeTrue) + } + + err = gc.CleanRepo(ctx, repoName) + So(err, ShouldBeNil) + + // Blob upload is recent it should still be there + uploads, err = imgStore.ListBlobUploads(repoName) + So(err, ShouldBeNil) + + if testcase.testCaseName == s3TestName { + // Remote sorage is written to only after the blob upload is finished, + // there should be no space used by blob uploads + So(uploads, ShouldEqual, []string{}) + } else { + // Local storage is used right away + So(uploads, ShouldEqual, []string{blobUploadID}) + } + + isPresent, _, _, err = imgStore.StatBlobUpload(repoName, blobUploadID) + + if testcase.testCaseName == s3TestName { + // Remote sorage is written to only after the blob upload is finished, + // there should be no space used by blob uploads + So(err, ShouldNotBeNil) + So(isPresent, ShouldBeFalse) + } else { + // Local storage is used right away + So(err, ShouldBeNil) + So(isPresent, ShouldBeTrue) + } + + time.Sleep(gcDelay + 1*time.Second) + + err = gc.CleanRepo(ctx, repoName) + So(err, ShouldBeNil) + + // Blob uploads should be GCed + uploads, err = imgStore.ListBlobUploads(repoName) + So(err, ShouldBeNil) + So(uploads, ShouldBeEmpty) + + isPresent, _, _, err = imgStore.StatBlobUpload(repoName, blobUploadID) + So(err, ShouldNotBeNil) + So(isPresent, ShouldBeFalse) + }) + }) + }) + } +}