diff --git a/pkg/retention/candidate.go b/pkg/retention/candidate.go index 176e5bb4..73abc172 100644 --- a/pkg/retention/candidate.go +++ b/pkg/retention/candidate.go @@ -1,6 +1,8 @@ package retention import ( + ispec "github.com/opencontainers/image-spec/specs-go/v1" + mTypes "zotregistry.dev/zot/pkg/meta/types" "zotregistry.dev/zot/pkg/retention/types" ) @@ -27,3 +29,25 @@ func GetCandidates(repoMeta mTypes.RepoMeta) []*types.Candidate { return candidates } + +func GetCandidatesFromIndex(index ispec.Index) []*types.Candidate { + candidates := make([]*types.Candidate, 0) + + // collect all manifests in the repo + for _, manifest := range index.Manifests { + tag, ok := manifest.Annotations[ispec.AnnotationRefName] + if !ok { + continue + } + + candidate := &types.Candidate{ + MediaType: manifest.MediaType, + DigestStr: string(manifest.Digest), + Tag: tag, + } + + candidates = append(candidates, candidate) + } + + return candidates +} diff --git a/pkg/retention/retention.go b/pkg/retention/retention.go index efeaa508..fe9e0e24 100644 --- a/pkg/retention/retention.go +++ b/pkg/retention/retention.go @@ -98,7 +98,51 @@ func (p policyManager) getRules(tagPolicy config.KeepTagsPolicy) []types.Rule { return rules } -func (p policyManager) GetRetainedTags(ctx context.Context, repoMeta mTypes.RepoMeta, index ispec.Index) []string { +// GetRetainedTagsFromIndex uses only index information to match tags against patterns and determine +// a list of tags to be retained. This function is to be used only in case MetaDB information is not available, +// if the DB is not instantiated. +func (p policyManager) GetRetainedTagsFromIndex(ctx context.Context, repo string, index ispec.Index) []string { + candidates := GetCandidatesFromIndex(index) + retainTags := make([]string, 0) + + // group all tags by tag policy + grouped := p.groupCandidatesByTagPolicy(repo, candidates) + + for _, candidates := range grouped { + if zcommon.IsContextDone(ctx) { + return nil + } + + for _, retainCandidate := range candidates.candidates { + // there may be duplicates + if !zcommon.Contains(retainTags, retainCandidate.Tag) { + reason := fmt.Sprintf(retainedStrFormat, retainCandidate.RetainedBy) + + logAction(repo, "keep", reason, retainCandidate, p.config.DryRun, &p.log) + + retainTags = append(retainTags, retainCandidate.Tag) + } + } + } + + // log tags which will be removed + for _, candidate := range candidates { + if !zcommon.Contains(retainTags, candidate.Tag) { + logAction(repo, "delete", filteredByTagNames, candidate, p.config.DryRun, &p.log) + + if p.auditLog != nil { + logAction(repo, "delete", filteredByTagNames, candidate, p.config.DryRun, p.auditLog) + } + } + } + + return retainTags +} + +// GetRetainedTagsFromMetaDB uses MetaDB information to apply retention rules and obtain a list of tags to be retained. +func (p policyManager) GetRetainedTagsFromMetaDB(ctx context.Context, repoMeta mTypes.RepoMeta, + index ispec.Index, +) []string { repo := repoMeta.Name matchedByName := make([]string, 0) diff --git a/pkg/retention/types/types.go b/pkg/retention/types/types.go index 172770b2..6686d38f 100644 --- a/pkg/retention/types/types.go +++ b/pkg/retention/types/types.go @@ -22,7 +22,8 @@ type PolicyManager interface { HasDeleteReferrer(repo string) bool HasDeleteUntagged(repo string) bool HasTagRetention(repo string) bool - GetRetainedTags(ctx context.Context, repoMeta mTypes.RepoMeta, index ispec.Index) []string + GetRetainedTagsFromIndex(ctx context.Context, repo string, index ispec.Index) []string + GetRetainedTagsFromMetaDB(ctx context.Context, repoMeta mTypes.RepoMeta, index ispec.Index) []string } type Rule interface { diff --git a/pkg/storage/gc/gc.go b/pkg/storage/gc/gc.go index e3d0762f..c9638640 100644 --- a/pkg/storage/gc/gc.go +++ b/pkg/storage/gc/gc.go @@ -363,16 +363,22 @@ func (gc GarbageCollect) removeTagsPerRetentionPolicy(ctx context.Context, repo return nil } - repoMeta, err := gc.metaDB.GetRepoMeta(ctx, repo) - if err != nil { - gc.log.Error().Err(err).Str("module", "gc").Str("repository", repo). - Msg("failed to get repoMeta") + var retainTags []string - return err + if gc.metaDB != nil { + repoMeta, err := gc.metaDB.GetRepoMeta(ctx, repo) + if err != nil { + gc.log.Error().Err(err).Str("module", "gc").Str("repository", repo). + Msg("failed to get repoMeta") + + return err + } + + retainTags = gc.policyMgr.GetRetainedTagsFromMetaDB(ctx, repoMeta, *index) + } else { + retainTags = gc.policyMgr.GetRetainedTagsFromIndex(ctx, repo, *index) } - retainTags := gc.policyMgr.GetRetainedTags(ctx, repoMeta, *index) - // remove for _, desc := range index.Manifests { if zcommon.IsContextDone(ctx) { diff --git a/pkg/storage/gc/gc_test.go b/pkg/storage/gc/gc_test.go index c0c2661d..e02e7d31 100644 --- a/pkg/storage/gc/gc_test.go +++ b/pkg/storage/gc/gc_test.go @@ -57,7 +57,7 @@ var testCases = []struct { }, } -func TestGarbageCollectAndRetention(t *testing.T) { +func TestGarbageCollectAndRetentionMetaDB(t *testing.T) { log := zlog.NewLogger("debug", "") audit := zlog.NewAuditLogger("debug", "/dev/null") @@ -671,6 +671,61 @@ func TestGarbageCollectAndRetention(t *testing.T) { So(err, ShouldBeNil) }) + Convey("retain all tags if keeptags is not specified", func() { + gc := gc.NewGarbageCollect(imgStore, metaDB, gc.Options{ + Delay: storageConstants.DefaultGCDelay, + ImageRetention: config.ImageRetention{ + Delay: storageConstants.DefaultRetentionDelay, + Policies: []config.RetentionPolicy{ + { + Repositories: []string{"**"}, + DeleteReferrers: true, + DeleteUntagged: &trueVal, + }, + }, + }, + }, audit, log) + + err = gc.CleanRepo(ctx, "retention") + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("gc-test1", "0.0.1") + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("gc-test1", "0.0.2") + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("gc-test2", "0.0.1") + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("gc-test3", "0.0.1") + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("retention", "0.0.1") + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("retention", "0.0.2") + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("retention", "0.0.3") + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("retention", "0.0.4") + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("retention", "0.0.5") + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("retention", "0.0.6") + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("retention", "0.0.7") + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("retention", "0.0.8") + So(err, ShouldBeNil) + }) + Convey("retain new tags", func() { sevenDays := 7 * 24 * time.Hour @@ -1455,3 +1510,821 @@ func readTagsFromStorage(rootDir, repoName string, digest godigest.Digest) ([]st return result, nil } + +func TestGarbageCollectAndRetentionNoMetaDB(t *testing.T) { + log := zlog.NewLogger("debug", "") + audit := zlog.NewAuditLogger("debug", "/dev/null") + + metrics := monitoring.NewMetricsServer(false, log) + + trueVal := true + + for _, testcase := range testCases { + testcase := testcase + t.Run(testcase.testCaseName, func(t *testing.T) { + var imgStore storageTypes.ImageStore + + var metaDB mTypes.MetaDB + metaDB = nil + + if testcase.storageType == storageConstants.S3StorageDriverName { + tskip.SkipDynamo(t) + tskip.SkipS3(t) + + uuid, err := guuid.NewV4() + if err != nil { + panic(err) + } + + rootDir := path.Join("/oci-repo-test", uuid.String()) + cacheDir := t.TempDir() + + bucket := "zot-storage-test" + + storageDriverParams := map[string]interface{}{ + "rootDir": rootDir, + "name": "s3", + "region": region, + "bucket": bucket, + "regionendpoint": os.Getenv("S3MOCK_ENDPOINT"), + "accesskey": "minioadmin", + "secretkey": "minioadmin", + "secure": false, + "skipverify": false, + "forcepathstyle": true, + } + + storeName := fmt.Sprintf("%v", storageDriverParams["name"]) + + store, err := factory.Create(context.Background(), storeName, storageDriverParams) + if err != nil { + panic(err) + } + + defer store.Delete(context.Background(), rootDir) //nolint: errcheck + + // create bucket if it doesn't exists + _, err = resty.R().Put("http://" + os.Getenv("S3MOCK_ENDPOINT") + "/" + bucket) + if err != nil { + panic(err) + } + + imgStore = s3.NewImageStore(rootDir, cacheDir, true, false, log, metrics, nil, store, nil, nil, nil) + } else { + // Create temporary directory + rootDir := t.TempDir() + + // Create ImageStore + imgStore = local.NewImageStore(rootDir, false, false, log, metrics, nil, nil, nil, nil) + } + + storeController := storage.StoreController{} + storeController.DefaultStore = imgStore + + ctx := context.Background() + + Convey("setup gc images", t, func() { + // for gc testing + // basic images + gcTest1 := CreateRandomImage() + err := WriteImageToFileSystem(gcTest1, "gc-test1", "0.0.1", storeController) + So(err, ShouldBeNil) + + // also add same image(same digest) with another tag + err = WriteImageToFileSystem(gcTest1, "gc-test1", "0.0.2", storeController) + So(err, ShouldBeNil) + + gcTest2 := CreateRandomImage() + err = WriteImageToFileSystem(gcTest2, "gc-test2", "0.0.1", storeController) + So(err, ShouldBeNil) + + gcTest3 := CreateRandomImage() + err = WriteImageToFileSystem(gcTest3, "gc-test3", "0.0.1", storeController) + So(err, ShouldBeNil) + + gcTest4 := CreateRandomMultiarch() + err = WriteMultiArchImageToFileSystem(gcTest4, "gc-test4", "0.0.1", storeController) + So(err, ShouldBeNil) + + // referrers + ref1 := CreateRandomImageWith().Subject(gcTest1.DescriptorRef()).Build() + err = WriteImageToFileSystem(ref1, "gc-test1", ref1.DigestStr(), storeController) + So(err, ShouldBeNil) + + ref2 := CreateRandomImageWith().Subject(gcTest2.DescriptorRef()).Build() + err = WriteImageToFileSystem(ref2, "gc-test2", ref2.DigestStr(), storeController) + So(err, ShouldBeNil) + + ref3 := CreateRandomImageWith().Subject(gcTest3.DescriptorRef()).Build() + err = WriteImageToFileSystem(ref3, "gc-test3", ref3.DigestStr(), storeController) + So(err, ShouldBeNil) + + ref4 := CreateMultiarchWith().RandomImages(3).Subject(gcTest4.DescriptorRef()).Build() + err = WriteMultiArchImageToFileSystem(ref4, "gc-test4", ref4.DigestStr(), storeController) + So(err, ShouldBeNil) + + // referrers pointing to referrers + refOfRef1 := CreateRandomImageWith().Subject(ref1.DescriptorRef()).Build() + err = WriteImageToFileSystem(refOfRef1, "gc-test1", refOfRef1.DigestStr(), storeController) + So(err, ShouldBeNil) + + refOfRef2 := CreateRandomImageWith().Subject(ref2.DescriptorRef()).Build() + err = WriteImageToFileSystem(refOfRef2, "gc-test2", refOfRef2.DigestStr(), storeController) + So(err, ShouldBeNil) + + refOfRef3 := CreateRandomImageWith().Subject(ref3.DescriptorRef()).Build() + err = WriteImageToFileSystem(refOfRef3, "gc-test3", refOfRef3.DigestStr(), storeController) + So(err, ShouldBeNil) + + refOfRef4 := CreateMultiarchWith().RandomImages(3).Subject(ref4.DescriptorRef()).Build() + err = WriteMultiArchImageToFileSystem(refOfRef4, "gc-test4", refOfRef4.DigestStr(), storeController) + So(err, ShouldBeNil) + + // untagged images + gcUntagged1 := CreateRandomImage() + err = WriteImageToFileSystem(gcUntagged1, "gc-test1", gcUntagged1.DigestStr(), storeController) + So(err, ShouldBeNil) + + gcUntagged2 := CreateRandomImage() + err = WriteImageToFileSystem(gcUntagged2, "gc-test2", gcUntagged2.DigestStr(), storeController) + So(err, ShouldBeNil) + + gcUntagged3 := CreateRandomImage() + err = WriteImageToFileSystem(gcUntagged3, "gc-test3", gcUntagged3.DigestStr(), storeController) + So(err, ShouldBeNil) + + gcUntagged4 := CreateRandomMultiarch() + err = WriteMultiArchImageToFileSystem(gcUntagged4, "gc-test4", gcUntagged4.DigestStr(), storeController) + So(err, ShouldBeNil) + + // for image retention testing + // old images + gcOld1 := CreateRandomImage() + err = WriteImageToFileSystem(gcOld1, "retention", "0.0.1", storeController) + So(err, ShouldBeNil) + + gcOld2 := CreateRandomImage() + err = WriteImageToFileSystem(gcOld2, "retention", "0.0.2", storeController) + So(err, ShouldBeNil) + + gcOld3 := CreateRandomImage() + err = WriteImageToFileSystem(gcOld3, "retention", "0.0.3", storeController) + So(err, ShouldBeNil) + + gcOld4 := CreateRandomMultiarch() + err = WriteMultiArchImageToFileSystem(gcOld4, "retention", "0.0.7", storeController) + So(err, ShouldBeNil) + + // new images + gcNew1 := CreateRandomImage() + err = WriteImageToFileSystem(gcNew1, "retention", "0.0.4", storeController) + So(err, ShouldBeNil) + + gcNew2 := CreateRandomImage() + err = WriteImageToFileSystem(gcNew2, "retention", "0.0.5", storeController) + So(err, ShouldBeNil) + + gcNew3 := CreateRandomImage() + err = WriteImageToFileSystem(gcNew3, "retention", "0.0.6", storeController) + So(err, ShouldBeNil) + + gcNew4 := CreateRandomMultiarch() + err = WriteMultiArchImageToFileSystem(gcNew4, "retention", "0.0.8", storeController) + So(err, ShouldBeNil) + + Convey("should not gc anything", func() { + gc := gc.NewGarbageCollect(imgStore, metaDB, gc.Options{ + Delay: storageConstants.DefaultGCDelay, + ImageRetention: config.ImageRetention{ + Delay: storageConstants.DefaultRetentionDelay, + Policies: []config.RetentionPolicy{ + { + Repositories: []string{"**"}, + DeleteReferrers: true, + DeleteUntagged: &trueVal, + KeepTags: []config.KeepTagsPolicy{ + {}, + }, + }, + }, + }, + }, audit, log) + + err := gc.CleanRepo(ctx, "gc-test1") + So(err, ShouldBeNil) + + err = gc.CleanRepo(ctx, "gc-test2") + So(err, ShouldBeNil) + + err = gc.CleanRepo(ctx, "gc-test3") + So(err, ShouldBeNil) + + err = gc.CleanRepo(ctx, "gc-test4") + So(err, ShouldBeNil) + + err = gc.CleanRepo(ctx, "retention") + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("gc-test1", gcTest1.DigestStr()) + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("gc-test1", gcUntagged1.DigestStr()) + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("gc-test1", ref1.DigestStr()) + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("gc-test1", refOfRef1.DigestStr()) + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("gc-test2", gcTest2.DigestStr()) + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("gc-test2", gcUntagged2.DigestStr()) + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("gc-test2", ref2.DigestStr()) + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("gc-test2", refOfRef2.DigestStr()) + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("gc-test3", gcTest3.DigestStr()) + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("gc-test3", gcUntagged3.DigestStr()) + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("gc-test3", ref3.DigestStr()) + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("gc-test3", refOfRef3.DigestStr()) + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("gc-test4", gcTest4.DigestStr()) + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("gc-test4", gcUntagged4.DigestStr()) + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("gc-test4", ref4.DigestStr()) + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("gc-test4", refOfRef4.DigestStr()) + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("retention", "0.0.1") + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("retention", "0.0.2") + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("retention", "0.0.3") + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("retention", "0.0.4") + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("retention", "0.0.5") + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("retention", "0.0.6") + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("retention", "0.0.7") + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("retention", "0.0.8") + So(err, ShouldBeNil) + }) + + Convey("gc untagged manifests", func() { + gc := gc.NewGarbageCollect(imgStore, metaDB, gc.Options{ + Delay: storageConstants.DefaultGCDelay, + ImageRetention: config.ImageRetention{ + Delay: 1 * time.Millisecond, + Policies: []config.RetentionPolicy{ + { + Repositories: []string{"**"}, + DeleteReferrers: true, + DeleteUntagged: &trueVal, + KeepTags: []config.KeepTagsPolicy{}, + }, + }, + }, + }, audit, log) + + err := gc.CleanRepo(ctx, "gc-test1") + So(err, ShouldBeNil) + + err = gc.CleanRepo(ctx, "gc-test2") + So(err, ShouldBeNil) + + err = gc.CleanRepo(ctx, "gc-test3") + So(err, ShouldBeNil) + + err = gc.CleanRepo(ctx, "gc-test4") + So(err, ShouldBeNil) + + err = gc.CleanRepo(ctx, "retention") + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("gc-test1", gcTest1.DigestStr()) + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("gc-test1", gcUntagged1.DigestStr()) + So(err, ShouldNotBeNil) + + _, _, _, err = imgStore.GetImageManifest("gc-test1", ref1.DigestStr()) + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("gc-test1", refOfRef1.DigestStr()) + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("gc-test2", gcTest2.DigestStr()) + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("gc-test2", gcUntagged2.DigestStr()) + So(err, ShouldNotBeNil) + + _, _, _, err = imgStore.GetImageManifest("gc-test2", ref2.DigestStr()) + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("gc-test2", refOfRef2.DigestStr()) + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("gc-test3", gcTest3.DigestStr()) + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("gc-test3", gcUntagged3.DigestStr()) + So(err, ShouldNotBeNil) + + _, _, _, err = imgStore.GetImageManifest("gc-test3", ref3.DigestStr()) + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("gc-test3", refOfRef3.DigestStr()) + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("gc-test4", gcTest4.DigestStr()) + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("gc-test4", gcUntagged4.DigestStr()) + So(err, ShouldNotBeNil) + + _, _, _, err = imgStore.GetImageManifest("gc-test4", ref4.DigestStr()) + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("gc-test4", refOfRef4.DigestStr()) + So(err, ShouldBeNil) + }) + + Convey("gc all tags, untagged, and afterwards referrers", func() { + gc := gc.NewGarbageCollect(imgStore, metaDB, gc.Options{ + Delay: 1 * time.Millisecond, + ImageRetention: config.ImageRetention{ + Delay: 1 * time.Millisecond, + Policies: []config.RetentionPolicy{ + { + Repositories: []string{"gc-test1"}, + DeleteReferrers: true, + DeleteUntagged: &trueVal, + KeepTags: []config.KeepTagsPolicy{ + { + Patterns: []string{"v1"}, // should not match any tag + }, + }, + }, + }, + }, + }, audit, log) + + err := gc.CleanRepo(ctx, "gc-test1") + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("gc-test1", gcUntagged1.DigestStr()) + So(err, ShouldNotBeNil) + + // although we have two tags both should be deleted + _, _, _, err = imgStore.GetImageManifest("gc-test1", gcTest1.DigestStr()) + So(err, ShouldNotBeNil) + + _, _, _, err = imgStore.GetImageManifest("gc-test1", ref1.DigestStr()) + So(err, ShouldNotBeNil) + + _, _, _, err = imgStore.GetImageManifest("gc-test1", refOfRef1.DigestStr()) + So(err, ShouldNotBeNil) + + // now repo should get gc'ed + repos, err := imgStore.GetRepositories() + So(err, ShouldBeNil) + So(repos, ShouldNotContain, "gc-test1") + So(repos, ShouldContain, "gc-test2") + So(repos, ShouldContain, "gc-test3") + So(repos, ShouldContain, "gc-test4") + So(repos, ShouldContain, "retention") + }) + + Convey("gc with dry-run all tags, untagged, and afterwards referrers", func() { + gc := gc.NewGarbageCollect(imgStore, metaDB, gc.Options{ + Delay: 1 * time.Millisecond, + ImageRetention: config.ImageRetention{ + Delay: 1 * time.Millisecond, + DryRun: true, + Policies: []config.RetentionPolicy{ + { + Repositories: []string{"gc-test1"}, + DeleteReferrers: true, + DeleteUntagged: &trueVal, + KeepTags: []config.KeepTagsPolicy{ + { + Patterns: []string{"v1"}, // should not match any tag + }, + }, + }, + }, + }, + }, audit, log) + + err := gc.CleanRepo(ctx, "gc-test1") + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("gc-test1", gcUntagged1.DigestStr()) + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("gc-test1", ref1.DigestStr()) + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("gc-test1", refOfRef1.DigestStr()) + So(err, ShouldBeNil) + + // now repo should not be gc'ed + repos, err := imgStore.GetRepositories() + So(err, ShouldBeNil) + So(repos, ShouldContain, "gc-test1") + So(repos, ShouldContain, "gc-test2") + So(repos, ShouldContain, "gc-test3") + So(repos, ShouldContain, "gc-test4") + So(repos, ShouldContain, "retention") + + tags, err := imgStore.GetImageTags("gc-test1") + So(err, ShouldBeNil) + So(tags, ShouldContain, "0.0.1") + So(tags, ShouldContain, "0.0.2") + }) + + Convey("all tags matches for retention", func() { + gc := gc.NewGarbageCollect(imgStore, metaDB, gc.Options{ + Delay: storageConstants.DefaultGCDelay, + ImageRetention: config.ImageRetention{ + Delay: storageConstants.DefaultRetentionDelay, + Policies: []config.RetentionPolicy{ + { + Repositories: []string{"**"}, + DeleteReferrers: true, + DeleteUntagged: &trueVal, + KeepTags: []config.KeepTagsPolicy{ + { + Patterns: []string{"0.0.*"}, + }, + }, + }, + }, + }, + }, audit, log) + + err = gc.CleanRepo(ctx, "retention") + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("gc-test1", "0.0.1") + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("gc-test1", "0.0.2") + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("gc-test2", "0.0.1") + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("gc-test3", "0.0.1") + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("retention", "0.0.1") + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("retention", "0.0.2") + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("retention", "0.0.3") + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("retention", "0.0.4") + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("retention", "0.0.5") + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("retention", "0.0.6") + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("retention", "0.0.7") + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("retention", "0.0.8") + So(err, ShouldBeNil) + }) + + Convey("retain all tags if keeptags is not specified", func() { + gc := gc.NewGarbageCollect(imgStore, metaDB, gc.Options{ + Delay: storageConstants.DefaultGCDelay, + ImageRetention: config.ImageRetention{ + Delay: storageConstants.DefaultRetentionDelay, + Policies: []config.RetentionPolicy{ + { + Repositories: []string{"**"}, + DeleteReferrers: true, + DeleteUntagged: &trueVal, + }, + }, + }, + }, audit, log) + + err = gc.CleanRepo(ctx, "retention") + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("gc-test1", "0.0.1") + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("gc-test1", "0.0.2") + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("gc-test2", "0.0.1") + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("gc-test3", "0.0.1") + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("retention", "0.0.1") + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("retention", "0.0.2") + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("retention", "0.0.3") + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("retention", "0.0.4") + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("retention", "0.0.5") + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("retention", "0.0.6") + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("retention", "0.0.7") + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("retention", "0.0.8") + So(err, ShouldBeNil) + }) + + Convey("retain a subset of all tags based on patterns only", func() { + gc := gc.NewGarbageCollect(imgStore, metaDB, gc.Options{ + Delay: storageConstants.DefaultGCDelay, + ImageRetention: config.ImageRetention{ + Delay: storageConstants.DefaultRetentionDelay, + Policies: []config.RetentionPolicy{ + { + Repositories: []string{"**"}, + DeleteReferrers: true, + DeleteUntagged: &trueVal, + KeepTags: []config.KeepTagsPolicy{ + { + Patterns: []string{"0.0.1"}, + }, + }, + }, + }, + }, + }, audit, log) + + err = gc.CleanRepo(ctx, "retention") + So(err, ShouldBeNil) + + tags, err := imgStore.GetImageTags("retention") + So(err, ShouldBeNil) + t.Log(tags) + + So(tags, ShouldContain, "0.0.1") + So(tags, ShouldNotContain, "0.0.2") + So(tags, ShouldNotContain, "0.0.3") + So(tags, ShouldNotContain, "0.0.4") + So(tags, ShouldNotContain, "0.0.5") + So(tags, ShouldNotContain, "0.0.6") + So(tags, ShouldNotContain, "0.0.7") + So(tags, ShouldNotContain, "0.0.8") + }) + + Convey("retain a subset of all tags based on patterns only using multiple rules", func() { + gc := gc.NewGarbageCollect(imgStore, metaDB, gc.Options{ + Delay: storageConstants.DefaultGCDelay, + ImageRetention: config.ImageRetention{ + Delay: storageConstants.DefaultRetentionDelay, + Policies: []config.RetentionPolicy{ + { + Repositories: []string{"**"}, + DeleteReferrers: true, + DeleteUntagged: &trueVal, + KeepTags: []config.KeepTagsPolicy{ + { + Patterns: []string{"0.0.1"}, + }, + { + Patterns: []string{"0.0.2"}, + }, + }, + }, + }, + }, + }, audit, log) + + err = gc.CleanRepo(ctx, "retention") + So(err, ShouldBeNil) + + tags, err := imgStore.GetImageTags("retention") + So(err, ShouldBeNil) + t.Log(tags) + + So(tags, ShouldContain, "0.0.1") + So(tags, ShouldContain, "0.0.2") + So(tags, ShouldNotContain, "0.0.3") + So(tags, ShouldNotContain, "0.0.4") + So(tags, ShouldNotContain, "0.0.5") + So(tags, ShouldNotContain, "0.0.6") + So(tags, ShouldNotContain, "0.0.7") + So(tags, ShouldNotContain, "0.0.8") + }) + + Convey("gc - do not match any repo", func() { + gc := gc.NewGarbageCollect(imgStore, metaDB, gc.Options{ + Delay: 1 * time.Millisecond, + ImageRetention: config.ImageRetention{ + Delay: 1 * time.Millisecond, + Policies: []config.RetentionPolicy{ + { + Repositories: []string{"no-match"}, + DeleteReferrers: true, + DeleteUntagged: &trueVal, + }, + }, + }, + }, audit, log) + + err := gc.CleanRepo(ctx, "gc-test1") + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("gc-test1", gcUntagged1.DigestStr()) + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("gc-test1", ref1.DigestStr()) + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("gc-test1", refOfRef1.DigestStr()) + So(err, ShouldBeNil) + + repos, err := imgStore.GetRepositories() + So(err, ShouldBeNil) + So(repos, ShouldContain, "gc-test1") + So(repos, ShouldContain, "gc-test2") + So(repos, ShouldContain, "gc-test3") + So(repos, ShouldContain, "retention") + }) + + Convey("gc with context done", func() { + gc := gc.NewGarbageCollect(imgStore, metaDB, gc.Options{ + Delay: storageConstants.DefaultGCDelay, + ImageRetention: config.ImageRetention{ + Delay: 1 * time.Millisecond, + Policies: []config.RetentionPolicy{ + { + Repositories: []string{"**"}, + DeleteReferrers: true, + DeleteUntagged: &trueVal, + KeepTags: []config.KeepTagsPolicy{ + { + Patterns: []string{"0.0.*"}, + }, + }, + }, + }, + }, + }, audit, log) + + ctx, cancel := context.WithCancel(ctx) + cancel() + + err := gc.CleanRepo(ctx, "gc-test1") + So(err, ShouldNotBeNil) + }) + + Convey("should gc only stale blob uploads", func() { + gcDelay := 1 * time.Second + repoName := "gc-test1" + + gc := gc.NewGarbageCollect(imgStore, metaDB, gc.Options{ + Delay: gcDelay, + ImageRetention: config.ImageRetention{ + Delay: storageConstants.DefaultRetentionDelay, + Policies: []config.RetentionPolicy{ + { + Repositories: []string{"**"}, + DeleteReferrers: true, + DeleteUntagged: &trueVal, + KeepTags: []config.KeepTagsPolicy{ + {}, + }, + }, + }, + }, + }, audit, log) + + blobUploadID, err := imgStore.NewBlobUpload(repoName) + So(err, ShouldBeNil) + + content := []byte("test-data3") + buf := bytes.NewBuffer(content) + _, err = imgStore.PutBlobChunkStreamed(repoName, blobUploadID, buf) + So(err, ShouldBeNil) + + // Blob upload should be there + uploads, err := imgStore.ListBlobUploads(repoName) + So(err, ShouldBeNil) + + if testcase.testCaseName == s3TestName { + // Remote sorage is written to only after the blob upload is finished, + // there should be no space used by blob uploads + So(uploads, ShouldEqual, []string{}) + } else { + // Local storage is used right away + So(uploads, ShouldEqual, []string{blobUploadID}) + } + + isPresent, _, _, err := imgStore.StatBlobUpload(repoName, blobUploadID) + + if testcase.testCaseName == s3TestName { + // Remote sorage is written to only after the blob upload is finished, + // there should be no space used by blob uploads + So(err, ShouldNotBeNil) + So(isPresent, ShouldBeFalse) + } else { + // Local storage is used right away + So(err, ShouldBeNil) + So(isPresent, ShouldBeTrue) + } + + err = gc.CleanRepo(ctx, repoName) + So(err, ShouldBeNil) + + // Blob upload is recent it should still be there + uploads, err = imgStore.ListBlobUploads(repoName) + So(err, ShouldBeNil) + + if testcase.testCaseName == s3TestName { + // Remote sorage is written to only after the blob upload is finished, + // there should be no space used by blob uploads + So(uploads, ShouldEqual, []string{}) + } else { + // Local storage is used right away + So(uploads, ShouldEqual, []string{blobUploadID}) + } + + isPresent, _, _, err = imgStore.StatBlobUpload(repoName, blobUploadID) + + if testcase.testCaseName == s3TestName { + // Remote sorage is written to only after the blob upload is finished, + // there should be no space used by blob uploads + So(err, ShouldNotBeNil) + So(isPresent, ShouldBeFalse) + } else { + // Local storage is used right away + So(err, ShouldBeNil) + So(isPresent, ShouldBeTrue) + } + + time.Sleep(gcDelay + 1*time.Second) + + err = gc.CleanRepo(ctx, repoName) + So(err, ShouldBeNil) + + // Blob uploads should be GCed + uploads, err = imgStore.ListBlobUploads(repoName) + So(err, ShouldBeNil) + So(uploads, ShouldBeEmpty) + + isPresent, _, _, err = imgStore.StatBlobUpload(repoName, blobUploadID) + So(err, ShouldNotBeNil) + So(isPresent, ShouldBeFalse) + }) + }) + }) + } +}