fix(gc): sync repodb when gc'ing manifests (#1819)

fix(gc): fix cleaning deduped blobs because they have the modTime of
the original blobs, fixed by updating the modTime when hard linking
the blobs.
fix(gc): failing to parse rootDir at zot startup when using s3 storage
because there are no files under rootDir and we can not create empty dirs
on s3, fixed by creating an empty file under rootDir.

Signed-off-by: Petu Eusebiu <peusebiu@cisco.com>

pkg/meta/boltdb/boltdb.go

@@ -404,6 +404,73 @@ func (bdw BoltDB) GetReferrersInfo(repo string, referredDigest godigest.Digest,
 	return referrersInfoResult, err
 }
 
+/*
+	RemoveRepoReference removes the tag from RepoMetadata if the reference is a tag,
+
+it also removes its corresponding digest from Statistics, Signatures and Referrers if there are no tags
+pointing to it.
+If the reference is a digest then it will remove the digest from Statistics, Signatures and Referrers only
+if there are no tags pointing to the digest, otherwise it's noop.
+*/
+func (bdw *BoltDB) RemoveRepoReference(repo, reference string, manifestDigest godigest.Digest) error {
+	err := bdw.DB.Update(func(tx *bbolt.Tx) error {
+		buck := tx.Bucket([]byte(RepoMetadataBucket))
+
+		repoMetaBlob := buck.Get([]byte(repo))
+
+		repoMeta := mTypes.RepoMetadata{
+			Name:       repo,
+			Tags:       map[string]mTypes.Descriptor{},
+			Statistics: map[string]mTypes.DescriptorStatistics{},
+			Signatures: map[string]mTypes.ManifestSignatures{},
+			Referrers:  map[string][]mTypes.ReferrerInfo{},
+		}
+
+		// object not found
+		if len(repoMetaBlob) > 0 {
+			err := json.Unmarshal(repoMetaBlob, &repoMeta)
+			if err != nil {
+				return err
+			}
+		}
+
+		if !common.ReferenceIsDigest(reference) {
+			delete(repoMeta.Tags, reference)
+		} else {
+			// remove all tags pointing to this digest
+			for tag, desc := range repoMeta.Tags {
+				if desc.Digest == reference {
+					delete(repoMeta.Tags, tag)
+				}
+			}
+		}
+
+		/* try to find at least one tag pointing to manifestDigest
+		if not found then we can also remove everything related to this digest */
+		var foundTag bool
+		for _, desc := range repoMeta.Tags {
+			if desc.Digest == manifestDigest.String() {
+				foundTag = true
+			}
+		}
+
+		if !foundTag {
+			delete(repoMeta.Statistics, manifestDigest.String())
+			delete(repoMeta.Signatures, manifestDigest.String())
+			delete(repoMeta.Referrers, manifestDigest.String())
+		}
+
+		repoMetaBlob, err := json.Marshal(repoMeta)
+		if err != nil {
+			return err
+		}
+
+		return buck.Put([]byte(repo), repoMetaBlob)
+	})
+
+	return err
+}
+
 func (bdw *BoltDB) SetRepoReference(repo string, reference string, manifestDigest godigest.Digest,
 	mediaType string,
 ) error {

pkg/meta/dynamodb/dynamodb.go

@@ -447,6 +447,79 @@ func (dwr DynamoDB) GetReferrersInfo(repo string, referredDigest godigest.Digest
 	return filteredResults, nil
 }
 
+/*
+	RemoveRepoReference removes the tag from RepoMetadata if the reference is a tag,
+
+it also removes its corresponding digest from Statistics, Signatures and Referrers if there are no tags
+pointing to it.
+If the reference is a digest then it will remove the digest from Statistics, Signatures and Referrers only
+if there are no tags pointing to the digest, otherwise it's noop.
+*/
+func (dwr *DynamoDB) RemoveRepoReference(repo, reference string, manifestDigest godigest.Digest,
+) error {
+	resp, err := dwr.Client.GetItem(context.TODO(), &dynamodb.GetItemInput{
+		TableName: aws.String(dwr.RepoMetaTablename),
+		Key: map[string]types.AttributeValue{
+			"RepoName": &types.AttributeValueMemberS{Value: repo},
+		},
+	})
+	if err != nil {
+		return err
+	}
+
+	repoMeta := mTypes.RepoMetadata{
+		Name:       repo,
+		Tags:       map[string]mTypes.Descriptor{},
+		Statistics: map[string]mTypes.DescriptorStatistics{},
+		Signatures: map[string]mTypes.ManifestSignatures{},
+		Referrers:  map[string][]mTypes.ReferrerInfo{},
+	}
+
+	if resp.Item != nil {
+		err := attributevalue.Unmarshal(resp.Item["RepoMetadata"], &repoMeta)
+		if err != nil {
+			return err
+		}
+	}
+
+	if !common.ReferenceIsDigest(reference) {
+		delete(repoMeta.Tags, reference)
+	} else {
+		// find all tags pointing to this digest
+		tags := []string{}
+		for tag, desc := range repoMeta.Tags {
+			if desc.Digest == reference {
+				tags = append(tags, tag)
+			}
+		}
+
+		// remove all tags
+		for _, tag := range tags {
+			delete(repoMeta.Tags, tag)
+		}
+	}
+
+	/* try to find at least one tag pointing to manifestDigest
+	if not found then we can also remove everything related to this digest */
+	var foundTag bool
+
+	for _, desc := range repoMeta.Tags {
+		if desc.Digest == manifestDigest.String() {
+			foundTag = true
+		}
+	}
+
+	if !foundTag {
+		delete(repoMeta.Statistics, manifestDigest.String())
+		delete(repoMeta.Signatures, manifestDigest.String())
+		delete(repoMeta.Referrers, manifestDigest.String())
+	}
+
+	err = dwr.SetRepoMeta(repo, repoMeta)
+
+	return err
+}
+
 func (dwr *DynamoDB) SetRepoReference(repo string, reference string, manifestDigest godigest.Digest,
 	mediaType string,
 ) error {

pkg/meta/hooks.go

@@ -109,7 +109,7 @@ func OnDeleteManifest(repo, reference, mediaType string, digest godigest.Digest,
 			manageRepoMetaSuccessfully = false
 		}
 	} else {
-		err = metaDB.DeleteRepoTag(repo, reference)
+		err = metaDB.RemoveRepoReference(repo, reference, digest)
 		if err != nil {
 			log.Info().Msg("metadb: restoring image store")
 

pkg/meta/hooks_test.go

@@ -4,7 +4,6 @@ import (
 	"encoding/json"
 	"errors"
 	"testing"
-	"time"
 
 	notreg "github.com/notaryproject/notation-go/registry"
 	godigest "github.com/opencontainers/go-digest"
@@ -32,9 +31,7 @@ func TestOnUpdateManifest(t *testing.T) {
 		storeController := storage.StoreController{}
 		log := log.NewLogger("debug", "")
 		metrics := monitoring.NewMetricsServer(false, log)
-		storeController.DefaultStore = local.NewImageStore(rootDir, true, true, 1*time.Second,
-			1*time.Second, true, true, log, metrics, nil, nil,
-		)
+		storeController.DefaultStore = local.NewImageStore(rootDir, true, true, log, metrics, nil, nil)
 
 		params := boltdb.DBParameters{
 			RootDir: rootDir,
@@ -73,9 +70,7 @@ func TestOnUpdateManifest(t *testing.T) {
 		storeController := storage.StoreController{}
 		log := log.NewLogger("debug", "")
 		metrics := monitoring.NewMetricsServer(false, log)
-		storeController.DefaultStore = local.NewImageStore(rootDir, true, true, 1*time.Second,
-			1*time.Second, true, true, log, metrics, nil, nil,
-		)
+		storeController.DefaultStore = local.NewImageStore(rootDir, true, true, log, metrics, nil, nil)
 
 		metaDB := mocks.MetaDBMock{
 			SetManifestDataFn: func(manifestDigest godigest.Digest, mm mTypes.ManifestData) error {

pkg/meta/meta_test.go

@@ -641,7 +641,7 @@ func RunMetaDBTests(t *testing.T, metaDB mTypes.MetaDB, preparationFuncs ...func
 			})
 		})
 
-		Convey("Test DeleteRepoTag", func() {
+		Convey("Test RemoveRepoReference and DeleteRepoTag", func() {
 			var (
 				repo            = "repo1"
 				tag1            = "0.0.1"
@@ -656,6 +656,62 @@ func RunMetaDBTests(t *testing.T, metaDB mTypes.MetaDB, preparationFuncs ...func
 			err = metaDB.SetRepoReference(repo, tag2, manifestDigest2, ispec.MediaTypeImageManifest)
 			So(err, ShouldBeNil)
 
+			Convey("Delete reference from repo", func() {
+				_, err := metaDB.GetRepoMeta(repo)
+				So(err, ShouldBeNil)
+
+				err = metaDB.RemoveRepoReference(repo, tag1, manifestDigest1)
+				So(err, ShouldBeNil)
+
+				repoMeta, err := metaDB.GetRepoMeta(repo)
+				So(err, ShouldBeNil)
+
+				_, ok := repoMeta.Tags[tag1]
+				So(ok, ShouldBeFalse)
+				So(repoMeta.Tags[tag2].Digest, ShouldResemble, manifestDigest2.String())
+			})
+
+			Convey("Delete a reference from repo", func() {
+				_, err := metaDB.GetRepoMeta(repo)
+				So(err, ShouldBeNil)
+
+				// shouldn't do anything because there is tag1 pointing to it
+				err = metaDB.RemoveRepoReference(repo, manifestDigest1.String(), manifestDigest1)
+				So(err, ShouldBeNil)
+
+				repoMeta, err := metaDB.GetRepoMeta(repo)
+				So(err, ShouldBeNil)
+
+				_, ok := repoMeta.Tags[tag1]
+				So(ok, ShouldBeFalse)
+				So(repoMeta.Tags[tag2].Digest, ShouldResemble, manifestDigest2.String())
+			})
+
+			Convey("Delete inexistent reference from repo", func() {
+				inexistentDigest := godigest.FromBytes([]byte("inexistent"))
+				err := metaDB.RemoveRepoReference(repo, inexistentDigest.String(), inexistentDigest)
+				So(err, ShouldBeNil)
+
+				repoMeta, err := metaDB.GetRepoMeta(repo)
+				So(err, ShouldBeNil)
+
+				So(repoMeta.Tags[tag1].Digest, ShouldResemble, manifestDigest1.String())
+				So(repoMeta.Tags[tag2].Digest, ShouldResemble, manifestDigest2.String())
+			})
+
+			Convey("Delete reference from inexistent repo", func() {
+				inexistentDigest := godigest.FromBytes([]byte("inexistent"))
+
+				err := metaDB.RemoveRepoReference("InexistentRepo", inexistentDigest.String(), inexistentDigest)
+				So(err, ShouldBeNil)
+
+				repoMeta, err := metaDB.GetRepoMeta(repo)
+				So(err, ShouldBeNil)
+
+				So(repoMeta.Tags[tag1].Digest, ShouldResemble, manifestDigest1.String())
+				So(repoMeta.Tags[tag2].Digest, ShouldResemble, manifestDigest2.String())
+			})
+
 			Convey("Delete from repo a tag", func() {
 				_, err := metaDB.GetRepoMeta(repo)
 				So(err, ShouldBeNil)
@@ -682,7 +738,7 @@ func RunMetaDBTests(t *testing.T, metaDB mTypes.MetaDB, preparationFuncs ...func
 				So(repoMeta.Tags[tag2].Digest, ShouldResemble, manifestDigest2.String())
 			})
 
-			Convey("Delete from inexistent repo", func() {
+			Convey("Delete tag from inexistent repo", func() {
 				err := metaDB.DeleteRepoTag("InexistentRepo", "InexistentTag")
 				So(err, ShouldBeNil)
 

pkg/meta/parse_test.go

@@ -400,7 +400,7 @@ func TestParseStorageDynamoWrapper(t *testing.T) {
 
 func RunParseStorageTests(rootDir string, metaDB mTypes.MetaDB) {
 	Convey("Test with simple case", func() {
-		imageStore := local.NewImageStore(rootDir, false, false, 0, 0, false, false,
+		imageStore := local.NewImageStore(rootDir, false, false,
 			log.NewLogger("debug", ""), monitoring.NewMetricsServer(false, log.NewLogger("debug", "")), nil, nil)
 
 		storeController := storage.StoreController{DefaultStore: imageStore}
@@ -486,7 +486,7 @@ func RunParseStorageTests(rootDir string, metaDB mTypes.MetaDB) {
 	})
 
 	Convey("Accept orphan signatures", func() {
-		imageStore := local.NewImageStore(rootDir, false, false, 0, 0, false, false,
+		imageStore := local.NewImageStore(rootDir, false, false,
 			log.NewLogger("debug", ""), monitoring.NewMetricsServer(false, log.NewLogger("debug", "")), nil, nil)
 
 		storeController := storage.StoreController{DefaultStore: imageStore}
@@ -543,7 +543,7 @@ func RunParseStorageTests(rootDir string, metaDB mTypes.MetaDB) {
 	})
 
 	Convey("Check statistics after load", func() {
-		imageStore := local.NewImageStore(rootDir, false, false, 0, 0, false, false,
+		imageStore := local.NewImageStore(rootDir, false, false,
 			log.NewLogger("debug", ""), monitoring.NewMetricsServer(false, log.NewLogger("debug", "")), nil, nil)
 
 		storeController := storage.StoreController{DefaultStore: imageStore}

pkg/meta/types/types.go

@@ -45,6 +45,16 @@ type MetaDB interface { //nolint:interfacebloat
 	// SetRepoReference sets the reference of a manifest in the tag list of a repo
 	SetRepoReference(repo string, reference string, manifestDigest godigest.Digest, mediaType string) error
 
+	/*
+	   	RemoveRepoReference removes the tag from RepoMetadata if the reference is a tag,
+
+	   it also removes its corresponding digest from Statistics, Signatures and Referrers if there are no tags
+	   pointing to it.
+	   If the reference is a digest then it will remove the digest from Statistics, Signatures and Referrers only
+	   if there are no tags pointing to the digest, otherwise it's noop
+	*/
+	RemoveRepoReference(repo, reference string, manifestDigest godigest.Digest) error
+
 	// DeleteRepoTag delets the tag from the tag list of a repo
 	DeleteRepoTag(repo string, tag string) error