feat(startup): update logic for metadb update on startup, skip unmodified repos (#2024)

- MetaDB stores the time of the last update of a repo
- During startup we check if the layout has been updated after the last recorded change in the db
- If this is the case, the repo is parsed and updated in the DB otherwise it's skipped

Signed-off-by: Laurentiu Niculae <niculae.laurentiu1@gmail.com>

pkg/storage/gc/gc_test.go

@@ -51,8 +51,8 @@ var testCases = []struct {
 }
 
 func TestGarbageCollectAndRetention(t *testing.T) {
-	log := zlog.NewLogger("info", "")
-	audit := zlog.NewAuditLogger("debug", "")
+	log := zlog.NewLogger("info", "/dev/null")
+	audit := zlog.NewAuditLogger("debug", "/dev/null")
 
 	metrics := monitoring.NewMetricsServer(false, log)
 

pkg/storage/imagestore/imagestore.go

@@ -1505,6 +1505,25 @@ func (is *ImageStore) GetIndexContent(repo string) ([]byte, error) {
 	return buf, nil
 }
 
+func (is *ImageStore) StatIndex(repo string) (bool, int64, time.Time, error) {
+	repoIndexPath := path.Join(is.rootDir, repo, "index.json")
+
+	fileInfo, err := is.storeDriver.Stat(repoIndexPath)
+	if err != nil {
+		if errors.As(err, &driver.PathNotFoundError{}) {
+			is.log.Error().Err(err).Str("indexFile", repoIndexPath).Msg("index.json doesn't exist")
+
+			return false, 0, time.Time{}, zerr.ErrRepoNotFound
+		}
+
+		is.log.Error().Err(err).Str("indexFile", repoIndexPath).Msg("failed to read index.json")
+
+		return false, 0, time.Time{}, err
+	}
+
+	return true, fileInfo.Size(), fileInfo.ModTime(), nil
+}
+
 func (is *ImageStore) PutIndexContent(repo string, index ispec.Index) error {
 	dir := path.Join(is.rootDir, repo)
 

pkg/storage/local/local_test.go

@@ -2994,6 +2994,24 @@ func TestPullRange(t *testing.T) {
 	})
 }
 
+func TestStatIndex(t *testing.T) {
+	Convey("NewImageStore", t, func() {
+		dir := t.TempDir()
+		log := zlog.Logger{Logger: zerolog.New(os.Stdout)}
+		metrics := monitoring.NewMetricsServer(false, log)
+		imgStore := local.NewImageStore(dir, true, true, log, metrics, nil, nil)
+
+		err := WriteImageToFileSystem(CreateRandomImage(), "repo", "tag",
+			storage.StoreController{DefaultStore: imgStore})
+		So(err, ShouldBeNil)
+
+		Convey("StatIndex PathNotFoundError", func() {
+			_, _, _, err := imgStore.StatIndex("not-found")
+			So(err, ShouldNotBeNil)
+		})
+	})
+}
+
 func TestStorageDriverErr(t *testing.T) {
 	dir := t.TempDir()
 

pkg/storage/types/types.go

@@ -54,6 +54,7 @@ type ImageStore interface { //nolint:interfacebloat
 	CleanupRepo(repo string, blobs []godigest.Digest, removeRepo bool) (int, error)
 	GetIndexContent(repo string) ([]byte, error)
 	PutIndexContent(repo string, index ispec.Index) error
+	StatIndex(repo string) (bool, int64, time.Time, error)
 	GetBlobContent(repo string, digest godigest.Digest) ([]byte, error)
 	GetReferrers(repo string, digest godigest.Digest, artifactTypes []string) (ispec.Index, error)
 	GetOrasReferrers(repo string, digest godigest.Digest, artifactType string) ([]artifactspec.Descriptor, error)