mirror of
https://github.com/project-zot/zot.git
synced 2026-06-18 05:28:07 +08:00
6c1f1bdd40
* feat(metrics): add Prometheus GC metrics Track garbage collection activity with three new metrics: - zot_gc_runs_total (counter, label: error) — GC run count - zot_gc_duration_seconds (summary) — GC run duration - zot_gc_deleted_total (counter, label: type) — items deleted by type: blob, manifest, upload MetricServer is added to GarbageCollect and wired through all callers (controller, verify-feature retention, tests). Signed-off-by: Benoit Tigeot <benoit.tigeot@lifen.fr> * fix(test): add missing metrics var in GCS GC tests TestGCSGarbageCollectImageIndex and TestGCSGarbageCollectChainedImageIndexes were missing the metrics variable required by NewGarbageCollect after the MetricServer parameter was added. Signed-off-by: Benoit Tigeot <benoit.tigeot@lifen.fr> * fix(test): add defer metrics.Stop() in GC tests Prevent goroutine/port leaks by stopping MetricsServer in storage_test.go (3 functions) and gcs_test.go (also add missing metrics declaration in TestGCSGarbageCollectImageManifest). Signed-off-by: Benoit Tigeot <benoit.tigeot@lifen.fr> * fix(test): cover `CleanRepo` error path Add test that exercises the error branch in `CleanRepo` where `cleanRepo` fails, covering the metrics calls and log lines flagged by Codecov. Signed-off-by: Benoit Tigeot <benoit.tigeot@lifen.fr> * test: Cover GC error paths for codecov Add three tests in gc_internal_test.go to cover previously untested error branches in `removeBlobUploads` and `removeUnreferencedBlobs`: `ListBlobUploads` failure, `addIndexBlobsToReferences` failure, and `PathNotFoundError` from `GetAllBlobs`. Signed-off-by: Benoit Tigeot <benoit.tigeot@lifen.fr> * test(gc): cover remaining error paths Cover `StatBlobUpload`, `digest.Validate()`, `isBlobOlderThan`, and `CleanupRepo` error branches in `removeBlobUploads` and `removeUnreferencedBlobs`. `removeUnreferencedBlobs` now at 100% coverage, `removeBlobUploads` from 78.3% to 91.3%. Signed-off-by: Benoit Tigeot <benoit.tigeot@lifen.fr> * test: cover `sanityChecks` label name mismatch Try to avoid -0.09% coverage regression on `minimal.go` by exercising the uncovered branch in `sanityChecks` where label names have correct count but wrong values. Signed-off-by: Benoit Tigeot <benoit.tigeot@lifen.fr> * test(gc): exercise real GC path in metrics test TestGCMetrics was calling metric helpers directly instead of running actual garbage collection, so it couldn't catch wiring regressions where `CleanRepo` stops recording metrics. Now uploads an orphaned blob and runs `gc.CleanRepo` end-to-end, verifying metrics appear on the Prometheus endpoint. Suggestion from Copilot: https://github.com/project-zot/zot/pull/3863#discussion_r3129324719 Signed-off-by: Benoit Tigeot <benoit.tigeot@lifen.fr> * fix(gc): skip deletion metrics when DryRun is enabled https://github.com/project-zot/zot/pull/3863#discussion_r3129324684 Signed-off-by: Benoit Tigeot <benoit.tigeot@lifen.fr> * fix(test): stop leaked MetricsServer goroutines in GCS tests https://github.com/project-zot/zot/pull/3863#discussion_r3129324657 Signed-off-by: Benoit Tigeot <benoit.tigeot@lifen.fr> * refactor(test): drop unnecessary zlog import alias Signed-off-by: Benoit Tigeot <benoit.tigeot@lifen.fr> * fix(monitoring): expose metric types outside build tag `MetricsCopy` and related types were only visible under `\!metrics`, causing a typecheck failure when golangci-lint runs with `-tags metrics`. Moving the type definitions to `common.go` makes them unconditionally available. Signed-off-by: Benoit Tigeot <benoit.tigeot@lifen.fr> * fix(monitoring): remove extra blank line for gci Signed-off-by: Benoit Tigeot <benoit.tigeot@lifen.fr> * test(gc): cover both dry-run and real deletion metrics And fix issue with build tag with metrics Signed-off-by: Benoit Tigeot <benoit.tigeot@lifen.fr> * Satisfy testpackage linter for gc metrics test The `testpackage` linter allows `package gc` only in files named `*_internal_test.go`; rename to follow that convention. Signed-off-by: Benoit Tigeot <benoit.tigeot@lifen.fr> --------- Signed-off-by: Benoit Tigeot <benoit.tigeot@lifen.fr>
107 lines
2.7 KiB
Go
107 lines
2.7 KiB
Go
//go:build !metrics
|
|
|
|
package gc
|
|
|
|
import (
|
|
"context"
|
|
"testing"
|
|
"time"
|
|
|
|
. "github.com/smartystreets/goconvey/convey"
|
|
|
|
"zotregistry.dev/zot/v2/pkg/api/config"
|
|
"zotregistry.dev/zot/v2/pkg/extensions/monitoring"
|
|
zlog "zotregistry.dev/zot/v2/pkg/log"
|
|
"zotregistry.dev/zot/v2/pkg/storage"
|
|
"zotregistry.dev/zot/v2/pkg/storage/cache"
|
|
"zotregistry.dev/zot/v2/pkg/storage/local"
|
|
. "zotregistry.dev/zot/v2/pkg/test/image-utils"
|
|
)
|
|
|
|
func TestGCDeletedMetrics(t *testing.T) {
|
|
trueVal := true
|
|
|
|
Convey("Given a repo with a kept and a deletable tag", t, func() {
|
|
dir := t.TempDir()
|
|
|
|
log := zlog.NewTestLogger()
|
|
audit := zlog.NewAuditLogger("debug", "")
|
|
|
|
metrics := monitoring.NewMetricsServer(true, log)
|
|
defer metrics.Stop()
|
|
|
|
cacheDriver, _ := storage.Create("boltdb", cache.BoltDBDriverParameters{
|
|
RootDir: dir,
|
|
Name: "cache",
|
|
UseRelPaths: true,
|
|
}, log)
|
|
imgStore := local.NewImageStore(dir, true, true, log, metrics, nil, cacheDriver, nil, nil)
|
|
|
|
err := WriteImageToFileSystem(CreateDefaultImage(), repoName, "keep-me",
|
|
storage.StoreController{DefaultStore: imgStore})
|
|
So(err, ShouldBeNil)
|
|
err = WriteImageToFileSystem(CreateDefaultImage(), repoName, "delete-me",
|
|
storage.StoreController{DefaultStore: imgStore})
|
|
So(err, ShouldBeNil)
|
|
|
|
retentionPolicies := []config.RetentionPolicy{
|
|
{
|
|
Repositories: []string{"**"},
|
|
DeleteUntagged: &trueVal,
|
|
KeepTags: []config.KeepTagsPolicy{
|
|
{Patterns: []string{"keep-me"}},
|
|
},
|
|
},
|
|
}
|
|
|
|
Convey("DryRun should not emit deleted metrics", func() {
|
|
gc := NewGarbageCollect(imgStore, nil, Options{
|
|
Delay: 1 * time.Millisecond,
|
|
ImageRetention: config.ImageRetention{
|
|
Delay: 1 * time.Millisecond,
|
|
DryRun: true,
|
|
Policies: retentionPolicies,
|
|
},
|
|
}, audit, log, metrics)
|
|
|
|
err = gc.CleanRepo(context.Background(), repoName)
|
|
So(err, ShouldBeNil)
|
|
|
|
So(gcDeletedCount(metrics, "manifest"), ShouldEqual, 0)
|
|
})
|
|
|
|
Convey("Real GC should emit deleted metrics", func() {
|
|
gc := NewGarbageCollect(imgStore, nil, Options{
|
|
Delay: 1 * time.Millisecond,
|
|
ImageRetention: config.ImageRetention{
|
|
Delay: 1 * time.Millisecond,
|
|
Policies: retentionPolicies,
|
|
},
|
|
}, audit, log, metrics)
|
|
|
|
err = gc.CleanRepo(context.Background(), repoName)
|
|
So(err, ShouldBeNil)
|
|
|
|
So(gcDeletedCount(metrics, "manifest"), ShouldBeGreaterThan, 0)
|
|
})
|
|
})
|
|
}
|
|
|
|
func gcDeletedCount(metrics monitoring.MetricServer, artifactType string) int {
|
|
data := metrics.ReceiveMetrics()
|
|
|
|
metricsCopy, ok := data.(monitoring.MetricsCopy)
|
|
if !ok {
|
|
return -1
|
|
}
|
|
|
|
for _, counter := range metricsCopy.Counters {
|
|
if counter.Name == "zot.gc.deleted" &&
|
|
len(counter.LabelValues) > 0 && counter.LabelValues[0] == artifactType {
|
|
return counter.Count
|
|
}
|
|
}
|
|
|
|
return 0
|
|
}
|