Files
zot/pkg/storage/gc/gc_metrics_internal_test.go
Benoit Tigeot 6c1f1bdd40 feat(metrics): add Prometheus GC metrics (#3863)
* feat(metrics): add Prometheus GC metrics

Track garbage collection activity with three new metrics:
- zot_gc_runs_total (counter, label: error) — GC run count
- zot_gc_duration_seconds (summary) — GC run duration
- zot_gc_deleted_total (counter, label: type) — items deleted
  by type: blob, manifest, upload

MetricServer is added to GarbageCollect and wired through
all callers (controller, verify-feature retention, tests).

Signed-off-by: Benoit Tigeot <benoit.tigeot@lifen.fr>

* fix(test): add missing metrics var in GCS GC tests

TestGCSGarbageCollectImageIndex and
TestGCSGarbageCollectChainedImageIndexes were missing the
metrics variable required by NewGarbageCollect after the
MetricServer parameter was added.

Signed-off-by: Benoit Tigeot <benoit.tigeot@lifen.fr>

* fix(test): add defer metrics.Stop() in GC tests

Prevent goroutine/port leaks by stopping MetricsServer in
storage_test.go (3 functions) and gcs_test.go (also add
missing metrics declaration in TestGCSGarbageCollectImageManifest).

Signed-off-by: Benoit Tigeot <benoit.tigeot@lifen.fr>

* fix(test): cover `CleanRepo` error path

Add test that exercises the error branch in
`CleanRepo` where `cleanRepo` fails, covering
the metrics calls and log lines flagged by Codecov.

Signed-off-by: Benoit Tigeot <benoit.tigeot@lifen.fr>

* test: Cover GC error paths for codecov

Add three tests in gc_internal_test.go to cover previously
untested error branches in `removeBlobUploads` and
`removeUnreferencedBlobs`: `ListBlobUploads` failure,
`addIndexBlobsToReferences` failure, and `PathNotFoundError`
from `GetAllBlobs`.

Signed-off-by: Benoit Tigeot <benoit.tigeot@lifen.fr>

* test(gc): cover remaining error paths

Cover `StatBlobUpload`, `digest.Validate()`,
`isBlobOlderThan`, and `CleanupRepo` error branches
in `removeBlobUploads` and `removeUnreferencedBlobs`.

`removeUnreferencedBlobs` now at 100% coverage,
`removeBlobUploads` from 78.3% to 91.3%.

Signed-off-by: Benoit Tigeot <benoit.tigeot@lifen.fr>

* test: cover `sanityChecks` label name mismatch

Try to avoid -0.09% coverage regression on `minimal.go`
by exercising the uncovered branch in `sanityChecks`
where label names have correct count but wrong values.

Signed-off-by: Benoit Tigeot <benoit.tigeot@lifen.fr>

* test(gc): exercise real GC path in metrics test

TestGCMetrics was calling metric helpers directly instead of
running actual garbage collection, so it couldn't catch wiring
regressions where `CleanRepo` stops recording metrics.

Now uploads an orphaned blob and runs `gc.CleanRepo` end-to-end,
verifying metrics appear on the Prometheus endpoint.

Suggestion from Copilot: https://github.com/project-zot/zot/pull/3863#discussion_r3129324719

Signed-off-by: Benoit Tigeot <benoit.tigeot@lifen.fr>

* fix(gc): skip deletion metrics when DryRun is enabled

https://github.com/project-zot/zot/pull/3863#discussion_r3129324684

Signed-off-by: Benoit Tigeot <benoit.tigeot@lifen.fr>

* fix(test): stop leaked MetricsServer goroutines in GCS tests

https://github.com/project-zot/zot/pull/3863#discussion_r3129324657

Signed-off-by: Benoit Tigeot <benoit.tigeot@lifen.fr>

* refactor(test): drop unnecessary zlog import alias

Signed-off-by: Benoit Tigeot <benoit.tigeot@lifen.fr>

* fix(monitoring): expose metric types outside build tag

`MetricsCopy` and related types were only visible under `\!metrics`,
causing a typecheck failure when golangci-lint runs with `-tags metrics`.
Moving the type definitions to `common.go` makes them unconditionally available.

Signed-off-by: Benoit Tigeot <benoit.tigeot@lifen.fr>

* fix(monitoring): remove extra blank line for gci

Signed-off-by: Benoit Tigeot <benoit.tigeot@lifen.fr>

* test(gc): cover both dry-run and real deletion metrics

And fix issue with build tag with metrics

Signed-off-by: Benoit Tigeot <benoit.tigeot@lifen.fr>

* Satisfy testpackage linter for gc metrics test

The `testpackage` linter allows `package gc` only in files named
`*_internal_test.go`; rename to follow that convention.

Signed-off-by: Benoit Tigeot <benoit.tigeot@lifen.fr>

---------

Signed-off-by: Benoit Tigeot <benoit.tigeot@lifen.fr>
2026-05-16 23:03:36 -07:00

107 lines
2.7 KiB
Go

//go:build !metrics
package gc
import (
"context"
"testing"
"time"
. "github.com/smartystreets/goconvey/convey"
"zotregistry.dev/zot/v2/pkg/api/config"
"zotregistry.dev/zot/v2/pkg/extensions/monitoring"
zlog "zotregistry.dev/zot/v2/pkg/log"
"zotregistry.dev/zot/v2/pkg/storage"
"zotregistry.dev/zot/v2/pkg/storage/cache"
"zotregistry.dev/zot/v2/pkg/storage/local"
. "zotregistry.dev/zot/v2/pkg/test/image-utils"
)
func TestGCDeletedMetrics(t *testing.T) {
trueVal := true
Convey("Given a repo with a kept and a deletable tag", t, func() {
dir := t.TempDir()
log := zlog.NewTestLogger()
audit := zlog.NewAuditLogger("debug", "")
metrics := monitoring.NewMetricsServer(true, log)
defer metrics.Stop()
cacheDriver, _ := storage.Create("boltdb", cache.BoltDBDriverParameters{
RootDir: dir,
Name: "cache",
UseRelPaths: true,
}, log)
imgStore := local.NewImageStore(dir, true, true, log, metrics, nil, cacheDriver, nil, nil)
err := WriteImageToFileSystem(CreateDefaultImage(), repoName, "keep-me",
storage.StoreController{DefaultStore: imgStore})
So(err, ShouldBeNil)
err = WriteImageToFileSystem(CreateDefaultImage(), repoName, "delete-me",
storage.StoreController{DefaultStore: imgStore})
So(err, ShouldBeNil)
retentionPolicies := []config.RetentionPolicy{
{
Repositories: []string{"**"},
DeleteUntagged: &trueVal,
KeepTags: []config.KeepTagsPolicy{
{Patterns: []string{"keep-me"}},
},
},
}
Convey("DryRun should not emit deleted metrics", func() {
gc := NewGarbageCollect(imgStore, nil, Options{
Delay: 1 * time.Millisecond,
ImageRetention: config.ImageRetention{
Delay: 1 * time.Millisecond,
DryRun: true,
Policies: retentionPolicies,
},
}, audit, log, metrics)
err = gc.CleanRepo(context.Background(), repoName)
So(err, ShouldBeNil)
So(gcDeletedCount(metrics, "manifest"), ShouldEqual, 0)
})
Convey("Real GC should emit deleted metrics", func() {
gc := NewGarbageCollect(imgStore, nil, Options{
Delay: 1 * time.Millisecond,
ImageRetention: config.ImageRetention{
Delay: 1 * time.Millisecond,
Policies: retentionPolicies,
},
}, audit, log, metrics)
err = gc.CleanRepo(context.Background(), repoName)
So(err, ShouldBeNil)
So(gcDeletedCount(metrics, "manifest"), ShouldBeGreaterThan, 0)
})
})
}
func gcDeletedCount(metrics monitoring.MetricServer, artifactType string) int {
data := metrics.ReceiveMetrics()
metricsCopy, ok := data.(monitoring.MetricsCopy)
if !ok {
return -1
}
for _, counter := range metricsCopy.Counters {
if counter.Name == "zot.gc.deleted" &&
len(counter.LabelValues) > 0 && counter.LabelValues[0] == artifactType {
return counter.Count
}
}
return 0
}