Added storage latency histogram metric

Signed-off-by: Alexei Dodon <adodon@cisco.com>

pkg/extensions/monitoring/common.go

@@ -2,7 +2,6 @@ package monitoring
 
 import (
 	"fmt"
-	"math"
 	"os"
 	"path/filepath"
 )
@@ -15,10 +14,6 @@ type MetricServer interface {
 	IsEnabled() bool
 }
 
-func GetDefaultBuckets() []float64 {
-	return []float64{.05, .5, 1, 5, 30, 60, 600, math.MaxFloat64}
-}
-
 func getDirSize(path string) (int64, error) {
 	var size int64
 

pkg/extensions/monitoring/extension.go

@@ -74,6 +74,15 @@ var (
 		},
 		[]string{"commit", "binaryType", "goVersion", "version"},
 	)
+	storageLockLatency = promauto.NewHistogramVec( // nolint: gochecknoglobals
+		prometheus.HistogramOpts{
+			Namespace: metricsNamespace,
+			Name:      "storage_lock_latency_seconds",
+			Help:      "Latency of serving HTTP requests",
+			Buckets:   GetStorageLatencyBuckets(),
+		},
+		[]string{"storageName", "lockType"},
+	)
 )
 
 type metricServer struct {
@@ -81,6 +90,14 @@ type metricServer struct {
 	log     log.Logger
 }
 
+func GetDefaultBuckets() []float64 {
+	return []float64{.05, .5, 1, 5, 30, 60, 600}
+}
+
+func GetStorageLatencyBuckets() []float64 {
+	return []float64{.001, .01, 0.1, 1, 5, 10, 15, 30, 60}
+}
+
 func NewMetricsServer(enabled bool, log log.Logger) MetricServer {
 	return &metricServer{
 		enabled: enabled,
@@ -174,3 +191,9 @@ func SetServerInfo(ms MetricServer, lvalues ...string) {
 		serverInfo.WithLabelValues(lvalues...).Set(0)
 	})
 }
+
+func ObserveStorageLockLatency(ms MetricServer, latency time.Duration, storageName, lockType string) {
+	ms.SendMetric(func() {
+		storageLockLatency.WithLabelValues(storageName, lockType).Observe(latency.Seconds())
+	})
+}

pkg/extensions/monitoring/minimal.go

@@ -27,7 +27,8 @@ const (
 	// Summary.
 	httpRepoLatencySeconds = metricsNamespace + ".http.repo.latency.seconds"
 	// Histogram.
-	httpMethodLatencySeconds = metricsNamespace + ".http.method.latency.seconds"
+	httpMethodLatencySeconds  = metricsNamespace + ".http.method.latency.seconds"
+	storageLockLatencySeconds = metricsNamespace + ".storage.lock.latency.seconds"
 
 	metricsScrapeTimeout       = 2 * time.Minute
 	metricsScrapeCheckInterval = 30 * time.Second
@@ -87,6 +88,14 @@ type HistogramValue struct {
 	LabelValues []string
 }
 
+func GetDefaultBuckets() []float64 {
+	return []float64{.05, .5, 1, 5, 30, 60, 600, math.MaxFloat64}
+}
+
+func GetStorageLatencyBuckets() []float64 {
+	return []float64{.001, .01, 0.1, 1, 5, 10, 15, 30, 60, math.MaxFloat64}
+}
+
 // implements the MetricServer interface.
 func (ms *metricServer) SendMetric(metric interface{}) {
 	if ms.enabled {
@@ -172,7 +181,7 @@ func NewMetricsServer(enabled bool, log log.Logger) MetricServer {
 	// convert to a map for returning easily the string corresponding to a bucket
 	bucketsFloat2String := map[float64]string{}
 
-	for _, fvalue := range GetDefaultBuckets() {
+	for _, fvalue := range append(GetDefaultBuckets(), GetStorageLatencyBuckets()...) {
 		if fvalue == math.MaxFloat64 {
 			bucketsFloat2String[fvalue] = "+Inf"
 		} else {
@@ -219,7 +228,8 @@ func GetSummaries() map[string][]string {
 
 func GetHistograms() map[string][]string {
 	return map[string][]string{
-		httpMethodLatencySeconds: {"method"},
+		httpMethodLatencySeconds:  {"method"},
+		storageLockLatencySeconds: {"storageName", "lockType"},
 	}
 }
 
@@ -366,7 +376,7 @@ func (ms *metricServer) HistogramObserve(hv *HistogramValue) {
 		// The HistogramValue not found: add it
 		buckets := make(map[string]int)
 
-		for _, fvalue := range GetDefaultBuckets() {
+		for _, fvalue := range GetBuckets(hv.Name) {
 			if hv.Sum <= fvalue {
 				buckets[ms.bucketsF2S[fvalue]] = 1
 			} else {
@@ -381,7 +391,7 @@ func (ms *metricServer) HistogramObserve(hv *HistogramValue) {
 		cachedH := ms.cache.Histograms[index]
 		cachedH.Count++
 		cachedH.Sum += hv.Sum
-		for _, fvalue := range GetDefaultBuckets() {
+		for _, fvalue := range GetBuckets(hv.Name) {
 			if hv.Sum <= fvalue {
 				cachedH.Buckets[ms.bucketsF2S[fvalue]]++
 			}
@@ -497,6 +507,25 @@ func SetServerInfo(ms MetricServer, lvs ...string) {
 	ms.ForceSendMetric(info)
 }
 
+func ObserveStorageLockLatency(ms MetricServer, latency time.Duration, storageName, lockType string) {
+	h := HistogramValue{
+		Name:        storageLockLatencySeconds,
+		Sum:         latency.Seconds(), // convenient temporary store for Histogram latency value
+		LabelNames:  []string{"storageName", "lockType"},
+		LabelValues: []string{storageName, lockType},
+	}
+	ms.SendMetric(h)
+}
+
 func GetMaxIdleScrapeInterval() time.Duration {
 	return metricsScrapeTimeout + metricsScrapeCheckInterval
 }
+
+func GetBuckets(metricName string) []float64 {
+	switch metricName {
+	case storageLockLatencySeconds:
+		return GetStorageLatencyBuckets()
+	default:
+		return GetDefaultBuckets()
+	}
+}