collector: collect releases per cluster

Our last iteration on this (4229528)
did not account for the fact that we did have releases grouped by
cluster (but not other attributes, as the original code did). So we're
partially backtracking here, and grouping releases into a grand total
per target cluster.

cmd/shipper-state-metrics/collector.go

@@ -31,6 +31,13 @@ var (
 		nil,
 	)
 
+	relsPerClusterDesc = prometheus.NewDesc(
+		fqn("releases_per_cluster"),
+		"Number of Release objects per cluster",
+		[]string{"cluster"},
+		nil,
+	)
+
 	relDurationDesc = prometheus.NewDesc(
 		fqn("release_durations"),
 		"Duration of release objects",
@@ -156,7 +163,8 @@ func (ssm ShipperStateMetrics) collectReleases(ch chan<- prometheus.Metric) {
 	now := time.Now()
 	relAgesByCondition := make(map[string][]float64)
 
-	breakdown := make(map[string]float64)
+	releasesPerCluster := make(map[string]float64)
+	releasesPerCondition := make(map[string]float64)
 	conditions := []shipper.ReleaseConditionType{
 		shipper.ReleaseConditionTypeScheduled,
 		shipper.ReleaseConditionTypeComplete,
@@ -171,6 +179,11 @@ func (ssm ShipperStateMetrics) collectReleases(ch chan<- prometheus.Metric) {
 			appName = "unknown"
 		}
 
+		clusters := strings.Split(rel.Annotations[shipper.ReleaseClustersAnnotation], ",")
+		for _, cluster := range clusters {
+			releasesPerCluster[cluster]++
+		}
+
 		for _, c := range conditions {
 			var reason, status string
 
@@ -189,7 +202,7 @@ func (ssm ShipperStateMetrics) collectReleases(ch chan<- prometheus.Metric) {
 			}
 
 			// it's either this or map[string]map[string]map[string]map[string]float64
-			breakdown[key(rel.Namespace, appName, string(c), status, reason)]++
+			releasesPerCondition[key(rel.Namespace, appName, string(c), status, reason)]++
 		}
 
 		// We're only interested in incomplete releases, as this metric
@@ -210,12 +223,14 @@ func (ssm ShipperStateMetrics) collectReleases(ch chan<- prometheus.Metric) {
 
 	}
 
-	klog.V(4).Infof("releases: %v", breakdown)
-
-	for k, v := range breakdown {
+	for k, v := range releasesPerCondition {
 		ch <- prometheus.MustNewConstMetric(relsDesc, prometheus.GaugeValue, v, unkey(k)...)
 	}
 
+	for cluster, v := range releasesPerCluster {
+		ch <- prometheus.MustNewConstMetric(relsPerClusterDesc, prometheus.GaugeValue, v, cluster)
+	}
+
 	for condition, ages := range relAgesByCondition {
 		count := uint64(len(ages))
 		sum := Sum(ages)