Add wpa_namespace label to metrics and split namespacedname in logs (#…

…144)
DataDog · Mar 25, 2022 · 799f4d9 · 799f4d9
1 parent ecf76ae
commit 799f4d9
Show file tree

Hide file tree

Showing 3 changed files with 32 additions and 13 deletions.
diff --git a/controllers/metrics.go b/controllers/metrics.go
@@ -20,6 +20,7 @@ const (
 	subsystem = "wpa_controller"
 	// Label keys
 	wpaNamePromLabel           = "wpa_name"
+	wpaNamespacePromLabel      = "wpa_namespace"
 	resourceNamePromLabel      = "resource_name"
 	resourceKindPromLabel      = "resource_kind"
 	resourceNamespacePromLabel = "resource_namespace"
@@ -47,6 +48,7 @@ var (
 		},
 		[]string{
 			wpaNamePromLabel,
+			wpaNamespacePromLabel,
 			metricNamePromLabel,
 			resourceNamespacePromLabel,
 			resourceNamePromLabel,
@@ -60,6 +62,7 @@ var (
 		},
 		[]string{
 			wpaNamePromLabel,
+			wpaNamespacePromLabel,
 			resourceNamespacePromLabel,
 			resourceNamePromLabel,
 			resourceKindPromLabel,
@@ -73,6 +76,7 @@ var (
 		},
 		[]string{
 			wpaNamePromLabel,
+			wpaNamespacePromLabel,
 			resourceNamespacePromLabel,
 			resourceNamePromLabel,
 			resourceKindPromLabel,
@@ -86,6 +90,7 @@ var (
 		},
 		[]string{
 			wpaNamePromLabel,
+			wpaNamespacePromLabel,
 			transitionPromLabel,
 			resourceNamespacePromLabel,
 			resourceNamePromLabel,
@@ -99,6 +104,7 @@ var (
 		},
 		[]string{
 			wpaNamePromLabel,
+			wpaNamespacePromLabel,
 			resourceNamespacePromLabel,
 			resourceNamePromLabel,
 			resourceKindPromLabel,
@@ -112,6 +118,7 @@ var (
 		},
 		[]string{
 			wpaNamePromLabel,
+			wpaNamespacePromLabel,
 			resourceNamespacePromLabel,
 			resourceNamePromLabel,
 			resourceKindPromLabel,
@@ -125,6 +132,7 @@ var (
 		},
 		[]string{
 			wpaNamePromLabel,
+			wpaNamespacePromLabel,
 			resourceNamespacePromLabel,
 			resourceNamePromLabel,
 			resourceKindPromLabel,
@@ -138,6 +146,7 @@ var (
 		},
 		[]string{
 			wpaNamePromLabel,
+			wpaNamespacePromLabel,
 			resourceNamespacePromLabel,
 			resourceNamePromLabel,
 			resourceKindPromLabel,
@@ -150,6 +159,7 @@ var (
 		},
 		[]string{
 			wpaNamePromLabel,
+			wpaNamespacePromLabel,
 			reasonPromLabel,
 			resourceNamespacePromLabel,
 			resourceNamePromLabel,
@@ -163,6 +173,7 @@ var (
 		},
 		[]string{
 			wpaNamePromLabel,
+			wpaNamespacePromLabel,
 			resourceNamespacePromLabel,
 			resourceNamePromLabel,
 			resourceKindPromLabel,
@@ -175,6 +186,7 @@ var (
 		},
 		[]string{
 			wpaNamePromLabel,
+			wpaNamespacePromLabel,
 			resourceNamespacePromLabel,
 			resourceNamePromLabel,
 			resourceKindPromLabel,
@@ -187,6 +199,7 @@ var (
 		},
 		[]string{
 			wpaNamePromLabel,
+			wpaNamespacePromLabel,
 			resourceNamespacePromLabel,
 			resourceNamePromLabel,
 			resourceKindPromLabel,
@@ -197,7 +210,7 @@ var (
 			Name:      "labels_info",
 			Help:      "Info metric for additional labels to associate to metrics as tags",
 		},
-		append(extraPromLabels, wpaNamePromLabel, resourceNamespacePromLabel),
+		append(extraPromLabels, wpaNamePromLabel, wpaNamespacePromLabel, resourceNamespacePromLabel),
 	)
 )
 
@@ -220,6 +233,7 @@ func init() {
 func cleanupAssociatedMetrics(wpa *datadoghqv1alpha1.WatermarkPodAutoscaler, onlyMetricsSpecific bool) {
 	promLabelsForWpa := prometheus.Labels{
 		wpaNamePromLabel:           wpa.Name,
+		wpaNamespacePromLabel:      wpa.Namespace,
 		resourceNamespacePromLabel: wpa.Namespace,
 		resourceNamePromLabel:      wpa.Spec.ScaleTargetRef.Name,
 		resourceKindPromLabel:      wpa.Spec.ScaleTargetRef.Kind,
@@ -242,7 +256,7 @@ func cleanupAssociatedMetrics(wpa *datadoghqv1alpha1.WatermarkPodAutoscaler, onl
 		transitionCountdown.Delete(promLabelsForWpa)
 		delete(promLabelsForWpa, transitionPromLabel)
 
-		promLabelsInfo := prometheus.Labels{wpaNamePromLabel: wpa.Name, resourceNamespacePromLabel: wpa.Namespace}
+		promLabelsInfo := prometheus.Labels{wpaNamePromLabel: wpa.Name, wpaNamespacePromLabel: wpa.Namespace, resourceNamespacePromLabel: wpa.Namespace}
 		for _, eLabel := range extraPromLabels {
 			eLabelValue := wpa.Labels[eLabel]
 			promLabelsInfo[eLabel] = eLabelValue

diff --git a/controllers/replica_calculator.go b/controllers/replica_calculator.go
@@ -102,6 +102,7 @@ func (c *ReplicaCalculator) GetExternalMetricReplicas(logger logr.Logger, target
 		// When we add official support for several metrics, move this Delete to only occur if no metric is available at all.
 		labelsWithReason := prometheus.Labels{
 			wpaNamePromLabel:           wpa.Name,
+			wpaNamespacePromLabel:      wpa.Namespace,
 			resourceNamespacePromLabel: wpa.Namespace,
 			resourceNamePromLabel:      wpa.Spec.ScaleTargetRef.Name,
 			resourceKindPromLabel:      wpa.Spec.ScaleTargetRef.Kind,
@@ -144,6 +145,7 @@ func (c *ReplicaCalculator) GetResourceReplicas(logger logr.Logger, target *auto
 		// When we add official support for several metrics, move this Delete to only occur if no metric is available at all.
 		labelsWithReason := prometheus.Labels{
 			wpaNamePromLabel:           wpa.Name,
+			wpaNamespacePromLabel:      wpa.Namespace,
 			resourceNamespacePromLabel: wpa.Namespace,
 			resourceNamePromLabel:      wpa.Spec.ScaleTargetRef.Name,
 			resourceKindPromLabel:      wpa.Spec.ScaleTargetRef.Kind,
@@ -207,8 +209,8 @@ func getReplicaCount(logger logr.Logger, currentReplicas, currentReadyReplicas i
 	adjustedHM := float64(highMark.MilliValue() + highMark.MilliValue()*wpa.Spec.Tolerance.MilliValue()/1000)
 	adjustedLM := float64(lowMark.MilliValue() - lowMark.MilliValue()*wpa.Spec.Tolerance.MilliValue()/1000)
 
-	labelsWithReason := prometheus.Labels{wpaNamePromLabel: wpa.Name, resourceNamespacePromLabel: wpa.Namespace, resourceNamePromLabel: wpa.Spec.ScaleTargetRef.Name, resourceKindPromLabel: wpa.Spec.ScaleTargetRef.Kind, reasonPromLabel: withinBoundsPromLabelVal}
-	labelsWithMetricName := prometheus.Labels{wpaNamePromLabel: wpa.Name, resourceNamespacePromLabel: wpa.Namespace, resourceNamePromLabel: wpa.Spec.ScaleTargetRef.Name, resourceKindPromLabel: wpa.Spec.ScaleTargetRef.Kind, metricNamePromLabel: name}
+	labelsWithReason := prometheus.Labels{wpaNamePromLabel: wpa.Name, wpaNamespacePromLabel: wpa.Namespace, resourceNamespacePromLabel: wpa.Namespace, resourceNamePromLabel: wpa.Spec.ScaleTargetRef.Name, resourceKindPromLabel: wpa.Spec.ScaleTargetRef.Kind, reasonPromLabel: withinBoundsPromLabelVal}
+	labelsWithMetricName := prometheus.Labels{wpaNamePromLabel: wpa.Name, wpaNamespacePromLabel: wpa.Namespace, resourceNamespacePromLabel: wpa.Namespace, resourceNamePromLabel: wpa.Spec.ScaleTargetRef.Name, resourceKindPromLabel: wpa.Spec.ScaleTargetRef.Kind, metricNamePromLabel: name}
 
 	switch {
 	case adjustedUsage > adjustedHM:

diff --git a/controllers/watermarkpodautoscaler_controller.go b/controllers/watermarkpodautoscaler_controller.go
@@ -91,7 +91,7 @@ type WatermarkPodAutoscalerReconciler struct {
 // The Controller will requeue the Request to be processed again if the returned error is non-nil or
 // Result.Requeue is true, otherwise upon completion it will remove the work from the queue.
 func (r *WatermarkPodAutoscalerReconciler) Reconcile(ctx context.Context, request ctrl.Request) (ctrl.Result, error) {
-	log := r.Log.WithValues("watermarkpodautoscaler", request.NamespacedName)
+	log := r.Log.WithValues("watermarkpodautoscaler", request.NamespacedName, "wpa_name", request.Name, "wpa_namespace", request.Namespace)
 	var err error
 	// resRepeat will be returned if we want to re-run reconcile process
 	// NB: we can't return non-nil err, as the "reconcile" msg will be added to the rate-limited queue
@@ -210,7 +210,7 @@ func (r *WatermarkPodAutoscalerReconciler) reconcileWPA(ctx context.Context, log
 	} else {
 		setCondition(wpa, datadoghqv1alpha1.WatermarkPodAutoscalerStatusDryRunCondition, corev1.ConditionFalse, "DryRun mode disabled", "Scaling changes can be applied")
 	}
-	dryRun.With(prometheus.Labels{wpaNamePromLabel: wpa.Name, resourceNamespacePromLabel: wpa.Namespace, resourceNamePromLabel: wpa.Spec.ScaleTargetRef.Name, resourceKindPromLabel: wpa.Spec.ScaleTargetRef.Kind}).Set(float64(dryRunMetricValue))
+	dryRun.With(prometheus.Labels{wpaNamePromLabel: wpa.Name, wpaNamespacePromLabel: wpa.Namespace, resourceNamespacePromLabel: wpa.Namespace, resourceNamePromLabel: wpa.Spec.ScaleTargetRef.Name, resourceKindPromLabel: wpa.Spec.ScaleTargetRef.Kind}).Set(float64(dryRunMetricValue))
 
 	reference := fmt.Sprintf("%s/%s/%s", wpa.Spec.ScaleTargetRef.Kind, wpa.Namespace, wpa.Spec.ScaleTargetRef.Name)
 	setCondition(wpa, autoscalingv2.AbleToScale, corev1.ConditionTrue, datadoghqv1alpha1.ConditionReasonSuccessfulGetScale, "the WPA controller was able to get the target's current scale")
@@ -303,10 +303,10 @@ func (r *WatermarkPodAutoscalerReconciler) reconcileWPA(ctx context.Context, log
 		desiredReplicas = currentReplicas
 	}
 
-	replicaEffective.With(prometheus.Labels{wpaNamePromLabel: wpa.Name, resourceNamespacePromLabel: wpa.Namespace, resourceNamePromLabel: wpa.Spec.ScaleTargetRef.Name, resourceKindPromLabel: wpa.Spec.ScaleTargetRef.Kind}).Set(float64(desiredReplicas))
+	replicaEffective.With(prometheus.Labels{wpaNamePromLabel: wpa.Name, wpaNamespacePromLabel: wpa.Namespace, resourceNamespacePromLabel: wpa.Namespace, resourceNamePromLabel: wpa.Spec.ScaleTargetRef.Name, resourceKindPromLabel: wpa.Spec.ScaleTargetRef.Kind}).Set(float64(desiredReplicas))
 
 	// add additional labels to info metric
-	promLabels := prometheus.Labels{wpaNamePromLabel: wpa.Name, resourceNamespacePromLabel: wpa.Namespace}
+	promLabels := prometheus.Labels{wpaNamePromLabel: wpa.Name, wpaNamespacePromLabel: wpa.Namespace, resourceNamespacePromLabel: wpa.Namespace}
 	for _, eLabel := range extraPromLabels {
 		eLabelValue := wpa.Labels[eLabel]
 		promLabels[eLabel] = eLabelValue
@@ -364,19 +364,19 @@ func shouldScale(logger logr.Logger, wpa *datadoghqv1alpha1.WatermarkPodAutoscal
 	downscaleCountdown := wpa.Status.LastScaleTime.Add(downscaleForbiddenWindow).Sub(timestamp).Seconds()
 
 	if downscaleCountdown > 0 {
-		transitionCountdown.With(prometheus.Labels{wpaNamePromLabel: wpa.Name, transitionPromLabel: "downscale", resourceNamespacePromLabel: wpa.Namespace, resourceNamePromLabel: wpa.Spec.ScaleTargetRef.Name, resourceKindPromLabel: wpa.Spec.ScaleTargetRef.Kind}).Set(downscaleCountdown)
+		transitionCountdown.With(prometheus.Labels{wpaNamePromLabel: wpa.Name, wpaNamespacePromLabel: wpa.Namespace, transitionPromLabel: "downscale", resourceNamespacePromLabel: wpa.Namespace, resourceNamePromLabel: wpa.Spec.ScaleTargetRef.Name, resourceKindPromLabel: wpa.Spec.ScaleTargetRef.Kind}).Set(downscaleCountdown)
 		setCondition(wpa, autoscalingv2.AbleToScale, corev1.ConditionFalse, datadoghqv1alpha1.ConditionReasonBackOffDownscale, "the time since the previous scale is still within the downscale forbidden window")
 		backoffDown = true
 		logger.Info("Too early to downscale", "lastScaleTime", wpa.Status.LastScaleTime, "nextDownscaleTimestamp", metav1.Time{Time: wpa.Status.LastScaleTime.Add(downscaleForbiddenWindow)}, "lastMetricsTimestamp", metav1.Time{Time: timestamp})
 	} else {
-		transitionCountdown.With(prometheus.Labels{wpaNamePromLabel: wpa.Name, transitionPromLabel: "downscale", resourceNamespacePromLabel: wpa.Namespace, resourceNamePromLabel: wpa.Spec.ScaleTargetRef.Name, resourceKindPromLabel: wpa.Spec.ScaleTargetRef.Kind}).Set(0)
+		transitionCountdown.With(prometheus.Labels{wpaNamePromLabel: wpa.Name, wpaNamespacePromLabel: wpa.Namespace, transitionPromLabel: "downscale", resourceNamespacePromLabel: wpa.Namespace, resourceNamePromLabel: wpa.Spec.ScaleTargetRef.Name, resourceKindPromLabel: wpa.Spec.ScaleTargetRef.Kind}).Set(0)
 	}
 	upscaleForbiddenWindow := time.Duration(wpa.Spec.UpscaleForbiddenWindowSeconds) * time.Second
 	upscaleCountdown := wpa.Status.LastScaleTime.Add(upscaleForbiddenWindow).Sub(timestamp).Seconds()
 
 	// Only upscale if there was no rescaling in the last upscaleForbiddenWindow
 	if upscaleCountdown > 0 {
-		transitionCountdown.With(prometheus.Labels{wpaNamePromLabel: wpa.Name, transitionPromLabel: "upscale", resourceNamespacePromLabel: wpa.Namespace, resourceNamePromLabel: wpa.Spec.ScaleTargetRef.Name, resourceKindPromLabel: wpa.Spec.ScaleTargetRef.Kind}).Set(upscaleCountdown)
+		transitionCountdown.With(prometheus.Labels{wpaNamePromLabel: wpa.Name, wpaNamespacePromLabel: wpa.Namespace, transitionPromLabel: "upscale", resourceNamespacePromLabel: wpa.Namespace, resourceNamePromLabel: wpa.Spec.ScaleTargetRef.Name, resourceKindPromLabel: wpa.Spec.ScaleTargetRef.Kind}).Set(upscaleCountdown)
 		backoffUp = true
 		logger.Info("Too early to upscale", "lastScaleTime", wpa.Status.LastScaleTime, "nextUpscaleTimestamp", metav1.Time{Time: wpa.Status.LastScaleTime.Add(upscaleForbiddenWindow)}, "lastMetricsTimestamp", metav1.Time{Time: timestamp})
 
@@ -386,7 +386,7 @@ func shouldScale(logger logr.Logger, wpa *datadoghqv1alpha1.WatermarkPodAutoscal
 			setCondition(wpa, autoscalingv2.AbleToScale, corev1.ConditionFalse, datadoghqv1alpha1.ConditionReasonBackOffUpscale, "the time since the previous scale is still within the upscale forbidden window")
 		}
 	} else {
-		transitionCountdown.With(prometheus.Labels{wpaNamePromLabel: wpa.Name, transitionPromLabel: "upscale", resourceNamespacePromLabel: wpa.Namespace, resourceNamePromLabel: wpa.Spec.ScaleTargetRef.Name, resourceKindPromLabel: wpa.Spec.ScaleTargetRef.Kind}).Set(0)
+		transitionCountdown.With(prometheus.Labels{wpaNamePromLabel: wpa.Name, wpaNamespacePromLabel: wpa.Namespace, transitionPromLabel: "upscale", resourceNamespacePromLabel: wpa.Namespace, resourceNamePromLabel: wpa.Spec.ScaleTargetRef.Name, resourceKindPromLabel: wpa.Spec.ScaleTargetRef.Kind}).Set(0)
 	}
 
 	return canScale(logger, backoffUp, backoffDown, currentReplicas, desiredReplicas)
@@ -436,7 +436,7 @@ func setStatus(wpa *datadoghqv1alpha1.WatermarkPodAutoscaler, currentReplicas, d
 func (r *WatermarkPodAutoscalerReconciler) computeReplicasForMetrics(logger logr.Logger, wpa *datadoghqv1alpha1.WatermarkPodAutoscaler, scale *autoscalingv1.Scale) (replicas int32, metric string, statuses []autoscalingv2.MetricStatus, timestamp time.Time, err error) {
 	statuses = make([]autoscalingv2.MetricStatus, len(wpa.Spec.Metrics))
 
-	labels := prometheus.Labels{wpaNamePromLabel: wpa.Name, resourceNamespacePromLabel: wpa.Namespace, resourceNamePromLabel: wpa.Spec.ScaleTargetRef.Name, resourceKindPromLabel: wpa.Spec.ScaleTargetRef.Kind}
+	labels := prometheus.Labels{wpaNamePromLabel: wpa.Name, wpaNamespacePromLabel: wpa.Namespace, resourceNamespacePromLabel: wpa.Namespace, resourceNamePromLabel: wpa.Spec.ScaleTargetRef.Name, resourceKindPromLabel: wpa.Spec.ScaleTargetRef.Kind}
 	minReplicas := float64(0)
 	if wpa.Spec.MinReplicas != nil {
 		minReplicas = float64(*wpa.Spec.MinReplicas)
@@ -460,6 +460,7 @@ func (r *WatermarkPodAutoscalerReconciler) computeReplicasForMetrics(logger logr
 
 				promLabelsForWpaWithMetricName := prometheus.Labels{
 					wpaNamePromLabel:           wpa.Name,
+					wpaNamespacePromLabel:      wpa.Namespace,
 					resourceNamespacePromLabel: wpa.Namespace,
 					resourceNamePromLabel:      wpa.Spec.ScaleTargetRef.Name,
 					resourceKindPromLabel:      wpa.Spec.ScaleTargetRef.Kind,
@@ -502,6 +503,7 @@ func (r *WatermarkPodAutoscalerReconciler) computeReplicasForMetrics(logger logr
 				metricNameProposal = fmt.Sprintf("%s{%v}", metricSpec.Resource.Name, metricSpec.Resource.MetricSelector.MatchLabels)
 				promLabelsForWpaWithMetricName := prometheus.Labels{
 					wpaNamePromLabel:           wpa.Name,
+					wpaNamespacePromLabel:      wpa.Namespace,
 					resourceNamespacePromLabel: wpa.Namespace,
 					resourceNamePromLabel:      wpa.Spec.ScaleTargetRef.Name,
 					resourceKindPromLabel:      wpa.Spec.ScaleTargetRef.Kind,
@@ -633,6 +635,7 @@ func convertDesiredReplicasWithRules(logger logr.Logger, wpa *datadoghqv1alpha1.
 	scaleDownLimit := calculateScaleDownLimit(wpa, currentReplicas)
 	promLabelsForWpa := prometheus.Labels{
 		wpaNamePromLabel:           wpa.Name,
+		wpaNamespacePromLabel:      wpa.Namespace,
 		resourceNamespacePromLabel: wpa.Namespace,
 		resourceNamePromLabel:      wpa.Spec.ScaleTargetRef.Name,
 		resourceKindPromLabel:      wpa.Spec.ScaleTargetRef.Kind,