Merge branch 'master' into docs-update-new-check-policy

CHANGELOG.md

@@ -1,13 +1,15 @@
 ## 0.4.0 (Unreleased)
 
 IMPROVEMENTS:
+ * Scaling policies can now detail any number of external metrics and thresholds to check. The internal autoscaler will perform lookups on each iteration of the external source, to check whether scaling is needed. Currently `prometheus` is supported as an external source [[GH-104]](https://github.com/jrasell/sherpa/pull/104)
  * Metrics to measure policy backend request latencies [[GH-93]](https://github.com/jrasell/sherpa/pull/93)
  * Metrics to measure scaling state backend request latencies[[GH-94]](https://github.com/jrasell/sherpa/pull/94)
  * Metrics to measure autoscaler performance and health [[GH-95]](https://github.com/jrasell/sherpa/pull/95)
  * Add prometheus format option to metrics endpoint [[GH-101]](https://github.com/jrasell/sherpa/pull/101)
 
 BUG FIXES:
  * Scale out/in endpoints should use POST rather than PUT method [[GH-103]](https://github.com/jrasell/sherpa/pull/103)
+ * Reuse the Consul API client rather than create new for each use [[GH-106]](https://github.com/jrasell/sherpa/pull/106)
 
 REFACTOR:
  * Move the system API endpoint into the `server/endpoints/v1` package [[GH-99]](https://github.com/jrasell/sherpa/pull/99)

cmd/server/command.go

@@ -26,6 +26,7 @@ func RegisterCommand(rootCmd *cobra.Command) error {
 	serverCfg.RegisterTLSConfig(cmd)
 	serverCfg.RegisterTelemetryConfig(cmd)
 	serverCfg.RegisterClusterConfig(cmd)
+	serverCfg.RegisterMetricProviderConfig(cmd)
 	logCfg.RegisterConfig(cmd)
 	rootCmd.AddCommand(cmd)
 
@@ -37,6 +38,7 @@ func runServer(_ *cobra.Command, _ []string) {
 	tlsConfig := serverCfg.GetTLSConfig()
 	telemetryConfig := serverCfg.GetTelemetryConfig()
 	clusterConfig := serverCfg.GetClusterConfig()
+	metricProviderConfig := serverCfg.GetMetricProviderConfig()
 
 	if err := verifyServerConfig(serverConfig); err != nil {
 		fmt.Println(err)
@@ -51,10 +53,11 @@ func runServer(_ *cobra.Command, _ []string) {
 	}
 
 	cfg := &server.Config{
-		Cluster:   &clusterConfig,
-		Server:    &serverConfig,
-		TLS:       &tlsConfig,
-		Telemetry: &telemetryConfig,
+		Cluster:        &clusterConfig,
+		MetricProvider: metricProviderConfig,
+		Server:         &serverConfig,
+		TLS:            &tlsConfig,
+		Telemetry:      &telemetryConfig,
 	}
 	srv := server.New(log.Logger, cfg)
 

docs/configuration/README.md

@@ -14,6 +14,7 @@ The Sherpa server can be configured by supplying either CLI flags or using envir
 * `--log-format` (string: "auto") - Specify the log format ("auto", "zerolog" or "human").
 * `--log-level` (string: "info") - Change the level used for logging.
 * `--log-use-color` (bool: true) - Use ANSI colors in logging output.
+* `--metric-provider-prometheus-addr` (string: "") The address of the Prometheus endpoint in the form <protocol>://<addr>:<port>.
 * `--policy-engine-api-enabled` (bool: true) - Enable the Sherpa API to manage scaling policies.
 * `--policy-engine-nomad-meta-enabled` (bool: false) - Enable Nomad job meta lookups to manage scaling policies.
 * `--policy-engine-strict-checking-enabled` (bool: true) - When enabled, all scaling activities must pass through policy checks.

docs/guides/telemetry.md

@@ -278,16 +278,10 @@ Autoscale metrics allow operators to get insight into how the autoscaler is func
     <td>Summary</td>
   </tr>
   <tr>
-    <td>`sherpa.autoscale.evaluation.error`</td>
-    <td>Number of autoscaling evaluation errors across all jobs</td>
-    <td>Number of errors</td>
-    <td>Counter</td>
-  </tr>
-  <tr>
-    <td>`sherpa.autoscale.{job}.evaluation.error`</td>
-    <td>Number of autoscaling evaluation errors for the job named {job}</td>
-    <td>Number of errors</td>
-    <td>Counter</td>
+    <td>`sherpa.autoscale.{job}.{group}.evaluation`</td>
+    <td>The time taken to perform the autoscaling evaluation for the job named {job} and group named {group}</td>
+    <td>Milliseconds</td>
+    <td>Summary</td>
   </tr>
   <tr>
     <td>`sherpa.autoscale.trigger.error`</td>
@@ -313,4 +307,22 @@ Autoscale metrics allow operators to get insight into how the autoscaler is func
     <td>Number of successes</td>
     <td>Counter</td>
   </tr>
+  <tr>
+    <td>`sherpa.autoscale.prometheus.get_value`</td>
+    <td>The time taken to query Prometheus for a metric value</td>
+    <td>Milliseconds</td>
+    <td>Summary</td>
+  </tr>
+  <tr>
+    <td>`sherpa.autoscale.prometheus.error`</td>
+    <td>Number of errors querying Prometheus for a metric value</td>
+    <td>Number of errors</td>
+    <td>Counter</td>
+  </tr>
+    <tr>
+      <td>`sherpa.autoscale.prometheus.success`</td>
+      <td>Number of successful queries of Prometheus for a metric value</td>
+      <td>Number of successes</td>
+      <td>Counter</td>
+    </tr>
 </table>

pkg/api/policy.go

@@ -20,10 +20,21 @@ type JobGroupPolicy struct {
 	MinCount                          int
 	ScaleOutCount                     int
 	ScaleInCount                      int
-	ScaleOutCPUPercentageThreshold    int
-	ScaleOutMemoryPercentageThreshold int
-	ScaleInCPUPercentageThreshold     int
-	ScaleInMemoryPercentageThreshold  int
+	ScaleOutCPUPercentageThreshold    *int
+	ScaleOutMemoryPercentageThreshold *int
+	ScaleInCPUPercentageThreshold     *int
+	ScaleInMemoryPercentageThreshold  *int
+	ExternalChecks                    map[string]*ExternalCheck
+}
+
+// ExternalCheck represents an individual external check within a group scaling policy.
+type ExternalCheck struct {
+	Enabled            bool
+	Provider           string
+	Query              string
+	ComparisonOperator string
+	ComparisonValue    int
+	Action             string
 }
 
 func (p *Policies) List() (*map[string]map[string]*JobGroupPolicy, error) {