Skip to content

Commit

Permalink
Move config loading out of main and add tests for it (#413)
Browse files Browse the repository at this point in the history
The loading of config was previously untested.
  • Loading branch information
gabegorelick committed Feb 4, 2020
1 parent 3bfd542 commit 30a4392
Show file tree
Hide file tree
Showing 5 changed files with 198 additions and 142 deletions.
144 changes: 3 additions & 141 deletions autospotting.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,16 +6,12 @@ package main
import (
"context"
"encoding/json"
"fmt"
"log"
"os"

autospotting "github.com/AutoSpotting/AutoSpotting/core"
"github.com/aws/aws-lambda-go/events"
"github.com/aws/aws-lambda-go/lambda"
"github.com/aws/aws-sdk-go/aws/endpoints"
ec2instancesinfo "github.com/cristim/ec2-instances-info"
"github.com/namsral/flag"
)

var conf autospotting.Config
Expand All @@ -34,43 +30,7 @@ func main() {
func run() {

log.Println("Starting autospotting agent, build", Version)

log.Printf("Parsed command line flags: "+
"regions='%s' "+
"min_on_demand_number=%d "+
"min_on_demand_percentage=%.1f "+
"allowed_instance_types=%v "+
"disallowed_instance_types=%v "+
"on_demand_price_multiplier=%.2f "+
"spot_price_buffer_percentage=%.3f "+
"bidding_policy=%s "+
"tag_filters=%s "+
"tag_filter_mode=%s "+
"spot_product_description=%v "+
"instance_termination_method=%s "+
"termination_notification_action=%s "+
"cron_schedule=%s "+
"cron_schedule_state=%s "+
"license=%s "+
"patch_beanstalk_userdata=%s \n",
conf.Regions,
conf.MinOnDemandNumber,
conf.MinOnDemandPercentage,
conf.AllowedInstanceTypes,
conf.DisallowedInstanceTypes,
conf.OnDemandPriceMultiplier,
conf.SpotPriceBufferPercentage,
conf.BiddingPolicy,
conf.FilterByTags,
conf.TagFilteringMode,
conf.SpotProductDescription,
conf.InstanceTerminationMethod,
conf.TerminationNotificationAction,
conf.CronSchedule,
conf.CronScheduleState,
conf.LicenseType,
conf.PatchBeanstalkUserdata,
)
log.Printf("Configuration flags: %#v", conf)

autospotting.Run(&conf)
log.Println("Execution completed, nothing left to do")
Expand All @@ -79,30 +39,10 @@ func run() {
// this is the equivalent of a main for when running from Lambda, but on Lambda
// the run() is executed within the handler function every time we have an event
func init() {
var region string

if r := os.Getenv("AWS_REGION"); r != "" {
region = r
} else {
region = endpoints.UsEast1RegionID
}

conf = autospotting.Config{
LogFile: os.Stdout,
LogFlag: log.Ldate | log.Ltime | log.Lshortfile,
MainRegion: region,
SleepMultiplier: 1,
Version: Version,
LicenseType: os.Getenv("LICENSE"),
Version: Version,
}

parseCommandLineFlags()

data, err := ec2instancesinfo.Data()
if err != nil {
log.Fatal(err.Error())
}
conf.InstanceData = data
autospotting.ParseConfig(&conf)
}

// Handler implements the AWS Lambda handler
Expand Down Expand Up @@ -152,81 +92,3 @@ func Handler(ctx context.Context, rawEvent json.RawMessage) {
run()
}
}

func parseCommandLineFlags() {
flag.StringVar(&conf.AllowedInstanceTypes, "allowed_instance_types", "",
"\n\tIf specified, the spot instances will be searched only among these types.\n\tIf missing, any instance type is allowed.\n"+
"\tAccepts a list of comma or whitespace separated instance types (supports globs).\n"+
"\tExample: ./AutoSpotting -allowed_instance_types 'c5.*,c4.xlarge'\n")
flag.StringVar(&conf.BiddingPolicy, "bidding_policy", autospotting.DefaultBiddingPolicy,
"\n\tPolicy choice for spot bid. If set to 'normal', we bid at the on-demand price(times the multiplier).\n"+
"\tIf set to 'aggressive', we bid at a percentage value above the spot price \n"+
"\tconfigurable using the spot_price_buffer_percentage.\n")
flag.StringVar(&conf.DisallowedInstanceTypes, "disallowed_instance_types", "",
"\n\tIf specified, the spot instances will _never_ be of these types.\n"+
"\tAccepts a list of comma or whitespace separated instance types (supports globs).\n"+
"\tExample: ./AutoSpotting -disallowed_instance_types 't2.*,c4.xlarge'\n")
flag.StringVar(&conf.InstanceTerminationMethod, "instance_termination_method", autospotting.DefaultInstanceTerminationMethod,
"\n\tInstance termination method. Must be one of '"+autospotting.DefaultInstanceTerminationMethod+"' (default),\n"+
"\t or 'detach' (compatibility mode, not recommended)\n")
flag.StringVar(&conf.TerminationNotificationAction, "termination_notification_action", autospotting.DefaultTerminationNotificationAction,
"\n\tTermination Notification Action.\n"+
"\tValid choices:\n"+
"\t'"+autospotting.DefaultTerminationNotificationAction+
"' (terminate if lifecyclehook else detach) | 'terminate' (lifecyclehook triggered)"+
" | 'detach' (lifecyclehook not triggered)\n")
flag.Int64Var(&conf.MinOnDemandNumber, "min_on_demand_number", autospotting.DefaultMinOnDemandValue,
"\n\tNumber of on-demand nodes to be kept running in each of the groups.\n\t"+
"Can be overridden on a per-group basis using the tag "+autospotting.OnDemandNumberLong+".\n")
flag.Float64Var(&conf.MinOnDemandPercentage, "min_on_demand_percentage", 0.0,
"\n\tPercentage of the total number of instances in each group to be kept on-demand\n\t"+
"Can be overridden on a per-group basis using the tag "+autospotting.OnDemandPercentageTag+
"\n\tIt is ignored if min_on_demand_number is also set.\n")
flag.Float64Var(&conf.OnDemandPriceMultiplier, "on_demand_price_multiplier", 1.0,
"\n\tMultiplier for the on-demand price. Numbers less than 1.0 are useful for volume discounts.\n"+
"\tExample: ./AutoSpotting -on_demand_price_multiplier 0.6 will have the on-demand price "+
"considered at 60% of the actual value.\n")
flag.StringVar(&conf.Regions, "regions", "",
"\n\tRegions where it should be activated (separated by comma or whitespace, also supports globs).\n"+
"\tBy default it runs on all regions.\n"+
"\tExample: ./AutoSpotting -regions 'eu-*,us-east-1'\n")
flag.Float64Var(&conf.SpotPriceBufferPercentage, "spot_price_buffer_percentage", autospotting.DefaultSpotPriceBufferPercentage,
"\n\tBid a given percentage above the current spot price.\n\tProtects the group from running spot"+
"instances that got significantly more expensive than when they were initially launched\n"+
"\tThe tag "+autospotting.SpotPriceBufferPercentageTag+" can be used to override this on a group level.\n"+
"\tIf the bid exceeds the on-demand price, we place a bid at on-demand price itself.\n")
flag.StringVar(&conf.SpotProductDescription, "spot_product_description", autospotting.DefaultSpotProductDescription,
"\n\tThe Spot Product to use when looking up spot price history in the market.\n"+
"\tValid choices: Linux/UNIX | SUSE Linux | Windows | Linux/UNIX (Amazon VPC) | \n"+
"\tSUSE Linux (Amazon VPC) | Windows (Amazon VPC)\n\tDefault value: "+autospotting.DefaultSpotProductDescription+"\n")
flag.StringVar(&conf.TagFilteringMode, "tag_filtering_mode", "opt-in", "\n\tControls the behavior of the tag_filters option.\n"+
"\tValid choices: opt-in | opt-out\n\tDefault value: 'opt-in'\n\tExample: ./AutoSpotting --tag_filtering_mode opt-out\n")
flag.StringVar(&conf.FilterByTags, "tag_filters", "", "\n\tSet of tags to filter the ASGs on.\n"+
"\tDefault if no value is set will be the equivalent of -tag_filters 'spot-enabled=true'\n"+
"\tIn case the tag_filtering_mode is set to opt-out, it defaults to 'spot-enabled=false'\n"+
"\tExample: ./AutoSpotting --tag_filters 'spot-enabled=true,Environment=dev,Team=vision'\n")

flag.StringVar(&conf.CronSchedule, "cron_schedule", "* *", "\n\tCron-like schedule in which to"+
"\tperform(or not) spot replacement actions. Format: hour day-of-week\n"+
"\tExample: ./AutoSpotting --cron_schedule '9-18 1-5' # workdays during the office hours \n")

flag.StringVar(&conf.CronScheduleState, "cron_schedule_state", "on", "\n\tControls whether to take actions "+
"inside or outside the schedule defined by cron_schedule. Allowed values: on|off\n"+
"\tExample: ./AutoSpotting --cron_schedule_state='off' --cron_schedule '9-18 1-5' # would only take action outside the defined schedule\n")
flag.StringVar(&conf.LicenseType, "license", "evaluation", "\n\tControls the terms under which you use AutoSpotting"+
"Allowed values: evaluation|I_am_supporting_it_on_Patreon|I_contributed_to_development_within_the_last_year|I_built_it_from_source_code\n"+
"\tExample: ./AutoSpotting --license evaluation\n")
flag.StringVar(&conf.PatchBeanstalkUserdata, "patch_beanstalk_userdata", "", "\n\tControls whether AutoSpotting patches Elastic Beanstalk UserData scripts to use the instance role when calling CloudFormation helpers instead of the standard CloudFormation authentication method\n"+
"\tExample: ./AutoSpotting --patch_beanstalk_userdata true\n")

v := flag.Bool("version", false, "Print version number and exit.\n")
flag.Parse()
printVersion(v)
}

func printVersion(v *bool) {
if *v {
fmt.Println("AutoSpotting build:", Version)
os.Exit(0)
}
}
107 changes: 107 additions & 0 deletions core/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,10 +4,15 @@
package autospotting

import (
"fmt"
"io"
"log"
"os"
"time"

"github.com/aws/aws-sdk-go/aws/endpoints"
ec2instancesinfo "github.com/cristim/ec2-instances-info"
"github.com/namsral/flag"
)

const (
Expand Down Expand Up @@ -88,3 +93,105 @@ type Config struct {
// authentication method
PatchBeanstalkUserdata string
}

// ParseConfig loads configuration from command line flags, environments variables, and config files.
func ParseConfig(conf *Config) {

// The use of FlagSet allows us to parse config multiple times, which is useful for unit tests.
flagSet := flag.NewFlagSet("AutoSpotting", flag.ExitOnError)

var region string

if r := os.Getenv("AWS_REGION"); r != "" {
region = r
} else {
region = endpoints.UsEast1RegionID
}

conf.LogFile = os.Stdout
conf.LogFlag = log.Ldate | log.Ltime | log.Lshortfile
conf.MainRegion = region
conf.SleepMultiplier = 1

flagSet.StringVar(&conf.AllowedInstanceTypes, "allowed_instance_types", "",
"\n\tIf specified, the spot instances will be searched only among these types.\n\tIf missing, any instance type is allowed.\n"+
"\tAccepts a list of comma or whitespace separated instance types (supports globs).\n"+
"\tExample: ./AutoSpotting -allowed_instance_types 'c5.*,c4.xlarge'\n")
flagSet.StringVar(&conf.BiddingPolicy, "bidding_policy", DefaultBiddingPolicy,
"\n\tPolicy choice for spot bid. If set to 'normal', we bid at the on-demand price(times the multiplier).\n"+
"\tIf set to 'aggressive', we bid at a percentage value above the spot price \n"+
"\tconfigurable using the spot_price_buffer_percentage.\n")
flagSet.StringVar(&conf.DisallowedInstanceTypes, "disallowed_instance_types", "",
"\n\tIf specified, the spot instances will _never_ be of these types.\n"+
"\tAccepts a list of comma or whitespace separated instance types (supports globs).\n"+
"\tExample: ./AutoSpotting -disallowed_instance_types 't2.*,c4.xlarge'\n")
flagSet.StringVar(&conf.InstanceTerminationMethod, "instance_termination_method", DefaultInstanceTerminationMethod,
"\n\tInstance termination method. Must be one of '"+DefaultInstanceTerminationMethod+"' (default),\n"+
"\t or 'detach' (compatibility mode, not recommended)\n")
flagSet.StringVar(&conf.TerminationNotificationAction, "termination_notification_action", DefaultTerminationNotificationAction,
"\n\tTermination Notification Action.\n"+
"\tValid choices:\n"+
"\t'"+DefaultTerminationNotificationAction+
"' (terminate if lifecyclehook else detach) | 'terminate' (lifecyclehook triggered)"+
" | 'detach' (lifecyclehook not triggered)\n")
flagSet.Int64Var(&conf.MinOnDemandNumber, "min_on_demand_number", DefaultMinOnDemandValue,
"\n\tNumber of on-demand nodes to be kept running in each of the groups.\n\t"+
"Can be overridden on a per-group basis using the tag "+OnDemandNumberLong+".\n")
flagSet.Float64Var(&conf.MinOnDemandPercentage, "min_on_demand_percentage", 0.0,
"\n\tPercentage of the total number of instances in each group to be kept on-demand\n\t"+
"Can be overridden on a per-group basis using the tag "+OnDemandPercentageTag+
"\n\tIt is ignored if min_on_demand_number is also set.\n")
flagSet.Float64Var(&conf.OnDemandPriceMultiplier, "on_demand_price_multiplier", 1.0,
"\n\tMultiplier for the on-demand price. Numbers less than 1.0 are useful for volume discounts.\n"+
"\tExample: ./AutoSpotting -on_demand_price_multiplier 0.6 will have the on-demand price "+
"considered at 60% of the actual value.\n")
flagSet.StringVar(&conf.Regions, "regions", "",
"\n\tRegions where it should be activated (separated by comma or whitespace, also supports globs).\n"+
"\tBy default it runs on all regions.\n"+
"\tExample: ./AutoSpotting -regions 'eu-*,us-east-1'\n")
flagSet.Float64Var(&conf.SpotPriceBufferPercentage, "spot_price_buffer_percentage", DefaultSpotPriceBufferPercentage,
"\n\tBid a given percentage above the current spot price.\n\tProtects the group from running spot"+
"instances that got significantly more expensive than when they were initially launched\n"+
"\tThe tag "+SpotPriceBufferPercentageTag+" can be used to override this on a group level.\n"+
"\tIf the bid exceeds the on-demand price, we place a bid at on-demand price itself.\n")
flagSet.StringVar(&conf.SpotProductDescription, "spot_product_description", DefaultSpotProductDescription,
"\n\tThe Spot Product to use when looking up spot price history in the market.\n"+
"\tValid choices: Linux/UNIX | SUSE Linux | Windows | Linux/UNIX (Amazon VPC) | \n"+
"\tSUSE Linux (Amazon VPC) | Windows (Amazon VPC)\n\tDefault value: "+DefaultSpotProductDescription+"\n")
flagSet.StringVar(&conf.TagFilteringMode, "tag_filtering_mode", "opt-in", "\n\tControls the behavior of the tag_filters option.\n"+
"\tValid choices: opt-in | opt-out\n\tDefault value: 'opt-in'\n\tExample: ./AutoSpotting --tag_filtering_mode opt-out\n")
flagSet.StringVar(&conf.FilterByTags, "tag_filters", "", "\n\tSet of tags to filter the ASGs on.\n"+
"\tDefault if no value is set will be the equivalent of -tag_filters 'spot-enabled=true'\n"+
"\tIn case the tag_filtering_mode is set to opt-out, it defaults to 'spot-enabled=false'\n"+
"\tExample: ./AutoSpotting --tag_filters 'spot-enabled=true,Environment=dev,Team=vision'\n")

flagSet.StringVar(&conf.CronSchedule, "cron_schedule", "* *", "\n\tCron-like schedule in which to"+
"\tperform(or not) spot replacement actions. Format: hour day-of-week\n"+
"\tExample: ./AutoSpotting --cron_schedule '9-18 1-5' # workdays during the office hours \n")

flagSet.StringVar(&conf.CronScheduleState, "cron_schedule_state", "on", "\n\tControls whether to take actions "+
"inside or outside the schedule defined by cron_schedule. Allowed values: on|off\n"+
"\tExample: ./AutoSpotting --cron_schedule_state='off' --cron_schedule '9-18 1-5' # would only take action outside the defined schedule\n")
flagSet.StringVar(&conf.LicenseType, "license", "evaluation", "\n\tControls the terms under which you use AutoSpotting"+
"Allowed values: evaluation|I_am_supporting_it_on_Patreon|I_contributed_to_development_within_the_last_year|I_built_it_from_source_code\n"+
"\tExample: ./AutoSpotting --license evaluation\n")
flagSet.StringVar(&conf.PatchBeanstalkUserdata, "patch_beanstalk_userdata", "", "\n\tControls whether AutoSpotting patches Elastic Beanstalk UserData scripts to use the instance role when calling CloudFormation helpers instead of the standard CloudFormation authentication method\n"+
"\tExample: ./AutoSpotting --patch_beanstalk_userdata true\n")

printVersion := flagSet.Bool("version", false, "Print version number and exit.\n")

if err := flagSet.Parse(os.Args[1:]); err != nil {
fmt.Printf("Error parsing config: %s\n", err.Error())
}

if *printVersion {
fmt.Println("AutoSpotting build:", conf.Version)
os.Exit(0)
}

data, err := ec2instancesinfo.Data()
if err != nil {
log.Fatal(err.Error())
}
conf.InstanceData = data
}
79 changes: 79 additions & 0 deletions core/config_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,79 @@
// Copyright (c) 2016-2019 Cristian Măgherușan-Stanciu
// Licensed under the Open Software License version 3.0

package autospotting

import (
"os"
"strings"
"testing"
"time"

"gotest.tools/v3/assert"
)

func TestParseConfig(t *testing.T) {
tests := []struct {
name string
environment map[string]string
}{
{
name: "default settings",
},
{
name: "with AWS_REGION set",
environment: map[string]string{
"AWS_REGION": "us-west-2",
},
},
{
name: "with LICENSE set",
environment: map[string]string{
"LICENSE": "I_built_it_from_source_code",
},
},
}

// save copy of environment before we run any tests
envVars := make(map[string]string)
for _, item := range os.Environ() {
e := strings.SplitN(item, "=", 2)
envVars[e[0]] = e[1]
}

for _, tt := range tests {
if tt.environment != nil {
for key, value := range tt.environment {
os.Setenv(key, value)
}
}

t.Run(tt.name, func(t *testing.T) {
config := Config{}
ParseConfig(&config)

if tt.environment != nil {
if tt.environment["AWS_REGION"] != "" {
assert.Equal(t, config.MainRegion, tt.environment["AWS_REGION"])
} else {
assert.Equal(t, config.MainRegion, "us-east-1", "MainRegion should default to us-east-1")
}

if tt.environment["LICENSE"] != "" {
assert.Equal(t, config.LicenseType, tt.environment["LICENSE"])
}
}

assert.Equal(t, config.LogFile, os.Stdout)
assert.Equal(t, config.SleepMultiplier, time.Duration(1))
assert.Assert(t, config.InstanceData != nil, "expected InstanceData to be initialized")
})

// reset environment variables
if tt.environment != nil {
for name := range tt.environment {
os.Setenv(name, envVars[name])
}
}
}
}
2 changes: 1 addition & 1 deletion go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -9,11 +9,11 @@ require (
github.com/davecgh/go-spew v1.1.1
github.com/mattn/goveralls v0.0.5 // indirect
github.com/namsral/flag v0.0.0-20170814194028-67f268f20922
github.com/pkg/errors v0.8.1 // indirect
github.com/robfig/cron v1.2.0
github.com/stretchr/objx v0.2.0 // indirect
golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f // indirect
golang.org/x/net v0.0.0-20190628185345-da137c7871d7 // indirect
golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb // indirect
golang.org/x/text v0.3.2 // indirect
gotest.tools/v3 v3.0.0
)
Loading

0 comments on commit 30a4392

Please sign in to comment.