Skip to content

Commit

Permalink
Ignore terminating spot instances that don't belong to AutoSpotting (#…
Browse files Browse the repository at this point in the history 
…391)

This is a reworking of #363 against `master`.

Credit for original code goes to @cfarrend.

Fixes #362
Closes #363
Closes #390
  • Loading branch information
@gabegorelick @cristim
gabegorelick authored and cristim committed Jan 28, 2020
1 parent 0eed8cd commit 33a444c
Show file tree
Hide file tree
Showing 4 changed files with 244 additions and 5 deletions.
17 changes: 13 additions & 4 deletions autospotting.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -133,11 +133,20 @@ func Handler(ctx context.Context, rawEvent json.RawMessage) {

// If event is Instance Spot Interruption
if cloudwatchEvent.DetailType == "EC2 Spot Instance Interruption Warning" {
if instanceID, err := autospotting.GetInstanceIDDueForTermination(cloudwatchEvent); err != nil {
instanceID, err := autospotting.GetInstanceIDDueForTermination(cloudwatchEvent)
if err != nil || instanceID == nil {
return
}

spotTermination := autospotting.NewSpotTermination(cloudwatchEvent.Region)
if spotTermination.IsInAutoSpottingASG(instanceID, conf.TagFilteringMode, conf.FilterByTags) {
err := spotTermination.ExecuteAction(instanceID, conf.TerminationNotificationAction)
if err != nil {
log.Printf("Error executing spot termination action: %s\n", err.Error())
}
} else {
log.Printf("Instance %s is not in AutoSpotting ASG\n", *instanceID)
return
} else if instanceID != nil {
spotTermination := autospotting.NewSpotTermination(cloudwatchEvent.Region)
spotTermination.ExecuteAction(instanceID, conf.TerminationNotificationAction)
}
} else {
// Event is Autospotting Cron Scheduling
Expand Down
7 changes: 6 additions & 1 deletion core/mock_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -151,7 +151,8 @@ type mockASG struct {
dto *autoscaling.DescribeTagsOutput

// Describe AutoScaling Group
dasgo *autoscaling.DescribeAutoScalingGroupsOutput
dasgo *autoscaling.DescribeAutoScalingGroupsOutput
dasgerr error

// Describe AutoScalingInstances
dasio *autoscaling.DescribeAutoScalingInstancesOutput
Expand Down Expand Up @@ -187,6 +188,10 @@ func (m mockASG) DescribeTagsPages(input *autoscaling.DescribeTagsInput, functio
return nil
}

func (m mockASG) DescribeAutoScalingGroups(input *autoscaling.DescribeAutoScalingGroupsInput) (*autoscaling.DescribeAutoScalingGroupsOutput, error) {
return m.dasgo, m.dasgerr
}

func (m mockASG) DescribeAutoScalingGroupsPages(input *autoscaling.DescribeAutoScalingGroupsInput, function func(*autoscaling.DescribeAutoScalingGroupsOutput, bool) bool) error {
function(m.dasgo, true)
return nil
Expand Down
49 changes: 49 additions & 0 deletions core/spot_termination.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ package autospotting
import (
"encoding/json"
"errors"
"strings"

"github.com/aws/aws-lambda-go/events"
"github.com/aws/aws-sdk-go/aws"
Expand Down Expand Up @@ -209,3 +210,51 @@ func (s *SpotTermination) asgHasTerminationLifecycleHook(autoScalingGroupName *s

return hasHook
}

// IsInAutoSpottingASG checks to see whether an instance is in an AutoSpotting ASG as defined by its tags.
// If the ASG does not have the required tags, it is not an AutoSpotting ASG and should be left alone.
func (s *SpotTermination) IsInAutoSpottingASG(instanceID *string, tagFilteringMode string, filterByTags string) bool {
var optInFilterMode = (tagFilteringMode != "opt-out")

asgName, err := s.getAsgName(instanceID)

if err != nil {
logger.Printf("Failed get ASG name for %s with err: %s\n", *instanceID, err.Error())
return false
} else if asgName == "" {
logger.Println("Instance", instanceID, "is not in an autoscaling group")
return false
}

asgGroupsOutput, err := s.asSvc.DescribeAutoScalingGroups(&autoscaling.DescribeAutoScalingGroupsInput{
AutoScalingGroupNames: []*string{
&asgName,
},
})

if err != nil {
logger.Printf("Failed to get ASG using ASG name %s with err: %s\n", asgName, err.Error())
return false
}

filters := replaceWhitespace(filterByTags)

var tagsToMatch = []Tag{}

for _, tagWithValue := range strings.Split(filters, ",") {
tag := splitTagAndValue(tagWithValue)
if tag != nil {
tagsToMatch = append(tagsToMatch, *tag)
}
}

isInASG := optInFilterMode == isASGWithMatchingTags(asgGroupsOutput.AutoScalingGroups[0], tagsToMatch)

if !isInASG {
logger.Printf("Skipping group %s because its tags, the currently "+
"configured filtering mode (%s) and tag filters do not align\n",
asgName, tagFilteringMode)
}

return isInASG
}
176 changes: 176 additions & 0 deletions core/spot_termination_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import (
"testing"

"github.com/aws/aws-lambda-go/events"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/service/autoscaling"
"github.com/aws/aws-sdk-go/service/ec2"
)
Expand Down Expand Up @@ -372,3 +373,178 @@ func TestExecuteAction(t *testing.T) {
})
}
}

func TestIsInAutoSpottingASG(t *testing.T) {
instanceID := "dummyInstanceID"

tests := []struct {
name string
spotTermination *SpotTermination
tagFilteringMode string
filterByTags string
expected bool
}{
{
name: "When instance is not in an ASG",
spotTermination: &SpotTermination{
asSvc: mockASG{
dasio: &autoscaling.DescribeAutoScalingInstancesOutput{
AutoScalingInstances: []*autoscaling.InstanceDetails{},
},
},
},
tagFilteringMode: "opt-in",
filterByTags: "spot-enabled=true",
expected: false,
},
{
name: "When instance is in ASG with matching tags",
spotTermination: &SpotTermination{
asSvc: mockASG{
dasgo: &autoscaling.DescribeAutoScalingGroupsOutput{
AutoScalingGroups: []*autoscaling.Group{
{
AutoScalingGroupName: aws.String("asg1"),
Tags: []*autoscaling.TagDescription{
{
Key: aws.String("spot-enabled"),
Value: aws.String("true"),
},
},
},
},
},
dasio: &autoscaling.DescribeAutoScalingInstancesOutput{
AutoScalingInstances: []*autoscaling.InstanceDetails{
{
AutoScalingGroupName: aws.String("asg1"),
},
},
},
},
},
tagFilteringMode: "opt-in",
filterByTags: "spot-enabled=true",
expected: true,
},
{
name: "When instance is in ASG without matching tag value",
spotTermination: &SpotTermination{
asSvc: mockASG{
dasgo: &autoscaling.DescribeAutoScalingGroupsOutput{
AutoScalingGroups: []*autoscaling.Group{
{
AutoScalingGroupName: aws.String("asg1"),
Tags: []*autoscaling.TagDescription{
{
Key: aws.String("spot-enabled"),
Value: aws.String("false"),
},
},
},
},
},
dasio: &autoscaling.DescribeAutoScalingInstancesOutput{
AutoScalingInstances: []*autoscaling.InstanceDetails{
{
AutoScalingGroupName: aws.String("asg1"),
},
},
},
},
},
tagFilteringMode: "opt-in",
filterByTags: "spot-enabled=true",
expected: false,
},
{
name: "When instance is in ASG with no tags",
spotTermination: &SpotTermination{
asSvc: mockASG{
dasgo: &autoscaling.DescribeAutoScalingGroupsOutput{
AutoScalingGroups: []*autoscaling.Group{
{
AutoScalingGroupName: aws.String("asg1"),
Tags: []*autoscaling.TagDescription{},
},
},
},
dasio: &autoscaling.DescribeAutoScalingInstancesOutput{
AutoScalingInstances: []*autoscaling.InstanceDetails{
{
AutoScalingGroupName: aws.String("asg1"),
},
},
},
},
},
tagFilteringMode: "opt-in",
filterByTags: "spot-enabled=true",
expected: false,
},
{
name: "When instance is in ASG that has opted out",
spotTermination: &SpotTermination{
asSvc: mockASG{
dasgo: &autoscaling.DescribeAutoScalingGroupsOutput{
AutoScalingGroups: []*autoscaling.Group{
{
AutoScalingGroupName: aws.String("asg1"),
Tags: []*autoscaling.TagDescription{
{
Key: aws.String("spot-enabled"),
Value: aws.String("false"),
},
},
},
},
},
dasio: &autoscaling.DescribeAutoScalingInstancesOutput{
AutoScalingInstances: []*autoscaling.InstanceDetails{
{
AutoScalingGroupName: aws.String("asg1"),
},
},
},
},
},
tagFilteringMode: "opt-out",
filterByTags: "spot-enabled=false",
expected: false,
},
{
name: "When instance is in ASG that has not opted out",
spotTermination: &SpotTermination{
asSvc: mockASG{
dasgo: &autoscaling.DescribeAutoScalingGroupsOutput{
AutoScalingGroups: []*autoscaling.Group{
{
AutoScalingGroupName: aws.String("asg1"),
},
},
},
dasio: &autoscaling.DescribeAutoScalingInstancesOutput{
AutoScalingInstances: []*autoscaling.InstanceDetails{
{
AutoScalingGroupName: aws.String("asg1"),
},
},
},
},
},
tagFilteringMode: "opt-out",
filterByTags: "spot-enabled=false",
expected: true,
},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {

actual := tc.spotTermination.IsInAutoSpottingASG(&instanceID, tc.tagFilteringMode, tc.filterByTags)

if tc.expected != actual {
t.Errorf("isInAutoSpottingASG received for %s: %v expected %v", tc.name, actual, tc.expected)
}
})
}
}

0 comments on commit 33a444c

Please sign in to comment.