diff --git a/CHANGELOG_PENDING.md b/CHANGELOG_PENDING.md index 8a1927a39ca..3dd2eaebad1 100644 --- a/CHANGELOG_PENDING.md +++ b/CHANGELOG_PENDING.md @@ -1,4 +1,5 @@ ### SDK Features +* `service/s3/s3crypto`: Updates to the Amazon S3 Encryption Client - This change includes fixes for issues that were reported by Sophie Schmieg from the Google ISE team, and for issues that were discovered by AWS Cryptography. ### SDK Enhancements diff --git a/aws/csm/reporter_test.go b/aws/csm/reporter_test.go index 9a835a80e48..7efdaccf06a 100644 --- a/aws/csm/reporter_test.go +++ b/aws/csm/reporter_test.go @@ -183,7 +183,7 @@ func TestReportingMetrics(t *testing.T) { { "Type": "ApiCallAttempt", "SdkException": request.ErrCodeRequestError, - "SdkExceptionMessage": request.ErrCodeRequestError+": sdk error", + "SdkExceptionMessage": request.ErrCodeRequestError + ": sdk error", "HttpStatusCode": float64(500), }, { diff --git a/aws/ec2metadata/service.go b/aws/ec2metadata/service.go index b8b2940d744..dc7e051e0c0 100644 --- a/aws/ec2metadata/service.go +++ b/aws/ec2metadata/service.go @@ -41,7 +41,7 @@ const ( enableTokenProviderHandlerName = "enableTokenProviderHandler" // TTL constants - defaultTTL = 21600 * time.Second + defaultTTL = 21600 * time.Second ttlExpirationWindow = 30 * time.Second ) diff --git a/service/s3/doc_custom.go b/service/s3/doc_custom.go index 4b65f71531a..7f7aca20859 100644 --- a/service/s3/doc_custom.go +++ b/service/s3/doc_custom.go @@ -104,19 +104,6 @@ // content from S3. The Encryption and Decryption clients can be used concurrently // once the client is created. // -// sess := session.Must(session.NewSession()) -// -// // Create the decryption client. -// svc := s3crypto.NewDecryptionClient(sess) -// -// // The object will be downloaded from S3 and decrypted locally. By metadata -// // about the object's encryption will instruct the decryption client how -// // decrypt the content of the object. By default KMS is used for keys. -// result, err := svc.GetObject(&s3.GetObjectInput { -// Bucket: aws.String(myBucket), -// Key: aws.String(myKey), -// }) -// // See the s3crypto package documentation for more information. // https://docs.aws.amazon.com/sdk-for-go/api/service/s3/s3crypto/ // diff --git a/service/s3/s3crypto/aes_cbc_content_cipher.go b/service/s3/s3crypto/aes_cbc_content_cipher.go index 30d6b8cb7b2..952632c18eb 100644 --- a/service/s3/s3crypto/aes_cbc_content_cipher.go +++ b/service/s3/s3crypto/aes_cbc_content_cipher.go @@ -2,7 +2,6 @@ package s3crypto import ( "io" - "strings" ) const ( @@ -15,18 +14,38 @@ type cbcContentCipherBuilder struct { padder Padder } -func (cbcContentCipherBuilder) isUsingDeprecatedFeatures() error { - return errDeprecatedCipherBuilder -} - -// AESCBCContentCipherBuilder returns a new encryption only mode structure with a specific cipher -// for the master key +// AESCBCContentCipherBuilder returns a new encryption only AES/CBC mode structure using the provided padder. The provided cipher data generator +// will be used to provide keys for content encryption. // -// deprecated: This content cipher builder has been deprecated. Users should migrate to AESGCMContentCipherBuilder +// deprecated: This feature is in maintenance mode, no new updates will be released. Please see https://docs.aws.amazon.com/general/latest/gr/aws_sdk_cryptography.html for more information. func AESCBCContentCipherBuilder(generator CipherDataGenerator, padder Padder) ContentCipherBuilder { return cbcContentCipherBuilder{generator: generator, padder: padder} } +// RegisterAESCBCContentCipher registers the AES/CBC cipher and padder with the provided CryptoRegistry. +// +// Example: +// cr := s3crypto.NewCryptoRegistry() +// if err := s3crypto.RegisterAESCBCContentCipher(cr, s3crypto.AESCBCPadder); err != nil { +// panic(err) // handle error +// } +// +// deprecated: This feature is in maintenance mode, no new updates will be released. Please see https://docs.aws.amazon.com/general/latest/gr/aws_sdk_cryptography.html for more information. +func RegisterAESCBCContentCipher(registry *CryptoRegistry, padder Padder) error { + if registry == nil { + return errNilCryptoRegistry + } + name := AESCBC + "/" + padder.Name() + err := registry.AddCEK(name, newAESCBCContentCipher) + if err != nil { + return err + } + if err := registry.AddPadder(name, padder); err != nil { + return err + } + return nil +} + func (builder cbcContentCipherBuilder) ContentCipher() (ContentCipher, error) { cd, err := builder.generator.GenerateCipherData(cbcKeySize, cbcNonceSize) if err != nil { @@ -37,11 +56,22 @@ func (builder cbcContentCipherBuilder) ContentCipher() (ContentCipher, error) { return newAESCBCContentCipher(cd) } +func (builder cbcContentCipherBuilder) isAWSFixture() bool { + return true +} + +func (cbcContentCipherBuilder) isEncryptionVersionCompatible(version clientVersion) error { + if version != v1ClientVersion { + return errDeprecatedIncompatibleCipherBuilder + } + return nil +} + // newAESCBCContentCipher will create a new aes cbc content cipher. If the cipher data's -// will set the CEK algorithm if it hasn't been set. +// will set the cek algorithm if it hasn't been set. func newAESCBCContentCipher(cd CipherData) (ContentCipher, error) { if len(cd.CEKAlgorithm) == 0 { - cd.CEKAlgorithm = strings.Join([]string{AESCBC, cd.Padder.Name()}, "/") + cd.CEKAlgorithm = AESCBC + "/" + cd.Padder.Name() } cipher, err := newAESCBC(cd, cd.Padder) if err != nil { @@ -77,3 +107,11 @@ func (cc *aesCBCContentCipher) DecryptContents(src io.ReadCloser) (io.ReadCloser func (cc aesCBCContentCipher) GetCipherData() CipherData { return cc.CipherData } + +var ( + _ ContentCipherBuilder = (*cbcContentCipherBuilder)(nil) + _ compatibleEncryptionFixture = (*cbcContentCipherBuilder)(nil) + _ awsFixture = (*cbcContentCipherBuilder)(nil) + + _ ContentCipher = (*aesCBCContentCipher)(nil) +) diff --git a/service/s3/s3crypto/aes_cbc_content_cipher_test.go b/service/s3/s3crypto/aes_cbc_content_cipher_test.go index 2d8fd920aef..61b9df2b5c2 100644 --- a/service/s3/s3crypto/aes_cbc_content_cipher_test.go +++ b/service/s3/s3crypto/aes_cbc_content_cipher_test.go @@ -1,14 +1,13 @@ -package s3crypto_test +package s3crypto import ( + "strings" "testing" - - "github.com/aws/aws-sdk-go/service/s3/s3crypto" ) func TestAESCBCBuilder(t *testing.T) { generator := mockGenerator{} - builder := s3crypto.AESCBCContentCipherBuilder(generator, s3crypto.NoPadder) + builder := AESCBCContentCipherBuilder(generator, NoPadder) if builder == nil { t.Fatal(builder) } @@ -18,3 +17,64 @@ func TestAESCBCBuilder(t *testing.T) { t.Fatal(err) } } + +func TestAesCBCContentCipher_isFixtureEncryptionCompatible(t *testing.T) { + generator := mockGenerator{} + builder := AESCBCContentCipherBuilder(generator, NoPadder) + if builder == nil { + t.Fatal("expected builder to not be nil") + } + + compatibility, ok := builder.(compatibleEncryptionFixture) + if !ok { + t.Fatal("expected builder to implement compatibleEncryptionFixture interface") + } + + if err := compatibility.isEncryptionVersionCompatible(v1ClientVersion); err != nil { + t.Errorf("expected builder to be compatible with v1 client") + } + + if err := compatibility.isEncryptionVersionCompatible(v2ClientVersion); err == nil { + t.Errorf("expected builder to not be compatible with v2 client") + } +} + +func TestRegisterAESCBCContentCipher(t *testing.T) { + cr := NewCryptoRegistry() + padder := AESCBCPadder + err := RegisterAESCBCContentCipher(cr, padder) + if err != nil { + t.Fatalf("expected no error, got %v", err) + } + + if v, ok := cr.GetCEK("AES/CBC/PKCS5Padding"); !ok { + t.Fatal("expected cek algorithm handler to registered") + } else if v == nil { + t.Fatal("expected non-nil cek handler to be registered") + } + + if v, ok := cr.GetPadder("AES/CBC/PKCS5Padding"); !ok { + t.Fatal("expected padder to be registered") + } else if v != padder { + t.Fatal("padder did not match provided value") + } + + // try to register padder again + err = RegisterAESCBCContentCipher(cr, padder) + if err == nil { + t.Fatal("expected error, got none") + } else if !strings.Contains(err.Error(), "duplicate cek registry entry") { + t.Errorf("expected duplicate cek entry, got %v", err) + } + + // try to regster padder with cek removed but padder entry still present + if _, ok := cr.RemoveCEK("AES/CBC/PKCS5Padding"); !ok { + t.Fatalf("expected value to be removed") + } + err = RegisterAESCBCContentCipher(cr, padder) + if err == nil { + t.Fatal("expected error, got none") + } else if !strings.Contains(err.Error(), "duplicate padder registry entry") { + t.Errorf("expected duplicate padder entry, got %v", err) + } +} diff --git a/service/s3/s3crypto/aes_gcm_content_cipher.go b/service/s3/s3crypto/aes_gcm_content_cipher.go index d4a4224e540..8bfdaa1c0e8 100644 --- a/service/s3/s3crypto/aes_gcm_content_cipher.go +++ b/service/s3/s3crypto/aes_gcm_content_cipher.go @@ -1,6 +1,7 @@ package s3crypto import ( + "fmt" "io" "github.com/aws/aws-sdk-go/aws" @@ -11,21 +12,62 @@ const ( gcmNonceSize = 12 ) -type gcmContentCipherBuilder struct { - generator CipherDataGenerator +// AESGCMContentCipherBuilder returns a new encryption only AES/GCM mode structure with a specific cipher data generator +// that will provide keys to be used for content encryption. +// +// Note: This uses the Go stdlib AEAD implementation for AES/GCM. Due to this objects to be encrypted or decrypted +// will be fully loaded into memory before encryption or decryption can occur. Caution must be taken to avoid memory +// allocation failures. +// +// deprecated: This feature is in maintenance mode, no new updates will be released. Please see https://docs.aws.amazon.com/general/latest/gr/aws_sdk_cryptography.html for more information. +func AESGCMContentCipherBuilder(generator CipherDataGenerator) ContentCipherBuilder { + return gcmContentCipherBuilder{generator} +} + +// AESGCMContentCipherBuilderV2 returns a new encryption only AES/GCM mode structure with a specific cipher data generator +// that will provide keys to be used for content encryption. This type is compatible with the V2 encryption client. +// +// Note: This uses the Go stdlib AEAD implementation for AES/GCM. Due to this objects to be encrypted or decrypted +// will be fully loaded into memory before encryption or decryption can occur. Caution must be taken to avoid memory +// allocation failures. +func AESGCMContentCipherBuilderV2(generator CipherDataGeneratorWithCEKAlg) ContentCipherBuilder { + return gcmContentCipherBuilderV2{generator} } -func (builder gcmContentCipherBuilder) isUsingDeprecatedFeatures() error { - if feature, ok := builder.generator.(deprecatedFeatures); ok { - return feature.isUsingDeprecatedFeatures() +// RegisterAESGCMContentCipher registers the AES/GCM content cipher algorithm with the provided CryptoRegistry. +// +// Example: +// cr := s3crypto.NewCryptoRegistry() +// if err := s3crypto.RegisterAESGCMContentCipher(cr); err != nil { +// panic(err) // handle error +// } +// +func RegisterAESGCMContentCipher(registry *CryptoRegistry) error { + if registry == nil { + return errNilCryptoRegistry + } + + err := registry.AddCEK(AESGCMNoPadding, newAESGCMContentCipher) + if err != nil { + return err + } + + // NoPadder is generic but required by this algorithm, so if it is already registered and is the expected implementation + // don't error. + padderName := NoPadder.Name() + if v, ok := registry.GetPadder(padderName); !ok { + if err := registry.AddPadder(padderName, NoPadder); err != nil { + return err + } + } else if _, ok := v.(noPadder); !ok { + return fmt.Errorf("%s is already registred but does not match expected type %T", padderName, NoPadder) } return nil } -// AESGCMContentCipherBuilder returns a new encryption only mode structure with a specific cipher -// for the master key -func AESGCMContentCipherBuilder(generator CipherDataGenerator) ContentCipherBuilder { - return gcmContentCipherBuilder{generator} +// gcmContentCipherBuilder is a AES/GCM content cipher to be used with the V1 client CipherDataGenerator interface +type gcmContentCipherBuilder struct { + generator CipherDataGenerator } func (builder gcmContentCipherBuilder) ContentCipher() (ContentCipher, error) { @@ -37,10 +79,6 @@ func (builder gcmContentCipherBuilder) ContentCipherWithContext(ctx aws.Context) var err error switch v := builder.generator.(type) { - case CipherDataGeneratorWithCEKAlgWithContext: - cd, err = v.GenerateCipherDataWithCEKAlgWithContext(ctx, gcmKeySize, gcmNonceSize, AESGCMNoPadding) - case CipherDataGeneratorWithCEKAlg: - cd, err = v.GenerateCipherDataWithCEKAlg(gcmKeySize, gcmNonceSize, AESGCMNoPadding) case CipherDataGeneratorWithContext: cd, err = v.GenerateCipherDataWithContext(ctx, gcmKeySize, gcmNonceSize) default: @@ -53,6 +91,52 @@ func (builder gcmContentCipherBuilder) ContentCipherWithContext(ctx aws.Context) return newAESGCMContentCipher(cd) } +// isFixtureEncryptionCompatible will ensure that this type may only be used with the V1 client +func (builder gcmContentCipherBuilder) isEncryptionVersionCompatible(version clientVersion) error { + if version != v1ClientVersion { + return errDeprecatedIncompatibleCipherBuilder + } + return nil +} + +func (builder gcmContentCipherBuilder) isAWSFixture() bool { + return true +} + +// gcmContentCipherBuilderV2 return a new builder for encryption content using AES/GCM/NoPadding. This type is meant +// to be used with key wrapping implementations that allow the cek algorithm to be provided when calling the +// cipher data generator. +type gcmContentCipherBuilderV2 struct { + generator CipherDataGeneratorWithCEKAlg +} + +func (builder gcmContentCipherBuilderV2) ContentCipher() (ContentCipher, error) { + return builder.ContentCipherWithContext(aws.BackgroundContext()) +} + +func (builder gcmContentCipherBuilderV2) ContentCipherWithContext(ctx aws.Context) (ContentCipher, error) { + cd, err := builder.generator.GenerateCipherDataWithCEKAlg(ctx, gcmKeySize, gcmNonceSize, AESGCMNoPadding) + if err != nil { + return nil, err + } + + return newAESGCMContentCipher(cd) +} + +// isFixtureEncryptionCompatible will ensure that this type may only be used with the V2 client +func (builder gcmContentCipherBuilderV2) isEncryptionVersionCompatible(version clientVersion) error { + if version != v2ClientVersion { + return errDeprecatedIncompatibleCipherBuilder + } + return nil +} + +// isAWSFixture will return whether this type was constructed with an AWS provided CipherDataGenerator +func (builder gcmContentCipherBuilderV2) isAWSFixture() bool { + v, ok := builder.generator.(awsFixture) + return ok && v.isAWSFixture() +} + func newAESGCMContentCipher(cd CipherData) (ContentCipher, error) { cd.CEKAlgorithm = AESGCMNoPadding cd.TagLength = "128" @@ -91,3 +175,25 @@ func (cc *aesGCMContentCipher) DecryptContents(src io.ReadCloser) (io.ReadCloser func (cc aesGCMContentCipher) GetCipherData() CipherData { return cc.CipherData } + +// assert ContentCipherBuilder implementations +var ( + _ ContentCipherBuilder = (*gcmContentCipherBuilder)(nil) + _ ContentCipherBuilder = (*gcmContentCipherBuilderV2)(nil) +) + +// assert ContentCipherBuilderWithContext implementations +var ( + _ ContentCipherBuilderWithContext = (*gcmContentCipherBuilder)(nil) + _ ContentCipherBuilderWithContext = (*gcmContentCipherBuilderV2)(nil) +) + +// assert ContentCipher implementations +var ( + _ ContentCipher = (*aesGCMContentCipher)(nil) +) + +// assert awsFixture implementations +var ( + _ awsFixture = (*gcmContentCipherBuilderV2)(nil) +) diff --git a/service/s3/s3crypto/aes_gcm_content_cipher_test.go b/service/s3/s3crypto/aes_gcm_content_cipher_test.go index eb0a1c8baa6..92f31af49ba 100644 --- a/service/s3/s3crypto/aes_gcm_content_cipher_test.go +++ b/service/s3/s3crypto/aes_gcm_content_cipher_test.go @@ -1,22 +1,23 @@ -package s3crypto_test +package s3crypto import ( + "strings" "testing" - "github.com/aws/aws-sdk-go/service/kms/kmsiface" - "github.com/aws/aws-sdk-go/service/s3/s3crypto" + "github.com/aws/aws-sdk-go/awstesting/unit" + "github.com/aws/aws-sdk-go/service/kms" ) func TestAESGCMContentCipherBuilder(t *testing.T) { generator := mockGenerator{} - if builder := s3crypto.AESGCMContentCipherBuilder(generator); builder == nil { + if builder := AESGCMContentCipherBuilder(generator); builder == nil { t.Error("expected non-nil value") } } func TestAESGCMContentCipherNewEncryptor(t *testing.T) { generator := mockGenerator{} - builder := s3crypto.AESGCMContentCipherBuilder(generator) + builder := AESGCMContentCipherBuilder(generator) cipher, err := builder.ContentCipher() if err != nil { @@ -28,6 +29,111 @@ func TestAESGCMContentCipherNewEncryptor(t *testing.T) { } } -type mockKMS struct { - kmsiface.KMSAPI +func TestAESGCMContentCipherBuilderV2(t *testing.T) { + builder := AESGCMContentCipherBuilderV2(mockGeneratorV2{}) + cipher, err := builder.ContentCipher() + + if err != nil { + t.Errorf("expected no error, but received %v", err) + } + + if cipher == nil { + t.Errorf("expected non-nil vaue") + } +} + +func TestGcmContentCipherBuilder_isFixtureEncryptionCompatible(t *testing.T) { + builder := AESGCMContentCipherBuilder(NewKMSKeyGenerator(mockKMS{}, "cmkID")) + features, ok := builder.(compatibleEncryptionFixture) + if !ok { + t.Errorf("expected to implement compatibleEncryptionFixture interface") + } + + if err := features.isEncryptionVersionCompatible(v1ClientVersion); err != nil { + t.Errorf("expected to recieve no error, got %v", err) + } + + if err := features.isEncryptionVersionCompatible(v2ClientVersion); err == nil { + t.Errorf("expected to recieve error, got nil") + } +} + +func TestGcmContentCipherBuilderV2_isFixtureEncryptionCompatible(t *testing.T) { + builder := AESGCMContentCipherBuilderV2(NewKMSContextKeyGenerator(mockKMS{}, "cmkID", nil)) + features, ok := builder.(compatibleEncryptionFixture) + if !ok { + t.Errorf("expected to implement compatibleEncryptionFixture interface") + } + + if err := features.isEncryptionVersionCompatible(v1ClientVersion); err == nil { + t.Error("expected to receive error, got nil") + } + + if err := features.isEncryptionVersionCompatible(v2ClientVersion); err != nil { + t.Errorf("expected to recieve no error, got %v", err) + } +} + +func TestRegisterAESGCMContentCipher(t *testing.T) { + cr := NewCryptoRegistry() + err := RegisterAESGCMContentCipher(cr) + if err != nil { + t.Fatalf("expected no error, got %v", err) + } + + if v, ok := cr.GetCEK("AES/GCM/NoPadding"); !ok { + t.Fatal("expected cek handler to be registered") + } else if v == nil { + t.Fatal("expected non-nil cek handler") + } + + if v, ok := cr.GetPadder("NoPadding"); !ok { + t.Fatal("expected padder to be registered") + } else if v != NoPadder { + t.Fatal("padder did not match expected type") + } + + err = RegisterAESGCMContentCipher(cr) + if err == nil { + t.Fatal("expected error, got none") + } else if !strings.Contains(err.Error(), "duplicate cek registry entry") { + t.Errorf("expected duplicate entry, got %v", err) + } + + if _, ok := cr.RemoveCEK("AES/GCM/NoPadding"); !ok { + t.Error("expected value to be removed") + } + err = RegisterAESGCMContentCipher(cr) + if err != nil { + t.Fatalf("expected no error, got %v", err) + } + + if _, ok := cr.RemoveCEK("AES/GCM/NoPadding"); !ok { + t.Fatalf("expected value to be removed") + } + if _, ok := cr.RemovePadder("NoPadding"); !ok { + t.Fatalf("expected value to be removed") + } + if err := cr.AddPadder("NoPadding", mockPadder{}); err != nil { + t.Fatalf("expected no error, got %v", err) + } + + err = RegisterAESGCMContentCipher(cr) + if err == nil { + t.Fatalf("expected error, got %v", err) + } else if !strings.Contains(err.Error(), "does not match expected type") { + t.Errorf("expected padder type error, got %v", err) + } +} + +func TestAESGCMContentCipherBuilderV2_isAWSFixture(t *testing.T) { + builder := AESGCMContentCipherBuilderV2(NewKMSContextKeyGenerator(kms.New(unit.Session.Copy()), "cmk", nil)) + if !builder.(awsFixture).isAWSFixture() { + t.Error("expected to be AWS ContentCipherBuilder constructed with a AWS CipherDataGenerator") + } + + builder = AESGCMContentCipherBuilderV2(mockGeneratorV2{}) + if builder.(awsFixture).isAWSFixture() { + t.Error("expected to return that this is not an AWS fixture") + } } diff --git a/service/s3/s3crypto/cipher_builder.go b/service/s3/s3crypto/cipher_builder.go index 17b59923860..bc4773eaa50 100644 --- a/service/s3/s3crypto/cipher_builder.go +++ b/service/s3/s3crypto/cipher_builder.go @@ -39,3 +39,10 @@ type CipherData struct { Padder Padder } + +// Clone returns a new copy of CipherData +func (cd CipherData) Clone() (v CipherData) { + v = cd + v.MaterialDescription = cd.MaterialDescription.Clone() + return v +} diff --git a/service/s3/s3crypto/cipher_util.go b/service/s3/s3crypto/cipher_util.go index 64eac775311..73056c3a959 100644 --- a/service/s3/s3crypto/cipher_util.go +++ b/service/s3/s3crypto/cipher_util.go @@ -6,20 +6,18 @@ import ( ) // AESGCMNoPadding is the constant value that is used to specify -// the CEK algorithm consiting of AES GCM with no padding. +// the cek algorithm consiting of AES GCM with no padding. const AESGCMNoPadding = "AES/GCM/NoPadding" // AESCBC is the string constant that signifies the AES CBC algorithm cipher. const AESCBC = "AES/CBC" -func encodeMeta(reader hashReader, cd CipherData) (Envelope, error) { +func encodeMeta(reader lengthReader, cd CipherData) (Envelope, error) { iv := base64.StdEncoding.EncodeToString(cd.IV) key := base64.StdEncoding.EncodeToString(cd.EncryptedKey) - md5 := reader.GetValue() contentLength := reader.GetContentLength() - md5Str := base64.StdEncoding.EncodeToString(md5) matdesc, err := cd.MaterialDescription.encodeDescription() if err != nil { return Envelope{}, err @@ -32,7 +30,6 @@ func encodeMeta(reader hashReader, cd CipherData) (Envelope, error) { WrapAlg: cd.WrapAlgorithm, CEKAlg: cd.CEKAlgorithm, TagLen: cd.TagLength, - UnencryptedMD5: md5Str, UnencryptedContentLen: strconv.FormatInt(contentLength, 10), }, nil } diff --git a/service/s3/s3crypto/cipher_util_test.go b/service/s3/s3crypto/cipher_util_test.go index 5c59160aa97..79b965cee51 100644 --- a/service/s3/s3crypto/cipher_util_test.go +++ b/service/s3/s3crypto/cipher_util_test.go @@ -15,14 +15,13 @@ import ( func TestWrapFactory(t *testing.T) { o := DecryptionClientOptions{ - WrapRegistry: map[string]WrapEntry{ + CryptoRegistry: initCryptoRegistryFrom(map[string]WrapEntry{ KMSWrap: (kmsKeyHandler{ kms: kms.New(unit.Session), }).decryptHandler, - }, - CEKRegistry: map[string]CEKEntry{ + }, map[string]CEKEntry{ AESGCMNoPadding: newAESGCMContentCipher, - }, + }, map[string]Padder{}), } env := Envelope{ WrapAlg: KMSWrap, @@ -43,14 +42,13 @@ func TestWrapFactory(t *testing.T) { } func TestWrapFactoryErrorNoWrap(t *testing.T) { o := DecryptionClientOptions{ - WrapRegistry: map[string]WrapEntry{ + CryptoRegistry: initCryptoRegistryFrom(map[string]WrapEntry{ KMSWrap: (kmsKeyHandler{ kms: kms.New(unit.Session), }).decryptHandler, - }, - CEKRegistry: map[string]CEKEntry{ + }, map[string]CEKEntry{ AESGCMNoPadding: newAESGCMContentCipher, - }, + }, map[string]Padder{}), } env := Envelope{ WrapAlg: "none", @@ -68,14 +66,13 @@ func TestWrapFactoryErrorNoWrap(t *testing.T) { func TestWrapFactoryCustomEntry(t *testing.T) { o := DecryptionClientOptions{ - WrapRegistry: map[string]WrapEntry{ + CryptoRegistry: initCryptoRegistryFrom(map[string]WrapEntry{ "custom": (kmsKeyHandler{ kms: kms.New(unit.Session), }).decryptHandler, - }, - CEKRegistry: map[string]CEKEntry{ + }, map[string]CEKEntry{ AESGCMNoPadding: newAESGCMContentCipher, - }, + }, map[string]Padder{}), } env := Envelope{ WrapAlg: "custom", @@ -108,17 +105,15 @@ func TestCEKFactory(t *testing.T) { }) o := DecryptionClientOptions{ - WrapRegistry: map[string]WrapEntry{ + CryptoRegistry: initCryptoRegistryFrom(map[string]WrapEntry{ KMSWrap: (kmsKeyHandler{ kms: kms.New(sess), }).decryptHandler, - }, - CEKRegistry: map[string]CEKEntry{ + }, map[string]CEKEntry{ AESGCMNoPadding: newAESGCMContentCipher, - }, - PadderRegistry: map[string]Padder{ + }, map[string]Padder{ NoPadder.Name(): NoPadder, - }, + }), } iv, err := hex.DecodeString("0d18e06c7c725ac9e362e1ce") if err != nil { @@ -167,17 +162,18 @@ func TestCEKFactoryNoCEK(t *testing.T) { }) o := DecryptionClientOptions{ - WrapRegistry: map[string]WrapEntry{ - KMSWrap: (kmsKeyHandler{ - kms: kms.New(sess), - }).decryptHandler, - }, - CEKRegistry: map[string]CEKEntry{ - AESGCMNoPadding: newAESGCMContentCipher, - }, - PadderRegistry: map[string]Padder{ - NoPadder.Name(): NoPadder, - }, + CryptoRegistry: initCryptoRegistryFrom( + map[string]WrapEntry{ + KMSWrap: (kmsKeyHandler{ + kms: kms.New(sess), + }).decryptHandler, + }, + map[string]CEKEntry{ + AESGCMNoPadding: newAESGCMContentCipher, + }, + map[string]Padder{ + NoPadder.Name(): NoPadder, + }), } iv, err := hex.DecodeString("0d18e06c7c725ac9e362e1ce") if err != nil { @@ -226,15 +222,14 @@ func TestCEKFactoryCustomEntry(t *testing.T) { }) o := DecryptionClientOptions{ - WrapRegistry: map[string]WrapEntry{ - KMSWrap: (kmsKeyHandler{ - kms: kms.New(sess), - }).decryptHandler, - }, - CEKRegistry: map[string]CEKEntry{ - "custom": newAESGCMContentCipher, - }, - PadderRegistry: map[string]Padder{}, + CryptoRegistry: initCryptoRegistryFrom( + map[string]WrapEntry{ + KMSWrap: (kmsKeyHandler{ + kms: kms.New(sess), + }).decryptHandler, + }, map[string]CEKEntry{ + "custom": newAESGCMContentCipher, + }, map[string]Padder{}), } iv, err := hex.DecodeString("0d18e06c7c725ac9e362e1ce") if err != nil { diff --git a/service/s3/s3crypto/crypto_registry.go b/service/s3/s3crypto/crypto_registry.go new file mode 100644 index 00000000000..ba58df6e928 --- /dev/null +++ b/service/s3/s3crypto/crypto_registry.go @@ -0,0 +1,158 @@ +package s3crypto + +import ( + "fmt" +) + +// CryptoRegistry is a collection of registries for configuring a decryption client with different key wrapping algorithms, +// content encryption algorithms, and padders. +type CryptoRegistry struct { + wrap map[string]WrapEntry + cek map[string]CEKEntry + padder map[string]Padder +} + +// NewCryptoRegistry creates a new CryptoRegistry to which wrapping algorithms, content encryption ciphers, and +// padders can be registered for use with the DecryptionClientV2. +func NewCryptoRegistry() *CryptoRegistry { + return &CryptoRegistry{ + wrap: map[string]WrapEntry{}, + cek: map[string]CEKEntry{}, + padder: map[string]Padder{}, + } +} + +// initCryptoRegistryFrom creates a CryptoRegistry from prepopulated values, this is used for the V1 client +func initCryptoRegistryFrom(wrapRegistry map[string]WrapEntry, cekRegistry map[string]CEKEntry, padderRegistry map[string]Padder) *CryptoRegistry { + cr := &CryptoRegistry{ + wrap: wrapRegistry, + cek: cekRegistry, + padder: padderRegistry, + } + return cr +} + +// GetWrap returns the WrapEntry identified by the given name. Returns false if the entry is not registered. +func (c CryptoRegistry) GetWrap(name string) (WrapEntry, bool) { + if c.wrap == nil { + return nil, false + } + entry, ok := c.wrap[name] + return entry, ok +} + +// AddWrap registers the provided WrapEntry under the given name, returns an error if a WrapEntry is already present +// for the given name. +// +// This method should only be used if you need to register custom wrapping algorithms. Please see the following methods +// for helpers to register AWS provided algorithms: +// RegisterKMSContextWrapWithAnyCMK (kms+context) +// RegisterKMSContextWrapWithCMK (kms+context) +// RegisterKMSWrapWithAnyCMK (kms) +// RegisterKMSWrapWithCMK (kms) +func (c *CryptoRegistry) AddWrap(name string, entry WrapEntry) error { + if entry == nil { + return errNilWrapEntry + } + + if _, ok := c.wrap[name]; ok { + return newErrDuplicateWrapEntry(name) + } + c.wrap[name] = entry + return nil +} + +// RemoveWrap removes the WrapEntry identified by name. If the WrapEntry is not present returns false. +func (c *CryptoRegistry) RemoveWrap(name string) (WrapEntry, bool) { + if c.wrap == nil { + return nil, false + } + entry, ok := c.wrap[name] + if ok { + delete(c.wrap, name) + } + return entry, ok +} + +// GetCEK returns the CEKEntry identified by the given name. Returns false if the entry is not registered. +func (c CryptoRegistry) GetCEK(name string) (CEKEntry, bool) { + if c.cek == nil { + return nil, false + } + entry, ok := c.cek[name] + return entry, ok +} + +// AddCEK registers CEKEntry under the given name, returns an error if a CEKEntry is already present for the given name. +// +// This method should only be used if you need to register custom content encryption algorithms. Please see the following methods +// for helpers to register AWS provided algorithms: +// RegisterAESGCMContentCipher (AES/GCM) +// RegisterAESCBCContentCipher (AES/CBC) +func (c *CryptoRegistry) AddCEK(name string, entry CEKEntry) error { + if entry == nil { + return errNilCEKEntry + } + if _, ok := c.cek[name]; ok { + return newErrDuplicateCEKEntry(name) + } + c.cek[name] = entry + return nil +} + +// RemoveCEK removes the CEKEntry identified by name. If the entry is not present returns false. +func (c *CryptoRegistry) RemoveCEK(name string) (CEKEntry, bool) { + if c.cek == nil { + return nil, false + } + entry, ok := c.cek[name] + if ok { + delete(c.cek, name) + } + return entry, ok +} + +// GetPadder returns the Padder identified by name. If the Padder is not present, returns false. +func (c *CryptoRegistry) GetPadder(name string) (Padder, bool) { + if c.padder == nil { + return nil, false + } + entry, ok := c.padder[name] + return entry, ok +} + +// AddPadder registers Padder under the given name, returns an error if a Padder is already present for the given name. +// +// This method should only be used to register custom padder implementations not provided by AWS. +func (c *CryptoRegistry) AddPadder(name string, padder Padder) error { + if padder == nil { + return errNilPadder + } + if _, ok := c.padder[name]; ok { + return newErrDuplicatePadderEntry(name) + } + c.padder[name] = padder + return nil +} + +// RemovePadder removes the Padder identified by name. If the entry is not present returns false. +func (c *CryptoRegistry) RemovePadder(name string) (Padder, bool) { + if c.padder == nil { + return nil, false + } + padder, ok := c.padder[name] + if ok { + delete(c.padder, name) + } + return padder, ok +} + +func (c CryptoRegistry) valid() error { + if len(c.wrap) == 0 { + return fmt.Errorf("at least one key wrapping algorithms must be provided") + } + if len(c.cek) == 0 { + return fmt.Errorf("at least one content decryption algorithms must be provided") + } + return nil +} diff --git a/service/s3/s3crypto/crypto_registry_test.go b/service/s3/s3crypto/crypto_registry_test.go new file mode 100644 index 00000000000..c98c5636db8 --- /dev/null +++ b/service/s3/s3crypto/crypto_registry_test.go @@ -0,0 +1,150 @@ +package s3crypto + +import ( + "strings" + "testing" +) + +func TestCryptoRegistry_Wrap(t *testing.T) { + cr := NewCryptoRegistry() + + mockWrap := WrapEntry(func(envelope Envelope) (CipherDataDecrypter, error) { + return nil, nil + }) + + if _, ok := cr.GetWrap("foo"); ok { + t.Errorf("expected wrapper to not be present") + } + + if _, ok := cr.RemoveWrap("foo"); ok { + t.Errorf("expected wrapped to not have been removed") + } + + if err := cr.AddWrap("foo", nil); err == nil { + t.Errorf("expected error, got none") + } + + if err := cr.AddWrap("foo", mockWrap); err != nil { + t.Errorf("expected no error, got %v", err) + } + + if err := cr.AddWrap("foo", mockWrap); err == nil { + t.Error("expected error, got none") + } + + if v, ok := cr.GetWrap("foo"); !ok || v == nil { + t.Error("expected wrapper to be present and not nil") + } + + if v, ok := cr.RemoveWrap("foo"); !ok || v == nil { + t.Error("expected wrapper to have been removed and not nil") + } + + if _, ok := cr.GetWrap("foo"); ok { + t.Error("expected wrapper to have been removed and not nil") + } +} + +func TestCryptoRegistry_CEK(t *testing.T) { + cr := NewCryptoRegistry() + + mockEntry := CEKEntry(func(data CipherData) (ContentCipher, error) { + return nil, nil + }) + + if _, ok := cr.GetCEK("foo"); ok { + t.Errorf("expected wrapper to not be present") + } + + if _, ok := cr.RemoveCEK("foo"); ok { + t.Errorf("expected wrapped to not have been removed") + } + + if err := cr.AddCEK("foo", nil); err == nil { + t.Errorf("expected error, got none") + } + + if err := cr.AddCEK("foo", mockEntry); err != nil { + t.Errorf("expected no error, got %v", err) + } + + if err := cr.AddCEK("foo", mockEntry); err == nil { + t.Error("expected error, got none") + } + + if v, ok := cr.GetCEK("foo"); !ok || v == nil { + t.Error("expected wrapper to be present and not nil") + } + + if v, ok := cr.RemoveCEK("foo"); !ok || v == nil { + t.Error("expected wrapper to have been removed and not nil") + } + + if _, ok := cr.GetCEK("foo"); ok { + t.Error("expected wrapper to have been removed and not nil") + } +} + +func TestCryptoRegistry_Padder(t *testing.T) { + cr := NewCryptoRegistry() + + padder := &mockPadder{} + + if _, ok := cr.GetPadder("foo"); ok { + t.Errorf("expected wrapper to not be present") + } + + if _, ok := cr.RemovePadder("foo"); ok { + t.Errorf("expected wrapped to not have been removed") + } + + if err := cr.AddPadder("foo", nil); err == nil { + t.Errorf("expected error, got none") + } + + if err := cr.AddPadder("foo", padder); err != nil { + t.Errorf("expected no error, got %v", err) + } + + if err := cr.AddPadder("foo", padder); err == nil { + t.Error("expected error, got none") + } + + if v, ok := cr.GetPadder("foo"); !ok || v == nil { + t.Error("expected wrapper to be present and not nil") + } + + if v, ok := cr.RemovePadder("foo"); !ok || v == nil { + t.Error("expected wrapper to have been removed and not nil") + } +} + +func TestCryptoRegistry_valid(t *testing.T) { + cr := NewCryptoRegistry() + + if err := cr.valid(); err == nil { + t.Errorf("expected error, got none") + } else if e, a := "at least one key wrapping algorithms must be provided", err.Error(); !strings.Contains(a, e) { + t.Errorf("expected %v, got %v", e, a) + } + + if err := cr.AddWrap("foo", func(envelope Envelope) (CipherDataDecrypter, error) { + return nil, nil + }); err != nil { + t.Fatalf("expected no error, got %v", err) + } + if err := cr.valid(); err == nil { + t.Fatalf("expected error, got none") + } else if e, a := "least one content decryption algorithms must be provided", err.Error(); !strings.Contains(a, e) { + t.Errorf("expected %v, got %v", e, a) + } + + if err := cr.AddCEK("foo", func(data CipherData) (ContentCipher, error) { + return nil, nil + }); err != nil { + t.Fatalf("expected no error, got %v", err) + } + if err := cr.valid(); err != nil { + t.Fatalf("expected no error, got %v", err) + } +} diff --git a/service/s3/s3crypto/decryption_client.go b/service/s3/s3crypto/decryption_client.go index 7e1c8cdc60c..5911b349d18 100644 --- a/service/s3/s3crypto/decryption_client.go +++ b/service/s3/s3crypto/decryption_client.go @@ -1,8 +1,6 @@ package s3crypto import ( - "strings" - "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/client" "github.com/aws/aws-sdk-go/aws/request" @@ -14,7 +12,7 @@ import ( // WrapEntry is builder that return a proper key decrypter and error type WrapEntry func(Envelope) (CipherDataDecrypter, error) -// CEKEntry is a builder thatn returns a proper content decrypter and error +// CEKEntry is a builder that returns a proper content decrypter and error type CEKEntry func(CipherData) (ContentCipher, error) // DecryptionClient is an S3 crypto client. The decryption client @@ -26,7 +24,7 @@ type CEKEntry func(CipherData) (ContentCipher, error) // * AES/GCM // * AES/CBC // -// deprecated: See DecryptionClientV2 +// deprecated: This feature is in maintenance mode, no new updates will be released. Please see https://docs.aws.amazon.com/general/latest/gr/aws_sdk_cryptography.html for more information. type DecryptionClient struct { S3Client s3iface.S3API // LoadStrategy is used to load the metadata either from the metadata of the object @@ -43,36 +41,36 @@ type DecryptionClient struct { // NewDecryptionClient instantiates a new S3 crypto client // // Example: -// sess := session.New() +// sess := session.Must(session.NewSession()) // svc := s3crypto.NewDecryptionClient(sess, func(svc *s3crypto.DecryptionClient{ // // Custom client options here // })) // -// deprecated: see NewDecryptionClientV2 +// deprecated: This feature is in maintenance mode, no new updates will be released. Please see https://docs.aws.amazon.com/general/latest/gr/aws_sdk_cryptography.html for more information. func NewDecryptionClient(prov client.ConfigProvider, options ...func(*DecryptionClient)) *DecryptionClient { s3client := s3.New(prov) s3client.Handlers.Build.PushBack(func(r *request.Request) { - request.AddToUserAgent(r, "S3Crypto") + request.AddToUserAgent(r, "S3CryptoV1n") }) + kmsClient := kms.New(prov) client := &DecryptionClient{ S3Client: s3client, LoadStrategy: defaultV2LoadStrategy{ client: s3client, }, WrapRegistry: map[string]WrapEntry{ - KMSWrap: (kmsKeyHandler{ - kms: kms.New(prov), - }).decryptHandler, + KMSWrap: NewKMSWrapEntry(kmsClient), + KMSContextWrap: newKMSContextWrapEntryWithAnyCMK(kmsClient), }, CEKRegistry: map[string]CEKEntry{ - AESGCMNoPadding: newAESGCMContentCipher, - strings.Join([]string{AESCBC, AESCBCPadder.Name()}, "/"): newAESCBCContentCipher, + AESGCMNoPadding: newAESGCMContentCipher, + AESCBC + "/" + AESCBCPadder.Name(): newAESCBCContentCipher, }, PadderRegistry: map[string]Padder{ - strings.Join([]string{AESCBC, AESCBCPadder.Name()}, "/"): AESCBCPadder, - "NoPadding": NoPadder, + AESCBC + "/" + AESCBCPadder.Name(): AESCBCPadder, + NoPadder.Name(): NoPadder, }, } for _, option := range options { @@ -94,14 +92,14 @@ func NewDecryptionClient(prov client.ConfigProvider, options ...func(*Decryption // }) // err := req.Send() // -// deprecated: see DecryptionClientV2.GetObjectRequest +// deprecated: This feature is in maintenance mode, no new updates will be released. Please see https://docs.aws.amazon.com/general/latest/gr/aws_sdk_cryptography.html for more information. func (c *DecryptionClient) GetObjectRequest(input *s3.GetObjectInput) (*request.Request, *s3.GetObjectOutput) { return getObjectRequest(c.getClientOptions(), input) } // GetObject is a wrapper for GetObjectRequest // -// deprecated: see DecryptionClientV2.GetObject +// deprecated: This feature is in maintenance mode, no new updates will be released. Please see https://docs.aws.amazon.com/general/latest/gr/aws_sdk_cryptography.html for more information. func (c *DecryptionClient) GetObject(input *s3.GetObjectInput) (*s3.GetObjectOutput, error) { return getObject(c.getClientOptions(), input) } @@ -114,7 +112,7 @@ func (c *DecryptionClient) GetObject(input *s3.GetObjectInput) (*s3.GetObjectOut // cause a panic. Use the Context to add deadlining, timeouts, etc. In the future // this may create sub-contexts for individual underlying requests. // -// deprecated: see DecryptionClientV2.GetObjectWithContext +// deprecated: This feature is in maintenance mode, no new updates will be released. Please see https://docs.aws.amazon.com/general/latest/gr/aws_sdk_cryptography.html for more information. func (c *DecryptionClient) GetObjectWithContext(ctx aws.Context, input *s3.GetObjectInput, opts ...request.Option) (*s3.GetObjectOutput, error) { return getObjectWithContext(c.getClientOptions(), ctx, input, opts...) } @@ -123,8 +121,6 @@ func (c *DecryptionClient) getClientOptions() DecryptionClientOptions { return DecryptionClientOptions{ S3Client: c.S3Client, LoadStrategy: c.LoadStrategy, - WrapRegistry: c.WrapRegistry, - CEKRegistry: c.CEKRegistry, - PadderRegistry: c.PadderRegistry, + CryptoRegistry: initCryptoRegistryFrom(c.WrapRegistry, c.CEKRegistry, c.PadderRegistry), } } diff --git a/service/s3/s3crypto/decryption_client_test.go b/service/s3/s3crypto/decryption_client_test.go index 50d421b5880..909d577701c 100644 --- a/service/s3/s3crypto/decryption_client_test.go +++ b/service/s3/s3crypto/decryption_client_test.go @@ -70,8 +70,6 @@ func TestGetObjectGCM(t *testing.T) { }, Body: ioutil.NopCloser(bytes.NewBuffer(b)), } - out.Metadata = make(map[string]*string) - out.Metadata["x-amz-wrap-alg"] = aws.String(s3crypto.KMSWrap) }) err := req.Send() if err != nil { @@ -139,8 +137,6 @@ func TestGetObjectCBC(t *testing.T) { }, Body: ioutil.NopCloser(bytes.NewBuffer(b)), } - out.Metadata = make(map[string]*string) - out.Metadata["x-amz-wrap-alg"] = aws.String(s3crypto.KMSWrap) }) err := req.Send() if err != nil { @@ -204,8 +200,6 @@ func TestGetObjectCBC2(t *testing.T) { }, Body: ioutil.NopCloser(bytes.NewBuffer(b)), } - out.Metadata = make(map[string]*string) - out.Metadata["x-amz-wrap-alg"] = aws.String(s3crypto.KMSWrap) }) err := req.Send() if err != nil { @@ -248,3 +242,56 @@ func TestGetObjectWithContext(t *testing.T) { t.Errorf("expected error message to contain %q, but did not %q", e, a) } } + +func TestDecryptionClient_GetObject_V2Artifact(t *testing.T) { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Fprintln(w, fmt.Sprintf("%s%s%s", `{"KeyId":"test-key-id","Plaintext":"`, "hJUv7S6K2cHF64boS9ixHX0TZAjBZLT4ZpEO4XxkGnY=", `"}`)) + })) + defer ts.Close() + + c := s3crypto.NewDecryptionClient(unit.Session.Copy(&aws.Config{Endpoint: &ts.URL})) + + input := &s3.GetObjectInput{ + Bucket: aws.String("test"), + Key: aws.String("test"), + } + + req, out := c.GetObjectRequest(input) + req.Handlers.Send.Clear() + req.Handlers.Send.PushBack(func(r *request.Request) { + b, err := hex.DecodeString("6b134eb7a353131de92faff64f594b2794e3544e31776cca26fe3bbeeffc68742d1007234f11c6670522602326868e29f37e9d2678f1614ec1a2418009b9772100929aadbed9a21a") + if err != nil { + t.Errorf("expected no error, but received %v", err) + } + + r.HTTPResponse = &http.Response{ + StatusCode: 200, + Header: http.Header{ + http.CanonicalHeaderKey("x-amz-meta-x-amz-key-v2"): []string{"PsuclPnlo2O0MQoov6kL1TBlaZG6oyNwWuAqmAgq7g8b9ZeeORi3VTMg624FU9jx"}, + http.CanonicalHeaderKey("x-amz-meta-x-amz-iv"): []string{"dqqlq2dRVSQ5hFRb"}, + http.CanonicalHeaderKey("x-amz-meta-x-amz-matdesc"): []string{`{"aws:x-amz-cek-alg": "AES/GCM/NoPadding"}`}, + http.CanonicalHeaderKey("x-amz-meta-x-amz-wrap-alg"): []string{s3crypto.KMSContextWrap}, + http.CanonicalHeaderKey("x-amz-meta-x-amz-cek-alg"): []string{"AES/GCM/NoPadding"}, + }, + Body: ioutil.NopCloser(bytes.NewBuffer(b)), + } + }) + err := req.Send() + if err != nil { + t.Fatalf("expected no error, got %v", err) + } + + actual, err := ioutil.ReadAll(out.Body) + if err != nil { + t.Fatalf("expected no error, got %v", err) + } + + expected, err := hex.DecodeString("af150d7156bf5b3f5c461e5c6ac820acc5a33aab7085d920666c250ff251209d5a4029b3bd78250fab6e11aed52fae948d407056a9519b68") + if err != nil { + t.Fatalf("expected no error, got %v", err) + } + + if bytes.Compare(expected, actual) != 0 { + t.Fatalf("expected content to match but it did not") + } +} diff --git a/service/s3/s3crypto/decryption_client_v2.go b/service/s3/s3crypto/decryption_client_v2.go index be37bd4bbc6..307c33b7e6f 100644 --- a/service/s3/s3crypto/decryption_client_v2.go +++ b/service/s3/s3crypto/decryption_client_v2.go @@ -1,12 +1,9 @@ package s3crypto import ( - "strings" - "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/client" "github.com/aws/aws-sdk-go/aws/request" - "github.com/aws/aws-sdk-go/service/kms" "github.com/aws/aws-sdk-go/service/s3" "github.com/aws/aws-sdk-go/service/s3/s3iface" ) @@ -33,58 +30,61 @@ type DecryptionClientOptions struct { // Defaults to our default load strategy. LoadStrategy LoadStrategy - WrapRegistry map[string]WrapEntry - CEKRegistry map[string]CEKEntry - PadderRegistry map[string]Padder + CryptoRegistry *CryptoRegistry } -// NewDecryptionClientV2 instantiates a new V2 S3 crypto client. The returned DecryptionClientV2 will be able to decrypt -// object encrypted by both the V1 and V2 clients. +// NewDecryptionClientV2 instantiates a new DecryptionClientV2. The NewDecryptionClientV2 must be configured with the +// desired key wrapping and content encryption algorithms that are required to be read by the client. These algorithms +// are registered by providing the client a CryptoRegistry that has been constructed with the desired configuration. +// NewDecryptionClientV2 will return an error if no key wrapping or content encryption algorithms have been provided. // // Example: // sess := session.Must(session.NewSession()) -// svc := s3crypto.NewDecryptionClientV2(sess, func(svc *s3crypto.DecryptionClientOptions{ +// cr := s3crypto.NewCryptoRegistry() +// if err := s3crypto.RegisterKMSContextWrapWithAnyCMK(cr, kms.New(sess)); err != nil { +// panic(err) // handle error +// } +// if err := s3crypto.RegisterAESGCMContentCipher(cr); err != nil { +// panic(err) // handle error +// } +// svc, err := s3crypto.NewDecryptionClientV2(sess, cr, func(o *s3crypto.DecryptionClientOptions) { // // Custom client options here -// })) -func NewDecryptionClientV2(prov client.ConfigProvider, options ...func(clientOptions *DecryptionClientOptions)) *DecryptionClientV2 { +// }) +// if err != nil { +// panic(err) // handle error +// } +func NewDecryptionClientV2( + prov client.ConfigProvider, cryptoRegistry *CryptoRegistry, + options ...func(clientOptions *DecryptionClientOptions), +) (*DecryptionClientV2, error) { s3client := s3.New(prov) s3client.Handlers.Build.PushBack(func(r *request.Request) { request.AddToUserAgent(r, "S3CryptoV2") }) - kmsClient := kms.New(prov) clientOptions := &DecryptionClientOptions{ S3Client: s3client, LoadStrategy: defaultV2LoadStrategy{ client: s3client, }, - WrapRegistry: map[string]WrapEntry{ - KMSWrap: NewKMSWrapEntry(kmsClient), - KMSContextWrap: NewKMSContextWrapEntry(kmsClient), - }, - CEKRegistry: map[string]CEKEntry{ - AESGCMNoPadding: newAESGCMContentCipher, - strings.Join([]string{AESCBC, AESCBCPadder.Name()}, "/"): newAESCBCContentCipher, - }, - PadderRegistry: map[string]Padder{ - strings.Join([]string{AESCBC, AESCBCPadder.Name()}, "/"): AESCBCPadder, - "NoPadding": NoPadder, - }, + CryptoRegistry: cryptoRegistry, } for _, option := range options { option(clientOptions) } - return &DecryptionClientV2{options: *clientOptions} + if err := cryptoRegistry.valid(); err != nil { + return nil, err + } + + return &DecryptionClientV2{options: *clientOptions}, nil } // GetObjectRequest will make a request to s3 and retrieve the object. In this process // decryption will be done. The SDK only supports V2 reads of KMS and GCM. // // Example: -// sess := session.Must(session.NewSession()) -// svc := s3crypto.NewDecryptionClientV2(sess) // req, out := svc.GetObjectRequest(&s3.GetObjectInput { // Key: aws.String("testKey"), // Bucket: aws.String("testBucket"), diff --git a/service/s3/s3crypto/decryption_client_v2_test.go b/service/s3/s3crypto/decryption_client_v2_test.go index 1261511367e..218025f3493 100644 --- a/service/s3/s3crypto/decryption_client_v2_test.go +++ b/service/s3/s3crypto/decryption_client_v2_test.go @@ -1,16 +1,28 @@ +// +build go1.7 + package s3crypto_test import ( + "bytes" + "encoding/hex" + "fmt" + "io/ioutil" + "net/http" + "net/http/httptest" + "strings" "testing" + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/request" "github.com/aws/aws-sdk-go/awstesting/unit" "github.com/aws/aws-sdk-go/service/kms" + "github.com/aws/aws-sdk-go/service/s3" "github.com/aws/aws-sdk-go/service/s3/s3crypto" ) func TestDecryptionClientV2_CheckDeprecatedFeatures(t *testing.T) { // AES/GCM/NoPadding with kms+context => allowed - builder := s3crypto.AESGCMContentCipherBuilder(s3crypto.NewKMSContextKeyGenerator(kms.New(unit.Session), "cmkID")) + builder := s3crypto.AESGCMContentCipherBuilderV2(s3crypto.NewKMSContextKeyGenerator(kms.New(unit.Session), "cmkID", s3crypto.MaterialDescription{})) _, err := s3crypto.NewEncryptionClientV2(unit.Session, builder) if err != nil { t.Errorf("expected no error, got %v", err) @@ -29,11 +41,294 @@ func TestDecryptionClientV2_CheckDeprecatedFeatures(t *testing.T) { if err == nil { t.Error("expected error, but got nil") } +} - // AES/CBC/PKCS5Padding with kms+context => not allowed - builder = s3crypto.AESCBCContentCipherBuilder(s3crypto.NewKMSContextKeyGenerator(kms.New(unit.Session), "cmkID"), s3crypto.NewPKCS7Padder(128)) - _, err = s3crypto.NewEncryptionClientV2(unit.Session, builder) +func TestDecryptionClientV2_GetObject(t *testing.T) { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Fprintln(w, fmt.Sprintf("%s%s%s", `{"KeyId":"test-key-id","Plaintext":"`, "hJUv7S6K2cHF64boS9ixHX0TZAjBZLT4ZpEO4XxkGnY=", `"}`)) + })) + defer ts.Close() + + kmsClient := kms.New(unit.Session.Copy(&aws.Config{Endpoint: &ts.URL})) + + cr := s3crypto.NewCryptoRegistry() + if err := s3crypto.RegisterKMSContextWrapWithAnyCMK(cr, kmsClient); err != nil { + t.Fatalf("expected no error, got %v", err) + } + if err := s3crypto.RegisterAESGCMContentCipher(cr); err != nil { + t.Fatalf("expected no error, got %v", err) + } + + c, err := s3crypto.NewDecryptionClientV2(unit.Session.Copy(), cr) + if err != nil { + t.Fatalf("expected no error, got %v", err) + } + + input := &s3.GetObjectInput{ + Bucket: aws.String("test"), + Key: aws.String("test"), + } + + req, out := c.GetObjectRequest(input) + req.Handlers.Send.Clear() + req.Handlers.Send.PushBack(func(r *request.Request) { + b, err := hex.DecodeString("6b134eb7a353131de92faff64f594b2794e3544e31776cca26fe3bbeeffc68742d1007234f11c6670522602326868e29f37e9d2678f1614ec1a2418009b9772100929aadbed9a21a") + if err != nil { + t.Errorf("expected no error, but received %v", err) + } + + r.HTTPResponse = &http.Response{ + StatusCode: 200, + Header: http.Header{ + http.CanonicalHeaderKey("x-amz-meta-x-amz-key-v2"): []string{"PsuclPnlo2O0MQoov6kL1TBlaZG6oyNwWuAqmAgq7g8b9ZeeORi3VTMg624FU9jx"}, + http.CanonicalHeaderKey("x-amz-meta-x-amz-iv"): []string{"dqqlq2dRVSQ5hFRb"}, + http.CanonicalHeaderKey("x-amz-meta-x-amz-matdesc"): []string{`{"aws:x-amz-cek-alg":"AES/GCM/NoPadding"}`}, + http.CanonicalHeaderKey("x-amz-meta-x-amz-wrap-alg"): []string{s3crypto.KMSContextWrap}, + http.CanonicalHeaderKey("x-amz-meta-x-amz-cek-alg"): []string{"AES/GCM/NoPadding"}, + }, + Body: ioutil.NopCloser(bytes.NewBuffer(b)), + } + }) + err = req.Send() + if err != nil { + t.Fatalf("expected no error, got %v", err) + } + + actual, err := ioutil.ReadAll(out.Body) + if err != nil { + t.Fatalf("expected no error, got %v", err) + } + + expected, err := hex.DecodeString("af150d7156bf5b3f5c461e5c6ac820acc5a33aab7085d920666c250ff251209d5a4029b3bd78250fab6e11aed52fae948d407056a9519b68") + if err != nil { + t.Fatalf("expected no error, got %v", err) + } + + if bytes.Compare(expected, actual) != 0 { + t.Fatalf("expected content to match but it did not") + } +} + +func TestDecryptionClientV2_GetObject_V1Interop_KMS_AESCBC(t *testing.T) { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Fprintln(w, fmt.Sprintf("%s%s%s", `{"KeyId":"test-key-id","Plaintext":"`, "7ItX9CTGNWWegC62RlaNu6EJ3+J9yGO7yAqDNU4CdeA=", `"}`)) + })) + defer ts.Close() + + kmsClient := kms.New(unit.Session.Copy(&aws.Config{Endpoint: &ts.URL})) + + cr := s3crypto.NewCryptoRegistry() + if err := s3crypto.RegisterKMSWrapWithAnyCMK(cr, kmsClient); err != nil { + t.Fatalf("expected no error, got %v", err) + } + if err := s3crypto.RegisterAESCBCContentCipher(cr, s3crypto.AESCBCPadder); err != nil { + t.Fatalf("expected no error, got %v", err) + } + + c, err := s3crypto.NewDecryptionClientV2(unit.Session.Copy(), cr) + if err != nil { + t.Fatalf("expected no error, got %v", err) + } + + input := &s3.GetObjectInput{ + Bucket: aws.String("test"), + Key: aws.String("test"), + } + + req, out := c.GetObjectRequest(input) + req.Handlers.Send.Clear() + req.Handlers.Send.PushBack(func(r *request.Request) { + b, err := hex.DecodeString("6f4f413a357a3c3a12289442fb835c5e4ecc8db1d86d3d1eab906ce07e1ad772180b2e9ec49c3fc667d8aceea8c46da6bb9738251a8e36241a473ad820f99c701906bac1f48578d5392e928889bbb1d9") + if err != nil { + t.Errorf("expected no error, but received %v", err) + } + + r.HTTPResponse = &http.Response{ + StatusCode: 200, + Header: http.Header{ + http.CanonicalHeaderKey("x-amz-meta-x-amz-key-v2"): []string{"/nJlgMtxMNk2ErKLLrLp3H7A7aQyJcJOClE2ldAIIFNZU4OhUMc1mMCHdIEC8fby"}, + http.CanonicalHeaderKey("x-amz-meta-x-amz-iv"): []string{"adO9U7pcEHxUTaguIkho9g=="}, + http.CanonicalHeaderKey("x-amz-meta-x-amz-matdesc"): []string{`{"kms_cmk_id":"test-key-id"}`}, + http.CanonicalHeaderKey("x-amz-meta-x-amz-wrap-alg"): []string{s3crypto.KMSWrap}, + http.CanonicalHeaderKey("x-amz-meta-x-amz-cek-alg"): []string{"AES/CBC/PKCS5Padding"}, + }, + Body: ioutil.NopCloser(bytes.NewBuffer(b)), + } + }) + err = req.Send() + if err != nil { + t.Fatalf("expected no error, got %v", err) + } + + actual, err := ioutil.ReadAll(out.Body) + if err != nil { + t.Fatalf("expected no error, got %v", err) + } + + expected, err := hex.DecodeString("a716e018ffecf4bb94d4352082af4662612d9c225efed6f389bf1f6f0447a9bce80cc712d7e66ee5e1c086af38e607ead351fd2c1a0247878e693ada73bd580b") + if err != nil { + t.Fatalf("expected no error, got %v", err) + } + + if bytes.Compare(expected, actual) != 0 { + t.Fatalf("expected content to match but it did not") + } +} + +func TestDecryptionClientV2_GetObject_V1Interop_KMS_AESGCM(t *testing.T) { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Fprintln(w, fmt.Sprintf("%s%s%s", `{"KeyId":"test-key-id","Plaintext":"`, "Hrjrkkt/vQwMYtqvK6+MiXh3xiMvviL1Ks7w2mgsJgU=", `"}`)) + })) + defer ts.Close() + + kmsClient := kms.New(unit.Session.Copy(&aws.Config{Endpoint: &ts.URL})) + + cr := s3crypto.NewCryptoRegistry() + if err := s3crypto.RegisterKMSWrapWithAnyCMK(cr, kmsClient); err != nil { + t.Fatalf("expected no error, got %v", err) + } + if err := s3crypto.RegisterAESGCMContentCipher(cr); err != nil { + t.Fatalf("expected no error, got %v", err) + } + + c, err := s3crypto.NewDecryptionClientV2(unit.Session.Copy(), cr) + if err != nil { + t.Fatalf("expected no error, got %v", err) + } + + input := &s3.GetObjectInput{ + Bucket: aws.String("test"), + Key: aws.String("test"), + } + + req, out := c.GetObjectRequest(input) + req.Handlers.Send.Clear() + req.Handlers.Send.PushBack(func(r *request.Request) { + b, err := hex.DecodeString("6370a90b9a118301c2160c23a90d96146761276acdcfa92e6cbcb783abdc2e1813891506d6850754ef87ed2ac3bf570dd5c9da9492b7769ae1e639d073d688bd284815404ce2648a") + if err != nil { + t.Errorf("expected no error, but received %v", err) + } + + r.HTTPResponse = &http.Response{ + StatusCode: 200, + Header: http.Header{ + http.CanonicalHeaderKey("x-amz-meta-x-amz-key-v2"): []string{"/7tu/RFXZU1UFwRzzf11IdF3b1wBxBZhnUMjVYHKKr5DjAHS602GvXt4zYcx/MJo"}, + http.CanonicalHeaderKey("x-amz-meta-x-amz-iv"): []string{"8Rlvyy8AoYj8v579"}, + http.CanonicalHeaderKey("x-amz-meta-x-amz-matdesc"): []string{`{"kms_cmk_id":"test-key-id"}`}, + http.CanonicalHeaderKey("x-amz-meta-x-amz-wrap-alg"): []string{s3crypto.KMSWrap}, + http.CanonicalHeaderKey("x-amz-meta-x-amz-cek-alg"): []string{"AES/GCM/NoPadding"}, + }, + Body: ioutil.NopCloser(bytes.NewBuffer(b)), + } + }) + err = req.Send() + if err != nil { + t.Fatalf("expected no error, got %v", err) + } + + actual, err := ioutil.ReadAll(out.Body) + if err != nil { + t.Fatalf("expected no error, got %v", err) + } + + expected, err := hex.DecodeString("75f6805afa7d7be4f56c5906adc27a5959158bf4af6e7c7e12bda3458300f6b1c8daaf9a5949f7a6bdbb8a9c072de05bf0541633421f42f8") + if err != nil { + t.Fatalf("expected no error, got %v", err) + } + + if bytes.Compare(expected, actual) != 0 { + t.Fatalf("expected content to match but it did not") + } +} + +func TestDecryptionClientV2_GetObject_OnlyDecryptsRegisteredAlgorithms(t *testing.T) { + dataHandler := func(r *request.Request) { + b, err := hex.DecodeString("1bd0271b25951fdef3dbe51a9b7af85f66b311e091aa10a346655068f657b9da9acc0843ea0522b0d1ae4a25a31b13605dd1ac5d002db8965d9d4652fd602693") + if err != nil { + t.Errorf("expected no error, but received %v", err) + } + + r.HTTPResponse = &http.Response{ + StatusCode: 200, + Header: http.Header{ + http.CanonicalHeaderKey("x-amz-meta-x-amz-key-v2"): []string{"gNuYjzkLTzfhOcIX9h1l8jApWcAAQqzlryOE166kdDojaHH/+7cCqR5HU8Bpxmij"}, + http.CanonicalHeaderKey("x-amz-meta-x-amz-iv"): []string{"Vmauu+TMEgaXa26ObqpARA=="}, + http.CanonicalHeaderKey("x-amz-meta-x-amz-matdesc"): []string{`{"kms_cmk_id":"test-key-id"}`}, + http.CanonicalHeaderKey("x-amz-meta-x-amz-wrap-alg"): []string{s3crypto.KMSWrap}, + http.CanonicalHeaderKey("x-amz-meta-x-amz-cek-alg"): []string{"AES/CBC/PKCS5Padding"}, + }, + Body: ioutil.NopCloser(bytes.NewBuffer(b)), + } + } + + cases := map[string]struct { + Client *s3crypto.DecryptionClientV2 + WantErr string + }{ + "unsupported wrap": { + Client: func() *s3crypto.DecryptionClientV2 { + cr := s3crypto.NewCryptoRegistry() + if err := s3crypto.RegisterKMSContextWrapWithAnyCMK(cr, kms.New(unit.Session.Copy())); err != nil { + t.Fatalf("expected no error, got %v", err) + } + if err := s3crypto.RegisterAESGCMContentCipher(cr); err != nil { + t.Fatalf("expected no error, got %v", err) + } + + c, err := s3crypto.NewDecryptionClientV2(unit.Session.Copy(), cr) + if err != nil { + t.Fatalf("expected no error, got %v", err) + } + return c + }(), + WantErr: "wrap algorithm isn't supported, kms", + }, + "unsupported cek": { + Client: func() *s3crypto.DecryptionClientV2 { + cr := s3crypto.NewCryptoRegistry() + if err := s3crypto.RegisterKMSWrapWithAnyCMK(cr, kms.New(unit.Session.Copy())); err != nil { + t.Fatalf("expected no error, got %v", err) + } + if err := s3crypto.RegisterAESGCMContentCipher(cr); err != nil { + t.Fatalf("expected no error, got %v", err) + } + c, err := s3crypto.NewDecryptionClientV2(unit.Session.Copy(), cr) + if err != nil { + t.Fatalf("expected no error, got %v", err) + } + return c + }(), + WantErr: "cek algorithm isn't supported, AES/CBC/PKCS5Padding", + }, + } + + for name, tt := range cases { + t.Run(name, func(t *testing.T) { + input := &s3.GetObjectInput{ + Bucket: aws.String("test"), + Key: aws.String("test"), + } + req, _ := tt.Client.GetObjectRequest(input) + req.Handlers.Send.Clear() + req.Handlers.Send.PushBack(dataHandler) + err := req.Send() + if err == nil { + t.Fatalf("expected error, got none") + } + if e, a := tt.WantErr, err.Error(); !strings.Contains(a, e) { + t.Errorf("expected %v, got %v", e, a) + } + }) + } +} + +func TestDecryptionClientV2_CheckValidCryptoRegistry(t *testing.T) { + cr := s3crypto.NewCryptoRegistry() + _, err := s3crypto.NewDecryptionClientV2(unit.Session.Copy(), cr) if err == nil { - t.Error("expected error, but got nil") + t.Fatal("expected error, got none") + } + if e, a := "at least one key wrapping algorithms must be provided", err.Error(); !strings.Contains(a, e) { + t.Fatalf("expected %v, got %v", e, a) } } diff --git a/service/s3/s3crypto/deprecations.go b/service/s3/s3crypto/deprecations.go deleted file mode 100644 index 8c54aa96682..00000000000 --- a/service/s3/s3crypto/deprecations.go +++ /dev/null @@ -1,10 +0,0 @@ -package s3crypto - -import "fmt" - -var errDeprecatedCipherBuilder = fmt.Errorf("attempted to use deprecated cipher builder") -var errDeprecatedCipherDataGenerator = fmt.Errorf("attempted to use deprecated cipher data generator") - -type deprecatedFeatures interface { - isUsingDeprecatedFeatures() error -} diff --git a/service/s3/s3crypto/deprecations_test.go b/service/s3/s3crypto/deprecations_test.go deleted file mode 100644 index 6a9bb0c9949..00000000000 --- a/service/s3/s3crypto/deprecations_test.go +++ /dev/null @@ -1,51 +0,0 @@ -package s3crypto - -import ( - "testing" - - "github.com/aws/aws-sdk-go/service/kms/kmsiface" -) - -type mockKMS struct { - kmsiface.KMSAPI -} - -func TestAESGCMContentCipherBuilder_isUsingDeprecatedFeatures(t *testing.T) { - builder := AESGCMContentCipherBuilder(NewKMSKeyGenerator(mockKMS{}, "cmkID")) - - features, ok := builder.(deprecatedFeatures) - if !ok { - t.Errorf("expected to implement deprecatedFeatures interface") - } - - err := features.isUsingDeprecatedFeatures() - if err == nil { - t.Errorf("expected to recieve error for using deprecated features") - } - - builder = AESGCMContentCipherBuilder(NewKMSContextKeyGenerator(mockKMS{}, "cmkID")) - - features, ok = builder.(deprecatedFeatures) - if !ok { - t.Errorf("expected to implement deprecatedFeatures interface") - } - - err = features.isUsingDeprecatedFeatures() - if err != nil { - t.Errorf("expected no error, got %v", err) - } -} - -func TestAESCBCContentCipherBuilder_isUsingDeprecatedFeatures(t *testing.T) { - builder := AESCBCContentCipherBuilder(nil, nil) - - features, ok := builder.(deprecatedFeatures) - if !ok { - t.Errorf("expected to implement deprecatedFeatures interface") - } - - err := features.isUsingDeprecatedFeatures() - if err == nil { - t.Errorf("expected to recieve error for using deprecated features") - } -} diff --git a/service/s3/s3crypto/doc.go b/service/s3/s3crypto/doc.go index 305b7e517df..55c639b71af 100644 --- a/service/s3/s3crypto/doc.go +++ b/service/s3/s3crypto/doc.go @@ -16,20 +16,41 @@ Creating an S3 cryptography client cmkID := "" sess := session.Must(session.NewSession()) + kmsClient := kms.New(sess) // Create the KeyProvider - handler := s3crypto.NewKMSContextKeyGenerator(kms.New(sess), cmkID) + var matdesc s3crypto.MaterialDescription + handler := s3crypto.NewKMSContextKeyGenerator(kmsClient, cmkID, matdesc) // Create an encryption and decryption client // We need to pass the session here so S3 can use it. In addition, any decryption that // occurs will use the KMS client. - svc := s3crypto.NewEncryptionClientV2(sess, s3crypto.AESGCMContentCipherBuilder(handler)) - svc := s3crypto.NewDecryptionClientV2(sess) + svc, err := s3crypto.NewEncryptionClientV2(sess, s3crypto.AESGCMContentCipherBuilderV2(handler)) + if err != nil { + panic(err) // handle error + } + + // Create a CryptoRegistry and register the algorithms you wish to use for decryption + cr := s3crypto.NewCryptoRegistry() + + if err := s3crypto.RegisterAESGCMContentCipher(cr); err != nil { + panic(err) // handle error + } + + if err := s3crypto.RegisterKMSContextWrapWithAnyCMK(cr, kmsClient); err != nil { + panic(err) // handle error + } + + // Create a decryption client to decrypt artifacts + svc, err := s3crypto.NewDecryptionClientV2(sess, cr) + if err != nil { + panic(err) // handle error + } Configuration of the S3 cryptography client sess := session.Must(session.NewSession()) - handler := s3crypto.NewKMSContextKeyGenerator(kms.New(sess), cmkID) - svc := s3crypto.NewEncryptionClientV2(sess, s3crypto.AESGCMContentCipherBuilder(handler), func (o *s3crypto.EncryptionClientOptions) { + handler := s3crypto.NewKMSContextKeyGenerator(kms.New(sess), cmkID, s3crypto.MaterialDescription{}) + svc, err := s3crypto.NewEncryptionClientV2(sess, s3crypto.AESGCMContentCipherBuilderV2(handler), func (o *s3crypto.EncryptionClientOptions) { // Save instruction files to separate objects o.SaveStrategy = NewS3SaveStrategy(sess, "") @@ -43,20 +64,39 @@ Configuration of the S3 cryptography client // instead of writing the contents to a temp file. o.MinFileSize = int64(1024 * 1024 * 1024) }) + if err != nil { + panic(err) // handle error + } -The default SaveStrategy is to the object's header. +Object Metadata SaveStrategy -The InstructionFileSuffix defaults to .instruction. Careful here though, if you do this, be sure you know -what that suffix is in grabbing data. All requests will look for fooKey.example instead of fooKey.instruction. -This suffix only affects gets and not puts. Put uses the keyprovider's suffix. +The default SaveStrategy is to save metadata to an object's headers. An alternative SaveStrategy can be provided to the EncryptionClientV2. +For example, the S3SaveStrategy can be used to save the encryption metadata to a instruction file that is stored in S3 +using the objects KeyName+InstructionFileSuffix. The InstructionFileSuffix defaults to .instruction. If using this strategy you will need to +configure the DecryptionClientV2 to use the matching S3LoadStrategy LoadStrategy in order to decrypt object using this save strategy. -Registration of new wrap or cek algorithms are also supported by the SDK. Let's say we want to support `AES Wrap` -and `AES CTR`. Let's assume we have already defined the functionality. +Custom Key Wrappers and Custom Content Encryption Algorithms - svc := s3crypto.NewDecryptionClientV2(sess, func(o *s3crypto.DecryptionClientOptions) { - o.WrapRegistry["CustomWrap"] = NewCustomWrap - o.CEKRegistry["CustomCEK"] = NewCustomCEK - }) +Registration of custom key wrapping or content encryption algorithms not provided by AWS is allowed by the SDK, but +security and compatibility with custom types can not be guaranteed. For example if you want to support `CustomWrap` +key wrapping algorithm and `CustomCEK` content encryption algorithm. You can use the CryptoRegistry to register these types. + + cr := s3crypto.NewCryptoRegistry() + + // Register a custom key wrap algorithm to the CryptoRegistry + if err := cr.AddWrap("CustomWrap", NewCustomWrapEntry); err != nil { + panic(err) // handle error + } + + // Register a custom content encryption algorithm to the CryptoRegistry + if err := cr.AddCEK("CustomCEK", NewCustomCEKEntry); err != nil { + panic(err) // handle error + } + + svc, err := s3crypto.NewDecryptionClientV2(sess, cr) + if err != nil { + panic(err) // handle error + } We have now registered these new algorithms to the decryption client. When the client calls `GetObject` and sees the wrap as `CustomWrap` then it'll use that wrap algorithm. This is also true for `CustomCEK`. @@ -69,27 +109,30 @@ defined ciphers. // Our content cipher builder, NewCustomCEKContentBuilder svc := s3crypto.NewEncryptionClientV2(sess, NewCustomCEKContentBuilder(handler)) -Deprecations +Maintenance Mode Notification for V1 Clients -The EncryptionClient and DecryptionClient types and their associated constructor functions have been deprecated. -Users of these clients should migrate to EncryptionClientV2 and DecryptionClientV2 types and constructor functions. +The EncryptionClient and DecryptionClient are in maintenance mode, no new updates will be released. Please see https://docs.aws.amazon.com/general/latest/gr/aws_sdk_cryptography.html for more information. +Users of these clients should migrate to EncryptionClientV2 and DecryptionClientV2. EncryptionClientV2 removes encryption support of the following features - * AES/CBC/PKCS5Padding (content cipher) + * AES/CBC (content cipher) * kms (key wrap algorithm) Attempting to construct an EncryptionClientV2 with deprecated features will result in an error returned back to the calling application during construction of the client. -Users of `AES/CBC/PKCS5Padding` will need to migrate usage to `AES/GCM/NoPadding`. +Users of `AES/CBC` will need to migrate usage to `AES/GCM`. Users of `kms` key provider will need to migrate `kms+context`. DecryptionClientV2 client adds support for the `kms+context` key provider and maintains backwards comparability with -objects encrypted with the deprecated EncryptionClient. +objects encrypted with the V1 EncryptionClient. Migrating from V1 to V2 Clients Examples of how to migrate usage of the V1 clients to the V2 equivalents have been documented as usage examples of the NewEncryptionClientV2 and NewDecryptionClientV2 functions. + +Please see the AWS SDK for Go Developer Guide for additional migration steps https://docs.aws.amazon.com/sdk-for-go/v1/developer-guide/s3-encryption-migration.html + */ package s3crypto diff --git a/service/s3/s3crypto/encryption_client.go b/service/s3/s3crypto/encryption_client.go index b3816e89ac7..e00e9664de1 100644 --- a/service/s3/s3crypto/encryption_client.go +++ b/service/s3/s3crypto/encryption_client.go @@ -2,6 +2,7 @@ package s3crypto import ( "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/awserr" "github.com/aws/aws-sdk-go/aws/client" "github.com/aws/aws-sdk-go/aws/request" "github.com/aws/aws-sdk-go/service/s3" @@ -17,7 +18,7 @@ const DefaultMinFileSize = 1024 * 512 * 5 // AES GCM will load all data into memory. However, the rest of the content algorithms // do not load the entire contents into memory. // -// deprecated: See EncryptionClientV2 +// deprecated: This feature is in maintenance mode, no new updates will be released. Please see https://docs.aws.amazon.com/general/latest/gr/aws_sdk_cryptography.html for more information. type EncryptionClient struct { S3Client s3iface.S3API ContentCipherBuilder ContentCipherBuilder @@ -33,6 +34,19 @@ type EncryptionClient struct { MinFileSize int64 } +func validateV1EncryptionClientConstruction(c *EncryptionClient) error { + builder, ok := c.ContentCipherBuilder.(compatibleEncryptionFixture) + if !ok { + return nil + } + + err := builder.isEncryptionVersionCompatible(v1ClientVersion) + if err != nil { + return awserr.New(clientConstructionErrorCode, "invalid client configuration", err) + } + return nil +} + // NewEncryptionClient instantiates a new S3 crypto client // // Example: @@ -41,12 +55,12 @@ type EncryptionClient struct { // handler := s3crypto.NewKMSKeyGenerator(kms.New(sess), cmkID) // svc := s3crypto.NewEncryptionClient(sess, s3crypto.AESGCMContentCipherBuilder(handler)) // -// deprecated: See NewEncryptionClientV2 +// deprecated: This feature is in maintenance mode, no new updates will be released. Please see https://docs.aws.amazon.com/general/latest/gr/aws_sdk_cryptography.html for more information. func NewEncryptionClient(prov client.ConfigProvider, builder ContentCipherBuilder, options ...func(*EncryptionClient)) *EncryptionClient { s3client := s3.New(prov) s3client.Handlers.Build.PushBack(func(r *request.Request) { - request.AddToUserAgent(r, "S3Crypto") + request.AddToUserAgent(r, "S3CryptoV1n") }) client := &EncryptionClient{ @@ -67,7 +81,7 @@ func NewEncryptionClient(prov client.ConfigProvider, builder ContentCipherBuilde // that data to S3. // // Example: -// svc := s3crypto.New(session.New(), s3crypto.AESGCMContentCipherBuilder(handler)) +// svc := s3crypto.NewEncryptionClient(session.Must(session.NewSession()), s3crypto.AESGCMContentCipherBuilder(handler)) // req, out := svc.PutObjectRequest(&s3.PutObjectInput { // Key: aws.String("testKey"), // Bucket: aws.String("testBucket"), @@ -75,15 +89,34 @@ func NewEncryptionClient(prov client.ConfigProvider, builder ContentCipherBuilde // }) // err := req.Send() // -// deprecated: See EncryptionClientV2.PutObjectRequest +// deprecated: This feature is in maintenance mode, no new updates will be released. Please see https://docs.aws.amazon.com/general/latest/gr/aws_sdk_cryptography.html for more information. func (c *EncryptionClient) PutObjectRequest(input *s3.PutObjectInput) (*request.Request, *s3.PutObjectOutput) { - return putObjectRequest(c.getClientOptions(), input) + req, out := putObjectRequest(c.getClientOptions(), input) + if err := validateV1EncryptionClientConstruction(c); err != nil { + errHandler := setReqError(err) + req.Error = err + req.Handlers.Build.Clear() + req.Handlers.Send.Clear() + req.Handlers.Validate.PushFront(errHandler) + req.Handlers.Build.PushFront(errHandler) + req.Handlers.Send.PushFront(errHandler) + } + return req, out +} + +func setReqError(err error) func(*request.Request) { + return func(r *request.Request) { + r.Error = err + } } // PutObject is a wrapper for PutObjectRequest // -// deprecated: See EncryptionClientV2.PutObject +// deprecated: This feature is in maintenance mode, no new updates will be released. Please see https://docs.aws.amazon.com/general/latest/gr/aws_sdk_cryptography.html for more information. func (c *EncryptionClient) PutObject(input *s3.PutObjectInput) (*s3.PutObjectOutput, error) { + if err := validateV1EncryptionClientConstruction(c); err != nil { + return nil, err + } return putObject(c.getClientOptions(), input) } @@ -96,8 +129,11 @@ func (c *EncryptionClient) PutObject(input *s3.PutObjectInput) (*s3.PutObjectOut // this may create sub-contexts for individual underlying requests. // PutObject is a wrapper for PutObjectRequest // -// deprecated: See EncryptionClientV2.PutObjectWithContext +// deprecated: This feature is in maintenance mode, no new updates will be released. Please see https://docs.aws.amazon.com/general/latest/gr/aws_sdk_cryptography.html for more information. func (c *EncryptionClient) PutObjectWithContext(ctx aws.Context, input *s3.PutObjectInput, opts ...request.Option) (*s3.PutObjectOutput, error) { + if err := validateV1EncryptionClientConstruction(c); err != nil { + return nil, err + } return putObjectWithContext(c.getClientOptions(), ctx, input, opts...) } diff --git a/service/s3/s3crypto/encryption_client_test.go b/service/s3/s3crypto/encryption_client_test.go index 5f444c12dfd..450788d1bfb 100644 --- a/service/s3/s3crypto/encryption_client_test.go +++ b/service/s3/s3crypto/encryption_client_test.go @@ -1,4 +1,4 @@ -package s3crypto_test +package s3crypto import ( "bytes" @@ -16,7 +16,6 @@ import ( "github.com/aws/aws-sdk-go/awstesting/unit" "github.com/aws/aws-sdk-go/service/kms" "github.com/aws/aws-sdk-go/service/s3" - "github.com/aws/aws-sdk-go/service/s3/s3crypto" ) func TestDefaultConfigValues(t *testing.T) { @@ -26,9 +25,9 @@ func TestDefaultConfigValues(t *testing.T) { Region: aws.String("us-west-2"), }) svc := kms.New(sess) - handler := s3crypto.NewKMSKeyGenerator(svc, "testid") + handler := NewKMSKeyGenerator(svc, "testid") - c := s3crypto.NewEncryptionClient(sess, s3crypto.AESGCMContentCipherBuilder(handler)) + c := NewEncryptionClient(sess, AESGCMContentCipherBuilder(handler)) if c == nil { t.Error("expected non-vil client value") @@ -52,7 +51,7 @@ func TestPutObject(t *testing.T) { S3ForcePathStyle: aws.Bool(true), Region: aws.String("us-west-2"), }) - c := s3crypto.NewEncryptionClient(sess, cb) + c := NewEncryptionClient(sess, cb) if c == nil { t.Error("expected non-vil client value") } @@ -86,7 +85,7 @@ func TestPutObjectWithContext(t *testing.T) { generator := mockGenerator{} cb := mockCipherBuilder{generator} - c := s3crypto.NewEncryptionClient(unit.Session, cb) + c := NewEncryptionClient(unit.Session, cb) ctx := &awstesting.FakeContext{DoneCh: make(chan struct{})} ctx.Error = fmt.Errorf("context canceled") @@ -109,3 +108,51 @@ func TestPutObjectWithContext(t *testing.T) { t.Errorf("expected error message to contain %q, but did not %q", e, a) } } + +func TestEncryptionClient_PutObject_InvalidClientConstruction(t *testing.T) { + generator := mockGeneratorV2{} + cb := mockCipherBuilderV2{generator: generator} + + c := NewEncryptionClient(unit.Session, cb) + if c == nil { + t.Errorf("expected client not to be nil") + } + + input := s3.PutObjectInput{ + Bucket: aws.String("test"), + Key: aws.String("test"), + Body: bytes.NewReader([]byte{}), + } + _, err := c.PutObject(&input) + if err == nil { + t.Fatalf("expected error, did not get one") + } + if e, a := "invalid client configuration", err.Error(); !strings.Contains(a, e) { + t.Errorf("expected %v, got %v", e, a) + } + + _, err = c.PutObjectWithContext(aws.BackgroundContext(), &input) + if err == nil { + t.Fatalf("expected error, did not get one") + } + if e, a := "invalid client configuration", err.Error(); !strings.Contains(a, e) { + t.Errorf("expected %v, got %v", e, a) + } + + _, err = c.PutObjectWithContext(aws.BackgroundContext(), &input) + if err == nil { + t.Fatalf("expected error, did not get one") + } + if e, a := "invalid client configuration", err.Error(); !strings.Contains(a, e) { + t.Errorf("expected %v, got %v", e, a) + } + + req, _ := c.PutObjectRequest(&input) + err = req.Send() + if err == nil { + t.Fatalf("expected error, did not get one") + } + if e, a := "invalid client configuration", err.Error(); !strings.Contains(a, e) { + t.Errorf("expected %v, got %v", e, a) + } +} diff --git a/service/s3/s3crypto/encryption_client_v2.go b/service/s3/s3crypto/encryption_client_v2.go index 3f8961f2187..928c267654b 100644 --- a/service/s3/s3crypto/encryption_client_v2.go +++ b/service/s3/s3crypto/encryption_client_v2.go @@ -8,6 +8,8 @@ import ( "github.com/aws/aws-sdk-go/service/s3/s3iface" ) +const customTypeWarningMessage = "WARNING: The S3 Encryption Client is configured to write encrypted objects using types not provided by AWS. Security and compatibility with these types can not be guaranteed." + // EncryptionClientV2 is an S3 crypto client. By default the SDK will use Authentication mode which // will use KMS for key wrapping and AES GCM for content encryption. // AES GCM will load all data into memory. However, the rest of the content algorithms @@ -33,19 +35,19 @@ type EncryptionClientOptions struct { } // NewEncryptionClientV2 instantiates a new S3 crypto client. An error will be returned to the caller if the provided -// contentCipherBuilder has been deprecated, or if it uses other deprecated components. +// contentCipherBuilder has been deprecated or was constructed with a deprecated component. // // Example: // cmkID := "arn:aws:kms:region:000000000000:key/00000000-0000-0000-0000-000000000000" // sess := session.Must(session.NewSession()) -// handler := s3crypto.NewKMSContextKeyGenerator(kms.New(sess), cmkID) -// svc := s3crypto.NewEncryptionClientV2(sess, s3crypto.AESGCMContentCipherBuilder(handler)) +// var matdesc s3crypto.MaterialDescription +// handler := s3crypto.NewKMSContextKeyGenerator(kms.New(sess), cmkID, matdesc) +// svc := s3crypto.NewEncryptionClientV2(sess, s3crypto.AESGCMContentCipherBuilderV2(handler)) func NewEncryptionClientV2(prov client.ConfigProvider, contentCipherBuilder ContentCipherBuilder, options ...func(clientOptions *EncryptionClientOptions), ) ( client *EncryptionClientV2, err error, ) { s3client := s3.New(prov) - s3client.Handlers.Build.PushBack(func(r *request.Request) { request.AddToUserAgent(r, "S3CryptoV2") }) @@ -56,17 +58,25 @@ func NewEncryptionClientV2(prov client.ConfigProvider, contentCipherBuilder Cont SaveStrategy: HeaderV2SaveStrategy{}, MinFileSize: DefaultMinFileSize, } - for _, option := range options { option(clientOptions) } - if feature, ok := contentCipherBuilder.(deprecatedFeatures); ok { - if err := feature.isUsingDeprecatedFeatures(); err != nil { + // Check that the configured client uses a compatible ContentCipherBuilder. + // User provided types will not implement this method + if fixture, ok := contentCipherBuilder.(compatibleEncryptionFixture); ok { + if err := fixture.isEncryptionVersionCompatible(v2ClientVersion); err != nil { return nil, err } } + // Check if the passed in type is an fixture, if not log a warning message to the user + if fixture, ok := contentCipherBuilder.(awsFixture); !ok || !fixture.isAWSFixture() { + if s3client.Config.Logger != nil { + s3client.Config.Logger.Log(customTypeWarningMessage) + } + } + client = &EncryptionClientV2{ *clientOptions, } @@ -78,9 +88,6 @@ func NewEncryptionClientV2(prov client.ConfigProvider, contentCipherBuilder Cont // that data to S3. // // Example: -// sess := session.Must(session.NewSession()) -// handler := s3crypto.NewKMSContextKeyGenerator(kms.New(sess), "cmkID") -// svc := s3crypto.NewEncryptionClientV2(sess, s3crypto.AESGCMContentCipherBuilder(handler)) // req, out := svc.PutObjectRequest(&s3.PutObjectInput { // Key: aws.String("testKey"), // Bucket: aws.String("testBucket"), diff --git a/service/s3/s3crypto/encryption_client_v2_test.go b/service/s3/s3crypto/encryption_client_v2_test.go new file mode 100644 index 00000000000..055156b8fd0 --- /dev/null +++ b/service/s3/s3crypto/encryption_client_v2_test.go @@ -0,0 +1,211 @@ +// +build go1.7 + +package s3crypto + +import ( + "bytes" + "encoding/hex" + "fmt" + "io" + "io/ioutil" + "net/http" + "net/http/httptest" + "reflect" + "strings" + "testing" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/request" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/awstesting/unit" + "github.com/aws/aws-sdk-go/service/kms" + "github.com/aws/aws-sdk-go/service/s3" +) + +func sessionWithLogCheck(message string) (*session.Session, *bool) { + gotWarning := false + + u := unit.Session.Copy(&aws.Config{Logger: aws.LoggerFunc(func(i ...interface{}) { + if len(i) == 0 { + return + } + s, ok := i[0].(string) + if !ok { + return + } + if s == message { + gotWarning = true + } + })}) + + return u, &gotWarning +} + +func TestNewEncryptionClientV2(t *testing.T) { + tUnit, gotWarning := sessionWithLogCheck(customTypeWarningMessage) + + mcb := AESGCMContentCipherBuilderV2(NewKMSContextKeyGenerator(nil, "id", nil)) + v2, err := NewEncryptionClientV2(tUnit, mcb) + if err != nil { + t.Fatalf("expected no error, got %v", err) + } + if v2 == nil { + t.Fatal("expected client to not be nil") + } + + if *gotWarning { + t.Errorf("expected no warning for aws provided custom cipher builder") + } + + if !reflect.DeepEqual(mcb, v2.options.ContentCipherBuilder) { + t.Errorf("content cipher builder did not match provided value") + } + + _, ok := v2.options.SaveStrategy.(HeaderV2SaveStrategy) + if !ok { + t.Errorf("expected default save strategy to be s3 header strategy") + } + + if v2.options.S3Client == nil { + t.Errorf("expected s3 client not be nil") + } + + if e, a := DefaultMinFileSize, v2.options.MinFileSize; int64(e) != a { + t.Errorf("expected %v, got %v", e, a) + } + + if e, a := "", v2.options.TempFolderPath; e != a { + t.Errorf("expected %v, got %v", e, a) + } +} + +func TestNewEncryptionClientV2_NonDefaults(t *testing.T) { + tUnit, gotWarning := sessionWithLogCheck(customTypeWarningMessage) + + s3Client := s3.New(tUnit) + + mcb := mockCipherBuilderV2{} + v2, err := NewEncryptionClientV2(tUnit, nil, func(clientOptions *EncryptionClientOptions) { + clientOptions.S3Client = s3Client + clientOptions.ContentCipherBuilder = mcb + clientOptions.TempFolderPath = "/mock/path" + clientOptions.MinFileSize = 42 + clientOptions.SaveStrategy = S3SaveStrategy{} + }) + if err != nil { + t.Fatalf("expected no error, got %v", err) + } + if v2 == nil { + t.Fatal("expected client to not be nil") + } + + if !*gotWarning { + t.Errorf("expected warning for custom provided content cipher builder") + } + + if !reflect.DeepEqual(mcb, v2.options.ContentCipherBuilder) { + t.Errorf("content cipher builder did not match provided value") + } + + _, ok := v2.options.SaveStrategy.(S3SaveStrategy) + if !ok { + t.Errorf("expected default save strategy to be s3 header strategy") + } + + if v2.options.S3Client != s3Client { + t.Errorf("expected s3 client not be nil") + } + + if e, a := 42, v2.options.MinFileSize; int64(e) != a { + t.Errorf("expected %v, got %v", e, a) + } + + if e, a := "/mock/path", v2.options.TempFolderPath; e != a { + t.Errorf("expected %v, got %v", e, a) + } +} + +// cdgWithStaticTestIV is a test structure that wraps a CipherDataGeneratorWithCEKAlg and stubs in a static IV +// so that encryption tests can be guranteed to be consistent. +type cdgWithStaticTestIV struct { + IV []byte + CipherDataGeneratorWithCEKAlg +} + +// isAWSFixture will avoid the warning log message when doing tests that need to mock the IV +func (k cdgWithStaticTestIV) isAWSFixture() bool { + return true +} + +func (k cdgWithStaticTestIV) GenerateCipherDataWithCEKAlg(ctx aws.Context, keySize, ivSize int, cekAlg string) (CipherData, error) { + cipherData, err := k.CipherDataGeneratorWithCEKAlg.GenerateCipherDataWithCEKAlg(ctx, keySize, ivSize, cekAlg) + if err == nil { + cipherData.IV = k.IV + } + return cipherData, err +} + +func TestEncryptionClientV2_PutObject_KMSCONTEXT_AESGCM(t *testing.T) { + ts := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { + fmt.Fprintln(writer, `{"CiphertextBlob":"8gSzlk7giyfFbLPUVgoVjvQebI1827jp8lDkO+n2chsiSoegx1sjm8NdPk0Bl70I","KeyId":"test-key-id","Plaintext":"lP6AbIQTmptyb/+WQq+ubDw+w7na0T1LGSByZGuaono="}`) + })) + + sess := unit.Session.Copy() + kmsClient := kms.New(sess.Copy(&aws.Config{Endpoint: &ts.URL})) + + var md MaterialDescription + iv, _ := hex.DecodeString("ae325acae2bfd5b9c3d0b813") + kmsWithStaticIV := cdgWithStaticTestIV{ + IV: iv, + CipherDataGeneratorWithCEKAlg: NewKMSContextKeyGenerator(kmsClient, "test-key-id", md), + } + contentCipherBuilderV2 := AESGCMContentCipherBuilderV2(kmsWithStaticIV) + client, err := NewEncryptionClientV2(sess, contentCipherBuilderV2) + if err != nil { + t.Fatalf("expected no error, got %v", err) + } + + req, _ := client.PutObjectRequest(&s3.PutObjectInput{ + Bucket: aws.String("test-bucket"), + Key: aws.String("test-key"), + Body: func() io.ReadSeeker { + content, _ := hex.DecodeString("8f2c59c6dbfcacf356f3da40788cbde67ca38161a4702cbcf757af663e1c24a600001b2f500417dbf5a050f57db6737422b2ed6a44c75e0d") + return bytes.NewReader(content) + }(), + }) + + req.Handlers.Send.Clear() + req.Handlers.Send.PushFront(func(r *request.Request) { + all, err := ioutil.ReadAll(r.Body) + if err != nil { + t.Fatalf("expected no error, got %v", err) + } + + expected, _ := hex.DecodeString("4cd8e95a1c9b8b19640e02838b02c8c09e66250703a602956695afbc23cbb8647d51645955ab63b89733d0766f9a264adb88571b1d467b734ff72eb73d31de9a83670d59688c54ea") + + if !bytes.Equal(all, expected) { + t.Error("encrypted bytes did not match expected") + } + + req.HTTPResponse = &http.Response{ + Status: http.StatusText(200), + StatusCode: http.StatusOK, + Body: aws.ReadSeekCloser(bytes.NewReader([]byte{})), + } + }) + err = req.Send() + if err != nil { + t.Errorf("expected no error, got %v", err) + } +} + +func TestNewEncryptionClientV2_FailsOnIncompatibleFixtures(t *testing.T) { + sess := unit.Session.Copy() + _, err := NewEncryptionClientV2(sess, AESGCMContentCipherBuilder(NewKMSKeyGenerator(kms.New(sess), "cmkId"))) + if err == nil { + t.Fatal("expected to fail, but got nil") + } + if !strings.Contains(err.Error(), "attempted to use deprecated or incompatible cipher builder") { + t.Errorf("expected to get error for using dperecated cipher builder") + } +} diff --git a/service/s3/s3crypto/envelope.go b/service/s3/s3crypto/envelope.go index 1db9f7ad38d..12c67ceca0e 100644 --- a/service/s3/s3crypto/envelope.go +++ b/service/s3/s3crypto/envelope.go @@ -34,11 +34,14 @@ type Envelope struct { // CipherKey is the randomly generated cipher key. CipherKey string `json:"x-amz-key-v2"` // MaterialDesc is a description to distinguish from other envelopes. - MatDesc string `json:"x-amz-matdesc"` - WrapAlg string `json:"x-amz-wrap-alg"` - CEKAlg string `json:"x-amz-cek-alg"` - TagLen string `json:"x-amz-tag-len"` - UnencryptedMD5 string `json:"-"` + MatDesc string `json:"x-amz-matdesc"` + WrapAlg string `json:"x-amz-wrap-alg"` + CEKAlg string `json:"x-amz-cek-alg"` + TagLen string `json:"x-amz-tag-len"` + + // deprecated: This MD5 hash is no longer populated + UnencryptedMD5 string `json:"-"` + UnencryptedContentLen string `json:"x-amz-unencrypted-content-length"` } diff --git a/service/s3/s3crypto/errors.go b/service/s3/s3crypto/errors.go new file mode 100644 index 00000000000..c003ba5b87d --- /dev/null +++ b/service/s3/s3crypto/errors.go @@ -0,0 +1,24 @@ +package s3crypto + +import "fmt" + +var errNilCryptoRegistry = fmt.Errorf("provided CryptoRegistry must not be nil") +var errNilWrapEntry = fmt.Errorf("wrap entry must not be nil") +var errNilCEKEntry = fmt.Errorf("cek entry must not be nil") +var errNilPadder = fmt.Errorf("padder must not be nil") + +func newErrDuplicateWrapEntry(name string) error { + return newErrDuplicateRegistryEntry("wrap", name) +} + +func newErrDuplicateCEKEntry(name string) error { + return newErrDuplicateRegistryEntry("cek", name) +} + +func newErrDuplicatePadderEntry(name string) error { + return newErrDuplicateRegistryEntry("padder", name) +} + +func newErrDuplicateRegistryEntry(registry, key string) error { + return fmt.Errorf("duplicate %v registry entry, %v", registry, key) +} diff --git a/service/s3/s3crypto/fixture.go b/service/s3/s3crypto/fixture.go new file mode 100644 index 00000000000..6664a484d9e --- /dev/null +++ b/service/s3/s3crypto/fixture.go @@ -0,0 +1,27 @@ +package s3crypto + +import "fmt" + +type clientVersion int + +const ( + v1ClientVersion clientVersion = 1 + iota + v2ClientVersion +) + +var errDeprecatedIncompatibleCipherBuilder = fmt.Errorf("attempted to use deprecated or incompatible cipher builder") + +// compatibleEncryptionFixture is an unexported interface to expose whether a given fixture is compatible for encryption +// given the provided client version. +type compatibleEncryptionFixture interface { + isEncryptionVersionCompatible(clientVersion) error +} + +// awsFixture is an unexported interface to expose whether a given fixture is an aws provided fixture, and whether that +// fixtures dependencies were constructed using aws types. +// +// This interface is used in v2 clients to warn users if they are using custom implementations of ContentCipherBuilder +// or CipherDataGenerator. +type awsFixture interface { + isAWSFixture() bool +} diff --git a/service/s3/s3crypto/hash_io.go b/service/s3/s3crypto/hash_io.go index cda54e63452..2f37103053b 100644 --- a/service/s3/s3crypto/hash_io.go +++ b/service/s3/s3crypto/hash_io.go @@ -1,19 +1,38 @@ package s3crypto import ( - "crypto/md5" "crypto/sha256" "hash" "io" ) -// hashReader is used for calculating SHA256 when following the sigv4 specification. -// Additionally this used for calculating the unencrypted MD5. -type hashReader interface { - GetValue() []byte +// lengthReader returns the content length +type lengthReader interface { GetContentLength() int64 } +type contentLengthReader struct { + contentLength int64 + body io.Reader +} + +func newContentLengthReader(f io.Reader) *contentLengthReader { + return &contentLengthReader{body: f} +} + +func (r *contentLengthReader) Read(b []byte) (int, error) { + n, err := r.body.Read(b) + if err != nil && err != io.EOF { + return n, err + } + r.contentLength += int64(n) + return n, err +} + +func (r *contentLengthReader) GetContentLength() int64 { + return r.contentLength +} + type sha256Writer struct { sha256 []byte hash hash.Hash @@ -31,31 +50,3 @@ func (r *sha256Writer) Write(b []byte) (int, error) { func (r *sha256Writer) GetValue() []byte { return r.hash.Sum(nil) } - -type md5Reader struct { - contentLength int64 - hash hash.Hash - body io.Reader -} - -func newMD5Reader(body io.Reader) *md5Reader { - return &md5Reader{hash: md5.New(), body: body} -} - -func (w *md5Reader) Read(b []byte) (int, error) { - n, err := w.body.Read(b) - if err != nil && err != io.EOF { - return n, err - } - w.contentLength += int64(n) - w.hash.Write(b[:n]) - return n, err -} - -func (w *md5Reader) GetValue() []byte { - return w.hash.Sum(nil) -} - -func (w *md5Reader) GetContentLength() int64 { - return w.contentLength -} diff --git a/service/s3/s3crypto/hash_io_test.go b/service/s3/s3crypto/hash_io_test.go index a27daa34fa9..2d4458c67d4 100644 --- a/service/s3/s3crypto/hash_io_test.go +++ b/service/s3/s3crypto/hash_io_test.go @@ -3,6 +3,10 @@ package s3crypto import ( "bytes" "encoding/hex" + "fmt" + "io" + "io/ioutil" + "strings" "testing" ) @@ -27,3 +31,52 @@ func TestSHA256_Case2(t *testing.T) { t.Errorf("expected equivalent sha values, but received otherwise") } } + +type mockReader struct { + err error +} + +func (m mockReader) Read(p []byte) (int, error) { + return len(p), m.err +} + +func TestContentLengthReader(t *testing.T) { + cases := []struct { + reader io.Reader + expected int64 + expectedErr string + }{ + { + reader: bytes.NewReader([]byte("foo bar baz")), + expected: 11, + }, + { + reader: bytes.NewReader(nil), + expected: 0, + }, + { + reader: mockReader{err: fmt.Errorf("not an EOF error")}, + expectedErr: "not an EOF error", + }, + } + + for _, tt := range cases { + reader := newContentLengthReader(tt.reader) + _, err := ioutil.ReadAll(reader) + if err != nil { + if len(tt.expectedErr) == 0 { + t.Errorf("expected no error, got %v", err) + } else if !strings.Contains(err.Error(), tt.expectedErr) { + t.Errorf("expected error %v, got %v", tt.expectedErr, err.Error()) + } + continue + } else if len(tt.expectedErr) > 0 { + t.Error("expected error, got none") + continue + } + actual := reader.GetContentLength() + if tt.expected != actual { + t.Errorf("expected %v, got %v", tt.expected, actual) + } + } +} diff --git a/service/s3/s3crypto/integration/main_test.go b/service/s3/s3crypto/integration/main_test.go index 6fff21828ca..55250f64381 100644 --- a/service/s3/s3crypto/integration/main_test.go +++ b/service/s3/s3crypto/integration/main_test.go @@ -77,9 +77,27 @@ func TestEncryptionV1_WithV2Interop(t *testing.T) { v1DC := s3crypto.NewDecryptionClient(config.Session, func(client *s3crypto.DecryptionClient) { client.S3Client = config.Clients.S3 }) - v2DC := s3crypto.NewDecryptionClientV2(config.Session, func(options *s3crypto.DecryptionClientOptions) { + + cr := s3crypto.NewCryptoRegistry() + if err = s3crypto.RegisterKMSWrapWithAnyCMK(cr, config.Clients.KMS); err != nil { + t.Fatalf("expected no error, got %v", err) + } + if err = s3crypto.RegisterKMSContextWrapWithAnyCMK(cr, config.Clients.KMS); err != nil { + t.Fatalf("expected no error, got %v", err) + } + if err = s3crypto.RegisterAESGCMContentCipher(cr); err != nil { + t.Fatalf("expected no error, got %v", err) + } + if err = s3crypto.RegisterAESCBCContentCipher(cr, s3crypto.AESCBCPadder); err != nil { + t.Fatalf("expected no error, got %v", err) + } + + v2DC, err := s3crypto.NewDecryptionClientV2(config.Session, cr, func(options *s3crypto.DecryptionClientOptions) { options.S3Client = config.Clients.S3 }) + if err != nil { + t.Fatalf("expected no error, got %v", err) + } cases := map[string]s3crypto.ContentCipherBuilder{ "AES/GCM/NoPadding": s3crypto.AESGCMContentCipherBuilder(kmsKeyGenerator), @@ -102,20 +120,35 @@ func TestEncryptionV1_WithV2Interop(t *testing.T) { } } -func TestEncryptionV2(t *testing.T) { - kmsKeyGenerator := s3crypto.NewKMSContextKeyGenerator(config.Clients.KMS, config.KMSKeyID) - gcmContentCipherBuilder := s3crypto.AESGCMContentCipherBuilder(kmsKeyGenerator) +func TestEncryptionV2_WithV1Interop(t *testing.T) { + kmsKeyGenerator := s3crypto.NewKMSContextKeyGenerator(config.Clients.KMS, config.KMSKeyID, s3crypto.MaterialDescription{}) + gcmContentCipherBuilder := s3crypto.AESGCMContentCipherBuilderV2(kmsKeyGenerator) ec, err := s3crypto.NewEncryptionClientV2(config.Session, gcmContentCipherBuilder, func(options *s3crypto.EncryptionClientOptions) { options.S3Client = config.Clients.S3 }) if err != nil { - t.Fatalf("failed to construct encryption client: %v", err) + t.Fatalf("failed to construct encryption decryptionClient: %v", err) } - dc := s3crypto.NewDecryptionClientV2(config.Session, func(options *s3crypto.DecryptionClientOptions) { + decryptionClient := s3crypto.NewDecryptionClient(config.Session, func(client *s3crypto.DecryptionClient) { + client.S3Client = config.Clients.S3 + }) + + cr := s3crypto.NewCryptoRegistry() + if err = s3crypto.RegisterKMSContextWrapWithAnyCMK(cr, config.Clients.KMS); err != nil { + t.Fatalf("expected no error, got %v", err) + } + if err = s3crypto.RegisterAESGCMContentCipher(cr); err != nil { + t.Fatalf("expected no error, got %v", err) + } + + decryptionClientV2, err := s3crypto.NewDecryptionClientV2(config.Session, cr, func(options *s3crypto.DecryptionClientOptions) { options.S3Client = config.Clients.S3 }) + if err != nil { + t.Fatalf("expected no error, got %v", err) + } // 1020 is chosen here as it is not cleanly divisible by the AES-256 block size testData := make([]byte, 1020) @@ -129,8 +162,11 @@ func TestEncryptionV2(t *testing.T) { // Upload V2 Objects with Encryption Client putObject(t, ec, keyId, bytes.NewReader(testData)) - // Verify V2 Object with Decryption Client - getObjectAndCompare(t, dc, keyId, testData) + // Verify V2 Object with V2 Decryption Client + getObjectAndCompare(t, decryptionClientV2, keyId, testData) + + // Verify V2 Object with V1 Decryption Client + getObjectAndCompare(t, decryptionClient, keyId, testData) } type Encryptor interface { diff --git a/service/s3/s3crypto/key_handler.go b/service/s3/s3crypto/key_handler.go index f74a74d07e4..755e51a24b2 100644 --- a/service/s3/s3crypto/key_handler.go +++ b/service/s3/s3crypto/key_handler.go @@ -24,16 +24,7 @@ type CipherDataGeneratorWithContext interface { // content cipher. CipherDataGenerator will also encrypt the key and store it in // the CipherData. type CipherDataGeneratorWithCEKAlg interface { - GenerateCipherDataWithCEKAlg(int, int, string) (CipherData, error) - - CipherDataGenerator // backwards comparability to plug into older interface -} - -// CipherDataGeneratorWithCEKAlgWithContext handles generating proper key and IVs of -// proper size for the content cipher. CipherDataGenerator will also encrypt -// the key and store it in the CipherData. -type CipherDataGeneratorWithCEKAlgWithContext interface { - GenerateCipherDataWithCEKAlgWithContext(aws.Context, int, int, string) (CipherData, error) + GenerateCipherDataWithCEKAlg(ctx aws.Context, keySize, ivSize int, cekAlgorithm string) (CipherData, error) } // CipherDataDecrypter is a handler to decrypt keys from the envelope. diff --git a/service/s3/s3crypto/kms_context_key_handler.go b/service/s3/s3crypto/kms_context_key_handler.go new file mode 100644 index 00000000000..86f3572fdc0 --- /dev/null +++ b/service/s3/s3crypto/kms_context_key_handler.go @@ -0,0 +1,199 @@ +package s3crypto + +import ( + "fmt" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/kms" + "github.com/aws/aws-sdk-go/service/kms/kmsiface" +) + +const ( + // KMSContextWrap is a constant used during decryption to build a kms+context key handler + KMSContextWrap = "kms+context" + kmsAWSCEKContextKey = "aws:" + cekAlgorithmHeader + + kmsReservedKeyConflictErrMsg = "conflict in reserved KMS Encryption Context key %s. This value is reserved for the S3 Encryption Client and cannot be set by the user" + kmsMismatchCEKAlg = "the content encryption algorithm used at encryption time does not match the algorithm stored for decryption time. The object may be altered or corrupted" +) + +// NewKMSContextKeyGenerator builds a new kms+context key provider using the customer key ID and material +// description. +// +// Example: +// sess := session.Must(session.NewSession()) +// cmkID := "KMS Key ARN" +// var matdesc s3crypto.MaterialDescription +// handler := s3crypto.NewKMSContextKeyGenerator(kms.New(sess), cmkID, matdesc) +func NewKMSContextKeyGenerator(client kmsiface.KMSAPI, cmkID string, matdesc MaterialDescription) CipherDataGeneratorWithCEKAlg { + return newKMSContextKeyHandler(client, cmkID, matdesc) +} + +// RegisterKMSContextWrapWithCMK registers the kms+context wrapping algorithm to the given WrapRegistry. The wrapper +// will be configured to only call KMS Decrypt using the provided CMK. +// +// Example: +// cr := s3crypto.NewCryptoRegistry() +// if err := RegisterKMSContextWrapWithCMK(); err != nil { +// panic(err) // handle error +// } +func RegisterKMSContextWrapWithCMK(registry *CryptoRegistry, client kmsiface.KMSAPI, cmkID string) error { + if registry == nil { + return errNilCryptoRegistry + } + return registry.AddWrap(KMSContextWrap, newKMSContextWrapEntryWithCMK(client, cmkID)) +} + +// RegisterKMSContextWrapWithAnyCMK registers the kms+context wrapping algorithm to the given WrapRegistry. The wrapper +// will be configured to call KMS decrypt without providing a CMK. +// +// Example: +// sess := session.Must(session.NewSession()) +// cr := s3crypto.NewCryptoRegistry() +// if err := s3crypto.RegisterKMSContextWrapWithAnyCMK(cr, kms.New(sess)); err != nil { +// panic(err) // handle error +// } +func RegisterKMSContextWrapWithAnyCMK(registry *CryptoRegistry, client kmsiface.KMSAPI) error { + if registry == nil { + return errNilCryptoRegistry + } + return registry.AddWrap(KMSContextWrap, newKMSContextWrapEntryWithAnyCMK(client)) +} + +// newKMSContextWrapEntryWithCMK builds returns a new kms+context key provider and its decrypt handler. +// The returned handler will be configured to calls KMS Decrypt API without specifying a specific KMS CMK. +func newKMSContextWrapEntryWithCMK(kmsClient kmsiface.KMSAPI, cmkID string) WrapEntry { + // These values are read only making them thread safe + kp := &kmsContextKeyHandler{ + kms: kmsClient, + cmkID: &cmkID, + } + + return kp.decryptHandler +} + +// newKMSContextWrapEntryWithAnyCMK builds returns a new kms+context key provider and its decrypt handler. +// The returned handler will be configured to calls KMS Decrypt API without specifying a specific KMS CMK. +func newKMSContextWrapEntryWithAnyCMK(kmsClient kmsiface.KMSAPI) WrapEntry { + // These values are read only making them thread safe + kp := &kmsContextKeyHandler{ + kms: kmsClient, + } + + return kp.decryptHandler +} + +// kmsContextKeyHandler wraps the kmsKeyHandler to explicitly make this type incompatible with the v1 client +// by not exposing the old interface implementations. +type kmsContextKeyHandler struct { + kms kmsiface.KMSAPI + cmkID *string + + CipherData +} + +func (kp *kmsContextKeyHandler) isAWSFixture() bool { + return true +} + +func newKMSContextKeyHandler(client kmsiface.KMSAPI, cmkID string, matdesc MaterialDescription) *kmsContextKeyHandler { + kp := &kmsContextKeyHandler{ + kms: client, + cmkID: &cmkID, + } + + if matdesc == nil { + matdesc = MaterialDescription{} + } + + kp.CipherData.WrapAlgorithm = KMSContextWrap + kp.CipherData.MaterialDescription = matdesc + + return kp +} + +func (kp *kmsContextKeyHandler) GenerateCipherDataWithCEKAlg(ctx aws.Context, keySize int, ivSize int, cekAlgorithm string) (CipherData, error) { + cd := kp.CipherData.Clone() + + if len(cekAlgorithm) == 0 { + return CipherData{}, fmt.Errorf("cek algorithm identifier must not be empty") + } + + if _, ok := cd.MaterialDescription[kmsAWSCEKContextKey]; ok { + return CipherData{}, fmt.Errorf(kmsReservedKeyConflictErrMsg, kmsAWSCEKContextKey) + } + cd.MaterialDescription[kmsAWSCEKContextKey] = &cekAlgorithm + + out, err := kp.kms.GenerateDataKeyWithContext(ctx, + &kms.GenerateDataKeyInput{ + EncryptionContext: cd.MaterialDescription, + KeyId: kp.cmkID, + KeySpec: aws.String("AES_256"), + }) + if err != nil { + return CipherData{}, err + } + + iv, err := generateBytes(ivSize) + if err != nil { + return CipherData{}, err + } + + cd.Key = out.Plaintext + cd.IV = iv + cd.EncryptedKey = out.CiphertextBlob + + return cd, nil +} + +// decryptHandler initializes a KMS keyprovider with a material description. This +// is used with Decrypting kms content, due to the cmkID being in the material description. +func (kp kmsContextKeyHandler) decryptHandler(env Envelope) (CipherDataDecrypter, error) { + if env.WrapAlg != KMSContextWrap { + return nil, fmt.Errorf("%s value `%s` did not match the expected algorithm `%s` for this handler", cekAlgorithmHeader, env.WrapAlg, KMSContextWrap) + } + + m := MaterialDescription{} + err := m.decodeDescription([]byte(env.MatDesc)) + if err != nil { + return nil, err + } + + if v, ok := m[kmsAWSCEKContextKey]; !ok { + return nil, fmt.Errorf("required key %v is missing from encryption context", kmsAWSCEKContextKey) + } else if v == nil || *v != env.CEKAlg { + return nil, fmt.Errorf(kmsMismatchCEKAlg) + } + + kp.MaterialDescription = m + kp.WrapAlgorithm = KMSContextWrap + + return &kp, nil +} + +// DecryptKey makes a call to KMS to decrypt the key. +func (kp *kmsContextKeyHandler) DecryptKey(key []byte) ([]byte, error) { + return kp.DecryptKeyWithContext(aws.BackgroundContext(), key) +} + +// DecryptKeyWithContext makes a call to KMS to decrypt the key with request context. +func (kp *kmsContextKeyHandler) DecryptKeyWithContext(ctx aws.Context, key []byte) ([]byte, error) { + out, err := kp.kms.DecryptWithContext(ctx, + &kms.DecryptInput{ + KeyId: kp.cmkID, // will be nil and not serialized if created with the AnyCMK constructor + EncryptionContext: kp.MaterialDescription, + CiphertextBlob: key, + GrantTokens: []*string{}, + }) + if err != nil { + return nil, err + } + return out.Plaintext, nil +} + +var ( + _ CipherDataGeneratorWithCEKAlg = (*kmsContextKeyHandler)(nil) + _ CipherDataDecrypter = (*kmsContextKeyHandler)(nil) + _ CipherDataDecrypterWithContext = (*kmsContextKeyHandler)(nil) + _ awsFixture = (*kmsContextKeyHandler)(nil) +) diff --git a/service/s3/s3crypto/kms_context_key_handler_test.go b/service/s3/s3crypto/kms_context_key_handler_test.go new file mode 100644 index 00000000000..0d9abd42427 --- /dev/null +++ b/service/s3/s3crypto/kms_context_key_handler_test.go @@ -0,0 +1,265 @@ +package s3crypto + +import ( + "bytes" + "encoding/base64" + "encoding/hex" + "encoding/json" + "fmt" + "io/ioutil" + "net/http" + "net/http/httptest" + "reflect" + "strings" + "testing" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/awstesting/unit" + "github.com/aws/aws-sdk-go/service/kms" +) + +func TestKmsContextKeyHandler_GenerateCipherDataWithCEKAlg(t *testing.T) { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + bodyBytes, err := ioutil.ReadAll(r.Body) + if err != nil { + w.WriteHeader(500) + return + } + var body map[string]interface{} + err = json.Unmarshal(bodyBytes, &body) + if err != nil { + w.WriteHeader(500) + return + } + + md, ok := body["EncryptionContext"].(map[string]interface{}) + if !ok { + w.WriteHeader(500) + return + } + + exEncContext := map[string]interface{}{ + "aws:" + cekAlgorithmHeader: "cekAlgValue", + } + + if e, a := exEncContext, md; !reflect.DeepEqual(e, a) { + w.WriteHeader(500) + t.Errorf("expected %v, got %v", e, a) + return + } + + fmt.Fprintln(w, `{"CiphertextBlob":"AQEDAHhqBCCY1MSimw8gOGcUma79cn4ANvTtQyv9iuBdbcEF1QAAAH4wfAYJKoZIhvcNAQcGoG8wbQIBADBoBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDJ6IcN5E4wVbk38MNAIBEIA7oF1E3lS7FY9DkoxPc/UmJsEwHzL82zMqoLwXIvi8LQHr8If4Lv6zKqY8u0+JRgSVoqCvZDx3p8Cn6nM=","KeyId":"arn:aws:kms:us-west-2:042062605278:key/c80a5cdb-8d09-4f9f-89ee-df01b2e3870a","Plaintext":"6tmyz9JLBE2yIuU7iXpArqpDVle172WSmxjcO6GNT7E="}`) + })) + defer ts.Close() + + sess := unit.Session.Copy(&aws.Config{ + MaxRetries: aws.Int(0), + Endpoint: aws.String(ts.URL), + DisableSSL: aws.Bool(true), + S3ForcePathStyle: aws.Bool(true), + Region: aws.String("us-west-2"), + }) + + svc := kms.New(sess) + handler := NewKMSContextKeyGenerator(svc, "testid", nil) + + keySize := 32 + ivSize := 16 + + cd, err := handler.GenerateCipherDataWithCEKAlg(aws.BackgroundContext(), keySize, ivSize, "cekAlgValue") + if err != nil { + t.Errorf("expected no error, but received %v", err) + } + if keySize != len(cd.Key) { + t.Errorf("expected %d, but received %d", keySize, len(cd.Key)) + } + if ivSize != len(cd.IV) { + t.Errorf("expected %d, but received %d", ivSize, len(cd.IV)) + } +} + +func TestKmsContextKeyHandler_GenerateCipherDataWithCEKAlg_ReservedKeyConflict(t *testing.T) { + svc := kms.New(unit.Session.Copy()) + handler := NewKMSContextKeyGenerator(svc, "testid", MaterialDescription{ + "aws:x-amz-cek-alg": aws.String("something unexpected"), + }) + + _, err := handler.GenerateCipherDataWithCEKAlg(aws.BackgroundContext(), 32, 16, "cekAlgValue") + if err == nil { + t.Errorf("expected error, but none") + } else if !strings.Contains(err.Error(), "conflict in reserved KMS Encryption Context key aws:x-amz-cek-alg") { + t.Errorf("expected reserved key error, got %v", err) + } +} + +func TestKmsContextKeyHandler_DecryptKey(t *testing.T) { + key, _ := hex.DecodeString("31bdadd96698c204aa9ce1448ea94ae1fb4a9a0b3c9d773b51bb1822666b8f22") + keyB64 := base64.URLEncoding.EncodeToString(key) + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + bodyBytes, err := ioutil.ReadAll(r.Body) + if err != nil { + t.Errorf("expected no error, got %v", err) + w.WriteHeader(500) + return + } + + var body map[string]interface{} + err = json.Unmarshal(bodyBytes, &body) + if err != nil { + w.WriteHeader(500) + return + } + + if _, ok := body["KeyId"]; ok { + t.Errorf("expected CMK to not be sent") + } + + md, ok := body["EncryptionContext"].(map[string]interface{}) + if !ok { + w.WriteHeader(500) + return + } + + exEncContext := map[string]interface{}{ + "aws:" + cekAlgorithmHeader: "AES/GCM/NoPadding", + } + + if e, a := exEncContext, md; !reflect.DeepEqual(e, a) { + w.WriteHeader(500) + t.Errorf("expected %v, got %v", e, a) + return + } + + fmt.Fprintln(w, fmt.Sprintf("%s%s%s", `{"KeyId":"test-key-id","Plaintext":"`, keyB64, `"}`)) + })) + defer ts.Close() + + sess := unit.Session.Copy(&aws.Config{ + MaxRetries: aws.Int(0), + Endpoint: aws.String(ts.URL), + DisableSSL: aws.Bool(true), + S3ForcePathStyle: aws.Bool(true), + Region: aws.String("us-west-2"), + }) + handler, err := newKMSContextWrapEntryWithAnyCMK(kms.New(sess))(Envelope{WrapAlg: KMSContextWrap, CEKAlg: "AES/GCM/NoPadding", MatDesc: `{"aws:x-amz-cek-alg": "AES/GCM/NoPadding"}`}) + if err != nil { + t.Fatalf("expected no error, but received %v", err) + } + + plaintextKey, err := handler.DecryptKey([]byte{1, 2, 3, 4}) + if err != nil { + t.Errorf("expected no error, but received %v", err) + } + + if !bytes.Equal(key, plaintextKey) { + t.Errorf("expected %v, but received %v", key, plaintextKey) + } +} + +func TestKmsContextKeyHandler_decryptHandler_MismatchCEK(t *testing.T) { + _, err := newKMSContextWrapEntryWithAnyCMK(kms.New(unit.Session.Copy()))(Envelope{WrapAlg: KMSContextWrap, CEKAlg: "MismatchCEKValue", MatDesc: `{"aws:x-amz-cek-alg": "AES/GCM/NoPadding"}`}) + if err == nil { + t.Fatal("expected error, but got none") + } + + if e, a := "algorithm used at encryption time does not match the algorithm stored", err.Error(); !strings.Contains(a, e) { + t.Errorf("expected error to contain %v, got %v", e, a) + } +} + +func TestKmsContextKeyHandler_decryptHandler_MissingContextKey(t *testing.T) { + _, err := newKMSContextWrapEntryWithAnyCMK(kms.New(unit.Session.Copy()))(Envelope{WrapAlg: KMSContextWrap, CEKAlg: "AES/GCM/NoPadding", MatDesc: `{}`}) + if err == nil { + t.Fatal("expected error, but got none") + } + + if e, a := "missing from encryption context", err.Error(); !strings.Contains(a, e) { + t.Errorf("expected error to contain %v, got %v", e, a) + } +} + +func TestKmsContextKeyHandler_decryptHandler_MismatchWrap(t *testing.T) { + _, err := newKMSContextWrapEntryWithAnyCMK(kms.New(unit.Session.Copy()))(Envelope{WrapAlg: KMSWrap, CEKAlg: "AES/GCM/NoPadding", MatDesc: `{}`}) + if err == nil { + t.Fatal("expected error, but got none") + } + + if e, a := "x-amz-cek-alg value `kms` did not match the expected algorithm `kms+context` for this handler", err.Error(); !strings.Contains(a, e) { + t.Errorf("expected error to contain %v, got %v", e, a) + } +} + +func TestKmsContextKeyHandler_DecryptKey_WithCMK(t *testing.T) { + key, _ := hex.DecodeString("31bdadd96698c204aa9ce1448ea94ae1fb4a9a0b3c9d773b51bb1822666b8f22") + keyB64 := base64.URLEncoding.EncodeToString(key) + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + body, err := ioutil.ReadAll(r.Body) + if err != nil { + t.Errorf("expected no error, got %v", err) + w.WriteHeader(500) + return + } + + if !bytes.Contains(body, []byte(`"KeyId":"thisKey"`)) { + t.Errorf("expected CMK to be sent") + } + + fmt.Fprintln(w, fmt.Sprintf("%s%s%s", `{"KeyId":"thisKey","Plaintext":"`, keyB64, `"}`)) + })) + defer ts.Close() + + sess := unit.Session.Copy(&aws.Config{ + MaxRetries: aws.Int(0), + Endpoint: aws.String(ts.URL), + DisableSSL: aws.Bool(true), + S3ForcePathStyle: aws.Bool(true), + Region: aws.String("us-west-2"), + }) + handler, err := newKMSContextWrapEntryWithCMK(kms.New(sess), "thisKey")(Envelope{WrapAlg: KMSContextWrap, CEKAlg: "AES/GCM/NoPadding", MatDesc: `{"aws:x-amz-cek-alg": "AES/GCM/NoPadding"}`}) + if err != nil { + t.Errorf("expected no error, but received %v", err) + } + + _, err = handler.DecryptKey([]byte{1, 2, 3, 4}) + if err != nil { + t.Errorf("expected no error, but received %v", err) + } +} + +func TestRegisterKMSContextWrapWithAnyCMK(t *testing.T) { + kmsClient := kms.New(unit.Session.Copy()) + + cr := NewCryptoRegistry() + if err := RegisterKMSContextWrapWithAnyCMK(cr, kmsClient); err != nil { + t.Errorf("expected no error, got %v", err) + } + + if wrap, ok := cr.GetWrap(KMSContextWrap); !ok { + t.Errorf("expected wrapped to be present") + } else if wrap == nil { + t.Errorf("expected wrap to not be nil") + } + + if err := RegisterKMSContextWrapWithCMK(cr, kmsClient, "test-key-id"); err == nil { + t.Error("expected error, got none") + } +} + +func TestRegisterKMSContextWrapWithCMK(t *testing.T) { + kmsClient := kms.New(unit.Session.Copy()) + + cr := NewCryptoRegistry() + if err := RegisterKMSContextWrapWithCMK(cr, kmsClient, "cmkId"); err != nil { + t.Errorf("expected no error, got %v", err) + } + + if wrap, ok := cr.GetWrap(KMSContextWrap); !ok { + t.Errorf("expected wrapped to be present") + } else if wrap == nil { + t.Errorf("expected wrap to not be nil") + } + + if err := RegisterKMSContextWrapWithAnyCMK(cr, kmsClient); err == nil { + t.Error("expected error, got none") + } +} diff --git a/service/s3/s3crypto/kms_key_handler.go b/service/s3/s3crypto/kms_key_handler.go index be1af51929f..44d40f124e8 100644 --- a/service/s3/s3crypto/kms_key_handler.go +++ b/service/s3/s3crypto/kms_key_handler.go @@ -1,8 +1,6 @@ package s3crypto import ( - "fmt" - "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/awserr" "github.com/aws/aws-sdk-go/service/kms" @@ -12,16 +10,15 @@ import ( const ( // KMSWrap is a constant used during decryption to build a KMS key handler. KMSWrap = "kms" - - // KMSContextWrap is a constant used during decryption to build a kms+context key handler - KMSContextWrap = "kms+context" ) // kmsKeyHandler will make calls to KMS to get the masterkey type kmsKeyHandler struct { - kms kmsiface.KMSAPI - cmkID *string - withContext bool + kms kmsiface.KMSAPI + cmkID *string + + // useProvidedCMK is toggled when using `kms` key wrapper with V2 client + useProvidedCMK bool CipherData } @@ -30,48 +27,32 @@ type kmsKeyHandler struct { // description. // // Example: -// sess := session.New(&aws.Config{}) +// sess := session.Must(session.NewSession()) // cmkID := "arn to key" // matdesc := s3crypto.MaterialDescription{} // handler := s3crypto.NewKMSKeyGenerator(kms.New(sess), cmkID) // -// deprecated: See NewKMSContextKeyGenerator +// deprecated: This feature is in maintenance mode, no new updates will be released. Please see https://docs.aws.amazon.com/general/latest/gr/aws_sdk_cryptography.html for more information. func NewKMSKeyGenerator(kmsClient kmsiface.KMSAPI, cmkID string) CipherDataGenerator { return NewKMSKeyGeneratorWithMatDesc(kmsClient, cmkID, MaterialDescription{}) } -// NewKMSContextKeyGenerator builds a new kms+context key provider using the customer key ID and material -// description. -// -// Example: -// sess := session.New(&aws.Config{}) -// cmkID := "arn to key" -// matdesc := s3crypto.MaterialDescription{} -// handler := s3crypto.NewKMSContextKeyGenerator(kms.New(sess), cmkID) -func NewKMSContextKeyGenerator(client kmsiface.KMSAPI, cmkID string) CipherDataGeneratorWithCEKAlg { - return NewKMSContextKeyGeneratorWithMatDesc(client, cmkID, MaterialDescription{}) -} - -func newKMSKeyHandler(client kmsiface.KMSAPI, cmkID string, withContext bool, matdesc MaterialDescription) *kmsKeyHandler { +func newKMSKeyHandler(client kmsiface.KMSAPI, cmkID string, matdesc MaterialDescription) *kmsKeyHandler { // These values are read only making them thread safe kp := &kmsKeyHandler{ - kms: client, - cmkID: &cmkID, - withContext: withContext, + kms: client, + cmkID: &cmkID, } if matdesc == nil { matdesc = MaterialDescription{} } - // These values are read only making them thread safe - if kp.withContext { - kp.CipherData.WrapAlgorithm = KMSContextWrap - } else { - matdesc["kms_cmk_id"] = &cmkID - kp.CipherData.WrapAlgorithm = KMSWrap - } + matdesc["kms_cmk_id"] = &cmkID + + kp.CipherData.WrapAlgorithm = KMSWrap kp.CipherData.MaterialDescription = matdesc + return kp } @@ -79,32 +60,20 @@ func newKMSKeyHandler(client kmsiface.KMSAPI, cmkID string, withContext bool, ma // description. // // Example: -// sess := session.New(&aws.Config{}) +// sess := session.Must(session.NewSession()) // cmkID := "arn to key" // matdesc := s3crypto.MaterialDescription{} // handler := s3crypto.NewKMSKeyGeneratorWithMatDesc(kms.New(sess), cmkID, matdesc) // -// deprecated: See NewKMSContextKeyGeneratorWithMatDesc +// deprecated: This feature is in maintenance mode, no new updates will be released. Please see https://docs.aws.amazon.com/general/latest/gr/aws_sdk_cryptography.html for more information. func NewKMSKeyGeneratorWithMatDesc(kmsClient kmsiface.KMSAPI, cmkID string, matdesc MaterialDescription) CipherDataGenerator { - return newKMSKeyHandler(kmsClient, cmkID, false, matdesc) -} - -// NewKMSContextKeyGeneratorWithMatDesc builds a new kms+context key provider using the customer key ID and material -// description. -// -// Example: -// sess := session.New(&aws.Config{}) -// cmkID := "arn to key" -// matdesc := s3crypto.MaterialDescription{} -// handler := s3crypto.NewKMSKeyGeneratorWithMatDesc(kms.New(sess), cmkID, matdesc) -func NewKMSContextKeyGeneratorWithMatDesc(kmsClient kmsiface.KMSAPI, cmkID string, matdesc MaterialDescription) CipherDataGeneratorWithCEKAlg { - return newKMSKeyHandler(kmsClient, cmkID, true, matdesc) + return newKMSKeyHandler(kmsClient, cmkID, matdesc) } // NewKMSWrapEntry builds returns a new KMS key provider and its decrypt handler. // // Example: -// sess := session.New(&aws.Config{}) +// sess := session.Must(session.NewSession()) // customKMSClient := kms.New(sess) // decryptHandler := s3crypto.NewKMSWrapEntry(customKMSClient) // @@ -112,34 +81,64 @@ func NewKMSContextKeyGeneratorWithMatDesc(kmsClient kmsiface.KMSAPI, cmkID strin // svc.WrapRegistry[s3crypto.KMSWrap] = decryptHandler // })) // -// deprecated: See NewKMSContextWrapEntry +// deprecated: This feature is in maintenance mode, no new updates will be released. Please see https://docs.aws.amazon.com/general/latest/gr/aws_sdk_cryptography.html for more information. func NewKMSWrapEntry(kmsClient kmsiface.KMSAPI) WrapEntry { - // These values are read only making them thread safe - kp := &kmsKeyHandler{ - kms: kmsClient, - } - + kp := newKMSWrapEntry(kmsClient) return kp.decryptHandler } -// NewKMSContextWrapEntry builds returns a new KMS key provider and its decrypt handler. +// RegisterKMSWrapWithCMK registers the `kms` wrapping algorithm to the given WrapRegistry. The wrapper will be +// configured to call KMS Decrypt with the provided CMK. // // Example: -// sess := session.New(&aws.Config{}) -// customKMSClient := kms.New(sess) -// decryptHandler := s3crypto.NewKMSContextWrapEntry(customKMSClient) +// sess := session.Must(session.NewSession()) +// cr := s3crypto.NewCryptoRegistry() +// if err := s3crypto.RegisterKMSWrapWithCMK(cr, kms.New(sess), "cmkId"); err != nil { +// panic(err) // handle error +// } // -// svc := s3crypto.NewDecryptionClient(sess, func(svc *s3crypto.DecryptionClient) { -// svc.WrapRegistry[s3crypto.KMSContextWrap] = decryptHandler -// })) -func NewKMSContextWrapEntry(kmsClient kmsiface.KMSAPI) WrapEntry { +// deprecated: This feature is in maintenance mode, no new updates will be released. Please see https://docs.aws.amazon.com/general/latest/gr/aws_sdk_cryptography.html for more information. +func RegisterKMSWrapWithCMK(registry *CryptoRegistry, client kmsiface.KMSAPI, cmkID string) error { + if registry == nil { + return errNilCryptoRegistry + } + return registry.AddWrap(KMSWrap, newKMSWrapEntryWithCMK(client, cmkID)) +} + +// RegisterKMSWrapWithAnyCMK registers the `kms` wrapping algorithm to the given WrapRegistry. The wrapper will be +// configured to call KMS Decrypt without providing a CMK. +// +// Example: +// sess := session.Must(session.NewSession()) +// cr := s3crypto.NewCryptoRegistry() +// if err := s3crypto.RegisterKMSWrapWithAnyCMK(cr, kms.New(sess)); err != nil { +// panic(err) // handle error +// } +// +// deprecated: This feature is in maintenance mode, no new updates will be released. Please see https://docs.aws.amazon.com/general/latest/gr/aws_sdk_cryptography.html for more information. +func RegisterKMSWrapWithAnyCMK(registry *CryptoRegistry, client kmsiface.KMSAPI) error { + if registry == nil { + return errNilCryptoRegistry + } + return registry.AddWrap(KMSWrap, NewKMSWrapEntry(client)) +} + +// newKMSWrapEntryWithCMK builds returns a new KMS key provider and its decrypt handler. The wrap entry will be configured +// to only attempt to decrypt the data key using the provided CMK. +func newKMSWrapEntryWithCMK(kmsClient kmsiface.KMSAPI, cmkID string) WrapEntry { + kp := newKMSWrapEntry(kmsClient) + kp.useProvidedCMK = true + kp.cmkID = &cmkID + return kp.decryptHandler +} + +func newKMSWrapEntry(kmsClient kmsiface.KMSAPI) *kmsKeyHandler { // These values are read only making them thread safe kp := &kmsKeyHandler{ - kms: kmsClient, - withContext: true, + kms: kmsClient, } - return kp.decryptHandler + return kp } // decryptHandler initializes a KMS keyprovider with a material description. This @@ -151,17 +150,14 @@ func (kp kmsKeyHandler) decryptHandler(env Envelope) (CipherDataDecrypter, error return nil, err } - cmkID, ok := m["kms_cmk_id"] - if !kp.withContext && !ok { + _, ok := m["kms_cmk_id"] + if !ok { return nil, awserr.New("MissingCMKIDError", "Material description is missing CMK ID", nil) } kp.CipherData.MaterialDescription = m - kp.cmkID = cmkID kp.WrapAlgorithm = KMSWrap - if kp.withContext { - kp.WrapAlgorithm = KMSContextWrap - } + return &kp, nil } @@ -172,12 +168,18 @@ func (kp *kmsKeyHandler) DecryptKey(key []byte) ([]byte, error) { // DecryptKeyWithContext makes a call to KMS to decrypt the key with request context. func (kp *kmsKeyHandler) DecryptKeyWithContext(ctx aws.Context, key []byte) ([]byte, error) { - out, err := kp.kms.DecryptWithContext(ctx, - &kms.DecryptInput{ - EncryptionContext: kp.CipherData.MaterialDescription, - CiphertextBlob: key, - GrantTokens: []*string{}, - }) + in := &kms.DecryptInput{ + EncryptionContext: kp.MaterialDescription, + CiphertextBlob: key, + GrantTokens: []*string{}, + } + + // useProvidedCMK will be true if a constructor was used with the new V2 client + if kp.useProvidedCMK { + in.KeyId = kp.cmkID + } + + out, err := kp.kms.DecryptWithContext(ctx, in) if err != nil { return nil, err } @@ -190,31 +192,14 @@ func (kp *kmsKeyHandler) GenerateCipherData(keySize, ivSize int) (CipherData, er return kp.GenerateCipherDataWithContext(aws.BackgroundContext(), keySize, ivSize) } -func (kp kmsKeyHandler) GenerateCipherDataWithCEKAlg(keySize, ivSize int, cekAlgorithm string) (CipherData, error) { - return kp.GenerateCipherDataWithCEKAlgWithContext(aws.BackgroundContext(), keySize, ivSize, cekAlgorithm) -} - // GenerateCipherDataWithContext makes a call to KMS to generate a data key, // Upon making the call, it also sets the encrypted key. func (kp *kmsKeyHandler) GenerateCipherDataWithContext(ctx aws.Context, keySize, ivSize int) (CipherData, error) { - return kp.GenerateCipherDataWithCEKAlgWithContext(ctx, keySize, ivSize, "") -} - -func (kp kmsKeyHandler) GenerateCipherDataWithCEKAlgWithContext(ctx aws.Context, keySize int, ivSize int, cekAlgorithm string) (CipherData, error) { - md := kp.CipherData.MaterialDescription - - wrapAlgorithm := KMSWrap - if kp.withContext { - wrapAlgorithm = KMSContextWrap - if len(cekAlgorithm) == 0 { - return CipherData{}, fmt.Errorf("CEK algorithm identifier must not be empty") - } - md["aws:"+cekAlgorithmHeader] = &cekAlgorithm - } + cd := kp.CipherData.Clone() out, err := kp.kms.GenerateDataKeyWithContext(ctx, &kms.GenerateDataKeyInput{ - EncryptionContext: md, + EncryptionContext: cd.MaterialDescription, KeyId: kp.cmkID, KeySpec: aws.String("AES_256"), }) @@ -227,19 +212,21 @@ func (kp kmsKeyHandler) GenerateCipherDataWithCEKAlgWithContext(ctx aws.Context, return CipherData{}, err } - cd := CipherData{ - Key: out.Plaintext, - IV: iv, - WrapAlgorithm: wrapAlgorithm, - MaterialDescription: md, - EncryptedKey: out.CiphertextBlob, - } + cd.Key = out.Plaintext + cd.IV = iv + cd.EncryptedKey = out.CiphertextBlob + return cd, nil } -func (kp *kmsKeyHandler) isUsingDeprecatedFeatures() error { - if !kp.withContext { - return errDeprecatedCipherDataGenerator - } - return nil +func (kp kmsKeyHandler) isAWSFixture() bool { + return true } + +var ( + _ CipherDataGenerator = (*kmsKeyHandler)(nil) + _ CipherDataGeneratorWithContext = (*kmsKeyHandler)(nil) + _ CipherDataDecrypter = (*kmsKeyHandler)(nil) + _ CipherDataDecrypterWithContext = (*kmsKeyHandler)(nil) + _ awsFixture = (*kmsKeyHandler)(nil) +) diff --git a/service/s3/s3crypto/kms_key_handler_test.go b/service/s3/s3crypto/kms_key_handler_test.go index 4c030b0e9cf..7c480829bfc 100644 --- a/service/s3/s3crypto/kms_key_handler_test.go +++ b/service/s3/s3crypto/kms_key_handler_test.go @@ -4,7 +4,6 @@ import ( "bytes" "encoding/base64" "encoding/hex" - "encoding/json" "fmt" "io/ioutil" "net/http" @@ -17,7 +16,7 @@ import ( "github.com/aws/aws-sdk-go/service/kms" ) -func TestBuildKMSEncryptHandler(t *testing.T) { +func TestNewKMSKeyGenerator(t *testing.T) { svc := kms.New(unit.Session) handler := NewKMSKeyGenerator(svc, "testid") if handler == nil { @@ -25,7 +24,7 @@ func TestBuildKMSEncryptHandler(t *testing.T) { } } -func TestBuildKMSEncryptHandlerWithMatDesc(t *testing.T) { +func TestNewKMSKeyGeneratorWithMatDesc(t *testing.T) { svc := kms.New(unit.Session) handler := NewKMSKeyGeneratorWithMatDesc(svc, "testid", MaterialDescription{ "Testing": aws.String("123"), @@ -45,7 +44,7 @@ func TestBuildKMSEncryptHandlerWithMatDesc(t *testing.T) { } } -func TestKMSGenerateCipherData(t *testing.T) { +func TestKmsKeyHandler_GenerateCipherData(t *testing.T) { ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { fmt.Fprintln(w, `{"CiphertextBlob":"AQEDAHhqBCCY1MSimw8gOGcUma79cn4ANvTtQyv9iuBdbcEF1QAAAH4wfAYJKoZIhvcNAQcGoG8wbQIBADBoBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDJ6IcN5E4wVbk38MNAIBEIA7oF1E3lS7FY9DkoxPc/UmJsEwHzL82zMqoLwXIvi8LQHr8If4Lv6zKqY8u0+JRgSVoqCvZDx3p8Cn6nM=","KeyId":"arn:aws:kms:us-west-2:042062605278:key/c80a5cdb-8d09-4f9f-89ee-df01b2e3870a","Plaintext":"6tmyz9JLBE2yIuU7iXpArqpDVle172WSmxjcO6GNT7E="}`) })) @@ -77,10 +76,19 @@ func TestKMSGenerateCipherData(t *testing.T) { } } -func TestKMSDecrypt(t *testing.T) { +func TestKmsKeyHandler_DecryptKey(t *testing.T) { key, _ := hex.DecodeString("31bdadd96698c204aa9ce1448ea94ae1fb4a9a0b3c9d773b51bb1822666b8f22") keyB64 := base64.URLEncoding.EncodeToString(key) ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + body, err := ioutil.ReadAll(r.Body) + if err != nil { + t.Errorf("expected no error, got %v", err) + w.WriteHeader(500) + return + } + if bytes.Contains(body, []byte(`"KeyId":"test"`)) { + t.Errorf("expected CMK to not be sent") + } fmt.Fprintln(w, fmt.Sprintf("%s%s%s", `{"KeyId":"test-key-id","Plaintext":"`, keyB64, `"}`)) })) defer ts.Close() @@ -92,7 +100,7 @@ func TestKMSDecrypt(t *testing.T) { S3ForcePathStyle: aws.Bool(true), Region: aws.String("us-west-2"), }) - handler, err := (kmsKeyHandler{kms: kms.New(sess)}).decryptHandler(Envelope{MatDesc: `{"kms_cmk_id":"test"}`}) + handler, err := (kmsKeyHandler{kms: kms.New(sess)}).decryptHandler(Envelope{WrapAlg: KMSWrap, MatDesc: `{"kms_cmk_id":"test"}`}) if err != nil { t.Errorf("expected no error, but received %v", err) } @@ -107,37 +115,22 @@ func TestKMSDecrypt(t *testing.T) { } } -func TestKMSContextGenerateCipherData(t *testing.T) { +func TestKmsKeyHandler_DecryptKey_WithCMK(t *testing.T) { + key, _ := hex.DecodeString("31bdadd96698c204aa9ce1448ea94ae1fb4a9a0b3c9d773b51bb1822666b8f22") + keyB64 := base64.URLEncoding.EncodeToString(key) ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - bodyBytes, err := ioutil.ReadAll(r.Body) + body, err := ioutil.ReadAll(r.Body) if err != nil { - w.WriteHeader(500) - return - } - var body map[string]interface{} - err = json.Unmarshal(bodyBytes, &body) - if err != nil { - w.WriteHeader(500) - return - } - - md, ok := body["EncryptionContext"].(map[string]interface{}) - if !ok { + t.Errorf("expected no error, got %v", err) w.WriteHeader(500) return } - exEncContext := map[string]interface{}{ - "aws:" + cekAlgorithmHeader: "cekAlgValue", - } - - if e, a := exEncContext, md; !reflect.DeepEqual(e, a) { - w.WriteHeader(500) - t.Errorf("expected %v, got %v", e, a) - return + if !bytes.Contains(body, []byte(`"KeyId":"thisKey"`)) { + t.Errorf("expected CMK to be sent") } - fmt.Fprintln(w, `{"CiphertextBlob":"AQEDAHhqBCCY1MSimw8gOGcUma79cn4ANvTtQyv9iuBdbcEF1QAAAH4wfAYJKoZIhvcNAQcGoG8wbQIBADBoBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDJ6IcN5E4wVbk38MNAIBEIA7oF1E3lS7FY9DkoxPc/UmJsEwHzL82zMqoLwXIvi8LQHr8If4Lv6zKqY8u0+JRgSVoqCvZDx3p8Cn6nM=","KeyId":"arn:aws:kms:us-west-2:042062605278:key/c80a5cdb-8d09-4f9f-89ee-df01b2e3870a","Plaintext":"6tmyz9JLBE2yIuU7iXpArqpDVle172WSmxjcO6GNT7E="}`) + fmt.Fprintln(w, fmt.Sprintf("%s%s%s", `{"KeyId":"test-key-id","Plaintext":"`, keyB64, `"}`)) })) defer ts.Close() @@ -148,79 +141,54 @@ func TestKMSContextGenerateCipherData(t *testing.T) { S3ForcePathStyle: aws.Bool(true), Region: aws.String("us-west-2"), }) - - svc := kms.New(sess) - handler := NewKMSContextKeyGenerator(svc, "testid") - - keySize := 32 - ivSize := 16 - - cd, err := handler.GenerateCipherDataWithCEKAlg(keySize, ivSize, "cekAlgValue") + handler, err := newKMSWrapEntryWithCMK(kms.New(sess), "thisKey")(Envelope{WrapAlg: KMSWrap, MatDesc: `{"kms_cmk_id":"test"}`}) if err != nil { t.Errorf("expected no error, but received %v", err) } - if keySize != len(cd.Key) { - t.Errorf("expected %d, but received %d", keySize, len(cd.Key)) + + plaintextKey, err := handler.DecryptKey([]byte{1, 2, 3, 4}) + if err != nil { + t.Errorf("expected no error, but received %v", err) } - if ivSize != len(cd.IV) { - t.Errorf("expected %d, but received %d", ivSize, len(cd.IV)) + if !bytes.Equal(key, plaintextKey) { + t.Errorf("expected %v, but received %v", key, plaintextKey) } } -func TestKMSContextDecrypt(t *testing.T) { - key, _ := hex.DecodeString("31bdadd96698c204aa9ce1448ea94ae1fb4a9a0b3c9d773b51bb1822666b8f22") - keyB64 := base64.URLEncoding.EncodeToString(key) - ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - bodyBytes, err := ioutil.ReadAll(r.Body) - if err != nil { - w.WriteHeader(500) - return - } - var body map[string]interface{} - err = json.Unmarshal(bodyBytes, &body) - if err != nil { - w.WriteHeader(500) - return - } +func TestRegisterKMSWrapWithAnyCMK(t *testing.T) { + kmsClient := kms.New(unit.Session.Copy()) - md, ok := body["EncryptionContext"].(map[string]interface{}) - if !ok { - w.WriteHeader(500) - return - } + cr := NewCryptoRegistry() + if err := RegisterKMSWrapWithAnyCMK(cr, kmsClient); err != nil { + t.Errorf("expected no error, got %v", err) + } - exEncContext := map[string]interface{}{ - "aws:" + cekAlgorithmHeader: "cekAlgValue", - } + if wrap, ok := cr.GetWrap(KMSWrap); !ok { + t.Errorf("expected wrapped to be present") + } else if wrap == nil { + t.Errorf("expected wrap to not be nil") + } - if e, a := exEncContext, md; !reflect.DeepEqual(e, a) { - w.WriteHeader(500) - t.Errorf("expected %v, got %v", e, a) - return - } + if err := RegisterKMSWrapWithCMK(cr, kmsClient, "test-key-id"); err == nil { + t.Error("expected error, got none") + } +} - fmt.Fprintln(w, fmt.Sprintf("%s%s%s", `{"KeyId":"test-key-id","Plaintext":"`, keyB64, `"}`)) - })) - defer ts.Close() +func TestRegisterKMSWrapWithCMK(t *testing.T) { + kmsClient := kms.New(unit.Session.Copy()) - sess := unit.Session.Copy(&aws.Config{ - MaxRetries: aws.Int(0), - Endpoint: aws.String(ts.URL), - DisableSSL: aws.Bool(true), - S3ForcePathStyle: aws.Bool(true), - Region: aws.String("us-west-2"), - }) - handler, err := NewKMSContextWrapEntry(kms.New(sess))(Envelope{MatDesc: `{"aws:x-amz-cek-alg": "cekAlgValue"}`}) - if err != nil { - t.Errorf("expected no error, but received %v", err) + cr := NewCryptoRegistry() + if err := RegisterKMSWrapWithCMK(cr, kmsClient, "cmkId"); err != nil { + t.Errorf("expected no error, got %v", err) } - plaintextKey, err := handler.DecryptKey([]byte{1, 2, 3, 4}) - if err != nil { - t.Errorf("expected no error, but received %v", err) + if wrap, ok := cr.GetWrap(KMSWrap); !ok { + t.Errorf("expected wrapped to be present") + } else if wrap == nil { + t.Errorf("expected wrap to not be nil") } - if !bytes.Equal(key, plaintextKey) { - t.Errorf("expected %v, but received %v", key, plaintextKey) + if err := RegisterKMSWrapWithAnyCMK(cr, kmsClient); err == nil { + t.Error("expected error, got none") } } diff --git a/service/s3/s3crypto/mat_desc.go b/service/s3/s3crypto/mat_desc.go index 301e20dad1d..315ef90b1e1 100644 --- a/service/s3/s3crypto/mat_desc.go +++ b/service/s3/s3crypto/mat_desc.go @@ -8,6 +8,18 @@ import ( // key has been used. type MaterialDescription map[string]*string +// Clone returns a copy of the MaterialDescription +func (md MaterialDescription) Clone() (clone MaterialDescription) { + if md == nil { + return nil + } + clone = make(MaterialDescription, len(md)) + for k, v := range md { + clone[k] = copyPtrString(v) + } + return clone +} + func (md *MaterialDescription) encodeDescription() ([]byte, error) { v, err := json.Marshal(&md) return v, err @@ -16,3 +28,11 @@ func (md *MaterialDescription) encodeDescription() ([]byte, error) { func (md *MaterialDescription) decodeDescription(b []byte) error { return json.Unmarshal(b, &md) } + +func copyPtrString(v *string) *string { + if v == nil { + return nil + } + ns := *v + return &ns +} diff --git a/service/s3/s3crypto/mat_desc_test.go b/service/s3/s3crypto/mat_desc_test.go index f87051b04be..056c56d89cd 100644 --- a/service/s3/s3crypto/mat_desc_test.go +++ b/service/s3/s3crypto/mat_desc_test.go @@ -1,3 +1,5 @@ +// +build go1.7 + package s3crypto import ( @@ -33,3 +35,32 @@ func TestDecodeMaterialDescription(t *testing.T) { t.Error("expected material description to be equivalent, but received otherwise") } } + +func TestMaterialDescription_Clone(t *testing.T) { + tests := map[string]struct { + md MaterialDescription + wantClone MaterialDescription + }{ + "it handles nil": { + md: nil, + wantClone: nil, + }, + "it copies all values": { + md: MaterialDescription{ + "key1": aws.String("value1"), + "key2": aws.String("value2"), + }, + wantClone: MaterialDescription{ + "key1": aws.String("value1"), + "key2": aws.String("value2"), + }, + }, + } + for name, tt := range tests { + t.Run(name, func(t *testing.T) { + if gotClone := tt.md.Clone(); !reflect.DeepEqual(gotClone, tt.wantClone) { + t.Errorf("Clone() = %v, want %v", gotClone, tt.wantClone) + } + }) + } +} diff --git a/service/s3/s3crypto/migrations_test.go b/service/s3/s3crypto/migrations_test.go index 6d76809d3cb..b4b353bc15b 100644 --- a/service/s3/s3crypto/migrations_test.go +++ b/service/s3/s3crypto/migrations_test.go @@ -24,12 +24,12 @@ func ExampleNewEncryptionClientV2_migration00() { // Usage of NewKMSKeyGenerator (kms) key wrapping algorithm must be migrated to NewKMSContextKeyGenerator (kms+context) key wrapping algorithm // // cipherDataGenerator := s3crypto.NewKMSKeyGenerator(kmsClient, cmkID) - cipherDataGenerator := s3crypto.NewKMSContextKeyGenerator(kmsClient, cmkID) + cipherDataGenerator := s3crypto.NewKMSContextKeyGenerator(kmsClient, cmkID, s3crypto.MaterialDescription{}) // Usage of AESCBCContentCipherBuilder (AES/CBC/PKCS5Padding) must be migrated to AESGCMContentCipherBuilder (AES/GCM/NoPadding) // // contentCipherBuilder := s3crypto.AESCBCContentCipherBuilder(cipherDataGenerator, s3crypto.AESCBCPadder) - contentCipherBuilder := s3crypto.AESGCMContentCipherBuilder(cipherDataGenerator) + contentCipherBuilder := s3crypto.AESGCMContentCipherBuilderV2(cipherDataGenerator) // Construction of an encryption client should be done using NewEncryptionClientV2 // @@ -59,9 +59,9 @@ func ExampleNewEncryptionClientV2_migration01() { kmsClient := kms.New(sess) cmkID := "1234abcd-12ab-34cd-56ef-1234567890ab" - cipherDataGenerator := s3crypto.NewKMSContextKeyGenerator(kmsClient, cmkID) + cipherDataGenerator := s3crypto.NewKMSContextKeyGenerator(kmsClient, cmkID, s3crypto.MaterialDescription{}) - contentCipherBuilder := s3crypto.AESGCMContentCipherBuilder(cipherDataGenerator) + contentCipherBuilder := s3crypto.AESGCMContentCipherBuilderV2(cipherDataGenerator) // Overriding of the encryption client options is possible by passing in functional arguments that override the // provided EncryptionClientOptions. @@ -98,7 +98,46 @@ func ExampleNewDecryptionClientV2_migration00() { // The V2 decryption client is able to decrypt object encrypted by the V1 client. // // decryptionClient := s3crypto.NewDecryptionClient(sess) - decryptionClient := s3crypto.NewDecryptionClientV2(sess) + + // The V2 decryption client requires you to explicitly register the key wrap algorithms and content encryption algorithms + // that you want to explicitly support decryption for. + registry := s3crypto.NewCryptoRegistry() + + kmsClient := kms.New(sess) + + // If you need support for unwrapping data keys wrapped using the `kms` wrap algorithm you can use RegisterKMSWrapWithAnyCMK. + // Alternatively you may use RegisterKMSWrapWithCMK if you wish to limit KMS decrypt calls to a specific CMK. + if err := s3crypto.RegisterKMSWrapWithAnyCMK(registry, kmsClient); err != nil { + fmt.Printf("error: %v", err) + return + } + + // For unwrapping data keys wrapped using the new `kms+context` key wrap algorithm you can use RegisterKMSContextWrapWithAnyCMK. + // Alternatively you may use RegisterKMSWrapWithCMK if you wish to limit KMS decrypt calls to a specific CMK. + if err := s3crypto.RegisterKMSContextWrapWithAnyCMK(registry, kmsClient); err != nil { + fmt.Printf("error: %v", err) + return + } + + // If you need to decrypt objects encrypted using the V1 AES/CBC/PCKS5Padding cipher you can do so with RegisterAESCBCContentCipher + if err := s3crypto.RegisterAESCBCContentCipher(registry, s3crypto.AESCBCPadder); err != nil { + fmt.Printf("error: %v", err) + return + } + + // For decrypting objects encrypted in V1 or V2 using AES/GCM/NoPadding cipher you can do so with RegisterAESGCMContentCipher. + if err := s3crypto.RegisterAESGCMContentCipher(registry); err != nil { + fmt.Printf("error: %v", err) + return + } + + // Instantiate a new decryption client, and provided the Wrap, cek, and Padder that have been registered + // with your desired algorithms. + decryptionClient, err := s3crypto.NewDecryptionClientV2(sess, registry) + if err != nil { + fmt.Printf("error: %v", err) + return + } getObject, err := decryptionClient.GetObject(&s3.GetObjectInput{ Bucket: aws.String("your_bucket"), @@ -127,9 +166,27 @@ func ExampleNewDecryptionClientV2_migration01() { // decryptionClient := s3crypto.NewDecryptionClient(sess, func(o *s3crypto.DecryptionClient) { // o.S3Client = s3.New(sess, &aws.Config{Region: aws.String("us-west-2")}) //}) - decryptionClient := s3crypto.NewDecryptionClientV2(sess, func(o *s3crypto.DecryptionClientOptions) { + registry := s3crypto.NewCryptoRegistry() + + kmsClient := kms.New(sess) + if err := s3crypto.RegisterKMSWrapWithAnyCMK(registry, kmsClient); err != nil { + fmt.Printf("error: %v", err) + return + } + + // If you need to decrypt objects encrypted using AES/GCM/NoPadding cipher you can do so with RegisterAESGCMContentCipher + if err := s3crypto.RegisterAESGCMContentCipher(registry); err != nil { + fmt.Printf("error: %v", err) + return + } + + decryptionClient, err := s3crypto.NewDecryptionClientV2(sess, registry, func(o *s3crypto.DecryptionClientOptions) { o.S3Client = s3.New(sess, &aws.Config{Region: aws.String("us-west-2")}) }) + if err != nil { + fmt.Printf("error: %v", err) + return + } getObject, err := decryptionClient.GetObject(&s3.GetObjectInput{ Bucket: aws.String("your_bucket"), diff --git a/service/s3/s3crypto/mock_test.go b/service/s3/s3crypto/mock_test.go index ac96d4cbf37..fa110af0549 100644 --- a/service/s3/s3crypto/mock_test.go +++ b/service/s3/s3crypto/mock_test.go @@ -1,38 +1,62 @@ -package s3crypto_test +package s3crypto import ( "bytes" + "fmt" "io" "io/ioutil" - "github.com/aws/aws-sdk-go/service/s3/s3crypto" + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/kms/kmsiface" ) type mockGenerator struct{} -func (m mockGenerator) GenerateCipherData(keySize, ivSize int) (s3crypto.CipherData, error) { - cd := s3crypto.CipherData{ +func (m mockGenerator) GenerateCipherData(keySize, ivSize int) (CipherData, error) { + cd := CipherData{ Key: make([]byte, keySize), IV: make([]byte, ivSize), } return cd, nil } -func (m mockGenerator) EncryptKey(key []byte) ([]byte, error) { - size := len(key) - b := bytes.Repeat([]byte{1}, size) - return b, nil +func (m mockGenerator) DecryptKey(key []byte) ([]byte, error) { + return make([]byte, 16), nil } -func (m mockGenerator) DecryptKey(key []byte) ([]byte, error) { +type mockGeneratorV2 struct{} + +func (m mockGeneratorV2) GenerateCipherDataWithCEKAlg(ctx aws.Context, keySize int, ivSize int, cekAlg string) (CipherData, error) { + cd := CipherData{ + Key: make([]byte, keySize), + IV: make([]byte, ivSize), + } + return cd, nil +} + +func (m mockGeneratorV2) DecryptKey(key []byte) ([]byte, error) { return make([]byte, 16), nil } +func (m mockGeneratorV2) isEncryptionVersionCompatible(version clientVersion) error { + if version != v2ClientVersion { + return fmt.Errorf("mock error about version") + } + return nil +} + type mockCipherBuilder struct { - generator s3crypto.CipherDataGenerator + generator CipherDataGenerator } -func (builder mockCipherBuilder) ContentCipher() (s3crypto.ContentCipher, error) { +func (builder mockCipherBuilder) isEncryptionVersionCompatible(version clientVersion) error { + if version != v1ClientVersion { + return fmt.Errorf("mock error about version") + } + return nil +} + +func (builder mockCipherBuilder) ContentCipher() (ContentCipher, error) { cd, err := builder.generator.GenerateCipherData(32, 16) if err != nil { return nil, err @@ -40,11 +64,30 @@ func (builder mockCipherBuilder) ContentCipher() (s3crypto.ContentCipher, error) return &mockContentCipher{cd}, nil } +type mockCipherBuilderV2 struct { + generator CipherDataGeneratorWithCEKAlg +} + +func (builder mockCipherBuilderV2) isEncryptionVersionCompatible(version clientVersion) error { + if version != v2ClientVersion { + return fmt.Errorf("mock error about version") + } + return nil +} + +func (builder mockCipherBuilderV2) ContentCipher() (ContentCipher, error) { + cd, err := builder.generator.GenerateCipherDataWithCEKAlg(aws.BackgroundContext(), 32, 16, "mock-cek-alg") + if err != nil { + return nil, err + } + return &mockContentCipher{cd}, nil +} + type mockContentCipher struct { - cd s3crypto.CipherData + cd CipherData } -func (cipher *mockContentCipher) GetCipherData() s3crypto.CipherData { +func (cipher *mockContentCipher) GetCipherData() CipherData { return cipher.cd } @@ -66,3 +109,22 @@ func (cipher *mockContentCipher) DecryptContents(src io.ReadCloser) (io.ReadClos size := len(b) return ioutil.NopCloser(bytes.NewReader(make([]byte, size))), nil } + +type mockKMS struct { + kmsiface.KMSAPI +} + +type mockPadder struct { +} + +func (m mockPadder) Pad(i []byte, i2 int) ([]byte, error) { + return i, nil +} + +func (m mockPadder) Unpad(i []byte) ([]byte, error) { + return i, nil +} + +func (m mockPadder) Name() string { + return "mockPadder" +} diff --git a/service/s3/s3crypto/shared_client.go b/service/s3/s3crypto/shared_client.go index 2c967d07467..1ad5eea0c0a 100644 --- a/service/s3/s3crypto/shared_client.go +++ b/service/s3/s3crypto/shared_client.go @@ -13,6 +13,12 @@ import ( "github.com/aws/aws-sdk-go/service/s3" ) +// clientConstructionErrorCode is used for operations that can't be completed due to invalid client construction +const clientConstructionErrorCode = "ClientConstructionError" + +// mismatchWrapError is an error returned if a wrapping handler receives an unexpected envelope +var mismatchWrapError = awserr.New(clientConstructionErrorCode, "wrap algorithm provided did not match handler", nil) + func putObjectRequest(c EncryptionClientOptions, input *s3.PutObjectInput) (*request.Request, *s3.PutObjectOutput) { req, out := c.S3Client.PutObjectRequest(input) @@ -45,9 +51,9 @@ func putObjectRequest(c EncryptionClientOptions, input *s3.PutObjectInput) (*req return } - md5 := newMD5Reader(input.Body) + lengthReader := newContentLengthReader(input.Body) sha := newSHA256Writer(dst) - reader, err := encryptor.EncryptContents(md5) + reader, err := encryptor.EncryptContents(lengthReader) if err != nil { r.Error = err return @@ -60,7 +66,7 @@ func putObjectRequest(c EncryptionClientOptions, input *s3.PutObjectInput) (*req } data := encryptor.GetCipherData() - env, err := encodeMeta(md5, data) + env, err := encodeMeta(lengthReader, data) if err != nil { r.Error = err return @@ -101,7 +107,7 @@ func getObjectRequest(options DecryptionClientOptions, input *s3.GetObjectInput) return } - // If KMS should return the correct CEK algorithm with the proper + // If KMS should return the correct cek algorithm with the proper // KMS key provider cipher, err := contentCipherFromEnvelope(options, r.Context(), env) if err != nil { @@ -143,8 +149,9 @@ func contentCipherFromEnvelope(options DecryptionClientOptions, ctx aws.Context, } func wrapFromEnvelope(options DecryptionClientOptions, env Envelope) (CipherDataDecrypter, error) { - f, ok := options.WrapRegistry[env.WrapAlg] + f, ok := options.CryptoRegistry.GetWrap(env.WrapAlg) if !ok || f == nil { + return nil, awserr.New( "InvalidWrapAlgorithmError", "wrap algorithm isn't supported, "+env.WrapAlg, @@ -155,7 +162,7 @@ func wrapFromEnvelope(options DecryptionClientOptions, env Envelope) (CipherData } func cekFromEnvelope(options DecryptionClientOptions, ctx aws.Context, env Envelope, decrypter CipherDataDecrypter) (ContentCipher, error) { - f, ok := options.CEKRegistry[env.CEKAlg] + f, ok := options.CryptoRegistry.GetCEK(env.CEKAlg) if !ok || f == nil { return nil, awserr.New( "InvalidCEKAlgorithmError", @@ -197,11 +204,11 @@ func cekFromEnvelope(options DecryptionClientOptions, ctx aws.Context, env Envel // If there wasn't a cek algorithm specific padder, we check the padder itself. // We return a no unpadder, if no unpadder was found. This means any customization // either contained padding within the cipher implementation, and to maintain -// backwards compatility we will simply not unpad anything. +// backwards compatibility we will simply not unpad anything. func getPadder(options DecryptionClientOptions, cekAlg string) Padder { - padder, ok := options.PadderRegistry[cekAlg] + padder, ok := options.CryptoRegistry.GetPadder(cekAlg) if !ok { - padder, ok = options.PadderRegistry[cekAlg[strings.LastIndex(cekAlg, "/")+1:]] + padder, ok = options.CryptoRegistry.GetPadder(cekAlg[strings.LastIndex(cekAlg, "/")+1:]) if !ok { return NoPadder } diff --git a/service/s3/s3crypto/strategy.go b/service/s3/s3crypto/strategy.go index 390b151a6bd..00e7646e6c1 100644 --- a/service/s3/s3crypto/strategy.go +++ b/service/s3/s3crypto/strategy.go @@ -118,7 +118,6 @@ func (load HeaderV2LoadStrategy) Load(req *request.Request) (Envelope, error) { env.WrapAlg = req.HTTPResponse.Header.Get(strings.Join([]string{metaHeader, wrapAlgorithmHeader}, "-")) env.CEKAlg = req.HTTPResponse.Header.Get(strings.Join([]string{metaHeader, cekAlgorithmHeader}, "-")) env.TagLen = req.HTTPResponse.Header.Get(strings.Join([]string{metaHeader, tagLengthHeader}, "-")) - env.UnencryptedMD5 = req.HTTPResponse.Header.Get(strings.Join([]string{metaHeader, unencryptedMD5Header}, "-")) env.UnencryptedContentLen = req.HTTPResponse.Header.Get(strings.Join([]string{metaHeader, unencryptedContentLengthHeader}, "-")) return env, nil }