Skip to content

Commit

Permalink
fix: support datadog plugin api key to allow for secret injection
Browse files Browse the repository at this point in the history 
Signed-off-by: nitintecg <nitin.tejuja12@gmail.com>
  • Loading branch information
@nitintecg
nitintecg committed Mar 11, 2024
1 parent 2f49673 commit e1ef3b7
Show file tree
Hide file tree
Showing 18 changed files with 312 additions and 51 deletions.
22 changes: 11 additions & 11 deletions apis/fluentbit/v1alpha2/plugins/filter/multiline_types.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,8 +21,8 @@ type Multi struct {
Parser string `json:"parser"`
//Key name that holds the content to process.
//Note that a Multiline Parser definition can already specify the key_content to use, but this option allows to overwrite that value for the purpose of the filter.
KeyContent string `json:"keyContent,omitempty"`
EmitterName string `json:"emitterName,omitempty"`
KeyContent string `json:"keyContent,omitempty"`
EmitterName string `json:"emitterName,omitempty"`
EmitterMemBufLimit string `json:"emitterMemBufLimit,omitempty"`
EmitterStorageType string `json:"emitterStorageType,omitempty"`
}
Expand All @@ -44,15 +44,15 @@ func (m *Multiline) Params(_ plugins.SecretLoader) (*params.KVs, error) {
if m.Multi.KeyContent != "" {
kvs.Insert("multiline.key_content", m.Multi.KeyContent)
}
if m.Multi.EmitterName != "" {
kvs.Insert("Emitter_Name", m.Multi.EmitterName)
}
if m.Multi.EmitterMemBufLimit != "" {
kvs.Insert("Emitter_Mem_Buf_Limit", m.Multi.EmitterMemBufLimit)
}
if m.Multi.EmitterStorageType != "" {
kvs.Insert("Emitter_Storage.type", m.Multi.EmitterStorageType)
}
if m.Multi.EmitterName != "" {
kvs.Insert("Emitter_Name", m.Multi.EmitterName)
}
if m.Multi.EmitterMemBufLimit != "" {
kvs.Insert("Emitter_Mem_Buf_Limit", m.Multi.EmitterMemBufLimit)
}
if m.Multi.EmitterStorageType != "" {
kvs.Insert("Emitter_Storage.type", m.Multi.EmitterStorageType)
}
}
return kvs, nil
}
2 changes: 1 addition & 1 deletion apis/fluentbit/v1alpha2/plugins/filter/rewritetag_types.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ type RewriteTag struct {
// When the filter emits a record under the new Tag, there is an internal emitter
// plugin that takes care of the job. Since this emitter expose metrics as any other
// component of the pipeline, you can use this property to configure an optional name for it.
EmitterName string `json:"emitterName,omitempty"`
EmitterName string `json:"emitterName,omitempty"`
EmitterMemBufLimit string `json:"emitterMemBufLimit,omitempty"`
EmitterStorageType string `json:"emitterStorageType,omitempty"`
}
Expand Down
10 changes: 7 additions & 3 deletions apis/fluentbit/v1alpha2/plugins/output/datadog_types.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ type DataDog struct {
// Datadog supports and recommends setting this to gzip.
Compress string `json:"compress,omitempty"`
// Your Datadog API key.
APIKey string `json:"apikey,omitempty"`
APIKey *plugins.Secret `json:"apikey,omitempty"`
// Specify an HTTP Proxy.
Proxy string `json:"proxy,omitempty"`
// To activate the remapping, specify configuration flag provider.
Expand Down Expand Up @@ -61,8 +61,12 @@ func (s *DataDog) Params(sl plugins.SecretLoader) (*params.KVs, error) {
if s.Compress != "" {
kvs.Insert("compress", s.Compress)
}
if s.APIKey != "" {
kvs.Insert("apikey", s.APIKey)
if s.APIKey != nil {
apiKey, err := sl.LoadSecret(*s.APIKey)
if err != nil {
return nil, err
}
kvs.Insert("apikey", apiKey)
}
if s.Proxy != "" {
kvs.Insert("proxy", s.Proxy)
Expand Down
2 changes: 0 additions & 2 deletions apis/fluentbit/v1alpha2/plugins/output/datadog_types_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,6 @@ func TestOutput_DataDog_Params(t *testing.T) {

dd := DataDog{
Host: "http-intake.logs.datadoghq.com",
APIKey: "1234apikey",
TLS: ptrBool(true),
Compress: "gzip",
Service: "service_name",
Expand All @@ -31,7 +30,6 @@ func TestOutput_DataDog_Params(t *testing.T) {
expected.Insert("Host", "http-intake.logs.datadoghq.com")
expected.Insert("TLS", "true")
expected.Insert("compress", "gzip")
expected.Insert("apikey", "1234apikey")
expected.Insert("json_date_key", "timestamp")
expected.Insert("include_tag_key", "true")
expected.Insert("tag_key", "tagkey")
Expand Down
5 changes: 5 additions & 0 deletions apis/fluentbit/v1alpha2/plugins/output/zz_generated.deepcopy.go
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

29 changes: 4 additions & 25 deletions charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_clusterfilters.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -486,24 +486,11 @@ spec:
alias:
description: Alias for the plugin
type: string
emitterMemBufLimit:
type: string
emitterName:
description: When the filter emits a record under the new
Tag, there is an internal emitter plugin that takes care
of the job. Since this emitter expose metrics as any other
component of the pipeline, you can use this property to
configure an optional name for it.
type: string
emitterStorageType:
description: Specify the emitter buffering mechanism to use. It can be
memory or filesystem
enum:
- filesystem
- memory
type: string
emitterMemBufLimit:
description: Set a limit of memory that Emitter plugin can use when
appending data to the Engine. If the limit is reach, it will
be paused; when the data is flushed it resumes.
type: string
keyContent:
description: Key name that holds the content to process.
Expand Down Expand Up @@ -659,6 +646,8 @@ spec:
alias:
description: Alias for the plugin
type: string
emitterMemBufLimit:
type: string
emitterName:
description: When the filter emits a record under the new
Tag, there is an internal emitter plugin that takes care
Expand All @@ -667,16 +656,6 @@ spec:
configure an optional name for it.
type: string
emitterStorageType:
description: Specify the emitter buffering mechanism to use. It can be
memory or filesystem
enum:
- filesystem
- memory
type: string
emitterMemBufLimit:
description: Set a limit of memory that Emitter plugin can use when
appending data to the Engine. If the limit is reach, it will
be paused; when the data is flushed it resumes.
type: string
retryLimit:
description: 'RetryLimit describes how many times fluent-bit
Expand Down
27 changes: 26 additions & 1 deletion charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_clusteroutputs.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -348,7 +348,32 @@ spec:
properties:
apikey:
description: Your Datadog API key.
type: string
properties:
valueFrom:
description: ValueSource defines how to find a value's key.
properties:
secretKeyRef:
description: Selects a key of a secret in the pod's namespace
properties:
key:
description: The key of the secret to select from. Must
be a valid secret key.
type: string
name:
description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
TODO: Add other useful fields. apiVersion, kind,
uid?'
type: string
optional:
description: Specify whether the Secret or its key
must be defined
type: boolean
required:
- key
type: object
x-kubernetes-map-type: atomic
type: object
type: object
compress:
description: Compress the payload in GZIP format. Datadog supports
and recommends setting this to gzip.
Expand Down
10 changes: 10 additions & 0 deletions charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_filters.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -488,6 +488,12 @@ spec:
alias:
description: Alias for the plugin
type: string
emitterMemBufLimit:
type: string
emitterName:
type: string
emitterStorageType:
type: string
keyContent:
description: Key name that holds the content to process.
Note that a Multiline Parser definition can already specify
Expand Down Expand Up @@ -642,13 +648,17 @@ spec:
alias:
description: Alias for the plugin
type: string
emitterMemBufLimit:
type: string
emitterName:
description: When the filter emits a record under the new
Tag, there is an internal emitter plugin that takes care
of the job. Since this emitter expose metrics as any other
component of the pipeline, you can use this property to
configure an optional name for it.
type: string
emitterStorageType:
type: string
retryLimit:
description: 'RetryLimit describes how many times fluent-bit
should retry to send data to a specific output. If set
Expand Down
27 changes: 26 additions & 1 deletion charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_outputs.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -348,7 +348,32 @@ spec:
properties:
apikey:
description: Your Datadog API key.
type: string
properties:
valueFrom:
description: ValueSource defines how to find a value's key.
properties:
secretKeyRef:
description: Selects a key of a secret in the pod's namespace
properties:
key:
description: The key of the secret to select from. Must
be a valid secret key.
type: string
name:
description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
TODO: Add other useful fields. apiVersion, kind,
uid?'
type: string
optional:
description: Specify whether the Secret or its key
must be defined
type: boolean
required:
- key
type: object
x-kubernetes-map-type: atomic
type: object
type: object
compress:
description: Compress the payload in GZIP format. Datadog supports
and recommends setting this to gzip.
Expand Down
10 changes: 10 additions & 0 deletions config/crd/bases/fluentbit.fluent.io_clusterfilters.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -486,6 +486,12 @@ spec:
alias:
description: Alias for the plugin
type: string
emitterMemBufLimit:
type: string
emitterName:
type: string
emitterStorageType:
type: string
keyContent:
description: Key name that holds the content to process.
Note that a Multiline Parser definition can already specify
Expand Down Expand Up @@ -640,13 +646,17 @@ spec:
alias:
description: Alias for the plugin
type: string
emitterMemBufLimit:
type: string
emitterName:
description: When the filter emits a record under the new
Tag, there is an internal emitter plugin that takes care
of the job. Since this emitter expose metrics as any other
component of the pipeline, you can use this property to
configure an optional name for it.
type: string
emitterStorageType:
type: string
retryLimit:
description: 'RetryLimit describes how many times fluent-bit
should retry to send data to a specific output. If set
Expand Down
27 changes: 26 additions & 1 deletion config/crd/bases/fluentbit.fluent.io_clusteroutputs.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -348,7 +348,32 @@ spec:
properties:
apikey:
description: Your Datadog API key.
type: string
properties:
valueFrom:
description: ValueSource defines how to find a value's key.
properties:
secretKeyRef:
description: Selects a key of a secret in the pod's namespace
properties:
key:
description: The key of the secret to select from. Must
be a valid secret key.
type: string
name:
description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
TODO: Add other useful fields. apiVersion, kind,
uid?'
type: string
optional:
description: Specify whether the Secret or its key
must be defined
type: boolean
required:
- key
type: object
x-kubernetes-map-type: atomic
type: object
type: object
compress:
description: Compress the payload in GZIP format. Datadog supports
and recommends setting this to gzip.
Expand Down
10 changes: 10 additions & 0 deletions config/crd/bases/fluentbit.fluent.io_filters.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -486,6 +486,12 @@ spec:
alias:
description: Alias for the plugin
type: string
emitterMemBufLimit:
type: string
emitterName:
type: string
emitterStorageType:
type: string
keyContent:
description: Key name that holds the content to process.
Note that a Multiline Parser definition can already specify
Expand Down Expand Up @@ -640,13 +646,17 @@ spec:
alias:
description: Alias for the plugin
type: string
emitterMemBufLimit:
type: string
emitterName:
description: When the filter emits a record under the new
Tag, there is an internal emitter plugin that takes care
of the job. Since this emitter expose metrics as any other
component of the pipeline, you can use this property to
configure an optional name for it.
type: string
emitterStorageType:
type: string
retryLimit:
description: 'RetryLimit describes how many times fluent-bit
should retry to send data to a specific output. If set
Expand Down
27 changes: 26 additions & 1 deletion config/crd/bases/fluentbit.fluent.io_outputs.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -348,7 +348,32 @@ spec:
properties:
apikey:
description: Your Datadog API key.
type: string
properties:
valueFrom:
description: ValueSource defines how to find a value's key.
properties:
secretKeyRef:
description: Selects a key of a secret in the pod's namespace
properties:
key:
description: The key of the secret to select from. Must
be a valid secret key.
type: string
name:
description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
TODO: Add other useful fields. apiVersion, kind,
uid?'
type: string
optional:
description: Specify whether the Secret or its key
must be defined
type: boolean
required:
- key
type: object
x-kubernetes-map-type: atomic
type: object
type: object
compress:
description: Compress the payload in GZIP format. Datadog supports
and recommends setting this to gzip.
Expand Down
3 changes: 3 additions & 0 deletions docs/plugins/fluentbit/filter/multiline.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,3 +7,6 @@
| ----- | ----------- | ------ |
| parser | Specify one or multiple Multiline Parsing definitions to apply to the content. You can specify multiple multiline parsers to detect different formats by separating them with a comma. | string |
| keyContent | Key name that holds the content to process. Note that a Multiline Parser definition can already specify the key_content to use, but this option allows to overwrite that value for the purpose of the filter. | string |
| emitterName | | string |
| emitterMemBufLimit | | string |
| emitterStorageType | | string |
2 changes: 2 additions & 0 deletions docs/plugins/fluentbit/filter/rewritetag.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,3 +7,5 @@ RewriteTag define a `rewrite_tag` filter, allows to re-emit a record under a new
| ----- | ----------- | ------ |
| rules | Defines the matching criteria and the format of the Tag for the matching record. The Rule format have four components: KEY REGEX NEW_TAG KEEP. | []string |
| emitterName | When the filter emits a record under the new Tag, there is an internal emitter plugin that takes care of the job. Since this emitter expose metrics as any other component of the pipeline, you can use this property to configure an optional name for it. | string |
| emitterMemBufLimit | | string |
| emitterStorageType | | string |
Loading

0 comments on commit e1ef3b7

Please sign in to comment.