OpenSearch Spike

docker-compose.yml

@@ -638,6 +638,8 @@ services:
       resources:
         limits:
           memory: 100M
+    environment:
+      - "GF_INSTALL_PLUGINS=grafana-opensearch-datasource"
     volumes:
       - ./src/grafana/grafana.ini:/etc/grafana/grafana.ini
       - ./src/grafana/provisioning/:/etc/grafana/provisioning/
@@ -692,6 +694,37 @@ services:
       - "${PROMETHEUS_SERVICE_PORT}:${PROMETHEUS_SERVICE_PORT}"
     logging: *logging
 
+  opensearch:
+    image: opensearchproject/opensearch:latest
+    container_name: opensearch
+    environment:
+      - cluster.name=demo-cluster
+      - node.name=demo-node
+      - bootstrap.memory_lock=true
+      - discovery.type=single-node
+      - "OPENSEARCH_JAVA_OPTS=-Xms512m -Xmx512m"
+      - "DISABLE_INSTALL_DEMO_CONFIG=true"
+      - "DISABLE_SECURITY_PLUGIN=true"
+    ulimits:
+      memlock:
+        soft: -1
+        hard: -1
+      nofile:
+        soft: 65536
+        hard: 65536
+    ports:
+      - "9200:9200"
+    logging: *logging
+
+  dataprepper:
+    image: opensearchproject/data-prepper:latest
+    volumes:
+      - ./src/opensearch/pipelines.yaml:/usr/share/data-prepper/pipelines/pipelines.yaml
+      - ./src/opensearch/data-prepper-config.yaml:/usr/share/data-prepper/config/data-prepper-config.yaml
+    ports:
+      - "21892"
+    logging: *logging
+
   # *****
   # Tests
   # *****

src/grafana/grafana.ini

@@ -406,7 +406,7 @@ serve_from_sub_path = true
 ;token_rotation_interval_minutes = 10
 
 # Set to true to disable (hide) the login form, useful if you use OAuth, defaults to false
-disable_login_form = true 
+disable_login_form = false 
 
 # Set to true to disable the sign out link in the side menu. Useful if you use auth.proxy or auth.jwt, defaults to false
 ;disable_signout_menu = false

src/grafana/provisioning/datasources/opensearch.yaml

@@ -0,0 +1,20 @@
+# Copyright The OpenTelemetry Authors
+# SPDX-License-Identifier: Apache-2.0
+
+apiVersion: 1
+
+datasources:
+  - name: OpenSearch
+    type: grafana-opensearch-datasource
+    url: http://opensearch:9200/
+    access: proxy
+    editable: true
+    isDefault: false
+    jsonData:
+      database: 'otel'
+      interval: Daily
+      timeField: 'time'
+      version: '2.10.0'
+      flavor: 'opensearch'
+      logMessageField: body
+      logLevelField: severitynumber

src/opensearch/data-prepper-config.yaml

@@ -0,0 +1,4 @@
+# Copyright The OpenTelemetry Authors
+# SPDX-License-Identifier: Apache-2.0
+
+ssl: false

src/opensearch/pipelines.yaml

@@ -0,0 +1,14 @@
+# Copyright The OpenTelemetry Authors
+# SPDX-License-Identifier: Apache-2.0
+
+demo-pipeline:
+  source:
+    otel_logs_source:
+      ssl: false
+  sink:
+    - opensearch:
+        hosts: ["http://opensearch:9200"]
+        insecure: true
+        username: admin
+        password: admin
+        index: otel

src/otelcollector/otelcol-observability.yml

@@ -7,6 +7,8 @@ exporters:
     endpoint: "jaeger:4317"
     tls:
       insecure: true
+  otlp/logs:
+    endpoint: "dataprepper:21892"
   otlphttp/prometheus:
     endpoint: "http://prometheus:9090/api/v1/otlp"
     tls:
@@ -18,3 +20,5 @@ service:
       exporters: [otlp, logging, spanmetrics]
     metrics:
       exporters: [otlphttp/prometheus, logging]
+    logs:
+      exporters: [otlp/logs, logging]