OpenSearch Spike (open-telemetry#1151)

* add opensearch * fix env * fix yamllint * fix lint * fix errors in datasource * remove stdout export
observIQ · Oct 25, 2023 · ed24c3a · ed24c3a
1 parent 2d00774
commit ed24c3a
Show file tree

Hide file tree

Showing 5 changed files with 77 additions and 0 deletions.
diff --git a/docker-compose.yml b/docker-compose.yml
@@ -638,6 +638,8 @@ services:
       resources:
         limits:
           memory: 100M
+    environment:
+      - "GF_INSTALL_PLUGINS=grafana-opensearch-datasource"
     volumes:
       - ./src/grafana/grafana.ini:/etc/grafana/grafana.ini
       - ./src/grafana/provisioning/:/etc/grafana/provisioning/
@@ -692,6 +694,37 @@ services:
       - "${PROMETHEUS_SERVICE_PORT}:${PROMETHEUS_SERVICE_PORT}"
     logging: *logging
 
+  opensearch:
+    image: opensearchproject/opensearch:latest
+    container_name: opensearch
+    environment:
+      - cluster.name=demo-cluster
+      - node.name=demo-node
+      - bootstrap.memory_lock=true
+      - discovery.type=single-node
+      - "OPENSEARCH_JAVA_OPTS=-Xms512m -Xmx512m"
+      - "DISABLE_INSTALL_DEMO_CONFIG=true"
+      - "DISABLE_SECURITY_PLUGIN=true"
+    ulimits:
+      memlock:
+        soft: -1
+        hard: -1
+      nofile:
+        soft: 65536
+        hard: 65536
+    ports:
+      - "9200:9200"
+    logging: *logging
+
+  dataprepper:
+    image: opensearchproject/data-prepper:latest
+    volumes:
+      - ./src/opensearch/pipelines.yaml:/usr/share/data-prepper/pipelines/pipelines.yaml
+      - ./src/opensearch/data-prepper-config.yaml:/usr/share/data-prepper/config/data-prepper-config.yaml
+    ports:
+      - "21892"
+    logging: *logging
+
   # *****
   # Tests
   # *****

diff --git a/src/grafana/provisioning/datasources/opensearch.yaml b/src/grafana/provisioning/datasources/opensearch.yaml
@@ -0,0 +1,20 @@
+# Copyright The OpenTelemetry Authors
+# SPDX-License-Identifier: Apache-2.0
+
+apiVersion: 1
+
+datasources:
+  - name: OpenSearch
+    type: grafana-opensearch-datasource
+    url: http://opensearch:9200/
+    access: proxy
+    editable: true
+    isDefault: false
+    jsonData:
+      database: 'otel'
+      interval: Daily
+      timeField: 'time'
+      version: '2.10.0'
+      flavor: 'opensearch'
+      logMessageField: body
+      logLevelField: severitynumber
diff --git a/src/opensearch/data-prepper-config.yaml b/src/opensearch/data-prepper-config.yaml
@@ -0,0 +1,4 @@
+# Copyright The OpenTelemetry Authors
+# SPDX-License-Identifier: Apache-2.0
+
+ssl: false
diff --git a/src/opensearch/pipelines.yaml b/src/opensearch/pipelines.yaml
@@ -0,0 +1,14 @@
+# Copyright The OpenTelemetry Authors
+# SPDX-License-Identifier: Apache-2.0
+
+demo-pipeline:
+  source:
+    otel_logs_source:
+      ssl: false
+  sink:
+    - opensearch:
+        hosts: ["http://opensearch:9200"]
+        insecure: true
+        username: admin
+        password: admin
+        index: otel
diff --git a/src/otelcollector/otelcol-observability.yml b/src/otelcollector/otelcol-observability.yml
@@ -7,6 +7,10 @@ exporters:
     endpoint: "jaeger:4317"
     tls:
       insecure: true
+  otlp/logs:
+    endpoint: "dataprepper:21892"
+    tls:
+      insecure: true
   otlphttp/prometheus:
     endpoint: "http://prometheus:9090/api/v1/otlp"
     tls:
@@ -18,3 +22,5 @@ service:
       exporters: [otlp, logging, spanmetrics]
     metrics:
       exporters: [otlphttp/prometheus, logging]
+    logs:
+      exporters: [otlp/logs, logging]