scripts/ci/docker-compose/integration-otel.yml

# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.
---
services:
  otel-collector:
    image: otel/opentelemetry-collector-contrib:0.155.0
    labels:
      org.apache.airflow.breeze: "true"
      org.apache.airflow.breeze.worktree: "${BREEZE_WORKTREE_PATH:-}"
      breeze.description: "Integration required for OTEL/opentelemetry hooks."
    container_name: "breeze-otel-collector"
    command: [--config=/etc/otel-collector-config.yml]
    volumes:
      - ./otel-collector-config.yml:/etc/otel-collector-config.yml
    ports:
      - "24318:4318"    # OTLP http receiver
      - "28889:8889"    # Prometheus exporter metrics

  prometheus:
    labels:
      org.apache.airflow.breeze: "true"
      org.apache.airflow.breeze.worktree: "${BREEZE_WORKTREE_PATH:-}"
    image: prom/prometheus:v3.5.4
    container_name: "breeze-prometheus"
    user: "0"
    command:
      - "--config.file=/etc/prometheus/prometheus.yml"
      - "--enable-feature=native-histograms"
    ports:
      - "29090:9090"
    volumes:
      - ./prometheus-config.yml:/etc/prometheus/prometheus.yml

  grafana:
    labels:
      org.apache.airflow.breeze: "true"
      org.apache.airflow.breeze.worktree: "${BREEZE_WORKTREE_PATH:-}"
    image: grafana/grafana:8.2.4
    container_name: "breeze-grafana"
    environment:
      GF_AUTH_ANONYMOUS_ENABLED: true
      GF_AUTH_ANONYMOUS_ORG_NAME: "Main Org."
      GF_AUTH_ANONYMOUS_ORG_ROLE: "Admin"
      GF_PATHS_PROVISIONING: /grafana/provisioning
    ports:
      - "23000:3000"
    volumes:
      - ./grafana/volume/data:/grafana
      - ./grafana/volume/datasources:/grafana/datasources
      - ./grafana/volume/dashboards:/grafana/dashboards
      - ./grafana/volume/provisioning:/grafana/provisioning

  jaeger:
    labels:
      org.apache.airflow.breeze: "true"
      org.apache.airflow.breeze.worktree: "${BREEZE_WORKTREE_PATH:-}"
    image: jaegertracing/all-in-one:1.57
    container_name: "breeze-jaeger"
    environment:
      COLLECTOR_OTLP_ENABLED: true
      COLLLECTOR_ZIPKIN_HOST_PORT: 9411
    # Host ports are intentionally below 32768 to stay outside Linux's default
    # ephemeral port range (32768-60999). Picking high ports inside that range
    # causes intermittent CI failures when the kernel transiently grabs the
    # same port for an outbound connection from another process on the runner.
    ports:
      - "26686:16686"
      - "24317:4317"
      - "27318:4318"
      - "29422:9422"

  airflow:
    environment:
      - INTEGRATION_OTEL=true
      - OTEL_SERVICE_NAME=test
      - OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf
      - OTEL_TRACES_EXPORTER=otlp_proto_http
      - OTEL_METRICS_EXPORTER=otlp
      - OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=http://breeze-otel-collector:4318/v1/traces
      - OTEL_EXPORTER_OTLP_METRICS_ENDPOINT=http://breeze-otel-collector:4318/v1/metrics
      - OTEL_METRIC_EXPORT_INTERVAL=30000
      - AIRFLOW__METRICS__OTEL_ON=True
      - AIRFLOW__TRACES__OTEL_ON=True
      - AIRFLOW__TRACES__OTEL_TASK_LOG_EVENT=True  # todo: deprecate

    depends_on:
      - otel-collector
      - prometheus
      - grafana
      - jaeger