[SOLVED] Elastic APM server is not available in Docker

Elastic APM server is not available in Docker

I'm trying to run apm-server to collect and send to elasticsearch from java based app using apm-agent. Here's my compose file:

services:
  order-service:
    image: apm-java/order-service:1.0.0
    build:
      context: .
      dockerfile: Dockerfile
      args:
        - "JAR_PATH=target/order-service-0.0.1.jar"
    environment:
      - "JAR_NAME=order-service-0.0.1.jar"
      - "JVM_OPTIONS="
      - "ELASTIC_APM_SERVICE_NAME=order-service"
      - "ELASTIC_APM_APPLICATION_PACKAGES=org.elk.orderservice"
      - "ELASTIC_APM_SERVER_URLS=http://apm:8200"
    ports:
      - 8080:8080
    networks:
      - app-network
    depends_on:
      - postgres
      - redis
      - apm

  kibana:
    image: kibana:8.11.3
    environment:
      - "LOGGING_QUIET=true"
    ports:
      - 5601:5601
    networks:
      - app-network
    depends_on:
      - elasticsearch

  elasticsearch:
    image: apm-java/elasticsearch:1.0.0
    build:
      context: ./docker/elasticsearch
      dockerfile: Dockerfile
    environment:
      - cluster.name=apm-java
      - bootstrap.memory_lock=true
      - "ES_JAVA_OPTS=-Xms256m -Xmx256m"
      - ELASTIC_PASSWORD=$ELASTIC_PASSWORD
    ulimits:
      memlock:
        soft: -1
        hard: -1
    networks:
      - app-network
    ports:
      - 9200:9200

  apm:
    image: apm-java/apm-server:1.0.0
    build:
      context: ./docker/apm
      dockerfile: Dockerfile
    environment:
      - "ELASTICSEARCH_USERNAME=elastic"
      - "ELASTICSEARCH_PASSWORD=$ELASTIC_PASSWORD"
    ports:
      - 8200:8200
    networks:
      - app-network
    depends_on:
      - elasticsearch
      - kibana

  postgres:
    image: 'postgres:16.1'
    environment:
      - 'POSTGRES_DB=mydatabase'
      - 'POSTGRES_PASSWORD=secret'
      - 'POSTGRES_USER=myuser'
    ports:
      - '5432:5432'
    networks:
      - app-network

  redis:
    image: 'redis:7.2.3'
    ports:
      - '32780:6379'
    networks:
      - app-network

networks:
  app-network:
    driver: bridge
    name: app-network

Java app dockerfile:

FROM eclipse-temurin:21-jdk

EXPOSE 8080
RUN mkdir -p /opt/app
WORKDIR /opt/app

ARG JAR_PATH
COPY $JAR_PATH /opt/app
RUN wget -O apm-agent.jar https://search.maven.org/remotecontent?filepath=co/elastic/apm/elastic-apm-agent/1.45.0/elastic-apm-agent-1.45.0.jar

CMD java -javaagent:/opt/app/apm-agent.jar $JVM_OPTIONS -jar $JAR_NAME

apm-server yml, dockerfile:

apm-server:
  host: "localhost:8200"
  concurrent_requests: 5
  rum:
    enabled: true

queue.mem.events: 4096
max_procs: 4

output.elasticsearch:
  hosts: ["https://elasticsearch:9200"]
  username: "elastic"
  password: "changeme"
  ssl.verification_mode: none


setup.kibana.host: "kibana:5601"
setup.dashboards.enabled: true

#logging.level: info
#logging.to_files: false

logging.level: debug
logging.to_files: false

FROM docker.elastic.co/apm/apm-server:8.11.3
COPY --chmod=0644 --chown=1000:1000 apm-server.yml /usr/share/apm-server/apm-server.yml

Elasticsearch conf, dockerfile:

cluster.name: ${cluster.name}
network.host: 0.0.0.0

# minimum_master_nodes need to be explicitly set when bound on a public IP
# set to 1 to allow single node clusters
# Details: https://github.com/elastic/elasticsearch/pull/17288
discovery.type: single-node

FROM docker.elastic.co/elasticsearch/elasticsearch:8.11.3
COPY --chown=elasticsearch:elasticsearch elasticsearch.yml /usr/share/elasticsearch/config/

CMD ["elasticsearch", "-Elogger.level=INFO"]

And I'm able to connect from apm-server to elasticsearch with apm-server test output command:

elasticsearch: https://elasticsearch:9200...
  parse url... OK
  connection...
    parse host... OK
    dns lookup... OK
    addresses: 172.23.0.3
    dial up... OK
  TLS...
    security... WARN server's certificate chain verification is disabled
    handshake... OK
    TLS version: TLSv1.3
    dial up... OK
  talk to server... OK
  version: 8.11.3

But in java app logs getting following error:

2024-01-10 08:05:29,528 [main] INFO  co.elastic.apm.agent.configuration.StartupInfo - Starting Elastic APM 1.45.0 as order-service (0.0.1) on Java 21.0.1 Runtime version: 21.0.1+12-LTS VM version: 21.0.1+12-LTS (Eclipse Adoptium) Linux 6.5.11-linuxkit
2024-01-10T08:05:29.529074548Z 2024-01-10 08:05:29,528 [main] INFO  co.elastic.apm.agent.configuration.StartupInfo - server_urls: 'http://apm:8200' (source: Environment Variables)
2024-01-10T08:05:29.529416131Z 2024-01-10 08:05:29,529 [main] INFO  co.elastic.apm.agent.configuration.StartupInfo - application_packages: 'org.elk.orderservice' (source: Environment Variables)
2024-01-10T08:05:30.587938173Z 2024-01-10 08:05:30,587 [elastic-apm-server-healthcheck] WARN  co.elastic.apm.agent.report.ApmServerHealthChecker - Elastic APM server http://apm:8200/ is not available (Connection refused)
2024-01-10T08:05:30.589589381Z 2024-01-10 08:05:30,589 [main] INFO  co.elastic.apm.agent.impl.ElasticApmTracer - Tracer switched to RUNNING state
2024-01-10T08:05:31.430460173Z 2024-01-10 08:05:31,429 [elastic-apm-remote-config-poller] ERROR co.elastic.apm.agent.configuration.ApmServerConfigurationSource - Connection refused

All services binded to single "app-network". So what can be the cause of the connection issue between java-agent and apm-server? How I can debug flow from java to elk?

Solution

APM Server is bound to localhost, preventing access when exposing the port. To remedy this, change your apm.server.yml from:

apm-server:
  host: "localhost:8200"

to:

apm-server:
  host: "0.0.0.0:8200"