I have the following docker-compose.yml file:
services:
elasticsearch:
image: elasticsearch:7.10.1
container_name: elasticsearch
environment:
- discovery.type=single-node
- http.host=0.0.0.0
- http.port=9201
- ES_JAVA_OPTS=-Xms512m -Xmx512m
- xpack.security.enabled=false
ports:
- "9201:9201"
- "9300:9300"
expose:
- "9201"
healthcheck:
test: ["CMD-SHELL", "curl --silent --fail localhost:9201/_cluster/health || exit 1"]
interval: 10s
timeout: 10s
retries: 3
networks:
- elastic
command: >
bash -c '
until curl -sS "http://localhost:9201/_cat/health?h=status" | grep -q "green\|yellow"; do
sleep 1
done
curl -X PUT "http://localhost:9201/products" -H "Content-Type: application/json" -d'
{
"settings": {
"number_of_shards": 1,
"number_of_replicas": 1,
"analysis": {
"analyzer": "standard"
}
},
"mappings": {
"properties": {
"title": {
"type": "text",
"analyzer": "standard"
},
"author": {
"type": "text",
"analyzer": "standard"
},
"description": {
"type": "text",
"analyzer": "standard"
},
"published_date": {
"type": "date"
},
"purchase_url": {
"type": "keyword"
}
}
}
}'
'
deploy:
resources:
limits:
memory: 2GB
opensearch-node1A:
image: opensearchproject/opensearch:1.2.3
container_name: opensearch-node1A
environment:
- cluster.name=opensearch-cluster
- node.name=opensearch-node1A
- discovery.seed_hosts=opensearch-node1A,opensearch-node2A
- cluster.initial_master_nodes=opensearch-node1A,opensearch-node2A
- bootstrap.memory_lock=true # along with the memlock settings below, disables swapping
- "OPENSEARCH_JAVA_OPTS=-Xms512m -Xmx512m" # minimum and maximum Java heap size, recommend setting both to 50% of system RAM
ulimits:
memlock:
soft: -1
hard: -1
nofile:
soft: 65536 # maximum number of open files for the OpenSearch user, set to at least 65536 on modern systems
hard: 65536
volumes:
- opensearch-data1:/usr/share/opensearch/data
ports:
- 9200:9200
- 9600:9600 # required for Performance Analyzer
networks:
- opensearch-net
opensearch-node2A:
image: opensearchproject/opensearch:1.2.3
container_name: opensearch-node2A
environment:
- cluster.name=opensearch-cluster
- node.name=opensearch-node2A
- discovery.seed_hosts=opensearch-node1A,opensearch-node2A
- cluster.initial_master_nodes=opensearch-node1A,opensearch-node2A
- bootstrap.memory_lock=true
- "OPENSEARCH_JAVA_OPTS=-Xms512m -Xmx512m"
ulimits:
memlock:
soft: -1
hard: -1
nofile:
soft: 65536
hard: 65536
volumes:
- opensearch-data2:/usr/share/opensearch/data
networks:
- opensearch-net
opensearch-dashboardsA:
image: opensearchproject/opensearch-dashboards:1.1.0
container_name: opensearch-dashboardsA
ports:
- 5601:5601
expose:
- "5601"
environment:
OPENSEARCH_HOSTS: '["https://opensearch-node1A:9200","https://opensearch-node2A:9200"]'
networks:
- opensearch-net
logstash-with-plugin:
image: opensearchproject/logstash-oss-with-opensearch-output-plugin:7.16.2
container_name: logstash-with-plugin
networks:
- opensearch-net
volumes:
opensearch-data1:
opensearch-data2:
networks:
opensearch-net:
driver: bridge
elastic:
driver: bridge
I want to do a POC about transferring data from Elasticsearch to Opensearch, but I want them running over containers with docker-compose.yml.
So I set Elasticsearch to run on port 9201 while Opensearch runs on port 9200.
However, from the logs, I see that the Elasticsearch container fails to to start:
logstash-with-plugin | [2024-09-26T04:29:39,365][INFO ][logstash.javapipeline ][main] Pipeline Java execution initialization time {"seconds"=>0.32}
logstash-with-plugin | [2024-09-26T04:29:39,376][INFO ][logstash.inputs.beats ][main] Starting input listener {:address=>"0.0.0.0:5044"}
logstash-with-plugin | [2024-09-26T04:29:39,384][INFO ][logstash.javapipeline ][main] Pipeline started {"pipeline.id"=>"main"}
logstash-with-plugin | [2024-09-26T04:29:39,413][INFO ][logstash.agent ] Pipelines running {:count=>1, :running_pipelines=>[:main], :non_running_pipelines=>[]}
logstash-with-plugin | [2024-09-26T04:29:39,418][INFO ][org.logstash.beats.Server][main][0710cad67e8f47667bc7612580d5b91f691dd8262a4187d9eca8cf87229d04aa] Starting server on port: 5044
elasticsearch | curl: (7) Failed to connect to localhost port 9201: Connection refused
elasticsearch | curl: (7) Failed to connect to localhost port 9201: Connection refused
elasticsearch | curl: (7) Failed to connect to localhost port 9201: Connection refused
elasticsearch | curl: (7) Failed to connect to localhost port 9201: Connection refused
elasticsearch | curl: (7) Failed to connect to localhost port 9201: Connection refused
How can I have both running ?
In elasticsearch service section you have to remove command part and create products index manually or you can use docker compose file below:
services:
elasticsearch:
image: elasticsearch:7.10.1
container_name: elasticsearch
environment:
- discovery.type=single-node
- http.host=0.0.0.0
- http.port=9201
- ES_JAVA_OPTS=-Xms512m -Xmx512m
- xpack.security.enabled=false
ports:
- "9201:9201"
- "9300:9300"
expose:
- "9201"
healthcheck:
test: ["CMD-SHELL", "curl --silent --fail localhost:9201/_cluster/health || exit 1"]
interval: 10s
timeout: 10s
retries: 3
networks:
- elastic
command: >
bash -c '
/usr/local/bin/docker-entrypoint.sh &
until [ "$(curl -sS "http://localhost:9201/_cat/health?h=status" 2>&1 | grep "green")" = "green" ]; do
echo "Status is not green, retrying..."
sleep 1
done
echo "Status is green"
curl -X PUT "http://localhost:9201/products" -H "Content-Type: application/json" -d "{\"settings\":{\"number_of_shards\":1,\"number_of_replicas\":1,\"analysis\":{\"analyzer\":\"standard\"}},\"mappings\":{\"properties\":{\"title\":{\"type\":\"text\",\"analyzer\":\"standard\"},\"author\":{\"type\":\"text\",\"analyzer\":\"standard\"},\"description\":{\"type\":\"text\",\"analyzer\":\"standard\"},\"published_date\":{\"type\":\"date\"},\"purchase_url\":{\"type\":\"keyword\"}}}}"
exec tail -f /dev/null
'
deploy:
resources:
limits:
memory: 2GB
opensearch-node1A:
image: opensearchproject/opensearch:1.2.3
container_name: opensearch-node1A
environment:
- cluster.name=opensearch-cluster
- node.name=opensearch-node1A
- discovery.seed_hosts=opensearch-node1A,opensearch-node2A
- cluster.initial_master_nodes=opensearch-node1A,opensearch-node2A
- bootstrap.memory_lock=true # along with the memlock settings below, disables swapping
- "OPENSEARCH_JAVA_OPTS=-Xms512m -Xmx512m" # minimum and maximum Java heap size, recommend setting both to 50% of system RAM
ulimits:
memlock:
soft: -1
hard: -1
nofile:
soft: 65536 # maximum number of open files for the OpenSearch user, set to at least 65536 on modern systems
hard: 65536
volumes:
- opensearch-data1:/usr/share/opensearch/data
ports:
- 9200:9200
- 9600:9600 # required for Performance Analyzer
networks:
- opensearch-net
opensearch-node2A:
image: opensearchproject/opensearch:1.2.3
container_name: opensearch-node2A
environment:
- cluster.name=opensearch-cluster
- node.name=opensearch-node2A
- discovery.seed_hosts=opensearch-node1A,opensearch-node2A
- cluster.initial_master_nodes=opensearch-node1A,opensearch-node2A
- bootstrap.memory_lock=true
- "OPENSEARCH_JAVA_OPTS=-Xms512m -Xmx512m"
ulimits:
memlock:
soft: -1
hard: -1
nofile:
soft: 65536
hard: 65536
volumes:
- opensearch-data2:/usr/share/opensearch/data
networks:
- opensearch-net
opensearch-dashboardsA:
image: opensearchproject/opensearch-dashboards:1.1.0
container_name: opensearch-dashboardsA
ports:
- 5601:5601
expose:
- "5601"
environment:
OPENSEARCH_HOSTS: '["https://opensearch-node1A:9200","https://opensearch-node2A:9200"]'
networks:
- opensearch-net
logstash-with-plugin:
image: opensearchproject/logstash-oss-with-opensearch-output-plugin:7.16.2
container_name: logstash-with-plugin
networks:
- opensearch-net
volumes:
opensearch-data1:
opensearch-data2:
networks:
opensearch-net:
driver: bridge
elastic:
driver: bridge
In the above docker compose file, I just edited the command section of the elasticsearch service to match.