I am new to ELK and I have deployed Elastic and Kibana on K8s, I would like to monitor an standalone Nginx server which is running in a standalone Ubuntu server, I have installed the filebeat and logstash on that machine (Ubuntu server) and configured the filebeat to gather the Nginx access logs and push it to Logstash, I want to enrich the logs with GeoIP data, so that I can create map on the Kibana, below is my Filebeat and Logstash configurations, what issue I am facing is that I am not getting the GeoIP data and the GeoIP fields "geo.location" gets created but I get geo failure tags:
"tags": [
"beats_input_codec_plain_applied",
"geoip-city-failed",
"geoip-asn-failed"
]
checking if field is created:
GET /endpoints/_search
{
"_source": ["geo.location"],
"size": 1
}
*************
{
"took": 1,
"timed_out": false,
"_shards": {
"total": 1,
"successful": 1,
"skipped": 0,
"failed": 0
},
"hits": {
"total": {
"value": 160,
"relation": "eq"
},
"max_score": 1,
"hits": [
{
"_index": "endpoints",
"_id": "99eNaYoB1eIOU-vBqNqc",
"_score": 1,
"_ignored": [
"message.keyword",
"event.original.keyword"
],
"_source": {}
}
]
}
}
getting mapping detials:
{
"endpoints": {
"mappings": {
"properties": {
"@timestamp": {
"type": "date"
},
"@version": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"agent": {
"properties": {
"ephemeral_id": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"id": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"name": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"type": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"version": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
},
"ecs": {
"properties": {
"version": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
},
"event": {
"properties": {
"dataset": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"module": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"original": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"timezone": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
},
"fileset": {
"properties": {
"name": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
},
"geo": {
"properties": {
"location": {
"type": "geo_point"
}
}
},
"host": {
"properties": {
"name": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
},
"input": {
"properties": {
"type": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
},
"log": {
"properties": {
"file": {
"properties": {
"path": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
},
"offset": {
"type": "long"
}
}
},
"log_type": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"message": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"service": {
"properties": {
"type": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
},
"tags": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
}
}
}
filebeat.yml
filebeat.inputs:
- type: filestream
enabled: true
paths:
- /var/log/nginx/*.log
fields:
log_type: nginx
fields_under_root: true
close_inactive: 24h
filebeat.config.modules:
path: ${path.config}/modules.d/*.yml
reload.enabled: false
output.logstash:
hosts: ["localhost:5044"]
setup.kibana:
host: "192.168.30.14:5601"
username: "$user"
password: "$mypassword"
logstash.conf
input {
beats {
port => 5044
}
}
filter {
geoip {
default_database_type => "City"
source => "clientip"
target => "geo"
tag_on_failure => ["geoip-city-failed"]
}
geoip {
default_database_type => "ASN"
source => "clientip"
target => "geo"
tag_on_failure => ["geoip-asn-failed"]
}
}
output {
elasticsearch {
hosts => ["https://192.168.30.12:9200"]
index => "endpoints"
user => "$myuser"
password => "$mypass"
ssl => true
ssl_certificate_authorities => "/$HOME/ca.crt"
ssl_certificate_verification => false
}
}
this is the data I am getting in ES:
{
"_index": "endpoints",
"_id": "iNfbaYoB1eIOU-vBaNs7",
"_version": 1,
"_score": 0,
"_source": {
"tags": [
"beats_input_codec_plain_applied",
"geoip-city-failed",
"geoip-asn-failed"
],
"@timestamp": "2023-09-06T09:38:20.908Z",
"message": "106.202.45.121 - - [06/Sep/2023:09:38:19 +0000] \"GET / HTTP/1.1\" 304 0 \"-\" \"Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Mobile Safari/537.36\"",
"agent": {
"id": "518bc225-d4dd-4e3f-a334-b7aca04e1c43",
"version": "8.9.1",
"ephemeral_id": "29ae20d4-b4c7-4e7e-9491-7f9f60e5c65b",
"type": "filebeat",
"name": "ubuntu"
},
"ecs": {
"version": "1.12.0"
},
"log": {
"offset": 966163,
"file": {
"path": "/var/log/nginx/access.log"
}
},
"@version": "1",
"event": {
"original": "106.202.45.121 - - [06/Sep/2023:09:38:19 +0000] \"GET / HTTP/1.1\" 304 0 \"-\" \"Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Mobile Safari/537.36\"",
"timezone": "+00:00",
"dataset": "nginx.access",
"module": "nginx"
},
"host": {
"name": "ubuntu"
},
"service": {
"type": "nginx"
},
"input": {
"type": "log"
},
"fileset": {
"name": "access"
}
},
"fields": {
"agent.version.keyword": [
"8.9.1"
],
"service.type.keyword": [
"nginx"
],
"input.type.keyword": [
"log"
],
"host.name.keyword": [
"ubuntu"
],
"event.dataset.keyword": [
"nginx.access"
],
"tags.keyword": [
"beats_input_codec_plain_applied",
"geoip-city-failed",
"geoip-asn-failed"
],
"service.type": [
"nginx"
],
"agent.type": [
"filebeat"
],
"ecs.version.keyword": [
"1.12.0"
],
"event.module": [
"nginx"
],
"@version": [
"1"
],
"agent.name": [
"ubuntu"
],
"host.name": [
"ubuntu"
],
"event.timezone": [
"+00:00"
],
"log.file.path.keyword": [
"/var/log/nginx/access.log"
],
"agent.type.keyword": [
"filebeat"
],
"agent.ephemeral_id.keyword": [
"29ae20d4-b4c7-4e7e-9491-7f9f60e5c65b"
],
"event.original": [
"106.202.45.121 - - [06/Sep/2023:09:38:19 +0000] \"GET / HTTP/1.1\" 304 0 \"-\" \"Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Mobile Safari/537.36\""
],
"agent.name.keyword": [
"ubuntu"
],
"agent.id.keyword": [
"518bc225-d4dd-4e3f-a334-b7aca04e1c43"
],
"fileset.name": [
"access"
],
"@version.keyword": [
"1"
],
"input.type": [
"log"
],
"log.offset": [
966163
],
"message": [
"106.202.45.121 - - [06/Sep/2023:09:38:19 +0000] \"GET / HTTP/1.1\" 304 0 \"-\" \"Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Mobile Safari/537.36\""
],
"tags": [
"beats_input_codec_plain_applied",
"geoip-city-failed",
"geoip-asn-failed"
],
"fileset.name.keyword": [
"access"
],
"@timestamp": [
"2023-09-06T09:38:20.908Z"
],
"agent.id": [
"518bc225-d4dd-4e3f-a334-b7aca04e1c43"
],
"ecs.version": [
"1.12.0"
],
"event.original.keyword": [
"106.202.45.121 - - [06/Sep/2023:09:38:19 +0000] \"GET / HTTP/1.1\" 304 0 \"-\" \"Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Mobile Safari/537.36\""
],
"log.file.path": [
"/var/log/nginx/access.log"
],
"message.keyword": [
"106.202.45.121 - - [06/Sep/2023:09:38:19 +0000] \"GET / HTTP/1.1\" 304 0 \"-\" \"Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Mobile Safari/537.36\""
],
"event.module.keyword": [
"nginx"
],
"agent.ephemeral_id": [
"29ae20d4-b4c7-4e7e-9491-7f9f60e5c65b"
],
"agent.version": [
"8.9.1"
],
"event.dataset": [
"nginx.access"
],
"event.timezone.keyword": [
"+00:00"
]
}
}
First, the condition to enable the geoip filter works on a field that does not exist in your documents: [fileset][module], so no event get through your geoip filter.
Moreover, once you fix that condition, COMBINEDAPACHELOG obviously won't parse your nginx access logs as you expect (Apache != Nginx) and as a result the message field that geoip will parse won't contain a valid IP address.
In summary, you need to:
%{IPORHOST:clientip} - %{DATA:user_name} \[%{HTTPDATE:time}\] "%{WORD:method}%{DATA:url} HTTP/%{NUMBER:http_version}" %{NUMBER:response_code} %{NUMBER:body_sent:bytes} "%{DATA:referrer}" "%{DATA:agent}"Mapping:
PUT /nginx
{
"mappings": {
"properties": {
"geo": {
"properties": {
"geo": {
"properties": {
"location": {
"type": "geo_point"
}
}
}
}
}
}
}
}