I configured it using Logstash and Filebeat. And I used MongoDB output plugin.
I want to make data in mongoDB like this:
{
"_id": {"$oid": "62ea3197736a54952d56e2fe"},
"category": "TEST",
"date": "2022-08-03T17:28:07+09:00",
"level": "WARNING",
"message": "statistics_sales.go:136: sdb.FindUserByID, error: mongo: no documents in result",
"title": "statistics_sales"
}
It is orginal log data:
ERROR: 2022/08/16 13:27:42.292305 statistics_sales.go:136: sdb.FindUserByID, error: mongo: no documents in result
So I configure logstash.conf like this:
filter {
grok {
match => {
"message" => "%{LOGLEVEL:level}: %{YEAR:year}/%{MONTHNUM:month}/%{MONTHDAY:day} %{TIME:time} %{GREEDYDATA:message}"
}
overwrite => [ "message" ]
}
grok {
match => {
"message" => "%{DATA:title}\.go%{GREEDYDATA}"
}
}
grok {
match => {
"time" => "%{DATA:sec}\.%{GREEDYDATA}"
}
}
grok {
add_field => {
"date" => "%{year}-%{month}-%{day}T%{time}+09:00"
}
}
grok {
remove_field => ["year", "month", "day", "time", "event", "@version", "agent", "host", "@timestamp", "ecs"]
}
}
But, I'm getting an error from under the time(sec) configuration.
I tryed many way, But I can't find what is the problem.
It is Logstash result:
{
"month" => "08",
"input" => {
"type" => "log"
},
"agent" => {
"name" => "HOSTNAME",
"version" => "8.3.3",
"id" => "8a97df17-1977-472f-b6f1-6c7a7359c372",
"type" => "filebeat",
"ephemeral_id" => "34333de9-c892-42a0-bc35-ee49a0f06721"
},
"log" => {
"offset" => 575,
"file" => {
"path" => "/home/bjkang/LogCollector/Log/service_log_2022-08-23.log"
}
},
"ecs" => {
"version" => "8.0.0"
},
"title" => "statistics_sales",
"@timestamp" => 2022-08-23T11:30:42.407Z,
"time" => "13:27:42.293679",
"sec" => "13:27:42",
"host" => {
"name" => "HOSTNAME"
},
"tags" => [
[0] "_grokparsefailure"
],
"@version" => "1",
"message" => "statistics_sales.go:136: sdb.FindUserByID, error: mongo: no documents in result",
"day" => "16",
"level" => "ERROR",
"year" => "2022",
"event" => {
"original" => "ERROR: 2022/08/16 13:27:42.293679 statistics_sales.go:136: sdb.FindUserByID, error: mongo: no documents in result"
}
}
Can I get some hint for this?
I solved it by setting it like this:
filter {
grok {
match => {
"message" => "%{LOGLEVEL:level}: %{YEAR:year}/%{MONTHNUM:month}/%{MONTHDAY:day} %{TIME:time} %{GREEDYDATA:message}"
}
overwrite => [ "message" ]
}
grok {
match => {
"message" => "%{DATA:title}\.go%{GREEDYDATA}"
}
}
grok {
match => {
"time" => "%{DATA:hhmmss}\.%{GREEDYDATA}"
}
}
mutate {
add_field => {
"date" => "%{year}-%{month}-%{day}T%{hhmmss}+09:00"
}
}
prune {
whitelist_names => ["level", "message", "title", "date", "@timestamp", "@version"]
}
}
But, I can't insert doc without metadata[@timestamp, @version]
maybe someone know this, plz comment :)