I'm trying to use OpenSearch over Logstash logs. I have two logs, but the second log does not apply to OpenSearch available fields.
Here is my two logs and each has a different log4j pattern. It is a third-party solution, so these logs pattern can't change.
Log1
%d %5p [%c] [%X{txIp}] [%X{mbrNo}] %m%n
Log2
[%-5p][%d{yyyyMMdd HH:mm:ss.SSS}][%t]%c{4}.%M(%L) - %X{reqCommand}|%X{svcTrId}|%m%n
and this is my logstash.conf
filter:
filter {
if [fields][index] == "log1"{
grok {
match => {
"message" => "%{TIMESTAMP_ISO8601:date} %{LOGLEVEL:logLevel} \[%{DATA:class}\] \[%{DATA:txIp}\] \[%{DATA:mbrNo}\] %{GREEDYDATA:message}"
}
}
} else if [fields][index] == "log2" {
grok {
match => {
"message" => "\[%{LOGLEVEL:logLevel}\]\[%{TIMESTAMP_ISO8601:date}\]\[%{DATA:thread}\]%{DATA:class}.%{DATA:method}\(%{DATA:line}\) - %{DATA:reqCommand}\|%{DATA:svcTrId}\|%{GREEDYDATA:message}"
}
}
}
date {
match => ["date", "ISO8601"]
}
}
Finally It works.
This is my Filter.
filter {
if [fields][index] == "log1" {
grok {
match => {
"message" => "%{TIMESTAMP_ISO8601:logTimestamp} %{DATA:logLevel} \[%{DATA:class}\] \[%{DATA:txIp}\] \[%{DATA:mbrNo}\] %{GREEDYDATA:message}"
}
}
date {
match => ["logTimestamp", "ISO8601"]
target => "@timestamp"
timezone => "Asia/Seoul"
}
} else if [fields][index] == "log2" {
grok {
match => {
"message" => "\[%{DATA:logLevel}]\[%{YEAR:year}%{MONTHNUM:month}%{MONTHDAY:day} %{TIME:time}\]\[%{DATA:thread}\]%{DATA:class}.%{DATA:method}\(%{DATA:line}\) - %{DATA:reqCommand}\|%{DATA:svcTrId}\|%{GREEDYDATA:message}"
}
}
mutate {
add_field => {
"logtimestamp" => "%{year}-%{month}-%{day}T%{time}"
}
}
date {
match => ["logtimestamp", "ISO8601"]
target => "@timestamp"
timezone => "Asia/Seoul"
}
}
}