logstashgrok

Logstash grok filter grok parse failure


I configured it using Logstash and Filebeat. And I used MongoDB output plugin.

I want to make data in mongoDB like this:

  {
    "_id": {"$oid": "62ea3197736a54952d56e2fe"},
    "category": "TEST",
    "date": "2022-08-03T17:28:07+09:00",
    "level": "WARNING",
    "message": "statistics_sales.go:136: sdb.FindUserByID, error:  mongo: no documents in result",
    "title": "statistics_sales"
  }

It is orginal log data:

ERROR: 2022/08/16 13:27:42.292305 statistics_sales.go:136: sdb.FindUserByID, error:  mongo: no documents in result

So I configure logstash.conf like this:

filter {
  grok {
    match => {
      "message" => "%{LOGLEVEL:level}: %{YEAR:year}/%{MONTHNUM:month}/%{MONTHDAY:day} %{TIME:time} %{GREEDYDATA:message}"
    }

    overwrite => [ "message" ]
  }

  grok {
    match => {
      "message" => "%{DATA:title}\.go%{GREEDYDATA}"
    }
  }

  grok {
    match => {
      "time" => "%{DATA:sec}\.%{GREEDYDATA}"
    }
  }

  grok {
    add_field => {
      "date" => "%{year}-%{month}-%{day}T%{time}+09:00"
    }
  }

  grok {
    remove_field => ["year", "month", "day", "time", "event", "@version", "agent", "host", "@timestamp", "ecs"]
  }
}

But, I'm getting an error from under the time(sec) configuration.

I tryed many way, But I can't find what is the problem.

It is Logstash result:

{
         "month" => "08",
         "input" => {
        "type" => "log"
    },
         "agent" => {
                "name" => "HOSTNAME",
             "version" => "8.3.3",
                  "id" => "8a97df17-1977-472f-b6f1-6c7a7359c372",
                "type" => "filebeat",
        "ephemeral_id" => "34333de9-c892-42a0-bc35-ee49a0f06721"
    },
           "log" => {
        "offset" => 575,
          "file" => {
            "path" => "/home/bjkang/LogCollector/Log/service_log_2022-08-23.log"
        }
    },
           "ecs" => {
        "version" => "8.0.0"
    },
         "title" => "statistics_sales",
    "@timestamp" => 2022-08-23T11:30:42.407Z,
          "time" => "13:27:42.293679",
           "sec" => "13:27:42",
          "host" => {
        "name" => "HOSTNAME"
    },
          "tags" => [
        [0] "_grokparsefailure"
    ],
      "@version" => "1",
       "message" => "statistics_sales.go:136: sdb.FindUserByID, error:  mongo: no documents in result",
           "day" => "16",
         "level" => "ERROR",
          "year" => "2022",
         "event" => {
        "original" => "ERROR: 2022/08/16 13:27:42.293679 statistics_sales.go:136: sdb.FindUserByID, error:  mongo: no documents in result"
    }
}

Can I get some hint for this?


Solution

  • I solved it by setting it like this:

    filter {
      grok {
        match => {
          "message" => "%{LOGLEVEL:level}: %{YEAR:year}/%{MONTHNUM:month}/%{MONTHDAY:day} %{TIME:time} %{GREEDYDATA:message}"
        }
    
        overwrite => [ "message" ]
      }
    
      grok {
        match => {
          "message" => "%{DATA:title}\.go%{GREEDYDATA}"
        }
      }
    
      grok {
        match => {
          "time" => "%{DATA:hhmmss}\.%{GREEDYDATA}"
        }
      }
      mutate {
        add_field => {
          "date" => "%{year}-%{month}-%{day}T%{hhmmss}+09:00"
        }
      }
    
      prune {
        whitelist_names => ["level", "message", "title", "date", "@timestamp", "@version"]
      }
    }
    

    But, I can't insert doc without metadata[@timestamp, @version]

    maybe someone know this, plz comment :)