logstashlogstash-filter

Logstash filter not replacing variable


I am performing the following mutate/add_field transofmations in a logstash filter:

mutate {
     add_field => { "[retrieved_sessionid_new]" => "%{[retrieved_sessionid][0]}" }
     add_field => { "[totalBytes]" => "%{[retrieved_sessionid_new][totalBytes]}" }
}

Here is how it comes up in stdout (and elasticsearch of course)

"retrieved_sessionid" => [
        [0] "{\"srcBytes\":\"376\",\"ElapsedTime\":\"0\",\"@version\":\"1\",\"@timestamp\":\"2018-11-29T13:31:11.944Z\",\"dstBytes\":\"450\",\"SessionID\":\"39680\",\"tags\":[\"bar\",\"foo\",\"traffic_event\"],\"totalBytes\":\"826\"}"
    ]

"retrieved_sessionid_new" => "{\"srcBytes\":\"537\",\"ElapsedTime\":\"8\",\"@version\":\"1\",\"@timestamp\":\"2018-11-29T13:31:03.931Z\",\"dstBytes\":\"526\",\"SessionID\":\"6131\",\"tags\":[\"bar\",\"boo\",\"traffic_event\"],\"totalBytes\":\"1063\"}",

"totalBytes" => "%{[retrieved_sessionid_new][totalBytes]}",

Why isn't the totalBytes variable not being interpolated properly?

edit:

original json event:

"retrieved_sessionid": [
      "{\"srcBytes\":\"381\",\"ElapsedTime\":\"4\",\"@version\":\"1\",\"@timestamp\":\"2018-11-29T13:12:26.928Z\",\"dstBytes\":\"526\",\"SessionID\":\"56276\",\"tags\":[\"bar\",\"foo\",\"traffic_event\"],\"totalBytes\":\"907\"}"
    ],
"totalBytes": "%{[retrieved_sessionid_new][totalBytes]}",
"retrieved_sessionid_new": "{\"srcBytes\":\"381\",\"ElapsedTime\":\"4\",\"@version\":\"1\",\"@timestamp\":\"2018-11-29T13:12:26.928Z\",\"dstBytes\":\"526\",\"SessionID\":\"56276\",\"tags\":[\"bar\",\"foo\",\"traffic_event\"],\"totalBytes\":\"907\"}"

Solution

  • You should decode your data as JSON.

    Example using json filter (tested with LS 6.5.1):

    input {
      stdin { }
    }
    
    filter {
      json {
        source => "message"
        add_field => { "[totalBytes]" => "%{[retrieved_sessionid][0][totalBytes]}" }
      }
    }
    
    output {
      stdout {}
    }
    

    Output:

    {
                    "message" => "{\"retrieved_sessionid\": [{\"srcBytes\":\"376\",\"ElapsedTime\":\"0\",\"@version\":\"1\",\"@timestamp\":\"2018-11-29T13:31:11.944Z\",\"dstBytes\":\"450\",\"SessionID\":\"39680\",\"tags\":[\"bar\",\"foo\",\"traffic_event\"],\"totalBytes\":\"826\"}]}",
                 "@timestamp" => 2018-08-24T14:08:32.080Z,
        "retrieved_sessionid" => [
            [0] {
                  "SessionID" => "39680",
                 "@timestamp" => "2018-11-29T13:31:11.944Z",
                "ElapsedTime" => "0",
                       "tags" => [
                    [0] "bar",
                    [1] "foo",
                    [2] "traffic_event"
                ],
                   "dstBytes" => "450",
                   "@version" => "1",
                   "srcBytes" => "376",
                 "totalBytes" => "826"
            }
        ],
                       "host" => "localhost.localdomain",
                   "@version" => "1",
                 "totalBytes" => "826"
    }