elasticsearchkibanaaws-elasticsearchkibana-7

Kibana query - aggregate counts using terms, range in single bucket


I have a query in Kibana searching in index pattern that has pipeline execution metadata, which include pipelineid, dev grief count, etc I'd like to split the result into a bucket of pipeline ids with total execution count and executions count that has dev_grief > 0

My query -

GET pipeline-execution/_search
{
  "size": 0,
  "aggs": {
    "Terms_Aggregation": {
      "terms": {
        "field": "PipelineId",
        "size": 1000
      }
    }
  }
}

The above query is giving me total executions by pipeline id based on terms aggregation, but I would like the count of executions that has GriefToDeveloper value greater than 0.

Hence, I added a filter using range -

GET pipeline-execution/_search
{
  "size": 0,
  "aggs": {
    "DeveloperGriefCount": {
      "filter": {
        "range": {
          "GriefToDeveloper": {
            "gte": 1
          }
        }
      },
      "aggs": {
        "Pipeline": {
          "terms": {
            "field": "PipelineId",
            "size": 1000
          }
        }
      }
    }
  }
}

Now, this works, but as you can see there is no total execution count -

enter image description here

I would like to merge these two queries, and just retrieve one bucket whose elements would look something like -

{
   "buckets": [
      {
         "key": "pipeline_id_3",
         "doc_count": 10789,
         "grief_count": 950
      },
      {
         "key": "pipeline_id_4",
         "doc_count": 7666,
         "grief_count": 25
      }
   ]
}

UPDATE1 - Sample document -

{
    "_index": "pipeline-execution",
    "_type": "_doc",
    "_id": "FUM_MHkBjRE1lX_78952a",
    "_score": 1,
    "_source": {
        "CreatedOn": "2021-05-03T03:22:41.715000",
        "PipelineId": "pipeline_id_1",
        "ExecIdentifier": "147895632145",
        "Source": 45,
        "Good": 40,
        "PayloadHashValue": "95d59a7c80ebc4974f11995c4f4004ef",
        "GriefToDeveloper": 5
    }
}

Index Map/Template-

{
    "CreatedOn": {
        "type": "date"
    },
    "PipelineId": {
        "type": "keyword"
    },
    "ExecIdentifier": {
        "type": "keyword"
    },
    "Source": {
        "type": "integer"
    },
    "Good": {
        "type": "integer"
    },
    "PayloadHashValue": {
        "type": "text"
    },
    "GriefToDeveloper": {
        "type": "integer"
    }
}

**UPDATE2 - ** Here's the query I wanted to see -

GET pipeline-execution/_search
{
  "size": 0,
  "aggs": {
    "Pipelines": {
      "terms": {
        "field": "PipelineId",
        "size": 1000
      },
      "aggs": {
        "total_exec": {
          "value_count": {
            "field": "PipelineId"
          }
        },
        "dev_grief": {
            "value_count": {
            "field": "PipelineId"
          }
        },
        "Grief%": {
          "bucket_script": {
            "buckets_path": {
              "TotalExecutions": "total_exec",
              "DeveloperGrief": "dev_grief"
            },
            "script": "(params.TotalExecutions/params.DeveloperGrief)*100"
          }
        }
      }
    }
  }
}

Response:

"Pipelines": {
   "doc_count_error_upper_bound": 0,
   "sum_other_doc_count": 0,
   "buckets": [
      {
         "key": "pipeline_id_1",
         "doc_count": 2,
         "total_exec": {
            "value": 2
         },
         "dev_grief": {
            "value": 1
         },
         "Grief%": {
            "value": 50
         }
      }
   ]
}

Any help is greatly appreciated.

Thank you


Solution

  • Your idea was correct. You needed to make filter aggregation a sub aggregation of terms instead of other way around

    Query

    {
      "size": 0,
      "aggs": {
        "pipelines": {
          "terms": {
            "field": "PipelineId",
            "size": 10
          },
          "aggs": {
            "grief_count": {
              "filter": {
                "range": {
                  "GriefToDeveloper": {
                    "gte": 1
                  }
                }
              }
            }
          }
        }
      }
    }
    

    Result

    "pipelines" : {
          "doc_count_error_upper_bound" : 0,
          "sum_other_doc_count" : 0,
          "buckets" : [
            {
              "key" : "pipeline_id_1",
              "doc_count" : 2,
              "grief_count" : {
                "doc_count" : 1
              }
            }
          ]
        }
    

    Update

    {
      "size": 0,
      "aggs": {
        "Pipelines": {
          "terms": {
            "field": "PipelineId",
            "size": 1000
          },
          "aggs": {
            "total_exec": {
              "value_count": {
                "field": "PipelineId"
              }
            },
            "grief_count": {
              "filter": {
                "range": {
                  "GriefToDeveloper": {
                    "gte": 1
                  }
                }
              },
              "aggs": {
                "dev_grief": {
                  "value_count": {
                    "field": "PipelineId"
                  }
                }
              }
            },
            "Grief%": {
              "bucket_script": {
                "buckets_path": {
                  "TotalExecutions": "total_exec",
                  "DeveloperGrief": "grief_count>dev_grief"
                },
                "script": "(params.TotalExecutions/params.DeveloperGrief)*100"
              }
            }
          }
        }
      }
    }
    

    Result from updated query -

    "Pipelines" : {
       "doc_count_error_upper_bound": 0,
       "sum_other_doc_count": 0,
       "buckets": [
          {
             "key": "pipeline_id_1",
             "doc_count": 1315291,
             "grief_count": {
                "doc_count": 4447,
                "dev_grief": {
                   "value": 4447
                }
             },
             "total_exec": {
                "value": 1315291
             },
             "Grief%": {
                "value": 0.33810008
             }
          }
       ]
    }