ELK Stack Integration not working with WSO2 API Manager

I'm currently trying to integrate a WSO2 Api Manager with the ELK stack, I followed the instructions in Apim Docs but so far I didn't got a single log or analytics.

I'm sending all my Filebeat, Logstash, Kibana and Elasticsearch configs and the WSO2 repository/config too.

Filebeat config file:

filebeat.inputs:


- type: log

  paths:
    - /home/aluno/wso2am-4.2.0/repository/logs/apim_metrics.log
  include_lines: ['(apimMetrics):']

filebeat.config.modules:
  path: ${path.config}/modules.d/*.yml
  reload.enabled: false

setup.template.settings:
  index.number_of_shards: 1

setup.kibana:

output.logstash:
  # The Logstash hosts
  hosts: ["192.168.56.104:5044"]

processors:
  - add_host_metadata:
      when.not.contains.tags: forwarded
  - add_cloud_metadata: ~
  - add_docker_metadata: ~
  - add_kubernetes_metadata: ~

Logstash config file:

input {
    beats {
        port => 5044
    }
}

filter {
    grok {
        match => ["message", "%{GREEDYDATA:UNWANTED}\ apimMetrics:%{GREEDYDATA:apimMetrics}\, %{GREEDYDATA:UNWANTED} \:%{GREEDYDATA:properties}"]
    }
    json {
        source => "properties"
    }
}
output {
    if [apimMetrics] == " apim:response" {
        elasticsearch {
            hosts => ["http://192.168.56.104:9200"]
            index => "apim_event_response"
            user => "elastic"
            password => "alunowso2"
        }
    } else if [apimMetrics] == " apim:faulty" {
        elasticsearch {
            hosts => ["http://192.168.56.104:9200"]
            index => "apim_event_faulty"
            user => "elastic"
            password => "alunowso2"
        }
    }
}

Kibana:

server.port: 5601

server.publicBaseUrl: "http://192.168.56.104:5601"

elasticsearch.username: "kibana_system"
elasticsearch.password: "alunowso2"
  appenders:
    file:
      type: file
      fileName: /var/log/kibana/kibana.log
      layout:
        type: json
  root:
    appenders:
      - default
      - file

Elasticsearch:

node.name: wso2-elastic
path.data: /var/lib/elasticsearch
path.logs: /var/log/elasticsearch


xpack.security.enabled: true

xpack.security.enrollment.enabled: true

xpack.security.http.ssl:
  enabled: true
  keystore.path: certs/http.p12
xpack.security.transport.ssl:
  enabled: true
  verification_mode: certificate
  keystore.path: certs/transport.p12
  truststore.path: certs/transport.p12
http.host: 0.0.0.0

I've configured the [apim.analytics] to be true and use "elk" in wso2 repository/config

I was expecting with this configuration I had acess to logs provided by WSO2 APIM but I'm getting Kibana Dashboard this screen and seems that my apim_metrics.log is empty too, what can I do?

Deployment.toml:

[server]
hostname = "wso2.test"
#offset=0
base_path = "${carbon.protocol}://${carbon.host}:${carbon.management.port}"
#discard_empty_caches = false
server_role = "default"

[super_admin]
username = "admin"
password = "admin"
create_admin_account = true

[user_store]
type = "database_unique_id"

[database.apim_db]
type = "h2"
url = "jdbc:h2:./repository/database/WSO2AM_DB;AUTO_SERVER=TRUE;DB_CLOSE_ON_EXIT=FALSE"
username = "wso2carbon"
password = "wso2carbon"

[database.shared_db]
type = "h2"
url = "jdbc:h2:./repository/database/WSO2SHARED_DB;DB_CLOSE_ON_EXIT=FALSE"
username = "wso2carbon"
password = "wso2carbon"

[keystore.tls]
file_name =  "wso2carbon.jks"
type =  "JKS"
password =  "wso2carbon"
alias =  "wso2carbon"
key_password =  "wso2carbon"

#[keystore.listener_profile]
#bind_address = "0.0.0.0"

#[keystore.primary]
#file_name =  "wso2carbon.jks"
#type =  "JKS"
#password =  "wso2carbon"
#alias =  "wso2carbon"
#key_password =  "wso2carbon"

#[keystore.internal]
#file_name =  "wso2carbon.jks"
#type =  "JKS"
#password =  "wso2carbon"
#alias =  "wso2carbon"
#key_password =  "wso2carbon"

[[apim.gateway.environment]]
name = "Default"
type = "hybrid"
provider = "wso2"
display_in_api_console = true
description = "This is a hybrid gateway that handles both production and sandbox token traffic."
show_as_token_endpoint_url = true
service_url = "https://localhost:${mgt.transport.https.port}/services/"
username= "${admin.username}"
password= "${admin.password}"
ws_endpoint = "ws://localhost:9099"
wss_endpoint = "wss://localhost:8099"
http_endpoint = "http://localhost:${http.nio.port}"
https_endpoint = "https://localhost:${https.nio.port}"
websub_event_receiver_http_endpoint = "http://localhost:9021"
websub_event_receiver_https_endpoint = "https://localhost:8021"

[apim.sync_runtime_artifacts.gateway]
gateway_labels =["Default"]

#[apim.cache.gateway_token]
#enable = true
#expiry_time = "900s"

#[apim.cache.resource]
#enable = true
#expiry_time = "900s"

#[apim.cache.km_token]
#enable = false
#expiry_time = "15m"

#[apim.cache.recent_apis]
#enable = false

#[apim.cache.scopes]
#enable = true

#[apim.cache.publisher_roles]
#enable = true

#[apim.cache.jwt_claim]
#enable = true
#expiry_time = "15m"

#[apim.cache.tags]
#expiry_time = "2m"

[apim.analytics]
enable = true
type = "elk"

[apim.key_manager]
enable_apikey_subscription_validation = true
#service_url = "https://localhost:${mgt.transport.https.port}/services/"
#username = "$ref{super_admin.username}"
#password = "$ref{super_admin.password}"
#pool.init_idle_capacity = 50
#pool.max_idle = 100
#key_validation_handler_type = "default"
#key_validation_handler_type = "custom"
#key_validation_handler_impl = "org.wso2.carbon.apimgt.keymgt.handlers.DefaultKeyValidationHandler"

#[apim.idp]
#server_url = "https://localhost:${mgt.transport.https.port}"
#authorize_endpoint = "https://localhost:${mgt.transport.https.port}/oauth2/authorize"
#oidc_logout_endpoint = "https://localhost:${mgt.transport.https.port}/oidc/logout"
#oidc_check_session_endpoint = "https://localhost:${mgt.transport.https.port}/oidc/checksession"

#[apim.jwt]
#enable = true
#encoding = "base64" # base64,base64url
#generator_impl = "org.wso2.carbon.apimgt.keymgt.token.JWTGenerator"
#claim_dialect = "http://wso2.org/claims"
#convert_dialect = false
#header = "X-JWT-Assertion"
#signing_algorithm = "SHA256withRSA"
#enable_user_claims = true
#claims_extractor_impl = "org.wso2.carbon.apimgt.impl.token.ExtendedDefaultClaimsRetriever"

#[apim.oauth_config]
#enable_outbound_auth_header = false
#auth_header = "Authorization"
#revoke_endpoint = "https://localhost:${https.nio.port}/revoke"
#enable_token_encryption = false
#enable_token_hashing = false

#[apim.devportal]
#url = "https://localhost:${mgt.transport.https.port}/devportal"
#enable_application_sharing = false
#if application_sharing_type, application_sharing_impl both defined priority goes to application_sharing_impl
#application_sharing_type = "default" #changed type, saml, default #todo: check the new config for rest api
#application_sharing_impl = "org.wso2.carbon.apimgt.impl.SAMLGroupIDExtractorImpl"
#display_multiple_versions = false
#display_deprecated_apis = false
#enable_comments = true
#enable_ratings = true
#enable_forum = true
#enable_anonymous_mode=true
#enable_cross_tenant_subscriptions = true
#default_reserved_username = "apim_reserved_user"

[apim.cors]
allow_origins = "*"
allow_methods = ["GET","PUT","POST","DELETE","PATCH","OPTIONS"]
allow_headers = ["authorization","Access-Control-Allow-Origin","Content-Type","SOAPAction","apikey","Internal-Key"]
allow_credentials = false

#[apim.throttling]
#enable_data_publishing = true
#enable_policy_deploy = true
#enable_blacklist_condition = true
#enable_persistence = true
#throttle_decision_endpoints = ["tcp://localhost:5672","tcp://localhost:5672"]

#[apim.throttling.blacklist_condition]
#start_delay = "5m"
#period = "1h"

#[apim.throttling.jms]
#start_delay = "5m"

#[apim.throttling.event_sync]
#hostName = "0.0.0.0"
#port = 11224

#[apim.throttling.event_management]
#hostName = "0.0.0.0"
#port = 10005

#[[apim.throttling.url_group]]
#traffic_manager_urls = ["tcp://localhost:9611","tcp://localhost:9611"]
#traffic_manager_auth_urls = ["ssl://localhost:9711","ssl://localhost:9711"]
#type = "loadbalance"

#[[apim.throttling.url_group]]
#traffic_manager_urls = ["tcp://localhost:9611","tcp://localhost:9611"]
#traffic_manager_auth_urls = ["ssl://localhost:9711","ssl://localhost:9711"]
#type = "failover"

#[apim.workflow]
#enable = false
#service_url = "https://localhost:9445/bpmn"
#username = "$ref{super_admin.username}"
#password = "$ref{super_admin.password}"
#callback_endpoint = "https://localhost:${mgt.transport.https.port}/api/am/admin/v0.17/workflows/update-workflow-status"
#token_endpoint = "https://localhost:${https.nio.port}/token"
#client_registration_endpoint = "https://localhost:${mgt.transport.https.port}/client-registration/v0.17/register"
#client_registration_username = "$ref{super_admin.username}"
#client_registration_password = "$ref{super_admin.password}"

#data bridge config
#[transport.receiver]
#type = "binary"
#worker_threads = 10
#session_timeout = "30m"
#keystore.file_name = "$ref{keystore.tls.file_name}"
#keystore.password = "$ref{keystore.tls.password}"
#tcp_port = 9611
#ssl_port = 9711
#ssl_receiver_thread_pool_size = 100
#tcp_receiver_thread_pool_size = 100
#ssl_enabled_protocols = ["TLSv1","TLSv1.1","TLSv1.2"]
#ciphers = ["SSL_RSA_WITH_RC4_128_MD5","SSL_RSA_WITH_RC4_128_SHA"]

#[apim.notification]
#from_address = "APIM.com"
#username = "APIM"
#password = "APIM+123"
#hostname = "localhost"
#port = 3025
#enable_start_tls = false
#enable_authentication = true

#[apim.token.revocation]
#notifier_impl = "org.wso2.carbon.apimgt.keymgt.events.TokenRevocationNotifierImpl"
#enable_realtime_notifier = true
#realtime_notifier.ttl = 5000
#enable_persistent_notifier = true
#persistent_notifier.hostname = "https://localhost:2379/v2/keys/jti/"
#persistent_notifier.ttl = 5000
#persistent_notifier.username = "root"
#persistent_notifier.password = "root"

[[event_handler]]
name="userPostSelfRegistration"
subscriptions=["POST_ADD_USER"]

[service_provider]
sp_name_regex = "^[\\sa-zA-Z0-9._-]*$"

[database.local]
url = "jdbc:h2:./repository/database/WSO2CARBON_DB;DB_CLOSE_ON_EXIT=FALSE"

[[event_listener]]
id = "token_revocation"
type = "org.wso2.carbon.identity.core.handler.AbstractIdentityHandler"
name = "org.wso2.is.notification.ApimOauthEventInterceptor"
order = 1
[event_listener.properties]
notification_endpoint = "https://localhost:${mgt.transport.https.port}/internal/data/v1/notify"
username = "${admin.username}"
password = "${admin.password}"
'header.X-WSO2-KEY-MANAGER' = "default"

[oauth.grant_type.token_exchange]
enable = true
allow_refresh_tokens = true
iat_validity_period = "1h"

[transport.https.properties]
proxyPort = 443

(I've ommited the comments because StackOverflow was accusing spam)

What is the API Manager version that you are trying out? ELK analytics was introduced into API Manager from 4.2.0 onward and please verify whether you are using a supported version. Furthermore, share your APIM /repository/conf/deployment.toml config snippet for [apim.analytics]

If you can see analytics logs in the repository/logs/apim_metrics.log file. Then you can verify the ELK data flow by adding stdout {} into the logstash config file.

please refer to the following sample

output {
    if [apimMetrics] == " apim:response" {
        elasticsearch {
            hosts => ["http://localhost:9200"]
            index => "apim_event_response"
            user => "elastic"
            password => "xxx"
            data_stream => false
        }
    } else if [apimMetrics] == " apim:faulty" {
        elasticsearch {
            hosts => ["http://localhost:9200"]
            index => "apim_event_faulty"
            user => "elastic"
            password => "xxx"
            data_stream => false
        }
    }
    stdout {}
}

And also in your above Elasticsearch configuration, you have enabled xpack.security hence you need to call Elasticsearch via HTTPs. Change the Logstash host URLs to HTTPs and configure the certs.