I am using the Serilog elasticsearch sink (version 8.4.1, elastic 7.8.0) within ASP.NET Core 3.1 with the following configuration:
{
"Serilog": {
"MinimumLevel": {
"Default": "Information",
"Override": {
"Serilog.AspNetCore": "Information"
}
},
"Enrich": [ "FromLogContext" ],
"WriteTo": [
{
"Name": "Logger",
"Args": {
"configureLogger": {
"Filter": [
{
"Name": "ByIncludingOnly",
"Args": {
"expression": "SourceContext = 'Serilog.AspNetCore.RequestLoggingMiddleware'"
}
}
],
"WriteTo": [
{
"Name": "Elasticsearch",
"Args": {
"nodeUris": "http://localhost:9200",
"indexFormat": "request-logs-{0:yyyy.MM.dd}",
"period": 1,
"connectionTimeout": 5,
"typeName": "_doc",
"inlineFields": true,
"restrictToMinimumLevel": "Information"
}
}
]
}
}
}
]
}
}
I'm using the RequestLogging by Serilog.AspNetCore
app.UseSerilogRequestLogging();
and enrich the IDiagnosticContext within a custom middleware with an object that requires special mapping of GeoSpatial and IP properties.
HttpDiagnostics diagnostics = new HttpDiagnostics
{
Host = host.ToString(),
IsHttps = isHttps,
LocalIp = localIpAddress,
LocalPort = localPort,
Protocol = protocol,
RemoteIp = remoteIpAddress,
RemotePort = remotePort,
RequestContentLength = requestContentLength,
RequestContentType = requestContentType,
Scheme = scheme,
UserAgent = userAgent,
ResponseContentLength = responseContentLength,
ResponseContentType = responseContentType
};
this.diagnosticContext.Set("Http", diagnostics, true);
My approach to ending up with a correct mapping of the fields in the logevent was: remodelling the LogEvent type with my custom Object and creating a mapping for an index template with a NEST client.
ElasticClient client = new ElasticClient(new Uri(settings.Uri));
PutIndexTemplateResponse response = client.Indices.PutTemplate(
settings.Name,
p => p.IndexPatterns(settings.IndexPattern)
.Settings(s => s.DefaultPipeline("geoip"))
.Map<SerilogDiagnosticsLogEvent>(m => m.AutoMap()));
The index template mapping is created correctly
{
"_doc": {
"properties": {
"traceId": {
"fields": {
"keyword": {
"ignore_above": 256,
"type": "keyword"
}
},
"type": "text"
},
"level": {
"fields": {
"keyword": {
"ignore_above": 256,
"type": "keyword"
}
},
"type": "text"
},
"requestMethod": {
"fields": {
"keyword": {
"ignore_above": 256,
"type": "keyword"
}
},
"type": "text"
},
"message": {
"fields": {
"keyword": {
"ignore_above": 256,
"type": "keyword"
}
},
"type": "text"
},
"sourceContext": {
"fields": {
"keyword": {
"ignore_above": 256,
"type": "keyword"
}
},
"type": "text"
},
"parentId": {
"fields": {
"keyword": {
"ignore_above": 256,
"type": "keyword"
}
},
"type": "text"
},
"elapsed": {
"type": "double"
},
"spanId": {
"fields": {
"keyword": {
"ignore_above": 256,
"type": "keyword"
}
},
"type": "text"
},
"@timestamp": {
"type": "date"
},
"requestId": {
"fields": {
"keyword": {
"ignore_above": 256,
"type": "keyword"
}
},
"type": "text"
},
"http": {
"type": "object",
"properties": {
"responseContentLength": {
"type": "long"
},
"requestContentLength": {
"type": "long"
},
"geoIp": {
"type": "object",
"properties": {
"cityName": {
"fields": {
"keyword": {
"ignore_above": 256,
"type": "keyword"
}
},
"type": "text"
},
"countryIsoCode": {
"fields": {
"keyword": {
"ignore_above": 256,
"type": "keyword"
}
},
"type": "text"
},
"regionName": {
"fields": {
"keyword": {
"ignore_above": 256,
"type": "keyword"
}
},
"type": "text"
},
"location": {
"type": "geo_point"
},
"continentName": {
"fields": {
"keyword": {
"ignore_above": 256,
"type": "keyword"
}
},
"type": "text"
}
}
},
"remoteIp": {
"type": "ip"
},
"localPort": {
"type": "integer"
},
"scheme": {
"fields": {
"keyword": {
"ignore_above": 256,
"type": "keyword"
}
},
"type": "text"
},
"remotePort": {
"type": "integer"
},
"userAgent": {
"fields": {
"keyword": {
"ignore_above": 256,
"type": "keyword"
}
},
"type": "text"
},
"protocol": {
"fields": {
"keyword": {
"ignore_above": 256,
"type": "keyword"
}
},
"type": "text"
},
"responseContentType": {
"fields": {
"keyword": {
"ignore_above": 256,
"type": "keyword"
}
},
"type": "text"
},
"host": {
"fields": {
"keyword": {
"ignore_above": 256,
"type": "keyword"
}
},
"type": "text"
},
"isHttps": {
"type": "boolean"
},
"localIp": {
"type": "ip"
},
"requestContentType": {
"fields": {
"keyword": {
"ignore_above": 256,
"type": "keyword"
}
},
"type": "text"
}
}
},
"connectionId": {
"fields": {
"keyword": {
"ignore_above": 256,
"type": "keyword"
}
},
"type": "text"
},
"messageTemplate": {
"fields": {
"keyword": {
"ignore_above": 256,
"type": "keyword"
}
},
"type": "text"
},
"requestPath": {
"fields": {
"keyword": {
"ignore_above": 256,
"type": "keyword"
}
},
"type": "text"
},
"statusCode": {
"type": "integer"
}
}
}
}
So now I am expecting the documents indexed by Serilog to be using the mapping of the index template I have created. But what actually happens is, the indices created are NOT using the correct mapping from the template. So the IP fields are indexed as strings, which results into the GeoIp pipeline not working on the fields.
Now I am asking myself: Is my configuration faulty? Is Serilog always indexing the logevents with THEIR mapping? How do I apply the correct mapping to custom properties within a logevent?
So as described in this Github issue https://github.com/serilog/serilog-sinks-elasticsearch/issues/366 ...
turns out the index-template mapping created with the NEST client, was using camel-case for property names and the custom properties on the LogEvent came as pascal-case, which triggered dynamic-mapping of elasticsearch and lead to duplicate mapping entries.
EDIT
You can work around this by annotating the properties of your model with mapping attributes and set the names to pascal-case.