serilog-sinks-elasticsearch

Resend failed messages with Serilog.Sinks.Elasticsearch


I'm using the Serilog.Sinks.Elasticsearch nuget package and Elasticsearch was missing shards for a while. As a result I've got a lot of failed messages saved as invalid-**.json files. How do I resend those files?


Solution

  • This seems to work. Install Elasticsearch.Net nuget package

        private static async Task RetryLogMessages(string url, string directory)
        {
            string doneDirectoryPath = Path.Combine(directory, "Done");
            Directory.CreateDirectory(doneDirectoryPath);
            string[] filePaths = Directory.GetFiles(directory, "invalid*.json");
            ElasticLowLevelClient client = new(new ConnectionConfiguration(new Uri(url)));
            foreach (string filePath in filePaths)
            {
                string[] json = await File.ReadAllLinesAsync(filePath, Encoding.UTF8);
    
                DynamicResponse response = await client.BulkAsync<DynamicResponse>(PostData.MultiJson(json));
                await Task.Delay(100);// Take it easy??
    
                if (response.Success)
                {
                    string fileName = Path.GetFileName(filePath);
                    string destFileName = Path.Combine(doneDirectoryPath, fileName);
                    File.Move(filePath, destFileName);
                    Console.WriteLine($"{fileName} done");
                    // File.Delete(filePath);
                }
                else
                {
                    Console.WriteLine(response.HttpStatusCode);
                    Console.WriteLine(response.OriginalException);
                }
            }
        }