I am a bit lost here, I am totally new to BenchmarkDotNet and trying to create a working benchmark with nice rendered output. I think I have the output part covered. However I did hit a view walls and every time I thought I had found a solution another problem pops up.
But first things first.
I want to test is the performance of collections (SortedDictionary, ImmutableSortedDictionary, SortedList) for different scenarios.
For the beginning I did start with an ItemInsert tests. I wanted to know how fast can I insert 10 randomly generated items into an existing collection of 0, 10, 100, 1000, 10_000, 100_000, 1_000_000, 10_000_000 and 100_000_000 items.
Problems I ran into (Related to BenchmarkDotNet not shortcommings of generic datatypes in general ;) )
Benchmarks are not isolated. I ran into this when trying to prefill the collections in the GlobalSetup. After the first Benchmark the next benchmark gets the collection with the itemd added by the first benchmark. Thats not what I want. So where is the place to prepare the collection to test? IterationSetup would be an option but there are warnings everywhere not to use it and also rebuilding the collection before each benchmark would be damn slow. And doing it in the Benchmark itself would mess with the measurement I guess.
So how do I write a benchmark that prefills a collection with items and uses exactly this prefilled list for all benchmarks without the need to regenerate the collection before each benchmark.
Ok, just for others to find. I did not find a more elegant way than this.
What I solved: I had to circumvent that there is no or in the generic where clause. And still only allow specific known Datatypes. I needed to run all benchmarks in the same class because prefill of the collections take long so I wanted to avoid unnecessary prefills at all cost. More Collections can added without much problem, because I want to add my own implementations later. Thats the hole point of that benchmark. I tried to avoid dynamic where possible.
What I did not solve: I did not add lookup and remove item tests now but that should be easy since preparing the prefilled collections work. I did not check Report output, but as long as the benchmark is generating Artefacts report problems can be fixed. I could not avoid the use of IterationSetup. I even had to use targets.
This is my BaseClass just in case I need more than one Benchmark class of this kind
namespace Benchmarks.Base;
using BenchmarkDotNet.Attributes;
using BenchmarkDotNet.Jobs;
using JetBrains.Annotations;
[CategoriesColumn]
[MinColumn]
[MaxColumn]
[MemoryDiagnoser(false)]
[HideColumns("Job")]
[ShortRunJob(RuntimeMoniker.Net80)]
[ShortRunJob(RuntimeMoniker.Net90)]
//[ShortRunJob(RuntimeMoniker.NativeAot80)] // Bug
//[ShortRunJob(RuntimeMoniker.NativeAot90)] // Bug
public class BenchmarkBase
{
[Params(0, 10, 100, 1_000, 10_000, 100_000, 1_000_000, 10_000_000, 100_000_000)]
public int ItemsAlreadyPresent { get; [UsedImplicitly] set; }
[Params(10)]
public int Inserts { get; [UsedImplicitly] set; }
[Params(1_00, 2_000, 5_000, 10_000)] public int PriceRange { get; [UsedImplicitly] set; }
}
And this is the Benchmark class
namespace Benchmarks;
using System.Collections.Immutable;
using System.ComponentModel.DataAnnotations;
using System.Numerics;
using Base;
using BenchmarkDotNet.Attributes;
using BenchmarkDotNetVisualizer;
using JetBrains.Annotations;
using Nethermind.Int256;
using Utils;
using OrderDataBI = SortedDictionary.BigInteger.OrderData;
using OrderDataKeyBI = SortedDictionary.BigInteger.OrderDataKey;
using OrderDataUI = SortedDictionary.UInt256.OrderData;
using OrderDataKeyUI = SortedDictionary.UInt256.OrderDataKey;
[PublicAPI]
[RichHtmlExporter("Benchmark of Collection Item Insert FlatBigint",
["Inserts", "PriceRange"],
["Mean", "Allocated"],
["Mean", "Min", "Max", "Allocated"],
dividerMode: RenderTableDividerMode.SeparateTables,
htmlWrapMode: HtmlDocumentWrapMode.RichDataTables)]
[GenericTypeArguments(typeof(OrderDataKeyBI), typeof(OrderDataBI))]
[GenericTypeArguments(typeof(OrderDataKeyUI), typeof(OrderDataUI))]
[Display(Name = "Benchmark of flat dictionaries item insert for {0}-{1} type", GroupName = "Benchmark of Collection Item Insert")]
// ReSharper disable once ClassCanBeSealed.Global
public class AddItemBenchmarkFlat<TKey, TData> : BenchmarkBase
where TKey : notnull
where TData : notnull
{
private IDictionary<TKey, TData> _collection = default!;
private bool _immutable;
private TData[] _prefillArray = default!;
public string[] GetDataTypeName { get; } = [typeof(TData).GetFormattedName()];
[ParamsSource(nameof(GetDataTypeName), Priority = -1)]
public string DataType { get; set; } = default!;
[GlobalSetup]
public void A_Global_Setup()
{
_orderDataKeys = new TKey[Inserts];
_orderDataArray = new TData[Inserts];
_prefillArray = new TData[ItemsAlreadyPresent];
var rnd = new Random();
if (typeof(TData) == typeof(OrderDataBI))
{
Parallel.For(0,
Inserts,
i =>
{
var price = new BigInteger(rnd.NextInt64(0, PriceRange));
var orderId = new Guid(i + ItemsAlreadyPresent, 0, 0, [0, 0, 0, 0, 0, 0, 0, 0]);
_orderDataKeys[i] = (dynamic)new OrderDataKeyBI(price, orderId);
_orderDataArray[i] = (dynamic)new OrderDataBI(price, orderId, 1);
});
Parallel.For(0,
ItemsAlreadyPresent,
i =>
{
var price = new BigInteger(rnd.NextInt64(0, PriceRange));
var orderId = new Guid(i, 0, 0, [0, 0, 0, 0, 0, 0, 0, 0]);
_prefillArray[i] = (dynamic)new OrderDataBI(price, orderId, 1);
});
}
else if (typeof(TData) == typeof(OrderDataUI))
{
Parallel.For(0,
Inserts,
i =>
{
var price = new UInt256((ulong)rnd.NextInt64(0, PriceRange));
var orderId = new Guid(i + ItemsAlreadyPresent, 0, 0, [0, 0, 0, 0, 0, 0, 0, 0]);
_orderDataKeys[i] = (dynamic)new OrderDataKeyUI(price, orderId);
_orderDataArray[i] = (dynamic)new OrderDataUI(price, orderId, 1);
});
Parallel.For(0,
ItemsAlreadyPresent,
i =>
{
var price = new UInt256((ulong)rnd.NextInt64(0, PriceRange));
var orderId = new Guid(i, 0, 0, [0, 0, 0, 0, 0, 0, 0, 0]);
_prefillArray[i] = (dynamic)new OrderDataUI(price, orderId, 1);
});
}
else { throw new Exception("Unsupported Type"); }
}
[IterationSetup(Target = "SortedDictionary_AddItem")]
public void A_Iteration_Setup_SortedDictionary()
{
if (typeof(TData) == typeof(OrderDataBI))
{
_collection =
(dynamic)new SortedDictionary<OrderDataKeyBI, OrderDataBI>(((OrderDataBI[])(dynamic)_prefillArray).ToDictionary(static pair =>
new OrderDataKeyBI(pair.Price, pair.OrderId),
static pair => pair));
}
else
{
_collection =
(dynamic)new SortedDictionary<OrderDataKeyUI, OrderDataUI>(((OrderDataUI[])(dynamic)_prefillArray).ToDictionary(static pair =>
new OrderDataKeyUI(pair.Price, pair.OrderId),
static pair => pair));
}
}
[IterationSetup(Target = "SortedImmutableDictionary_AddItem")]
public void A_Iteration_Setup_SortedImmutableDictionary()
{
if (typeof(TData) == typeof(OrderDataBI))
{
_collection = (dynamic)((OrderDataBI[])(dynamic)_prefillArray).ToImmutableSortedDictionary(static p => new OrderDataKeyBI(p.Price, p.OrderId), p => p);
}
else { _collection = (dynamic)((OrderDataUI[])(dynamic)_prefillArray).ToImmutableSortedDictionary(static p => new OrderDataKeyUI(p.Price, p.OrderId), p => p); }
}
[IterationSetup(Target = "SortedList_AddItem")]
public void A_Iteration_Setup_SortedList()
{
if (typeof(TData) == typeof(OrderDataBI))
{
_collection =
(dynamic)new SortedList<OrderDataKeyBI, OrderDataBI>(((OrderDataBI[])(dynamic)_prefillArray).ToDictionary(static pair =>
new OrderDataKeyBI(pair.Price, pair.OrderId),
static pair => pair));
}
else
{
_collection =
(dynamic)new SortedList<OrderDataKeyUI, OrderDataUI>(((OrderDataUI[])(dynamic)_prefillArray).ToDictionary(static pair =>
new OrderDataKeyUI(pair.Price, pair.OrderId),
static pair => pair));
}
}
[Benchmark]
public void SortedDictionary_AddItem()
{
BenchmarkHelper.FlatCollection(_orderDataKeys, _orderDataArray, ref _collection);
}
[Benchmark]
public void SortedImmutableDictionary_AddItem()
{
var immutable = (IImmutableDictionary<TKey, TData>)_collection;
BenchmarkHelper.FlatImmutableCollection(_orderDataKeys, _orderDataArray, ref immutable);
}
[Benchmark]
public void SortedList_AddItem()
{
BenchmarkHelper.FlatCollection(_orderDataKeys, _orderDataArray, ref _collection);
}
// ReSharper disable NullableWarningSuppressionIsUsed
private TData[] _orderDataArray = default!;
private TKey[] _orderDataKeys = default!;
// ReSharper restore NullableWarningSuppressionIsUsed
}