using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Globalization;
using System.Linq;
using Azimuth;
namespace Glass.PerformanceTests
{
public struct SteadyStateResult
{
public double MedianNanoseconds;
public double MinimumNanoseconds;
public double BytesPerCall;
public Vector Fingerprint;
///
/// One line of text, for passing from the process that measured it to the one reporting it.
///
public string ToLine()
{
return string.Join(" ", new double[]
{
MedianNanoseconds, MinimumNanoseconds, BytesPerCall, Fingerprint.X, Fingerprint.Y,
}
.Select(value => value.ToString("R", CultureInfo.InvariantCulture)));
}
public static SteadyStateResult FromLine(string line)
{
double[] values = line.Split(' ')
.Select(value => double.Parse(value, CultureInfo.InvariantCulture))
.ToArray();
return new SteadyStateResult
{
MedianNanoseconds = values[0],
MinimumNanoseconds = values[1],
BytesPerCall = values[2],
Fingerprint = new Vector(values[3], values[4]),
};
}
}
public sealed class ProcessVariant
{
public const string X86 = "x86";
public const string X64 = "x64";
public readonly string Label;
public readonly string ExePath;
public ProcessVariant(string label, string exePath)
{
Label = label;
ExePath = exePath;
}
}
public static class Measurement
{
public const double WarmUpMilliseconds = 100;
public const double BatchMilliseconds = 5;
///
/// Odd, so the median is a sample rather than an average of two.
///
public const int Rounds = 31;
public const int FirstCallLaunches = 5;
public const string SteadyStateArgument = "--steady-state";
public const string FirstCallArgument = "--first-call";
public static double NanosecondsFor(long ticks) => ticks * 1e9 / Stopwatch.Frequency;
public static double Median(List samples)
{
var sorted = new List(samples);
sorted.Sort();
return sorted[sorted.Count / 2];
}
public static double Minimum(List samples)
{
double returnMe = double.MaxValue;
foreach (double sample in samples)
{
returnMe = Math.Min(returnMe, sample);
}
return returnMe;
}
///
/// Runs `run` long enough to JIT it and settle the caches, and returns how many calls fill one batch.
///
public static int WarmUp(Func run, Workload workload)
{
long budget = (long)(WarmUpMilliseconds * Stopwatch.Frequency / 1000);
long start = Stopwatch.GetTimestamp();
long calls = 0;
long elapsed;
do
{
run(workload);
++calls;
elapsed = Stopwatch.GetTimestamp() - start;
}
while (elapsed < budget);
double millisecondsPerCall = elapsed * 1000.0 / Stopwatch.Frequency / calls;
return Math.Max(1, (int)(BatchMilliseconds / millisecondsPerCall));
}
public static double NanosecondsPerCall(Func run, Workload workload, int calls)
{
long start = Stopwatch.GetTimestamp();
for (int i = 0; i < calls; ++i)
{
run(workload);
}
return NanosecondsFor(Stopwatch.GetTimestamp() - start) / calls;
}
public static double BytesPerCall(Func run, Workload workload, int calls)
{
long before = GC.GetAllocatedBytesForCurrentThread();
for (int i = 0; i < calls; ++i)
{
run(workload);
}
return (double)(GC.GetAllocatedBytesForCurrentThread() - before) / calls;
}
///
/// Times each engine on the scenario once warm, one batch at a time, taking turns.
///
public static SteadyStateResult[] TimeSteadyState(Scenario scenario, IReadOnlyList engines)
{
var runs = new Func[engines.Count];
var callsPerBatch = new int[engines.Count];
var samples = new List[engines.Count];
for (int engine = 0; engine < engines.Count; ++engine)
{
runs[engine] = engines[engine].For(scenario.Operation);
callsPerBatch[engine] = WarmUp(runs[engine], scenario.Workload);
samples[engine] = new List(Rounds);
}
// Interleaved, and each round starting from a different engine, so drift in clock speed as the CPU
// heats up lands on every engine alike rather than on whichever ran last.
for (int round = 0; round < Rounds; ++round)
{
for (int turn = 0; turn < engines.Count; ++turn)
{
int engine = (round + turn) % engines.Count;
samples[engine].Add(NanosecondsPerCall(runs[engine], scenario.Workload, callsPerBatch[engine]));
}
}
var returnMe = new SteadyStateResult[engines.Count];
for (int engine = 0; engine < engines.Count; ++engine)
{
returnMe[engine] = new SteadyStateResult
{
MedianNanoseconds = Median(samples[engine]),
MinimumNanoseconds = Minimum(samples[engine]),
BytesPerCall = BytesPerCall(runs[engine], scenario.Workload, callsPerBatch[engine]),
Fingerprint = runs[engine](scenario.Workload),
};
}
return returnMe;
}
///
/// Times a single call, which in a fresh process includes JIT and every one-time setup it triggers.
///
public static long TicksForFirstCall(Scenario scenario, Engine engine)
{
Func run = engine.For(scenario.Operation);
long start = Stopwatch.GetTimestamp();
run(scenario.Workload);
return Stopwatch.GetTimestamp() - start;
}
public static string RunInFreshProcess(ProcessVariant variant, string arguments)
{
// Shares this console, since CreateNoWindow starts a console host per child, ~45ms each.
var startInfo = new ProcessStartInfo(variant.ExePath, arguments)
{
UseShellExecute = false,
RedirectStandardOutput = true,
RedirectStandardError = true,
};
using (Process child = Process.Start(startInfo))
{
string output = child.StandardOutput.ReadToEnd();
string errors = child.StandardError.ReadToEnd();
child.WaitForExit();
if (child.ExitCode != 0)
{
throw new InvalidOperationException(
$"{variant.Label} run with \"{arguments}\" exited with {child.ExitCode}: {errors}");
}
return output;
}
}
///
/// Every engine's steady-state result on a scenario, measured in a fresh process so nothing an earlier
/// scenario JITted or warmed carries over.
///
public static SteadyStateResult[] SteadyStateInFreshProcess(ProcessVariant variant, int scenarioIndex)
{
return RunInFreshProcess(variant, $"{SteadyStateArgument} {scenarioIndex}")
.Split(new[] { '\r', '\n' }, StringSplitOptions.RemoveEmptyEntries)
.Select(SteadyStateResult.FromLine)
.ToArray();
}
///
/// The median time of an engine's first call on a scenario, each in a fresh process.
///
public static double FirstCallNanoseconds(ProcessVariant variant, int scenarioIndex, int engineIndex)
{
var samples = new List(FirstCallLaunches);
for (int launch = 0; launch < FirstCallLaunches; ++launch)
{
string ticks = RunInFreshProcess(variant, $"{FirstCallArgument} {scenarioIndex} {engineIndex}");
samples.Add(NanosecondsFor(long.Parse(ticks.Trim(), CultureInfo.InvariantCulture)));
}
return Median(samples);
}
}
}