using System; using System.Collections.Generic; using System.Diagnostics; using System.Globalization; using System.Linq; using Azimuth; namespace Glass.PerformanceTests { public struct SteadyStateResult { public double MedianNanoseconds; public double MinimumNanoseconds; public double BytesPerCall; public Vector Fingerprint; /// /// One line of text, for passing from the process that measured it to the one reporting it. /// public string ToLine() { return string.Join(" ", new double[] { MedianNanoseconds, MinimumNanoseconds, BytesPerCall, Fingerprint.X, Fingerprint.Y, } .Select(value => value.ToString("R", CultureInfo.InvariantCulture))); } public static SteadyStateResult FromLine(string line) { double[] values = line.Split(' ') .Select(value => double.Parse(value, CultureInfo.InvariantCulture)) .ToArray(); return new SteadyStateResult { MedianNanoseconds = values[0], MinimumNanoseconds = values[1], BytesPerCall = values[2], Fingerprint = new Vector(values[3], values[4]), }; } } public sealed class ProcessVariant { public const string X86 = "x86"; public const string X64 = "x64"; public readonly string Label; public readonly string ExePath; public ProcessVariant(string label, string exePath) { Label = label; ExePath = exePath; } } public static class Measurement { public const double WarmUpMilliseconds = 100; public const double BatchMilliseconds = 5; /// /// Odd, so the median is a sample rather than an average of two. /// public const int Rounds = 31; public const int FirstCallLaunches = 5; public const string SteadyStateArgument = "--steady-state"; public const string FirstCallArgument = "--first-call"; public static double NanosecondsFor(long ticks) => ticks * 1e9 / Stopwatch.Frequency; public static double Median(List samples) { var sorted = new List(samples); sorted.Sort(); return sorted[sorted.Count / 2]; } public static double Minimum(List samples) { double returnMe = double.MaxValue; foreach (double sample in samples) { returnMe = Math.Min(returnMe, sample); } return returnMe; } /// /// Runs `run` long enough to JIT it and settle the caches, and returns how many calls fill one batch. /// public static int WarmUp(Func run, Workload workload) { long budget = (long)(WarmUpMilliseconds * Stopwatch.Frequency / 1000); long start = Stopwatch.GetTimestamp(); long calls = 0; long elapsed; do { run(workload); ++calls; elapsed = Stopwatch.GetTimestamp() - start; } while (elapsed < budget); double millisecondsPerCall = elapsed * 1000.0 / Stopwatch.Frequency / calls; return Math.Max(1, (int)(BatchMilliseconds / millisecondsPerCall)); } public static double NanosecondsPerCall(Func run, Workload workload, int calls) { long start = Stopwatch.GetTimestamp(); for (int i = 0; i < calls; ++i) { run(workload); } return NanosecondsFor(Stopwatch.GetTimestamp() - start) / calls; } public static double BytesPerCall(Func run, Workload workload, int calls) { long before = GC.GetAllocatedBytesForCurrentThread(); for (int i = 0; i < calls; ++i) { run(workload); } return (double)(GC.GetAllocatedBytesForCurrentThread() - before) / calls; } /// /// Times each engine on the scenario once warm, one batch at a time, taking turns. /// public static SteadyStateResult[] TimeSteadyState(Scenario scenario, IReadOnlyList engines) { var runs = new Func[engines.Count]; var callsPerBatch = new int[engines.Count]; var samples = new List[engines.Count]; for (int engine = 0; engine < engines.Count; ++engine) { runs[engine] = engines[engine].For(scenario.Operation); callsPerBatch[engine] = WarmUp(runs[engine], scenario.Workload); samples[engine] = new List(Rounds); } // Interleaved, and each round starting from a different engine, so drift in clock speed as the CPU // heats up lands on every engine alike rather than on whichever ran last. for (int round = 0; round < Rounds; ++round) { for (int turn = 0; turn < engines.Count; ++turn) { int engine = (round + turn) % engines.Count; samples[engine].Add(NanosecondsPerCall(runs[engine], scenario.Workload, callsPerBatch[engine])); } } var returnMe = new SteadyStateResult[engines.Count]; for (int engine = 0; engine < engines.Count; ++engine) { returnMe[engine] = new SteadyStateResult { MedianNanoseconds = Median(samples[engine]), MinimumNanoseconds = Minimum(samples[engine]), BytesPerCall = BytesPerCall(runs[engine], scenario.Workload, callsPerBatch[engine]), Fingerprint = runs[engine](scenario.Workload), }; } return returnMe; } /// /// Times a single call, which in a fresh process includes JIT and every one-time setup it triggers. /// public static long TicksForFirstCall(Scenario scenario, Engine engine) { Func run = engine.For(scenario.Operation); long start = Stopwatch.GetTimestamp(); run(scenario.Workload); return Stopwatch.GetTimestamp() - start; } public static string RunInFreshProcess(ProcessVariant variant, string arguments) { // Shares this console, since CreateNoWindow starts a console host per child, ~45ms each. var startInfo = new ProcessStartInfo(variant.ExePath, arguments) { UseShellExecute = false, RedirectStandardOutput = true, RedirectStandardError = true, }; using (Process child = Process.Start(startInfo)) { string output = child.StandardOutput.ReadToEnd(); string errors = child.StandardError.ReadToEnd(); child.WaitForExit(); if (child.ExitCode != 0) { throw new InvalidOperationException( $"{variant.Label} run with \"{arguments}\" exited with {child.ExitCode}: {errors}"); } return output; } } /// /// Every engine's steady-state result on a scenario, measured in a fresh process so nothing an earlier /// scenario JITted or warmed carries over. /// public static SteadyStateResult[] SteadyStateInFreshProcess(ProcessVariant variant, int scenarioIndex) { return RunInFreshProcess(variant, $"{SteadyStateArgument} {scenarioIndex}") .Split(new[] { '\r', '\n' }, StringSplitOptions.RemoveEmptyEntries) .Select(SteadyStateResult.FromLine) .ToArray(); } /// /// The median time of an engine's first call on a scenario, each in a fresh process. /// public static double FirstCallNanoseconds(ProcessVariant variant, int scenarioIndex, int engineIndex) { var samples = new List(FirstCallLaunches); for (int launch = 0; launch < FirstCallLaunches; ++launch) { string ticks = RunInFreshProcess(variant, $"{FirstCallArgument} {scenarioIndex} {engineIndex}"); samples.Add(NanosecondsFor(long.Parse(ticks.Trim(), CultureInfo.InvariantCulture))); } return Median(samples); } } }