using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Globalization;
using System.IO;
using System.Reflection;
using System.Runtime.InteropServices;
using System.Threading;
using Azimuth;
namespace Glass.PerformanceTests
{
///
/// Times every text layout engine on the same scenarios, for the numbers deciding whether one can replace
/// another.
///
public static class Program
{
public static bool IsJitOptimizerDisabled(Assembly assembly)
{
var debuggable = assembly.GetCustomAttribute();
return debuggable != null && debuggable.IsJITOptimizerDisabled;
}
///
/// Why timings taken in this process wouldn't mean anything, or null when they would.
///
public static string ReasonTimingsAreMeaningless()
{
if (Debugger.IsAttached)
{
return "a debugger is attached";
}
foreach (Assembly assembly in new[] { typeof(Program).Assembly, typeof(TextLayout).Assembly })
{
if (IsJitOptimizerDisabled(assembly))
{
return $"{assembly.GetName().Name} is a Debug build";
}
}
return null;
}
///
/// One core at high priority, so the scheduler neither migrates the timing loop nor preempts it for background
/// work.
///
public static void ClaimOneCore()
{
Process self = Process.GetCurrentProcess();
self.PriorityClass = ProcessPriorityClass.High;
self.ProcessorAffinity = (IntPtr)(1L << (Environment.ProcessorCount - 1));
}
public const float BusyMachinePercent = 20;
public const int BusySampleMilliseconds = 500;
[DllImport("kernel32.dll", SetLastError = true)]
[return: MarshalAs(UnmanagedType.Bool)]
private static extern bool GetSystemTimes(out long idleTime, out long kernelTime, out long userTime);
///
/// How busy every processor was over a short sample, or null where Windows won't say.
///
public static float? SystemBusyPercent()
{
if (!GetSystemTimes(out long idleBefore, out long kernelBefore, out long userBefore))
{
return null;
}
Thread.Sleep(BusySampleMilliseconds);
if (!GetSystemTimes(out long idleAfter, out long kernelAfter, out long userAfter))
{
return null;
}
// Kernel time includes idle time.
long total = kernelAfter - kernelBefore + userAfter - userBefore;
long idle = idleAfter - idleBefore;
if (total <= 0)
{
return null;
}
return 100f * (total - idle) / total;
}
public static string FormatFingerprint(Vector fingerprint)
{
return string.Format(CultureInfo.InvariantCulture, "{0:0.#} x {1:0.#}",
(double)fingerprint.X, (double)fingerprint.Y);
}
public static string FormatRatio(double value, double baseline)
{
return string.Format(CultureInfo.InvariantCulture, "{0,7:0.000}x", value / baseline);
}
public static Scenario ScenarioAt(string index)
{
return Scenarios.BuildAll()[int.Parse(index, CultureInfo.InvariantCulture)];
}
public static int RunSteadyState(string scenarioIndex)
{
ClaimOneCore();
foreach (SteadyStateResult result in Measurement.TimeSteadyState(ScenarioAt(scenarioIndex), Engines.All))
{
Console.WriteLine(result.ToLine());
}
return 0;
}
public static int RunFirstCall(string scenarioIndex, string engineIndex)
{
ClaimOneCore();
Scenario scenario = ScenarioAt(scenarioIndex);
Engine engine = Engines.All[int.Parse(engineIndex, CultureInfo.InvariantCulture)];
Console.WriteLine(Measurement.TicksForFirstCall(scenario, engine).ToString(CultureInfo.InvariantCulture));
return 0;
}
///
/// The 32- and 64-bit builds of this program the options ask for, whichever of them were built, found next to
/// the running one.
///
public static List FindVariants(RunOptions options)
{
string directory = Path.GetDirectoryName(Process.GetCurrentProcess().MainModule.FileName);
var returnMe = new List();
foreach ((string label, string exeName) in new[]
{
(ProcessVariant.X86, "Glass.PerformanceTests.exe"),
(ProcessVariant.X64, "Glass.PerformanceTests64.exe"),
})
{
if (!options.Wants(label))
{
continue;
}
string path = Path.Combine(directory, exeName);
if (File.Exists(path))
{
returnMe.Add(new ProcessVariant(label, path));
}
else
{
Console.WriteLine($"WARNING: {path} isn't built, so there are no {label} results.");
}
}
return returnMe;
}
public static void PrintHeader(bool comparing, bool timingFirstCalls, float? busyPercent)
{
Console.WriteLine($".NET {Environment.Version}, {Environment.ProcessorCount} logical processors, " +
$"timer at {Stopwatch.Frequency / 1e6:0.##} MHz. Every measurement runs in a fresh process, " +
"pinned to the last processor at high priority.");
if (busyPercent > BusyMachinePercent)
{
Console.WriteLine($"WARNING: the machine was {busyPercent:0}% busy before starting. Absolute times " +
"will be inflated and noisy; engines take turns, so ratios within this run still compare.");
}
var fonts = new[] { SampleFonts.Philosopher18, SampleFonts.LiberationSans16 };
foreach (SpriteFontDefinition font in fonts)
{
Console.WriteLine($"{SampleFonts.Describe(font)}: {font.GlyphDefinitions.Count} glyphs, " +
$"{font.Kernings.Count} kerning pairs");
}
Console.WriteLine();
if (timingFirstCalls)
{
Console.WriteLine("Times are per call, warm unless marked first call; first call is the median of " +
$"{Measurement.FirstCallLaunches} fresh processes, timed once per font and text for an engine " +
"that reads nothing else.");
}
else
{
Console.WriteLine($"Times are per call, warm; {RunOptions.SteadyOnlyOption} skipped first calls.");
}
if (comparing)
{
Console.WriteLine($"Ratios are against {Engines.All[0].Name} at the same bitness.");
}
string header = $"{"engine",-16} {"median us",10} {"min us",10} {"ns/char",8} {"bytes",8}";
if (timingFirstCalls)
{
header += $" {"first us",10}";
}
if (comparing)
{
header += $" {"median",8}";
if (timingFirstCalls)
{
header += $" {"first",8}";
}
}
Console.WriteLine(header + " result");
}
/// Null when first calls weren't timed.
public static void PrintScenario(Scenario scenario, IReadOnlyList engines,
IReadOnlyList variants, SteadyStateResult[][] steady, double[][] firstCall)
{
Console.WriteLine();
Console.WriteLine(scenario.Name);
int characters = scenario.Workload.Text.Length;
for (int variant = 0; variant < variants.Count; ++variant)
{
for (int engine = 0; engine < engines.Count; ++engine)
{
SteadyStateResult result = steady[variant][engine];
string line = string.Format(CultureInfo.InvariantCulture,
"{0,-16} {1,10:0.00} {2,10:0.00} {3,8:0.0} {4,8:0.#}",
$"{engines[engine].Name} {variants[variant].Label}",
result.MedianNanoseconds / 1000, result.MinimumNanoseconds / 1000,
result.MedianNanoseconds / characters, result.BytesPerCall);
if (firstCall != null)
{
line += string.Format(CultureInfo.InvariantCulture, " {0,10:0.0}",
firstCall[variant][engine] / 1000);
}
if (engines.Count > 1)
{
line += " " + FormatRatio(result.MedianNanoseconds, steady[variant][0].MedianNanoseconds);
if (firstCall != null)
{
line += " " + FormatRatio(firstCall[variant][engine], firstCall[variant][0]);
}
}
Console.WriteLine(line + " " + FormatFingerprint(result.Fingerprint));
}
}
}
///
/// Each engine's first call on a scenario, reusing the time from an earlier scenario that did the same work.
///
/// Times already taken, keyed by engine and the earliest scenario doing that work.
public static double[] TimeFirstCalls(ProcessVariant variant, List scenarios, int scenarioIndex,
IReadOnlyList engines, Dictionary<(int engine, int scenario), double> timed)
{
var returnMe = new double[engines.Count];
for (int engine = 0; engine < engines.Count; ++engine)
{
int earliest = scenarios.FindIndex(
other => engines[engine].DoesSameWork(other, scenarios[scenarioIndex]));
if (!timed.TryGetValue((engine, earliest), out returnMe[engine]))
{
returnMe[engine] = Measurement.FirstCallNanoseconds(variant, earliest, engine);
timed.Add((engine, earliest), returnMe[engine]);
}
}
return returnMe;
}
public static int Main(string[] args)
{
string reason = ReasonTimingsAreMeaningless();
if (reason != null)
{
Console.Error.WriteLine($"Refusing to benchmark: {reason}. Run a Release build without a debugger.");
return 1;
}
if (args.Length == 2 && args[0] == Measurement.SteadyStateArgument)
{
return RunSteadyState(args[1]);
}
if (args.Length == 3 && args[0] == Measurement.FirstCallArgument)
{
return RunFirstCall(args[1], args[2]);
}
RunOptions options = RunOptions.Parse(args);
if (options == null)
{
Console.Error.WriteLine(RunOptions.Usage);
return 1;
}
float? busyPercent = SystemBusyPercent();
IReadOnlyList engines = Engines.All;
List variants = FindVariants(options);
List scenarios = Scenarios.BuildAll();
PrintHeader(comparing: engines.Count > 1, timingFirstCalls: !options.SteadyOnly, busyPercent);
var firstCallsTimed = new Dictionary<(int engine, int scenario), double>[variants.Count];
for (int variant = 0; variant < variants.Count; ++variant)
{
firstCallsTimed[variant] = new Dictionary<(int engine, int scenario), double>();
}
for (int index = 0; index < scenarios.Count; ++index)
{
Scenario scenario = scenarios[index];
if (scenario.Name.IndexOf(options.ScenarioFilter, StringComparison.OrdinalIgnoreCase) < 0)
{
continue;
}
var steady = new SteadyStateResult[variants.Count][];
double[][] firstCall = options.SteadyOnly ? null : new double[variants.Count][];
for (int variant = 0; variant < variants.Count; ++variant)
{
steady[variant] = Measurement.SteadyStateInFreshProcess(variants[variant], index);
if (firstCall != null)
{
firstCall[variant] = TimeFirstCalls(variants[variant], scenarios, index, engines,
firstCallsTimed[variant]);
}
}
PrintScenario(scenario, engines, variants, steady, firstCall);
}
return 0;
}
}
}