using System; using System.Collections.Generic; using System.Diagnostics; using System.Globalization; using System.IO; using System.Reflection; using System.Runtime.InteropServices; using System.Threading; using Azimuth; namespace Glass.PerformanceTests { /// /// Times every text layout engine on the same scenarios, for the numbers deciding whether one can replace /// another. /// public static class Program { public static bool IsJitOptimizerDisabled(Assembly assembly) { var debuggable = assembly.GetCustomAttribute(); return debuggable != null && debuggable.IsJITOptimizerDisabled; } /// /// Why timings taken in this process wouldn't mean anything, or null when they would. /// public static string ReasonTimingsAreMeaningless() { if (Debugger.IsAttached) { return "a debugger is attached"; } foreach (Assembly assembly in new[] { typeof(Program).Assembly, typeof(TextLayout).Assembly }) { if (IsJitOptimizerDisabled(assembly)) { return $"{assembly.GetName().Name} is a Debug build"; } } return null; } /// /// One core at high priority, so the scheduler neither migrates the timing loop nor preempts it for background /// work. /// public static void ClaimOneCore() { Process self = Process.GetCurrentProcess(); self.PriorityClass = ProcessPriorityClass.High; self.ProcessorAffinity = (IntPtr)(1L << (Environment.ProcessorCount - 1)); } public const float BusyMachinePercent = 20; public const int BusySampleMilliseconds = 500; [DllImport("kernel32.dll", SetLastError = true)] [return: MarshalAs(UnmanagedType.Bool)] private static extern bool GetSystemTimes(out long idleTime, out long kernelTime, out long userTime); /// /// How busy every processor was over a short sample, or null where Windows won't say. /// public static float? SystemBusyPercent() { if (!GetSystemTimes(out long idleBefore, out long kernelBefore, out long userBefore)) { return null; } Thread.Sleep(BusySampleMilliseconds); if (!GetSystemTimes(out long idleAfter, out long kernelAfter, out long userAfter)) { return null; } // Kernel time includes idle time. long total = kernelAfter - kernelBefore + userAfter - userBefore; long idle = idleAfter - idleBefore; if (total <= 0) { return null; } return 100f * (total - idle) / total; } public static string FormatFingerprint(Vector fingerprint) { return string.Format(CultureInfo.InvariantCulture, "{0:0.#} x {1:0.#}", (double)fingerprint.X, (double)fingerprint.Y); } public static string FormatRatio(double value, double baseline) { return string.Format(CultureInfo.InvariantCulture, "{0,7:0.000}x", value / baseline); } public static Scenario ScenarioAt(string index) { return Scenarios.BuildAll()[int.Parse(index, CultureInfo.InvariantCulture)]; } public static int RunSteadyState(string scenarioIndex) { ClaimOneCore(); foreach (SteadyStateResult result in Measurement.TimeSteadyState(ScenarioAt(scenarioIndex), Engines.All)) { Console.WriteLine(result.ToLine()); } return 0; } public static int RunFirstCall(string scenarioIndex, string engineIndex) { ClaimOneCore(); Scenario scenario = ScenarioAt(scenarioIndex); Engine engine = Engines.All[int.Parse(engineIndex, CultureInfo.InvariantCulture)]; Console.WriteLine(Measurement.TicksForFirstCall(scenario, engine).ToString(CultureInfo.InvariantCulture)); return 0; } /// /// The 32- and 64-bit builds of this program the options ask for, whichever of them were built, found next to /// the running one. /// public static List FindVariants(RunOptions options) { string directory = Path.GetDirectoryName(Process.GetCurrentProcess().MainModule.FileName); var returnMe = new List(); foreach ((string label, string exeName) in new[] { (ProcessVariant.X86, "Glass.PerformanceTests.exe"), (ProcessVariant.X64, "Glass.PerformanceTests64.exe"), }) { if (!options.Wants(label)) { continue; } string path = Path.Combine(directory, exeName); if (File.Exists(path)) { returnMe.Add(new ProcessVariant(label, path)); } else { Console.WriteLine($"WARNING: {path} isn't built, so there are no {label} results."); } } return returnMe; } public static void PrintHeader(bool comparing, bool timingFirstCalls, float? busyPercent) { Console.WriteLine($".NET {Environment.Version}, {Environment.ProcessorCount} logical processors, " + $"timer at {Stopwatch.Frequency / 1e6:0.##} MHz. Every measurement runs in a fresh process, " + "pinned to the last processor at high priority."); if (busyPercent > BusyMachinePercent) { Console.WriteLine($"WARNING: the machine was {busyPercent:0}% busy before starting. Absolute times " + "will be inflated and noisy; engines take turns, so ratios within this run still compare."); } var fonts = new[] { SampleFonts.Philosopher18, SampleFonts.LiberationSans16 }; foreach (SpriteFontDefinition font in fonts) { Console.WriteLine($"{SampleFonts.Describe(font)}: {font.GlyphDefinitions.Count} glyphs, " + $"{font.Kernings.Count} kerning pairs"); } Console.WriteLine(); if (timingFirstCalls) { Console.WriteLine("Times are per call, warm unless marked first call; first call is the median of " + $"{Measurement.FirstCallLaunches} fresh processes, timed once per font and text for an engine " + "that reads nothing else."); } else { Console.WriteLine($"Times are per call, warm; {RunOptions.SteadyOnlyOption} skipped first calls."); } if (comparing) { Console.WriteLine($"Ratios are against {Engines.All[0].Name} at the same bitness."); } string header = $"{"engine",-16} {"median us",10} {"min us",10} {"ns/char",8} {"bytes",8}"; if (timingFirstCalls) { header += $" {"first us",10}"; } if (comparing) { header += $" {"median",8}"; if (timingFirstCalls) { header += $" {"first",8}"; } } Console.WriteLine(header + " result"); } /// Null when first calls weren't timed. public static void PrintScenario(Scenario scenario, IReadOnlyList engines, IReadOnlyList variants, SteadyStateResult[][] steady, double[][] firstCall) { Console.WriteLine(); Console.WriteLine(scenario.Name); int characters = scenario.Workload.Text.Length; for (int variant = 0; variant < variants.Count; ++variant) { for (int engine = 0; engine < engines.Count; ++engine) { SteadyStateResult result = steady[variant][engine]; string line = string.Format(CultureInfo.InvariantCulture, "{0,-16} {1,10:0.00} {2,10:0.00} {3,8:0.0} {4,8:0.#}", $"{engines[engine].Name} {variants[variant].Label}", result.MedianNanoseconds / 1000, result.MinimumNanoseconds / 1000, result.MedianNanoseconds / characters, result.BytesPerCall); if (firstCall != null) { line += string.Format(CultureInfo.InvariantCulture, " {0,10:0.0}", firstCall[variant][engine] / 1000); } if (engines.Count > 1) { line += " " + FormatRatio(result.MedianNanoseconds, steady[variant][0].MedianNanoseconds); if (firstCall != null) { line += " " + FormatRatio(firstCall[variant][engine], firstCall[variant][0]); } } Console.WriteLine(line + " " + FormatFingerprint(result.Fingerprint)); } } } /// /// Each engine's first call on a scenario, reusing the time from an earlier scenario that did the same work. /// /// Times already taken, keyed by engine and the earliest scenario doing that work. public static double[] TimeFirstCalls(ProcessVariant variant, List scenarios, int scenarioIndex, IReadOnlyList engines, Dictionary<(int engine, int scenario), double> timed) { var returnMe = new double[engines.Count]; for (int engine = 0; engine < engines.Count; ++engine) { int earliest = scenarios.FindIndex( other => engines[engine].DoesSameWork(other, scenarios[scenarioIndex])); if (!timed.TryGetValue((engine, earliest), out returnMe[engine])) { returnMe[engine] = Measurement.FirstCallNanoseconds(variant, earliest, engine); timed.Add((engine, earliest), returnMe[engine]); } } return returnMe; } public static int Main(string[] args) { string reason = ReasonTimingsAreMeaningless(); if (reason != null) { Console.Error.WriteLine($"Refusing to benchmark: {reason}. Run a Release build without a debugger."); return 1; } if (args.Length == 2 && args[0] == Measurement.SteadyStateArgument) { return RunSteadyState(args[1]); } if (args.Length == 3 && args[0] == Measurement.FirstCallArgument) { return RunFirstCall(args[1], args[2]); } RunOptions options = RunOptions.Parse(args); if (options == null) { Console.Error.WriteLine(RunOptions.Usage); return 1; } float? busyPercent = SystemBusyPercent(); IReadOnlyList engines = Engines.All; List variants = FindVariants(options); List scenarios = Scenarios.BuildAll(); PrintHeader(comparing: engines.Count > 1, timingFirstCalls: !options.SteadyOnly, busyPercent); var firstCallsTimed = new Dictionary<(int engine, int scenario), double>[variants.Count]; for (int variant = 0; variant < variants.Count; ++variant) { firstCallsTimed[variant] = new Dictionary<(int engine, int scenario), double>(); } for (int index = 0; index < scenarios.Count; ++index) { Scenario scenario = scenarios[index]; if (scenario.Name.IndexOf(options.ScenarioFilter, StringComparison.OrdinalIgnoreCase) < 0) { continue; } var steady = new SteadyStateResult[variants.Count][]; double[][] firstCall = options.SteadyOnly ? null : new double[variants.Count][]; for (int variant = 0; variant < variants.Count; ++variant) { steady[variant] = Measurement.SteadyStateInFreshProcess(variants[variant], index); if (firstCall != null) { firstCall[variant] = TimeFirstCalls(variants[variant], scenarios, index, engines, firstCallsTimed[variant]); } } PrintScenario(scenario, engines, variants, steady, firstCall); } return 0; } } }