using System; using System.Collections.Generic; using System.Globalization; using System.IO; using System.Linq; using Glass.Typesetting; using UnitTestSharp; namespace Glass.UnitTests.Typesetting { public class UnicodePropertiesTests : TestFixture { /// /// Where the build unpacks 3rdParty\Unicode\UCD-*.7z, next to this binary. /// public static readonly string DataDirectory = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "Unicode"); public const int CodePointCount = UnicodeProperties.MaxCodePoint + 1; public const int DottedCircle = 0x25CC; public const int SoftHyphen = 0x00AD; public static string[] ReadProperty(string fileName, string valueWhereUnlisted) { var returnMe = Enumerable.Repeat(valueWhereUnlisted, CodePointCount).ToArray(); foreach ((int first, int last, string value) in ReadRanges(fileName)) { for (int codePoint = first; codePoint <= last; ++codePoint) { returnMe[codePoint] = value; } } return returnMe; } public static IEnumerable<(int First, int Last, string Value)> ReadRanges(string fileName) { foreach (string line in File.ReadLines(Path.Combine(DataDirectory, fileName))) { string data = line.Split('#')[0].Trim(); if (data.Length == 0) { continue; } string[] fields = data.Split(';'); string[] range = fields[0].Trim().Split(new[] { ".." }, StringSplitOptions.None); yield return (int.Parse(range[0], NumberStyles.HexNumber), int.Parse(range[range.Length - 1], NumberStyles.HexNumber), fields[1].Trim()); } } /// /// LB1's default resolutions, plus QU split by whether it's initial or final punctuation. /// public static LineBreakClass ResolvedClass(string lineBreak, string generalCategory) { switch (lineBreak) { case "AI": case "SG": case "XX": return LineBreakClass.AL; case "SA": return generalCategory == "Mn" || generalCategory == "Mc" ? LineBreakClass.CM : LineBreakClass.AL; case "CJ": return LineBreakClass.NS; case "QU" when generalCategory == "Pi": return LineBreakClass.QU_Pi; case "QU" when generalCategory == "Pf": return LineBreakClass.QU_Pf; default: return (LineBreakClass)Enum.Parse(typeof(LineBreakClass), lineBreak); } } public static string Describe(CharacterProperties properties) { return $"{properties.LineBreakClass()}" + (properties.IsEastAsian() ? " east-asian" : "") + (properties.IsUnassignedExtendedPictographic() ? " unassigned-pictographic" : ""); } public class OfTests : TestFixture { [CpuTimeout(3000)] public void EveryCodePoint_MatchesTheCharacterDatabase() { string[] lineBreaks = ReadProperty("LineBreak.txt", valueWhereUnlisted: "XX"); string[] widths = ReadProperty("EastAsianWidth.txt", valueWhereUnlisted: "N"); string[] categories = ReadProperty("DerivedGeneralCategory.txt", valueWhereUnlisted: "Cn"); var pictographic = new bool[CodePointCount]; foreach ((int first, int last, string value) in ReadRanges("emoji-data.txt")) { for (int codePoint = first; codePoint <= last; ++codePoint) { pictographic[codePoint] |= value == "Extended_Pictographic"; } } var mismatches = new List(); for (int codePoint = 0; codePoint < CodePointCount; ++codePoint) { var expected = (CharacterProperties)(codePoint == DottedCircle ? LineBreakClass.DottedCircle : codePoint == SoftHyphen ? LineBreakClass.SoftHyphen : ResolvedClass(lineBreaks[codePoint], categories[codePoint])); if (widths[codePoint] == "F" || widths[codePoint] == "W" || widths[codePoint] == "H") { expected |= CharacterProperties.EastAsian; } if (pictographic[codePoint] && categories[codePoint] == "Cn") { expected |= CharacterProperties.UnassignedExtendedPictographic; } CharacterProperties actual = UnicodeProperties.Of(codePoint); if (actual != expected && mismatches.Count < 10) { mismatches.Add($"U+{codePoint:X4}: expected {Describe(expected)}, was {Describe(actual)}"); } } CheckEqual("", string.Join("; ", mismatches)); } public void Letter_IsAlphabeticAndNotEastAsian() { CheckEqual("AL", Describe(UnicodeProperties.Of('A'))); } public void Ideograph_IsIdeographicAndEastAsian() { CheckEqual("ID east-asian", Describe(UnicodeProperties.Of(0x5B57))); } public void FullwidthParenthesis_IsEastAsian_WhereTheAsciiOneIsNot() { CheckEqual("OP east-asian", Describe(UnicodeProperties.Of(0xFF08))); CheckEqual("OP", Describe(UnicodeProperties.Of('('))); } public void ComplexContextLetter_ResolvesToAlphabetic() { CheckEqual(LineBreakClass.AL, UnicodeProperties.Of(0x0E01).LineBreakClass()); } public void ComplexContextMark_ResolvesToCombiningMark() { CheckEqual(LineBreakClass.CM, UnicodeProperties.Of(0x0E31).LineBreakClass()); } public void AmbiguousCharacter_ResolvesToAlphabetic() { CheckEqual(LineBreakClass.AL, UnicodeProperties.Of(0x00A7).LineBreakClass()); } public void ConditionalJapaneseStarter_ResolvesToNonstarter() { CheckEqual(LineBreakClass.NS, UnicodeProperties.Of(0x3041).LineBreakClass()); } public void UnassignedCodePoint_ResolvesToAlphabetic() { CheckEqual(LineBreakClass.AL, UnicodeProperties.Of(0x0378).LineBreakClass()); } public void Surrogate_ResolvesToAlphabetic() { CheckEqual(LineBreakClass.AL, UnicodeProperties.Of(0xD800).LineBreakClass()); } public void QuotationMarks_SplitByInitialAndFinalPunctuation() { char[] quotes = { '"', (char)0x00AB, (char)0x00BB }; CheckEqual(new[] { LineBreakClass.QU, LineBreakClass.QU_Pi, LineBreakClass.QU_Pf }, quotes.Select(quote => UnicodeProperties.Of(quote).LineBreakClass())); } public void DottedCircle_HasAClassOfItsOwn() { CheckEqual("DottedCircle", Describe(UnicodeProperties.Of(DottedCircle))); } public void SoftHyphen_HasAClassOfItsOwn() { CheckEqual("SoftHyphen", Describe(UnicodeProperties.Of(SoftHyphen))); } public void UnassignedPictograph_IsFlagged() { CheckTrue(UnicodeProperties.Of(0x1F02C).IsUnassignedExtendedPictographic()); } public void AssignedEmoji_IsNotFlaggedAsUnassigned() { CheckFalse(UnicodeProperties.Of(0x1F600).IsUnassignedExtendedPictographic()); } public void LastCodePoint_IsLookedUp() { CheckEqual(LineBreakClass.AL, UnicodeProperties.Of(UnicodeProperties.MaxCodePoint).LineBreakClass()); } public void NegativeValue_Throws() { CheckThrow(typeof(ArgumentOutOfRangeException)); UnicodeProperties.Of(-1); } public void ValuePastTheLastCodePoint_Throws() { CheckThrow(typeof(ArgumentOutOfRangeException)); UnicodeProperties.Of(UnicodeProperties.MaxCodePoint + 1); } } public class IsDefaultIgnorableTests : TestFixture { public void EveryCodePoint_MatchesTheCharacterDatabase() { var expected = new bool[CodePointCount]; foreach ((int first, int last, string value) in ReadRanges("DerivedCoreProperties.txt")) { for (int codePoint = first; codePoint <= last; ++codePoint) { expected[codePoint] |= value == "Default_Ignorable_Code_Point"; } } var mismatches = new List(); for (int codePoint = 0; codePoint < CodePointCount; ++codePoint) { if (UnicodeProperties.IsDefaultIgnorable(codePoint) != expected[codePoint] && mismatches.Count < 10) { mismatches.Add($"U+{codePoint:X4}: expected {expected[codePoint]}"); } } CheckEqual("", string.Join("; ", mismatches)); } public void NegativeValue_IsNot() { CheckFalse(UnicodeProperties.IsDefaultIgnorable(-1)); } } public class SpaceSeparatorsTests : TestFixture { public void MatchesTheCharacterDatabase() { string[] categories = ReadProperty("DerivedGeneralCategory.txt", valueWhereUnlisted: "Cn"); CheckEqual(Enumerable.Range(0, CodePointCount).Where(codePoint => categories[codePoint] == "Zs"), UnicodeProperties.SpaceSeparators); } } public class OfDecodedTests : TestFixture { public void EveryCodePoint_MatchesOf() { int mismatches = 0; for (int codePoint = 0; codePoint <= UnicodeProperties.MaxCodePoint; ++codePoint) { if (UnicodeProperties.OfDecoded(codePoint) != UnicodeProperties.Of(codePoint)) { ++mismatches; } } CheckEqual(0, mismatches); } } } public class CharacterPropertiesExtensionsTests : TestFixture { public static readonly CharacterProperties BothFlags = CharacterProperties.EastAsian | CharacterProperties.UnassignedExtendedPictographic; public class LineBreakClassTests : TestFixture { public void IgnoresBothFlags() { CheckEqual(LineBreakClass.OP, ((CharacterProperties)LineBreakClass.OP | BothFlags).LineBreakClass()); } } public class IsEastAsianTests : TestFixture { public void ReadsOnlyItsOwnFlag() { CheckTrue(CharacterProperties.EastAsian.IsEastAsian()); CheckFalse((CharacterProperties.UnassignedExtendedPictographic | CharacterProperties.LineBreakClassMask) .IsEastAsian()); } } public class IsUnassignedExtendedPictographicTests : TestFixture { public void ReadsOnlyItsOwnFlag() { CheckTrue(CharacterProperties.UnassignedExtendedPictographic.IsUnassignedExtendedPictographic()); CheckFalse((CharacterProperties.EastAsian | CharacterProperties.LineBreakClassMask) .IsUnassignedExtendedPictographic()); } } } }