using System; using System.Collections.Generic; using System.IO; using System.Linq; using Glass.Typesetting; using UnitTestSharp; using Conformance = Glass.UnitTests.TextLayoutTests.WordWrapUax14Conformance; namespace Glass.UnitTests.Typesetting { public class LineBreakIteratorTests : TestFixture { public static List BreaksIn(LineBreakIterator iterator) { var returnMe = new List(); foreach (LineBreak lineBreak in iterator) { returnMe.Add(lineBreak); } return returnMe; } public static List BreaksIn(StringView text) => BreaksIn(new LineBreakIterator(text)); public static int[] PositionsIn(StringView text) => BreaksIn(text).Select(each => each.Position).ToArray(); public static string CodePointsOf(string text) { var codePoints = new List(); for (int i = 0; i < text.Length; i += char.IsSurrogatePair(text, i) ? 2 : 1) { codePoints.Add(char.ConvertToUtf32(text, i).ToString("X4")); } return string.Join(" ", codePoints); } public static string Describe(IEnumerable breaks) { return string.Join(",", breaks.Select(each => each.Position + (each.IsMandatory ? "!" : ""))); } /// /// Where `iteratorFor` disagrees with the Unicode Consortium's LineBreakTest.txt, or "" where it doesn't. /// public static string DisagreementsWithLineBreakTest(System.Func iteratorFor) { int mismatchCount = 0; var firstMismatches = new List(); foreach (string line in File.ReadLines(Conformance.TestFilePath)) { if (line.Length == 0 || line[0] == '#') { continue; } Conformance.Case testCase = Conformance.Parse(line); var expected = new HashSet(testCase.Boundaries.Where(each => each.Breaks) .Select(each => each.Index)) { testCase.Text.Length }; var actual = new HashSet(BreaksIn(iteratorFor(testCase.Text)).Select(each => each.Position)); if (expected.SetEquals(actual)) { continue; } ++mismatchCount; if (firstMismatches.Count < 10) { firstMismatches.Add($"[{CodePointsOf(testCase.Text)}] expected breaks at " + $"{string.Join(",", expected.OrderBy(each => each))} but got " + $"{string.Join(",", actual.OrderBy(each => each))}"); } } return mismatchCount == 0 ? "" : $"{mismatchCount} cases differ, including: " + string.Join("; ", firstMismatches); } /// /// One code point for each combination of line break class and flags. /// public static readonly int[] Representatives = FindRepresentatives(); public static int[] FindRepresentatives() { var firstByProperties = new Dictionary(); for (int codePoint = 0; codePoint < 0x30000; ++codePoint) { // Surrogate code points can't be put in a string alone without pairing up by accident. if (codePoint >= 0xD800 && codePoint <= 0xDFFF) { continue; } byte packed = (byte)UnicodeProperties.Of(codePoint); if (!firstByProperties.ContainsKey(packed)) { firstByProperties[packed] = codePoint; } } return firstByProperties.Values.ToArray(); } public class MoveNextTests : TestFixture { [CpuTimeout(3000)] public void AgreesWithLineBreakTest_AtEveryBoundary() { CheckEqual("", DisagreementsWithLineBreakTest(text => new LineBreakIterator(text))); } public void EmptyText_HasNoBreaks() { CheckEqual(new LineBreak[0], BreaksIn("")); } public void EndOfText_IsAMandatoryBreak() { CheckEqual(new[] { new LineBreak(1, isMandatory: true) }, BreaksIn("a")); } public void AfterASpace_IsAnOptionalBreak() { CheckEqual(new[] { new LineBreak(2, isMandatory: false, trailingSpaces: 1), new LineBreak(3, isMandatory: true), }, BreaksIn("a b")); } public void AfterALineFeed_IsAMandatoryBreak() { CheckEqual(new[] { new LineBreak(2, isMandatory: true, ending: LineBreakEnding.HardBreak), new LineBreak(3, isMandatory: true), }, BreaksIn("a\nb")); } public void CarriageReturnLineFeed_BreaksOnceAfterBoth() { CheckEqual(new[] { 3, 4 }, PositionsIn("a\r\nb")); } public void TrailingLineFeed_BreaksOnceAtTheEnd() { CheckEqual(new[] { new LineBreak(2, isMandatory: true, ending: LineBreakEnding.HardBreak) }, BreaksIn("a\n")); } public void SurrogatePair_IsNeverSplit() { string grinningFace = char.ConvertFromUtf32(0x1F600); CheckEqual(new[] { 2, 4 }, PositionsIn(grinningFace + grinningFace)); } public void LoneSurrogate_ActsAsAlphabetic() { CheckEqual(new[] { 3 }, PositionsIn("a" + (char)0xD800 + "b")); } public void View_PositionsAreRelativeToItsStart() { CheckEqual(new[] { 2, 3 }, PositionsIn(new StringView("xx a b", 3, 6))); } public void View_LooksAheadNoFurtherThanItsEnd() { // With the 5 in view, LB15c would allow a break before the decimal mark. CheckEqual(new[] { 3 }, PositionsIn(new StringView("a .5", 0, 3))); } public void View_LooksBackNoFurtherThanItsStart() { // With the 1 in view, LB25 would keep the percent sign with the bracket. CheckEqual(new[] { 1, 2 }, PositionsIn(new StringView("1)%", 1, 3))); } public void AffixAfterABracketedNumber_StaysWithIt() { CheckEqual(new[] { 4 }, PositionsIn("(1)%")); } public void AffixAfterAMarkedSeparator_StaysWithTheNumber() { CheckEqual(new[] { 4 }, PositionsIn("1," + (char)0x0301 + "%")); } public void DigitAfterASlash_BreaksUnlessTheSlashFollowsADigit() { CheckEqual(new[] { 2, 3 }, PositionsIn("a/2")); CheckEqual(new[] { 3 }, PositionsIn("1/2")); } public void LettersAndDigitsInterleaved_NeverBreak() { CheckEqual(new[] { 9, 10 }, PositionsIn("ab12cd34 x")); } public void LetterAfterNumberPunctuation_CanBreak() { CheckEqual(new[] { 2, 3 }, PositionsIn("1-a")); } public void NumberRunEndingTheTextInPunctuation_BreaksOnlyAtTheEnd() { CheckEqual(new[] { 5 }, PositionsIn("ab12-")); } public void HyphenEndingANumberAfterASpace_StillBreaksBeforeALetter() { CheckEqual(new[] { 2, 4, 5 }, PositionsIn("x 1-a")); CheckEqual(new[] { 2, 5, 6 }, PositionsIn("x a1-b")); } public void MarkedZeroWidthSpace_IsNotLookedPastSpacesForAsOne() { // The mark is its own item (LB10), so LB8 doesn't reach past the space and LB15b holds. CheckEqual(new[] { 1, 4 }, PositionsIn("" + (char)0x200B + (char)0x0301 + " " + (char)0x00BB)); } public void OpeningQuotationBeforeSpaces_KeepsWhatFollows() { CheckEqual(new[] { 3 }, PositionsIn((char)0x00AB + " x")); } public void QuotationThatDoesNotOpen_LetsSpacesBreak() { CheckEqual(new[] { 3, 4 }, PositionsIn("a" + (char)0x00AB + " x")); } public void OpenBracketWithAJoinerBeforeSpaces_KeepsWhatFollows() { CheckEqual(new[] { 4 }, PositionsIn("(" + (char)0x200D + " x")); } public void SpacesAtTheStart_BreakAfterThem() { CheckEqual(new[] { 2, 3 }, PositionsIn(" x")); } public void RunOfLetters_KeepsAMarkAfterIt() { CheckEqual(new[] { 4 }, PositionsIn("ab" + (char)0x0301 + "c")); } public void RunOfDigits_StaysWithLettersAndAffixesAfterIt() { CheckEqual(new[] { 4 }, PositionsIn("123%")); CheckEqual(new[] { 4 }, PositionsIn("12ab")); } public void NumberWithSeparators_NeverBreaksInside() { CheckEqual(new[] { 16 }, PositionsIn("12,450.50/3:15;7")); CheckEqual(new[] { 4 }, PositionsIn("1--2")); CheckEqual(new[] { 4 }, PositionsIn("1/-2")); } public void NumberEndingInASeparator_LeavesItToWhatFollows() { CheckEqual(new[] { 2, 3 }, PositionsIn("3-a")); CheckEqual(new[] { 2, 3 }, PositionsIn("3/a")); CheckEqual(new[] { 3 }, PositionsIn("3,a")); CheckEqual(new[] { 4 }, PositionsIn("1,2%")); } public void NumberFollowedByASpace_BreaksAfterIt() { CheckEqual(new[] { 4, 5 }, PositionsIn("12. x")); CheckEqual(new[] { 4, 5 }, PositionsIn("1:2 3")); } public void NumberStartingWithANonAsciiDigit_NeverBreaksInside() { CheckEqual(new[] { 3 }, PositionsIn((char)0x0661 + ",2")); } public void View_EndsARunAtItsEnd() { CheckEqual(new[] { 3 }, PositionsIn(new StringView("abcdef", 0, 3))); } public void EmptyView_HasNoBreaks() { CheckEqual(new int[0], PositionsIn(new StringView("abc", 1, 1))); } public void LeadingLetterOrDigit_KeepsAMarkAfterIt() { CheckEqual(new[] { 3 }, PositionsIn("a" + (char)0x0301 + "c")); CheckEqual(new[] { 3 }, PositionsIn("1" + (char)0x0301 + "%")); } public void LeadingHyphen_KeepsTheWordAfterIt() { CheckEqual(new[] { 2 }, PositionsIn("-c")); } public void HyphenAfterARunOfLetters_LetsTheNextWordBreakAway() { // LB20a would keep the c if the item before the hyphen were a space rather than a letter. CheckEqual(new[] { 3, 4 }, PositionsIn("ab-c")); CheckEqual(new[] { 1, 3 }, PositionsIn(" -c")); } public static readonly string RegionalIndicator = char.ConvertFromUtf32(0x1F1E6); public void RegionalIndicators_BreakBetweenPairs() { CheckEqual(new[] { 4, 8 }, PositionsIn(string.Concat(Enumerable.Repeat(RegionalIndicator, 4)))); } public void RegionalIndicators_KeepPairingThroughALongRun() { CheckEqual(new[] { 4, 8, 12, 16, 18 }, PositionsIn(string.Concat(Enumerable.Repeat(RegionalIndicator, 9)))); } public void RegionalIndicators_PairAcrossAJoinerTheyAbsorb() { string joined = RegionalIndicator + (char)0x200D + RegionalIndicator; CheckEqual(new[] { 5, 7 }, PositionsIn(joined + RegionalIndicator)); } public void RegionalIndicators_PairAfreshAfterSomethingElse() { string pair = RegionalIndicator + RegionalIndicator; CheckEqual(new[] { 4, 5, 9 }, PositionsIn(pair + "a" + pair)); } public const string Sentence = "The quick (brown) fox—jumps “over” 12.5%."; [AllocationTest] public void AllocatesNothing() { int count = 0; foreach (LineBreak lineBreak in new LineBreakIterator(Sentence)) { ++count; } CheckNoAllocations(); CheckEqual(PositionsIn(Sentence).Length, count); } } public static bool IsHardBreak(char character) => UnicodeProperties.Of(character).LineBreakClass().Is(LineBreakClassSets.HardBreak); public static bool IsSpace(char character) => UnicodeProperties.Of(character).LineBreakClass() == LineBreakClass.SP; /// /// Where the spaces and the hard break before a break at `position` begin, worked out from each character's /// class rather than the way the iterator does. /// public static (int ContentEnd, int SpacesEnd) ExpectedTrailing(string text, int position) { int spacesEnd = position; if (spacesEnd > 0 && IsHardBreak(text[spacesEnd - 1])) { --spacesEnd; if (text[spacesEnd] == '\n' && spacesEnd > 0 && text[spacesEnd - 1] == '\r') { --spacesEnd; } } int contentEnd = spacesEnd; while (contentEnd > 0 && IsSpace(text[contentEnd - 1])) { --contentEnd; } return (contentEnd, spacesEnd); } /// /// Where the iterator's trailing positions disagree with on LineBreakTest.txt. /// public static string TrailingDisagreementsWithLineBreakTest() { var firstMismatches = new List(); foreach (string line in File.ReadLines(Conformance.TestFilePath)) { if (line.Length == 0 || line[0] == '#') { continue; } string text = Conformance.Parse(line).Text; var iterator = new LineBreakIterator(text); int previousEnd = 0; while (iterator.MoveNext()) { (int contentEnd, int spacesEnd) = ExpectedTrailing(text, iterator.Current.Position); LineBreak lineBreak = iterator.Current; bool staysInTheWord = lineBreak.ContentEnd >= previousEnd; if ((lineBreak.ContentEnd != contentEnd || lineBreak.SpacesEnd != spacesEnd || !staysInTheWord) && firstMismatches.Count < 10) { firstMismatches.Add($"[{CodePointsOf(text)}] at {lineBreak.Position}: " + $"{lineBreak.ContentEnd}/{lineBreak.SpacesEnd}, expected {contentEnd}/{spacesEnd}"); } previousEnd = iterator.Current.Position; } } return string.Join("; ", firstMismatches); } /// /// Each break's position, then where its trailing spaces and hard break begin. /// public static string DescribeTrailing(StringView text) { var descriptions = new List(); var iterator = new LineBreakIterator(text); while (iterator.MoveNext()) { LineBreak lineBreak = iterator.Current; descriptions.Add($"{lineBreak.Position}:{lineBreak.ContentEnd}/{lineBreak.SpacesEnd}"); } return string.Join(" ", descriptions); } // Each break's word shape, as MoveNext reports it: content end, spaces end and ending. public class MoveNextTrailingTests : TestFixture { [CpuTimeout(3000)] public void OnLineBreakTest_ContentAndSpacesEndWhereTheCharacterClassesSay() { CheckEqual("", TrailingDisagreementsWithLineBreakTest()); } public void Spaces_AreBeforeTheBreak() { CheckEqual("4:2/4 6:6/6", DescribeTrailing("ab cd")); } public void NoSpaces_IsTheBreaksPosition() { CheckEqual("2:2/2 3:3/3", DescribeTrailing("a-b")); } public void SpacesInsideAWord_AreContent() { CheckEqual("5:4/5 6:6/6", DescribeTrailing("( ab b")); } public void WordEndingInContent_AfterSpacesInsideIt_HasNoTrailingSpaces() { CheckEqual("4:4/4", DescribeTrailing("( ab")); } public void SpacesEndingTheText_AreBeforeTheLastBreak() { CheckEqual("4:2/4", DescribeTrailing("ab ")); } public void SpacesBeforeTheView_AreNotCounted() { CheckEqual("1:0/0 3:3/3", DescribeTrailing(((StringView)" \nab")[1, 4])); } public void View_PositionsAreRelativeToItsStart() { CheckEqual("4:2/4 6:6/6", DescribeTrailing(((StringView)"xxab cd")[2, 8])); } public void SpacesAfterAHardBreak_AreAWordWithNoContent() { CheckEqual("1:0/0 4:1/3 5:5/5", DescribeTrailing("\n \nx")); } public void LineFeed_IsAfterTheSpaces() { CheckEqual("4:2/3 6:6/6", DescribeTrailing("ab \ncd")); } public void CarriageReturnLineFeed_IsOneHardBreak() { CheckEqual("4:2/2 6:6/6", DescribeTrailing("ab\r\ncd")); } public void CarriageReturnLineFeedEndingTheText_IsOneHardBreak() { CheckEqual("4:2/2", DescribeTrailing("ab\r\n")); } public void JoinerAfterASoftHyphenEndingTheText_HidesTheSoftHyphen_AndEndsPlain() { const char zeroWidthJoiner = (char)0x200D; CheckEqual(LineBreakEnding.Plain, BreaksIn($"a{LineBreakIterator.SoftHyphen}{zeroWidthJoiner}").Last().Ending); } public void LoneCarriageReturn_IsOneCharacter() { CheckEqual("3:2/2 5:5/5", DescribeTrailing("ab\rcd")); } public void CarriageReturnBeforeTheView_IsNotPartOfTheHardBreak() { CheckEqual("1:0/0 3:3/3", DescribeTrailing(((StringView)"\r\nab")[1, 4])); } public void Endings_NameWhatEndsEachWord() { LineBreakEnding[] endings = BreaksIn($"a\r\nb\nc{LineBreakIterator.SoftHyphen}d") .Select(each => each.Ending).ToArray(); CheckEqual(new[] { LineBreakEnding.CarriageReturnLineFeed, LineBreakEnding.HardBreak, LineBreakEnding.SoftHyphen, LineBreakEnding.Plain, }, endings); } } public class TextTests : TestFixture { public void IsTheViewItWasMadeOn() { const string whole = "xxab cd"; StringView view = new LineBreakIterator(((StringView)whole)[2, 6]).Text; CheckEqual(new object[] { whole, 2, 6 }, new object[] { view.Text, view.StartIndex, view.EndIndex }); } public void StaysTheSameAsBreaksAreRead() { var iterator = new LineBreakIterator("ab cd"); iterator.MoveNext(); CheckEqual("ab cd", iterator.Text.ToString()); } } public class GetEnumeratorTests : TestFixture { public void EachForeach_StartsFromTheBeginning() { var iterator = new LineBreakIterator("a b"); var first = new List(); var second = new List(); foreach (LineBreak lineBreak in iterator) { first.Add(lineBreak.Position); } foreach (LineBreak lineBreak in iterator) { second.Add(lineBreak.Position); } CheckEqual(new[] { 2, 3 }, first); CheckEqual(first, second); } } public class ByRulesAloneTests : TestFixture { [CpuTimeout(3000)] public void AgreesWithLineBreakTest_AtEveryBoundary() { CheckEqual("", DisagreementsWithLineBreakTest(text => LineBreakIterator.ByRulesAlone(text))); } [CpuTimeout(3000)] public void AgreesWithThePairTable_OnRandomText() { var random = new System.Random(1); var firstMismatches = new List(); for (int trial = 0; trial < 50000 && firstMismatches.Count < 5; ++trial) { var text = new System.Text.StringBuilder(); int length = 1 + random.Next(8); for (int i = 0; i < length; ++i) { text.Append(char.ConvertFromUtf32(Representatives[random.Next(Representatives.Length)])); } string candidate = text.ToString(); string byTable = Describe(BreaksIn(candidate)); string byRules = Describe(BreaksIn(LineBreakIterator.ByRulesAlone(candidate))); if (byTable != byRules) { firstMismatches.Add($"[{CodePointsOf(candidate)}] table {byTable}, rules {byRules}"); } } CheckEqual("", string.Join("; ", firstMismatches)); } } public class DecideByClassesAloneTests : TestFixture { public void AfterAHardBreak_IsMandatory() { CheckEqual(LineBreakBoundary.Mandatory, LineBreakIterator.DecideByClassesAlone(LineBreakClass.BK, LineBreakClass.AL)); } public void ClosingBeforeANonstarter_IsProhibited() { CheckEqual(LineBreakBoundary.Prohibited, LineBreakIterator.DecideByClassesAlone(LineBreakClass.CL, LineBreakClass.NS)); } public void NoRuleForbidding_IsAllowed() { CheckEqual(LineBreakBoundary.Allowed, LineBreakIterator.DecideByClassesAlone(LineBreakClass.AL, LineBreakClass.ID)); } } } }