using System;
using System.Collections.Generic;
using System.Globalization;
using System.IO;
using System.Linq;
using Glass.Typesetting;
using UnitTestSharp;
namespace Glass.UnitTests.Typesetting
{
public class UnicodePropertiesTests : TestFixture
{
///
/// Where the build unpacks 3rdParty\Unicode\UCD-*.7z, next to this binary.
///
public static readonly string DataDirectory = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "Unicode");
public const int CodePointCount = UnicodeProperties.MaxCodePoint + 1;
public const int DottedCircle = 0x25CC;
public const int SoftHyphen = 0x00AD;
public static string[] ReadProperty(string fileName, string valueWhereUnlisted)
{
var returnMe = Enumerable.Repeat(valueWhereUnlisted, CodePointCount).ToArray();
foreach ((int first, int last, string value) in ReadRanges(fileName))
{
for (int codePoint = first; codePoint <= last; ++codePoint)
{
returnMe[codePoint] = value;
}
}
return returnMe;
}
public static IEnumerable<(int First, int Last, string Value)> ReadRanges(string fileName)
{
foreach (string line in File.ReadLines(Path.Combine(DataDirectory, fileName)))
{
string data = line.Split('#')[0].Trim();
if (data.Length == 0)
{
continue;
}
string[] fields = data.Split(';');
string[] range = fields[0].Trim().Split(new[] { ".." }, StringSplitOptions.None);
yield return (int.Parse(range[0], NumberStyles.HexNumber),
int.Parse(range[range.Length - 1], NumberStyles.HexNumber), fields[1].Trim());
}
}
///
/// LB1's default resolutions, plus QU split by whether it's initial or final punctuation.
///
public static LineBreakClass ResolvedClass(string lineBreak, string generalCategory)
{
switch (lineBreak)
{
case "AI":
case "SG":
case "XX":
return LineBreakClass.AL;
case "SA":
return generalCategory == "Mn" || generalCategory == "Mc" ? LineBreakClass.CM : LineBreakClass.AL;
case "CJ":
return LineBreakClass.NS;
case "QU" when generalCategory == "Pi":
return LineBreakClass.QU_Pi;
case "QU" when generalCategory == "Pf":
return LineBreakClass.QU_Pf;
default:
return (LineBreakClass)Enum.Parse(typeof(LineBreakClass), lineBreak);
}
}
public static string Describe(CharacterProperties properties)
{
return $"{properties.LineBreakClass()}" + (properties.IsEastAsian() ? " east-asian" : "")
+ (properties.IsUnassignedExtendedPictographic() ? " unassigned-pictographic" : "");
}
public class OfTests : TestFixture
{
[CpuTimeout(3000)]
public void EveryCodePoint_MatchesTheCharacterDatabase()
{
string[] lineBreaks = ReadProperty("LineBreak.txt", valueWhereUnlisted: "XX");
string[] widths = ReadProperty("EastAsianWidth.txt", valueWhereUnlisted: "N");
string[] categories = ReadProperty("DerivedGeneralCategory.txt", valueWhereUnlisted: "Cn");
var pictographic = new bool[CodePointCount];
foreach ((int first, int last, string value) in ReadRanges("emoji-data.txt"))
{
for (int codePoint = first; codePoint <= last; ++codePoint)
{
pictographic[codePoint] |= value == "Extended_Pictographic";
}
}
var mismatches = new List();
for (int codePoint = 0; codePoint < CodePointCount; ++codePoint)
{
var expected = (CharacterProperties)(codePoint == DottedCircle ? LineBreakClass.DottedCircle
: codePoint == SoftHyphen ? LineBreakClass.SoftHyphen
: ResolvedClass(lineBreaks[codePoint], categories[codePoint]));
if (widths[codePoint] == "F" || widths[codePoint] == "W" || widths[codePoint] == "H")
{
expected |= CharacterProperties.EastAsian;
}
if (pictographic[codePoint] && categories[codePoint] == "Cn")
{
expected |= CharacterProperties.UnassignedExtendedPictographic;
}
CharacterProperties actual = UnicodeProperties.Of(codePoint);
if (actual != expected && mismatches.Count < 10)
{
mismatches.Add($"U+{codePoint:X4}: expected {Describe(expected)}, was {Describe(actual)}");
}
}
CheckEqual("", string.Join("; ", mismatches));
}
public void Letter_IsAlphabeticAndNotEastAsian()
{
CheckEqual("AL", Describe(UnicodeProperties.Of('A')));
}
public void Ideograph_IsIdeographicAndEastAsian()
{
CheckEqual("ID east-asian", Describe(UnicodeProperties.Of(0x5B57)));
}
public void FullwidthParenthesis_IsEastAsian_WhereTheAsciiOneIsNot()
{
CheckEqual("OP east-asian", Describe(UnicodeProperties.Of(0xFF08)));
CheckEqual("OP", Describe(UnicodeProperties.Of('(')));
}
public void ComplexContextLetter_ResolvesToAlphabetic()
{
CheckEqual(LineBreakClass.AL, UnicodeProperties.Of(0x0E01).LineBreakClass());
}
public void ComplexContextMark_ResolvesToCombiningMark()
{
CheckEqual(LineBreakClass.CM, UnicodeProperties.Of(0x0E31).LineBreakClass());
}
public void AmbiguousCharacter_ResolvesToAlphabetic()
{
CheckEqual(LineBreakClass.AL, UnicodeProperties.Of(0x00A7).LineBreakClass());
}
public void ConditionalJapaneseStarter_ResolvesToNonstarter()
{
CheckEqual(LineBreakClass.NS, UnicodeProperties.Of(0x3041).LineBreakClass());
}
public void UnassignedCodePoint_ResolvesToAlphabetic()
{
CheckEqual(LineBreakClass.AL, UnicodeProperties.Of(0x0378).LineBreakClass());
}
public void Surrogate_ResolvesToAlphabetic()
{
CheckEqual(LineBreakClass.AL, UnicodeProperties.Of(0xD800).LineBreakClass());
}
public void QuotationMarks_SplitByInitialAndFinalPunctuation()
{
char[] quotes = { '"', (char)0x00AB, (char)0x00BB };
CheckEqual(new[] { LineBreakClass.QU, LineBreakClass.QU_Pi, LineBreakClass.QU_Pf },
quotes.Select(quote => UnicodeProperties.Of(quote).LineBreakClass()));
}
public void DottedCircle_HasAClassOfItsOwn()
{
CheckEqual("DottedCircle", Describe(UnicodeProperties.Of(DottedCircle)));
}
public void SoftHyphen_HasAClassOfItsOwn()
{
CheckEqual("SoftHyphen", Describe(UnicodeProperties.Of(SoftHyphen)));
}
public void UnassignedPictograph_IsFlagged()
{
CheckTrue(UnicodeProperties.Of(0x1F02C).IsUnassignedExtendedPictographic());
}
public void AssignedEmoji_IsNotFlaggedAsUnassigned()
{
CheckFalse(UnicodeProperties.Of(0x1F600).IsUnassignedExtendedPictographic());
}
public void LastCodePoint_IsLookedUp()
{
CheckEqual(LineBreakClass.AL, UnicodeProperties.Of(UnicodeProperties.MaxCodePoint).LineBreakClass());
}
public void NegativeValue_Throws()
{
CheckThrow(typeof(ArgumentOutOfRangeException));
UnicodeProperties.Of(-1);
}
public void ValuePastTheLastCodePoint_Throws()
{
CheckThrow(typeof(ArgumentOutOfRangeException));
UnicodeProperties.Of(UnicodeProperties.MaxCodePoint + 1);
}
}
public class IsDefaultIgnorableTests : TestFixture
{
public void EveryCodePoint_MatchesTheCharacterDatabase()
{
var expected = new bool[CodePointCount];
foreach ((int first, int last, string value) in ReadRanges("DerivedCoreProperties.txt"))
{
for (int codePoint = first; codePoint <= last; ++codePoint)
{
expected[codePoint] |= value == "Default_Ignorable_Code_Point";
}
}
var mismatches = new List();
for (int codePoint = 0; codePoint < CodePointCount; ++codePoint)
{
if (UnicodeProperties.IsDefaultIgnorable(codePoint) != expected[codePoint] && mismatches.Count < 10)
{
mismatches.Add($"U+{codePoint:X4}: expected {expected[codePoint]}");
}
}
CheckEqual("", string.Join("; ", mismatches));
}
public void NegativeValue_IsNot()
{
CheckFalse(UnicodeProperties.IsDefaultIgnorable(-1));
}
}
public class SpaceSeparatorsTests : TestFixture
{
public void MatchesTheCharacterDatabase()
{
string[] categories = ReadProperty("DerivedGeneralCategory.txt", valueWhereUnlisted: "Cn");
CheckEqual(Enumerable.Range(0, CodePointCount).Where(codePoint => categories[codePoint] == "Zs"),
UnicodeProperties.SpaceSeparators);
}
}
public class OfDecodedTests : TestFixture
{
public void EveryCodePoint_MatchesOf()
{
int mismatches = 0;
for (int codePoint = 0; codePoint <= UnicodeProperties.MaxCodePoint; ++codePoint)
{
if (UnicodeProperties.OfDecoded(codePoint) != UnicodeProperties.Of(codePoint))
{
++mismatches;
}
}
CheckEqual(0, mismatches);
}
}
}
public class CharacterPropertiesExtensionsTests : TestFixture
{
public static readonly CharacterProperties BothFlags =
CharacterProperties.EastAsian | CharacterProperties.UnassignedExtendedPictographic;
public class LineBreakClassTests : TestFixture
{
public void IgnoresBothFlags()
{
CheckEqual(LineBreakClass.OP, ((CharacterProperties)LineBreakClass.OP | BothFlags).LineBreakClass());
}
}
public class IsEastAsianTests : TestFixture
{
public void ReadsOnlyItsOwnFlag()
{
CheckTrue(CharacterProperties.EastAsian.IsEastAsian());
CheckFalse((CharacterProperties.UnassignedExtendedPictographic | CharacterProperties.LineBreakClassMask)
.IsEastAsian());
}
}
public class IsUnassignedExtendedPictographicTests : TestFixture
{
public void ReadsOnlyItsOwnFlag()
{
CheckTrue(CharacterProperties.UnassignedExtendedPictographic.IsUnassignedExtendedPictographic());
CheckFalse((CharacterProperties.EastAsian | CharacterProperties.LineBreakClassMask)
.IsUnassignedExtendedPictographic());
}
}
}
}