Delete checked-in Unicode data files in favor of runtime-assets package (dotnet#1743)

IDNA files (in System.Globalization.Extensions.Tests) not yet deleted; will tackle this later

Author: GrabYourPitchforks

src/libraries/Common/tests/CoreFx.Private.TestUtilities.Unicode/System/Text/Unicode/UnicodeData.cs

@@ -2,6 +2,7 @@
 // The .NET Foundation licenses this file to you under the MIT license.
 // See the LICENSE file in the project root for more information.
 
+using System.Globalization;
 using System.Threading;
 
 namespace System.Text.Unicode
@@ -13,7 +14,7 @@ public static class UnicodeData
 
         public static CodePoint GetData(int codePoint)
         {
-            if ((uint)codePoint >= _codePointData.Length)
+            if ((uint)codePoint >= (uint)_codePointData.Length)
             {
                 throw new ArgumentOutOfRangeException(
                     message: FormattableString.Invariant($"Value U+{(uint)codePoint:X4} is not a valid code point."),
@@ -36,5 +37,29 @@ public static CodePoint GetData(int codePoint)
         public static CodePoint GetData(uint codePoint) => GetData((int)codePoint);
 
         public static CodePoint GetData(Rune rune) => GetData(rune.Value);
+
+        /*
+         * Helper methods
+         */
+
+        public static UnicodeCategory GetUnicodeCategory(int codePoint) => GetData(codePoint).GeneralCategory;
+
+        public static bool IsLetter(int codePoint)
+        {
+            switch (GetUnicodeCategory(codePoint))
+            {
+                case UnicodeCategory.UppercaseLetter:
+                case UnicodeCategory.LowercaseLetter:
+                case UnicodeCategory.TitlecaseLetter:
+                case UnicodeCategory.ModifierLetter:
+                case UnicodeCategory.OtherLetter:
+                    return true;
+
+                default:
+                    return false;
+            }
+        }
+
+        public static bool IsWhiteSpace(int codePoint) => (GetData(codePoint).Flags & CodePointFlags.White_Space) != 0;
     }
 }

src/libraries/Common/tests/Data/CaseFolding-12.1.0.txt

@@ -4,6 +4,7 @@
     <TestRuntime>true</TestRuntime>
     <IncludeRemoteExecutor>true</IncludeRemoteExecutor>
     <Configurations>$(NetCoreAppCurrent)-Debug;$(NetCoreAppCurrent)-Release</Configurations>
+    <UnicodeUcdVersion>12.1</UnicodeUcdVersion>
   </PropertyGroup>
   <ItemGroup>
     <Compile Include="CompareInfo\CompareInfoTests.cs" />
@@ -117,13 +118,13 @@
   </ItemGroup>
   <ItemGroup>
     <PackageReference Include="System.Private.Runtime.UnicodeData" Version="$(SystemPrivateRuntimeUnicodeDataVersion)" ExcludeAssets="contentFiles" GeneratePathProperty="true" />
-    <EmbeddedResource Include="$(PkgSystem_Private_Runtime_UnicodeData)\contentFiles\any\any\12.1.0\ucd\UnicodeData.txt">
-      <Link>CharUnicodeInfo\UnicodeData.12.1.txt</Link>
-      <LogicalName>UnicodeData.12.1.txt</LogicalName>
+    <EmbeddedResource Include="$(PkgSystem_Private_Runtime_UnicodeData)\contentFiles\any\any\$(UnicodeUcdVersion).0\ucd\UnicodeData.txt">
+      <Link>CharUnicodeInfo\UnicodeData.$(UnicodeUcdVersion).txt</Link>
+      <LogicalName>UnicodeData.txt</LogicalName>
     </EmbeddedResource>
-    <EmbeddedResource Include="$(PkgSystem_Private_Runtime_UnicodeData)\contentFiles\any\any\12.1.0\ucd\auxiliary\GraphemeBreakTest.txt">
-      <Link>CharUnicodeInfo\GraphemeBreakTest-12.1.0.txt</Link>
-      <LogicalName>GraphemeBreakTest-12.1.0.txt</LogicalName>
+    <EmbeddedResource Include="$(PkgSystem_Private_Runtime_UnicodeData)\contentFiles\any\any\$(UnicodeUcdVersion).0\ucd\auxiliary\GraphemeBreakTest.txt">
+      <Link>CharUnicodeInfo\GraphemeBreakTest-$(UnicodeUcdVersion).0.txt</Link>
+      <LogicalName>GraphemeBreakTest.txt</LogicalName>
     </EmbeddedResource>
   </ItemGroup>
   <ItemGroup>

src/libraries/Common/tests/Data/PropList-12.1.0.txt

@@ -13,7 +13,7 @@ public static class CharUnicodeInfoTestData
         private static readonly Lazy<List<CharUnicodeInfoTestCase>> s_testCases = new Lazy<List<CharUnicodeInfoTestCase>>(() =>
         {
             List<CharUnicodeInfoTestCase> testCases = new List<CharUnicodeInfoTestCase>();
-            string fileName = "UnicodeData.12.1.txt";
+            string fileName = "UnicodeData.txt";
             Stream stream = typeof(CharUnicodeInfoTestData).GetTypeInfo().Assembly.GetManifestResourceStream(fileName);
             using (StreamReader reader = new StreamReader(stream))
             {

src/libraries/Common/tests/Data/UnicodeData.11.0.txt

@@ -94,7 +94,7 @@ public void ReplacementCharHasTypeOther()
 
         private static IEnumerable<(Rune[][] clusters, string line)> GetGraphemeBreakTestData()
         {
-            using Stream stream = typeof(GraphemeBreakTest).Assembly.GetManifestResourceStream("GraphemeBreakTest-12.1.0.txt");
+            using Stream stream = typeof(GraphemeBreakTest).Assembly.GetManifestResourceStream("GraphemeBreakTest.txt");
             using StreamReader reader = new StreamReader(stream);
 
             string line;

src/libraries/Common/tests/Data/UnicodeData.12.1.txt

@@ -252,7 +252,6 @@
     <Compile Include="System\Text\ASCIIUtilityTests.cs" />
     <Compile Include="System\Text\RuneTests.cs" />
     <Compile Include="System\Text\RuneTests.TestData.cs" />
-    <Compile Include="System\Text\Unicode\UnicodeData.cs" />
     <Compile Include="System\Text\Unicode\Utf16UtilityTests.ValidateChars.cs" />
     <Compile Include="System\Text\Unicode\Utf8Tests.cs" />
     <Compile Include="System\Text\Unicode\Utf8UtilityTests.ValidateBytes.cs" />
@@ -276,24 +275,13 @@
       <Link>Common\System\Runtime\Serialization\Formatters\BinaryFormatterHelpers.cs</Link>
     </Compile>
   </ItemGroup>
-  <ItemGroup>
-    <EmbeddedResource Include="$(CommonTestPath)Data\UnicodeData.12.1.txt">
-      <Link>System\Text\Unicode\UnicodeData.12.1.txt</Link>
-      <LogicalName>UnicodeData.12.1.txt</LogicalName>
-    </EmbeddedResource>
-    <EmbeddedResource Include="$(CommonTestPath)Data\CaseFolding-12.1.0.txt">
-      <Link>System\Text\Unicode\CaseFolding-12.1.0.txt</Link>
-      <LogicalName>CaseFolding-12.1.0.txt</LogicalName>
-    </EmbeddedResource>
-    <EmbeddedResource Include="$(CommonTestPath)Data\PropList-12.1.0.txt">
-      <Link>System\Text\Unicode\PropList-12.1.0.txt</Link>
-      <LogicalName>PropList-12.1.0.txt</LogicalName>
-    </EmbeddedResource>
-  </ItemGroup>
   <ItemGroup>
     <ProjectReference Include="TestLoadAssembly\TestLoadAssembly.csproj" />
     <ProjectReference Include="TestCollectibleAssembly\TestCollectibleAssembly.csproj" />
     <ProjectReference Include="TestModule\System.Reflection.TestModule.ilproj" />
     <ProjectReference Include="TestStructs\System.TestStructs.ilproj" />
   </ItemGroup>
+  <ItemGroup>
+    <ProjectReference Include="$(CommonTestPath)CoreFx.Private.TestUtilities.Unicode\CoreFx.Private.TestUtilities.Unicode.csproj" />
+  </ItemGroup>
 </Project>

src/libraries/Common/tests/Data/UnicodeData.8.0.txt

@@ -5,7 +5,7 @@
 using System.Collections.Generic;
 using System.Globalization;
 using System.Text;
-using System.Text.Unicode.Tests;
+using System.Text.Unicode;
 using Xunit;
 using Xunit.Sdk;
 
@@ -1116,7 +1116,7 @@ public static void GetUnicodeCategory_Char_AllInputs()
             // This tests calls char.GetUnicodeCategory for every possible input, ensuring that
             // the runtime agrees with the data in the core Unicode files.
 
-            for (uint i = 0; i <= char.MaxValue; i++)
+            for (int i = 0; i <= char.MaxValue; i++)
             {
                 UnicodeCategory expected;
 
@@ -1250,7 +1250,7 @@ public static void IsWhiteSpace_Char_AllInputs()
             // This tests calls char.IsWhiteSpace for every possible input, ensuring that
             // the runtime agrees with the data in the core Unicode files.
 
-            for (uint i = 0; i <= char.MaxValue; i++)
+            for (int i = 0; i <= char.MaxValue; i++)
             {
                 if (UnicodeData.IsWhiteSpace(i) != char.IsWhiteSpace((char)i))
                 {

src/libraries/Common/tests/Data/UnicodeData63.txt

@@ -4,7 +4,7 @@
 
 using System.Buffers;
 using System.Globalization;
-using System.Text.Unicode.Tests;
+using System.Text.Unicode;
 using Xunit;
 using Xunit.Sdk;
 
@@ -350,11 +350,11 @@ public static void GetUnicodeCategory_AllInputs()
 
             foreach (Rune rune in AllRunes())
             {
-                if (UnicodeData.GetUnicodeCategory((uint)rune.Value) != Rune.GetUnicodeCategory(rune))
+                if (UnicodeData.GetUnicodeCategory(rune.Value) != Rune.GetUnicodeCategory(rune))
                 {
                     // We'll build up the exception message ourselves so the dev knows what code point failed.
                     throw new AssertActualExpectedException(
-                        expected: UnicodeData.GetUnicodeCategory((uint)rune.Value),
+                        expected: UnicodeData.GetUnicodeCategory(rune.Value),
                         actual: Rune.GetUnicodeCategory(rune),
                         userMessage: FormattableString.Invariant($@"Rune.GetUnicodeCategory(U+{rune.Value:X4}) returned wrong value."));
                 }
@@ -455,7 +455,7 @@ public static void IsWhiteSpace_AllInputs()
 
             foreach (Rune rune in AllRunes())
             {
-                Assert.Equal(UnicodeData.IsWhiteSpace((uint)rune.Value), Rune.IsWhiteSpace(rune));
+                Assert.Equal(UnicodeData.IsWhiteSpace(rune.Value), Rune.IsWhiteSpace(rune));
             }
         }