Skip to content

Commit 52fe6df

Browse files
papehpapeh
committed
Filter XML Files (#34)
* If translatable_elements is specified, exclude files with no matching elements or attributes * Does not check for translatable content in the referenced elements or attributes
1 parent bf24822 commit 52fe6df

10 files changed

+382
-133
lines changed

‎Overcrowdin.sln.DotSettings

+1
Original file line numberDiff line numberDiff line change
@@ -2,4 +2,5 @@
22
<s:Boolean x:Key="/Default/UserDictionary/Words/=Crowdin/@EntryIndexedValue">True</s:Boolean>
33
<s:Boolean x:Key="/Default/UserDictionary/Words/=localizers/@EntryIndexedValue">True</s:Boolean>
44
<s:Boolean x:Key="/Default/UserDictionary/Words/=Overcrowdin/@EntryIndexedValue">True</s:Boolean>
5+
<s:Boolean x:Key="/Default/UserDictionary/Words/=precreating/@EntryIndexedValue">True</s:Boolean>
56
<s:Boolean x:Key="/Default/UserDictionary/Words/=subdirectory/@EntryIndexedValue">True</s:Boolean></wpf:ResourceDictionary>

‎OvercrowdinTests/CommandUtilitiesTests.cs

+52-98
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,9 @@
1-
using System;
1+
using System;
22
using System.Collections.Generic;
33
using System.IO;
44
using System.IO.Abstractions.TestingHelpers;
55
using System.Linq;
66
using System.Text;
7-
using System.Xml.Linq;
87
using Crowdin.Api.Typed;
98
using Microsoft.Extensions.Configuration;
109
using Newtonsoft.Json.Linq;
@@ -306,8 +305,8 @@ public void AdditionalOptionsForXmlFiles()
306305
const string trElt0 = "//string[@txt]";
307306
const string trElt1a = "/cheese/wheel";
308307
const string trElt1b = "/round[@round]";
309-
mockFileSystem.File.WriteAllText(fileName0, "<br/>");
310-
mockFileSystem.File.WriteAllText(fileName1, "<br/>");
308+
mockFileSystem.File.WriteAllText(fileName0, "<string txt='something'/>");
309+
mockFileSystem.File.WriteAllText(fileName1, "<cheese><wheel>swiss</wheel></cheese>");
311310
dynamic configJson = SetUpConfig(fileName0);
312311
var files = configJson.files;
313312
files[0].translate_content = 0;
@@ -607,86 +606,14 @@ public void BatchEmptyFilesListDoesntCrash()
607606
Assert.Empty(result.First().Files);
608607
}
609608

610-
// Tests for filtering unnecessary .resx files:
611-
612-
[Fact]
613-
public void ResxFilterExcludesEmptyValues()
614-
{
615-
var elt = new XElement("data", new XAttribute("name", "someString"), new XElement("value"));
616-
Assert.False(CommandUtilities.HasLocalizableData(elt));
617-
}
618-
619-
[Theory]
620-
[InlineData("something.Icon")]
621-
[InlineData("something.Name")]
622-
public void ResxFilterExcludesNonLocalizableStrings(string dataName)
623-
{
624-
var elt = new XElement("data", new XAttribute("name", dataName), new XElement("value", "content"));
625-
Assert.False(CommandUtilities.HasLocalizableData(elt));
626-
}
627-
628-
[Theory]
629-
[InlineData("someString")]
630-
[InlineData("something.Text")]
631-
[InlineData("something.AccessibleName")]
632-
[InlineData("something.AccessibleDescription")]
633-
public void ResxFilterIncludesLocalizableStrings(string dataName)
634-
{
635-
var elt = new XElement("data", new XAttribute("name", dataName), new XElement("value", "content"));
636-
Assert.True(CommandUtilities.HasLocalizableData(elt));
637-
}
638-
639-
[Fact]
640-
public void ResxFilterIncludesLocalizableDocuments()
641-
{
642-
var doc = XDocument.Load(new StringReader(ResxOpenTag + ResxLocalizableData + ResxNonLocalizableData + ResxCloseTag));
643-
Assert.True(CommandUtilities.HasLocalizableData(doc));
644-
}
645-
646-
/// <remarks>
647-
/// Crowdin imports whitespaces strings, but they are hidden by default and localizers are instructed not to localize spaces.
648-
/// </remarks>
649-
[Fact]
650-
public void ResxFilterExcludesLocalizableWhitespace()
651-
{
652-
var doc = XDocument.Load(new StringReader(ResxOpenTag + ResxLocalizableWhitespace + ResxCloseTag));
653-
Assert.False(CommandUtilities.HasLocalizableData(doc));
654-
}
655-
656609
[Fact]
657-
public void ResxFilterExcludesNonLocalizableDocuments()
658-
{
659-
var doc = XDocument.Load(new StringReader(ResxOpenTag + ResxEmptyLocalizableData + ResxNonLocalizableData + ResxCloseTag));
660-
Assert.False(CommandUtilities.HasLocalizableData(doc));
661-
}
662-
663-
[Theory]
664-
[InlineData(true)]
665-
[InlineData(false)]
666-
public void FilterFiltersResx(bool hasLocalizableData)
667-
{
668-
var mockFileSystem = new MockFileSystem();
669-
const string fileName = "test.resx";
670-
mockFileSystem.File.WriteAllText(fileName, ResxOpenTag + (hasLocalizableData ? ResxLocalizableData : string.Empty) + ResxCloseTag);
671-
Assert.Equal(hasLocalizableData, CommandUtilities.IsLocalizable(fileName, mockFileSystem));
672-
}
673-
674-
[Fact]
675-
public void FilterFiltersOnlyResx()
676-
{
677-
var mockFileSystem = new MockFileSystem();
678-
const string fileName = "test.xml";
679-
mockFileSystem.File.WriteAllText(fileName, ResxOpenTag + ResxCloseTag);
680-
Assert.True(CommandUtilities.IsLocalizable(fileName, mockFileSystem));
681-
}
682-
683-
[Fact]
684-
public void MatchedFilesAreFiltered()
610+
public void ResxFilesAreFiltered()
685611
{
686612
var mockFileSystem = new MockFileSystem();
687613
const string localizableFileName = "full.resx";
688-
mockFileSystem.File.WriteAllText(localizableFileName, ResxOpenTag + ResxLocalizableData + ResxCloseTag);
689-
mockFileSystem.File.WriteAllText("empty.resx", ResxOpenTag + ResxCloseTag);
614+
mockFileSystem.File.WriteAllText(localizableFileName,
615+
ResxFilterTests.ResxOpenTag + ResxFilterTests.ResxLocalizableData + ResxFilterTests.ResxCloseTag);
616+
mockFileSystem.File.WriteAllText("empty.resx", ResxFilterTests.ResxOpenTag + ResxFilterTests.ResxCloseTag);
690617
var configJson = SetUpConfig("*.resx");
691618
var fileParamsList = new List<AddFileParameters>();
692619

@@ -702,23 +629,50 @@ public void MatchedFilesAreFiltered()
702629
Assert.Equal(localizableFileName, fileParams.Files.Keys.First());
703630
}
704631

705-
private const string ResxOpenTag = @"<?xml version=""1.0"" encoding=""utf-8""?><root>";
706-
private const string ResxLocalizableData = @"
707-
<data name=""$this.AccessibleName"" xml:space=""preserve"">
708-
<value>Date matcher</value>
709-
</data>";
710-
private const string ResxLocalizableWhitespace = @"
711-
<data name=""ksSingleSpace"" xml:space=""preserve"">
712-
<value> </value>
713-
</data>";
714-
private const string ResxNonLocalizableData = @"
715-
<data name=""&gt;&gt;$this.Name"" xml:space=""preserve"">
716-
<value>SimpleDateMatchDlg</value>
717-
</data>";
718-
private const string ResxEmptyLocalizableData = @"
719-
<data name=""$this.Text"" xml:space=""preserve"">
720-
<value></value>
721-
</data>";
722-
private const string ResxCloseTag = "</root>";
632+
[Theory]
633+
[InlineData("Add")]
634+
[InlineData("Update")]
635+
public void XmlFilesAreFiltered(string operation)
636+
{
637+
var mockFileSystem = new MockFileSystem();
638+
const string fileNameNotFiltered = "notFiltered.xml";
639+
mockFileSystem.File.WriteAllText(fileNameNotFiltered, XmlFilterTests.XmlOpenTag + XmlFilterTests.XmlCloseTag);
640+
const string fileNamePassesFilter = "filterPass.xml";
641+
mockFileSystem.File.WriteAllText(fileNamePassesFilter,
642+
XmlFilterTests.XmlOpenTag + XmlFilterTests.XmlGroupCorrect + XmlFilterTests.XmlCloseTag);
643+
mockFileSystem.File.WriteAllText("filterFail.xml", XmlFilterTests.XmlOpenTag + XmlFilterTests.XmlCloseTag);
644+
dynamic configJson = SetUpConfig(fileNameNotFiltered);
645+
dynamic file = new JObject();
646+
file.source = "filter*.xml";
647+
file.translatable_elements = new JArray { XmlFilterTests.XpathToWrongAttribute, XmlFilterTests.XpathToTranslatableElements };
648+
configJson.files.Add(file);
649+
var fileParamsList = new List<FileParameters>();
650+
651+
using (var memStream = new MemoryStream(Encoding.UTF8.GetBytes(configJson.ToString())))
652+
{
653+
var config = new ConfigurationBuilder().AddNewtonsoftJsonStream(memStream).Build();
654+
switch (operation)
655+
{
656+
case "Add":
657+
var addFileParamsList = new List<AddFileParameters>();
658+
CommandUtilities.GetFilesFromConfiguration(config, mockFileSystem, addFileParamsList, new SortedSet<string>());
659+
fileParamsList.AddRange(addFileParamsList);
660+
break;
661+
case "Update":
662+
var updateFileParamsList = new List<UpdateFileParameters>();
663+
CommandUtilities.GetFilesFromConfiguration(config, mockFileSystem, updateFileParamsList, new SortedSet<string>());
664+
fileParamsList.AddRange(updateFileParamsList);
665+
break;
666+
}
667+
}
668+
669+
Assert.Equal(2, fileParamsList.Count);
670+
var fileParams = fileParamsList[0];
671+
Assert.Single(fileParams.Files);
672+
Assert.Equal(fileNameNotFiltered, fileParams.Files.Keys.First());
673+
fileParams = fileParamsList[1];
674+
Assert.Single(fileParams.Files);
675+
Assert.Equal(fileNamePassesFilter, fileParams.Files.Keys.First());
676+
}
723677
}
724678
}

‎OvercrowdinTests/ContentFilteringTests.cs

+49
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
using System.IO.Abstractions.TestingHelpers;
2+
using Overcrowdin.ContentFiltering;
3+
using Xunit;
4+
5+
namespace OvercrowdinTests
6+
{
7+
public class ContentFilteringTests
8+
{
9+
[Theory]
10+
[InlineData(true)]
11+
[InlineData(false)]
12+
public void FilterFiltersResx(bool hasLocalizableData)
13+
{
14+
var mockFileSystem = new MockFileSystem();
15+
const string fileName = "test.resx";
16+
mockFileSystem.File.WriteAllText(fileName,
17+
ResxFilterTests.ResxOpenTag +
18+
(hasLocalizableData ? ResxFilterTests.ResxLocalizableData : string.Empty) +
19+
ResxFilterTests.ResxCloseTag);
20+
Assert.Equal(hasLocalizableData, ContentFiltering.IsLocalizable(mockFileSystem, fileName));
21+
}
22+
23+
[Theory]
24+
[InlineData(true)]
25+
[InlineData(false)]
26+
public void FilterFiltersXml(bool hasLocalizableData)
27+
{
28+
var mockFileSystem = new MockFileSystem();
29+
const string fileName = "test.xml";
30+
mockFileSystem.File.WriteAllText(fileName,
31+
XmlFilterTests.XmlOpenTag +
32+
(hasLocalizableData ? XmlFilterTests.XmlGroupCorrect : string.Empty) +
33+
XmlFilterTests.XmlCloseTag);
34+
// ReSharper disable once CoVariantArrayConversion - precreating a 2D array is necessary to replicate production behavior
35+
Assert.Equal(hasLocalizableData,
36+
ContentFiltering.IsLocalizable(mockFileSystem, fileName, XmlFilterTests.XpathArrToTranslatableElements));
37+
}
38+
39+
[Theory]
40+
[InlineData("test.xml")]
41+
[InlineData("test.txt")]
42+
public void FilterPassesUnfilterableFiles(string fileName)
43+
{
44+
var mockFileSystem = new MockFileSystem();
45+
mockFileSystem.File.WriteAllText(fileName, ResxFilterTests.ResxOpenTag + ResxFilterTests.ResxCloseTag);
46+
Assert.True(ContentFiltering.IsLocalizable(mockFileSystem, fileName));
47+
}
48+
}
49+
}

‎OvercrowdinTests/ResxFilterTests.cs

+80
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,80 @@
1+
using System.IO;
2+
using System.Xml.Linq;
3+
using Overcrowdin.ContentFiltering;
4+
using Xunit;
5+
6+
namespace OvercrowdinTests
7+
{
8+
public class ResxFilterTests
9+
{
10+
[Fact]
11+
public void ResxFilterExcludesEmptyValues()
12+
{
13+
var elt = new XElement("data", new XAttribute("name", "someString"), new XElement("value"));
14+
Assert.False(ResxFilter.HasLocalizableData(elt));
15+
}
16+
17+
[Theory]
18+
[InlineData("something.Icon")]
19+
[InlineData("something.Name")]
20+
public void ResxFilterExcludesNonLocalizableStrings(string dataName)
21+
{
22+
var elt = new XElement("data", new XAttribute("name", dataName), new XElement("value", "content"));
23+
Assert.False(ResxFilter.HasLocalizableData(elt));
24+
}
25+
26+
[Theory]
27+
[InlineData("someString")]
28+
[InlineData("something.Text")]
29+
[InlineData("something.AccessibleName")]
30+
[InlineData("something.AccessibleDescription")]
31+
public void ResxFilterIncludesLocalizableStrings(string dataName)
32+
{
33+
var elt = new XElement("data", new XAttribute("name", dataName), new XElement("value", "content"));
34+
Assert.True(ResxFilter.HasLocalizableData(elt));
35+
}
36+
37+
[Fact]
38+
public void ResxFilterIncludesLocalizableDocuments()
39+
{
40+
var doc = XDocument.Load(new StringReader(ResxOpenTag + ResxLocalizableData + ResxNonLocalizableData + ResxCloseTag));
41+
Assert.True(ResxFilter.HasLocalizableData(doc));
42+
}
43+
44+
/// <remarks>
45+
/// Crowdin imports whitespaces strings, but they are hidden by default and localizers are instructed not to localize spaces.
46+
/// </remarks>
47+
[Fact]
48+
public void ResxFilterExcludesLocalizableWhitespace()
49+
{
50+
var doc = XDocument.Load(new StringReader(ResxOpenTag + ResxLocalizableWhitespace + ResxCloseTag));
51+
Assert.False(ResxFilter.HasLocalizableData(doc));
52+
}
53+
54+
[Fact]
55+
public void ResxFilterExcludesNonLocalizableDocuments()
56+
{
57+
var doc = XDocument.Load(new StringReader(ResxOpenTag + ResxEmptyLocalizableData + ResxNonLocalizableData + ResxCloseTag));
58+
Assert.False(ResxFilter.HasLocalizableData(doc));
59+
}
60+
61+
internal const string ResxOpenTag = @"<?xml version=""1.0"" encoding=""utf-8""?><root>";
62+
internal const string ResxLocalizableData = @"
63+
<data name=""$this.AccessibleName"" xml:space=""preserve"">
64+
<value>Date matcher</value>
65+
</data>";
66+
private const string ResxLocalizableWhitespace = @"
67+
<data name=""ksSingleSpace"" xml:space=""preserve"">
68+
<value> </value>
69+
</data>";
70+
private const string ResxNonLocalizableData = @"
71+
<data name=""&gt;&gt;$this.Name"" xml:space=""preserve"">
72+
<value>SimpleDateMatchDlg</value>
73+
</data>";
74+
private const string ResxEmptyLocalizableData = @"
75+
<data name=""$this.Text"" xml:space=""preserve"">
76+
<value></value>
77+
</data>";
78+
internal const string ResxCloseTag = "</root>";
79+
}
80+
}

‎OvercrowdinTests/XmlFilterTests.cs

+68
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,68 @@
1+
using System.IO;
2+
using System.Xml.Linq;
3+
using Overcrowdin.ContentFiltering;
4+
using Xunit;
5+
6+
namespace OvercrowdinTests
7+
{
8+
public class XmlFilterTests
9+
{
10+
[Fact]
11+
public static void TranslatableElementsUnspecified_Passes()
12+
{
13+
var doc = XDocument.Load(new StringReader(XmlOpenTag + XmlCloseTag));
14+
Assert.True(XmlFilter.HasTranslatableItems(doc));
15+
}
16+
17+
[Theory]
18+
[InlineData(true)]
19+
[InlineData(false)]
20+
public static void TranslatableElementsSpecified_Filters(bool hasTranslatableItems) {
21+
var doc = XDocument.Load(new StringReader(XmlOpenTag + (hasTranslatableItems ? XmlGroupCorrect : string.Empty) + XmlCloseTag));
22+
// ReSharper disable once CoVariantArrayConversion - precreating a 2D array is necessary to replicate production behavior
23+
Assert.Equal(hasTranslatableItems, XmlFilter.HasTranslatableItems(doc, XpathArrToTranslatableElements));
24+
}
25+
26+
[Theory]
27+
[InlineData("Wrong Attribute", XmlGroupWithWrongAttribute)]
28+
[InlineData("Wrong Element", XmlGroupWithWrongElement)]
29+
[InlineData("No Attribute", XmlGroupWithoutAttribute)]
30+
public static void ExcludesWrongItems(string message, string xmlGroup)
31+
{
32+
var doc = XDocument.Load(new StringReader(XmlOpenTag + xmlGroup + XmlCloseTag));
33+
// ReSharper disable once CoVariantArrayConversion - precreating a 2D array is necessary to replicate production behavior
34+
Assert.False(XmlFilter.HasTranslatableItems(doc, XpathArrToTranslatableElements), message);
35+
}
36+
37+
[Fact]
38+
public static void MultipleXpaths_MatchesOnlySecond_Passes()
39+
{
40+
var doc = XDocument.Load(new StringReader(XmlOpenTag + XmlGroupWithWrongAttribute + XmlCloseTag));
41+
var xPaths = new[] { new[] { XpathToTranslatableElements, XpathToWrongAttribute } };
42+
// ReSharper disable once CoVariantArrayConversion - precreating a 2D array is necessary to replicate production behavior
43+
Assert.True(XmlFilter.HasTranslatableItems(doc, xPaths));
44+
}
45+
46+
internal static readonly string[][] XpathArrToTranslatableElements = {new[] {XpathToTranslatableElements}};
47+
internal const string XpathToTranslatableElements = "/strings//group/string[@txt]";
48+
internal const string XpathToWrongAttribute = "/strings//group/string[@wrong]";
49+
internal const string XmlOpenTag = @"<?xml version='1.0' encoding='UTF-8'?><strings>";
50+
internal const string XmlGroupCorrect = @"
51+
<group id='TopGroup1'>
52+
<string txt='No Records'/>
53+
</group>";
54+
internal const string XmlGroupWithWrongElement = @"
55+
<group id='TopGroupWrongElt'>
56+
<wrong txt='No Records'/>
57+
</group>";
58+
internal const string XmlGroupWithWrongAttribute = @"
59+
<group id='TopGroupWrongAtt'>
60+
<string wrong='No Records'/>
61+
</group>";
62+
internal const string XmlGroupWithoutAttribute = @"
63+
<group id='TopGroupWithoutAtt'>
64+
<string>just for giggles</string>
65+
</group>";
66+
internal const string XmlCloseTag = @"</strings>";
67+
}
68+
}

0 commit comments

Comments
 (0)