Switching from regex to parser

MultinetInteractive · Sep 25, 2017 · 8ee7952 · 8ee7952
1 parent af84fb5
commit 8ee7952
Show file tree

Hide file tree

Showing 3 changed files with 101 additions and 44 deletions.
diff --git a/MN.L10n/L10nBuildTask.cs b/MN.L10n/L10nBuildTask.cs
@@ -7,7 +7,6 @@
 using System.Diagnostics;
 using System.IO;
 using System.Linq;
-using System.Text.RegularExpressions;
 
 namespace MN.L10n
 {
@@ -16,15 +15,15 @@ public class L10nBuildTask : Task
 		public override bool Execute()
 		{
 			var fi = new FileInfo(BuildEngine.ProjectFileOfTaskNode);
-			
+
 			var baseDir = fi.Directory;
 			var sourceDir = Environment.CurrentDirectory;
 			Log.LogMessage(MessageImportance.High, "info l10n: L10n - beginning work: " + sourceDir);
 
 			Debugger.Break();
 			Stopwatch stw = new Stopwatch();
 			stw.Start();
-			
+
 			L10nConfig config = null;
 
 			while (!baseDir.GetFiles("*.sln").Any())
@@ -39,10 +38,10 @@ public override bool Execute()
 			{
 				config = Jil.JSON.Deserialize<L10nConfig>(File.ReadAllText(cfgFile.FullName));
 			}
-			
+
 			L10n PhraseInstance = L10n.CreateInstance(
-				new NullLanguageProvider(), 
-				new FileDataProvider(solutionDir), 
+				new NullLanguageProvider(),
+				new FileDataProvider(solutionDir),
 				new FileResolver()
 			);
 
@@ -57,9 +56,10 @@ public override bool Execute()
 				"/bin", "\\bin",
 				"/obj", "\\obj",
 				".dll", ".designer.cs",
-				"/packages", "\\packages"
+				"/packages", "\\packages",
+				".min.js", ".css"
 			};
-			
+
 			List<string> fileList = new List<string>();
 
 			if (config != null)
@@ -99,49 +99,31 @@ public override bool Execute()
 
 			var phraseRewriter = new PhrasesRewriter(PhraseInstance);
 
-			var noParam = new Regex(@"(?:MN\.)?(?:L10n\.)?(?:L10n\.)?_[sm]\(['""](.*?)['""]\)", RegexOptions.Compiled);
-			var withParam = new Regex(@"(?:MN\.)?(?:L10n\.)?(?:L10n\.)?_[sm]\(['""](.*?)['""],.*?{(.*?)}\)", RegexOptions.Compiled);
-
-			var findParam = new Regex(@"(\$[a-zA-Z_]*?\$)", RegexOptions.Compiled);
+			var parser = new L10nParser();
 
 			foreach (var file in fileList.Distinct())
 			{
 				var fileContents = File.ReadAllText(file);
-				var m = noParam.Matches(fileContents).Cast<Match>().ToList();
-				m.AddRange(withParam.Matches(fileContents).Cast<Match>().ToList());
-				if (m.Count > 0)
+				var invocations = parser.Parse(fileContents);
+
+				foreach (var _phrase in invocations)
 				{
-					foreach (Match match in m)
+					if (!PhraseInstance.Phrases.ContainsKey(_phrase))
 					{
-						var phraseText = match.Groups[1].Value.Trim();
-						var args = match.Groups[2].Value.Trim();
-
-						if (match.Groups.Count == 2)
-						{
-							// Here we check if the regex matched too much, until we get a better regex...
-							var hasParams = findParam.Matches(match.Value).Cast<Match>().ToList();
-							if (hasParams.Any())
-							{
-								continue;
-							}
-						}
-
-						if (!PhraseInstance.Phrases.ContainsKey(phraseText))
-						{
-							PhraseInstance.Phrases.Add(phraseText, new L10nPhrase() { });
-						}
-						else
-						{
-							PhraseInstance.Phrases[phraseText].Usages++;
-						}
-
-						if (phraseRewriter.unusedPhrases.Contains(phraseText))
-						{
-							phraseRewriter.unusedPhrases.Remove(phraseText);
-						}
+						PhraseInstance.Phrases.Add(_phrase, new L10nPhrase() { });
+					}
+					else
+					{
+						PhraseInstance.Phrases[_phrase].Usages++;
+					}
+
+					if (phraseRewriter.unusedPhrases.Contains(_phrase))
+					{
+						phraseRewriter.unusedPhrases.Remove(_phrase);
 					}
 				}
-				Log.LogMessage(MessageImportance.High, "info l10n: Checked phrases in: " + file + ", found " + m.Count + " phrases");
+
+				Log.LogMessage(MessageImportance.High, "info l10n: Checked phrases in: " + file + ", found " + invocations.Count + " phrases");
 			};
 
 			phraseRewriter.SavePhrasesToFile();

diff --git a/MN.L10n/L10nParser.cs b/MN.L10n/L10nParser.cs
@@ -0,0 +1,75 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+
+namespace MN.L10n
+{
+	public class L10nParser
+	{
+		public List<string> Invocations { get; set; } = new List<string>();
+
+		public List<string> Parse(string source)
+		{
+			Invocations.Clear();
+			int _pos = -1;
+			bool inToken = false;
+			StringBuilder _tokenContent = new StringBuilder();
+			char _stringContainer = '"';
+
+			for (_pos = 0; _pos < source.Length; _pos++)
+			{
+				switch (source[_pos])
+				{
+					case '_': // Possible _s/_m, peek to see
+						if (!inToken)
+						{
+							_tokenContent.Clear();
+
+							var peek = source[_pos + 1];
+							switch (peek)
+							{
+								case 's':
+								case 'm':
+									// Even more likely to be _s/_m, proceed
+									peek = source[_pos + 2];
+									if (peek == '(')
+									{
+										peek = source[_pos + 3];
+										if (peek == '"' || peek == '\'')
+										{
+											_stringContainer = peek;
+											inToken = true;
+											_pos += 4;
+										}
+									}
+									break;
+								default:
+									continue;
+							}
+						}
+						if (inToken)
+						{
+							_tokenContent.Append(source[_pos]);
+						}
+
+						break;
+					default:
+						if (inToken)
+						{
+							var tail = source[_pos - 1];
+							if (source[_pos] == _stringContainer && tail != '\\' && tail != '(')
+							{
+								Invocations.Add(_tokenContent.ToString());
+								inToken = false;
+							}
+							_tokenContent.Append(source[_pos]);
+						}
+						break;
+				}
+			}
+
+			return Invocations;
+		}
+	}
+}
diff --git a/MN.L10n/MN.L10n.csproj b/MN.L10n/MN.L10n.csproj
@@ -12,7 +12,7 @@ Translation package</Description>
     <PackageProjectUrl>https://github.com/MultinetInteractive/MN.L10n</PackageProjectUrl>
     <RepositoryType>git</RepositoryType>
     <Copyright>© 20XX MultiNet Interactive AB</Copyright>
-    <Version>1.0.97</Version>
+    <Version>1.0.100</Version>
 	<AutoIncrementPackageRevision>True</AutoIncrementPackageRevision>
   </PropertyGroup>