Limited regular expressions

src/libraries/Microsoft.PowerFx.Core/Binding/BinderUtils.cs

@@ -2,7 +2,8 @@
 // Licensed under the MIT license.
 
 using System.Collections.Generic;
-using System.Linq;
+using System.Globalization;
+using System.Linq;
 using Microsoft.CodeAnalysis;
 using Microsoft.PowerFx.Core.App.Controls;
 using Microsoft.PowerFx.Core.App.ErrorContainers;
@@ -14,6 +15,7 @@
 using Microsoft.PowerFx.Core.Localization;
 using Microsoft.PowerFx.Core.Logging.Trackers;
 using Microsoft.PowerFx.Core.Texl;
+using Microsoft.PowerFx.Core.Texl.Builtins;
 using Microsoft.PowerFx.Core.Types;
 using Microsoft.PowerFx.Core.Types.Enums;
 using Microsoft.PowerFx.Core.Utils;
@@ -1502,10 +1504,32 @@ public static bool TryGetConstantValue(CheckTypesContext context, TexlNode node,
             Contracts.AssertValue(node);
             nodeValue = null;
             switch (node.Kind)
-            {
+            {
                 case NodeKind.StrLit:
                     nodeValue = node.AsStrLit().Value;
-                    return true;
+                    return true;
+                case NodeKind.StrInterp:
+                    var strInterpNode = node.AsStrInterp();
+                    var segments = new List<string>();
+                    foreach (var segmentNode in strInterpNode.Children)
+                    {
+                        if (TryGetConstantValue(context, segmentNode, out var segmentValue))
+                        {
+                            segments.Append(segmentValue);
+                        }
+                        else
+                        {
+                            break;
+                        }
+                    }
+
+                    if (segments.Count == strInterpNode.Children.Count)
+                    {
+                        nodeValue = string.Join(string.Empty, segments);
+                        return true;
+                    }
+
+                    break;
                 case NodeKind.BinaryOp:
                     var binaryOpNode = node.AsBinaryOp();
                     if (binaryOpNode.Op == BinaryOp.Concat)
@@ -1540,7 +1564,43 @@ public static bool TryGetConstantValue(CheckTypesContext context, TexlNode node,
                             nodeValue = string.Join(string.Empty, parameters);
                             return true;
                         }
-                    }
+                    }
+                    else if ((callNode.Head.Name.Value == BuiltinFunctionsCore.Char.Name || callNode.Head.Name.Value == BuiltinFunctionsCore.UniChar.Name) && callNode.Args.Children.Count == 1)
+                    {
+                        int val = -1;
+
+                        if (callNode.Args.Children[0].Kind == NodeKind.DecLit)
+                        {
+                            val = (int)((DecLitNode)callNode.Args.Children[0]).ActualDecValue;
+                        }
+                        else if (callNode.Args.Children[0].Kind == NodeKind.NumLit)
+                        {
+                            val = (int)((NumLitNode)callNode.Args.Children[0]).ActualNumValue;
+                        }
+                        else if (callNode.Args.Children[0].Kind == NodeKind.Call)
+                        {
+                            var hexCallNode = callNode.Args.Children[0].AsCall();
+                            if (hexCallNode.Head.Name.Value == BuiltinFunctionsCore.Hex2Dec.Name && hexCallNode.Args.Children[0].Kind == NodeKind.StrLit)
+                            {
+                                var hexStr = hexCallNode.Args.Children[0].AsStrLit().Value;
+
+                                // check for 10 hex digits is the same as in LibraryMath.cs/Hex2Dec, Excel functions works on a 40 bit number
+                                // may result in a negative val, but that will be checked below
+                                if (hexStr.Length > 10 || !int.TryParse(hexStr, System.Globalization.NumberStyles.HexNumber, null, out val))
+                                {
+                                    return false;
+                                }
+                            }
+                        }
+
+                        if (val < 1 || (val > 255 && callNode.Head.Name.Value == BuiltinFunctionsCore.Char.Name) || val > 0x10FFFF || (val >= 0xD800 && val <= 0xDFFF))
+                        {
+                            return false;
+                        }
+
+                        nodeValue = char.ConvertFromUtf32(val);
+                        return true;
+                    }
 
                     break;
                 case NodeKind.FirstName:

src/libraries/Microsoft.PowerFx.Core/Localization/Strings.cs

@@ -743,10 +743,62 @@ internal static class TexlStrings
         public static ErrorResourceKey ErrDecimalRequiresPowerFxV1 = new ErrorResourceKey("ErrDecimalNeedsPowerFxV1");
 
         public static ErrorResourceKey ErrInvalidRegEx = new ErrorResourceKey("ErrInvalidRegEx");
+        public static ErrorResourceKey ErrInvalidRegExWithContext = new ErrorResourceKey("ErrInvalidRegExWithContext");
+        public static ErrorResourceKey ErrInvalidRegExBadInlineOptions = new ErrorResourceKey("ErrInvalidRegExBadInlineOptions");
+        public static ErrorResourceKey ErrInvalidRegExInlineOptionNotAtStart = new ErrorResourceKey("ErrInvalidRegExInlineOptionNotAtStart");
+        public static ErrorResourceKey ErrInvalidRegExBadOctal = new ErrorResourceKey("ErrInvalidRegExBadOctal");
+        public static ErrorResourceKey ErrInvalidRegExBadCurly = new ErrorResourceKey("ErrInvalidRegExBadCurly");
+        public static ErrorResourceKey ErrInvalidRegExBadSquare = new ErrorResourceKey("ErrInvalidRegExBadSquare");
+        public static ErrorResourceKey ErrInvalidRegExUnterminatedSquare = new ErrorResourceKey("ErrInvalidRegExUnterminatedSquare");
+        public static ErrorResourceKey ErrInvalidRegExBadParen = new ErrorResourceKey("ErrInvalidRegExBadParen");
+        public static ErrorResourceKey ErrInvalidRegExBadEscapeInsideCharacterClass = new ErrorResourceKey("ErrInvalidRegExBadEscapeInsideCharacterClass");
+        public static ErrorResourceKey ErrInvalidRegExBadEscapeInsideNegativeCharacterClass = new ErrorResourceKey("ErrInvalidRegExBadEscapeInsideNegativeCharacterClass");
+        public static ErrorResourceKey ErrInvalidRegExBadEscapeOutsideCharacterClass = new ErrorResourceKey("ErrInvalidRegExBadEscapeOutsideCharacterClass");
+        public static ErrorResourceKey ErrInvalidRegExRepeatInCharClass = new ErrorResourceKey("ErrInvalidRegExRepeatInCharClass");
+        public static ErrorResourceKey ErrInvalidRegExRepeatedInlineOption = new ErrorResourceKey("ErrInvalidRegExRepeatedInlineOption");
+        public static ErrorResourceKey ErrInvalidRegExInlineOptionConflictsWithNumberedSubMatches = new ErrorResourceKey("ErrInvalidRegExInlineOptionConflictsWithNumberedSubMatches");
+        public static ErrorResourceKey ErrInvalidRegExBadQuantifier = new ErrorResourceKey("ErrInvalidRegExBadQuantifier");
+        public static ErrorResourceKey ErrInvalidRegExBadExactQuantifier = new ErrorResourceKey("ErrInvalidRegExBadExactQuantifier");
+        public static ErrorResourceKey ErrInvalidRegExBadBackRefSelfReferencing = new ErrorResourceKey("ErrInvalidRegExBadBackRefSelfReferencing");
+        public static ErrorResourceKey ErrInvalidRegExBadBackRefNotDefined = new ErrorResourceKey("ErrInvalidRegExBadBackRefNotDefined");
+        public static ErrorResourceKey ErrInvalidRegExBadBalancing = new ErrorResourceKey("ErrInvalidRegExBadBalancing");
+        public static ErrorResourceKey ErrInvalidRegExBadSingleQuoteNamedCapture = new ErrorResourceKey("ErrInvalidRegExBadSingleQuoteNamedCapture");
+        public static ErrorResourceKey ErrInvalidRegExBadEscape = new ErrorResourceKey("ErrInvalidRegExBadEscape");
+        public static ErrorResourceKey ErrInvalidRegExBadConditional = new ErrorResourceKey("ErrInvalidRegExBadConditional");
+        public static ErrorResourceKey ErrInvalidRegExBadNamedCaptureAlreadyExists = new ErrorResourceKey("ErrInvalidRegExBadNamedCaptureAlreadyExists");
+        public static ErrorResourceKey ErrInvalidRegExBadNamedCaptureName = new ErrorResourceKey("ErrInvalidRegExBadNamedCaptureName");
+        public static ErrorResourceKey ErrInvalidRegExUnclosedCaptureGroups = new ErrorResourceKey("ErrInvalidRegExUnclosedCaptureGroups");
+        public static ErrorResourceKey ErrInvalidRegExUnclosedInlineComment = new ErrorResourceKey("ErrInvalidRegExUnclosedInlineComment");
+        public static ErrorResourceKey ErrInvalidRegExUnclosedInlineCommentWithContext = new ErrorResourceKey("ErrInvalidRegExUnclosedInlineCommentWithContext");
+        public static ErrorResourceKey ErrInvalidRegExUnopenedCaptureGroups = new ErrorResourceKey("ErrInvalidRegExUnopenedCaptureGroups");
+        public static ErrorResourceKey ErrInvalidRegExMixingNamedAndNumberedSubMatches = new ErrorResourceKey("ErrInvalidRegExMixingNamedAndNumberedSubMatches");
+        public static ErrorResourceKey ErrInvalidRegExNumberedSubMatchesDisabled = new ErrorResourceKey("ErrInvalidRegExNumberedSubMatchesDisabled");
+        public static ErrorResourceKey ErrInvalidRegExLiteralHyphenInCharacterClass = new ErrorResourceKey("ErrInvalidRegExLiteralHyphenInCharacterClass");
+        public static ErrorResourceKey ErrInvalidRegExUnescapedCharInCharacterClass = new ErrorResourceKey("ErrInvalidRegExUnescapedCharInCharacterClass");
+        public static ErrorResourceKey ErrInvalidRegExBadUnicodeCategory = new ErrorResourceKey("ErrInvalidRegExBadUnicodeCategory");
+        public static ErrorResourceKey ErrInvalidRegExEmptyCharacterClass = new ErrorResourceKey("ErrInvalidRegExEmptyCharacterClass");
+        public static ErrorResourceKey ErrInvalidRegExQuantifiedCapture = new ErrorResourceKey("ErrInvalidRegExQuantifiedCapture");
+        public static ErrorResourceKey ErrInvalidRegExCaptureInLookAround = new ErrorResourceKey("ErrInvalidRegExCaptureInLookAround");
+        public static ErrorResourceKey ErrInvalidRegExNestedLookAround = new ErrorResourceKey("ErrInvalidRegExNestedLookAround");
+        public static ErrorResourceKey ErrInvalidRegExUnlimitedQuantifierInLookAround = new ErrorResourceKey("ErrInvalidRegExUnlimitedQuantifierInLookAround");
+        public static ErrorResourceKey ErrInvalidRegExBackRefToZeroCapture = new ErrorResourceKey("ErrInvalidRegExBackRefToZeroCapture");
+        public static ErrorResourceKey ErrInvalidRegExGroupTrackerOverflow = new ErrorResourceKey("ErrInvalidRegExGroupTrackerOverflow");
+        public static ErrorResourceKey ErrInvalidRegExQuantifierOursideLookAround = new ErrorResourceKey("ErrInvalidRegExQuantifierOursideLookAround");
+        public static ErrorResourceKey ErrInvalidRegExNamedCaptureNameTooLong = new ErrorResourceKey("ErrInvalidRegExNamedCaptureNameTooLong");
+        public static ErrorResourceKey ErrInvalidRegExLowHighQuantifierFlip = new ErrorResourceKey("ErrInvalidRegExLowHighQuantifierFlip");
+        public static ErrorResourceKey ErrInvalidRegExLookbehindTooManyChars = new ErrorResourceKey("ErrInvalidRegExLookbehindTooManyChars");
+        public static ErrorResourceKey ErrInvalidRegExNumberOverflow = new ErrorResourceKey("ErrInvalidRegExNumberOverflow");
+        public static ErrorResourceKey ErrInvalidRegExV1Options = new ErrorResourceKey("ErrInvalidRegExV1Options");
+        public static ErrorResourceKey ErrInvalidRegExCharacterClassCategoryUse = new ErrorResourceKey("ErrInvalidRegExCharacterClassCategoryUse");
+        public static ErrorResourceKey ErrInvalidRegExCharacterClassRangeReverse = new ErrorResourceKey("ErrInvalidRegExCharacterClassRangeReverse");
+        public static ErrorResourceKey ErrInvalidRegExEndsWithBackslash = new ErrorResourceKey("ErrInvalidRegExEndsWithBackslash");
+        public static ErrorResourceKey ErrInvalidRegExQuantifierOnNothing = new ErrorResourceKey("ErrInvalidRegExQuantifierOnNothing");
+        public static ErrorResourceKey ErrInvalidRegExBackRefToCaptureInLookaround = new ErrorResourceKey("ErrInvalidRegExBackRefToCaptureInLookaround");
+
         public static ErrorResourceKey ErrVariableRegEx = new ErrorResourceKey("ErrVariableRegEx");
-        public static ErrorResourceKey InfoRegExCaptureNameHidesPredefinedFullMatchField = new ErrorResourceKey("InfoRegExCaptureNameHidesPredefinedFullMatchField");
-        public static ErrorResourceKey InfoRegExCaptureNameHidesPredefinedSubMatchesField = new ErrorResourceKey("InfoRegExCaptureNameHidesPredefinedSubMatchesField");
-        public static ErrorResourceKey InfoRegExCaptureNameHidesPredefinedStartMatchField = new ErrorResourceKey("InfoRegExCaptureNameHidesPredefinedStartMatchField");
+        public static ErrorResourceKey ErrVariableRegExOptions = new ErrorResourceKey("ErrVariableRegExOptions");
+        public static ErrorResourceKey InfoRegExCaptureNameHidesPredefinedSingular = new ErrorResourceKey("InfoRegExCaptureNameHidesPredefinedSingular");
+        public static ErrorResourceKey InfoRegExCaptureNameHidesPredefinedPlural = new ErrorResourceKey("InfoRegExCaptureNameHidesPredefinedPlural");
 
         public static ErrorResourceKey ErrErrorIrrelevantField = new ErrorResourceKey("ErrErrorIrrelevantField");
         public static ErrorResourceKey ErrAsNotInContext = new ErrorResourceKey("ErrAsNotInContext");

src/libraries/Microsoft.PowerFx.Core/Public/Config/RegexTypeCache.cs

@@ -3,6 +3,9 @@
 
 using System;
 using System.Collections.Concurrent;
+using Microsoft.PowerFx.Core.Errors;
+using Microsoft.PowerFx.Core.Localization;
+using Microsoft.PowerFx.Core.Texl.Builtins;
 using Microsoft.PowerFx.Core.Types;
 
 namespace Microsoft.PowerFx
@@ -13,9 +16,10 @@ namespace Microsoft.PowerFx
     /// </summary>
     internal class RegexTypeCache
     {
-        // Key is ("tbl_" or "rec_" + regex expression)
+        // Key is ("tbl_" or "rec_" + schema altering options + regex expression)
+        // DType can be null if we have validated the regular expression, but didn't need the type for IsMatch
         // See Match.cs code for details
-        internal ConcurrentDictionary<string, Tuple<DType, bool, bool, bool>> Cache { get; }
+        internal ConcurrentDictionary<string, RegexTypeCacheEntry> Cache { get; }
 
         internal int CacheSize { get; }
 
@@ -27,7 +31,46 @@ public RegexTypeCache(int regexCacheSize)
             }
 
             CacheSize = regexCacheSize;
-            Cache = regexCacheSize == -1 ? null : new ConcurrentDictionary<string, Tuple<DType, bool, bool, bool>>();
+            Cache = regexCacheSize == -1 ? null : new ConcurrentDictionary<string, RegexTypeCacheEntry>();
         }
+
+        public void Add(string key, RegexTypeCacheEntry entry)
+        {
+            if (Cache == null)
+            {
+                // nothing to do, we drop the entry
+                return;
+            }
+
+            if (Cache.Count >= CacheSize)
+            {
+                // To preserve memory during authoring, we clear the cache if it gets
+                // too large. This should only happen in a minority of cases and
+                // should have no impact on deployed apps.
+                Cache.Clear();
+            }
+
+            Cache[key] = entry;
+        }
+
+        public bool TryLookup(string key, out RegexTypeCacheEntry entry)
+        {
+            if (Cache != null && Cache.ContainsKey(key))
+            {
+                entry = Cache[key];
+                return true;
+            }
+
+            entry = null;
+            return false;
+        }
+    }
+
+    internal class RegexTypeCacheEntry
+    {
+        public DType ReturnType;
+        public ErrorResourceKey? Error;
+        public DocumentErrorSeverity ErrorSeverity;
+        public string ErrorParam;
     }
 }