Switch to native support for `..."<NLD>" and multiline string expressions

Author: schuellc-nvidia

examples/v2_x/language_reference/use_llms/interaction_loop/main.co

@@ -10,7 +10,7 @@ flow main
         when unhandled user intent
             llm continue interaction
         or when user was silent 12.0
-            $response = i"A random fun fact"
+            $response = ..."A random fun fact"
             bot say $response
         or when user expressed greeting
             bot say "Hi there!"

examples/v2_x/tutorial/guardrails_1/rails.co

@@ -24,6 +24,6 @@ flow input rails $input_text
     abort
 
 flow check user utterance $input_text -> $input_safe
-  $is_safe = i"Consider the following user utterance: '{$input_text}'. Assign 'True' if appropriate, 'False' if inappropriate."
+  $is_safe = ..."Consider the following user utterance: '{$input_text}'. Assign 'True' if appropriate, 'False' if inappropriate."
   print $is_safe
   return $is_safe

nemoguardrails/colang/v2_x/lang/expansion.py

@@ -41,7 +41,10 @@
 )
 from nemoguardrails.colang.v2_x.runtime.errors import ColangSyntaxError
 from nemoguardrails.colang.v2_x.runtime.flows import FlowConfig, InternalEvents
-from nemoguardrails.colang.v2_x.runtime.utils import new_var_uid
+from nemoguardrails.colang.v2_x.runtime.utils import (
+    escape_special_string_characters,
+    new_var_uid,
+)
 
 
 def expand_elements(
@@ -653,11 +656,12 @@ def _expand_assignment_stmt_element(element: Assignment) -> List[ElementType]:
     new_elements: List[ElementType] = []
 
     # Check if the expression is an NLD instruction
-    nld_instruction_pattern = r"^\s*i\"(.*)\"|^\s*i'(.*)'"
+    nld_instruction_pattern = r'\.\.\.("""|\'\'\'|"|\')((?:\\\1|(?!\1)[\s\S])*?)\1'
     match = re.search(nld_instruction_pattern, element.expression)
 
     if match:
         # Replace the assignment with the GenerateValueAction system action
+        instruction = escape_special_string_characters(match.group(2))
         new_elements.append(
             SpecOp(
                 op="await",
@@ -666,7 +670,7 @@ def _expand_assignment_stmt_element(element: Assignment) -> List[ElementType]:
                     spec_type=SpecType.ACTION,
                     arguments={
                         "var_name": f'"{element.key}"',
-                        "instructions": f'"{match.group(1) or match.group(2)}"',
+                        "instructions": f'"{instruction}"',
                     },
                 ),
                 return_var_name=element.key,

nemoguardrails/colang/v2_x/lang/grammar/colang.lark

@@ -221,8 +221,8 @@ _NEWLINE: (/\r?\n[\t ]*/)+
 
 // Primitive values
 
-STRING: /i?("(?!"").*?(?<!\\)(\\\\)*?"|i?'(?!'').*?(?<!\\)(\\\\)*?')/i
-LONG_STRING: /(""".*?(?<!\\)(\\\\)*?"""|'''.*?(?<!\\)(\\\\)*?''')/is
+STRING: /(\.\.\.)?("(?!"").*?(?<!\\)(\\\\)*?"|(\.\.\.)?'(?!'').*?(?<!\\)(\\\\)*?')/i
+LONG_STRING: /(\.\.\.)?(""".*?(?<!\\)(\\\\)*?"""|(\.\.\.)?'''.*?(?<!\\)(\\\\)*?''')/is
 
 _SPECIAL_DEC: "0".."9"        ("_"?  "0".."9"                       )*
 DEC_NUMBER:   "1".."9"        ("_"?  "0".."9"                       )*

nemoguardrails/colang/v2_x/lang/parser.py

@@ -59,9 +59,6 @@ def _apply_pre_parsing_expansions(content: str):
 
         Currently, only the "..." is expanded.
         """
-        # Replace ..."NLD" with i"NLD"
-        content = re.sub(r"\.\.\.(['\"])(.*?)\1", r'i"\2"', content)
-
         # We make sure to capture the correct indentation level and use that.
         content = re.sub(
             r"\n( +)\.\.\.",

nemoguardrails/colang/v2_x/runtime/eval.py

@@ -26,7 +26,10 @@
 from nemoguardrails.colang.v2_x.runtime import system_functions
 from nemoguardrails.colang.v2_x.runtime.errors import ColangValueError
 from nemoguardrails.colang.v2_x.runtime.flows import FlowState, State
-from nemoguardrails.colang.v2_x.runtime.utils import AttributeDict
+from nemoguardrails.colang.v2_x.runtime.utils import (
+    AttributeDict,
+    escape_special_string_characters,
+)
 from nemoguardrails.eval.cli.simplify_formatter import SimplifyFormatter
 from nemoguardrails.utils import new_uid
 
@@ -90,20 +93,10 @@ def eval_expression(expr: str, context: dict) -> Any:
                             f"Error evaluating inner expression: '{inner_expression}'"
                         ) from ex
 
-                    # Escape quotation characters
-                    value = str(value).replace("'", "\\'").replace('"', '\\"')
-
-                    # Escape escaped characters
-                    escaped_characters_map = {
-                        "\n": "\\n",
-                        "\t": "\\t",
-                        "\r": "\\r",
-                        "\b": "\\b",
-                        "\f": "\\f",
-                        "\v": "\\v",
-                    }
-                    for c, s in escaped_characters_map.items():
-                        value = str(value).replace(c, s)
+                    value = str(value)
+
+                    # Escape special characters
+                    value = escape_special_string_characters(value)
 
                     inner_expression_values.append(value)
                 string_expression = re.sub(

nemoguardrails/colang/v2_x/runtime/utils.py

@@ -13,6 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import re
 import uuid
 
 
@@ -40,3 +41,23 @@ def new_readable_uid(name: str) -> str:
 def new_var_uid() -> str:
     """Creates a new uuid that is compatible with variable names."""
     return str(uuid.uuid4()).replace("-", "_")
+
+
+def escape_special_string_characters(string: str) -> str:
+    """Escapes all occurrences of special characters."""
+    # Replace " or ' with \\" or \\' if not already escaped
+    string = re.sub(r"(^|[^\\])('|\")", r"\1\\\2", string)
+    # Replace other special characters
+    escaped_characters_map = {
+        "\n": "\\n",
+        "\t": "\\t",
+        "\r": "\\r",
+        "\b": "\\b",
+        "\f": "\\f",
+        "\v": "\\v",
+    }
+
+    for c, s in escaped_characters_map.items():
+        string = str(string).replace(c, s)
+
+    return string

tests/v2_x/test_llm_value_generation.py

@@ -0,0 +1,65 @@
+# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+
+from rich.logging import RichHandler
+
+from nemoguardrails import RailsConfig
+from tests.utils import TestChat
+
+FORMAT = "%(message)s"
+logging.basicConfig(
+    level=logging.DEBUG,
+    format=FORMAT,
+    datefmt="[%X,%f]",
+    handlers=[RichHandler(markup=True)],
+)
+
+
+def test_1():
+    """Test use of expression as statements."""
+    config = RailsConfig.from_content(
+        colang_content="""
+        flow main
+          match UtteranceUserActionFinished()
+          $v1 = ..."Generate the company' name\\" from users input"
+          $v2 = ...'Generate the company\\' name" from users input'
+          $v3 = ...'''Generate the company' name" from users input'''
+          $v4 = ...'''Generate the company'
+                    name" from users input'''
+
+          await UtteranceBotAction(script="{$v1}{$v2}{$v3}{$v4}")
+        """,
+        yaml_content="""
+        colang_version: "2.x"
+        models:
+        - type: main
+          engine: openai
+          model: gpt-3.5-turbo-instruct
+        """,
+    )
+
+    chat = TestChat(
+        config,
+        llm_completions=["'1'", "'2'", "'3'", "'4'"],
+    )
+
+    chat >> "hi"
+    chat << "1234"
+
+
+if __name__ == "__main__":
+    test_1()