Merge pull request #554 from NVIDIA/feature/colang-NLD-intents

Feature/colang nld intents

Author: schuellc-nvidia

nemoguardrails/actions/llm/generation.py

@@ -44,6 +44,7 @@
 )
 from nemoguardrails.colang import parse_colang_file
 from nemoguardrails.colang.v2_x.lang.colang_ast import Flow, Spec, SpecOp
+from nemoguardrails.colang.v2_x.runtime.eval import eval_expression
 from nemoguardrails.context import (
     generation_options_var,
     llm_call_info_var,
@@ -150,8 +151,9 @@ def _extract_user_message_example(self, flow: Flow):
                     return
 
                 # Extract the message and remove the double quotes
-                message = spec.arguments["final_transcript"][1:-1]
-                self.user_messages[flow.name] = [message]
+                message = eval_expression(spec.arguments["final_transcript"], {})
+                if isinstance(message, str):
+                    self.user_messages[flow.name] = [message]
 
             elif el.op == "await":
                 spec = cast(SpecOp, el).spec
@@ -169,11 +171,11 @@ def _extract_user_message_example(self, flow: Flow):
                     ):
                         continue
 
-                    message = spec.arguments["$0"][1:-1]
-                    if flow.name not in self.user_messages:
-                        self.user_messages[flow.name] = []
-
-                    self.user_messages[flow.name].append(message)
+                    message = eval_expression(spec.arguments["$0"], {})
+                    if isinstance(message, str):
+                        if flow.name not in self.user_messages:
+                            self.user_messages[flow.name] = []
+                        self.user_messages[flow.name].append(message)
 
     def _extract_bot_message_example(self, flow: Flow):
         # Quick heuristic to identify the user utterance examples

nemoguardrails/actions/v2_x/generation.py

@@ -17,7 +17,7 @@
 import logging
 import re
 from ast import literal_eval
-from typing import Any, List, Optional
+from typing import Any, List, Optional, Tuple
 
 from langchain.llms import BaseLLM
 
@@ -140,30 +140,9 @@ async def _init_flows_index(self) -> None:
         if self.instruction_flows_index is None:
             self.instruction_flows_index = self.flows_index
 
-    @action(name="GetLastUserMessageAction", is_system_action=True)
-    async def get_last_user_message(
-        self, events: List[dict], llm: Optional[BaseLLM] = None
-    ) -> str:
-        event = get_last_user_utterance_event_v2_x(events)
-        assert event and event["type"] == "UtteranceUserActionFinished"
-        return event["final_transcript"]
-
-    @action(name="GenerateUserIntentAction", is_system_action=True, execute_async=True)
-    async def generate_user_intent(
-        self,
-        state: State,
-        events: List[dict],
-        user_action: str,
-        max_example_flows: int = 5,
-        llm: Optional[BaseLLM] = None,
-    ) -> str:
-        """Generate the canonical form for what the user said i.e. user intent."""
-
-        # Use action specific llm if registered else fallback to main llm
-        llm = llm or self.llm
-
-        log.info("Phase 1 :: Generating user intent")
-
+    async def _collect_user_intent_and_examples(
+        self, state: State, user_action: str, max_example_flows: int
+    ) -> Tuple[List[str], str]:
         # We search for the most relevant similar user intents
         examples = ""
         potential_user_intents = []
@@ -182,25 +161,68 @@ async def generate_user_intent(
         heads = find_all_active_event_matchers(state)
         for head in heads:
             element = get_element_from_head(state, head)
-            event = get_event_from_element(
-                state, state.flow_states[head.flow_state_uid], element
-            )
+            flow_state = state.flow_states[head.flow_state_uid]
+            event = get_event_from_element(state, flow_state, element)
             if (
                 event.name == InternalEvents.FLOW_FINISHED
                 and "flow_id" in event.arguments
             ):
                 flow_id = event.arguments["flow_id"]
+                if not isinstance(flow_id, str):
+                    continue
+
                 flow_config = state.flow_configs.get(flow_id, None)
-                if isinstance(flow_id, str) and (
-                    flow_config is None
+                element_flow_state_instance = state.flow_id_states[flow_id]
+                if flow_config is not None and (
+                    flow_config.has_meta_tag("user_intent")
                     or (
-                        flow_config.has_meta_tag("user_intent")
-                        and flow_id not in potential_user_intents
+                        element_flow_state_instance
+                        and "_user_intent" in element_flow_state_instance[0].context
                     )
                 ):
-                    examples += f"user intent: {flow_id}\n\n"
-                    potential_user_intents.append(flow_id)
+                    if flow_config.elements[1]["_type"] == "doc_string_stmt":
+                        examples += "user action: <" + (
+                            flow_config.elements[1]["elements"][0]["elements"][0][
+                                "elements"
+                            ][0][3:-3]
+                            + ">\n"
+                        )
+                        examples += f"user intent: {flow_id}\n\n"
+                    elif flow_id not in potential_user_intents:
+                        examples += f"user intent: {flow_id}\n\n"
+                        potential_user_intents.append(flow_id)
         examples = examples.strip("\n")
+        return (potential_user_intents, examples)
+
+    @action(name="GetLastUserMessageAction", is_system_action=True)
+    async def get_last_user_message(
+        self, events: List[dict], llm: Optional[BaseLLM] = None
+    ) -> str:
+        event = get_last_user_utterance_event_v2_x(events)
+        assert event and event["type"] == "UtteranceUserActionFinished"
+        return event["final_transcript"]
+
+    @action(name="GenerateUserIntentAction", is_system_action=True, execute_async=True)
+    async def generate_user_intent(
+        self,
+        state: State,
+        events: List[dict],
+        user_action: str,
+        max_example_flows: int = 5,
+        llm: Optional[BaseLLM] = None,
+    ) -> str:
+        """Generate the canonical form for what the user said i.e. user intent."""
+
+        # Use action specific llm if registered else fallback to main llm
+        llm = llm or self.llm
+
+        log.info("Phase 1 :: Generating user intent")
+        (
+            potential_user_intents,
+            examples,
+        ) = await self._collect_user_intent_and_examples(
+            state, user_action, max_example_flows
+        )
 
         prompt = self.llm_task_manager.render_task_prompt(
             task=Task.GENERATE_USER_INTENT_FROM_USER_ACTION,
@@ -257,43 +279,12 @@ async def generate_user_intent_and_bot_action(
 
         log.info("Phase 1 :: Generating user intent and bot action")
 
-        # We search for the most relevant similar user intents
-        examples = ""
-        potential_user_intents = []
-
-        if self.user_message_index:
-            results = await self.user_message_index.search(
-                text=user_action, max_results=max_example_flows
-            )
-
-            # We add these in reverse order so the most relevant is towards the end.
-            for result in reversed(results):
-                examples += f"user action: user said \"{result.text}\"\nuser intent: {result.meta['intent']}\n\n"
-                potential_user_intents.append(result.meta["intent"])
-
-        # We add all currently active user intents (heads on match statements)
-        heads = find_all_active_event_matchers(state)
-        for head in heads:
-            element = get_element_from_head(state, head)
-            event = get_event_from_element(
-                state, state.flow_states[head.flow_state_uid], element
-            )
-            if (
-                event.name == InternalEvents.FLOW_FINISHED
-                and "flow_id" in event.arguments
-            ):
-                flow_id = event.arguments["flow_id"]
-                flow_config = state.flow_configs.get(flow_id, None)
-                if isinstance(flow_id, str) and (
-                    flow_config is None
-                    or (
-                        flow_config.has_meta_tag("user_intent")
-                        and flow_id not in potential_user_intents
-                    )
-                ):
-                    examples += f"user intent: {flow_id}\n\n"
-                    potential_user_intents.append(flow_id)
-        examples = examples.strip("\n")
+        (
+            potential_user_intents,
+            examples,
+        ) = await self._collect_user_intent_and_examples(
+            state, user_action, max_example_flows
+        )
 
         prompt = self.llm_task_manager.render_task_prompt(
             task=Task.GENERATE_USER_INTENT_FROM_USER_ACTION,

nemoguardrails/colang/v2_x/library/llm.co

@@ -101,21 +101,22 @@ flow continuation on unhandled user utterance
     send UserIntentLog(flow_id=$user_intent, parameter=None)
     # If the user intent was not handled
 
-    if $bot_intent is not None
-      $exists = await CheckValidFlowExistsAction(flow_id=$bot_intent)
-      $flow_info = await CreateFlowAction(name=$bot_intent, body=$bot_action)
+    # Create and start new flow or an existing on with same name as bot intent
+    $flow_exists = False
+    if $bot_intent is None
+      $bot_intent = "bot custom action"
     else
-      $exists = False
-      $flow_info = await CreateFlowAction(name="bot action", body=$bot_action)
+      $flow_exists = await CheckValidFlowExistsAction(flow_id=$bot_intent)
 
-    if $exists == False
+    if $flow_exists == False and $bot_action is not None
+      $flow_info = await CreateFlowAction(name=$bot_intent, body=$bot_action)
       $flows = await AddFlowsAction(config=$flow_info.body)
       if len($flows) == 0
         print "Failed parsing created bot action flow!"
         return
-      else
-        await await_flow_by_name $flow_info.name
-        await RemoveFlowsAction(flow_ids=[$flow_info.name])
+      # Start
+      await await_flow_by_name $flow_info.name
+      await RemoveFlowsAction(flow_ids=[$flow_info.name])
 
 
 flow unhandled user intent -> $intent

nemoguardrails/llm/prompts/general.yml

@@ -116,19 +116,13 @@ prompts:
     # Prompt for detecting the user message canonical form.
     - task: generate_user_intent_from_user_action
       content: |-
-          """
-          {{ general_instructions }}
-          """
-
           # This is how a conversation between a user and the bot can go:
           {{ sample_conversation }}
 
           # These are the most likely user intents:
           {{ examples }}
 
-          # This is the current conversation between the user and the bot:
-          {{ history | colang }}
-
+          # Continuation of interaction using only specified user intents from the section 'These are the most likely user intents:':
           user action: {{ user_action }}
           user intent:
       stop:

tests/v2_x/test_llm_user_intents_detection.py

@@ -0,0 +1,85 @@
+# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+
+from nemoguardrails import RailsConfig
+from tests.utils import TestChat
+
+colang_content = '''
+    import core
+    import llm
+
+    flow main
+        activate llm continuation
+        activate greeting
+        activate other reactions
+
+    flow greeting
+        user expressed greeting
+        bot say "Hello world!"
+
+    flow other reactions
+        user expressed to be bored
+        bot say "No problem!"
+
+    flow user expressed greeting
+        """"User expressed greeting in any way or form."""
+        user said "hi"
+
+    flow user expressed to be bored
+        """"User expressed to be bored."""
+        user said "This is boring"
+    '''
+
+yaml_content = """
+colang_version: "2.x"
+models:
+  - type: main
+    engine: openai
+    model: gpt-3.5-turbo-instruct
+
+    """
+
+
+def test_1():
+    config = RailsConfig.from_content(colang_content, yaml_content)
+
+    chat = TestChat(
+        config,
+        llm_completions=["user expressed greeting"],
+    )
+
+    chat >> "hi"
+    chat << "Hello world!"
+
+    chat >> "hello"
+    chat << "Hello world!"
+
+
+def test_2():
+    config = RailsConfig.from_content(colang_content, yaml_content)
+
+    chat = TestChat(
+        config,
+        llm_completions=["user expressed to be bored"],
+    )
+
+    chat >> "You are boring me!"
+    chat << "No problem!"
+
+
+if __name__ == "__main__":
+    test_2()