fix(hallucination-check): Rename check_hallucination to self_check_hallucination

This commit renames all instances of 'check_hallucination' to
'self_check_hallucination' across multiple files. This includes changes in
function names, task names, and documentation.

Author: Pouyanpi

docs/user_guides/guardrails-library.md

@@ -323,14 +323,14 @@ To use the hallucination rail, you should:
 rails:
   input:
     flows:
-      - self check hallucinations
+      - self check hallucination
 ```
 
-2. Define a `self_check_hallucinations` prompt in the `prompts.yml` file:
+2. Define a `self_check_hallucination` prompt in the `prompts.yml` file:
 
 ```yaml
 prompts:
-  - task: self_check_hallucinations
+  - task: self_check_hallucination
     content: |-
       You are given a task to identify if the hypothesis is in agreement with the context below.
       You will only use the contents of the context and not rely on external knowledge.

examples/configs/rag/custom_rag_output_rails/config.yml

@@ -7,7 +7,7 @@ rails:
   output:
     flows:
       - self check facts
-      - check hallucination
+      - self check hallucination
 
 prompts:
   - task: self_check_facts

examples/sample_config.yml

@@ -34,7 +34,7 @@ rails:
     enable_on_predefined_messages: false
     flows:
       - self check facts
-      - check hallucination
+      - self check hallucination
       - activefence moderation
       - check sensitive data
       - gotitai rag truthcheck

nemoguardrails/eval/evaluate_hallucination.py

@@ -102,7 +102,7 @@ def get_extra_responses(self, prompt, num_responses=2):
 
         return extra_responses
 
-    def check_hallucination(self):
+    def self_check_hallucination(self):
         """
         Run the hallucination rail evaluation.
         For each prompt, generate 2 extra responses from the LLM and check consistency with the bot response.
@@ -150,7 +150,7 @@ def check_hallucination(self):
             else:
                 paragraph = ". ".join(extra_responses)
                 hallucination_check_prompt = self.llm_task_manager.render_task_prompt(
-                    Task.CHECK_HALLUCINATION,
+                    Task.SELF_CHECK_HALLUCINATION,
                     {"paragraph": paragraph, "statement": bot_response},
                 )
                 hallucination = self.llm(hallucination_check_prompt)
@@ -177,7 +177,7 @@ def run(self):
             hallucination_check_predictions,
             num_flagged,
             num_error,
-        ) = self.check_hallucination()
+        ) = self.self_check_hallucination()
         print(
             f"% of samples flagged as hallucinations: {num_flagged/len(self.dataset) * 100}"
         )

nemoguardrails/library/hallucination/actions.py

@@ -40,7 +40,7 @@
 
 
 @action()
-async def check_hallucination(
+async def self_check_hallucination(
     llm_task_manager: LLMTaskManager,
     context: Optional[dict] = None,
     llm: Optional[BaseLLM] = None,
@@ -58,14 +58,17 @@ async def check_hallucination(
             "The langchain_openai module is not installed. Please install it using pip: pip install langchain_openai"
         )
 
+    print(hasattr(llm, "best_of"))
+    print("***" * 100)
     bot_response = context.get("bot_message")
     last_bot_prompt_string = context.get("_last_bot_prompt")
 
     if bot_response and last_bot_prompt_string:
         num_responses = HALLUCINATION_NUM_EXTRA_RESPONSES
         # Use beam search for the LLM call, to get several completions with only one call.
         # At the current moment, only OpenAI LLM engines are supported for computing the additional completions.
-        if type(llm) != OpenAI:
+        #
+        if "openai" not in str(type(llm)).lower():
             log.warning(
                 f"Hallucination rail can only be used with OpenAI LLM engines."
                 f"Current LLM engine is {type(llm).__name__}."
@@ -77,7 +80,17 @@ async def check_hallucination(
         chain = LLMChain(prompt=last_bot_prompt, llm=llm)
 
         # Generate multiple responses with temperature 1.
-        with llm_params(llm, temperature=1.0, n=num_responses, best_of=num_responses):
+        with llm_params(llm, temperature=1.0, n=num_responses):
+            # best_of
+            # integer or null
+            #
+            # Optional
+            # Defaults to 1
+            # Generates best_of completions server-side and returns the "best" (the one with the highest log probability per token). Results cannot be streamed.
+            #
+            # When used with n, best_of controls the number of candidate completions and n specifies how many to return – best_of must be greater than n.
+            #
+            # Note: Because this parameter generates many completions, it can quickly consume your token quota. Use carefully and ensure that you have reasonable settings for max_tokens and stop.
             extra_llm_response = await chain.agenerate(
                 [{"text": last_bot_prompt_string}],
                 run_manager=logging_callback_manager_for_chain,
@@ -112,16 +125,16 @@ async def check_hallucination(
         if use_llm_checking:
             # Only support LLM-based agreement check in current version
             prompt = llm_task_manager.render_task_prompt(
-                task=Task.CHECK_HALLUCINATION,
+                task=Task.SELF_CHECK_HALLUCINATION,
                 context={
                     "statement": bot_response,
                     "paragraph": ". ".join(extra_responses),
                 },
             )
 
             # Initialize the LLMCallInfo object
-            llm_call_info_var.set(LLMCallInfo(task=Task.CHECK_HALLUCINATION.value))
-            stop = llm_task_manager.get_stop_tokens(task=Task.CHECK_HALLUCINATION)
+            llm_call_info_var.set(LLMCallInfo(task=Task.SELF_CHECK_HALLUCINATION.value))
+            stop = llm_task_manager.get_stop_tokens(task=Task.SELF_CHECK_HALLUCINATION)
 
             with llm_params(llm, temperature=config.lowest_temperature):
                 agreement = await llm_call(llm, prompt, stop=stop)

nemoguardrails/library/hallucination/flows.co

@@ -5,7 +5,7 @@ define flow hallucination warning
   """Warning rail for hallucination."""
   bot ...
   if $hallucination_warning == True
-    $is_hallucination = execute check_hallucination
+    $is_hallucination = execute self_check_hallucination
     $hallucination_warning = False
 
     if $is_hallucination
@@ -16,10 +16,10 @@ define bot inform answer prone to hallucination
   "The above response may have been hallucinated, and should be independently verified."
 
 
-define subflow check hallucination
+define subflow self check hallucination
   """Output rail for checking hallucinations."""
   if $check_hallucination == True
-    $is_hallucination = execute check_hallucination
+    $is_hallucination = execute self_check_hallucination
     $check_hallucination = False
 
     if $is_hallucination

nemoguardrails/llm/filters.py

@@ -62,7 +62,7 @@ def co_v2(
         "zapier_nla_query",
         "call activefence api",
         "jailbreak_detection_heuristics",
-        "check_hallucination",
+        "self_check_hallucination",
         "llama_guard_check_input",
         "llama_guard_check_output",
         "alignscore_check_facts",

nemoguardrails/llm/prompts/nemollm.yml

@@ -119,16 +119,14 @@ prompts:
       You will only use the contents of the evidence and not rely on external knowledge.
       Answer with yes/no. "evidence": {{ evidence }} "hypothesis": {{ response }} "entails":
 
-
-  - task: check_hallucination
+  - task: self_check_hallucination
     models:
       - nemollm
     content: |-
       You are given a task to identify if the hypothesis is in agreement with the context below.
       You will only use the contents of the context and not rely on external knowledge.
       Answer with yes/no. "context": {{ paragraph }} "hypothesis": {{ statement }} "agreement":
 
-
   # Prompts for compact mode for dialogue rails
   # Prompt for detecting the user message canonical form in compact form.
   - task: generate_user_intent

nemoguardrails/llm/types.py

@@ -46,4 +46,4 @@ class Task(Enum):
     )
 
     SELF_CHECK_FACTS = "fact_checking"
-    CHECK_HALLUCINATION = "check_hallucination"
+    SELF_CHECK_HALLUCINATION = "self_check_hallucination"

tests/test_configs/with_prompt_modes/prompts/prompts.yml

@@ -13,13 +13,13 @@ prompts:
     content: |-
       <<This custom prompt generates the user intent>>
 
-  - task: check_hallucination
+  - task: self_check_hallucination
     models:
       - nemollm
     content: |-
       <<This is a long placeholder prompt to check for hallucinations>>
 
-  - task: check_hallucination
+  - task: self_check_hallucination
     models:
       - nemollm
     mode: compact

tests/test_prompt_modes.py

@@ -33,10 +33,14 @@
         Task.GENERATE_USER_INTENT,
         "<<This is a placeholder for a custom prompt for generating the user intent using gpt-3.5-turbo>>",
     ),
-    ("task3_nemo_compact", Task.CHECK_HALLUCINATION, "<<Check for hallucinations>>"),
+    (
+        "task3_nemo_compact",
+        Task.SELF_CHECK_HALLUCINATION,
+        "<<Check for hallucinations>>",
+    ),
     (
         "task4_nemo_standard",
-        Task.CHECK_HALLUCINATION,
+        Task.SELF_CHECK_HALLUCINATION,
         "<<This is a long placeholder prompt to check for hallucinations>>",
     ),
 ]