Basic Synthetic Conversation pipeline (#6)

* fix: update JSON from backend

* fix: JSON parsing from backend

* rename testing

* rename testing

* google learnLM new agent for comparison

* introduction of google ai models

* updates

* synthetic conversation gen v1

Author: neagualexa

.dockerignore

@@ -144,4 +144,16 @@ README.md
 data/
 
 # Test reports
-reports/
+reports/
+
+# Synthetic data conversations
+src/agents/utils/example_inputs/
+src/agents/utils/synthetic_conversations/
+src/agents/utils/synthetic_conversation_generator.py
+src/agents/utils/testbench_prompts.py
+src/agents/utils/langgraph_viz.py
+
+# development agents
+src/agents/student_agent/
+src/agents/development_agents/
+src/agents/google_learnML_agent/

.gitignore

@@ -132,4 +132,7 @@ dmypy.json
 .vscode
 
 .DS_Store
-evaluation_function/db_analytics
+evaluation_function/db_analytics
+
+# Synthetic data conversations
+src/agents/utils/synthetic_conversations/*.json

requirements.txt

@@ -3,6 +3,7 @@ langchain-chroma
 langchain-community
 langchain-core
 langchain-openai
+langchain_google_genai
 langchain-text-splitters
 langchainhub
 langdetect

src/agents/google_learnLM_agent/google_learnLM_agent.py

@@ -0,0 +1,203 @@
+try:
+    from ..llm_factory import GoogleAILLMs
+    from .google_learnLM_prompts import \
+        role_prompt, conv_pref_prompt, update_conv_pref_prompt, summary_prompt, update_summary_prompt
+    from ..utils.types import InvokeAgentResponseType
+except ImportError:
+    from src.agents.llm_factory import GoogleAILLMs
+    from src.agents.google_learnLM_agent.google_learnLM_prompts import \
+        role_prompt, conv_pref_prompt, update_conv_pref_prompt, summary_prompt, update_summary_prompt
+    from src.agents.utils.types import InvokeAgentResponseType
+
+from langgraph.graph import StateGraph, START, END
+from langchain_core.messages import SystemMessage, RemoveMessage, HumanMessage, AIMessage
+from langchain_core.runnables.config import RunnableConfig
+from langgraph.graph.message import add_messages
+from typing import Annotated, TypeAlias
+from typing_extensions import TypedDict
+
+"""
+GOOGLE's LearnLM-Tutor agent based on https://arxiv.org/pdf/2412.16429v2 available experimentally on Google AI Studio.
+Docs: https://ai.google.dev/gemini-api/docs/learnlm
+[LLM workflow with a summarisation, profiling, and chat agent that receives an external conversation history].
+
+Used as one of the baselines for comparison for the other agents.
+
+This agent is designed to:
+- [summarise_prompt]        summarise the conversation after 'max_messages_to_summarize' number of messages is reached in the conversation
+- [conv_pref_prompt]        analyse the conversation style of the student 
+- [role_prompt]             role of a tutor to answer student's questions on the topic  
+"""
+
+ValidMessageTypes: TypeAlias = SystemMessage | HumanMessage | AIMessage
+AllMessageTypes: TypeAlias = ValidMessageTypes | RemoveMessage
+
+class State(TypedDict):
+    messages: Annotated[list[AllMessageTypes], add_messages]
+    summary: str
+    conversationalStyle: str
+
+class GoogleLearnLMAgent:
+    def __init__(self):
+        llm = GoogleAILLMs()
+        self.llm = llm.get_llm()
+        summarisation_llm = GoogleAILLMs()
+        self.summarisation_llm = summarisation_llm.get_llm()
+        self.summary = ""
+        self.conversationalStyle = ""
+
+        # Define Agent's specific Parameters
+        self.max_messages_to_summarize = 11
+        self.role_prompt = role_prompt
+        self.summary_prompt = summary_prompt
+        self.update_summary_prompt = update_summary_prompt
+        self.conversation_preference_prompt = conv_pref_prompt
+        self.update_conversation_preference_prompt = update_conv_pref_prompt
+
+        # Define a new graph for the conversation & compile it
+        self.workflow = StateGraph(State)
+        self.workflow_definition()
+        self.app = self.workflow.compile()
+
+    def call_model(self, state: State, config: RunnableConfig) -> str:
+        """Call the LLM model knowing the role system prompt, the summary and the conversational style."""
+        
+        # Default AI tutor role prompt
+        system_message = self.role_prompt
+
+        # Adding external student progress and question context details from data queries
+        question_response_details = config["configurable"].get("question_response_details", "")
+        if question_response_details:
+            system_message += f"## Known Question Materials: {question_response_details} \n\n"
+
+        # Adding summary and conversational style to the system message
+        summary = state.get("summary", "")
+        conversationalStyle = state.get("conversationalStyle", "")
+        if summary:
+            system_message += f"## Summary of conversation earlier: {summary} \n\n"
+        if conversationalStyle:
+            system_message += f"## Known conversational style and preferences of the student for this conversation: {conversationalStyle}. \n\nYour answer must be in line with this conversational style."
+
+        messages = [SystemMessage(content=system_message)] + state['messages']
+
+        valid_messages = self.check_for_valid_messages(messages)
+        response = self.llm.invoke(valid_messages)
+
+        # Save summary for fetching outside the class
+        self.summary = summary
+        self.conversationalStyle = conversationalStyle
+
+        return {"summary": summary, "messages": [response]}
+    
+    def check_for_valid_messages(self, messages: list[AllMessageTypes]) -> list[ValidMessageTypes]:
+        """ Removing the RemoveMessage() from the list of messages """
+
+        valid_messages: list[ValidMessageTypes] = []
+        for message in messages:
+            if message.type != 'remove':
+                valid_messages.append(message)
+        return valid_messages
+    
+    def summarize_conversation(self, state: State, config: RunnableConfig) -> dict:
+        """Summarize the conversation."""
+
+        summary = state.get("summary", "")
+        previous_summary = config["configurable"].get("summary", "")
+        previous_conversationalStyle = config["configurable"].get("conversational_style", "")
+        if previous_summary:
+            summary = previous_summary
+        
+        if summary:
+            summary_message = (
+                f"This is summary of the conversation to date: {summary}\n\n" +
+                self.update_summary_prompt
+            )
+        else:
+            summary_message = self.summary_prompt
+        
+        if previous_conversationalStyle:
+            conversationalStyle_message = (
+                f"This is the previous conversational style of the student for this conversation: {previous_conversationalStyle}\n\n" +
+                self.update_conversation_preference_prompt
+            )
+        else:
+            conversationalStyle_message = self.conversation_preference_prompt
+
+        # STEP 1: Summarize the conversation
+        messages = state["messages"][:-1] + [SystemMessage(content=summary_message)] 
+        valid_messages = self.check_for_valid_messages(messages)
+        summary_response = self.summarisation_llm.invoke(valid_messages)
+
+        # STEP 2: Analyze the conversational style
+        messages = state["messages"][:-1] + [SystemMessage(content=conversationalStyle_message)]
+        valid_messages = self.check_for_valid_messages(messages)
+        conversationalStyle_response = self.summarisation_llm.invoke(valid_messages)
+
+        # Delete messages that are no longer wanted, except the last ones
+        delete_messages: list[AllMessageTypes] = [RemoveMessage(id=m.id) for m in state["messages"][:-3]]
+
+        return {"summary": summary_response.content, "conversationalStyle": conversationalStyle_response.content, "messages": delete_messages}
+    
+    def should_summarize(self, state: State) -> str:
+        """
+        Return the next node to execute. 
+        If there are more than X messages, then we summarize the conversation.
+        Otherwise, we call the LLM.
+        """
+
+        messages = state["messages"]
+        valid_messages = self.check_for_valid_messages(messages)
+        nr_messages = len(valid_messages)
+        if "system" in valid_messages[-1].type:
+            nr_messages -= 1
+
+        # always pairs of (sent, response) + 1 latest message
+        if nr_messages > self.max_messages_to_summarize:
+            return "summarize_conversation"
+        return "call_llm"    
+
+    def workflow_definition(self) -> None:
+        self.workflow.add_node("call_llm", self.call_model)
+        self.workflow.add_node("summarize_conversation", self.summarize_conversation)
+
+        self.workflow.add_conditional_edges(source=START, path=self.should_summarize)
+        self.workflow.add_edge("summarize_conversation", "call_llm")
+        self.workflow.add_edge("call_llm", END)
+
+    def get_summary(self) -> str:
+        return self.summary
+    
+    def get_conversational_style(self) -> str:
+        return self.conversationalStyle
+
+    def print_update(self, update: dict) -> None:
+        for k, v in update.items():
+            for m in v["messages"]:
+                m.pretty_print()
+            if "summary" in v:
+                print(v["summary"])
+
+    def pretty_response_value(self, event: dict) -> str:
+        return event["messages"][-1].content
+    
+agent = GoogleLearnLMAgent()
+def invoke_google_learnlm_agent(query: str, conversation_history: list, summary: str, conversationalStyle: str, question_response_details: str, session_id: str) -> InvokeAgentResponseType:
+    """
+    Call an agent that has no conversation memory and expects to receive all past messages in the params and the latest human request in the query.
+    If conversation history longer than X, the agent will summarize the conversation and will provide a conversational style analysis.
+    """
+    print(f'in invoke_google_learnlm_agent(), query = {query}, thread_id = {session_id}')
+
+    config = {"configurable": {"thread_id": session_id, "summary": summary, "conversational_style": conversationalStyle, "question_response_details": question_response_details}}
+    response_events = agent.app.invoke({"messages": conversation_history + [HumanMessage(content=query)]}, config=config, stream_mode="values") #updates
+    pretty_printed_response = agent.pretty_response_value(response_events) # get last event/ai answer in the response
+
+    # Gather Metadata from the agent
+    summary = agent.get_summary()
+    conversationalStyle = agent.get_conversational_style()
+
+    return {
+        "input": query,
+        "output": pretty_printed_response,
+        "intermediate_steps": [str(summary), conversationalStyle, conversation_history]
+    }

src/agents/google_learnLM_agent/google_learnLM_prompts.py

@@ -0,0 +1,78 @@
+# NOTE:
+# First person view prompts proven to be more effective in generating responses from the model (Dec 2024)
+# 'Keep your responses open for further questions and encourage the student's curiosity.' -> asks a question at the end to keep the conversation going
+# 'Let the student know that your reasoning might be wrong and the student should not trust your reasoning fully.' -> not relliant
+
+# PROMPTS generated with the help of ChatGPT GPT-4o Nov 2024
+
+role_prompt = "You are an excellent tutor that aims to provide clear and concise explanations to students. I am the student. Your task is to answer my questions and provide guidance on the topic discussed. Ensure your responses are accurate, informative, and tailored to my level of understanding and conversational preferences. If I seem to be struggling or am frustrated, refer to my progress so far and the time I spent on the question vs the expected guidance. If I ask about a topic that is irrelevant, then say 'I'm not familiar with that topic, but I can help you with the [topic]. You do not need to end your messages with a concluding statement.\n\n"
+
+pref_guidelines = """**Guidelines:**
+- Use concise, objective language.
+- Note the student's educational goals, such as understanding foundational concepts, passing an exam, getting top marks, code implementation, hands-on practice, etc.
+- Note any specific preferences in how the student learns, such as asking detailed questions, seeking practical examples, requesting quizes, requesting clarifications, etc.
+- Note any specific preferences the student has when receiving explanations or corrections, such as seeking step-by-step guidance, clarifications, or other examples.
+- Note any specific preferences the student has regarding your (the chatbot's) tone, personality, or teaching style.
+- Avoid assumptions about motivation; observe only patterns evident in the conversation.
+- If no particular preference is detectable, state "No preference observed."
+"""
+
+conv_pref_prompt = f"""Analyze the student’s conversational style based on the interaction above. Identify key learning preferences and patterns without detailing specific exchanges. Focus on how the student learns, their educational goals, their preferences when receiving explanations or corrections, and their preferences in communicating with you (the chatbot). Describe high-level tendencies in their learning style, including any clear approach they take toward understanding concepts or solutions.
+
+{pref_guidelines}
+
+Examples:
+
+Example 1:
+**Conversation:**
+Student: "I understand that the derivative gives us the slope of a function, but what if we want to know the rate of change over an interval? Do we still use the derivative?"
+AI: "Good question! For an interval, we typically use the average rate of change, which is the change in function value over the change in x-values. The derivative gives the instantaneous rate of change at a specific point."
+
+**Expected Answer:**
+The student prefers in-depth conceptual understanding and asks thoughtful questions that differentiate between similar concepts. They seem comfortable discussing foundational ideas in calculus.
+
+Example 2:
+**Conversation:**
+Student: "I’m trying to solve this physics problem: if I throw a ball upwards at 10 m/s, how long will it take to reach the top? I thought I could just divide by gravity, but I’m not sure."
+AI: "You're on the right track! Since acceleration due to gravity is 9.8 m/s², you can divide the initial velocity by gravity to find the time to reach the peak, which would be around 1.02 seconds."
+
+**Expected Answer:**
+The student prefers practical problem-solving and is open to corrections. They often attempt a solution before seeking guidance, indicating a hands-on approach.
+
+Example 3:
+**Conversation:**
+Student: "Can you explain the difference between meiosis and mitosis? I know both involve cell division, but I’m confused about how they differ."
+AI: "Certainly! Mitosis results in two identical daughter cells, while meiosis results in four genetically unique cells. Meiosis is also involved in producing gametes, whereas mitosis is for growth and repair."
+
+**Expected Answer:**
+The student prefers clear, comparative explanations when learning complex biological processes. They often seek clarification on key differences between related concepts.
+
+Example 4:
+**Conversation:**
+Student: "I wrote this Python code to reverse a string, but it’s not working. Here’s what I tried: `for char in string: new_string = char + new_string`."
+AI: "You’re close! Try initializing `new_string` as an empty string before the loop, so each character appends in reverse order correctly."
+
+**Expected Answer:**
+The student prefers hands-on guidance with code, often sharing specific code snippets. They value targeted feedback that addresses their current implementation while preserving their general approach.
+
+"""
+
+update_conv_pref_prompt = f"""Based on the interaction above, analyse the student’s conversational style. Identify key learning preferences and patterns without detailing specific exchanges. Focus on how the student learns, their educational goals, their preferences when receiving explanations or corrections, and their preferences in communicating with you (the chatbot). Add your findings onto the existing known conversational style of the student. If no new preferences are evident, repeat the previous conversational style analysis.
+
+{pref_guidelines}
+"""
+
+summary_prompt = """
+You are an AI assistant specializing in concise and accurate summarization. Your task is to summarize the previous conversation, capturing the main topics, key points, user questions, and your responses in a clear and organized format. 
+
+Ensure the summary is:
+
+Concise: Keep the summary brief while including all essential information.
+Structured: Organize the summary into sections such as 'Topics Discussed,' 'Key Questions and Responses,' and 'Follow-Up Suggestions' if applicable.
+Neutral and Accurate: Avoid adding interpretations or opinions; focus only on the content shared.
+When summarizing: If the conversation is technical, highlight significant concepts, solutions, and terminology. If context involves problem-solving, detail the problem and the steps or solutions provided. If the user asks for creative input, briefly describe the ideas presented.
+
+Provide the summary in a bulleted format for clarity. Avoid redundant details while preserving the core intent of the discussion.
+"""
+
+update_summary_prompt = "Update the summary by taking into account the new messages above:"

src/agents/llm_factory.py

@@ -6,13 +6,14 @@
 from langchain_community.embeddings import OllamaEmbeddings
 from langchain_openai import ChatOpenAI
 from langchain_openai import OpenAIEmbeddings
+from langchain_google_genai import ChatGoogleGenerativeAI
 
 class AzureLLMs:
-    def __init__(self):
+    def __init__(self, temperature: int = 0):
         self._azure_llm = AzureChatOpenAI(
                         openai_api_version=os.environ["AZURE_OPENAI_API_VERSION"],
                         azure_deployment=os.environ["AZURE_OPENAI_CHAT_DEPLOYMENT_NAME"],
-                        temperature=0,
+                        temperature=temperature,
                         max_tokens=None,
                     )
         self._azure_embedding = AzureOpenAIEmbeddings(azure_deployment=os.environ['AZURE_OPENAI_EMBEDDING_1536_DEPLOYMENT'], 
@@ -51,10 +52,10 @@ def get_embedding(self):
         return self._ollama_embedding
 
 class OpenAILLMs:
-    def __init__(self):
+    def __init__(self, temperature: int = 0):
         self._openai_llm = ChatOpenAI(
             model=os.environ['OPENAI_MODEL'],
-            temperature=0,
+            temperature=temperature,
             api_key=os.environ["OPENAI_API_KEY"],
         )
 
@@ -68,3 +69,15 @@ def get_llm(self):
 
     def get_embedding(self):
         return self._openai_embedding
+
+class GoogleAILLMs:
+    def __init__(self, temperature: int = 0):
+
+        self._google_llm = ChatGoogleGenerativeAI(
+            model=os.environ['GOOGLE_AI_MODEL'],
+            temperature=temperature,
+            google_api_key=os.environ['GOOGLE_AI_API_KEY'],
+        )
+    
+    def get_llm(self):
+        return self._google_llm