Improve Summarisation for agents (#12)

* student-tutor sythetic conversation basic (with student personal and basic fluency)

* save conversation in csv

* review and update socratic agent

* useful print

* dockerignore file update

* refactor summary stage in all agents, remove student conversation style for now

* fix pytests

Author: neagualexa

.dockerignore

@@ -149,11 +149,12 @@ reports/
 # Synthetic data conversations
 src/agents/utils/example_inputs/
 src/agents/utils/synthetic_conversations/
-src/agents/utils/synthetic_conversation_generator.py
+src/agents/utils/synthetic_conversation_generation.py
 src/agents/utils/testbench_prompts.py
 src/agents/utils/langgraph_viz.py
 
 # development agents
+src/agents/base_agent/
 src/agents/student_agent/
 src/agents/development_agents/
-src/agents/google_learnML_agent/
+src/agents/google_learnLM_agent/

index.py

@@ -8,7 +8,7 @@ def handler(event, context):
     """
     Lambda handler function
     """
-    # Log the input event
+    # Log the input event TODO: remove this line
     print("Received event:", json.dumps(event, indent=2))
 
     if "message" not in event:

index_test.py

@@ -31,7 +31,7 @@ def test_missing_argument(self):
         for arg in arguments:
             event = {
                 "message": "Hello, World",
-                "params": {"conversation_id": "1234Test"}
+                "params": {"conversation_id": "1234Test", "conversation_history": [{"type": "user", "content": "Hello, World"}]}
             }
             event.pop(arg)
 
@@ -42,7 +42,7 @@ def test_missing_argument(self):
     def test_correct_arguments(self):
         event = {
             "message": "Hello, World",
-            "params": {"conversation_id": "1234Test"}
+            "params": {"conversation_id": "1234Test", "conversation_history": [{"type": "user", "content": "Hello, World"}]}
         }
 
         result = handler(event, None)
@@ -52,7 +52,7 @@ def test_correct_arguments(self):
     def test_correct_response(self):
         event = {
             "message": "Hello, World",
-            "params": {"conversation_id": "1234Test"}
+            "params": {"conversation_id": "1234Test", "conversation_history": [{"type": "user", "content": "Hello, World"}]}
         }
 
         result = handler(event, None)

src/agents/base_agent/base_agent.py

@@ -1,12 +1,12 @@
 try:
     from ..llm_factory import OpenAILLMs
     from .base_prompts import \
-        role_prompt, conv_pref_prompt, update_conv_pref_prompt, summary_prompt, update_summary_prompt
+        role_prompt, conv_pref_prompt, update_conv_pref_prompt, summary_prompt, update_summary_prompt, summary_system_prompt
     from ..utils.types import InvokeAgentResponseType
 except ImportError:
     from src.agents.llm_factory import OpenAILLMs
     from src.agents.base_agent.base_prompts import \
-        role_prompt, conv_pref_prompt, update_conv_pref_prompt, summary_prompt, update_summary_prompt
+        role_prompt, conv_pref_prompt, update_conv_pref_prompt, summary_prompt, update_summary_prompt, summary_system_prompt
     from src.agents.utils.types import InvokeAgentResponseType
 
 from langgraph.graph import StateGraph, START, END
@@ -70,7 +70,7 @@ def call_model(self, state: State, config: RunnableConfig) -> str:
         summary = state.get("summary", "")
         conversationalStyle = state.get("conversationalStyle", "")
         if summary:
-            system_message += f"## Summary of conversation earlier: {summary} \n\n"
+            system_message += summary_system_prompt.format(summary=summary)
         if conversationalStyle:
             system_message += f"## Known conversational style and preferences of the student for this conversation: {conversationalStyle}. \n\nYour answer must be in line with this conversational style."
 
@@ -144,6 +144,8 @@ def should_summarize(self, state: State) -> str:
         messages = state["messages"]
         valid_messages = self.check_for_valid_messages(messages)
         nr_messages = len(valid_messages)
+        if len(valid_messages) == 0:
+            raise Exception("Internal Error: No valid messages found in the conversation history. Conversation history might be empty.")
         if "system" in valid_messages[-1].type:
             nr_messages -= 1
 
@@ -185,7 +187,7 @@ def invoke_base_agent(query: str, conversation_history: list, summary: str, conv
     print(f'in invoke_base_agent(), query = {query}, thread_id = {session_id}')
 
     config = {"configurable": {"thread_id": session_id, "summary": summary, "conversational_style": conversationalStyle, "question_response_details": question_response_details}}
-    response_events = agent.app.invoke({"messages": conversation_history + [HumanMessage(content=query)]}, config=config, stream_mode="values") #updates
+    response_events = agent.app.invoke({"messages": conversation_history, "summary": summary, "conversational_style": conversationalStyle}, config=config, stream_mode="values") #updates
     pretty_printed_response = agent.pretty_response_value(response_events) # get last event/ai answer in the response
 
     # Gather Metadata from the agent

src/agents/base_agent/base_prompts.py

@@ -62,17 +62,20 @@
 {pref_guidelines}
 """
 
-summary_prompt = """
-You are an AI assistant specializing in concise and accurate summarization. Your task is to summarize the previous conversation, capturing the main topics, key points, user questions, and your responses in a clear and organized format. 
-
-Ensure the summary is:
+summary_guidelines = """Ensure the summary is:
 
 Concise: Keep the summary brief while including all essential information.
-Structured: Organize the summary into sections such as 'Topics Discussed,' 'Key Questions and Responses,' and 'Follow-Up Suggestions' if applicable.
+Structured: Organize the summary into sections such as 'Topics Discussed' and 'Top 3 Key Detailed Ideas'.
 Neutral and Accurate: Avoid adding interpretations or opinions; focus only on the content shared.
 When summarizing: If the conversation is technical, highlight significant concepts, solutions, and terminology. If context involves problem-solving, detail the problem and the steps or solutions provided. If the user asks for creative input, briefly describe the ideas presented.
+Last messages: Include the most recent 4 messages to provide context for the summary.
 
-Provide the summary in a bulleted format for clarity. Avoid redundant details while preserving the core intent of the discussion.
-"""
+Provide the summary in a bulleted format for clarity. Avoid redundant details while preserving the core intent of the discussion."""
+
+summary_prompt = f"""Summarize the conversation between a student and a tutor. Your summary should highlight the major topics discussed during the session, followed by a detailed recollection of the last five significant points or ideas. Ensure the summary flows smoothly to maintain the continuity of the discussion."""
+
+update_summary_prompt = f"""Update the summary by taking into account the new messages above.
+
+{summary_guidelines}"""
 
-update_summary_prompt = "Update the summary by taking into account the new messages above:"
+summary_system_prompt = "You are continuing a tutoring session with the student. Background context: {summary}. Use this context to inform your understanding but do not explicitly restate, refer to, or incorporate the details directly in your responses unless the user brings them up. Respond naturally to the user's current input, assuming prior knowledge from the summary."

src/agents/google_learnLM_agent/google_learnLM_agent.py

@@ -1,12 +1,12 @@
 try:
     from ..llm_factory import GoogleAILLMs
     from .google_learnLM_prompts import \
-        role_prompt, conv_pref_prompt, update_conv_pref_prompt, summary_prompt, update_summary_prompt
+        role_prompt, conv_pref_prompt, update_conv_pref_prompt, summary_prompt, update_summary_prompt, summary_system_prompt
     from ..utils.types import InvokeAgentResponseType
 except ImportError:
     from src.agents.llm_factory import GoogleAILLMs
     from src.agents.google_learnLM_agent.google_learnLM_prompts import \
-        role_prompt, conv_pref_prompt, update_conv_pref_prompt, summary_prompt, update_summary_prompt
+        role_prompt, conv_pref_prompt, update_conv_pref_prompt, summary_prompt, update_summary_prompt, summary_system_prompt
     from src.agents.utils.types import InvokeAgentResponseType
 
 from langgraph.graph import StateGraph, START, END
@@ -74,7 +74,7 @@ def call_model(self, state: State, config: RunnableConfig) -> str:
         summary = state.get("summary", "")
         conversationalStyle = state.get("conversationalStyle", "")
         if summary:
-            system_message += f"## Summary of conversation earlier: {summary} \n\n"
+            system_message += summary_system_prompt.format(summary=summary)
         if conversationalStyle:
             system_message += f"## Known conversational style and preferences of the student for this conversation: {conversationalStyle}. \n\nYour answer must be in line with this conversational style."
 
@@ -148,6 +148,8 @@ def should_summarize(self, state: State) -> str:
         messages = state["messages"]
         valid_messages = self.check_for_valid_messages(messages)
         nr_messages = len(valid_messages)
+        if len(valid_messages) == 0:
+            raise Exception("Internal Error: No valid messages found in the conversation history. Conversation history might be empty.")
         if "system" in valid_messages[-1].type:
             nr_messages -= 1
 
@@ -189,7 +191,7 @@ def invoke_google_learnlm_agent(query: str, conversation_history: list, summary:
     print(f'in invoke_google_learnlm_agent(), query = {query}, thread_id = {session_id}')
 
     config = {"configurable": {"thread_id": session_id, "summary": summary, "conversational_style": conversationalStyle, "question_response_details": question_response_details}}
-    response_events = agent.app.invoke({"messages": conversation_history + [HumanMessage(content=query)]}, config=config, stream_mode="values") #updates
+    response_events = agent.app.invoke({"messages": conversation_history, "summary": summary, "conversational_style": conversationalStyle}, config=config, stream_mode="values") #updates
     pretty_printed_response = agent.pretty_response_value(response_events) # get last event/ai answer in the response
 
     # Gather Metadata from the agent

src/agents/google_learnLM_agent/google_learnLM_prompts.py

@@ -62,17 +62,20 @@
 {pref_guidelines}
 """
 
-summary_prompt = """
-You are an AI assistant specializing in concise and accurate summarization. Your task is to summarize the previous conversation, capturing the main topics, key points, user questions, and your responses in a clear and organized format. 
-
-Ensure the summary is:
+summary_guidelines = """Ensure the summary is:
 
 Concise: Keep the summary brief while including all essential information.
-Structured: Organize the summary into sections such as 'Topics Discussed,' 'Key Questions and Responses,' and 'Follow-Up Suggestions' if applicable.
+Structured: Organize the summary into sections such as 'Topics Discussed' and 'Top 3 Key Detailed Ideas'.
 Neutral and Accurate: Avoid adding interpretations or opinions; focus only on the content shared.
 When summarizing: If the conversation is technical, highlight significant concepts, solutions, and terminology. If context involves problem-solving, detail the problem and the steps or solutions provided. If the user asks for creative input, briefly describe the ideas presented.
+Last messages: Include the most recent 4 messages to provide context for the summary.
 
-Provide the summary in a bulleted format for clarity. Avoid redundant details while preserving the core intent of the discussion.
-"""
+Provide the summary in a bulleted format for clarity. Avoid redundant details while preserving the core intent of the discussion."""
+
+summary_prompt = f"""Summarize the conversation between a student and a tutor. Your summary should highlight the major topics discussed during the session, followed by a detailed recollection of the last five significant points or ideas. Ensure the summary flows smoothly to maintain the continuity of the discussion."""
+
+update_summary_prompt = f"""Update the summary by taking into account the new messages above.
+
+{summary_guidelines}"""
 
-update_summary_prompt = "Update the summary by taking into account the new messages above:"
+summary_system_prompt = "You are continuing a tutoring session with the student. Background context: {summary}. Use this context to inform your understanding but do not explicitly restate, refer to, or incorporate the details directly in your responses unless the user brings them up. Respond naturally to the user's current input, assuming prior knowledge from the summary."

src/agents/informational_agent/informational_agent.py

@@ -1,12 +1,12 @@
 try:
     from ..llm_factory import OpenAILLMs
     from .informational_prompts import \
-        informational_role_prompt, conv_pref_prompt, update_conv_pref_prompt, summary_prompt, update_summary_prompt
+        informational_role_prompt, conv_pref_prompt, update_conv_pref_prompt, summary_prompt, update_summary_prompt, summary_system_prompt
     from ..utils.types import InvokeAgentResponseType
 except ImportError:
     from src.agents.llm_factory import OpenAILLMs
     from src.agents.informational_agent.informational_prompts import \
-        informational_role_prompt, conv_pref_prompt, update_conv_pref_prompt, summary_prompt, update_summary_prompt
+        informational_role_prompt, conv_pref_prompt, update_conv_pref_prompt, summary_prompt, update_summary_prompt, summary_system_prompt
     from src.agents.utils.types import InvokeAgentResponseType
 
 from langgraph.graph import StateGraph, START, END
@@ -25,6 +25,8 @@
 - [informational_role_prompt]   role of a tutor to answer student's ALL questions on any topic  
 """
 
+# TODO: return/uncomment improved conversational style use & analysis
+
 ValidMessageTypes: TypeAlias = SystemMessage | HumanMessage | AIMessage
 AllMessageTypes: TypeAlias = ValidMessageTypes | RemoveMessage
 
@@ -70,9 +72,9 @@ def call_model(self, state: State, config: RunnableConfig) -> str:
         summary = state.get("summary", "")
         conversationalStyle = state.get("conversationalStyle", "")
         if summary:
-            system_message += f"## Summary of conversation earlier: {summary} \n\n"
-        if conversationalStyle:
-            system_message += f"## Known conversational style and preferences of the student for this conversation: {conversationalStyle}. \n\nYour answer must be in line with this conversational style."
+            system_message += summary_system_prompt.format(summary=summary)
+        # if conversationalStyle:
+        #     system_message += f"## Known conversational style and preferences of the student for this conversation: {conversationalStyle}. \n\nYour answer must be in line with this conversational style."
 
         messages = [SystemMessage(content=system_message)] + state['messages']
 
@@ -99,7 +101,7 @@ def summarize_conversation(self, state: State, config: RunnableConfig) -> dict:
 
         summary = state.get("summary", "")
         previous_summary = config["configurable"].get("summary", "")
-        previous_conversationalStyle = config["configurable"].get("conversational_style", "")
+        # previous_conversationalStyle = config["configurable"].get("conversational_style", "")
         if previous_summary:
             summary = previous_summary
 
@@ -111,28 +113,29 @@ def summarize_conversation(self, state: State, config: RunnableConfig) -> dict:
         else:
             summary_message = self.summary_prompt
 
-        if previous_conversationalStyle:
-            conversationalStyle_message = (
-                f"This is the previous conversational style of the student for this conversation: {previous_conversationalStyle}\n\n" +
-                self.update_conversation_preference_prompt
-            )
-        else:
-            conversationalStyle_message = self.conversation_preference_prompt
+        # if previous_conversationalStyle:
+        #     conversationalStyle_message = (
+        #         f"This is the previous conversational style of the student for this conversation: {previous_conversationalStyle}\n\n" +
+        #         self.update_conversation_preference_prompt
+        #     )
+        # else:
+        #     conversationalStyle_message = self.conversation_preference_prompt
 
         # STEP 1: Summarize the conversation
         messages = state["messages"][:-1] + [SystemMessage(content=summary_message)] 
         valid_messages = self.check_for_valid_messages(messages)
         summary_response = self.summarisation_llm.invoke(valid_messages)
 
         # STEP 2: Analyze the conversational style
-        messages = state["messages"][:-1] + [SystemMessage(content=conversationalStyle_message)]
-        valid_messages = self.check_for_valid_messages(messages)
-        conversationalStyle_response = self.summarisation_llm.invoke(valid_messages)
+        # messages = state["messages"][:-1] + [SystemMessage(content=conversationalStyle_message)]
+        # valid_messages = self.check_for_valid_messages(messages)
+        # conversationalStyle_response = self.summarisation_llm.invoke(valid_messages)
 
         # Delete messages that are no longer wanted, except the last ones
-        delete_messages: list[AllMessageTypes] = [RemoveMessage(id=m.id) for m in state["messages"][:-3]]
+        delete_messages: list[AllMessageTypes] = [RemoveMessage(id=m.id) for m in state["messages"][:-5]]
 
-        return {"summary": summary_response.content, "conversationalStyle": conversationalStyle_response.content, "messages": delete_messages}
+        # return {"summary": summary_response.content, "conversationalStyle": conversationalStyle_response.content, "messages": delete_messages}
+        return {"summary": summary_response.content, "messages": delete_messages}
 
     def should_summarize(self, state: State) -> str:
         """
@@ -144,6 +147,8 @@ def should_summarize(self, state: State) -> str:
         messages = state["messages"]
         valid_messages = self.check_for_valid_messages(messages)
         nr_messages = len(valid_messages)
+        if len(valid_messages) == 0:
+            raise Exception("Internal Error: No valid messages found in the conversation history. Conversation history might be empty.")
         if "system" in valid_messages[-1].type:
             nr_messages -= 1
 
@@ -181,7 +186,7 @@ def invoke_informational_agent(query: str, conversation_history: list, summary:
     print(f'in invoke_informational_agent(), query = {query}, thread_id = {session_id}')
 
     config = {"configurable": {"thread_id": session_id, "summary": summary, "conversational_style": conversationalStyle, "question_response_details": question_response_details}}
-    response_events = agent.app.invoke({"messages": conversation_history + [HumanMessage(content=query)]}, config=config, stream_mode="values") #updates
+    response_events = agent.app.invoke({"messages": conversation_history, "summary": summary, "conversational_style": conversationalStyle}, config=config, stream_mode="values") #updates
     pretty_printed_response = agent.pretty_response_value(response_events) # get last event/ai answer in the response
 
     # Gather Metadata from the agent