Testbench for better prompts & simplified prompt

.gitignore

@@ -135,4 +135,5 @@ dmypy.json
 
 # Synthetic data conversations
 src/agents/utils/synthetic_conversations/*.json
-src/agents/utils/synthetic_conversations/*.csv
+src/agents/utils/synthetic_conversations/*.csv
+src/agents/utils/synthetic_conversations/*.tsv

src/agents/informational_agent/informational_agent.py

@@ -36,7 +36,7 @@ class State(TypedDict):
     conversationalStyle: str
 
 class InformationalAgent:
-    def __init__(self):
+    def __init__(self, informational_role_prompt: str = informational_role_prompt, conv_pref_prompt: str = conv_pref_prompt, update_conv_pref_prompt: str = update_conv_pref_prompt, summary_prompt: str = summary_prompt, update_summary_prompt: str = update_summary_prompt):
         llm = GoogleAILLMs()
         self.llm = llm.get_llm()
         summarisation_llm = OpenAILLMs()
@@ -181,7 +181,7 @@ def pretty_response_value(self, event: dict) -> str:
         return event["messages"][-1].content
 
 agent = InformationalAgent()
-def invoke_informational_agent(query: str, conversation_history: list, summary: str, conversationalStyle: str, question_response_details: str, session_id: str) -> InvokeAgentResponseType:
+def invoke_informational_agent(query: str, conversation_history: list, summary: str, conversationalStyle: str, question_response_details: str, session_id: str, agent: InformationalAgent = agent) -> InvokeAgentResponseType:
     print(f'in invoke_informational_agent(), thread_id = {session_id}')
 
     config = {"configurable": {"thread_id": session_id, "summary": summary, "conversational_style": conversationalStyle, "question_response_details": question_response_details}}

src/agents/informational_agent/informational_prompts.py

@@ -4,36 +4,30 @@
 
 # informational_role_prompt = "You are an excellent tutor that aims to provide clear and concise explanations to students. I am the student. Your task is to answer my questions and provide guidance on the topic discussed. Ensure your responses are accurate, informative, and tailored to my level of understanding and conversational preferences. If I seem to be struggling or am frustrated, refer to my progress so far and the time I spent on the question vs the expected guidance. You do not need to end your messages with a concluding statement.\n\n"
 
+# TODO: try using "A tutor and a student work together to solve the following math word problem."
 informational_role_prompt = """You are a highly skilled and patient AI tutor designed to assist me, the student, in discovering answers and mastering concepts. Your teaching style emphasizes student-centric learning, encouraging deep thinking, active engagement, and confidence building.
 
 ## Teaching Methods:
 Step-by-Step Learning: Break complex problems into smaller, manageable parts, solving one step at a time. Avoid giving the final answer upfront; instead, offer hints or intermediate steps to nudge the student toward the solution. Provide the full answer only when it’s clear the student needs it to move forward. If the student explicitly asks for the answer, direct them to the worked solutions or answer provided below, while encouraging them to engage with the chat for deeper understanding.
 Error Analysis: Treat mistakes as learning opportunities by helping students reflect on why they occurred and how to address them.
 Active Participation: Encourage students to take an active role in solving problems, providing guidance without overtaking their learning process.
-Tailored Feedback: Adapt your explanations, questions, and support to the student's level, needs, and progress. If the student is close to the solution, provide encouragement or subtle hints. If they seem stuck, gradually increase the specificity of your support.
 
 ## Key Qualities:
-Awareness: Use the known learning materials to base your responses on.
+Awareness: Use the known learning materials to base your responses on. Refer back to the learning materials when necessary, but do not repeat them verbatim. Instead, summarize or paraphrase the content to ensure understanding.
 Patience: Allow students ample time to think, process, and respond without rushing them.
 Clarity: Simplify complex ideas into clear, actionable steps.
 Encouragement: Celebrate student efforts and achievements to keep motivation high.
 Adaptability: Customize teaching approaches based on the student's learning preferences and evolving needs.
 Curiosity-Building: Inspire students to ask thoughtful questions, fostering a love for learning.
 Consistency: Reinforce concepts regularly to build lasting understanding.
-Conversation Flow:
-Frequently conclude interactions with a question to keep the dialogue active and gauge the student's comprehension and comfort with the material.
-Continuously adapt to the student's problem-solving style, preferred level of guidance, and feedback.
-
-Example Conversation Style:
-
-If the student asks, "How do I solve this equation?" respond with:
-"Let's start by identifying what you know. What operation do you think comes first?"
-Follow up with guided hints or clarifications based on their response.
+Authenticity: Maintain authenticity and a constructive learning environment by keeping feedback clear and focused.
+Conversation Flow: Frequently conclude interactions with a question to keep the dialogue active and gauge the student's comprehension and comfort with the material.
+Tailored Feedback: Adapt your explanations, questions, and support to the student's level, needs, and progress. If the student is close to the solution, provide encouragement or subtle hints. If the student seems stuck, reflect on their progress and the time spent on the topic, offering the expected guidance. Then gradually increase the specificity of your support.
 
 ## Flexibility:
-Restrict your response's length to quickly resolve the student's query. However, adjust your approach dynamically, if the student seeks detailed guidance, prefers a hands-off approach, or demonstrates unique problem-solving strategies. If the student struggles or seems frustrated, reflect on their progress and the time spent on the topic, offering the expected guidance. If the student asks about an irrelevant topic, politely redirect them back to the topic. Do not end your responses with a concluding statement.
+Directly answer the student's question. Keep your answer short. If the student asks about an irrelevant topic, politely redirect them back to the topic. Do not end your responses with a concluding statement.
 
-## Governance
+## Governance:
 You are a chatbot deployed in Lambda Feedback, an online self-study platform. You are discussing with students from Imperial College London."""
 
 pref_guidelines = """**Guidelines:**

src/agents/utils/parse_json_to_prompt.py

@@ -163,7 +163,7 @@ def format_response_area_details(responseArea: ResponseAreaDetails, studentSumma
         return f"""
         ## Response Area: {responseArea.position + 1}
         {f'Area task: What is {responseArea.preResponseText} ?' if responseArea.preResponseText else ''}
-        (Secret - not to be shared) Expected Answer: {responseArea.answer};
+        (Keep it Secret) Expected Answer: {responseArea.answer};
         {submissionDetails}"""
 
     def format_part_details(part: PartDetails, currentPart: CurrentPart, summary: List[StudentWorkResponseArea]) -> str:

src/agents/utils/prompt_value_analysis.py

@@ -0,0 +1,106 @@
+import networkx as nx
+import matplotlib.pyplot as plt
+import numpy as np
+from sentence_transformers import SentenceTransformer
+from sklearn.metrics.pairwise import cosine_similarity
+import pandas as pd
+from matplotlib import cm
+from matplotlib.lines import Line2D 
+
+# Load paragraphs from CSV
+path = "src/agents/utils/synthetic_conversations/"
+df = pd.read_csv(path+"prompts_importance.tsv", delimiter="\t")  # Replace with your actual file name
+print(df.columns)
+df["response"] = df["response"].astype(str).str.replace("$$", "", regex=False)
+df["response"] = df["response"].astype(str).str.replace("\\", "", regex=False)
+paragraphs = df["response"].tolist()  
+messages = df["message"].tolist()
+prompts = df["prompt"].tolist()
+missing_prompts = df["prompt_missing"].tolist()
+print(f"Loaded {len(paragraphs)} paragraphs, {len(messages)} messages, {len(prompts)} prompts, and {len(missing_prompts)} missing prompts")
+
+# Load embedding model
+model = SentenceTransformer("sentence-transformers/all-MiniLM-L6-v2")
+
+# Compute embeddings
+embeddings = model.encode(paragraphs, convert_to_numpy=True)
+
+# Compute similarity matrix
+similarity_matrix = cosine_similarity(embeddings)
+
+# Create graph
+G = nx.Graph()
+
+# Create a mapping of messages to colours
+message_to_color = {msg: cm.viridis(i / len(set(messages))) for i, msg in enumerate(set(messages))}
+
+# Add nodes with message-based colours
+node_colors = []
+for i, paragraph in enumerate(paragraphs):
+    msg = messages[i]
+    color = message_to_color[msg]  # Get colour based on message
+    G.add_node(i, text=paragraph, message=msg, color=color, prompt=prompts[i], missing_prompt=missing_prompts[i])  
+    node_colors.append(color)  # Add node colour for visualization
+
+# Define a similarity threshold for edges
+threshold = 0.5
+for i in range(len(paragraphs)):
+    for j in range(i + 1, len(paragraphs)):
+        if similarity_matrix[i, j] > threshold:
+            G.add_edge(i, j, weight=similarity_matrix[i, j])
+
+# Draw graph
+fig, ax = plt.subplots(figsize=(12, 6))
+pos = nx.spring_layout(G)  # Positioning of nodes
+nx.draw(G, pos, with_labels=False, node_color=node_colors, edge_color="white", ax=ax)
+
+# Create annotation for hover effect
+hover_text = ax.text(0.5, -0.1, "", transform=ax.transAxes, ha="center", va="top", fontsize=10, wrap=True)
+hover_text.set_visible(False)
+
+# Function to update hover text and wrap it
+def update_hover_text(ind):
+    node_idx = ind["ind"][0]
+    node_pos = pos[node_idx]
+    hover_text.set_position((0.5, -0.05))  # Position the text box at the bottom
+    hover_text.set_text("Message: "+ G.nodes[node_idx]["message"]+ "\nResponse: "+ G.nodes[node_idx]["text"])  # Set the text
+    hover_text.set_visible(True)
+    plt.draw()
+
+# Mouse hover event
+def hover(event):
+    if event.inaxes == ax:
+        for i, (x, y) in pos.items():
+            if np.linalg.norm([x - event.xdata, y - event.ydata]) < 0.05:  # Adjust hover sensitivity
+                update_hover_text({"ind": [i]})
+                return
+    hover_text.set_visible(False)  # Hide text when not hovering over nodes
+    plt.draw()
+
+# Mouse click event
+def on_click(event):
+    if event.inaxes == ax:
+        for i, (x, y) in pos.items():
+            if np.linalg.norm([x - event.xdata, y - event.ydata]) < 0.05:  # Click sensitivity
+                node_idx = i
+                message = G.nodes[node_idx]["message"]
+                prompt = G.nodes[node_idx]["prompt"]
+                missing_prompt = G.nodes[node_idx]["missing_prompt"]
+                text = G.nodes[node_idx]["text"]
+                print(f"Clicked node {node_idx} \n-- Message: {message}\n-- Response: {text}\n-- Prompt: {prompt}\n-- Missing Prompt: {missing_prompt}")
+                print("====================")
+
+# Create legend
+legend_handles = []
+for msg, color in message_to_color.items():
+    legend_handles.append(Line2D([0], [0], marker='o', color='w', markerfacecolor=color, markersize=10, label=msg))
+
+ax.legend(handles=legend_handles, title="Messages", bbox_to_anchor=(0.3, 0.0), loc='lower center', 
+          borderaxespad=1, ncol=1, fontsize=10, columnspacing=1, frameon=False)
+
+# Connect events
+fig.canvas.mpl_connect("motion_notify_event", hover)
+fig.canvas.mpl_connect("button_press_event", on_click)
+
+plt.subplots_adjust(bottom=0.2)  # Add space for the bottom bar
+plt.show()