Some renames

Author: pamelafox

README.md

@@ -16,6 +16,14 @@ Plus these scripts to demonstrate additional features:
 * [`chat_langchain.py`](./chat_langchain.py): Uses the langchain SDK to generate chat completions. [Learn more from Langchain docs](https://python.langchain.com/docs/get_started/quickstart)
 * [`chat_llamaindex.py`](./chat_llamaindex.py): Uses the LlamaIndex SDK to generate chat completions. [Learn more from LlamaIndex docs](https://docs.llamaindex.ai/en/stable/)
 
+These scripts for RAG:
+
+* [`rag_csv.py`](./rag.py): Retrieves matching results from a CSV file and uses them to answer user's question.
+* [`rag_multiturn.py`](./rag_multiturn.py): The same idea, but with a back-and-forth chat interface using `input()` which keeps track of past messages and sends them with each chat completion call.
+* [`rag_queryrewrite.py`](./rag_queryrewrite.py): Adds a query rewriting step to the RAG process, where the user's question is rewritten to improve the retrieval results.
+* [`rag_documents_ingestion.py`](./rag_ingestion.py): Ingests PDFs by using pymupdf to convert to markdown, then using Langchain to split into chunks, then using OpenAI to embed the chunks, and finally storing in a local JSON file.
+* [`rag_documents_flow.py`](./rag_pdfs.py): A RAG flow that retrieves matching results from the local JSON file created by `rag_documents_ingestion.py`.
+
 ## Setting up the environment
 
 If you open this up in a Dev Container or GitHub Codespaces, everything will be setup for you.

rag.py renamed to rag_csv.py

@@ -33,35 +33,29 @@
     MODEL_NAME = os.environ["OPENAI_MODEL"]
 
 
-def search(query):
-    # Open the CSV and store in a list
-    with open("hybrid.csv") as file:
-        reader = csv.reader(file)
-        rows = list(reader)
-
-    # Normalize the user question to replace punctuation and make lowercase
-    normalized_message = query.lower().replace("?", "").replace("(", " ").replace(")", " ")
-    # Search the CSV for user question using very naive search
-    words = normalized_message.split()
-    matching_rows = []
-    for row in rows[1:]:
-        # if the word matches any word in row, add the row to the matches
-        if any(word in row[0].lower().split() for word in words) or any(
-            word in row[5].lower().split() for word in words
-        ):
-            matching_rows.append(row)
-    # Format as a markdown table, since language models understand markdown
-    matches_table = " | ".join(rows[0]) + "\n" + " | ".join(" --- " for _ in range(len(rows[0]))) + "\n"
-    matches_table += "\n".join(" | ".join(row) for row in matches)
-    return matches_table
-
-
-user_question = "how fast is the prius v?"
-
-matches = search(user_question)
-
-print("Found matches:")
-print(matches)
+USER_MESSAGE = "how fast is the prius v?"
+
+# Open the CSV and store in a list
+with open("hybrid.csv") as file:
+    reader = csv.reader(file)
+    rows = list(reader)
+
+# Normalize the user question to replace punctuation and make lowercase
+normalized_message = USER_MESSAGE.lower().replace("?", "").replace("(", " ").replace(")", " ")
+
+# Search the CSV for user question using very naive search
+words = normalized_message.split()
+matches = []
+for row in rows[1:]:
+    # if the word matches any word in row, add the row to the matches
+    if any(word in row[0].lower().split() for word in words) or any(word in row[5].lower().split() for word in words):
+        matches.append(row)
+
+# Format as a markdown table, since language models understand markdown
+matches_table = " | ".join(rows[0]) + "\n" + " | ".join(" --- " for _ in range(len(rows[0]))) + "\n"
+matches_table += "\n".join(" | ".join(row) for row in matches)
+print(f"Found {len(matches)} matches:")
+print(matches_table)
 
 # Now we can use the matches to generate a response
 SYSTEM_MESSAGE = """
@@ -74,7 +68,7 @@ def search(query):
     temperature=0.3,
     messages=[
         {"role": "system", "content": SYSTEM_MESSAGE},
-        {"role": "user", "content": user_question + "\nSources: " + matches},
+        {"role": "user", "content": USER_MESSAGE + "\nSources: " + matches_table},
     ],
 )