pamelafox · Mar 18, 2025
diff --git a/‎rag_documents_ingestion.py
+1-1 b/‎rag_documents_ingestion.py
+1-1
@@ -44,7 +44,7 @@
 
     # Split the text into smaller chunks
     text_splitter = RecursiveCharacterTextSplitter.from_tiktoken_encoder(
-        model_name="gpt-4o", chunk_size=500, chunk_overlap=0
+        model_name="gpt-4o", chunk_size=500, chunk_overlap=125
     )
     texts = text_splitter.create_documents([md_text])
     file_chunks = [{"id": f"{filename}-{(i + 1)}", "text": text.page_content} for i, text in enumerate(texts)]
Original file line number	Diff line number	Diff line change
`@@ -44,7 +44,7 @@`
`44`	`44`
`45`	`45`	`# Split the text into smaller chunks`
`46`	`46`	`text_splitter = RecursiveCharacterTextSplitter.from_tiktoken_encoder(`
`47`		`- model_name="gpt-4o", chunk_size=500, chunk_overlap=0`
	`47`	`+ model_name="gpt-4o", chunk_size=500, chunk_overlap=125`
`48`	`48`	`)`
`49`	`49`	`texts = text_splitter.create_documents([md_text])`
`50`	`50`	`file_chunks = [{"id": f"{filename}-{(i + 1)}", "text": text.page_content} for i, text in enumerate(texts)]`