feat: Add asynchronous document enhancement method and adjust minimum page content length

Author: a-klos

admin-api-lib/src/admin_api_lib/impl/api_endpoints/default_confluence_loader.py

@@ -83,7 +83,6 @@ def __init__(
         self._extractor_api = extractor_api
         self._rag_api = rag_api
         self._settings = settings
-        self._sanitize_document_name()
         self._key_value_store = key_value_store
         self._information_mapper = information_mapper
         self._information_enhancer = information_enhancer
@@ -157,6 +156,13 @@ async def process_confluence(index):
         for t in threads:
             t.join()
 
+    async def _aenhance_langchain_documents(self, documents: list[Document]):
+        try:
+            return await self._information_enhancer.ainvoke(documents)
+        except Exception as e:
+            logger.error("Exception occured while enhancing confluence langchain document %s" % e)
+            raise e
+
     async def _delete_previous_information_pieces(self, index=0):
         try:
             await self._document_deleter.adelete_document(self._settings.document_name[index])

extractor-api-lib/src/extractor_api_lib/impl/api_endpoints/default_confluence_extractor.py

@@ -13,7 +13,7 @@
 class DefaultConfluenceExtractor(ConfluenceExtractor):
     """Default implementation of the FileExtractor interface."""
 
-    MIN_PAGE_CONTENT_LENGTH = 20
+    MIN_PAGE_CONTENT_LENGTH = 10
 
     def __init__(
         self,