Added pre-flight service test for RAG

FIxed #438

Author: clusterzx

routes/setup.js

@@ -11,6 +11,7 @@ const debugService = require('../services/debugService.js');
 const configFile = require('../config/config.js');
 const ChatService = require('../services/chatService.js');
 const documentsService = require('../services/documentsService.js');
+const RAGService = require('../services/ragService.js');
 const fs = require('fs').promises;
 const path = require('path');
 const jwt = require('jsonwebtoken');
@@ -4226,4 +4227,18 @@ router.get('/api/processing-status', async (req, res) => {
   }
 });
 
+router.get('/api/rag-test', async (req, res) => {
+  RAGService.initialize();
+  try { 
+    if(await RAGService.sendDocumentsToRAGService()){
+      res.status(200).json({ success: true });
+    }else{
+      res.status(500).json({ success: false });
+    }    
+  } catch (error) {
+    res.status(500).json({ error: 'Failed to fetch processing status' });
+  }
+}
+);
+
 module.exports = router;

services/customService.js

@@ -142,7 +142,7 @@ class CustomOpenAIService {
             content: truncatedContent
           }
         ],
-        temperature: 0.3,
+        ...(model !== 'o3-mini' && { temperature: 0.3 }),
       });
 
       // Handle response

services/manualService.js

@@ -87,7 +87,7 @@ class ManualService {
                 content: content
             }
             ],
-            temperature: 0.3,
+            ...(model !== 'o3-mini' && { temperature: 0.3 }),
         });
 
         let jsonContent = response.choices[0].message.content;
@@ -183,7 +183,7 @@ class ManualService {
                     content: content
                 }
                 ],
-                temperature: 0.3,
+                ...(model !== 'o3-mini' && { temperature: 0.3 }),
             });
 
             let jsonContent = response.choices[0].message.content;

services/openaiService.js

@@ -302,7 +302,7 @@ class OpenAIService {
             content: truncatedContent
           }
         ],
-        temperature: 0.3,
+        ...(model !== 'o3-mini' && { temperature: 0.3 }),
       });
 
       // Handle response

services/paperlessService.js

@@ -761,6 +761,55 @@ class PaperlessService {
     }
   }
 
+  async getDocumentsForRAGService () {
+    /**
+     * Get all documents with metadata (title, tags, correspondent, created date and content).
+     * 
+     * @returns    An array of documents with metadata.
+     * @throws     An error if the request fails.
+     * @note       This method is used to get all documents with metadata for further processing 
+     */
+    
+    this.initialize();
+    try {
+      let response;
+      let page = 1;
+      let hasMore = true;
+  
+      while (hasMore) {
+        try {
+          const params = {
+            params: { fields: 'id,title,tags,correspondent,created,content' },
+            page,
+            page_size: 100,  // Maximale Seitengröße für effizientes Laden
+            ordering: 'name'  // Optional: Sortierung nach Namen
+          };
+
+          response = await this.client.get('/documents/', { params });
+
+          if (!response?.data?.results || !Array.isArray(response.data.results)) {
+            console.error(`[DEBUG] Invalid API response on page ${page}`);
+            break;
+          }
+
+          hasMore = response.data.next !== null;
+          page++;
+        
+        } catch (error) {
+          console.error(`[ERROR] fetching documents page ${page}:`, error.message);
+          if (error.response) {
+            console.error('[ERROR] Response status:', error.response.status);
+          }
+          break;
+        }
+      }  
+      return response.data.results;
+    } catch (error) {
+      console.error('[ERROR] fetching documents with metadata:', error.message);
+      return [];
+    }
+  }
+
 
   // Aktualisierte getDocuments Methode
   async getDocuments() {

services/ragService.js

@@ -0,0 +1,75 @@
+//ragService.js
+
+// Used to collect send all documents of paperless-ngx with
+// body: { "title": $TITLE, "content": $CONTENT, "metadata": { "tags": $TAGS, "correspondent": $CORRESPONDENT} }
+// to rag endpoint (http://localhost:5000/api/v1/content) and get the response
+
+const paperlessService = require('./paperlessService');
+
+const axios = require('axios');
+const FormData = require('form-data');
+const fs = require('fs');
+const path = require('path');
+
+class RAGService {
+  constructor() {
+    this.apiUrl = "http://localhost:5000/api/v1";
+    this.apiToken = "test-key-123";
+  }
+
+    initialize() {
+        if (!this.client && this.apiUrl && this.apiToken) {
+        this.client = axios.create({
+            baseURL: this.apiUrl,
+            headers: {
+            'Authorization': `Token ${this.apiToken}`,
+            'Content-Type': 'application/json'
+            }
+        });
+        }
+    }
+
+    async fetchAllDocumentsWithMetaData() {
+        const documents = await paperlessService.getDocumentsForRAGService();
+        return documents;
+    }
+
+    async sendDocumentsToRAGService() {
+        try {
+            const documents = await this.fetchAllDocumentsWithMetaData();
+            for (const document of documents) {
+                const tags = document.tags.join(",");
+                const correspondent = document.correspondent;
+                const title = document.title;
+                const content = document.content;
+                const body = {
+                    title,
+                    content,
+                    metadata: {
+                        tags,
+                        correspondent
+                    }
+                }
+                this.client.post('/content', body)
+                .then(response => {
+                    console.log(response.data);
+                })
+                .catch(error => {
+                    console.error(error);
+                }).finally(() => {
+                    console.log("Document sent to RAG service");
+                });
+            }
+            return true;
+        } catch (error) {
+            console.error("Error initializing RAG service client", error);
+            return false;
+        }       
+    }
+
+
+
+
+}
+
+module.exports = new RAGService();