feat: Add content map creator from XML sitemap

.gitignore

@@ -1,3 +1,6 @@
+contentmap.db
+
+
  Byte-compiled / optimized / DLL files
 __pycache__/
 *.py[cod]

contentmap/core.py

@@ -1,21 +1,43 @@
 from typing import List, Dict, TypedDict
+from datetime import datetime
 import sqlite3
+import importlib.metadata
 
 
 class ContentMapCreator:
 
-    def __init__(self, contents: List[Dict[str, str]]):
+    def __init__(
+            self,
+            contents: List[Dict[str, str]],
+            database: str = "contentmap.db"
+    ):
         self.contents = contents
-        self.db = sqlite3.connect("contentmap.db")
+        self.db = sqlite3.connect(database)
         self.db.row_factory = sqlite3.Row
         self.cursor = self.db.cursor()
 
+    def init_db(self):
+        self.cursor.execute("CREATE TABLE IF NOT EXISTS content (url, content)")
+        self.cursor.execute("CREATE TABLE IF NOT EXISTS config (cat, value)")
+        self.db.commit()
+
+    def add_config(self):
+        data = [
+            {"Version:": "1"},
+            {"Generated with:": "Contentmap lib"},
+            {"Date:": datetime.now().strftime("%Y-%m-%d %H:%M:%S")},
+            {"Embeddings:": "mistral-embed"},
+            {"FTSE:": None},
+        ]
+        data = [{"cat": k, "value": v} for row in data for k, v in row.items()]
+        self.cursor.executemany("INSERT INTO config VALUES (:cat, :value)", data)
+        self.db.commit()
+
     def build(self):
-        self.cursor.execute("CREATE TABLE contentmap (url, content)")
+        self.init_db()
+        self.add_config()
         self.cursor.executemany(
-            "INSERT INTO contentmap VALUES (:url, :content)",
+            "INSERT INTO content VALUES (:url, :content)",
             self.contents
         )
-
-    def test_release_dummy(self):
-        return 1
+        self.db.commit()

contentmap/sitemap.py

@@ -0,0 +1,57 @@
+import asyncio
+import logging
+import requests
+
+import aiohttp
+import trafilatura
+from tqdm.asyncio import tqdm_asyncio
+from lxml import etree
+
+from contentmap.core import ContentMapCreator
+
+
+class SitemapToContentDatabase:
+
+    def __init__(self, sitemap_url, seconds_timeout=10, concurrency=None):
+        self.sitemap_url = sitemap_url
+        self.semaphore = asyncio.Semaphore(concurrency) if concurrency is not None else None
+        self.timeout = aiohttp.ClientTimeout(
+            sock_connect=seconds_timeout,
+            sock_read=seconds_timeout
+        )
+
+    def load(self):
+        urls = self.get_urls()
+        loop = asyncio.get_event_loop()
+        contents = loop.run_until_complete(self.get_contents(urls))
+        cm = ContentMapCreator(contents)
+        cm.build()
+
+    def get_urls(self):
+        r = requests.get(self.sitemap_url)
+        tree = etree.fromstring(r.content)
+        urls = [
+            url.text for url
+            in tree.findall(".//{http://www.sitemaps.org/schemas/sitemap/0.9}loc")
+        ]
+        return urls
+
+    async def get_contents(self, urls):
+        async with aiohttp.ClientSession(timeout=self.timeout) as session:
+            tasks = [self.fetch_content(session, url) for url in urls]
+            return await tqdm_asyncio.gather(*tasks)
+
+    async def fetch_content(self, session, url):
+        try:
+            if not self.semaphore:
+                async with session.get(url) as response:
+                    raw = await response.text()
+            else:
+                async with self.semaphore, session.get(url) as response:
+                    raw = await response.text()
+            content = trafilatura.extract(raw)
+            return {"url": url, "content": content}
+
+        except aiohttp.ClientConnectionError as e:
+            logging.error(f"Error while fetching {url}: {e.__repr__}")
+            return None