Refactor codebase, update README, add KeyboardInterrupt handling, and fix import issues

Author: meghost77

Diff for: db/database.py

@@ -1,88 +1,49 @@
-from typing import Optional
-
 from motor.motor_asyncio import AsyncIOMotorClient
 
 from config import settings
 
-# Global variables for MongoDB client and database
-client = None
-db = None
-
-# Initialize the database connection
-
-
-async def init_db():
-    global client, db
-    # Create a new MongoDB client
-    client = AsyncIOMotorClient(settings.MONGODB_URI)
-    # Select the database
-    db = client[settings.DB_NAME]
-
-# Close the database connection
-
-
-async def close_db():
-    global client
-    if client:
-        # Close the MongoDB client connection
-        client.close()
-
-# Get a specific collection from the database
-
-
-async def get_collection(name):
-    return db[name]
-
-# Update a single document in the collection
-
-
-async def update_one(collection_name, filter, update, upsert=False):
-    collection = await get_collection(collection_name)
-    # Update a document in the collection with the specified filter and update
-    return await collection.update_one(filter, update, upsert=upsert)
-
-# Find documents in the collection with optional sorting, skipping, and limiting
-
-
-async def find(collection_name, filter={}, sort=None, skip=0, limit=0):
-    collection = await get_collection(collection_name)
-    cursor = collection.find(filter)
-    if sort:
-        cursor = cursor.sort(sort)
-    if skip:
-        cursor = cursor.skip(skip)
-    if limit:
-        cursor = cursor.limit(limit)
-    # Convert the cursor to a list with the specified limit
-    return await cursor.to_list(length=limit)
-
-# Find a single document in the collection
-
-
-async def find_one(collection_name, filter):
-    collection = await get_collection(collection_name)
-    # Find one document that matches the filter
-    return await collection.find_one(filter)
-
-# List all collection names in the database
-
-
-async def list_collection_names():
-    return await db.list_collection_names()
-
-# Count the number of posts matching the query and optional city
-
-
-async def count_posts(collection_name: str):
-    collection = await get_collection(collection_name)
-    return await collection.count_documents({'_id': {'$ne': 'last_update'}})
-
-# Get the latest posts sorted by post date
-
 
-async def get_latest_posts(limit: int):
-    collection = await get_collection('posts')
-    # Find the latest posts sorted by post date in descending order
-    cursor = collection.find().sort([('postDate', -1)]).limit(limit)
-    # Convert the cursor to a list with the specified limit
-    return await cursor.to_list(length=limit)
+class Database:
+    client = None
+    db = None
+
+    @classmethod
+    async def connect(cls):
+        cls.client = AsyncIOMotorClient(settings.MONGODB_URI)
+        cls.db = cls.client[settings.DB_NAME]
+
+    @classmethod
+    async def close(cls):
+        if cls.client:
+            cls.client.close()
+
+    @classmethod
+    async def get_collection(cls, name):
+        return cls.db[name]
+
+    @classmethod
+    async def update_one(cls, collection_name, filter, update, upsert=False):
+        collection = await cls.get_collection(collection_name)
+        return await collection.update_one(filter, update, upsert=upsert)
+
+    @classmethod
+    async def find(cls, collection_name, filter={}, sort=None, skip=0, limit=0):
+        collection = await cls.get_collection(collection_name)
+        cursor = collection.find(filter)
+        if sort:
+            cursor = cursor.sort(sort)
+        return await cursor.skip(skip).limit(limit).to_list(length=limit)
+
+    @classmethod
+    async def find_one(cls, collection_name, filter):
+        collection = await cls.get_collection(collection_name)
+        return await collection.find_one(filter)
+
+    @classmethod
+    async def list_collection_names(cls):
+        return await cls.db.list_collection_names()
+
+    @classmethod
+    async def count_posts(cls, collection_name: str):
+        collection = await cls.get_collection(collection_name)
+        return await collection.count_documents({'_id': {'$ne': 'last_update'}})

Diff for: db/models.py

@@ -3,32 +3,29 @@
 
 from pydantic import BaseModel, Field
 
-# Base model for a post
-
 
 class Price(BaseModel):
     formattedPrice: Optional[str] = Field(
         None, description="Formatted price of the item")
 
 
 class Post(BaseModel):
-    id: Union[str, int]  # ID can be either string or integer
-    bodyHTML: str  # HTML content of the post
-    title: str  # Title of the post
-    URL: str  # URL of the post
-    city: str  # City related to the post
-    postDate: datetime  # Date when the post was created
-    updateDate: datetime  # Date when the post was last updated
-    firstImage: Optional[str] = None  # Optional field for the first image URL
-    commentCount: int  # Number of comments on the post
+    id: Union[str, int]
+    bodyHTML: str
+    title: str
+    URL: str
+    city: str
+    postDate: datetime
+    updateDate: datetime
+    firstImage: Optional[str] = None
+    commentCount: int
     tags: Optional[List[str]] = None
     authorUsername: Optional[str] = None
     price: Optional[Price] = None
-# Detailed model for a post extending the base Post model
 
 
 class PostDetail(Post):
-    bodyHTML: str  # HTML content of the post
+    pass
 
 
 class SearchResponse(BaseModel):

Diff for: main.py

@@ -1,38 +1,35 @@
 import logging
+import signal
+import sys
 from contextlib import asynccontextmanager
 
+import uvicorn
 from apscheduler.schedulers.asyncio import AsyncIOScheduler
 from apscheduler.triggers.interval import IntervalTrigger
 from fastapi import FastAPI
 
-from config import settings  # Import settings from config
-from db.database import close_db, init_db
+from config import settings
+from db.database import Database
 from utils.routes import router
-from utils.services import update_all_collections
+from utils.services import PostService
 
-# Logging Setup
 logging.basicConfig(level=logging.INFO,
                     format='%(asctime)s - %(name)s - %(levelname)s - %(message)s')
 logger = logging.getLogger(__name__)
 
-# Initialize the scheduler for periodic tasks
 scheduler = AsyncIOScheduler()
 
-# Context manager for application lifespan
-
 
 @asynccontextmanager
 async def lifespan(app: FastAPI):
-    # Startup
     logger.info("Initializing database connection...")
-    await init_db()  # Establish database connection
+    await Database.connect()
     logger.info("Database connection initialized.")
 
     logger.info("Starting scheduler...")
-    scheduler.start()  # Start the scheduler
-    # Add a job to update all collections every hour
+    scheduler.start()
     scheduler.add_job(
-        update_all_collections,
+        PostService.update_all_collections,
         IntervalTrigger(hours=1),
         id='periodic_update',
         replace_existing=True
@@ -42,51 +39,34 @@ async def lifespan(app: FastAPI):
     try:
         yield
     finally:
-        # Shutdown
         logger.info("Shutting down scheduler...")
-        scheduler.shutdown()  # Shutdown the scheduler
+        scheduler.shutdown()
         logger.info("Scheduler shut down.")
 
         logger.info("Closing database connection...")
-        await close_db()  # Close database connection
+        await Database.close()
         logger.info("Database connection closed.")
 
-# Create FastAPI app with the custom lifespan context manager
 app = FastAPI(lifespan=lifespan)
-# Include the application routes from the router
 app.include_router(router)
 
-# Data update function with logging
 
+def signal_handler(sig, frame):
+    logger.info("Received shutdown signal. Initiating graceful shutdown...")
+    scheduler.shutdown(wait=False)
+    sys.exit(0)
 
-async def update_all_collections():
-    logger.info("Starting the data collection process...")
-    try:
-        # Logic for data update
-        logger.info("Data collection process completed successfully.")
-    except Exception as e:
-        logger.error(f"An error occurred during data collection: {e}")
 
 if __name__ == "__main__":
-    import signal
-    import sys
-
-    import uvicorn
-
-    # Graceful shutdown handler
-    def handle_exit(*args):
-        logger.info("Shutting down gracefully...")
-        sys.exit(0)
-
-    # Register signal handlers for graceful shutdown
-    for sig in (signal.SIGINT, signal.SIGTERM):
-        signal.signal(sig, handle_exit)
+    signal.signal(signal.SIGINT, signal_handler)
+    signal.signal(signal.SIGTERM, signal_handler)
 
     try:
-        logger.info("Starting the script...")
-        # Run the application using the configured port
+        logger.info("Starting the Haraj Scraper Backend...")
         uvicorn.run(app, host="0.0.0.0", port=settings.PORT)
     except KeyboardInterrupt:
-        handle_exit()
+        logger.info("Keyboard interrupt received. Shutting down...")
+    except Exception as e:
+        logger.error(f"An error occurred: {str(e)}")
     finally:
-        logger.info("Script has been stopped.")
+        logger.info("Application shutdown complete.")