Merge pull request #18 from Matanga1-2/automate-URLs

added automatic browsing option

src/app.py

@@ -11,7 +11,7 @@
 
 
 class Yad2ScraperApp:
-    def __init__(self, client: Yad2Client, address_matcher: AddressMatcher):
+    def __init__(self, client: Yad2Client, address_matcher: AddressMatcher, search_urls: dict):
         # Initialize database
         self.db = Database()
         self.db.create_tables()
@@ -21,6 +21,7 @@ def __init__(self, client: Yad2Client, address_matcher: AddressMatcher):
         self.client = client
         self.client.saved_items_repo = self.saved_items_repo  # Set the repo on the existing client
         self.address_matcher = address_matcher
+        self.search_urls = search_urls
         self.feed_items = None
 
     def run(self) -> None:
@@ -45,9 +46,10 @@ def _process_menu_choice(self) -> bool:
         print("2. Get feed items")
         print("3. Process current feed")
         print("4. Store saved items")
-        print("5. Exit")
+        print("5. Go to all URLs")
+        print("6. Exit")
 
-        choice = input("\nEnter your choice (1-5): ").strip()
+        choice = input("\nEnter your choice (1-6): ").strip()
 
         if choice == '1':
             self._handle_new_url()
@@ -58,6 +60,8 @@ def _process_menu_choice(self) -> bool:
         elif choice == '4':
             self._handle_store_saved_items()
         elif choice == '5':
+            self._handle_go_to_all_urls()
+        elif choice == '6':
             print("Goodbye!")
             return False
         else:
@@ -98,6 +102,17 @@ def _handle_new_url(self) -> None:
         self.client.navigate_to(url)
         self.feed_items = None
 
+
+    def _handle_go_to_all_urls(self) -> None:
+        self._handle_store_saved_items()
+        for name, url in self.search_urls.items():
+            print(f"Navigating to URL {name}...")
+            self.client.navigate_to(url)
+            self._handle_get_feed()
+            self._handle_process_feed()
+        print("Done going through all URLs!")
+        return False
+
     def _handle_get_feed(self) -> None:
         print("Fetching feed items...")
         self.feed_items = self.client.get_feed_items()

src/main.py

@@ -1,5 +1,6 @@
 #!/usr/bin/env python3
 
+import json
 import logging
 import os
 import signal
@@ -57,8 +58,9 @@ def main():
     try:
         client = Yad2Client(headless=False)
         address_matcher = AddressMatcher(get_resource_path(os.path.join('consts', 'supported_streets.json')))
+        search_urls = json.load(open(get_resource_path(os.path.join('consts', 'search_url.json'))))
 
-        app = Yad2ScraperApp(client, address_matcher)
+        app = Yad2ScraperApp(client, address_matcher, search_urls)
         app.run()
 
     except Exception as e:

tests/test_app.py

@@ -1,4 +1,5 @@
-from unittest.mock import MagicMock
+from unittest.mock import MagicMock, patch
+import json
 
 import pytest
 
@@ -18,8 +19,17 @@ def mock_address_matcher():
     return MagicMock(spec=AddressMatcher)
 
 @pytest.fixture
-def app(mock_client, mock_address_matcher):
-    return Yad2ScraperApp(mock_client, mock_address_matcher)
+def mock_search_urls():
+    """Mock the search URLs loaded from search_url.json."""
+    return {
+        "petach_tikva_1": "https://www.yad2.co.il/realestate/forsale?multiNeighborhood=2001002%2C763%2C765%2C756%2C2001001&propertyGroup=apartments&property=1&rooms=3-4&price=2300000-2600000&floor=1--1&parking=1&elevator=1&balcony=1&shelter=1",
+        "petach_tikva_2": "https://www.yad2.co.il/realestate/forsale?multiNeighborhood=2001002%2C763%2C765%2C756%2C2001001&propertyGroup=apartments&property=1&rooms=3-4&price=2300000-2600000&floor=1--1&parking=1&elevator=1&balcony=1&shelter=1&page=2",
+    }
+
+@pytest.fixture
+def app(mock_client, mock_address_matcher, mock_search_urls):
+    """Create an instance of Yad2ScraperApp with mocked dependencies."""
+    return Yad2ScraperApp(mock_client, mock_address_matcher, mock_search_urls)
 
 def test_handle_store_saved_items_success(app, capsys):
     """Test successful storing of saved items."""