diff --git a/src/app.py b/src/app.py index 1992db2..4aa8574 100644 --- a/src/app.py +++ b/src/app.py @@ -11,7 +11,7 @@ class Yad2ScraperApp: - def __init__(self, client: Yad2Client, address_matcher: AddressMatcher): + def __init__(self, client: Yad2Client, address_matcher: AddressMatcher, search_urls: dict): # Initialize database self.db = Database() self.db.create_tables() @@ -21,6 +21,7 @@ def __init__(self, client: Yad2Client, address_matcher: AddressMatcher): self.client = client self.client.saved_items_repo = self.saved_items_repo # Set the repo on the existing client self.address_matcher = address_matcher + self.search_urls = search_urls self.feed_items = None def run(self) -> None: @@ -45,9 +46,10 @@ def _process_menu_choice(self) -> bool: print("2. Get feed items") print("3. Process current feed") print("4. Store saved items") - print("5. Exit") + print("5. Go to all URLs") + print("6. Exit") - choice = input("\nEnter your choice (1-5): ").strip() + choice = input("\nEnter your choice (1-6): ").strip() if choice == '1': self._handle_new_url() @@ -58,6 +60,8 @@ def _process_menu_choice(self) -> bool: elif choice == '4': self._handle_store_saved_items() elif choice == '5': + self._handle_go_to_all_urls() + elif choice == '6': print("Goodbye!") return False else: @@ -98,6 +102,17 @@ def _handle_new_url(self) -> None: self.client.navigate_to(url) self.feed_items = None + + def _handle_go_to_all_urls(self) -> None: + self._handle_store_saved_items() + for name, url in self.search_urls.items(): + print(f"Navigating to URL {name}...") + self.client.navigate_to(url) + self._handle_get_feed() + self._handle_process_feed() + print("Done going through all URLs!") + return False + def _handle_get_feed(self) -> None: print("Fetching feed items...") self.feed_items = self.client.get_feed_items() diff --git a/src/db/yad2scraper.db b/src/db/yad2scraper.db index a4e2dda..dfc0a5a 100644 Binary files a/src/db/yad2scraper.db and b/src/db/yad2scraper.db differ diff --git a/src/main.py b/src/main.py index 830fe77..963be61 100644 --- a/src/main.py +++ b/src/main.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 +import json import logging import os import signal @@ -57,8 +58,9 @@ def main(): try: client = Yad2Client(headless=False) address_matcher = AddressMatcher(get_resource_path(os.path.join('consts', 'supported_streets.json'))) + search_urls = json.load(open(get_resource_path(os.path.join('consts', 'search_url.json')))) - app = Yad2ScraperApp(client, address_matcher) + app = Yad2ScraperApp(client, address_matcher, search_urls) app.run() except Exception as e: diff --git a/tests/test_app.py b/tests/test_app.py index 70b715b..e92c045 100644 --- a/tests/test_app.py +++ b/tests/test_app.py @@ -1,4 +1,5 @@ -from unittest.mock import MagicMock +from unittest.mock import MagicMock, patch +import json import pytest @@ -18,8 +19,17 @@ def mock_address_matcher(): return MagicMock(spec=AddressMatcher) @pytest.fixture -def app(mock_client, mock_address_matcher): - return Yad2ScraperApp(mock_client, mock_address_matcher) +def mock_search_urls(): + """Mock the search URLs loaded from search_url.json.""" + return { + "petach_tikva_1": "https://www.yad2.co.il/realestate/forsale?multiNeighborhood=2001002%2C763%2C765%2C756%2C2001001&propertyGroup=apartments&property=1&rooms=3-4&price=2300000-2600000&floor=1--1&parking=1&elevator=1&balcony=1&shelter=1", + "petach_tikva_2": "https://www.yad2.co.il/realestate/forsale?multiNeighborhood=2001002%2C763%2C765%2C756%2C2001001&propertyGroup=apartments&property=1&rooms=3-4&price=2300000-2600000&floor=1--1&parking=1&elevator=1&balcony=1&shelter=1&page=2", + } + +@pytest.fixture +def app(mock_client, mock_address_matcher, mock_search_urls): + """Create an instance of Yad2ScraperApp with mocked dependencies.""" + return Yad2ScraperApp(mock_client, mock_address_matcher, mock_search_urls) def test_handle_store_saved_items_success(app, capsys): """Test successful storing of saved items."""