feat: add infinite scrolling

Author: VinciGit00

scrapegraph-py/examples/async/async_smartscraper_infinite_scroll_example.py

@@ -0,0 +1,71 @@
+import asyncio
+from typing import List
+from pydantic import BaseModel
+
+from scrapegraph_py import AsyncClient
+from scrapegraph_py.logger import sgai_logger
+
+sgai_logger.set_logging(level="INFO")
+
+# Define the output schema
+class Company(BaseModel):
+    name: str
+    category: str
+    location: str
+
+class CompaniesResponse(BaseModel):
+    companies: List[Company]
+
+async def scrape_companies(client: AsyncClient, url: str, batch: str) -> None:
+    """Scrape companies from a specific YC batch with infinite scroll."""
+    try:
+        response = await client.smartscraper(
+            website_url=f"{url}?batch={batch}",
+            user_prompt="Extract all company names and their categories from the page",
+            output_schema=CompaniesResponse,
+            number_of_scrolls=10  # Scroll 10 times to load more companies
+        )
+
+        # Parse and print the results
+        result = CompaniesResponse.model_validate(response['result'])
+        print(f"\nCompanies from {batch} batch:")
+        print("=" * 80)
+        for company in result.companies:
+            print(f"Name: {company.name}")
+            print(f"Category: {company.category}")
+            print(f"Location: {company.location}")
+            print("-" * 80)
+
+    except Exception as e:
+        print(f"Error scraping {batch} batch: {e}")
+
+async def main():
+    # Initialize async client
+    sgai_client = AsyncClient(api_key="your-api-key-here")
+
+    try:
+        # Define batches to scrape
+        base_url = "https://www.ycombinator.com/companies"
+        batches = [
+            "Spring%202025",
+            "Winter%202025",
+            "Summer%202024"
+        ]
+
+        # Create tasks for each batch
+        tasks = [
+            scrape_companies(sgai_client, base_url, batch)
+            for batch in batches
+        ]
+
+        # Execute all scraping tasks concurrently
+        await asyncio.gather(*tasks)
+
+    except Exception as e:
+        print(f"An error occurred: {e}")
+
+    finally:
+        await sgai_client.close()
+
+if __name__ == "__main__":
+    asyncio.run(main()) 

scrapegraph-py/examples/sync/smartscraper_infinite_scroll_example.py

@@ -0,0 +1,46 @@
+from scrapegraph_py import Client
+from scrapegraph_py.logger import sgai_logger
+from pydantic import BaseModel
+from typing import List
+
+sgai_logger.set_logging(level="INFO")
+
+# Define the output schema
+class Company(BaseModel):
+    name: str
+    category: str
+    location: str
+
+class CompaniesResponse(BaseModel):
+    companies: List[Company]
+
+# Initialize the client with explicit API key
+sgai_client = Client(api_key="sgai-4cf4a4f5-87f7-457a-8c58-0790ecaf323e")
+
+try:
+    # SmartScraper request with infinite scroll
+    response = sgai_client.smartscraper(
+        website_url="https://www.ycombinator.com/companies?batch=Spring%202025",
+        user_prompt="Extract all company names and their categories from the page",
+        output_schema=CompaniesResponse,
+        number_of_scrolls=10  # Scroll 10 times to load more companies
+    )
+
+    # Print the response
+    print(f"Request ID: {response['request_id']}")
+    
+    # Parse and print the results in a structured way
+    result = CompaniesResponse.model_validate(response['result'])
+    print("\nExtracted Companies:")
+    print("-" * 80)
+    for company in result.companies:
+        print(f"Name: {company.name}")
+        print(f"Category: {company.category}")
+        print(f"Location: {company.location}")
+        print("-" * 80)
+
+except Exception as e:
+    print(f"An error occurred: {e}")
+
+finally:
+    sgai_client.close() 

scrapegraph-py/scrapegraph_py/async_client.py

@@ -174,6 +174,7 @@ async def smartscraper(
         website_html: Optional[str] = None,
         headers: Optional[dict[str, str]] = None,
         output_schema: Optional[BaseModel] = None,
+        number_of_scrolls: Optional[int] = None,
     ):
         """Send a smartscraper request"""
         logger.info("🔍 Starting smartscraper request")
@@ -183,6 +184,8 @@ async def smartscraper(
             logger.debug("📄 Using provided HTML content")
         if headers:
             logger.debug("🔧 Using custom headers")
+        if number_of_scrolls is not None:
+            logger.debug(f"🔄 Number of scrolls: {number_of_scrolls}")
         logger.debug(f"📝 Prompt: {user_prompt}")
 
         request = SmartScraperRequest(
@@ -191,6 +194,7 @@ async def smartscraper(
             headers=headers,
             user_prompt=user_prompt,
             output_schema=output_schema,
+            number_of_scrolls=number_of_scrolls,
         )
         logger.debug("✅ Request validation passed")
 

scrapegraph-py/scrapegraph_py/client.py

@@ -182,6 +182,7 @@ def smartscraper(
         website_html: Optional[str] = None,
         headers: Optional[dict[str, str]] = None,
         output_schema: Optional[BaseModel] = None,
+        number_of_scrolls: Optional[int] = None,
     ):
         """Send a smartscraper request"""
         logger.info("🔍 Starting smartscraper request")
@@ -191,6 +192,8 @@ def smartscraper(
             logger.debug("📄 Using provided HTML content")
         if headers:
             logger.debug("🔧 Using custom headers")
+        if number_of_scrolls is not None:
+            logger.debug(f"🔄 Number of scrolls: {number_of_scrolls}")
         logger.debug(f"📝 Prompt: {user_prompt}")
 
         request = SmartScraperRequest(
@@ -199,6 +202,7 @@ def smartscraper(
             headers=headers,
             user_prompt=user_prompt,
             output_schema=output_schema,
+            number_of_scrolls=number_of_scrolls,
         )
         logger.debug("✅ Request validation passed")
 

scrapegraph-py/scrapegraph_py/models/smartscraper.py

@@ -4,7 +4,7 @@
 from uuid import UUID
 
 from bs4 import BeautifulSoup
-from pydantic import BaseModel, Field, model_validator
+from pydantic import BaseModel, Field, model_validator, conint
 
 
 class SmartScraperRequest(BaseModel):
@@ -29,6 +29,11 @@ class SmartScraperRequest(BaseModel):
         description="Optional headers to send with the request, including cookies and user agent",
     )
     output_schema: Optional[Type[BaseModel]] = None
+    number_of_scrolls: Optional[conint(ge=0, le=100)] = Field(
+        default=None,
+        description="Number of times to scroll the page (0-100). If None, no scrolling will be performed.",
+        example=10
+    )
 
     @model_validator(mode="after")
     def validate_user_prompt(self) -> "SmartScraperRequest":