Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions examples/extras/.env.example
Original file line number Diff line number Diff line change
Expand Up @@ -2,3 +2,6 @@ OPENAI_API_KEY="YOUR_OPENAI_API_KEY"
BROWSER_BASE_PROJECT_ID="YOUR_BROWSER_BASE_PROJECT_ID"
BROWSER_BASE_API_KEY="YOUR_BROWSERBASE_API_KEY"
SCRAPE_DO_API_KEY="YOUR_SCRAPE_DO_API_KEY"
PROXYHAT_PROXY_SERVER="http://gate.proxyhat.com:8080"
PROXYHAT_USERNAME="YOUR_PROXYHAT_USERNAME"
PROXYHAT_PASSWORD="YOUR_PROXYHAT_PASSWORD"
49 changes: 49 additions & 0 deletions examples/extras/proxyhat_integration.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,49 @@
"""
Example of a SmartScraper pipeline running behind a residential proxy.

Any proxy that speaks the standard {server, username, password} format can be
passed through `loader_kwargs.proxy`. This example uses ProxyHat residential
proxies; swap in your own provider's credentials as needed.
"""

import json
import os

from dotenv import load_dotenv

from scrapegraphai.graphs import SmartScraperGraph

load_dotenv()

# ************************************************
# Define the configuration for the graph
# ************************************************

graph_config = {
"llm": {
"api_key": os.getenv("OPENAI_API_KEY"),
"model": "openai/gpt-4o",
},
"loader_kwargs": {
"proxy": {
"server": os.getenv("PROXYHAT_PROXY_SERVER"),
"username": os.getenv("PROXYHAT_USERNAME"),
"password": os.getenv("PROXYHAT_PASSWORD"),
},
},
"verbose": True,
"headless": True,
}

# ************************************************
# Create the SmartScraperGraph instance and run it
# ************************************************

smart_scraper_graph = SmartScraperGraph(
prompt="List me all the projects with their description",
source="https://perinim.github.io/projects/",
config=graph_config,
)

result = smart_scraper_graph.run()
print(json.dumps(result, indent=4))