File tree Expand file tree Collapse file tree
Expand file tree Collapse file tree Original file line number Diff line number Diff line change @@ -262,13 +262,15 @@ Ready-to-run examples are available in `examples/`:
262262- ` examples/async_extract.py `
263263- ` examples/async_scrape.py `
264264- ` examples/async_session_list.py `
265+ - ` examples/async_web_crawl.py `
265266- ` examples/async_web_fetch.py `
266267- ` examples/async_web_search.py `
267268- ` examples/sync_batch_fetch.py `
268269- ` examples/sync_crawl.py `
269270- ` examples/sync_extract.py `
270271- ` examples/sync_scrape.py `
271272- ` examples/sync_session_list.py `
273+ - ` examples/sync_web_crawl.py `
272274- ` examples/sync_web_fetch.py `
273275- ` examples/sync_web_search.py `
274276
Original file line number Diff line number Diff line change 1+ """
2+ Asynchronous web crawl example.
3+
4+ Run:
5+ export HYPERBROWSER_API_KEY="your_api_key"
6+ python3 examples/async_web_crawl.py
7+ """
8+
9+ import asyncio
10+
11+ from hyperbrowser import AsyncHyperbrowser
12+ from hyperbrowser .exceptions import HyperbrowserTimeoutError
13+ from hyperbrowser .models import StartWebCrawlJobParams
14+
15+
16+ async def main () -> None :
17+ async with AsyncHyperbrowser () as client :
18+ try :
19+ result = await client .web .crawl .start_and_wait (
20+ StartWebCrawlJobParams (
21+ url = "https://hyperbrowser.ai" ,
22+ ),
23+ poll_interval_seconds = 1.0 ,
24+ max_wait_seconds = 120.0 ,
25+ )
26+ except HyperbrowserTimeoutError :
27+ print ("Web crawl job timed out." )
28+ return
29+
30+ print (f"Status: { result .status } " )
31+ print (f"Crawled pages in batch: { len (result .data or [])} " )
32+
33+
34+ if __name__ == "__main__" :
35+ asyncio .run (main ())
Original file line number Diff line number Diff line change 1+ """
2+ Synchronous web crawl example.
3+
4+ Run:
5+ export HYPERBROWSER_API_KEY="your_api_key"
6+ python3 examples/sync_web_crawl.py
7+ """
8+
9+ from hyperbrowser import Hyperbrowser
10+ from hyperbrowser .exceptions import HyperbrowserTimeoutError
11+ from hyperbrowser .models import StartWebCrawlJobParams
12+
13+
14+ def main () -> None :
15+ with Hyperbrowser () as client :
16+ try :
17+ result = client .web .crawl .start_and_wait (
18+ StartWebCrawlJobParams (
19+ url = "https://hyperbrowser.ai" ,
20+ ),
21+ poll_interval_seconds = 1.0 ,
22+ max_wait_seconds = 120.0 ,
23+ )
24+ except HyperbrowserTimeoutError :
25+ print ("Web crawl job timed out." )
26+ return
27+
28+ print (f"Status: { result .status } " )
29+ print (f"Crawled pages in batch: { len (result .data or [])} " )
30+
31+
32+ if __name__ == "__main__" :
33+ main ()
You can’t perform that action at this time.
0 commit comments