Skip to content

Commit c55a73c

Browse files
Add sync and async web crawl usage examples
Co-authored-by: Shri Sukhani <shrisukhani@users.noreply.github.com>
1 parent d85215d commit c55a73c

3 files changed

Lines changed: 70 additions & 0 deletions

File tree

README.md

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -262,13 +262,15 @@ Ready-to-run examples are available in `examples/`:
262262
- `examples/async_extract.py`
263263
- `examples/async_scrape.py`
264264
- `examples/async_session_list.py`
265+
- `examples/async_web_crawl.py`
265266
- `examples/async_web_fetch.py`
266267
- `examples/async_web_search.py`
267268
- `examples/sync_batch_fetch.py`
268269
- `examples/sync_crawl.py`
269270
- `examples/sync_extract.py`
270271
- `examples/sync_scrape.py`
271272
- `examples/sync_session_list.py`
273+
- `examples/sync_web_crawl.py`
272274
- `examples/sync_web_fetch.py`
273275
- `examples/sync_web_search.py`
274276

examples/async_web_crawl.py

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
"""
2+
Asynchronous web crawl example.
3+
4+
Run:
5+
export HYPERBROWSER_API_KEY="your_api_key"
6+
python3 examples/async_web_crawl.py
7+
"""
8+
9+
import asyncio
10+
11+
from hyperbrowser import AsyncHyperbrowser
12+
from hyperbrowser.exceptions import HyperbrowserTimeoutError
13+
from hyperbrowser.models import StartWebCrawlJobParams
14+
15+
16+
async def main() -> None:
17+
async with AsyncHyperbrowser() as client:
18+
try:
19+
result = await client.web.crawl.start_and_wait(
20+
StartWebCrawlJobParams(
21+
url="https://hyperbrowser.ai",
22+
),
23+
poll_interval_seconds=1.0,
24+
max_wait_seconds=120.0,
25+
)
26+
except HyperbrowserTimeoutError:
27+
print("Web crawl job timed out.")
28+
return
29+
30+
print(f"Status: {result.status}")
31+
print(f"Crawled pages in batch: {len(result.data or [])}")
32+
33+
34+
if __name__ == "__main__":
35+
asyncio.run(main())

examples/sync_web_crawl.py

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
"""
2+
Synchronous web crawl example.
3+
4+
Run:
5+
export HYPERBROWSER_API_KEY="your_api_key"
6+
python3 examples/sync_web_crawl.py
7+
"""
8+
9+
from hyperbrowser import Hyperbrowser
10+
from hyperbrowser.exceptions import HyperbrowserTimeoutError
11+
from hyperbrowser.models import StartWebCrawlJobParams
12+
13+
14+
def main() -> None:
15+
with Hyperbrowser() as client:
16+
try:
17+
result = client.web.crawl.start_and_wait(
18+
StartWebCrawlJobParams(
19+
url="https://hyperbrowser.ai",
20+
),
21+
poll_interval_seconds=1.0,
22+
max_wait_seconds=120.0,
23+
)
24+
except HyperbrowserTimeoutError:
25+
print("Web crawl job timed out.")
26+
return
27+
28+
print(f"Status: {result.status}")
29+
print(f"Crawled pages in batch: {len(result.data or [])}")
30+
31+
32+
if __name__ == "__main__":
33+
main()

0 commit comments

Comments
 (0)