Merge remote-tracking branch 'origin/main' into close-inactive-contexts

Author: elacuesta

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.0.28
+current_version = 0.0.29
 commit = True
 tag = True
 

.github/workflows/tests.yml

@@ -9,7 +9,7 @@ jobs:
       fail-fast: false
       matrix:
         os: [ubuntu-latest, macos-latest]
-        python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"]
+        python-version: ["3.8", "3.9", "3.10", "3.11"]
 
     steps:
     - uses: actions/checkout@v2
@@ -22,9 +22,12 @@ jobs:
     - name: Install tox
       run: pip install tox
 
-    - name: Run tests
+    - name: Run asyncio tests
       run: tox -e py
 
+    - name: Run twisted tests
+      run: tox -e py-twisted
+
     - name: Upload coverage report
       run: |
         if [ "${{ runner.os }}" = "Linux" ]; then

.gitignore

@@ -4,11 +4,16 @@
 .mypy_cache/
 *.egg-info/
 .tox/
-.coverage
-.coverage.*
-htmlcov/
-coverage.xml
 build/
 dist/
 examples/*.png
 pip-wheel-metadata/
+
+# coverage
+.coverage
+.coverage.*
+htmlcov/
+coverage.xml
+coverage-*.xml
+coverage-asyncio/
+coverage-twisted/

README.md

@@ -363,10 +363,11 @@ See [Handling page events](#handling-page-events).
 ### `playwright_page_init_callback`
 Type `Optional[Union[Callable, str]]`, default `None`
 
-A coroutine function (`async def`) to be invoked immediately after creating
-a page for the request. It receives the page and the request as positional
-arguments. Useful for initialization code. Invoked only for newly created
-pages, ignored if the page for the request already exists (e.g. by passing
+A coroutine function (`async def`) to be invoked for newly created pages.
+Called after attaching page event handlers & setting up internal route
+handling, before making any request. It receives the Playwright page and the
+Scrapy request as positional arguments. Useful for initialization code.
+Ignored if the page for the request already exists (e.g. by passing
 `playwright_page`).
 
 ```python

docs/changelog.md

@@ -1,6 +1,12 @@
 # scrapy-playwright changelog
 
 
+### [v0.0.29](https://github.com/scrapy-plugins/scrapy-playwright/releases/tag/v0.0.29) (2023-08-11)
+
+* Set exc_info=True for warning log records (#219)
+* Invoke page_init_callback after setting route (#205)
+
+
 ### [v0.0.28](https://github.com/scrapy-plugins/scrapy-playwright/releases/tag/v0.0.28) (2023-08-05)
 
 * Retry page.content if necessary (#218)

scrapy_playwright/__init__.py

@@ -1 +1 @@
-__version__ = "0.0.28"
+__version__ = "0.0.29"

scrapy_playwright/handler.py

@@ -258,6 +258,7 @@ async def _create_page(self, request: Request, spider: Spider) -> Page:
                         "scrapy_request_method": request.method,
                         "exception": ex,
                     },
+                    exc_info=True,
                 )
 
         page.on("close", self._make_close_page_callback(context_name))
@@ -330,6 +331,10 @@ async def _download_request(self, request: Request, spider: Spider) -> Response:
             ),
         )
 
+        await _maybe_execute_page_init_callback(
+            page=page, request=request, context_name=context_name, spider=spider
+        )
+
         try:
             result = await self._download_request_with_page(request, page, spider)
         except Exception as ex:
@@ -346,6 +351,7 @@ async def _download_request(self, request: Request, spider: Spider) -> Response:
                         "scrapy_request_method": request.method,
                         "exception": ex,
                     },
+                    exc_info=True,
                 )
                 await page.close()
                 self.stats.inc_value("playwright/page_count/closed")
@@ -438,6 +444,7 @@ async def _apply_page_methods(self, page: Page, request: Request, spider: Spider
                             "scrapy_request_method": request.method,
                             "exception": ex,
                         },
+                        exc_info=True,
                     )
                 else:
                     pm.result = await _maybe_await(method(*pm.args, **pm.kwargs))
@@ -590,6 +597,7 @@ async def _request_handler(route: Route, playwright_request: PlaywrightRequest)
                             "playwright_request_method": playwright_request.method,
                             "exception": ex,
                         },
+                        exc_info=True,
                     )
                 else:
                     raise
@@ -621,6 +629,7 @@ def _attach_page_event_handlers(
                         "scrapy_request_method": request.method,
                         "exception": ex,
                     },
+                    exc_info=True,
                 )
 
 
@@ -643,6 +652,35 @@ async def _set_redirect_meta(request: Request, response: PlaywrightResponse) ->
         request.meta["redirect_reasons"] = list(reversed(redirect_reasons))
 
 
+async def _maybe_execute_page_init_callback(
+    page: Page,
+    request: Request,
+    context_name: str,
+    spider: Spider,
+) -> None:
+    page_init_callback = request.meta.get("playwright_page_init_callback")
+    if page_init_callback:
+        try:
+            page_init_callback = load_object(page_init_callback)
+            await page_init_callback(page, request)
+        except Exception as ex:
+            logger.warning(
+                "[Context=%s] Page init callback exception for %s exc_type=%s exc_msg=%s",
+                context_name,
+                repr(request),
+                type(ex),
+                str(ex),
+                extra={
+                    "spider": spider,
+                    "context_name": context_name,
+                    "scrapy_request_url": request.url,
+                    "scrapy_request_method": request.method,
+                    "exception": ex,
+                },
+                exc_info=True,
+            )
+
+
 def _make_request_logger(context_name: str, spider: Spider) -> Callable:
     async def _log_request(request: PlaywrightRequest) -> None:
         referrer = await request.header_value("referer")

setup.py

@@ -22,7 +22,6 @@
         "Development Status :: 3 - Alpha",
         "License :: OSI Approved :: BSD License",
         "Programming Language :: Python",
-        "Programming Language :: Python :: 3.7",
         "Programming Language :: Python :: 3.8",
         "Programming Language :: Python :: 3.9",
         "Programming Language :: Python :: 3.10",
@@ -33,7 +32,7 @@
         "Topic :: Software Development :: Libraries :: Application Frameworks",
         "Topic :: Software Development :: Libraries :: Python Modules",
     ],
-    python_requires=">=3.7",
+    python_requires=">=3.8",
     install_requires=[
         "scrapy>=2.0,!=2.4.0",
         "playwright>=1.15",