feat: Update Scrapy template and wrapper to use SDK 3.3.0

templates/python-scrapy/requirements.txt

@@ -1,5 +1,5 @@
 # Feel free to add your Python dependencies below. For formatting guidelines, see:
 # https://pip.pypa.io/en/latest/reference/requirements-file-format/
 
-apify[scrapy] < 4.0.0
-scrapy < 3.0.0
+apify[scrapy] >= 3.3.0, < 4.0.0
+scrapy >= 2.14.0, < 3.0.0

templates/python-scrapy/src/__main__.py

@@ -10,15 +10,8 @@
 We recommend you do not modify this file unless you really know what you are doing.
 """
 
-# ruff: noqa: E402
 from __future__ import annotations
 
-from scrapy.utils.reactor import install_reactor
-
-# Install Twisted's asyncio reactor before importing any other Twisted or
-# Scrapy components.
-install_reactor('twisted.internet.asyncioreactor.AsyncioSelectorReactor')
-
 import os
 
 from apify.scrapy import initialize_logging, run_scrapy_actor

templates/python-scrapy/src/main.py

@@ -23,8 +23,7 @@
 
 from apify import Actor
 from apify.scrapy import apply_apify_settings
-from scrapy.crawler import CrawlerRunner
-from scrapy.utils.defer import deferred_to_future
+from scrapy.crawler import AsyncCrawlerRunner
 
 # Import your Scrapy spider here.
 from .spiders import TitleSpider as Spider
@@ -42,11 +41,10 @@ async def main() -> None:
         # Apply Apify settings, which will override the Scrapy project settings.
         settings = apply_apify_settings(proxy_config=proxy_config)
 
-        # Create CrawlerRunner and execute the Scrapy spider.
-        crawler_runner = CrawlerRunner(settings)
-        crawl_deferred = crawler_runner.crawl(
+        # Create AsyncCrawlerRunner and execute the Scrapy spider.
+        crawler_runner = AsyncCrawlerRunner(settings)
+        await crawler_runner.crawl(
             Spider,
             start_urls=start_urls,
             allowed_domains=allowed_domains,
         )
-        await deferred_to_future(crawl_deferred)

templates/python-scrapy/src/settings.py

@@ -15,6 +15,8 @@
 TELNETCONSOLE_ENABLED = False
 # Do not change the Twisted reactor unless you really know what you are doing.
 TWISTED_REACTOR = 'twisted.internet.asyncioreactor.AsyncioSelectorReactor'
+HTTPCACHE_ENABLED = True
+HTTPCACHE_EXPIRATION_SECS = 7200
 ITEM_PIPELINES = {
     'src.pipelines.TitleItemPipeline': 123,
 }

wrappers/python-scrapy/requirements_apify.txt

@@ -1,5 +1,5 @@
 # Add your dependencies here.
 # See https://pip.pypa.io/en/latest/reference/requirements-file-format/
 # for how to format them
-apify[scrapy] < 3.0
-scrapy < 3.0
+apify[scrapy] >= 3.3.0, < 4.0.0
+scrapy >= 2.14.0, < 3.0.0

wrappers/python-scrapy/{projectFolder}/__main__.template.py

@@ -9,16 +9,9 @@
 
 We recommend you do not modify this file unless you really know what you are doing.
 """
-# ruff: noqa: E402
 
 from __future__ import annotations
 
-from scrapy.utils.reactor import install_reactor
-
-# Install Twisted's asyncio reactor before importing any other Twisted or
-# Scrapy components.
-install_reactor('twisted.internet.asyncioreactor.AsyncioSelectorReactor')
-
 import os
 
 from apify.scrapy import initialize_logging, run_scrapy_actor

wrappers/python-scrapy/{projectFolder}/main.template.py

@@ -22,8 +22,7 @@
 
 from apify import Actor
 from apify.scrapy import apply_apify_settings
-from scrapy.crawler import CrawlerRunner
-from scrapy.utils.defer import deferred_to_future
+from scrapy.crawler import AsyncCrawlerRunner
 
 # Import your Scrapy spider here.
 from {{spider_module_name}} import {{spider_class_name}} as Spider
@@ -44,11 +43,10 @@ async def main() -> None:
         # Apply Apify settings, which will override the Scrapy project settings.
         settings = apply_apify_settings(proxy_config=proxy_config)
 
-        # Create CrawlerRunner and execute the Scrapy spider.
-        crawler_runner = CrawlerRunner(settings)
-        crawl_deferred = crawler_runner.crawl(
+        # Create AsyncCrawlerRunner and execute the Scrapy spider.
+        crawler_runner = AsyncCrawlerRunner(settings)
+        await crawler_runner.crawl(
             Spider,
             start_urls=start_urls,
             allowed_domains=allowed_domains,
         )
-        await deferred_to_future(crawl_deferred)