docs: Add Migrating from Scrapy guide

docs/guides/code_examples/scrapy_migration/crawlee_concurrency.py

@@ -0,0 +1,35 @@
+import asyncio
+
+from crawlee import ConcurrencySettings
+from crawlee.crawlers import ParselCrawler, ParselCrawlingContext
+
+
+async def main() -> None:
+    # highlight-start
+    # `ConcurrencySettings` replaces Scrapy's `CONCURRENT_REQUESTS` and
+    # `DOWNLOAD_DELAY`.
+    concurrency_settings = ConcurrencySettings(
+        # Start with this many parallel tasks.
+        desired_concurrency=5,
+        # Never run more than this many in parallel.
+        max_concurrency=20,
+        # Cap total throughput across the whole pool.
+        max_tasks_per_minute=120,
+    )
+    # highlight-end
+
+    crawler = ParselCrawler(
+        concurrency_settings=concurrency_settings,
+        max_requests_per_crawl=50,
+    )
+
+    @crawler.router.default_handler
+    async def handler(context: ParselCrawlingContext) -> None:
+        context.log.info(f'Processing {context.request.url}')
+        await context.enqueue_links(selector='li.next a')
+
+    await crawler.run(['https://quotes.toscrape.com/'])
+
+
+if __name__ == '__main__':
+    asyncio.run(main())

docs/guides/code_examples/scrapy_migration/crawlee_crawlspider.py

@@ -0,0 +1,44 @@
+import asyncio
+
+from crawlee import Glob
+from crawlee.crawlers import ParselCrawler, ParselCrawlingContext
+
+
+async def main() -> None:
+    crawler = ParselCrawler(max_requests_per_crawl=50)
+
+    # The default handler plays the role of CrawlSpider's `rules`. It follows the
+    # pagination and enqueues each book detail page, routed by label.
+    @crawler.router.default_handler
+    async def listing_handler(context: ParselCrawlingContext) -> None:
+        context.log.info(f'Listing {context.request.url}')
+
+        # highlight-start
+        # `selector` is the `restrict_css` analog. `include` is the `allow` analog:
+        # it keeps only URLs matching the given globs.
+        await context.enqueue_links(
+            selector='article.product_pod h3 a',
+            include=[Glob('https://books.toscrape.com/catalogue/**')],
+            label='book',
+        )
+        # highlight-end
+
+        # Follow pagination without a label, like a `Rule` with no callback.
+        await context.enqueue_links(selector='li.next a')
+
+    # Routed by the 'book' label, like a `Rule` with `callback='parse_book'`.
+    @crawler.router.handler('book')
+    async def book_handler(context: ParselCrawlingContext) -> None:
+        context.log.info(f'Book {context.request.url}')
+        await context.push_data(
+            {
+                'title': context.selector.css('h1::text').get(),
+                'price': context.selector.css('p.price_color::text').get(),
+            }
+        )
+
+    await crawler.run(['https://books.toscrape.com/'])
+
+
+if __name__ == '__main__':
+    asyncio.run(main())

docs/guides/code_examples/scrapy_migration/crawlee_error_handling.py

@@ -0,0 +1,34 @@
+import asyncio
+
+from crawlee.crawlers import BasicCrawlingContext, ParselCrawler, ParselCrawlingContext
+
+
+async def main() -> None:
+    crawler = ParselCrawler(
+        # Retry each failed request up to this many times (the `RetryMiddleware` analog).
+        max_request_retries=3,
+        max_requests_per_crawl=50,
+    )
+
+    @crawler.router.default_handler
+    async def handler(context: ParselCrawlingContext) -> None:
+        for quote in context.selector.css('div.quote'):
+            await context.push_data({'text': quote.css('span.text::text').get()})
+
+    # highlight-start
+    # Runs between retries. It can inspect or adjust the request before the next try.
+    @crawler.error_handler
+    async def on_error(context: BasicCrawlingContext, error: Exception) -> None:
+        context.log.warning(f'Retrying {context.request.url}: {error}')
+
+    # Runs once a request has exhausted all retries, like Scrapy's `errback`.
+    @crawler.failed_request_handler
+    async def on_failed(context: BasicCrawlingContext, error: Exception) -> None:
+        context.log.error(f'Giving up on {context.request.url}: {error}')
+        # highlight-end
+
+    await crawler.run(['https://quotes.toscrape.com/'])
+
+
+if __name__ == '__main__':
+    asyncio.run(main())

docs/guides/code_examples/scrapy_migration/crawlee_export.py

@@ -0,0 +1,36 @@
+import asyncio
+
+from crawlee.crawlers import ParselCrawler, ParselCrawlingContext
+
+
+async def main() -> None:
+    crawler = ParselCrawler(max_requests_per_crawl=50)
+
+    @crawler.router.default_handler
+    async def handler(context: ParselCrawlingContext) -> None:
+        context.log.info(f'Processing {context.request.url}')
+
+        items = [
+            {
+                'text': quote.css('span.text::text').get(),
+                'author': quote.css('small.author::text').get(),
+            }
+            for quote in context.selector.css('div.quote')
+        ]
+        await context.push_data(items)
+
+        await context.enqueue_links(selector='li.next a')
+
+    await crawler.run(['https://quotes.toscrape.com/'])
+
+    # highlight-start
+    # Export the whole dataset to a file. The format follows the extension,
+    # which must be .json or .csv. It replaces Scrapy's `FEEDS` setting and
+    # the `-O output.json` CLI flag.
+    await crawler.export_data('quotes.json')
+    await crawler.export_data('quotes.csv')
+    # highlight-end
+
+
+if __name__ == '__main__':
+    asyncio.run(main())

docs/guides/code_examples/scrapy_migration/crawlee_labels.py

@@ -0,0 +1,42 @@
+import asyncio
+
+from crawlee.crawlers import ParselCrawler, ParselCrawlingContext
+
+
+async def main() -> None:
+    crawler = ParselCrawler(max_requests_per_crawl=50)
+
+    # The default handler processes listing pages: the entry point and each
+    # paginated page. It routes author links to a separate handler by label.
+    @crawler.router.default_handler
+    async def listing_handler(context: ParselCrawlingContext) -> None:
+        context.log.info(f'Listing {context.request.url}')
+
+        # highlight-start
+        # Enqueue author detail pages with a label. It replaces a Scrapy
+        # `Request(url, callback=self.parse_author)`.
+        await context.enqueue_links(selector='div.quote span a', label='author')
+        # highlight-end
+
+        # Follow the pagination link.
+        await context.enqueue_links(selector='li.next a')
+
+    # This handler runs only for requests labeled 'author'.
+    # highlight-next-line
+    @crawler.router.handler('author')
+    async def author_handler(context: ParselCrawlingContext) -> None:
+        context.log.info(f'Author {context.request.url}')
+
+        await context.push_data(
+            {
+                'name': context.selector.css('h3.author-title::text').get(),
+                'born': context.selector.css('span.author-born-date::text').get(),
+                'bio': context.selector.css('div.author-description::text').get(),
+            }
+        )
+
+    await crawler.run(['https://quotes.toscrape.com/'])
+
+
+if __name__ == '__main__':
+    asyncio.run(main())

docs/guides/code_examples/scrapy_migration/crawlee_playwright.py

@@ -0,0 +1,37 @@
+import asyncio
+
+from crawlee.crawlers import PlaywrightCrawler, PlaywrightCrawlingContext
+
+
+async def main() -> None:
+    # highlight-start
+    # `PlaywrightCrawler` renders JavaScript in a real browser. It replaces the
+    # `scrapy-playwright` package, with browser support built into the framework.
+    crawler = PlaywrightCrawler(
+        headless=True,
+        max_requests_per_crawl=50,
+    )
+    # highlight-end
+
+    @crawler.router.default_handler
+    async def handler(context: PlaywrightCrawlingContext) -> None:
+        context.log.info(f'Processing {context.request.url}')
+
+        # highlight-start
+        # `context.page` is a Playwright `Page`. Query the rendered DOM directly.
+        for quote in await context.page.locator('div.quote').all():
+            await context.push_data(
+                {
+                    'text': await quote.locator('span.text').text_content(),
+                    'author': await quote.locator('small.author').text_content(),
+                }
+            )
+        # highlight-end
+
+        await context.enqueue_links(selector='li.next a')
+
+    await crawler.run(['https://quotes.toscrape.com/js/'])
+
+
+if __name__ == '__main__':
+    asyncio.run(main())

docs/guides/code_examples/scrapy_migration/crawlee_post.py

@@ -0,0 +1,56 @@
+import asyncio
+from urllib.parse import urlencode
+
+from crawlee import Request
+from crawlee.crawlers import ParselCrawler, ParselCrawlingContext
+
+
+async def main() -> None:
+    crawler = ParselCrawler(max_requests_per_crawl=10)
+
+    @crawler.router.default_handler
+    async def login_page(context: ParselCrawlingContext) -> None:
+        # The CSRF token is tied to the session cookie issued for this GET.
+        if not context.session:
+            raise RuntimeError('Session not found')
+
+        token = context.selector.css('input[name="csrf_token"]::attr(value)').get()
+
+        # The CSRF token is required for the POST to succeed. If it's missing,
+        # the login will fail.
+        if not token:
+            raise RuntimeError('CSRF token not found')
+
+        form = {'csrf_token': token, 'username': 'user', 'password': 'pass'}
+
+        # highlight-start
+        # Crawlee's `payload` is the raw request body, so encode the fields yourself
+        # and set the `Content-Type`. Scrapy's `FormRequest` does both for you.
+        await context.add_requests(
+            [
+                Request.from_url(
+                    'https://quotes.toscrape.com/login',
+                    method='POST',
+                    payload=urlencode(form),
+                    headers={'content-type': 'application/x-www-form-urlencoded'},
+                    label='after-login',
+                    # Bind the POST to the same session so its CSRF cookie matches.
+                    session_id=context.session.id,
+                    # The POST shares the GET's URL. Include the method and payload
+                    # in the unique key, or the queue drops it as a duplicate.
+                    use_extended_unique_key=True,
+                )
+            ]
+        )
+        # highlight-end
+
+    @crawler.router.handler('after-login')
+    async def after_login(context: ParselCrawlingContext) -> None:
+        logged_in = context.selector.css('a[href="/logout"]').get() is not None
+        await context.push_data({'logged_in': logged_in})
+
+    await crawler.run(['https://quotes.toscrape.com/login'])
+
+
+if __name__ == '__main__':
+    asyncio.run(main())

docs/guides/code_examples/scrapy_migration/crawlee_proxy.py

@@ -0,0 +1,33 @@
+import asyncio
+
+from crawlee.crawlers import ParselCrawler, ParselCrawlingContext
+from crawlee.proxy_configuration import ProxyConfiguration
+
+
+async def main() -> None:
+    # highlight-start
+    # `ProxyConfiguration` replaces Scrapy's `HttpProxyMiddleware` and the
+    # `scrapy-rotating-proxies` package. The URLs rotate in a round-robin fashion.
+    proxy_configuration = ProxyConfiguration(
+        proxy_urls=[
+            'http://proxy-1.com/',
+            'http://proxy-2.com/',
+        ]
+    )
+    # highlight-end
+
+    crawler = ParselCrawler(
+        proxy_configuration=proxy_configuration,
+        max_requests_per_crawl=50,
+    )
+
+    @crawler.router.default_handler
+    async def handler(context: ParselCrawlingContext) -> None:
+        context.log.info(f'Processing {context.request.url}')
+        await context.enqueue_links(selector='li.next a')
+
+    await crawler.run(['https://quotes.toscrape.com/'])
+
+
+if __name__ == '__main__':
+    asyncio.run(main())

docs/guides/code_examples/scrapy_migration/crawlee_quotes.py

@@ -0,0 +1,33 @@
+import asyncio
+
+from crawlee.crawlers import ParselCrawler, ParselCrawlingContext
+
+
+async def main() -> None:
+    crawler = ParselCrawler(max_requests_per_crawl=50)
+
+    # The default handler runs for the entry point and every paginated page.
+    @crawler.router.default_handler
+    async def handler(context: ParselCrawlingContext) -> None:
+        context.log.info(f'Processing {context.request.url}')
+
+        # `context.selector` is a Parsel `Selector`, the same object Scrapy exposes
+        # as `response`. CSS and XPath queries carry over unchanged.
+        items = [
+            {
+                'text': quote.css('span.text::text').get(),
+                'author': quote.css('small.author::text').get(),
+                'tags': quote.css('div.tags a.tag::text').getall(),
+            }
+            for quote in context.selector.css('div.quote')
+        ]
+        await context.push_data(items)
+
+        # Follow the pagination link to the next page.
+        await context.enqueue_links(selector='li.next a')
+
+    await crawler.run(['https://quotes.toscrape.com/'])
+
+
+if __name__ == '__main__':
+    asyncio.run(main())

docs/guides/code_examples/scrapy_migration/crawlee_throttling.py

@@ -0,0 +1,40 @@
+import asyncio
+
+from crawlee.crawlers import ParselCrawler, ParselCrawlingContext
+from crawlee.request_loaders import ThrottlingRequestManager
+from crawlee.storages import RequestQueue
+
+
+async def main() -> None:
+    # A regular request queue holds requests for non-throttled domains.
+    request_queue = await RequestQueue.open()
+
+    # highlight-start
+    # `ThrottlingRequestManager` wraps the queue and adds per-domain backoff.
+    # It reacts to HTTP 429 responses and `robots.txt` crawl-delay directives, which
+    # makes it the closest built-in analog to Scrapy's `AutoThrottle`. The crawler
+    # feeds it those signals, so you only list the domains to watch.
+    request_manager = ThrottlingRequestManager(
+        inner=request_queue,
+        domains=['quotes.toscrape.com'],
+        request_manager_opener=RequestQueue.open,
+    )
+    # highlight-end
+
+    crawler = ParselCrawler(
+        request_manager=request_manager,
+        # Crawl-delay is only read when `robots.txt` handling is enabled.
+        respect_robots_txt_file=True,
+        max_requests_per_crawl=50,
+    )
+
+    @crawler.router.default_handler
+    async def handler(context: ParselCrawlingContext) -> None:
+        context.log.info(f'Processing {context.request.url}')
+        await context.enqueue_links(selector='li.next a')
+
+    await crawler.run(['https://quotes.toscrape.com/'])
+
+
+if __name__ == '__main__':
+    asyncio.run(main())