Add ScraperAPI as an option besides Zenrows

Co-authored-by: Luuk Timmermans <[email protected]>

Author: ThijsRay

README.md

@@ -6,17 +6,24 @@ Podimo is a proprietary podcasting player that enables you to listen to various
 This tool allows you to stream Podimo podcasts with your preferred podcast player, without having to use the Podimo app.
 </div>
 
-
 ## Usage
-The easiest way to use it is via [podimo.thijs.sh](https://podimo.thijs.sh). You can also host it yourself by following the instructions below. It's necessary to create a Zenrows account to bypass Podimo's anti-bot mechanisms.
+The easiest way to use it is via [podimo.thijs.sh](https://podimo.thijs.sh). You can also host it yourself by following the instructions below. It's necessary to either create a Zenrows or ScraperAPI account to bypass Podimo's anti-bot mechanisms.
 
-## Setting up a Zenrows account
+### Setting up a Zenrows account
 You can create a free account, which gives you 1000 free api credits.
 
 1. Go to [app.zenrows.com/register](https://app.zenrows.com/register) and create a free account
 2. Copy your API key and make sure to add it to the `ZENROWS_API` environment variable (`-e`) in the Docker run command
 
+### Setting up a ScraperAPI account
+
+You can create a free account, which gives you 1000 free API credits per month.
+
+1. Go to [dashboard.scraperapi.com/signup](https://dashboard.scraperapi.com/signup) and create a free account
+2. Copy your API key and make sure to add it to the `SCRAPER_API` environment variable (`-e`) in the Docker run command
+
 ## Instructions for self-hosting (with Docker)
+
 1. Clone this repository and enter the newly created directory
 ```sh
 git clone https://github.com/ThijsRay/podimo
@@ -28,10 +35,13 @@ cd podimo
 docker build -t podimo:latest .
 ```
 
-3. Run the Docker image
+3. Run the Docker image.
+Make sure you set the correct environment variables, and to add the API keys for
+ZenRows or ScraperAPI if you decide to use those services.
 ```sh
-docker run --rm -e PODIMO_HOSTNAME=yourip:12104 -e PODIMO_BIND_HOST=0.0.0.0:12104 -e PODIMO_PROTOCOL=http -e ZENROWS_API=APIKEY -p 12104:12104 podimo:latest
+docker run --rm -e PODIMO_HOSTNAME=yourip:12104 -e PODIMO_BIND_HOST=0.0.0.0:12104 -e PODIMO_PROTOCOL=http -p 12104:12104 podimo:latest
 ```
+
 For an explaination of what each environmental variable (`-e`) does, see the section on [configuration with environmental variables](#configuration).
 
 4. Visit http://yourip:12104. You should see the site now!
@@ -60,16 +70,15 @@ pip install -r requirements.txt
 ```sh
 python main.py
 ```
-4. Visit http://localhost:12104. You should see the site now!
-
+5. Visit http://localhost:12104. You should see the site now!
 
 ## Configuration
 There are a few environmental variables that can configure this tool
 - `PODIMO_HOSTNAME` Sets the hostname that is displayed in the interface to a custom value, defaults to `podimo.thijs.sh`
 - `PODIMO_BIND_HOST` Sets the IP and port to which this tool should bind, defaults to `127.0.0.1:12104`.
-- `PODIMO_PROTOCOL` Sets the protocol that is displayed in the interface. For local 
-deployments it can be useful to set this to `http`. Defaults to `https`.
+- `PODIMO_PROTOCOL` Sets the protocol that is displayed in the interface. For local deployments it can be useful to set this to `http`. Defaults to `https`.
 - `ZENROWS_API` Sets the Zenrows API key for it to be used.
+- `SCRAPER_API` Sets the ScraperAPI API key for it to be used.
 - `DEBUG` Shows a bit more information that can be useful while debugging
 - `HTTP_PROXY` A URL for an HTTP proxy that can be used to rotate IP addresses to avoid being blocked by CloudFlare.
 

podimo/client.py

@@ -17,15 +17,15 @@
 # See the Licence for the specific language governing
 # permissions and limitations under the Licence.
 
-from podimo.config import GRAPHQL_URL, ZENROWS_API
+from podimo.config import GRAPHQL_URL, SCRAPER_API, ZENROWS_API
 from podimo.utils import (is_correct_email_address, token_key,
                           randomFlyerId, generateHeaders as gHdrs, debug,
                           async_wrap)
 from podimo.cache import insertIntoPodcastCache, getCacheEntry, podcast_cache
 from time import time
-from os import getenv
-from zenrows import ZenRowsClient
 import sys
+if ZENROWS_API is not None:
+    from zenrows import ZenRowsClient
 
 class PodimoClient:
     def __init__(self, username: str, password: str, region: str, locale: str):
@@ -41,21 +41,26 @@ def __init__(self, username: str, password: str, region: str, locale: str):
             raise ValueError("Username or password are too long")
         if not is_correct_email_address(username):
             return ValueError("Email is not in the correct format")
-        
+
         self.key = token_key(username, password)
         self.token = None
 
     def generateHeaders(self, authorization):
-        return gHdrs(authorization, self.locale) 
+        return gHdrs(authorization, self.locale)
 
     async def post(self, headers, query, variables, scraper):
-        if ZENROWS_API is not None:
+        if SCRAPER_API is not None:
+            POST_URL = f"https://api.scraperapi.com?api_key={SCRAPER_API}&url={GRAPHQL_URL}&keep_headers=true"
+        elif ZENROWS_API is not None:
             scraper = ZenRowsClient(ZENROWS_API)
-        response = await async_wrap(scraper.post)(GRAPHQL_URL,
+            POST_URL = GRAPHQL_URL
+        else:
+            POST_URL = GRAPHQL_URL
+        response = await async_wrap(scraper.post)(POST_URL,
                                         headers=headers,
                                         cookies=self.cookie_jar,
                                         json={"query": query, "variables": variables},
-                                        timeout=(6.05, 12.05)
+                                        timeout=(6.05, 30)
                                     )
         if response is None:
             raise RuntimeError(f"Could not receive response for query: {query.strip()[:30]}...")
@@ -70,7 +75,6 @@ async def post(self, headers, query, variables, scraper):
     # as an anonymous user
     async def getPreregisterToken(self, scraper):
         headers = self.generateHeaders(None)
-        
         debug("AuthorizationPreregisterUser")
         query = """
             query AuthorizationPreregisterUser($locale: String!, $referenceUser: String, $countryCode: String, $appsFlyerId: String) {

podimo/config.py

@@ -30,6 +30,7 @@
 PODIMO_BIND_HOST = os.environ.get("PODIMO_BIND_HOST", "0.0.0.0:12104")
 PODIMO_PROTOCOL = os.environ.get("PODIMO_PROTOCOL", "http")
 ZENROWS_API = os.environ.get("ZENROWS_API", None)
+SCRAPER_API = os.environ.get("SCRAPER_API", None)
 
 # Enable extra logging in debugging mode
 DEBUG = os.environ.get("DEBUG", False)