Merge pull request #140 from phenobarbital/new-version

added some paid proxy services

proxylists/proxies/__init__.py

@@ -4,6 +4,9 @@
 from .proxydb import ProxyDB
 from .hidemy import Hidemy
 from .proxyworld import ProxyWorld
+from .oxylabs import Oxylabs
+from .smart import SmartProxy
+from .geonode import Geonode, GeonodeFree
 
 __all__ = ["FreeProxy", "Hidemy", "ProxyDB"]
 

proxylists/proxies/geonode.py

@@ -0,0 +1,96 @@
+"""
+Geonode Proxy information.
+"""
+import os
+from requests.models import PreparedRequest
+import aiohttp
+from .server import ProxyServer
+
+class Geonode(ProxyServer):
+    """
+    Get Proxy From Geonode.
+    """
+    geonode_dns = None
+    base_url: str = 'http://{username}:{password}@{geonode_dns}'
+
+    def __init__(self, **kwargs):
+        self.username = kwargs.get(
+            'username',
+            os.environ.get('GEONODE_USERNAME')
+        )
+        self.password = kwargs.get(
+            'password',
+            os.environ.get('GEONODE_PASSWORD')
+        )
+        self.geonode_dns: int = kwargs.get(
+            'geonode_dns',
+            os.environ.get(
+                'GEONODE_DNS',
+                'premium-residential.geonode.com:9000'
+            )
+        )
+        self.country: str = kwargs.get('country', 'US')
+        user = f"{self.username}-country-{self.country.upper()}"
+        self.proxy_url = self.base_url.format(
+            username=user,
+            password=self.password,
+            country=self.country,
+            geonode_dns=self.geonode_dns,
+        )
+
+    async def get_list(self):
+        proxies = []
+        proxies.append(self.proxy_url)
+        return proxies
+
+
+class GeonodeFree(ProxyServer):
+    """
+    Get Proxy From Geonode using Free List.
+    """
+    base_url: str = 'https://proxylist.geonode.com/api/proxy-list'
+
+    def __init__(self, **kwargs):
+        args = {
+            "limit": kwargs.get('limit', 20),
+            "page": 1,
+            "sort_by": "lastChecked",
+            "sort_type": "desc"
+        }
+        http_only = kwargs.get('http_only', False)
+        if http_only:
+            args['protocols'] = 'http'
+        req = PreparedRequest()
+        req.prepare_url(self.base_url, args)
+        self.url = req.url
+
+    async def get_proxies(self):
+        proxies = []
+        timeout = aiohttp.ClientTimeout(total=10)
+        async with aiohttp.ClientSession(
+            timeout=timeout,
+            trust_env=True
+        ) as session:
+            try:
+                async with session.get(
+                    self.url
+                ) as response:
+                    content = await response.json()
+                    for proxy in content['data']:
+                        print('PROXY ', proxy, '\n')
+                        if proxy.get('latency') < 250:
+                            for protocol in proxy.get('protocols'):
+                                proxies.append(
+                                    f"{protocol}://{proxy['ip']}:{proxy['port']}"
+                                )
+            except aiohttp.ClientProxyConnectionError as e:
+                print(f"Proxy connection error: {e}")
+            except aiohttp.ClientHttpProxyError as e:
+                print(f"Request timed out: {e}")
+            except aiohttp.ClientError as e:
+                print(f"Client error occurred: {e}")
+            except Exception as e:
+                print(f"An unexpected error occurred: {e}")
+            except Exception as e:
+                print(f'Proxy error: {e}')
+        return proxies

proxylists/proxies/oxylabs.py

@@ -0,0 +1,65 @@
+"""
+Oxylabs Proxy information.
+"""
+import os
+import logging
+import aiohttp
+from .server import ProxyServer
+
+class Oxylabs(ProxyServer):
+    url = 'customer-{username}-cc-{country}-sesstime-{session_time}:{password}@pr.oxylabs.io:7777'
+
+    def __init__(self, **kwargs):
+        self.username = kwargs.get(
+            'username',
+            os.environ.get('OXYLABS_USERNAME')
+        )
+        self.password = kwargs.get(
+            'password',
+            os.environ.get('OXYLABS_PASSWORD')
+        )
+        self.country: str = kwargs.get('country', 'us')
+        self.timeout: int = kwargs.get('timeout', 30.0)
+        self.session_time: int = kwargs.get('session_time', 5)
+        self.customer = self.url.format(
+            username=self.username,
+            password=self.password,
+            session_time=self.session_time,
+            country=self.country,
+        )
+        self.proxy = {
+            'http': f'http://{self.customer}',
+            'https': f'http://{self.customer}',
+        }
+
+    async def get_list(self):
+        proxies = []
+        proxies.append(self.customer)
+        return proxies
+
+    async def get_proxies(self):
+        proxies = []
+        timeout = aiohttp.ClientTimeout(total=self.timeout)
+        async with aiohttp.ClientSession(
+            timeout=timeout,
+            trust_env=True
+        ) as session:
+            try:
+                async with session.get(
+                    'https://ip.oxylabs.io/location',
+                    proxy=self.proxy.get('http'),
+                    allow_redirects=True
+                ) as response:
+                    content = await response.json()
+                    proxies.append(content['ip'])
+            except aiohttp.ClientProxyConnectionError as e:
+                print(f"Proxy connection error: {e}")
+            except aiohttp.ClientHttpProxyError as e:
+                print(f"Request timed out: {e}")
+            except aiohttp.ClientError as e:
+                print(f"Client error occurred: {e}")
+            except Exception as e:
+                print(f"An unexpected error occurred: {e}")
+            except Exception as e:
+                logging.error(f'Proxy error: {e}')
+        return proxies

proxylists/proxies/proxydb.py

@@ -1,5 +1,5 @@
-from .server import ProxyServer
 import logging
+from .server import ProxyServer
 
 
 class ProxyDB(ProxyServer):

proxylists/proxies/server.py

@@ -1,10 +1,9 @@
+from abc import abstractmethod
 import aiohttp
-import asyncio
 from lxml.html import fromstring
-from abc import abstractmethod
 
 
-class ProxyServer(object):
+class ProxyServer:
     url = ""
     parser = None
 
@@ -35,3 +34,10 @@ async def get_list(self):
     @abstractmethod
     async def get_proxies(self):
         pass
+
+    async def get_proxy_list(self) -> dict:
+        proxies = await self.get_list()
+        return {
+            'http': f'http://{proxies[0]}',
+            'https': f'http://{proxies[0]}',
+        }

proxylists/proxies/smart.py

@@ -0,0 +1,38 @@
+"""
+SmartProxy Proxy information.
+"""
+import os
+from .server import ProxyServer
+
+class SmartProxy(ProxyServer):
+    """
+    SmartProxy Proxy information.
+    """
+    base_url: str = 'http://{username}:{password}@{country}.smartproxy.com:{port}'
+
+    def __init__(self, **kwargs):
+        self.username = kwargs.get(
+            'username',
+            os.environ.get('SMARTPROXY_USERNAME')
+        )
+        self.password = kwargs.get(
+            'password',
+            os.environ.get('SMARTPROXY_PASSWORD')
+        )
+        self.port: int = kwargs.get(
+            'port',
+            os.environ.get('SMARTPROXY_PORT', 10001)
+        )
+        self.country: str = kwargs.get('country', 'us')
+        user = f"{self.username}-country-{self.country}"
+        self.proxy_url = self.base_url.format(
+            username=user,
+            password=self.password,
+            country=self.country,
+            port=self.port,
+        )
+
+    async def get_list(self):
+        proxies = []
+        proxies.append(self.proxy_url)
+        return proxies

proxylists/version.py

@@ -3,7 +3,7 @@
 __title__ = 'proxylists'
 __description__ = ('Package for getting useful proxy servers, '
                    'can use lists like hidemy or proxydb.')
-__version__ = '0.13.2'
+__version__ = '0.14.0'
 __author__ = 'Jesus Lara'
 __author_email__ = '[email protected]'
 __license__ = 'BSD'