From 4309450361d24d683d2e1310a403275f4a65527e Mon Sep 17 00:00:00 2001 From: Jesus Lara Date: Tue, 8 Oct 2024 04:02:22 +0200 Subject: [PATCH] added some paid proxy services --- proxylists/proxies/__init__.py | 3 ++ proxylists/proxies/geonode.py | 96 ++++++++++++++++++++++++++++++++++ proxylists/proxies/oxylabs.py | 65 +++++++++++++++++++++++ proxylists/proxies/proxydb.py | 2 +- proxylists/proxies/server.py | 12 +++-- proxylists/proxies/smart.py | 38 ++++++++++++++ proxylists/version.py | 2 +- 7 files changed, 213 insertions(+), 5 deletions(-) create mode 100644 proxylists/proxies/geonode.py create mode 100644 proxylists/proxies/oxylabs.py create mode 100644 proxylists/proxies/smart.py diff --git a/proxylists/proxies/__init__.py b/proxylists/proxies/__init__.py index 40b4f34..452cfdb 100644 --- a/proxylists/proxies/__init__.py +++ b/proxylists/proxies/__init__.py @@ -4,6 +4,9 @@ from .proxydb import ProxyDB from .hidemy import Hidemy from .proxyworld import ProxyWorld +from .oxylabs import Oxylabs +from .smart import SmartProxy +from .geonode import Geonode, GeonodeFree __all__ = ["FreeProxy", "Hidemy", "ProxyDB"] diff --git a/proxylists/proxies/geonode.py b/proxylists/proxies/geonode.py new file mode 100644 index 0000000..78570d2 --- /dev/null +++ b/proxylists/proxies/geonode.py @@ -0,0 +1,96 @@ +""" +Geonode Proxy information. +""" +import os +from requests.models import PreparedRequest +import aiohttp +from .server import ProxyServer + +class Geonode(ProxyServer): + """ + Get Proxy From Geonode. + """ + geonode_dns = None + base_url: str = 'http://{username}:{password}@{geonode_dns}' + + def __init__(self, **kwargs): + self.username = kwargs.get( + 'username', + os.environ.get('GEONODE_USERNAME') + ) + self.password = kwargs.get( + 'password', + os.environ.get('GEONODE_PASSWORD') + ) + self.geonode_dns: int = kwargs.get( + 'geonode_dns', + os.environ.get( + 'GEONODE_DNS', + 'premium-residential.geonode.com:9000' + ) + ) + self.country: str = kwargs.get('country', 'US') + user = f"{self.username}-country-{self.country.upper()}" + self.proxy_url = self.base_url.format( + username=user, + password=self.password, + country=self.country, + geonode_dns=self.geonode_dns, + ) + + async def get_list(self): + proxies = [] + proxies.append(self.proxy_url) + return proxies + + +class GeonodeFree(ProxyServer): + """ + Get Proxy From Geonode using Free List. + """ + base_url: str = 'https://proxylist.geonode.com/api/proxy-list' + + def __init__(self, **kwargs): + args = { + "limit": kwargs.get('limit', 20), + "page": 1, + "sort_by": "lastChecked", + "sort_type": "desc" + } + http_only = kwargs.get('http_only', False) + if http_only: + args['protocols'] = 'http' + req = PreparedRequest() + req.prepare_url(self.base_url, args) + self.url = req.url + + async def get_proxies(self): + proxies = [] + timeout = aiohttp.ClientTimeout(total=10) + async with aiohttp.ClientSession( + timeout=timeout, + trust_env=True + ) as session: + try: + async with session.get( + self.url + ) as response: + content = await response.json() + for proxy in content['data']: + print('PROXY ', proxy, '\n') + if proxy.get('latency') < 250: + for protocol in proxy.get('protocols'): + proxies.append( + f"{protocol}://{proxy['ip']}:{proxy['port']}" + ) + except aiohttp.ClientProxyConnectionError as e: + print(f"Proxy connection error: {e}") + except aiohttp.ClientHttpProxyError as e: + print(f"Request timed out: {e}") + except aiohttp.ClientError as e: + print(f"Client error occurred: {e}") + except Exception as e: + print(f"An unexpected error occurred: {e}") + except Exception as e: + print(f'Proxy error: {e}') + return proxies diff --git a/proxylists/proxies/oxylabs.py b/proxylists/proxies/oxylabs.py new file mode 100644 index 0000000..6a3ccf6 --- /dev/null +++ b/proxylists/proxies/oxylabs.py @@ -0,0 +1,65 @@ +""" +Oxylabs Proxy information. +""" +import os +import logging +import aiohttp +from .server import ProxyServer + +class Oxylabs(ProxyServer): + url = 'customer-{username}-cc-{country}-sesstime-{session_time}:{password}@pr.oxylabs.io:7777' + + def __init__(self, **kwargs): + self.username = kwargs.get( + 'username', + os.environ.get('OXYLABS_USERNAME') + ) + self.password = kwargs.get( + 'password', + os.environ.get('OXYLABS_PASSWORD') + ) + self.country: str = kwargs.get('country', 'us') + self.timeout: int = kwargs.get('timeout', 30.0) + self.session_time: int = kwargs.get('session_time', 5) + self.customer = self.url.format( + username=self.username, + password=self.password, + session_time=self.session_time, + country=self.country, + ) + self.proxy = { + 'http': f'http://{self.customer}', + 'https': f'http://{self.customer}', + } + + async def get_list(self): + proxies = [] + proxies.append(self.customer) + return proxies + + async def get_proxies(self): + proxies = [] + timeout = aiohttp.ClientTimeout(total=self.timeout) + async with aiohttp.ClientSession( + timeout=timeout, + trust_env=True + ) as session: + try: + async with session.get( + 'https://ip.oxylabs.io/location', + proxy=self.proxy.get('http'), + allow_redirects=True + ) as response: + content = await response.json() + proxies.append(content['ip']) + except aiohttp.ClientProxyConnectionError as e: + print(f"Proxy connection error: {e}") + except aiohttp.ClientHttpProxyError as e: + print(f"Request timed out: {e}") + except aiohttp.ClientError as e: + print(f"Client error occurred: {e}") + except Exception as e: + print(f"An unexpected error occurred: {e}") + except Exception as e: + logging.error(f'Proxy error: {e}') + return proxies diff --git a/proxylists/proxies/proxydb.py b/proxylists/proxies/proxydb.py index 37ae98c..e23c4b2 100644 --- a/proxylists/proxies/proxydb.py +++ b/proxylists/proxies/proxydb.py @@ -1,5 +1,5 @@ -from .server import ProxyServer import logging +from .server import ProxyServer class ProxyDB(ProxyServer): diff --git a/proxylists/proxies/server.py b/proxylists/proxies/server.py index f6e33b2..1518a41 100644 --- a/proxylists/proxies/server.py +++ b/proxylists/proxies/server.py @@ -1,10 +1,9 @@ +from abc import abstractmethod import aiohttp -import asyncio from lxml.html import fromstring -from abc import abstractmethod -class ProxyServer(object): +class ProxyServer: url = "" parser = None @@ -35,3 +34,10 @@ async def get_list(self): @abstractmethod async def get_proxies(self): pass + + async def get_proxy_list(self) -> dict: + proxies = await self.get_list() + return { + 'http': f'http://{proxies[0]}', + 'https': f'http://{proxies[0]}', + } diff --git a/proxylists/proxies/smart.py b/proxylists/proxies/smart.py new file mode 100644 index 0000000..99a5774 --- /dev/null +++ b/proxylists/proxies/smart.py @@ -0,0 +1,38 @@ +""" +SmartProxy Proxy information. +""" +import os +from .server import ProxyServer + +class SmartProxy(ProxyServer): + """ + SmartProxy Proxy information. + """ + base_url: str = 'http://{username}:{password}@{country}.smartproxy.com:{port}' + + def __init__(self, **kwargs): + self.username = kwargs.get( + 'username', + os.environ.get('SMARTPROXY_USERNAME') + ) + self.password = kwargs.get( + 'password', + os.environ.get('SMARTPROXY_PASSWORD') + ) + self.port: int = kwargs.get( + 'port', + os.environ.get('SMARTPROXY_PORT', 10001) + ) + self.country: str = kwargs.get('country', 'us') + user = f"{self.username}-country-{self.country}" + self.proxy_url = self.base_url.format( + username=user, + password=self.password, + country=self.country, + port=self.port, + ) + + async def get_list(self): + proxies = [] + proxies.append(self.proxy_url) + return proxies diff --git a/proxylists/version.py b/proxylists/version.py index 83e0bb8..49fd486 100644 --- a/proxylists/version.py +++ b/proxylists/version.py @@ -3,7 +3,7 @@ __title__ = 'proxylists' __description__ = ('Package for getting useful proxy servers, ' 'can use lists like hidemy or proxydb.') -__version__ = '0.13.2' +__version__ = '0.14.0' __author__ = 'Jesus Lara' __author_email__ = 'jesuslarag@gmail.com' __license__ = 'BSD'