From d7af4fc2754810c4a1aeeb9c4b5e657003381dd3 Mon Sep 17 00:00:00 2001 From: Shawn Elbaz Date: Tue, 2 Jan 2024 18:57:22 -0500 Subject: [PATCH] Update client.py --- zenrows/client.py | 44 +++++++++++++++++--------------------------- 1 file changed, 17 insertions(+), 27 deletions(-) diff --git a/zenrows/client.py b/zenrows/client.py index e3123aa..0de5bdb 100644 --- a/zenrows/client.py +++ b/zenrows/client.py @@ -5,20 +5,19 @@ from concurrent.futures import ThreadPoolExecutor import urllib3 from functools import partial - from .__version__ import __version__ - class ZenRowsClient: api_url = "https://api.zenrows.com/v1/" - def __init__(self, apikey: str, retries: int = 0, concurrency: int = 5): + def __init__(self, apikey: str, retries: int = 0, concurrency: int = 5, timeout: int = 10): self.apikey = apikey + self.timeout = timeout self.executor = ThreadPoolExecutor(max_workers=concurrency) - self.requests_session = requests.Session() - if (retries > 0): + + if retries > 0: max_retries = Retry().new( total=retries, backoff_factor=0.5, @@ -29,41 +28,32 @@ def __init__(self, apikey: str, retries: int = 0, concurrency: int = 5): self.requests_session.mount("https://", adapter) self.requests_session.mount("http://", adapter) - def get( - self, url: str, params: dict = None, headers: dict = None, **kwargs - ) -> requests.Response: + def get(self, url: str, params: dict = None, headers: dict = None, **kwargs) -> requests.Response: return self._worker("GET", url, params, headers, **kwargs) - async def get_async( - self, url: str, params: dict = None, headers: dict = None, **kwargs - ) -> requests.Response: + async def get_async(self, url: str, params: dict = None, headers: dict = None, **kwargs) -> requests.Response: loop = asyncio.get_event_loop() - return await loop.run_in_executor(self.executor, partial(self._worker, "GET", url, params, headers, **kwargs)) + return await loop.run_in_executor( + self.executor, partial(self._worker, "GET", url, params, headers, **kwargs) + ) - def post( - self, url: str, params: dict = None, headers: dict = None, data: dict = None, **kwargs - ) -> requests.Response: + def post(self, url: str, params: dict = None, headers: dict = None, data: dict = None, **kwargs) -> requests.Response: return self._worker("POST", url, params, headers, data, **kwargs) - async def post_async( - self, url: str, params: dict = None, headers: dict = None, data: dict = None, **kwargs - ) -> requests.Response: + async def post_async(self, url: str, params: dict = None, headers: dict = None, data: dict = None, **kwargs) -> requests.Response: loop = asyncio.get_event_loop() - return await loop.run_in_executor(self.executor, partial(self._worker, "POST", url, params, headers, data, **kwargs)) + return await loop.run_in_executor( + self.executor, partial(self._worker, "POST", url, params, headers, data, **kwargs) + ) - def _worker( - self, method, url: str, params: dict = None, headers: dict = None, data: dict = None, **kwargs - ): - final_params = {} + def _worker(self, method, url: str, params: dict = None, headers: dict = None, data: dict = None, **kwargs): + final_params = {"url": url, "apikey": self.apikey} if params: final_params.update(params) - final_params.update({"url": url, "apikey": self.apikey}) final_headers = {"User-Agent": f"zenrows/{__version__} python"} - if headers: final_params["custom_headers"] = True - final_headers["Accept"] = None final_headers["Accept-Encoding"] = urllib3.util.SKIP_HEADER final_headers["Connection"] = None @@ -73,5 +63,5 @@ def _worker( final_headers.update(headers) return self.requests_session.request( - method, self.api_url, params=final_params, headers=final_headers, data=data, **kwargs + method, self.api_url, params=final_params, headers=final_headers, data=data, timeout=self.timeout, **kwargs )