Skip to content

Include timeout because sometimes it takes too long #6

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 1 commit into
base: main
Choose a base branch
from
Open

Include timeout because sometimes it takes too long #6

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
44 changes: 17 additions & 27 deletions zenrows/client.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,20 +5,19 @@
from concurrent.futures import ThreadPoolExecutor
import urllib3
from functools import partial

from .__version__ import __version__


class ZenRowsClient:
api_url = "https://api.zenrows.com/v1/"

def __init__(self, apikey: str, retries: int = 0, concurrency: int = 5):
def __init__(self, apikey: str, retries: int = 0, concurrency: int = 5, timeout: int = 10):
self.apikey = apikey
self.timeout = timeout

self.executor = ThreadPoolExecutor(max_workers=concurrency)

self.requests_session = requests.Session()
if (retries > 0):

if retries > 0:
max_retries = Retry().new(
total=retries,
backoff_factor=0.5,
Expand All @@ -29,41 +28,32 @@ def __init__(self, apikey: str, retries: int = 0, concurrency: int = 5):
self.requests_session.mount("https://", adapter)
self.requests_session.mount("http://", adapter)

def get(
self, url: str, params: dict = None, headers: dict = None, **kwargs
) -> requests.Response:
def get(self, url: str, params: dict = None, headers: dict = None, **kwargs) -> requests.Response:
return self._worker("GET", url, params, headers, **kwargs)

async def get_async(
self, url: str, params: dict = None, headers: dict = None, **kwargs
) -> requests.Response:
async def get_async(self, url: str, params: dict = None, headers: dict = None, **kwargs) -> requests.Response:
loop = asyncio.get_event_loop()
return await loop.run_in_executor(self.executor, partial(self._worker, "GET", url, params, headers, **kwargs))
return await loop.run_in_executor(
self.executor, partial(self._worker, "GET", url, params, headers, **kwargs)
)

def post(
self, url: str, params: dict = None, headers: dict = None, data: dict = None, **kwargs
) -> requests.Response:
def post(self, url: str, params: dict = None, headers: dict = None, data: dict = None, **kwargs) -> requests.Response:
return self._worker("POST", url, params, headers, data, **kwargs)

async def post_async(
self, url: str, params: dict = None, headers: dict = None, data: dict = None, **kwargs
) -> requests.Response:
async def post_async(self, url: str, params: dict = None, headers: dict = None, data: dict = None, **kwargs) -> requests.Response:
loop = asyncio.get_event_loop()
return await loop.run_in_executor(self.executor, partial(self._worker, "POST", url, params, headers, data, **kwargs))
return await loop.run_in_executor(
self.executor, partial(self._worker, "POST", url, params, headers, data, **kwargs)
)

def _worker(
self, method, url: str, params: dict = None, headers: dict = None, data: dict = None, **kwargs
):
final_params = {}
def _worker(self, method, url: str, params: dict = None, headers: dict = None, data: dict = None, **kwargs):
final_params = {"url": url, "apikey": self.apikey}
if params:
final_params.update(params)
final_params.update({"url": url, "apikey": self.apikey})

final_headers = {"User-Agent": f"zenrows/{__version__} python"}

if headers:
final_params["custom_headers"] = True

final_headers["Accept"] = None
final_headers["Accept-Encoding"] = urllib3.util.SKIP_HEADER
final_headers["Connection"] = None
Expand All @@ -73,5 +63,5 @@ def _worker(
final_headers.update(headers)

return self.requests_session.request(
method, self.api_url, params=final_params, headers=final_headers, data=data, **kwargs
method, self.api_url, params=final_params, headers=final_headers, data=data, timeout=self.timeout, **kwargs
)