feature/added-v2-integration: removed return_text param, introduced post/out/delete request methods

Author: megabotan

.github/workflows/code_checks.yaml

@@ -6,7 +6,7 @@ jobs:
   code_checks:
     strategy:
       matrix:
-        pyver: [ '3.6', '3.7', '3.8', '3.9', '3.10' ]
+        pyver: ['3.7', '3.8', '3.9', '3.10', '3.11']
         os: [ ubuntu, macos, windows ]
       fail-fast: true
     runs-on: ${{ matrix.os }}-latest

README.md

@@ -23,7 +23,7 @@ python 3.6+.
 from scrapingant_client import ScrapingAntClient
 
 client = ScrapingAntClient(token='<YOUR-SCRAPINGANT-API-TOKEN>')
-# Scrape the example.com site.
+# Scrape the example.com site
 result = client.general_request('https://example.com')
 print(result.content)
 ```
@@ -62,17 +62,19 @@ Main class of this library.
 
 https://docs.scrapingant.com/request-response-format#available-parameters
 
-| Param             | Type                              | Default    |
-|-------------------|-----------------------------------|------------|
-| url               | <code>string</code>               |            |
-| cookies           | <code>List[Cookie]</code>         | None       |
-| headers           | <code>List[Dict[str, str]]</code> | None       |
-| js_snippet        | <code>string</code>               | None       |
-| proxy_type        | <code>ProxyType</code>            | datacenter | 
-| proxy_country     | <code>str</code>                  | None       | 
-| return_text       | <code>boolean</code>              | False      |
-| wait_for_selector | <code>str</code>                  | None       |
-| browser           | <code>boolean</code>              | True       |
+| Param             | Type                                                                                                                       | Default    |
+|-------------------|----------------------------------------------------------------------------------------------------------------------------|------------|
+| url               | <code>string</code>                                                                                                        |            |
+| method            | <code>string</code>                                                                                                        | GET        |
+| cookies           | <code>List[Cookie]</code>                                                                                                  | None       |
+| headers           | <code>List[Dict[str, str]]</code>                                                                                          | None       |
+| js_snippet        | <code>string</code>                                                                                                        | None       |
+| proxy_type        | <code>ProxyType</code>                                                                                                     | datacenter | 
+| proxy_country     | <code>str</code>                                                                                                           | None       | 
+| wait_for_selector | <code>str</code>                                                                                                           | None       |
+| browser           | <code>boolean</code>                                                                                                       | True       |
+| data              | same as [requests param 'data'](https://requests.readthedocs.io/en/latest/user/quickstart/#more-complicated-post-requests) | None       |
+| json              | same as [requests param 'json'](https://requests.readthedocs.io/en/latest/user/quickstart/#more-complicated-post-requests) | None       |
 
 **IMPORTANT NOTE:** <code>js_snippet</code> will be encoded to Base64 automatically by the ScrapingAnt client library.
 
@@ -82,7 +84,7 @@ https://docs.scrapingant.com/request-response-format#available-parameters
 
 Class defining cookie. Currently it supports only name and value
 
-| Param | Type                | 
+| Param |  Type               | 
 |-------|---------------------|
 | name  | <code>string</code> | 
 | value | <code>string</code> |
@@ -92,12 +94,12 @@ Class defining cookie. Currently it supports only name and value
 #### Response
 
 Class defining response from API.
-
-| Param       | Type                      |
-|-------------|---------------------------|
-| content     | <code>string</code>       |
-| cookies     | <code>List[Cookie]</code> |
-| status_code | <code>int</code>          |
+| Param       | Type                       |
+|-------------|----------------------------|
+| content     | <code>string</code>        |
+| cookies     | <code>List[Cookie]</code>  |
+| status_code | <code>int</code>           |
+| text        | <code>string</code>        |
 
 ## Exceptions
 
@@ -231,14 +233,38 @@ client = ScrapingAntClient(token='<YOUR-SCRAPINGANT-API-TOKEN>')
 
 
 async def main():
-    # Scrape the example.com site.
+    # Scrape the example.com site
     result = await client.general_request_async('https://example.com')
     print(result.content)
 
 
 asyncio.run(main())
 ```
 
+### Sending POST request
+
+```python3
+from scrapingant_client import ScrapingAntClient
+
+client = ScrapingAntClient(token='<YOUR-SCRAPINGANT-API-TOKEN>')
+
+# Sending POST request with json data
+result = client.general_request(
+    url="https://httpbin.org/post",
+    method="POST",
+    json={"test": "test"},
+)
+print(result.content)
+
+# Sending POST request with bytes data
+result = client.general_request(
+    url="https://httpbin.org/post",
+    method="POST",
+    data=b'test_bytes',
+)
+print(result.content)
+```
+
 ## Useful links
 
 - [Scrapingant API doumentation](https://docs.scrapingant.com)

scrapingant_client/__init__.py

@@ -1,4 +1,4 @@
-__version__ = "1.0.1"
+__version__ = "2.0.0"
 
 from scrapingant_client.client import ScrapingAntClient
 from scrapingant_client.cookie import Cookie

scrapingant_client/client.py

@@ -39,7 +39,6 @@ def _form_payload(
             js_snippet: Optional[str] = None,
             proxy_type: ProxyType = ProxyType.datacenter,
             proxy_country: Optional[str] = None,
-            return_text: bool = False,
             wait_for_selector: Optional[str] = None,
             browser: bool = True,
     ) -> Dict:
@@ -54,7 +53,6 @@ def _form_payload(
             request_data['proxy_country'] = proxy_country.lower()
         if wait_for_selector is not None:
             request_data['wait_for_selector'] = wait_for_selector
-        request_data['return_text'] = return_text
         request_data['browser'] = browser
         return request_data
 
@@ -69,44 +67,49 @@ def _parse_response(self, response_status_code: int, response_data: Dict, url: s
             raise ScrapingantDetectedException()
         elif response_status_code == 500:
             raise ScrapingantInternalException()
-        content = response_data['content']
+        content = response_data['html']
         cookies_string = response_data['cookies']
+        text = response_data['text']
         status_code = response_data['status_code']
         cookies_list = cookies_list_from_string(cookies_string)
         return Response(
             content=content,
             cookies=cookies_list,
+            text=text,
             status_code=status_code
         )
 
     def general_request(
             self,
             url: str,
+            method: str = 'GET',
             cookies: Optional[List[Cookie]] = None,
             headers: Optional[Dict[str, str]] = None,
             js_snippet: Optional[str] = None,
             proxy_type: ProxyType = ProxyType.datacenter,
             proxy_country: Optional[str] = None,
-            return_text: bool = False,
             wait_for_selector: Optional[str] = None,
             browser: bool = True,
+            data=None,
+            json=None,
     ) -> Response:
         request_data = self._form_payload(
             url=url,
             cookies=cookies,
             js_snippet=js_snippet,
             proxy_type=proxy_type,
             proxy_country=proxy_country,
-            return_text=return_text,
             wait_for_selector=wait_for_selector,
             browser=browser,
         )
         try:
-            response = self.requests_session.post(
-                SCRAPINGANT_API_BASE_URL + '/general',
-                json=request_data,
+            response = self.requests_session.request(
+                method=method,
+                url=SCRAPINGANT_API_BASE_URL + '/extended',
+                params=request_data,
                 headers=convert_headers(headers),
-                timeout=TIMEOUT_SECONDS
+                data=data,
+                json=json,
             )
         except requests.exceptions.Timeout:
             raise ScrapingantTimeoutException()
@@ -118,14 +121,16 @@ def general_request(
     async def general_request_async(
             self,
             url: str,
+            method: str = 'GET',
             cookies: Optional[List[Cookie]] = None,
             headers: Optional[Dict[str, str]] = None,
             js_snippet: Optional[str] = None,
             proxy_type: ProxyType = ProxyType.datacenter,
             proxy_country: Optional[str] = None,
-            return_text: bool = False,
             wait_for_selector: Optional[str] = None,
             browser: bool = True,
+            data=None,
+            json=None,
     ) -> Response:
         import httpx
 
@@ -135,7 +140,6 @@ async def general_request_async(
             js_snippet=js_snippet,
             proxy_type=proxy_type,
             proxy_country=proxy_country,
-            return_text=return_text,
             wait_for_selector=wait_for_selector,
             browser=browser,
         )
@@ -147,14 +151,16 @@ async def general_request_async(
                 timeout=TIMEOUT_SECONDS,
         ) as client:
             try:
-                response = await client.post(
-                    SCRAPINGANT_API_BASE_URL + '/general',
-                    json=request_data,
+                response = await client.request(
+                    method=method,
+                    url=SCRAPINGANT_API_BASE_URL + '/extended',
+                    params=request_data,
                     headers=convert_headers(headers),
+                    data=data,
+                    json=json,
                 )
             except httpx.TimeoutException:
                 raise ScrapingantTimeoutException()
-
         response_status_code = response.status_code
         response_data = response.json()
         parsed_response: Response = self._parse_response(response_status_code, response_data, url)

scrapingant_client/constants.py

@@ -1,2 +1,2 @@
-SCRAPINGANT_API_BASE_URL = 'https://api.scrapingant.com/v1'
+SCRAPINGANT_API_BASE_URL = 'https://api.scrapingant.com/v2'
 TIMEOUT_SECONDS = 120

scrapingant_client/proxy_type.py

@@ -4,3 +4,6 @@
 class ProxyType(str, Enum):
     datacenter = 'datacenter'
     residential = 'residential'
+
+    def __str__(self):
+        return self.value

scrapingant_client/response.py

@@ -4,7 +4,8 @@
 
 
 class Response:
-    def __init__(self, content: str, cookies: List[Cookie], status_code: Optional[int]):
+    def __init__(self, content: str, cookies: List[Cookie], text: str, status_code: Optional[int]):
         self.content = content
         self.cookies = cookies
+        self.text = text
         self.status_code = status_code

setup.py

@@ -34,7 +34,7 @@
         "License :: OSI Approved :: Apache Software License",
     ],
     keywords="scrapingant api scraper scraping",
-    python_requires='~=3.5',
+    python_requires='~=3.7',
     install_requires=['requests>=2,<3'],
     extras_require={
         'dev': [

tests/test_exceptions.py

@@ -16,7 +16,7 @@
 
 @responses.activate
 def test_invalid_token():
-    responses.add(responses.POST, SCRAPINGANT_API_BASE_URL + '/general',
+    responses.add(responses.GET, SCRAPINGANT_API_BASE_URL + '/extended',
                   json={'detail': 'wrong token'}, status=403)
     client = ScrapingAntClient(token='invalid_token')
     with pytest.raises(ScrapingantInvalidTokenException):
@@ -25,7 +25,7 @@ def test_invalid_token():
 
 @responses.activate
 def test_invalid_input():
-    responses.add(responses.POST, SCRAPINGANT_API_BASE_URL + '/general',
+    responses.add(responses.GET, SCRAPINGANT_API_BASE_URL + '/extended',
                   json={'detail': 'wrong url'}, status=422)
     client = ScrapingAntClient(token='some_token')
     with pytest.raises(ScrapingantInvalidInputException) as e:
@@ -35,7 +35,7 @@ def test_invalid_input():
 
 @responses.activate
 def test_internal_server_error():
-    responses.add(responses.POST, SCRAPINGANT_API_BASE_URL + '/general',
+    responses.add(responses.GET, SCRAPINGANT_API_BASE_URL + '/extended',
                   json={}, status=500)
     client = ScrapingAntClient(token='some_token')
     with pytest.raises(ScrapingantInternalException):
@@ -44,7 +44,7 @@ def test_internal_server_error():
 
 @responses.activate
 def test_not_reachable():
-    responses.add(responses.POST, SCRAPINGANT_API_BASE_URL + '/general',
+    responses.add(responses.GET, SCRAPINGANT_API_BASE_URL + '/extended',
                   json={}, status=404)
     client = ScrapingAntClient(token='some_token')
     with pytest.raises(ScrapingantSiteNotReachableException) as e:
@@ -54,7 +54,7 @@ def test_not_reachable():
 
 @responses.activate
 def test_detected():
-    responses.add(responses.POST, SCRAPINGANT_API_BASE_URL + '/general',
+    responses.add(responses.GET, SCRAPINGANT_API_BASE_URL + '/extended',
                   json={}, status=423)
     client = ScrapingAntClient(token='some_token')
     with pytest.raises(ScrapingantDetectedException) as e: