diff --git a/requirements.txt b/requirements.txt index aa74466..95cc56f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,7 +12,7 @@ attrs==25.4.0 # referencing babel==2.17.0 # via mkdocs-material -backrefs==5.9 +backrefs==6.0.1 # via mkdocs-material cachetools==6.2.1 # via google-auth @@ -22,9 +22,9 @@ cfgv==3.4.0 # via pre-commit chardet==5.2.0 # via frictionless -charset-normalizer==3.4.3 +charset-normalizer==3.4.4 # via requests -ckanapi==4.8 +ckanapi==4.9 # via hdx-python-api (pyproject.toml) click==8.3.0 # via @@ -32,7 +32,7 @@ click==8.3.0 # typer colorama==0.4.6 # via mkdocs-material -coverage==7.10.7 +coverage==7.11.3 # via pytest-cov defopt==7.0.0 # via hdx-python-api (pyproject.toml) @@ -44,7 +44,7 @@ docopt==0.6.2 # via # ckanapi # num2words -docutils==0.22.2 +docutils==0.22.3 # via defopt email-validator==2.3.0 # via hdx-python-api (pyproject.toml) @@ -56,7 +56,7 @@ frictionless==5.18.1 # via hdx-python-utilities ghp-import==2.1.0 # via mkdocs -google-auth==2.41.1 +google-auth==2.43.0 # via # google-auth-oauthlib # gspread @@ -70,7 +70,7 @@ hdx-python-utilities==3.9.5 # via # hdx-python-api (pyproject.toml) # hdx-python-country -humanize==4.13.0 +humanize==4.14.0 # via frictionless identify==2.6.15 # via pre-commit @@ -82,7 +82,7 @@ ijson==3.4.0.post0 # via hdx-python-utilities inflect==7.5.0 # via quantulum3 -iniconfig==2.1.0 +iniconfig==2.3.0 # via pytest isodate==0.7.2 # via frictionless @@ -110,14 +110,14 @@ loguru==0.7.3 # via hdx-python-utilities makefun==1.16.0 # via hdx-python-api (pyproject.toml) -markdown==3.9 +markdown==3.10 # via # mkdocs # mkdocs-material # pymdown-extensions markdown-it-py==4.0.0 # via rich -marko==2.2.0 +marko==2.2.1 # via frictionless markupsafe==3.0.3 # via @@ -137,7 +137,7 @@ mkdocs==1.6.1 # mkdocs-material mkdocs-get-deps==0.2.0 # via mkdocs -mkdocs-material==9.6.21 +mkdocs-material==9.7.0 # via mkapi mkdocs-material-extensions==1.3.1 # via mkdocs-material @@ -175,7 +175,7 @@ ply==3.11 # libhxl pockets==0.9.1 # via sphinxcontrib-napoleon -pre-commit==4.3.0 +pre-commit==4.4.0 # via hdx-python-api (pyproject.toml) pyasn1==0.6.1 # via @@ -183,20 +183,20 @@ pyasn1==0.6.1 # rsa pyasn1-modules==0.4.2 # via google-auth -pydantic==2.12.0 +pydantic==2.12.4 # via frictionless -pydantic-core==2.41.1 +pydantic-core==2.41.5 # via pydantic pygments==2.19.2 # via # mkdocs-material # pytest # rich -pymdown-extensions==10.16.1 +pymdown-extensions==10.17.1 # via mkdocs-material pyphonetics==0.5.3 # via hdx-python-utilities -pytest==8.4.2 +pytest==9.0.0 # via # hdx-python-api (pyproject.toml) # pytest-check @@ -232,7 +232,7 @@ quantulum3==0.9.2 # via hdx-python-api (pyproject.toml) ratelimit==2.2.1 # via hdx-python-utilities -referencing==0.36.2 +referencing==0.37.0 # via # jsonschema # jsonschema-specifications @@ -245,7 +245,7 @@ requests==2.32.5 # mkdocs-material # requests-file # requests-oauthlib -requests-file==2.1.0 +requests-file==3.0.1 # via hdx-python-utilities requests-oauthlib==2.0.0 # via google-auth-oauthlib @@ -253,13 +253,13 @@ rfc3986==2.0.0 # via frictionless rich==14.2.0 # via typer -rpds-py==0.27.1 +rpds-py==0.28.0 # via # jsonschema # referencing rsa==4.9.1 # via google-auth -ruamel-yaml==0.18.15 +ruamel-yaml==0.18.16 # via hdx-python-utilities ruamel-yaml-clib==0.2.14 # via ruamel-yaml @@ -279,7 +279,7 @@ six==1.17.0 # sphinxcontrib-napoleon sphinxcontrib-napoleon==0.7 # via defopt -structlog==25.4.0 +structlog==25.5.0 # via libhxl tableschema-to-template==0.0.13 # via hdx-python-utilities @@ -291,7 +291,7 @@ text-unidecode==1.3 # via python-slugify typeguard==4.4.4 # via inflect -typer==0.19.2 +typer==0.20.0 # via frictionless typing-extensions==4.15.0 # via @@ -313,7 +313,7 @@ urllib3==2.5.0 # requests validators==0.35.0 # via frictionless -virtualenv==20.35.3 +virtualenv==20.35.4 # via pre-commit watchdog==6.0.0 # via mkdocs diff --git a/src/hdx/data/resource.py b/src/hdx/data/resource.py index 187bca0..fa6ae34 100755 --- a/src/hdx/data/resource.py +++ b/src/hdx/data/resource.py @@ -15,6 +15,7 @@ from hdx.data.resource_view import ResourceView from hdx.utilities.dateparse import now_utc, now_utc_notz, parse_date from hdx.utilities.downloader import Download +from hdx.utilities.retriever import Retrieve from hdx.utilities.typehint import ListTuple from hdx.utilities.uuid import is_valid_uuid @@ -591,12 +592,15 @@ def search_in_hdx( logger.debug(result) return resources - def download(self, folder: Optional[str] = None) -> Tuple[str, str]: + def download( + self, folder: Optional[str] = None, retriever: Optional[Retrieve] = None + ) -> Tuple[str, str]: """Download resource store to provided folder or temporary folder if no folder supplied Args: folder (Optional[str]): Folder to download resource to. Defaults to None. + retriever (Optional[Retrieve]): Retrieve object to use. Defaults to None. Returns: Tuple[str, str]: (URL downloaded, Path to downloaded file) @@ -613,7 +617,7 @@ def download(self, folder: Optional[str] = None) -> Tuple[str, str]: filename = f"{filename}{file_format}" apikey = self.configuration.get_api_key() if apikey: - headers = {"Authorization": self.configuration.get_api_key()} + headers = {"Authorization": apikey} else: headers = None with Download( @@ -621,6 +625,8 @@ def download(self, folder: Optional[str] = None) -> Tuple[str, str]: use_env=False, headers=headers, ) as downloader: + if retriever: + downloader = retriever.clone(downloader) path = downloader.download_file(url, folder=folder, filename=filename) return url, path diff --git a/tests/hdx/data/test_resource.py b/tests/hdx/data/test_resource.py index 45688b2..b07b926 100755 --- a/tests/hdx/data/test_resource.py +++ b/tests/hdx/data/test_resource.py @@ -16,7 +16,9 @@ from hdx.data.resource import Resource from hdx.utilities.dateparse import parse_date from hdx.utilities.dictandlist import merge_two_dictionaries -from hdx.utilities.downloader import DownloadError +from hdx.utilities.downloader import Download, DownloadError +from hdx.utilities.path import get_temp_dir +from hdx.utilities.retriever import Retrieve resultdict = { "cache_last_updated": None, @@ -940,7 +942,6 @@ def test_search_in_hdx(self, configuration, search): def test_download(self, configuration, read): resource = Resource.read_from_hdx("74b74ae1-df0c-4716-829f-4f939a046811") - resource2 = Resource.read_from_hdx("74b74ae1-df0c-4716-829f-4f939a046814") url, path = resource.download() remove(path) assert ( @@ -948,9 +949,19 @@ def test_download(self, configuration, read): == "https://raw.githubusercontent.com/OCHA-DAP/hdx-python-api/main/tests/fixtures/test_data.csv" ) assert basename(path) == "MyResource1.csv" + tempdir = get_temp_dir() + retriever = Retrieve(Download(user_agent="test"), tempdir, tempdir, tempdir) + url, path = resource.download(retriever=retriever) + remove(path) + assert ( + url + == "https://raw.githubusercontent.com/OCHA-DAP/hdx-python-api/main/tests/fixtures/test_data.csv" + ) + resource["url"] = "" with pytest.raises(HDXError): resource.download() + resource2 = Resource.read_from_hdx("74b74ae1-df0c-4716-829f-4f939a046814") with pytest.raises(DownloadError): resource2.download()