Skip to content

HDXDSYS-2507 Add optional retriever parameter to resource download method of HDX Python API #94

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
mcarans merged 2 commits into from
Nov 12, 2025
Merged

HDXDSYS-2507 Add optional retriever parameter to resource download method of HDX Python API #94

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
212dfd4
Allow retriever to be passed to download method
mcarans Nov 12, 2025
0b1f278
Allow retriever to be passed to download method
mcarans Nov 12, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
46 changes: 23 additions & 23 deletions requirements.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ attrs==25.4.0
# referencing
babel==2.17.0
# via mkdocs-material
backrefs==5.9
backrefs==6.0.1
# via mkdocs-material
cachetools==6.2.1
# via google-auth
Expand All @@ -22,17 +22,17 @@ cfgv==3.4.0
# via pre-commit
chardet==5.2.0
# via frictionless
charset-normalizer==3.4.3
charset-normalizer==3.4.4
# via requests
ckanapi==4.8
ckanapi==4.9
# via hdx-python-api (pyproject.toml)
click==8.3.0
# via
# mkdocs
# typer
colorama==0.4.6
# via mkdocs-material
coverage==7.10.7
coverage==7.11.3
# via pytest-cov
defopt==7.0.0
# via hdx-python-api (pyproject.toml)
Expand All @@ -44,7 +44,7 @@ docopt==0.6.2
# via
# ckanapi
# num2words
docutils==0.22.2
docutils==0.22.3
# via defopt
email-validator==2.3.0
# via hdx-python-api (pyproject.toml)
Expand All @@ -56,7 +56,7 @@ frictionless==5.18.1
# via hdx-python-utilities
ghp-import==2.1.0
# via mkdocs
google-auth==2.41.1
google-auth==2.43.0
# via
# google-auth-oauthlib
# gspread
Expand All @@ -70,7 +70,7 @@ hdx-python-utilities==3.9.5
# via
# hdx-python-api (pyproject.toml)
# hdx-python-country
humanize==4.13.0
humanize==4.14.0
# via frictionless
identify==2.6.15
# via pre-commit
Expand All @@ -82,7 +82,7 @@ ijson==3.4.0.post0
# via hdx-python-utilities
inflect==7.5.0
# via quantulum3
iniconfig==2.1.0
iniconfig==2.3.0
# via pytest
isodate==0.7.2
# via frictionless
Expand Down Expand Up @@ -110,14 +110,14 @@ loguru==0.7.3
# via hdx-python-utilities
makefun==1.16.0
# via hdx-python-api (pyproject.toml)
markdown==3.9
markdown==3.10
# via
# mkdocs
# mkdocs-material
# pymdown-extensions
markdown-it-py==4.0.0
# via rich
marko==2.2.0
marko==2.2.1
# via frictionless
markupsafe==3.0.3
# via
Expand All @@ -137,7 +137,7 @@ mkdocs==1.6.1
# mkdocs-material
mkdocs-get-deps==0.2.0
# via mkdocs
mkdocs-material==9.6.21
mkdocs-material==9.7.0
# via mkapi
mkdocs-material-extensions==1.3.1
# via mkdocs-material
Expand Down Expand Up @@ -175,28 +175,28 @@ ply==3.11
# libhxl
pockets==0.9.1
# via sphinxcontrib-napoleon
pre-commit==4.3.0
pre-commit==4.4.0
# via hdx-python-api (pyproject.toml)
pyasn1==0.6.1
# via
# pyasn1-modules
# rsa
pyasn1-modules==0.4.2
# via google-auth
pydantic==2.12.0
pydantic==2.12.4
# via frictionless
pydantic-core==2.41.1
pydantic-core==2.41.5
# via pydantic
pygments==2.19.2
# via
# mkdocs-material
# pytest
# rich
pymdown-extensions==10.16.1
pymdown-extensions==10.17.1
# via mkdocs-material
pyphonetics==0.5.3
# via hdx-python-utilities
pytest==8.4.2
pytest==9.0.0
# via
# hdx-python-api (pyproject.toml)
# pytest-check
Expand Down Expand Up @@ -232,7 +232,7 @@ quantulum3==0.9.2
# via hdx-python-api (pyproject.toml)
ratelimit==2.2.1
# via hdx-python-utilities
referencing==0.36.2
referencing==0.37.0
# via
# jsonschema
# jsonschema-specifications
Expand All @@ -245,21 +245,21 @@ requests==2.32.5
# mkdocs-material
# requests-file
# requests-oauthlib
requests-file==2.1.0
requests-file==3.0.1
# via hdx-python-utilities
requests-oauthlib==2.0.0
# via google-auth-oauthlib
rfc3986==2.0.0
# via frictionless
rich==14.2.0
# via typer
rpds-py==0.27.1
rpds-py==0.28.0
# via
# jsonschema
# referencing
rsa==4.9.1
# via google-auth
ruamel-yaml==0.18.15
ruamel-yaml==0.18.16
# via hdx-python-utilities
ruamel-yaml-clib==0.2.14
# via ruamel-yaml
Expand All @@ -279,7 +279,7 @@ six==1.17.0
# sphinxcontrib-napoleon
sphinxcontrib-napoleon==0.7
# via defopt
structlog==25.4.0
structlog==25.5.0
# via libhxl
tableschema-to-template==0.0.13
# via hdx-python-utilities
Expand All @@ -291,7 +291,7 @@ text-unidecode==1.3
# via python-slugify
typeguard==4.4.4
# via inflect
typer==0.19.2
typer==0.20.0
# via frictionless
typing-extensions==4.15.0
# via
Expand All @@ -313,7 +313,7 @@ urllib3==2.5.0
# requests
validators==0.35.0
# via frictionless
virtualenv==20.35.3
virtualenv==20.35.4
# via pre-commit
watchdog==6.0.0
# via mkdocs
Expand Down
10 changes: 8 additions & 2 deletions src/hdx/data/resource.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
from hdx.data.resource_view import ResourceView
from hdx.utilities.dateparse import now_utc, now_utc_notz, parse_date
from hdx.utilities.downloader import Download
from hdx.utilities.retriever import Retrieve
from hdx.utilities.typehint import ListTuple
from hdx.utilities.uuid import is_valid_uuid

Expand Down Expand Up @@ -591,12 +592,15 @@ def search_in_hdx(
logger.debug(result)
return resources

def download(self, folder: Optional[str] = None) -> Tuple[str, str]:
def download(
self, folder: Optional[str] = None, retriever: Optional[Retrieve] = None
) -> Tuple[str, str]:
"""Download resource store to provided folder or temporary folder if no folder
supplied

Args:
folder (Optional[str]): Folder to download resource to. Defaults to None.
retriever (Optional[Retrieve]): Retrieve object to use. Defaults to None.

Returns:
Tuple[str, str]: (URL downloaded, Path to downloaded file)
Expand All @@ -613,14 +617,16 @@ def download(self, folder: Optional[str] = None) -> Tuple[str, str]:
filename = f"{filename}{file_format}"
apikey = self.configuration.get_api_key()
if apikey:
headers = {"Authorization": self.configuration.get_api_key()}
headers = {"Authorization": apikey}
else:
headers = None
with Download(
full_agent=self.configuration.get_user_agent(),
use_env=False,
headers=headers,
) as downloader:
if retriever:
downloader = retriever.clone(downloader)
path = downloader.download_file(url, folder=folder, filename=filename)
return url, path

Expand Down
15 changes: 13 additions & 2 deletions tests/hdx/data/test_resource.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,9 @@
from hdx.data.resource import Resource
from hdx.utilities.dateparse import parse_date
from hdx.utilities.dictandlist import merge_two_dictionaries
from hdx.utilities.downloader import DownloadError
from hdx.utilities.downloader import Download, DownloadError
from hdx.utilities.path import get_temp_dir
from hdx.utilities.retriever import Retrieve

resultdict = {
"cache_last_updated": None,
Expand Down Expand Up @@ -940,17 +942,26 @@ def test_search_in_hdx(self, configuration, search):

def test_download(self, configuration, read):
resource = Resource.read_from_hdx("74b74ae1-df0c-4716-829f-4f939a046811")
resource2 = Resource.read_from_hdx("74b74ae1-df0c-4716-829f-4f939a046814")
url, path = resource.download()
remove(path)
assert (
url
== "https://raw.githubusercontent.com/OCHA-DAP/hdx-python-api/main/tests/fixtures/test_data.csv"
)
assert basename(path) == "MyResource1.csv"
tempdir = get_temp_dir()
retriever = Retrieve(Download(user_agent="test"), tempdir, tempdir, tempdir)
url, path = resource.download(retriever=retriever)
remove(path)
assert (
url
== "https://raw.githubusercontent.com/OCHA-DAP/hdx-python-api/main/tests/fixtures/test_data.csv"
)

resource["url"] = ""
with pytest.raises(HDXError):
resource.download()
resource2 = Resource.read_from_hdx("74b74ae1-df0c-4716-829f-4f939a046814")
with pytest.raises(DownloadError):
resource2.download()

Expand Down
Loading