Skip to content

Commit 4e90bf3

Browse files
authored
Support context mananger in commit scheduler (#2670)
1 parent 1c10aaa commit 4e90bf3

File tree

2 files changed

+45
-2
lines changed

2 files changed

+45
-2
lines changed

src/huggingface_hub/_commit_scheduler.py

Lines changed: 28 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -30,8 +30,9 @@ class CommitScheduler:
3030
"""
3131
Scheduler to upload a local folder to the Hub at regular intervals (e.g. push to hub every 5 minutes).
3232
33-
The scheduler is started when instantiated and run indefinitely. At the end of your script, a last commit is
34-
triggered. Checkout the [upload guide](https://huggingface.co/docs/huggingface_hub/guides/upload#scheduled-uploads)
33+
The recommended way to use the scheduler is to use it as a context manager. This ensures that the scheduler is
34+
properly stopped and the last commit is triggered when the script ends. The scheduler can also be stopped manually
35+
with the `stop` method. Checkout the [upload guide](https://huggingface.co/docs/huggingface_hub/guides/upload#scheduled-uploads)
3536
to learn more about how to use it.
3637
3738
Args:
@@ -78,6 +79,22 @@ class CommitScheduler:
7879
>>> with csv_path.open("a") as f:
7980
... f.write("second line")
8081
```
82+
83+
Example using a context manager:
84+
```py
85+
>>> from pathlib import Path
86+
>>> from huggingface_hub import CommitScheduler
87+
88+
>>> with CommitScheduler(repo_id="test_scheduler", repo_type="dataset", folder_path="watched_folder", every=10) as scheduler:
89+
... csv_path = Path("watched_folder/data.csv")
90+
... with csv_path.open("a") as f:
91+
... f.write("first line")
92+
... (...)
93+
... with csv_path.open("a") as f:
94+
... f.write("second line")
95+
96+
# Scheduler is now stopped and last commit have been triggered
97+
```
8198
"""
8299

83100
def __init__(
@@ -144,6 +161,15 @@ def stop(self) -> None:
144161
"""
145162
self.__stopped = True
146163

164+
def __enter__(self) -> "CommitScheduler":
165+
return self
166+
167+
def __exit__(self, exc_type, exc_value, traceback) -> None:
168+
# Upload last changes before exiting
169+
self.trigger().result()
170+
self.stop()
171+
return
172+
147173
def _run_scheduler(self) -> None:
148174
"""Dumb thread waiting between each scheduled push to Hub."""
149175
while True:

tests/test_commit_scheduler.py

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -160,6 +160,23 @@ def test_sync_and_squash_history(self) -> None:
160160
self.assertEqual(len(commits), 1)
161161
self.assertEqual(commits[0].title, "Super-squash branch 'main' using huggingface_hub")
162162

163+
def test_context_manager(self) -> None:
164+
watched_folder = self.cache_dir / "watched_folder"
165+
watched_folder.mkdir(exist_ok=True, parents=True)
166+
file_path = watched_folder / "file.txt"
167+
168+
with CommitScheduler(
169+
folder_path=watched_folder,
170+
repo_id=self.repo_name,
171+
every=5, # every 5min
172+
hf_api=self.api,
173+
) as scheduler:
174+
with file_path.open("w") as f:
175+
f.write("first line\n")
176+
177+
assert "file.txt" in self.api.list_repo_files(scheduler.repo_id)
178+
assert scheduler._CommitScheduler__stopped # means the scheduler has been stopped when exiting the context
179+
163180

164181
@pytest.mark.usefixtures("fx_cache_dir")
165182
class TestPartialFileIO(unittest.TestCase):

0 commit comments

Comments
 (0)