From afb042833a7118670345176f8835e3dc1dc434c4 Mon Sep 17 00:00:00 2001 From: Daniel Dale Date: Sun, 13 Oct 2024 13:49:56 -0700 Subject: [PATCH] add support for ecosystem <= `2.5.0`, prepare for HF `clean_up_tokenization_spaces` behavior change, add reference to FTS enhanced support for `ModelParallelStrategy` enabling use of PyTorch composable distributed (e.g. ``fully_shard``, ``checkpoint``) and Tensor Parallelism (TP) APIs, temporarily disable docformatter pre-commit hook until an outstanding fix is merged --- .../finetuning-scheduler/finetuning-scheduler.py | 7 ++++++- lightning_examples/finetuning-scheduler/requirements.txt | 2 +- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/lightning_examples/finetuning-scheduler/finetuning-scheduler.py b/lightning_examples/finetuning-scheduler/finetuning-scheduler.py index 7cb539c0b..2b275380d 100644 --- a/lightning_examples/finetuning-scheduler/finetuning-scheduler.py +++ b/lightning_examples/finetuning-scheduler/finetuning-scheduler.py @@ -147,6 +147,8 @@ # # - ``ddp`` (and aliases ``ddp_find_unused_parameters_false``, ``ddp_find_unused_parameters_true``, ``ddp_spawn``, ``ddp_fork``, ``ddp_notebook``) # - ``fsdp`` (and alias ``fsdp_cpu_offload``) +# - **NEW**: ``ModelParallelStrategy`` +# - [See this example](https://finetuning-scheduler.readthedocs.io/en/stable/distributed/model_parallel_scheduled_fine_tuning.html) using FTS with PyTorch's composable distributed (e.g. ``fully_shard``, ``checkpoint``) and Tensor Parallelism (TP) APIs # # Custom or officially unsupported strategies can be used by setting [FinetuningScheduler.allow_untested](https://finetuning-scheduler.readthedocs.io/en/stable/api/finetuning_scheduler.fts.html?highlight=allow_untested#finetuning_scheduler.fts.FinetuningScheduler.params.allow_untested) to ``True``. # Note that most currently unsupported strategies are so because they require varying degrees of modification to be compatible. For example, ``deepspeed`` will require a [StrategyAdapter](https://finetuning-scheduler.readthedocs.io/en/stable/api/finetuning_scheduler.strategy_adapters.html#finetuning_scheduler.strategy_adapters.StrategyAdapter) to be written (similar to the one for ``FSDP``, [FSDPStrategyAdapter](https://finetuning-scheduler.readthedocs.io/en/stable/api/finetuning_scheduler.strategy_adapters.html#finetuning_scheduler.strategy_adapters.FSDPStrategyAdapter)) before support can be added (PRs welcome!), @@ -260,7 +262,10 @@ def __init__( self.save_hyperparameters() os.environ["TOKENIZERS_PARALLELISM"] = "true" if self.hparams.tokenizers_parallelism else "false" self.tokenizer = AutoTokenizer.from_pretrained( - self.hparams.model_name_or_path, use_fast=True, local_files_only=False + self.hparams.model_name_or_path, + use_fast=True, + local_files_only=False, + clean_up_tokenization_spaces=True, ) def prepare_data(self): diff --git a/lightning_examples/finetuning-scheduler/requirements.txt b/lightning_examples/finetuning-scheduler/requirements.txt index cdc2b0cff..9f19087c1 100644 --- a/lightning_examples/finetuning-scheduler/requirements.txt +++ b/lightning_examples/finetuning-scheduler/requirements.txt @@ -1,2 +1,2 @@ datasets >=2.17.0 # to allow explicitly setting `trust_remote_code` -finetuning-scheduler[examples] <=2.4.0 +finetuning-scheduler[examples] <=2.5.0