diff --git a/.github/workflows/job_jax_layer_tests.yml b/.github/workflows/job_jax_layer_tests.yml index dd57fad77645aa..5f3efd08abef56 100644 --- a/.github/workflows/job_jax_layer_tests.yml +++ b/.github/workflows/job_jax_layer_tests.yml @@ -48,6 +48,8 @@ jobs: INSTALL_DIR: ${{ github.workspace }}/install INSTALL_TEST_DIR: ${{ github.workspace }}/install/tests USE_SYSTEM_CACHE: False # Using remote HuggingFace cache + HF_HUB_VERBOSITY: debug + TRANSFORMERS_VERBOSITY: debug steps: # checkout action cleans up the workspace and have to be the first step - name: Fetch custom actions @@ -75,7 +77,6 @@ jobs: run: | echo "HF_HUB_CACHE=${{ runner.os == 'Linux' && env.HF_HUB_CACHE_LIN || env.HF_HUB_CACHE_WIN }}" >> "$GITHUB_ENV" echo "HUGGINGFACE_HUB_CACHE=${{ runner.os == 'Linux' && env.HF_HUB_CACHE_LIN || env.HF_HUB_CACHE_WIN }}" >> "$GITHUB_ENV" - echo "HF_HUB_OFFLINE=1" >> "$GITHUB_ENV" - name: Install OpenVINO dependencies (mac) if: runner.os == 'macOS' diff --git a/.github/workflows/job_jax_models_tests.yml b/.github/workflows/job_jax_models_tests.yml index 4e53f9f023215b..db1b322b2b4e9d 100644 --- a/.github/workflows/job_jax_models_tests.yml +++ b/.github/workflows/job_jax_models_tests.yml @@ -44,6 +44,8 @@ jobs: INSTALL_TEST_DIR: ${{ github.workspace }}/install/tests MODEL_HUB_TESTS_INSTALL_DIR: ${{ github.workspace }}/install/tests/model_hub_tests USE_SYSTEM_CACHE: False # Using remote HuggingFace cache + HF_HUB_VERBOSITY: debug + TRANSFORMERS_VERBOSITY: debug steps: # checkout action cleans up the workspace and have to be the first step - name: Fetch custom actions @@ -77,7 +79,6 @@ jobs: run: | echo "HF_HUB_CACHE=${{ runner.os == 'Linux' && env.HF_HUB_CACHE_LIN || env.HF_HUB_CACHE_WIN }}" >> "$GITHUB_ENV" echo "HUGGINGFACE_HUB_CACHE=${{ runner.os == 'Linux' && env.HF_HUB_CACHE_LIN || env.HF_HUB_CACHE_WIN }}" >> "$GITHUB_ENV" - echo "HF_HUB_OFFLINE=1" >> "$GITHUB_ENV" - name: Extract OpenVINO packages and tests run: pigz -dc openvino_tests.tar.gz | tar -xf - -v diff --git a/.github/workflows/job_onnx_models_tests.yml b/.github/workflows/job_onnx_models_tests.yml index ccca9a0a79f14f..bfb91111a15c1a 100644 --- a/.github/workflows/job_onnx_models_tests.yml +++ b/.github/workflows/job_onnx_models_tests.yml @@ -46,6 +46,8 @@ jobs: ONNX_MODEL_ZOO_SHA: "5faef4c33eba0395177850e1e31c4a6a9e634c82" USE_SYSTEM_CACHE: False # Using remote HuggingFace cache LOGS_FOLDER: ${{ github.workspace }}/onnx_models_tests_logs" + HF_HUB_VERBOSITY: debug + TRANSFORMERS_VERBOSITY: debug if: ${{ github.event_name != 'merge_group' }} steps: # checkout action cleans up the workspace and have to be the first step @@ -78,7 +80,6 @@ jobs: run: | echo "HF_HUB_CACHE=${{ runner.os == 'Linux' && env.HF_HUB_CACHE_LIN || env.HF_HUB_CACHE_WIN }}" >> "$GITHUB_ENV" echo "HUGGINGFACE_HUB_CACHE=$HF_HUB_CACHE" >> "$GITHUB_ENV" - echo "HF_HUB_OFFLINE=1" >> "$GITHUB_ENV" echo "MODELS_SHARE_PATH=/mount/testdata$((GITHUB_RUN_NUMBER % NUMBER_OF_REPLICAS))" >> "$GITHUB_ENV" # Issue 148922 diff --git a/.github/workflows/job_pytorch_fx_layer_tests.yml b/.github/workflows/job_pytorch_fx_layer_tests.yml index 69efb81dcff204..f93564f6c936fa 100644 --- a/.github/workflows/job_pytorch_fx_layer_tests.yml +++ b/.github/workflows/job_pytorch_fx_layer_tests.yml @@ -47,6 +47,8 @@ jobs: INSTALL_DIR: ${{ github.workspace }}/install INSTALL_TEST_DIR: ${{ github.workspace }}/install/tests USE_SYSTEM_CACHE: False # Using remote HuggingFace cache + HF_HUB_VERBOSITY: debug + TRANSFORMERS_VERBOSITY: debug steps: - name: Fetch setup_python and install wheels actions uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 @@ -74,7 +76,6 @@ jobs: run: | Add-Content -Path $env:GITHUB_ENV -Value "HF_HUB_CACHE=C:\\mount\\caches\\huggingface" Add-Content -Path $env:GITHUB_ENV -Value "HUGGINGFACE_HUB_CACHE=C:\\mount\\caches\\huggingface" - Add-Content -Path $env:GITHUB_ENV -Value "HF_HUB_OFFLINE=1" - name: Install OpenVINO dependencies (mac) if: runner.os == 'macOS' diff --git a/.github/workflows/job_pytorch_layer_tests.yml b/.github/workflows/job_pytorch_layer_tests.yml index 9b8abfc48217be..e1862f1f50faa3 100644 --- a/.github/workflows/job_pytorch_layer_tests.yml +++ b/.github/workflows/job_pytorch_layer_tests.yml @@ -49,6 +49,8 @@ jobs: INSTALL_TEST_DIR: ${{ github.workspace }}/install/tests LAYER_TESTS_INSTALL_DIR: ${{ github.workspace }}/install/tests/layer_tests USE_SYSTEM_CACHE: False # Using remote HuggingFace cache + HF_HUB_VERBOSITY: debug + TRANSFORMERS_VERBOSITY: debug steps: - name: Fetch setup_python and install wheels actions uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 @@ -75,7 +77,7 @@ jobs: run: | echo "HF_HUB_CACHE=${{ runner.os == 'Linux' && env.HF_HUB_CACHE_LIN || env.HF_HUB_CACHE_WIN }}" >> "$GITHUB_ENV" echo "HUGGINGFACE_HUB_CACHE=${{ runner.os == 'Linux' && env.HF_HUB_CACHE_LIN || env.HF_HUB_CACHE_WIN }}" >> "$GITHUB_ENV" - echo "HF_HUB_OFFLINE=1" >> "$GITHUB_ENV" + - name: Install OpenVINO dependencies (mac) if: runner.os == 'macOS' run: brew install pigz diff --git a/.github/workflows/job_pytorch_models_tests.yml b/.github/workflows/job_pytorch_models_tests.yml index d07ef4b52986e4..b285486b759fdb 100644 --- a/.github/workflows/job_pytorch_models_tests.yml +++ b/.github/workflows/job_pytorch_models_tests.yml @@ -43,6 +43,8 @@ jobs: INSTALL_TEST_DIR: ${{ github.workspace }}/install/tests MODEL_HUB_TESTS_INSTALL_DIR: ${{ github.workspace }}/install/tests/model_hub_tests USE_SYSTEM_CACHE: False # Using remote HuggingFace cache + HF_HUB_VERBOSITY: debug + TRANSFORMERS_VERBOSITY: debug steps: - name: Fetch setup_python and install wheels actions uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 @@ -75,7 +77,6 @@ jobs: run: | echo "HF_HUB_CACHE=${{ runner.os == 'Linux' && env.HF_HUB_CACHE_LIN || env.HF_HUB_CACHE_WIN }}" >> "$GITHUB_ENV" echo "HUGGINGFACE_HUB_CACHE=${{ runner.os == 'Linux' && env.HF_HUB_CACHE_LIN || env.HF_HUB_CACHE_WIN }}" >> "$GITHUB_ENV" - echo "HF_HUB_OFFLINE=1" >> "$GITHUB_ENV" - name: Extract OpenVINO artifacts run: pigz -dc openvino_tests.tar.gz | tar -xf - -v @@ -119,7 +120,6 @@ jobs: TYPE: ${{ inputs.model_scope == 'precommit' && 'precommit' || 'nightly' }} TEST_DEVICE: CPU OP_REPORT_FILE: ${{ env.INSTALL_TEST_DIR }}/TEST-torch_unsupported_ops.log - HF_HUB_OFFLINE: 0 # Can't use offline mode - these tests use `model_info` call, which can't be cached - name: PagedAttention Test if: ${{ inputs.model_scope == 'precommit' }} @@ -128,7 +128,6 @@ jobs: python3 -m pytest ${MODEL_HUB_TESTS_INSTALL_DIR}/transformation_tests/test_pa_transformation.py -m precommit --html=${INSTALL_TEST_DIR}/TEST-torch_pagedattention_tests.html --self-contained-html -vvv -s --tb=short -n 2 env: TEST_DEVICE: CPU - HF_HUB_OFFLINE: 0 - name: RoPE Test if: ${{ inputs.model_scope == 'precommit' }} diff --git a/.github/workflows/job_tensorflow_layer_tests.yml b/.github/workflows/job_tensorflow_layer_tests.yml index 5456e77c3d4986..1f2bdc18d0381e 100644 --- a/.github/workflows/job_tensorflow_layer_tests.yml +++ b/.github/workflows/job_tensorflow_layer_tests.yml @@ -48,6 +48,8 @@ jobs: INSTALL_DIR: ${{ github.workspace }}/install INSTALL_TEST_DIR: ${{ github.workspace }}/install/tests USE_SYSTEM_CACHE: False # Using remote HuggingFace cache + HF_HUB_VERBOSITY: debug + TRANSFORMERS_VERBOSITY: debug steps: - name: Fetch custom actions uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 @@ -80,7 +82,6 @@ jobs: run: | echo "HF_HUB_CACHE=${{ runner.os == 'Linux' && env.HF_HUB_CACHE_LIN || env.HF_HUB_CACHE_WIN }}" >> "$GITHUB_ENV" echo "HUGGINGFACE_HUB_CACHE=${{ runner.os == 'Linux' && env.HF_HUB_CACHE_LIN || env.HF_HUB_CACHE_WIN }}" >> "$GITHUB_ENV" - echo "HF_HUB_OFFLINE=1" >> "$GITHUB_ENV" - name: Install OpenVINO dependencies (mac) if: ${{ runner.os == 'macOS' }} diff --git a/.github/workflows/job_tensorflow_models_tests.yml b/.github/workflows/job_tensorflow_models_tests.yml index e62b7619d6345d..0c8e6a5e63cbae 100644 --- a/.github/workflows/job_tensorflow_models_tests.yml +++ b/.github/workflows/job_tensorflow_models_tests.yml @@ -43,6 +43,8 @@ jobs: INSTALL_TEST_DIR: ${{ github.workspace }}/install/tests NUMBER_OF_REPLICAS: 2 USE_SYSTEM_CACHE: False # Using remote HuggingFace cache + HF_HUB_VERBOSITY: debug + TRANSFORMERS_VERBOSITY: debug steps: - name: Fetch custom actions uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 @@ -76,7 +78,6 @@ jobs: echo "HF_HUB_CACHE=${{ runner.os == 'Linux' && env.HF_HUB_CACHE_LIN || env.HF_HUB_CACHE_WIN }}" >> "$GITHUB_ENV" echo "HUGGINGFACE_HUB_CACHE=${{ runner.os == 'Linux' && env.HF_HUB_CACHE_LIN || env.HF_HUB_CACHE_WIN }}" >> "$GITHUB_ENV" echo "TFHUB_CACHE_DIR=/mount/testdata$((GITHUB_RUN_NUMBER % NUMBER_OF_REPLICAS))/tfhub_models" >> "$GITHUB_ENV" - echo "HF_HUB_OFFLINE=1" >> "$GITHUB_ENV" - name: Extract OpenVINO artifacts (Linux and macOS) run: pigz -dc openvino_tests.tar.gz | tar -xf - -v diff --git a/tests/e2e_tests/test_utils/pytorch_loaders.py b/tests/e2e_tests/test_utils/pytorch_loaders.py index f1f4cfefbefc20..cad7b0e30e376d 100644 --- a/tests/e2e_tests/test_utils/pytorch_loaders.py +++ b/tests/e2e_tests/test_utils/pytorch_loaders.py @@ -6,6 +6,7 @@ import sys import torch +from huggingface_hub import snapshot_download class LoadPyTorchModel: @@ -70,7 +71,8 @@ def load_cadene_model(module, args): def load_hugging_face_model(module, args): module = importlib.import_module(module) - model = module.AutoModel.from_pretrained(args['model-name'], torchscript=True) + model_cached = snapshot_download(args['model-name']) # required to avoid HF rate limits + model = module.AutoModel.from_pretrained(model_cached, torchscript=True) return model diff --git a/tests/llm/accuracy_conformance.py b/tests/llm/accuracy_conformance.py index 141bccaeecbb64..0d84cf127b87e0 100644 --- a/tests/llm/accuracy_conformance.py +++ b/tests/llm/accuracy_conformance.py @@ -5,6 +5,7 @@ import tempfile import pytest +from huggingface_hub import snapshot_download from optimum.intel.openvino import (OVModelForCausalLM, OVWeightQuantizationConfig) from transformers import AutoModelForCausalLM, AutoTokenizer, set_seed @@ -100,8 +101,9 @@ def setup_model(model_id): logger.info(f"Setting up model: {model_id}") # Download original model - model = AutoModelForCausalLM.from_pretrained(model_id) - tokenizer = AutoTokenizer.from_pretrained(model_id) + model_cached = snapshot_download(model_id) # required to avoid HF rate limits + model = AutoModelForCausalLM.from_pretrained(model_cached) + tokenizer = AutoTokenizer.from_pretrained(model_cached) # Save original model model_path = get_model_path(model_id, "org") diff --git a/tests/model_hub_tests/jax/test_hf_transformers.py b/tests/model_hub_tests/jax/test_hf_transformers.py index 57fa1d7ba4b1d0..20a131455ee1d6 100644 --- a/tests/model_hub_tests/jax/test_hf_transformers.py +++ b/tests/model_hub_tests/jax/test_hf_transformers.py @@ -5,6 +5,7 @@ import os import pytest import requests +from huggingface_hub import snapshot_download from PIL import Image from models_hub_common.constants import hf_cache_dir, clean_hf_cache_dir from models_hub_common.utils import cleanup_dir, get_models_list, retry @@ -22,17 +23,18 @@ class TestTransformersModel(TestJaxConvertModel): @retry(3, exceptions=(OSError,), delay=1) def load_model(self, model_name, _): - model = FlaxAutoModel.from_pretrained(model_name) + model_cached = snapshot_download(model_name) # required to avoid HF rate limits + model = FlaxAutoModel.from_pretrained(model_cached) if model_name in ['google/vit-base-patch16-224-in21k']: url = "http://images.cocodataset.org/val2017/000000039769.jpg" image = Image.open(requests.get(url, stream=True).raw) - image_processor = AutoImageProcessor.from_pretrained(model_name) + image_processor = AutoImageProcessor.from_pretrained(model_cached) self.example = image_processor(images=image, return_tensors="np") elif model_name in ['albert/albert-base-v2', 'facebook/bart-base', 'ksmcg/Mistral-tiny']: - tokenizer = AutoTokenizer.from_pretrained(model_name) + tokenizer = AutoTokenizer.from_pretrained(model_cached) self.example = tokenizer("Hello, my dog is cute", return_tensors="np") elif model_name in ['openai/clip-vit-base-patch32']: - processor = AutoProcessor.from_pretrained(model_name) + processor = AutoProcessor.from_pretrained(model_cached) url = "http://images.cocodataset.org/val2017/000000039769.jpg" image = Image.open(requests.get(url, stream=True).raw) self.example = processor(text=["a photo of a cat", "a photo of a dog"], diff --git a/tests/model_hub_tests/pytorch/test_edsr.py b/tests/model_hub_tests/pytorch/test_edsr.py index 844c43985f8e71..d158d5a98b7abf 100644 --- a/tests/model_hub_tests/pytorch/test_edsr.py +++ b/tests/model_hub_tests/pytorch/test_edsr.py @@ -6,6 +6,7 @@ import pytest import random import torch +from huggingface_hub import snapshot_download from models_hub_common.constants import hf_cache_dir, clean_hf_cache_dir from models_hub_common.utils import cleanup_dir @@ -44,8 +45,9 @@ def load_model(self, model_name, model_link): image = Image.open(requests.get(url, stream=True).raw) assert model_name in name_to_class, "Unexpected model name" print(f"scale: {self.scale}") + model_cached = snapshot_download(f'eugenesiow/{model_name}') # required to avoid HF rate limits model = name_to_class[model_name].from_pretrained( - f'eugenesiow/{model_name}', scale=self.scale) + model_cached, scale=self.scale) inputs = ImageLoader.load_image(image) self.example = (torch.randn_like(inputs),) self.inputs = (inputs,) diff --git a/tests/model_hub_tests/pytorch/test_hf_transformers.py b/tests/model_hub_tests/pytorch/test_hf_transformers.py index 30fd450347b207..89fde03a35df5b 100644 --- a/tests/model_hub_tests/pytorch/test_hf_transformers.py +++ b/tests/model_hub_tests/pytorch/test_hf_transformers.py @@ -4,7 +4,7 @@ import os from datasets import Audio, load_dataset -from huggingface_hub import hf_hub_download, model_info +from huggingface_hub import hf_hub_download, model_info, snapshot_download from PIL import Image import pytest import torch @@ -59,12 +59,13 @@ def setup_class(self): def load_model(self, name, type): name, _, name_suffix = name.partition(':') + model_cached = snapshot_download(name) # required to avoid HF rate limits mi = model_info(name) auto_processor = None model = None example = None try: - config = AutoConfig.from_pretrained(name) + config = AutoConfig.from_pretrained(model_cached) except Exception: config = {} model_kwargs = {"torchscript": True} @@ -77,31 +78,31 @@ def load_model(self, name, type): except: auto_model = None if "clip_vision_model" in mi.tags: - preprocessor = CLIPFeatureExtractor.from_pretrained(name) + preprocessor = CLIPFeatureExtractor.from_pretrained(model_cached) encoded_input = preprocessor(self.image, return_tensors='pt') example = dict(encoded_input) elif 'xclip' in mi.tags: - model = XCLIPVisionModel.from_pretrained(name, **model_kwargs) + model = XCLIPVisionModel.from_pretrained(model_cached, **model_kwargs) example = {'pixel_values': torch.randn(16, 3, 224, 224)} elif 'audio-spectrogram-transformer' in mi.tags: example = {'input_values': torch.randn(1, 1024, 128)} elif 'mega' in mi.tags: - model = AutoModel.from_pretrained(name, **model_kwargs) + model = AutoModel.from_pretrained(model_cached, **model_kwargs) model.config.output_attentions = True model.config.output_hidden_states = True model.config.return_dict = True example = dict(model.dummy_inputs) elif 'bros' in mi.tags: - processor = AutoProcessor.from_pretrained(name) + processor = AutoProcessor.from_pretrained(model_cached) encoding = processor("to the moon!", return_tensors="pt") bbox = torch.randn([1, 6, 8]) example = dict( input_ids=encoding["input_ids"], bbox=bbox, attention_mask=encoding["attention_mask"]) elif 'upernet' in mi.tags: - processor = AutoProcessor.from_pretrained(name) + processor = AutoProcessor.from_pretrained(model_cached) example = dict(processor(images=self.image, return_tensors="pt")) elif 'deformable_detr' in mi.tags or 'oneformer' in mi.tags: - processor = AutoProcessor.from_pretrained(name) + processor = AutoProcessor.from_pretrained(model_cached) example = dict(processor(images=self.image, task_inputs=[ "semantic"], return_tensors="pt")) elif 'clap' in mi.tags: @@ -111,11 +112,11 @@ def load_model(self, name, type): } example = example_inputs_map[name_suffix] elif 'git' in mi.tags: - processor = AutoProcessor.from_pretrained(name) + processor = AutoProcessor.from_pretrained(model_cached) example = {'pixel_values': torch.randn(1, 3, 224, 224), 'input_ids': torch.randint(1, 100, size=(1, 13), dtype=torch.int64)} elif 'blip-2' in mi.tags: - processor = AutoProcessor.from_pretrained(name) + processor = AutoProcessor.from_pretrained(model_cached) example = dict(processor(images=self.image, return_tensors="pt")) example_inputs_map = { 'vision_model': {'pixel_values': torch.randn([1, 3, 224, 224])}, @@ -126,7 +127,7 @@ def load_model(self, name, type): } example = example_inputs_map[name_suffix] elif "t5" in mi.tags: - tokenizer = T5Tokenizer.from_pretrained(name) + tokenizer = T5Tokenizer.from_pretrained(model_cached) encoder = tokenizer( "Studies have been shown that owning a dog is good for you", return_tensors="pt") decoder = tokenizer("Studies show that", return_tensors="pt") @@ -137,8 +138,8 @@ def load_model(self, name, type): example = (wav_input_16khz,) elif "vit-gpt2" in name: model = VisionEncoderDecoderModel.from_pretrained( - name, **model_kwargs) - feature_extractor = ViTImageProcessor.from_pretrained(name) + model_cached, **model_kwargs) + feature_extractor = ViTImageProcessor.from_pretrained(model_cached) encoded_input = feature_extractor( images=[self.image], return_tensors="pt") @@ -147,7 +148,7 @@ def load_model(self, name, type): example["decoder_attention_mask"] = torch.ones( [1, 20], dtype=torch.int64) elif 'idefics' in mi.tags: - processor = AutoProcessor.from_pretrained(name) + processor = AutoProcessor.from_pretrained(model_cached) prompts = [[ "User: What is in this image?", @@ -167,16 +168,16 @@ def load_model(self, name, type): prompts, add_end_of_utterance_token=False, return_tensors="pt") example = dict(inputs) elif 'blip' in mi.tags and 'text2text-generation' in mi.tags: - processor = BlipProcessor.from_pretrained(name) + processor = BlipProcessor.from_pretrained(model_cached) model = BlipForConditionalGeneration.from_pretrained( - name, **model_kwargs) + model_cached, **model_kwargs) text = "a photography of" inputs = processor(self.image, text, return_tensors="pt") example = dict(inputs) elif 'speecht5' in mi.tags: - processor = SpeechT5Processor.from_pretrained(name) + processor = SpeechT5Processor.from_pretrained(model_cached) model = SpeechT5ForTextToSpeech.from_pretrained( - name, **model_kwargs) + model_cached, **model_kwargs) inputs = processor(text="Hello, my dog is cute.", return_tensors="pt") @@ -191,7 +192,7 @@ def load_model(self, name, type): example['decoder_input_values'] = torch.randn( [1, 20, model.config.num_mel_bins]) elif 'layoutlmv2' in mi.tags: - processor = LayoutLMv2Processor.from_pretrained(name) + processor = LayoutLMv2Processor.from_pretrained(model_cached) question = "What's the content of this image?" encoding = processor( @@ -199,8 +200,8 @@ def load_model(self, name, type): example = dict(encoding) elif 'pix2struct' in mi.tags: model = Pix2StructForConditionalGeneration.from_pretrained( - name, **model_kwargs) - processor = AutoProcessor.from_pretrained(name) + model_cached, **model_kwargs) + processor = AutoProcessor.from_pretrained(model_cached) import requests image_url = "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/transformers/tasks/ai2d-demo.jpg" @@ -217,25 +218,25 @@ def load_model(self, name, type): inputs = image_processor(images=self.image, return_tensors="pt") example = dict(inputs) elif "mms-lid" in name: - processor = AutoFeatureExtractor.from_pretrained(name) + processor = AutoFeatureExtractor.from_pretrained(model_cached) input_values = processor(torch.randn(16000).numpy(), sampling_rate=16_000, return_tensors="pt") example = {"input_values": input_values.input_values} elif "retribert" in mi.tags: - tokenizer = RetriBertTokenizer.from_pretrained(name) + tokenizer = RetriBertTokenizer.from_pretrained(model_cached) encoding1 = tokenizer( "How many cats are there?", return_tensors="pt") encoding2 = tokenizer("Second text", return_tensors="pt") example = (encoding1.input_ids, encoding1.attention_mask, encoding2.input_ids, encoding2.attention_mask) elif "mgp-str" in mi.tags or "clip_vision_model" in mi.tags: - processor = AutoProcessor.from_pretrained(name) + processor = AutoProcessor.from_pretrained(model_cached) encoded_input = processor(images=self.image, return_tensors="pt") example = (encoded_input.pixel_values,) elif "flava" in mi.tags: - model = FlavaImageModel.from_pretrained(name, **model_kwargs) - feature_extractor = AutoFeatureExtractor.from_pretrained(name) + model = FlavaImageModel.from_pretrained(model_cached, **model_kwargs) + feature_extractor = AutoFeatureExtractor.from_pretrained(model_cached) encoded_input = feature_extractor(images=[self.image], return_tensors="pt") @@ -243,11 +244,11 @@ def load_model(self, name, type): elif "vivit" in mi.tags: frames = list(torch.randint( 0, 255, [32, 3, 224, 224]).to(torch.float32)) - processor = VivitImageProcessor.from_pretrained(name) + processor = VivitImageProcessor.from_pretrained(model_cached) encoded_input = processor(images=frames, return_tensors="pt") example = (encoded_input.pixel_values,) elif "tvlt" in mi.tags: - processor = AutoProcessor.from_pretrained(name) + processor = AutoProcessor.from_pretrained(model_cached) num_frames = 8 images = list(torch.rand(num_frames, 3, 224, 224)) audio = list(torch.randn(10000)) @@ -255,26 +256,26 @@ def load_model(self, name, type): images, audio, sampling_rate=44100, return_tensors="pt") example = dict(input_dict) elif "gptsan-japanese" in mi.tags: - processor = AutoTokenizer.from_pretrained(name) + processor = AutoTokenizer.from_pretrained(model_cached) text = "織田信長は、" encoded_input = processor(text=[text], return_tensors="pt") example = dict(input_ids=encoded_input.input_ids, token_type_ids=encoded_input.token_type_ids) elif "videomae" in mi.tags or "timesformer" in mi.tags: - processor = AutoProcessor.from_pretrained(name) + processor = AutoProcessor.from_pretrained(model_cached) video = list(torch.randint( 0, 255, [16, 3, 224, 224]).to(torch.float32)) inputs = processor(video, return_tensors="pt") example = dict(inputs) elif 'text-to-speech' in mi.tags: - tokenizer = AutoTokenizer.from_pretrained(name) + tokenizer = AutoTokenizer.from_pretrained(model_cached) text = "some example text in the English language" inputs = tokenizer(text, return_tensors="pt") example = dict(inputs) elif 'musicgen' in mi.tags or "musicgen_melody" in mi.tags: - processor = AutoProcessor.from_pretrained(name) + processor = AutoProcessor.from_pretrained(model_cached) model = AutoModelForTextToWaveform.from_pretrained( - name, **model_kwargs) + model_cached, **model_kwargs) inputs = processor( text=["80s pop track with bassy drums and synth"], @@ -287,14 +288,14 @@ def load_model(self, name, type): example["decoder_input_ids"] = torch.ones( (inputs.input_ids.shape[0] * model.decoder.num_codebooks, 1), dtype=torch.long) * pad_token_id elif 'kosmos-2' in mi.tags or 'instructblip' in mi.tags: - processor = AutoProcessor.from_pretrained(name) + processor = AutoProcessor.from_pretrained(model_cached) prompt = "An image of" inputs = processor( text=prompt, images=self.image, return_tensors="pt") example = dict(inputs) elif 'vitmatte' in mi.tags: - processor = AutoImageProcessor.from_pretrained(name) + processor = AutoImageProcessor.from_pretrained(model_cached) filepath = hf_hub_download(repo_id="hf-internal-testing/image-matting-fixtures", filename="image.png", repo_type="dataset") @@ -307,7 +308,7 @@ def load_model(self, name, type): return_tensors="pt") example = dict(inputs) elif 'sam' in mi.tags: - processor = AutoProcessor.from_pretrained(name) + processor = AutoProcessor.from_pretrained(model_cached) input_points = [[[450, 600]]] inputs = processor(self.image, input_points=input_points, @@ -318,7 +319,7 @@ def load_model(self, name, type): if "reshaped_input_sizes" in example: del example["reshaped_input_sizes"] elif 'udop' in mi.tags: - processor = AutoProcessor.from_pretrained(name, apply_ocr=False) + processor = AutoProcessor.from_pretrained(model_cached, apply_ocr=False) dataset = load_dataset("nielsr/funsd-layoutlmv3", split="train") example = dataset[0] image = example["image"] @@ -335,7 +336,7 @@ def load_model(self, name, type): ds = ds.cast_column("audio", Audio(sampling_rate=22050)) sorted_audio = ds.sort("id").select(range(1))[:1]["audio"][0] _, audio, sr = sorted_audio.values() - processor = AutoProcessor.from_pretrained(name) + processor = AutoProcessor.from_pretrained(model_cached) inputs = processor(raw_speech=audio, sampling_rate=sr, text=text, return_tensors="pt") example = dict(inputs) @@ -364,7 +365,7 @@ def load_model(self, name, type): future_time_features=batch["future_time_features"] ) elif "seg-ment-tation" in mi.tags: - image_processor = AutoImageProcessor.from_pretrained(name) + image_processor = AutoImageProcessor.from_pretrained(model_cached) inputs = image_processor(images=self.image, return_tensors="pt") example = dict(inputs) elif "lxmert" in mi.tags: @@ -375,24 +376,24 @@ def load_model(self, name, type): else: try: if auto_model == "AutoModelForCausalLM": - tokenizer = AutoTokenizer.from_pretrained(name) + tokenizer = AutoTokenizer.from_pretrained(model_cached) text = "Replace me by any text you'd like." encoded_input = tokenizer(text, return_tensors='pt') example = dict(encoded_input) if "facebook/incoder" in name and "token_type_ids" in example: del example["token_type_ids"] elif auto_model == "AutoModelForMaskedLM": - tokenizer = AutoTokenizer.from_pretrained(name) + tokenizer = AutoTokenizer.from_pretrained(model_cached) text = "Replace me by any text you'd like." encoded_input = tokenizer(text, return_tensors='pt') example = dict(encoded_input) elif auto_model == "AutoModelForImageClassification": - processor = AutoProcessor.from_pretrained(name) + processor = AutoProcessor.from_pretrained(model_cached) encoded_input = processor( images=self.image, return_tensors="pt") example = dict(encoded_input) elif auto_model == "AutoModelForSeq2SeqLM": - tokenizer = AutoTokenizer.from_pretrained(name) + tokenizer = AutoTokenizer.from_pretrained(model_cached) inputs = tokenizer( "Studies have been shown that owning a dog is good for you", return_tensors="pt") decoder_inputs = tokenizer( @@ -403,19 +404,19 @@ def load_model(self, name, type): example = dict(input_ids=inputs.input_ids, decoder_input_ids=decoder_inputs.input_ids) elif auto_model == "AutoModelForSpeechSeq2Seq": - processor = AutoProcessor.from_pretrained(name) + processor = AutoProcessor.from_pretrained(model_cached) inputs = processor(torch.randn(1000).numpy(), sampling_rate=16000, return_tensors="pt") example = dict(inputs) elif auto_model == "AutoModelForCTC": - processor = AutoProcessor.from_pretrained(name) + processor = AutoProcessor.from_pretrained(model_cached) input_values = processor(torch.randn(1000).numpy(), return_tensors="pt") example = dict(input_values) elif auto_model == "AutoModelForTableQuestionAnswering": import pandas as pd - tokenizer = AutoTokenizer.from_pretrained(name) + tokenizer = AutoTokenizer.from_pretrained(model_cached) data = {"Actors": ["Brad Pitt", "Leonardo Di Caprio", "George Clooney"], "Number of movies": ["87", "53", "69"]} queries = ["What is the name of the first actor?", @@ -433,11 +434,11 @@ def load_model(self, name, type): else: text = "Replace me by any text you'd like." if auto_processor is not None and "Tokenizer" not in auto_processor: - processor = AutoProcessor.from_pretrained(name) + processor = AutoProcessor.from_pretrained(model_cached) encoded_input = processor( text=[text], images=self.image, return_tensors="pt", padding=True) else: - tokenizer = AutoTokenizer.from_pretrained(name) + tokenizer = AutoTokenizer.from_pretrained(model_cached) encoded_input = tokenizer(text, return_tensors='pt') example = dict(encoded_input) except: @@ -505,6 +506,7 @@ def teardown_method(self): @staticmethod def load_model_with_default_class(name, **kwargs): + model_cached = snapshot_download(name) # required to avoid HF rate limits try: mi = model_info(name) assert len({"owlv2", "owlvit", "vit_mae"}.intersection( @@ -513,9 +515,9 @@ def load_model_with_default_class(name, **kwargs): mi.config["architectures"]) == 1 class_name = mi.config["architectures"][0] model_class = getattr(transformers, class_name) - return model_class.from_pretrained(name, **kwargs) + return model_class.from_pretrained(model_cached, **kwargs) except: - return AutoModel.from_pretrained(name, **kwargs) + return AutoModel.from_pretrained(model_cached, **kwargs) @pytest.mark.parametrize("name,type", [("allenai/led-base-16384", "led"), ("bert-base-uncased", "bert"), diff --git a/tests/model_hub_tests/pytorch/test_llm.py b/tests/model_hub_tests/pytorch/test_llm.py index 7f3a214636d9ec..376ccf0cd80e4f 100644 --- a/tests/model_hub_tests/pytorch/test_llm.py +++ b/tests/model_hub_tests/pytorch/test_llm.py @@ -7,6 +7,7 @@ import numpy as np import pytest import torch +from huggingface_hub import snapshot_download from transformers import AutoConfig, AutoTokenizer, AutoModelForCausalLM from models_hub_common.utils import retry @@ -142,8 +143,9 @@ def setup_class(self): def load_model(self, name, type): model = None example = None + model_cached = snapshot_download(name) # required to avoid HF rate limits try: - config = AutoConfig.from_pretrained(name, trust_remote_code=True) + config = AutoConfig.from_pretrained(model_cached, trust_remote_code=True) except Exception: config = {} model_kwargs = {"torchscript": True, "trust_remote_code": True} @@ -159,9 +161,9 @@ def load_model(self, name, type): model_kwargs["torch_dtype"] = torch.float16 else: model_kwargs["torch_dtype"] = "auto" - - t = AutoTokenizer.from_pretrained(name, trust_remote_code=True) - self.model = AutoModelForCausalLM.from_pretrained(name, **model_kwargs) + model_cached = snapshot_download(name) # required to avoid HF rate limits + t = AutoTokenizer.from_pretrained(model_cached, trust_remote_code=True) + self.model = AutoModelForCausalLM.from_pretrained(model_cached, **model_kwargs) if is_quant: model = self.model else: diff --git a/tests/model_hub_tests/tensorflow/test_tf_convert_model.py b/tests/model_hub_tests/tensorflow/test_tf_convert_model.py index 5fb1981b4184c7..468712b7e909eb 100644 --- a/tests/model_hub_tests/tensorflow/test_tf_convert_model.py +++ b/tests/model_hub_tests/tensorflow/test_tf_convert_model.py @@ -12,6 +12,7 @@ import tensorflow_hub as hub # noinspection PyUnresolvedReferences import tensorflow_text # do not delete, needed for text models +from huggingface_hub import snapshot_download from models_hub_common.test_convert_model import TestConvertModel from models_hub_common.utils import get_models_list from openvino import Core @@ -43,13 +44,14 @@ def setup_class(self): def load_model(self, model_name, model_link: str): if is_hf_link(model_link): + model_cached = snapshot_download(model_name) # required to avoid HF rate limits library_type = model_link[3:] if library_type == "transformers": from transformers import TFAutoModel - return TFAutoModel.from_pretrained(model_name) + return TFAutoModel.from_pretrained(model_cached) elif library_type == "sentence-transformers": from tf_sentence_transformers import SentenceTransformer - return SentenceTransformer.from_pretrained(model_name) + return SentenceTransformer.from_pretrained(model_cached) elif 'storage.openvinotoolkit.org' in model_link: # this models is from public OpenVINO storage subprocess.check_call(["wget", "-nv", model_link], cwd=self.model_dir.name)