From 0428a860bd2cf59873e903e46504e08a8e426008 Mon Sep 17 00:00:00 2001 From: zRzRzRzRzRzRzR <2448370773@qq.com> Date: Thu, 6 Nov 2025 17:03:53 +0800 Subject: [PATCH 01/66] init --- docs/source/en/_toctree.yml | 2 + docs/source/en/model_doc/glm46v.md | 68 + src/transformers/models/__init__.py | 1 + .../models/auto/configuration_auto.py | 2 + src/transformers/models/auto/modeling_auto.py | 2 + src/transformers/models/glm46v/__init__.py | 29 + .../models/glm46v/configuration_glm46v.py | 338 ++++ .../models/glm46v/modeling_glm46v.py | 1701 +++++++++++++++++ .../models/glm46v/modular_glm46v.py | 141 ++ .../models/glm46v/processing_glm46v.py | 282 +++ .../models/glm46v/video_processing_glm46v.py | 272 +++ .../models/glm4v/modular_glm4v.py | 4 +- .../models/glm4v/processing_glm4v.py | 5 +- tests/models/glm46v/__init__.py | 0 tests/models/glm46v/test_modeling_glm46v.py | 586 ++++++ .../glm4v_moe/test_modeling_glm4v_moe.py | 2 +- 16 files changed, 3432 insertions(+), 3 deletions(-) create mode 100644 docs/source/en/model_doc/glm46v.md create mode 100644 src/transformers/models/glm46v/__init__.py create mode 100644 src/transformers/models/glm46v/configuration_glm46v.py create mode 100644 src/transformers/models/glm46v/modeling_glm46v.py create mode 100644 src/transformers/models/glm46v/modular_glm46v.py create mode 100644 src/transformers/models/glm46v/processing_glm46v.py create mode 100644 src/transformers/models/glm46v/video_processing_glm46v.py create mode 100644 tests/models/glm46v/__init__.py create mode 100644 tests/models/glm46v/test_modeling_glm46v.py diff --git a/docs/source/en/_toctree.yml b/docs/source/en/_toctree.yml index c5ce9fbdb9c4..7b743cd4f7c0 100644 --- a/docs/source/en/_toctree.yml +++ b/docs/source/en/_toctree.yml @@ -1064,6 +1064,8 @@ title: Gemma3n - local: model_doc/git title: GIT + - local: model_doc/glm46v + title: Glm46V - local: model_doc/glm4v title: glm4v - local: model_doc/glm4v_moe diff --git a/docs/source/en/model_doc/glm46v.md b/docs/source/en/model_doc/glm46v.md new file mode 100644 index 000000000000..2320c9072839 --- /dev/null +++ b/docs/source/en/model_doc/glm46v.md @@ -0,0 +1,68 @@ + + + +# Glm46V + +## Overview + +The Glm46V model was proposed in []() by . + + +The abstract from the paper is the following: + + + +Tips: + + + +This model was contributed by [INSERT YOUR HF USERNAME HERE](https://huggingface.co/). +The original code can be found [here](). + +## Usage examples + + + +## Glm46VConfig + +[[autodoc]] Glm46VConfig + +## Glm46VTextConfig + +[[autodoc]] Glm46VTextConfig + +## Glm46VForConditionalGeneration + +[[autodoc]] Glm46VForConditionalGeneration + +## Glm46VModel + +[[autodoc]] Glm46VModel + - forward + +## Glm46VPreTrainedModel + +[[autodoc]] Glm46VPreTrainedModel + - forward + +## Glm46VTextModel + +[[autodoc]] Glm46VTextModel + - forward \ No newline at end of file diff --git a/src/transformers/models/__init__.py b/src/transformers/models/__init__.py index 5630063f92ec..3b96d1a65691 100644 --- a/src/transformers/models/__init__.py +++ b/src/transformers/models/__init__.py @@ -141,6 +141,7 @@ from .git import * from .glm import * from .glm4 import * + from .glm46v import * from .glpn import * from .got_ocr2 import * from .gpt2 import * diff --git a/src/transformers/models/auto/configuration_auto.py b/src/transformers/models/auto/configuration_auto.py index 7e2e84a445ef..f4fab107a642 100644 --- a/src/transformers/models/auto/configuration_auto.py +++ b/src/transformers/models/auto/configuration_auto.py @@ -170,6 +170,7 @@ ("git", "GitConfig"), ("glm", "GlmConfig"), ("glm4", "Glm4Config"), + ("glm46v", "Glm46VConfig"), ("glm4_moe", "Glm4MoeConfig"), ("glm4v", "Glm4vConfig"), ("glm4v_moe", "Glm4vMoeConfig"), @@ -615,6 +616,7 @@ ("git", "GIT"), ("glm", "GLM"), ("glm4", "GLM4"), + ("glm46v", "Glm46V"), ("glm4_moe", "Glm4MoE"), ("glm4v", "GLM4V"), ("glm4v_moe", "GLM4VMOE"), diff --git a/src/transformers/models/auto/modeling_auto.py b/src/transformers/models/auto/modeling_auto.py index 197029464efd..26ecdaad0bdf 100644 --- a/src/transformers/models/auto/modeling_auto.py +++ b/src/transformers/models/auto/modeling_auto.py @@ -173,6 +173,7 @@ class _BaseModelWithGenerate(PreTrainedModel, GenerationMixin): ("git", "GitModel"), ("glm", "GlmModel"), ("glm4", "Glm4Model"), + ("glm46v", "Glm46VModel"), ("glm4_moe", "Glm4MoeModel"), ("glm4v", "Glm4vModel"), ("glm4v_moe", "Glm4vMoeModel"), @@ -1029,6 +1030,7 @@ class _BaseModelWithGenerate(PreTrainedModel, GenerationMixin): ("gemma3", "Gemma3ForConditionalGeneration"), ("gemma3n", "Gemma3nForConditionalGeneration"), ("git", "GitForCausalLM"), + ("glm46v", "Glm46VForConditionalGeneration"), ("glm4v", "Glm4vForConditionalGeneration"), ("glm4v_moe", "Glm4vMoeForConditionalGeneration"), ("got_ocr2", "GotOcr2ForConditionalGeneration"), diff --git a/src/transformers/models/glm46v/__init__.py b/src/transformers/models/glm46v/__init__.py new file mode 100644 index 000000000000..ae27f605ed0d --- /dev/null +++ b/src/transformers/models/glm46v/__init__.py @@ -0,0 +1,29 @@ +# coding=utf-8 +# Copyright 2025 the HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import TYPE_CHECKING + +from ...utils import _LazyModule +from ...utils.import_utils import define_import_structure + + +if TYPE_CHECKING: + from .configuration_glm46v import * + from .modeling_glm46v import * +else: + import sys + + _file = globals()["__file__"] + sys.modules[__name__] = _LazyModule(__name__, _file, define_import_structure(_file), module_spec=__spec__) diff --git a/src/transformers/models/glm46v/configuration_glm46v.py b/src/transformers/models/glm46v/configuration_glm46v.py new file mode 100644 index 000000000000..1dad5fdcbd60 --- /dev/null +++ b/src/transformers/models/glm46v/configuration_glm46v.py @@ -0,0 +1,338 @@ +# 🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨 +# This file was automatically generated from src/transformers/models/glm46v/modular_glm46v.py. +# Do NOT edit this file manually as any edits will be overwritten by the generation of +# the file from the modular. If any change should be done, please apply the change to the +# modular_glm46v.py file directly. One of our CI enforces this. +# 🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨 +# coding=utf-8 +# Copyright 2025 the HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Optional + +from ...configuration_utils import PreTrainedConfig +from ...modeling_rope_utils import RopeParameters, rope_config_validation, standardize_rope_params + + +class Glm46VVisionConfig(PreTrainedConfig): + r""" + This is the configuration class to store the configuration of a [`Glm46VVisionModel`]. It is used to instantiate an Glm46VVisionModel + model according to the specified arguments, defining the model architecture. Instantiating a configuration with the defaults will yield + a similar configuration to that of + GLM-4.1V-9B-Thinking [THUDM/GLM-4.1V-9B-Thinking](https://huggingface.co/THUDM/GLM-4.1V-9B-Thinking). + + Args: + hidden_size (`int`, *optional*, defaults to 1536): + Dimensionality of the encoder layers and the pooler layer. + depth (`int`, *optional*, defaults to 24): + Number of layers (depth) in the model. + attention_bias (`bool`, *optional*, defaults to `False`): + Whether to add a bias to the queries, keys and values. + intermediate_size (`int`, *optional*, defaults to 13696): + Dimensionality of the "intermediate" (i.e., feed-forward) layer in the Transformer encoder. + hidden_act (`str` or `function`, *optional*, defaults to `"selu"`): + The non-linear activation function (function or string) in the encoder and pooler. If string, `"gelu"`, + `"relu"`, `"selu"` and `"gelu_new"` are supported. + hidden_dropout_prob (`float`, *optional*, defaults to 0.0): + The dropout probability for all fully connected layers in the embeddings, encoder, and pooler. + attention_dropout (`float`, *optional*, defaults to 0.0): + Dropout probability for attention weights. + projection_dropout (`float`, *optional*, defaults to 0.0): + Dropout probability for the projection layer. + initializer_range (`float`, *optional*, defaults to 0.02): + The standard deviation of the truncated_normal_initializer for initializing all weight matrices. + image_size (`int` or `list[int]`, *optional*, defaults to `[336, 336]`): + The size (resolution) of each image. + patch_size (`int`, *optional*, defaults to `14`): + The size (resolution) of each patch. + num_channels (`int`, *optional*, defaults to 3): + The number of input channels. + out_hidden_size (`int`, *optional*, defaults to 4096): + The output hidden size of the vision model. + rms_norm_eps (`float`, *optional*, defaults to 1e-05): + The epsilon used by the rms normalization layers. + spatial_merge_size (`int`, *optional*, defaults to 2): + The size used for merging spatial dimensions. + temporal_patch_size (`int`, *optional*, defaults to 2): + The size used for patches along the temporal dimension. + Example: + + ```python + >>> from transformers import Glm46VVisionConfig, Glm46VVisionModel + + >>> # Initializing a Glm46VVisionConfig GLM-4.1V-9B style configuration + >>> configuration = Glm46VVisionConfig() + + >>> # Initializing a model (with random weights) from the GLM-4.1V-9B configuration + >>> model = Glm46VVisionModel(configuration) + + >>> # Accessing the model configuration + >>> configuration = model.config + ```""" + + model_type = "glm46v" + base_config_key = "vision_config" + + def __init__( + self, + depth=24, + hidden_size=1536, + hidden_act="silu", + attention_bias=False, + attention_dropout=0.0, + num_heads=12, + in_channels=3, + image_size=336, + patch_size=14, + rms_norm_eps=1e-05, + spatial_merge_size=2, + temporal_patch_size=2, + out_hidden_size=4096, + intermediate_size=13696, + initializer_range=0.02, + **kwargs, + ): + super().__init__(**kwargs) + + self.depth = depth + self.hidden_size = hidden_size + self.hidden_act = hidden_act + self.num_heads = num_heads + self.in_channels = in_channels + self.image_size = image_size + self.patch_size = patch_size + self.spatial_merge_size = spatial_merge_size + self.temporal_patch_size = temporal_patch_size + self.out_hidden_size = out_hidden_size + self.intermediate_size = intermediate_size + self.initializer_range = initializer_range + self.rms_norm_eps = rms_norm_eps + self.attention_bias = attention_bias + self.attention_dropout = attention_dropout + + +class Glm46VTextConfig(PreTrainedConfig): + r""" + This is the configuration class to store the configuration of a [`Glm46VModel`]. It is used to instantiate a + GLM-4.1V model according to the specified arguments, defining the model architecture. Instantiating a + configuration with the defaults will yield a similar configuration to that of + GLM-4.1V-9B-Thinking [THUDM/GLM-4.1V-9B-Thinking](https://huggingface.co/THUDM/GLM-4.1V-9B-Thinking). + + Configuration objects inherit from [`PreTrainedConfig`] and can be used to control the model outputs. Read the + documentation from [`PreTrainedConfig`] for more information. + + Args: + vocab_size (`int`, *optional*, defaults to 151552): + Vocabulary size of the Glm46V model. Defines the number of different tokens that can be represented by the + `inputs_ids` passed when calling [`Glm46VModel`] + hidden_size (`int`, *optional*, defaults to 4096): + Dimension of the hidden representations. + intermediate_size (`int`, *optional*, defaults to 13696): + Dimension of the MLP representations. + num_hidden_layers (`int`, *optional*, defaults to 40): + Number of hidden layers in the Transformer encoder. + num_attention_heads (`int`, *optional*, defaults to 32): + Number of attention heads for each attention layer in the Transformer encoder. + num_key_value_heads (`int`, *optional*, defaults to 2): + This is the number of key_value heads that should be used to implement Grouped Query Attention. If + `num_key_value_heads=num_attention_heads`, the model will use Multi Head Attention (MHA), if + `num_key_value_heads=1` the model will use Multi Query Attention (MQA) otherwise GQA is used. When + converting a multi-head checkpoint to a GQA checkpoint, each group key and value head should be constructed + by meanpooling all the original heads within that group. For more details checkout [this + paper](https://huggingface.co/papers/2305.13245). If it is not specified, will default to `32`. + hidden_act (`str` or `function`, *optional*, defaults to `"silu"`): + The non-linear activation function (function or string) in the decoder. + max_position_embeddings (`int`, *optional*, defaults to 32768): + The maximum sequence length that this model might ever be used with. + initializer_range (`float`, *optional*, defaults to 0.02): + The standard deviation of the truncated_normal_initializer for initializing all weight matrices. + rms_norm_eps (`float`, *optional*, defaults to 1e-05): + The epsilon used by the rms normalization layers. + use_cache (`bool`, *optional*, defaults to `True`): + Whether or not the model should return the last key/values attentions (not used by all models). Only + relevant if `config.is_decoder=True`. + tie_word_embeddings (`bool`, *optional*, defaults to `False`): + Whether the model's input and output word embeddings should be tied. + attention_dropout (`float`, *optional*, defaults to 0.0): + The dropout ratio for the attention probabilities. + rope_parameters (`RopeParameters`, *optional*): + Dictionary containing the configuration parameters for the RoPE embeddings. The dictionaty should contain + a value for `rope_theta` and optionally parameters used for scaling in case you want to use RoPE + with longer `max_position_embeddings`. + image_token_id (`int`, *optional*): + Token index used as placeholder for image embeddings. + video_token_id (`int`, *optional*): + Token index used as placeholder for video embeddings. + + ```python + >>> from transformers import Glm46VTextModel, Glm46VConfig + + >>> # Initializing a GLM-4.1V style configuration + >>> configuration = Glm46VConfig() + + >>> # Initializing a model from the GLM-4.1V style configuration + >>> model = Glm46VTextModel(configuration) + + >>> # Accessing the model configuration + >>> configuration = model.config + ```""" + + model_type = "glm46v_text" + base_config_key = "text_config" + keys_to_ignore_at_inference = ["past_key_values"] + # Default tensor parallel plan for base model `Glm46V` + base_model_tp_plan = { + "layers.*.self_attn.q_proj": "colwise", + "layers.*.self_attn.k_proj": "colwise", + "layers.*.self_attn.v_proj": "colwise", + "layers.*.self_attn.o_proj": "rowwise", + "layers.*.mlp.gate_up_proj": "colwise_rep", # we need to replicate here due to the `chunk` operation + "layers.*.mlp.down_proj": "rowwise_rep", # we need to replicate here due to the `chunk` operation + } + base_model_pp_plan = { + "embed_tokens": (["input_ids"], ["inputs_embeds"]), + "layers": (["hidden_states", "attention_mask"], ["hidden_states"]), + "norm": (["hidden_states"], ["hidden_states"]), + } + + def __init__( + self, + vocab_size: Optional[int] = 151552, + hidden_size: Optional[int] = 4096, + intermediate_size: Optional[int] = 13696, + num_hidden_layers: Optional[int] = 40, + num_attention_heads: Optional[int] = 32, + num_key_value_heads: Optional[int] = 2, + hidden_act: Optional[str] = "silu", + max_position_embeddings: Optional[int] = 32768, + initializer_range: Optional[float] = 0.02, + rms_norm_eps: Optional[int] = 1e-05, + use_cache: Optional[bool] = True, + tie_word_embeddings: Optional[bool] = False, + attention_dropout: Optional[float] = 0.0, + rope_parameters: Optional[RopeParameters | dict[str, RopeParameters]] = None, + image_token_id: Optional[int] = None, + video_token_id: Optional[int] = None, + **kwargs, + ): + self.vocab_size = vocab_size + self.max_position_embeddings = max_position_embeddings + self.hidden_size = hidden_size + self.intermediate_size = intermediate_size + self.num_hidden_layers = num_hidden_layers + self.num_attention_heads = num_attention_heads + + # for backward compatibility + if num_key_value_heads is None: + num_key_value_heads = num_attention_heads + + self.num_key_value_heads = num_key_value_heads + self.hidden_act = hidden_act + self.initializer_range = initializer_range + self.rms_norm_eps = rms_norm_eps + self.use_cache = use_cache + self.attention_dropout = attention_dropout + # Try to set `rope_scaling` if available, otherwise use `rope_parameters` + rope_scaling = kwargs.pop("rope_scaling", None) + self.rope_parameters = rope_scaling or rope_parameters + + # Validate the correctness of rotary position embeddings parameters + rope_theta = kwargs.get("rope_theta", 10000.0) + standardize_rope_params(self, rope_theta=rope_theta) + rope_config_validation(self, ignore_keys={"mrope_section"}) + self.image_token_id = image_token_id + self.video_token_id = video_token_id + + super().__init__(tie_word_embeddings=tie_word_embeddings, **kwargs) + + +class Glm46VConfig(PreTrainedConfig): + r""" + This is the configuration class to store the configuration of a [`Glm46VModel`]. It is used to instantiate a + GLM-4.1V model according to the specified arguments, defining the model architecture. Instantiating a + configuration with the defaults will yield a similar configuration to that of + GLM-4.1V-9B-Thinking [THUDM/GLM-4.1V-9B-Thinking](https://huggingface.co/THUDM/GLM-4.1V-9B-Thinking). + + Configuration objects inherit from [`PreTrainedConfig`] and can be used to control the model outputs. Read the + documentation from [`PreTrainedConfig`] for more information. + + + Args: + text_config (`Union[PreTrainedConfig, dict]`, *optional*, defaults to `Glm46VTextConfig`): + The config object or dictionary of the text backbone. + vision_config (`Union[PreTrainedConfig, dict]`, *optional*, defaults to `Glm46VVisionConfig`): + The config object or dictionary of the vision backbone. + image_token_id (`int`, *optional*, defaults to 151343): + The image token index to encode the image prompt. + video_token_id (`int`, *optional*, defaults to 151344): + The video token index to encode the image prompt. + image_start_token_id (`int`, *optional*, defaults to 151339): + The image start token index to encode the start of image. + image_end_token_id (`int`, *optional*, defaults to 151340): + The image end token index to encode the end of image. + video_start_token_id (`int`, *optional*, defaults to 151341): + The video start token index to encode the start of video. + video_end_token_id (`int`, *optional*, defaults to 151342): + The video end token index to encode the end of video. + + ```python + >>> from transformers import Glm46VForConditionalGeneration, Glm46VConfig + + >>> # Initializing a GLM-4.1V style configuration + >>> configuration = Glm46VConfig() + + >>> # Initializing a model from the GLM-4.1V style configuration + >>> model = Glm46VForConditionalGeneration(configuration) + + >>> # Accessing the model configuration + >>> configuration = model.config + ```""" + + model_type = "glm46v" + sub_configs = {"vision_config": Glm46VVisionConfig, "text_config": Glm46VTextConfig} + keys_to_ignore_at_inference = ["past_key_values"] + + def __init__( + self, + text_config=None, + vision_config=None, + image_token_id=151343, + video_token_id=151344, + image_start_token_id=151339, + image_end_token_id=151340, + video_start_token_id=151341, + video_end_token_id=151342, + **kwargs, + ): + if isinstance(vision_config, dict): + self.vision_config = self.sub_configs["vision_config"](**vision_config) + elif vision_config is None: + self.vision_config = self.sub_configs["vision_config"]() + + if isinstance(text_config, dict): + self.text_config = self.sub_configs["text_config"](**text_config) + elif text_config is None: + self.text_config = self.sub_configs["text_config"](**kwargs) + + self.image_token_id = image_token_id + self.video_token_id = video_token_id + self.video_start_token_id = video_start_token_id + self.video_end_token_id = video_end_token_id + self.image_start_token_id = image_start_token_id + self.image_end_token_id = image_end_token_id + + super().__init__(**kwargs) + + +__all__ = ["Glm46VConfig", "Glm46VTextConfig"] diff --git a/src/transformers/models/glm46v/modeling_glm46v.py b/src/transformers/models/glm46v/modeling_glm46v.py new file mode 100644 index 000000000000..b0361e5dc55a --- /dev/null +++ b/src/transformers/models/glm46v/modeling_glm46v.py @@ -0,0 +1,1701 @@ +# 🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨 +# This file was automatically generated from src/transformers/models/glm46v/modular_glm46v.py. +# Do NOT edit this file manually as any edits will be overwritten by the generation of +# the file from the modular. If any change should be done, please apply the change to the +# modular_glm46v.py file directly. One of our CI enforces this. +# 🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨 +# coding=utf-8 +# Copyright 2025 the HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import itertools +from collections.abc import Callable +from dataclasses import dataclass +from typing import Any, Optional, Union + +import torch +import torch.nn as nn +import torch.nn.functional as F +from torch.nn import LayerNorm + +from ...activations import ACT2FN +from ...cache_utils import Cache, DynamicCache +from ...generation import GenerationMixin +from ...integrations import use_kernel_forward_from_hub +from ...masking_utils import create_causal_mask +from ...modeling_flash_attention_utils import FlashAttentionKwargs +from ...modeling_layers import GradientCheckpointingLayer +from ...modeling_outputs import BaseModelOutputWithPast, ModelOutput +from ...modeling_rope_utils import ROPE_INIT_FUNCTIONS, dynamic_rope_update +from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel +from ...processing_utils import Unpack +from ...utils import TransformersKwargs, auto_docstring, can_return_tuple, is_torchdynamo_compiling +from ...utils.generic import check_model_inputs +from .configuration_glm46v import Glm46VConfig, Glm46VTextConfig, Glm46VVisionConfig + + +@use_kernel_forward_from_hub("RMSNorm") +class Glm46VRMSNorm(nn.Module): + def __init__(self, hidden_size, eps=1e-6): + """ + Glm46VRMSNorm is equivalent to T5LayerNorm + """ + super().__init__() + self.weight = nn.Parameter(torch.ones(hidden_size)) + self.variance_epsilon = eps + + def forward(self, hidden_states): + input_dtype = hidden_states.dtype + hidden_states = hidden_states.to(torch.float32) + variance = hidden_states.pow(2).mean(-1, keepdim=True) + hidden_states = hidden_states * torch.rsqrt(variance + self.variance_epsilon) + return self.weight * hidden_states.to(input_dtype) + + def extra_repr(self): + return f"{tuple(self.weight.shape)}, eps={self.variance_epsilon}" + + +class Glm4VisionMlp(nn.Module): + def __init__(self, config, bias: bool = False): + super().__init__() + self.hidden_size = config.hidden_size + self.intermediate_size = config.out_hidden_size + self.gate_proj = nn.Linear(self.hidden_size, self.intermediate_size, bias=bias) + self.up_proj = nn.Linear(self.hidden_size, self.intermediate_size, bias=bias) + self.down_proj = nn.Linear(self.intermediate_size, self.hidden_size, bias=bias) + self.act_fn = ACT2FN[config.hidden_act] + + def forward(self, hidden_state): + return self.down_proj(self.act_fn(self.gate_proj(hidden_state)) * self.up_proj(hidden_state)) + + +class Glm46VVisionPatchEmbed(nn.Module): + def __init__(self, config: Glm46VVisionConfig) -> None: + super().__init__() + self.patch_size = config.patch_size + self.temporal_patch_size = config.temporal_patch_size + self.in_channels = config.in_channels + self.embed_dim = config.hidden_size + + kernel_size = [self.temporal_patch_size, self.patch_size, self.patch_size] + self.proj = nn.Conv3d(self.in_channels, self.embed_dim, kernel_size=kernel_size, stride=kernel_size) + + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: + target_dtype = self.proj.weight.dtype + hidden_states = hidden_states.view( + -1, self.in_channels, self.temporal_patch_size, self.patch_size, self.patch_size + ) + hidden_states = self.proj(hidden_states.to(dtype=target_dtype)).view(-1, self.embed_dim) + return hidden_states + + +class Glm46VVisionRotaryEmbedding(nn.Module): + inv_freq: torch.Tensor # fix linting for `register_buffer` + + def __init__(self, dim: int, theta: float = 10000.0) -> None: + super().__init__() + inv_freq = 1.0 / (theta ** (torch.arange(0, dim, 2, dtype=torch.float) / dim)) + self.register_buffer("inv_freq", inv_freq, persistent=False) + + def forward(self, seqlen: int) -> torch.Tensor: + seq = torch.arange(seqlen, device=self.inv_freq.device, dtype=self.inv_freq.dtype) + freqs = torch.outer(seq, self.inv_freq) + return freqs + + +class Glm46VVisionPatchMerger(nn.Module): + def __init__(self, dim: int, context_dim: int, hidden_act: str, bias: bool = False) -> None: + super().__init__() + self.proj = nn.Linear(dim, dim, bias=bias) + self.post_projection_norm = LayerNorm(dim) + self.gate_proj = nn.Linear(dim, context_dim, bias=bias) + self.up_proj = nn.Linear(dim, context_dim, bias=bias) + self.down_proj = nn.Linear(context_dim, dim, bias=bias) + self.act1 = nn.GELU() + self.act_fn = ACT2FN[hidden_act] + + def forward(self, hidden_state: torch.Tensor) -> torch.Tensor: + hidden_state = self.proj(hidden_state) + hidden_state = self.act1(self.post_projection_norm(hidden_state)) + return self.down_proj(self.act_fn(self.gate_proj(hidden_state)) * self.up_proj(hidden_state)) + + +class Glm46VVisionEmbeddings(nn.Module): + def __init__(self, config: Glm46VVisionConfig): + super().__init__() + self.config = config + self.embed_dim = config.hidden_size + self.image_size = config.image_size + self.patch_size = config.patch_size + + self.num_patches = (self.image_size // self.patch_size) ** 2 + self.num_positions = self.num_patches + self.position_embedding = nn.Embedding(self.num_positions, self.embed_dim) + self.register_buffer("position_ids", torch.arange(self.num_positions).expand((1, -1)), persistent=False) + + def forward(self, embeddings, lengths, image_shapes, h_coords, w_coords) -> torch.Tensor: + """ + Forward pass with integrated position encoding adaptation using 2D interpolation. + + Args: + embeddings: Input embeddings tensor + lengths (torch.Tensor): Sequence lengths for each image in the batch. + image_shapes (torch.Tensor): Tensor of shape [batch_size, 3] representing the image shapes (t, h, w). + h_coords (torch.Tensor): Tensor of shape [total_seq] representing the h coordinate for each patch. + w_coords (torch.Tensor): Tensor of shape [total_seq] representing the w coordinate for each patch. + + Returns: + torch.Tensor: Embeddings with adapted position encoding added. + """ + # Get position embedding parameters + pos_embed_weight = self.position_embedding.weight + hidden_size = pos_embed_weight.shape[1] + total_seq = h_coords.shape[0] + device = pos_embed_weight.device + + # Move coordinates to correct device + h_coords, w_coords = h_coords.to(device), w_coords.to(device) + + # Handle empty sequence case + if total_seq == 0: + adapted_pos_embed = torch.empty(0, hidden_size, device=device, dtype=pos_embed_weight.dtype) + else: + # Convert inputs to tensors if needed + if isinstance(lengths, list): + lengths = torch.tensor(lengths, device=device, dtype=torch.long) + if not isinstance(image_shapes, torch.Tensor): + image_shapes = torch.tensor(image_shapes, device=device, dtype=torch.long) + + # Prepare 2D position embedding + orig_size_sq = pos_embed_weight.shape[0] + orig_size = int(orig_size_sq**0.5) + pos_embed_2d = ( + pos_embed_weight.view(orig_size, orig_size, hidden_size) + .permute(2, 0, 1) + .unsqueeze(0) + .to(device=device, dtype=torch.float32) + ) + + # Calculate target dimensions for each patch + target_h = torch.cat([image_shapes[i, 1].repeat(lengths[i]) for i in range(len(lengths))]).to( + device=device, dtype=torch.float32 + ) + target_w = torch.cat([image_shapes[i, 2].repeat(lengths[i]) for i in range(len(lengths))]).to( + device=device, dtype=torch.float32 + ) + + # Normalize coordinates to [-1, 1] range for grid_sample + h_coords = h_coords.to(device=device, dtype=torch.float32) + w_coords = w_coords.to(device=device, dtype=torch.float32) + norm_w = ((w_coords + 0.5) / target_w) * 2 - 1 + norm_h = ((h_coords + 0.5) / target_h) * 2 - 1 + + # Create sampling grid + grid = torch.stack((norm_w, norm_h), dim=-1).unsqueeze(0).unsqueeze(2) + + # Perform bicubic interpolation + interpolated_embed_fp32 = F.grid_sample( + pos_embed_2d, grid, mode="bicubic", align_corners=False, padding_mode="border" + ) + + # Reshape and convert back to original dtype + adapted_pos_embed_fp32 = interpolated_embed_fp32.squeeze(0).squeeze(-1).permute(1, 0) + adapted_pos_embed = adapted_pos_embed_fp32.to(pos_embed_weight.dtype).to(embeddings.device) + + # Add adapted position encoding to embeddings + embeddings = embeddings + adapted_pos_embed + return embeddings + + +def rotate_half(x): + """Rotates half the hidden dims of the input.""" + x1 = x[..., : x.shape[-1] // 2] + x2 = x[..., x.shape[-1] // 2 :] + return torch.cat((-x2, x1), dim=-1) + + +def apply_rotary_pos_emb_vision( + q: torch.Tensor, k: torch.Tensor, cos: torch.Tensor, sin: torch.Tensor +) -> tuple[torch.Tensor, torch.Tensor]: + orig_q_dtype = q.dtype + orig_k_dtype = k.dtype + q, k = q.float(), k.float() + cos, sin = cos.unsqueeze(-2).float(), sin.unsqueeze(-2).float() + q_embed = (q * cos) + (rotate_half(q) * sin) + k_embed = (k * cos) + (rotate_half(k) * sin) + q_embed = q_embed.to(orig_q_dtype) + k_embed = k_embed.to(orig_k_dtype) + return q_embed, k_embed + + +def repeat_kv(hidden_states: torch.Tensor, n_rep: int) -> torch.Tensor: + """ + This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch, + num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim) + """ + batch, num_key_value_heads, slen, head_dim = hidden_states.shape + if n_rep == 1: + return hidden_states + hidden_states = hidden_states[:, :, None, :, :].expand(batch, num_key_value_heads, n_rep, slen, head_dim) + return hidden_states.reshape(batch, num_key_value_heads * n_rep, slen, head_dim) + + +def eager_attention_forward( + module: nn.Module, + query: torch.Tensor, + key: torch.Tensor, + value: torch.Tensor, + attention_mask: Optional[torch.Tensor], + scaling: float, + dropout: float = 0.0, + **kwargs: Unpack[TransformersKwargs], +): + key_states = repeat_kv(key, module.num_key_value_groups) + value_states = repeat_kv(value, module.num_key_value_groups) + + attn_weights = torch.matmul(query, key_states.transpose(2, 3)) * scaling + if attention_mask is not None: + causal_mask = attention_mask[:, :, :, : key_states.shape[-2]] + attn_weights = attn_weights + causal_mask + + attn_weights = nn.functional.softmax(attn_weights, dim=-1, dtype=torch.float32).to(query.dtype) + attn_weights = nn.functional.dropout(attn_weights, p=dropout, training=module.training) + attn_output = torch.matmul(attn_weights, value_states) + attn_output = attn_output.transpose(1, 2).contiguous() + + return attn_output, attn_weights + + +class Glm46VVisionAttention(nn.Module): + def __init__(self, config: Glm46VVisionConfig) -> None: + super().__init__() + self.dim = config.hidden_size + self.num_heads = config.num_heads + self.head_dim = self.dim // self.num_heads + self.num_key_value_groups = 1 # needed for eager attention + self.qkv = nn.Linear(config.hidden_size, config.hidden_size * 3, bias=config.attention_bias) + self.proj = nn.Linear(config.hidden_size, config.hidden_size, bias=False) + self.scaling = self.head_dim**-0.5 + self.config = config + self.attention_dropout = config.attention_dropout + self.is_causal = False + + def forward( + self, + hidden_states: torch.Tensor, + cu_seqlens: torch.Tensor, + rotary_pos_emb: Optional[torch.Tensor] = None, + position_embeddings: Optional[tuple[torch.Tensor, torch.Tensor]] = None, + **kwargs, + ) -> torch.Tensor: + seq_length = hidden_states.shape[0] + query_states, key_states, value_states = ( + self.qkv(hidden_states).reshape(seq_length, 3, self.num_heads, -1).permute(1, 0, 2, 3).unbind(0) + ) + cos, sin = position_embeddings + query_states, key_states = apply_rotary_pos_emb_vision(query_states, key_states, cos, sin) + + query_states = query_states.transpose(0, 1).unsqueeze(0) + key_states = key_states.transpose(0, 1).unsqueeze(0) + value_states = value_states.transpose(0, 1).unsqueeze(0) + + attention_interface: Callable = eager_attention_forward + if self.config._attn_implementation != "eager": + attention_interface = ALL_ATTENTION_FUNCTIONS[self.config._attn_implementation] + + if self.config._attn_implementation == "flash_attention_2": + # Flash Attention 2: Use cu_seqlens for variable length attention + max_seqlen = (cu_seqlens[1:] - cu_seqlens[:-1]).max() + attn_output, _ = attention_interface( + self, + query_states, + key_states, + value_states, + attention_mask=None, + scaling=self.scaling, + dropout=0.0 if not self.training else self.attention_dropout, + cu_seq_lens_q=cu_seqlens, + cu_seq_lens_k=cu_seqlens, + max_length_q=max_seqlen, + max_length_k=max_seqlen, + is_causal=False, + **kwargs, + ) + else: + # Other implementations: Process each chunk separately + lengths = cu_seqlens[1:] - cu_seqlens[:-1] + splits = [ + torch.split(tensor, lengths.tolist(), dim=2) for tensor in (query_states, key_states, value_states) + ] + + attn_outputs = [ + attention_interface( + self, + q, + k, + v, + attention_mask=None, + scaling=self.scaling, + dropout=0.0 if not self.training else self.attention_dropout, + is_causal=False, + **kwargs, + )[0] + for q, k, v in zip(*splits) + ] + attn_output = torch.cat(attn_outputs, dim=1) + + attn_output = attn_output.reshape(seq_length, -1).contiguous() + attn_output = self.proj(attn_output) + return attn_output + + +class Glm46VisionMlp(nn.Module): + def __init__(self, config, bias: bool = False): + super().__init__() + self.hidden_size = config.hidden_size + self.intermediate_size = config.out_hidden_size + self.gate_proj = nn.Linear(self.hidden_size, self.intermediate_size, bias=bias) + self.up_proj = nn.Linear(self.hidden_size, self.intermediate_size, bias=bias) + self.down_proj = nn.Linear(self.intermediate_size, self.hidden_size, bias=bias) + self.act_fn = ACT2FN[config.hidden_act] + + def forward(self, hidden_state): + return self.down_proj(self.act_fn(self.gate_proj(hidden_state)) * self.up_proj(hidden_state)) + + +class Glm46VVisionBlock(GradientCheckpointingLayer): + def __init__(self, config) -> None: + super().__init__() + self.norm1 = Glm46VRMSNorm(config.hidden_size, eps=config.rms_norm_eps) + self.norm2 = Glm46VRMSNorm(config.hidden_size, eps=config.rms_norm_eps) + self.attn = Glm46VVisionAttention(config) + self.mlp = Glm46VisionMlp(config, bias=False) + + def forward( + self, + hidden_states: torch.Tensor, + cu_seqlens: torch.Tensor, + rotary_pos_emb: Optional[torch.Tensor] = None, + position_embeddings: Optional[tuple[torch.Tensor, torch.Tensor]] = None, + **kwargs, + ) -> torch.Tensor: + hidden_states = hidden_states + self.attn( + self.norm1(hidden_states), + cu_seqlens=cu_seqlens, + rotary_pos_emb=rotary_pos_emb, + position_embeddings=position_embeddings, + **kwargs, + ) + hidden_states = hidden_states + self.mlp(self.norm2(hidden_states)) + return hidden_states + + +class Glm46VTextRotaryEmbedding(nn.Module): + inv_freq: torch.Tensor # fix linting for `register_buffer` + + def __init__(self, config: Glm46VTextConfig, device=None): + super().__init__() + self.max_seq_len_cached = config.max_position_embeddings + self.original_max_seq_len = config.max_position_embeddings + + self.config = config + + self.rope_type = self.config.rope_parameters["rope_type"] + rope_init_fn: Callable = self.compute_default_rope_parameters + if self.rope_type != "default": + rope_init_fn = ROPE_INIT_FUNCTIONS[self.rope_type] + inv_freq, self.attention_scaling = rope_init_fn(self.config, device) + + self.register_buffer("inv_freq", inv_freq, persistent=False) + self.original_inv_freq = inv_freq + + @staticmethod + def compute_default_rope_parameters( + config: Optional[Glm46VTextConfig] = None, + device: Optional["torch.device"] = None, + seq_len: Optional[int] = None, + ) -> tuple["torch.Tensor", float]: + """ + Computes the inverse frequencies according to the original RoPE implementation + Args: + config ([`~transformers.PreTrainedConfig`]): + The model configuration. + device (`torch.device`): + The device to use for initialization of the inverse frequencies. + seq_len (`int`, *optional*): + The current sequence length. Unused for this type of RoPE. + Returns: + Tuple of (`torch.Tensor`, `float`), containing the inverse frequencies for the RoPE embeddings and the + post-processing scaling factor applied to the computed cos/sin (unused in this type of RoPE). + """ + base = config.rope_parameters["rope_theta"] + partial_rotary_factor = getattr(config, "partial_rotary_factor", 1.0) + head_dim = getattr(config, "head_dim", None) or config.hidden_size // config.num_attention_heads + dim = int(head_dim * partial_rotary_factor) + + attention_factor = 1.0 # Unused in this type of RoPE + + # Compute the inverse frequencies + inv_freq = 1.0 / ( + base ** (torch.arange(0, dim, 2, dtype=torch.int64).to(device=device, dtype=torch.float) / dim) + ) + return inv_freq, attention_factor + + @torch.no_grad() + @dynamic_rope_update # power user: used with advanced RoPE types (e.g. dynamic rope) + def forward(self, x, position_ids): + # In contrast to other models, GLM46V different position ids for the grids + # So we expand the inv_freq to shape (3, ...) + inv_freq_expanded = self.inv_freq[None, None, :, None].float().expand(3, position_ids.shape[1], -1, 1) + position_ids_expanded = position_ids[:, :, None, :].float() # shape (3, bs, 1, positions) + + device_type = x.device.type if isinstance(x.device.type, str) and x.device.type != "mps" else "cpu" + with torch.autocast(device_type=device_type, enabled=False): # Force float32 + freqs = (inv_freq_expanded.float() @ position_ids_expanded.float()).transpose(2, 3) + emb = torch.cat((freqs, freqs), dim=-1) + cos = emb.cos() * self.attention_scaling + sin = emb.sin() * self.attention_scaling + + return cos.to(dtype=x.dtype), sin.to(dtype=x.dtype) + + +def rotate_half_llm(x): + """Rotates half the hidden dims of the input.""" + x1 = x[..., 0::2] + x2 = x[..., 1::2] + return torch.stack((-x2, x1), dim=-1).flatten(-2) + + +def apply_multimodal_rotary_pos_emb(q, k, cos, sin, mrope_section, unsqueeze_dim=1): + """Applies Rotary Position Embedding with Multimodal Sections to the query and key tensors (https://qwenlm.github.io/blog/qwen2-vl/). + + Explanation: + Multimodal 3D rotary position embedding is an extension to 1D rotary position embedding. The input embedding + sequence contains vision (images / videos) embedding and text embedding or just contains text embedding. For + vision embedding part, we apply rotary position embedding on temporal, height and width dimension separately. + Here we split the channel dimension to 3 chunks for the temporal, height and width rotary position embedding. + For text embedding part, we just apply 1D rotary position embedding. The three rotary position index (temporal, + height and width) of text embedding is always the same, so the text embedding rotary position embedding has no + difference with modern LLMs. + + Args: + q (`torch.Tensor`): The query tensor. + k (`torch.Tensor`): The key tensor. + cos (`torch.Tensor`): The cosine part of the rotary embedding. + sin (`torch.Tensor`): The sine part of the rotary embedding. + mrope_section(`List(int)`): + Multimodal rope section is for channel dimension of temporal, height and width in rope calculation. + unsqueeze_dim (`int`, *optional*, defaults to 1): + The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and + sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note + that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and + k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes + cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have + the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2. + Returns: + `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding. + """ + mrope_section = mrope_section * 2 + cos = torch.cat([m[i % 3] for i, m in enumerate(cos.split(mrope_section, dim=-1))], dim=-1).unsqueeze( + unsqueeze_dim + ) + sin = torch.cat([m[i % 3] for i, m in enumerate(sin.split(mrope_section, dim=-1))], dim=-1).unsqueeze( + unsqueeze_dim + ) + + # Interleave them instead of usual shape + cos = cos[..., : cos.shape[-1] // 2].repeat_interleave(2, dim=-1) + sin = sin[..., : sin.shape[-1] // 2].repeat_interleave(2, dim=-1) + + # Keep half or full tensor for later concatenation + rotary_dim = cos.shape[-1] + q_rot, q_pass = q[..., :rotary_dim], q[..., rotary_dim:] + k_rot, k_pass = k[..., :rotary_dim], k[..., rotary_dim:] + + # Apply rotary embeddings on the first half or full tensor + q_embed = (q_rot * cos) + (rotate_half_llm(q_rot) * sin) + k_embed = (k_rot * cos) + (rotate_half_llm(k_rot) * sin) + + # Concatenate back to full shape + q_embed = torch.cat([q_embed, q_pass], dim=-1) + k_embed = torch.cat([k_embed, k_pass], dim=-1) + + return q_embed, k_embed + + +class Glm46VTextAttention(nn.Module): + """ + Multi-headed attention from 'Attention Is All You Need' paper. + and "Generating Long Sequences with Sparse Transformers". + """ + + def __init__(self, config: Glm46VTextConfig, layer_idx: Optional[int] = None): + super().__init__() + self.config = config + self.layer_idx = layer_idx + + self.hidden_size = config.hidden_size + self.num_heads = config.num_attention_heads + self.head_dim = self.hidden_size // self.num_heads + self.num_key_value_heads = config.num_key_value_heads + self.num_key_value_groups = self.num_heads // self.num_key_value_heads + self.is_causal = True + self.attention_dropout = config.attention_dropout + self.rope_parameters = config.rope_parameters + self.scaling = self.head_dim**-0.5 + + self.q_proj = nn.Linear(self.hidden_size, self.num_heads * self.head_dim, bias=True) + self.k_proj = nn.Linear(self.hidden_size, self.num_key_value_heads * self.head_dim, bias=True) + self.v_proj = nn.Linear(self.hidden_size, self.num_key_value_heads * self.head_dim, bias=True) + self.o_proj = nn.Linear(self.num_heads * self.head_dim, self.hidden_size, bias=False) + + def forward( + self, + hidden_states: torch.Tensor, + position_embeddings: Optional[tuple[torch.Tensor, torch.Tensor]] = None, + attention_mask: Optional[torch.Tensor] = None, + past_key_values: Optional[Cache] = None, + cache_position: Optional[torch.LongTensor] = None, + **kwargs: Unpack[FlashAttentionKwargs], + ) -> tuple[torch.Tensor, Optional[torch.Tensor], Optional[tuple[torch.Tensor]]]: + bsz, q_len, _ = hidden_states.size() + + query_states = self.q_proj(hidden_states) + key_states = self.k_proj(hidden_states) + value_states = self.v_proj(hidden_states) + + query_states = query_states.view(bsz, q_len, -1, self.head_dim).transpose(1, 2) + key_states = key_states.view(bsz, q_len, -1, self.head_dim).transpose(1, 2) + value_states = value_states.view(bsz, q_len, -1, self.head_dim).transpose(1, 2) + + cos, sin = position_embeddings + query_states, key_states = apply_multimodal_rotary_pos_emb( # diff with Llama + query_states, key_states, cos, sin, self.rope_parameters["mrope_section"] + ) + + if past_key_values is not None: + cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position} # Specific to RoPE models + key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx, cache_kwargs) + + attention_interface: Callable = eager_attention_forward + if self.config._attn_implementation != "eager": + attention_interface = ALL_ATTENTION_FUNCTIONS[self.config._attn_implementation] + + attn_output, attn_weights = attention_interface( + self, + query_states, + key_states, + value_states, + attention_mask, + dropout=0.0 if not self.training else self.attention_dropout, + scaling=self.scaling, + **kwargs, + ) + + attn_output = attn_output.reshape(bsz, q_len, -1).contiguous() + attn_output = self.o_proj(attn_output) + return attn_output, attn_weights + + +class Glm46VTextMLP(nn.Module): + def __init__(self, config): + super().__init__() + + self.config = config + self.gate_up_proj = nn.Linear(config.hidden_size, 2 * config.intermediate_size, bias=False) + self.down_proj = nn.Linear(config.intermediate_size, config.hidden_size, bias=False) + self.activation_fn = ACT2FN[config.hidden_act] + + def forward(self, hidden_states: torch.FloatTensor) -> torch.FloatTensor: + up_states = self.gate_up_proj(hidden_states) + + gate, up_states = up_states.chunk(2, dim=-1) + up_states = up_states * self.activation_fn(gate) + + return self.down_proj(up_states) + + +class Glm46VTextDecoderLayer(GradientCheckpointingLayer): + def __init__(self, config: Glm46VTextConfig, layer_idx: int): + super().__init__() + self.hidden_size = config.hidden_size + self.self_attn = Glm46VTextAttention(config, layer_idx) + self.mlp = Glm46VTextMLP(config) + self.input_layernorm = Glm46VRMSNorm(config.hidden_size, eps=config.rms_norm_eps) + self.post_attention_layernorm = Glm46VRMSNorm(config.hidden_size, eps=config.rms_norm_eps) + self.post_self_attn_layernorm = Glm46VRMSNorm(config.hidden_size, eps=config.rms_norm_eps) + self.post_mlp_layernorm = Glm46VRMSNorm(config.hidden_size, eps=config.rms_norm_eps) + + @auto_docstring + def forward( + self, + hidden_states: torch.Tensor, + position_embeddings: Optional[tuple[torch.Tensor, torch.Tensor]] = None, + attention_mask: Optional[torch.Tensor] = None, + position_ids: Optional[torch.LongTensor] = None, + past_key_values: Optional[Cache] = None, + use_cache: Optional[bool] = False, + cache_position: Optional[torch.LongTensor] = None, + **kwargs, + ) -> tuple[torch.FloatTensor, Optional[tuple[torch.FloatTensor, torch.FloatTensor]]]: + residual = hidden_states + + hidden_states = self.input_layernorm(hidden_states) + + # Self Attention + hidden_states, _ = self.self_attn( + hidden_states=hidden_states, + position_embeddings=position_embeddings, + attention_mask=attention_mask, + position_ids=position_ids, + past_key_values=past_key_values, + use_cache=use_cache, + cache_position=cache_position, + **kwargs, + ) + + hidden_states = self.post_self_attn_layernorm(hidden_states) + hidden_states = residual + hidden_states + + # Fully Connected + residual = hidden_states + hidden_states = self.post_attention_layernorm(hidden_states) + hidden_states = self.mlp(hidden_states) + hidden_states = self.post_mlp_layernorm(hidden_states) + hidden_states = residual + hidden_states + + return hidden_states + + +@dataclass +@auto_docstring( + custom_intro=""" + Base class for Llava outputs, with hidden states and attentions. + """ +) +class Glm46VModelOutputWithPast(ModelOutput): + r""" + past_key_values (`Cache`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`): + It is a [`~cache_utils.Cache`] instance. For more details, see our [kv cache guide](https://huggingface.co/docs/transformers/en/kv_cache). + + Contains pre-computed hidden-states (key and values in the self-attention blocks) that can be used (see + `past_key_values` input) to speed up sequential decoding. + rope_deltas (`torch.LongTensor` of shape `(batch_size, )`, *optional*): + The rope index difference between sequence length and multimodal rope. + """ + + last_hidden_state: Optional[torch.FloatTensor] = None + past_key_values: Optional[Cache] = None + hidden_states: Optional[tuple[torch.FloatTensor]] = None + attentions: Optional[tuple[torch.FloatTensor]] = None + rope_deltas: Optional[torch.LongTensor] = None + + +@auto_docstring +class Glm46VPreTrainedModel(PreTrainedModel): + config: Glm46VConfig + base_model_prefix = "model" + input_modalities = ["image", "video", "text"] + supports_gradient_checkpointing = True + _no_split_modules = ["Glm46VTextDecoderLayer", "Glm46VVisionBlock"] + _skip_keys_device_placement = "past_key_values" + _supports_flash_attn = True + _supports_sdpa = True + + _can_compile_fullgraph = True + _supports_attention_backend = True + _can_record_outputs = { + "hidden_states": Glm46VTextDecoderLayer, + "attentions": Glm46VTextAttention, + } + + +class Glm46VVisionModel(Glm46VPreTrainedModel): + config: Glm46VVisionConfig + input_modalities = ["image", "video"] + _no_split_modules = ["Glm46VVisionBlock"] + + def __init__(self, config) -> None: + super().__init__(config) + self.spatial_merge_size = config.spatial_merge_size + self.patch_size = config.patch_size + + self.embeddings = Glm46VVisionEmbeddings(config) + self.patch_embed = Glm46VVisionPatchEmbed(config) + + head_dim = config.hidden_size // config.num_heads + self.rotary_pos_emb = Glm46VVisionRotaryEmbedding(head_dim // 2) + + self.blocks = nn.ModuleList([Glm46VVisionBlock(config) for _ in range(config.depth)]) + self.merger = Glm46VVisionPatchMerger( + dim=config.out_hidden_size, context_dim=config.intermediate_size, hidden_act=config.hidden_act + ) + + self.post_conv_layernorm = Glm46VRMSNorm(config.hidden_size, eps=config.rms_norm_eps) + self.downsample = nn.Conv2d( + in_channels=config.hidden_size, + out_channels=config.out_hidden_size, + kernel_size=config.spatial_merge_size, + stride=config.spatial_merge_size, + ) + self.post_layernorm = Glm46VRMSNorm(config.hidden_size, eps=config.rms_norm_eps) + + self.gradient_checkpointing = False + self.post_init() + + def rot_pos_emb(self, grid_thw): + pos_ids = [] + for t, h, w in grid_thw: + hpos_ids = torch.arange(h).unsqueeze(1).expand(-1, w) + hpos_ids = hpos_ids.reshape( + h // self.spatial_merge_size, + self.spatial_merge_size, + w // self.spatial_merge_size, + self.spatial_merge_size, + ) + hpos_ids = hpos_ids.permute(0, 2, 1, 3) + hpos_ids = hpos_ids.flatten() + + wpos_ids = torch.arange(w).unsqueeze(0).expand(h, -1) + wpos_ids = wpos_ids.reshape( + h // self.spatial_merge_size, + self.spatial_merge_size, + w // self.spatial_merge_size, + self.spatial_merge_size, + ) + wpos_ids = wpos_ids.permute(0, 2, 1, 3) + wpos_ids = wpos_ids.flatten() + pos_ids.append(torch.stack([hpos_ids, wpos_ids], dim=-1).repeat(t, 1)) + pos_ids = torch.cat(pos_ids, dim=0) + max_grid_size = grid_thw[:, 1:].max() + rotary_pos_emb_full = self.rotary_pos_emb(max_grid_size) + rotary_pos_emb = rotary_pos_emb_full[pos_ids].flatten(1) + return rotary_pos_emb, pos_ids + + def forward(self, hidden_states: torch.Tensor, grid_thw: torch.Tensor) -> torch.Tensor: + """ + Args: + hidden_states (`torch.Tensor` of shape `(seq_len, hidden_size)`): + The final hidden states of the model. + grid_thw (`torch.Tensor` of shape `(num_images_or_videos, 3)`): + The temporal, height and width of feature shape of each image in LLM. + + Returns: + `torch.Tensor`: hidden_states. + """ + hidden_states = self.patch_embed(hidden_states) + hidden_states = self.post_conv_layernorm(hidden_states) + + rotary_pos_emb, image_type_ids = self.rot_pos_emb(grid_thw) + emb = torch.cat((rotary_pos_emb, rotary_pos_emb), dim=-1) + position_embeddings = (emb.cos(), emb.sin()) + + cu_seqlens = torch.repeat_interleave(grid_thw[:, 1] * grid_thw[:, 2], grid_thw[:, 0]).cumsum( + dim=0, + # Select dtype based on the following factors: + # - FA2 requires that cu_seqlens_q must have dtype int32 + # - torch.onnx.export requires that cu_seqlens_q must have same dtype as grid_thw + # See https://github.com/huggingface/transformers/pull/34852 for more information + dtype=grid_thw.dtype if torch.jit.is_tracing() else torch.int32, + ) + cu_seqlens = F.pad(cu_seqlens, (1, 0), value=0) + seqlens = (cu_seqlens[1:] - cu_seqlens[:-1]).tolist() + hidden_states = self.embeddings(hidden_states, seqlens, grid_thw, image_type_ids[:, 0], image_type_ids[:, 1]) + + for blk in self.blocks: + hidden_states = blk( + hidden_states, + cu_seqlens=cu_seqlens, + position_embeddings=position_embeddings, + ) + + hidden_states = self.post_layernorm(hidden_states) + + hidden_states = hidden_states.view( + -1, self.spatial_merge_size, self.spatial_merge_size, hidden_states.shape[-1] + ) + hidden_states = hidden_states.permute(0, 3, 1, 2) + hidden_states = self.downsample(hidden_states).view(-1, self.config.out_hidden_size) + + hidden_states = self.merger(hidden_states) + return hidden_states + + +@auto_docstring +class Glm46VTextModel(Glm46VPreTrainedModel): + config: Glm46VTextConfig + input_modalities = "text" + + def __init__(self, config: Glm46VTextConfig): + super().__init__(config) + self.padding_idx = config.pad_token_id + self.vocab_size = config.vocab_size + + self.embed_tokens = nn.Embedding(config.vocab_size, config.hidden_size, self.padding_idx) + self.layers = nn.ModuleList( + [Glm46VTextDecoderLayer(config, layer_idx) for layer_idx in range(config.num_hidden_layers)] + ) + self.norm = Glm46VRMSNorm(config.hidden_size, eps=config.rms_norm_eps) + self.rotary_emb = Glm46VTextRotaryEmbedding(config=config) + + self.gradient_checkpointing = False + # Initialize weights and apply final processing + self.post_init() + + @auto_docstring + @check_model_inputs() + def forward( + self, + input_ids: Optional[torch.LongTensor] = None, + attention_mask: Optional[torch.Tensor] = None, + position_ids: Optional[torch.LongTensor] = None, + past_key_values: Optional[Cache] = None, + inputs_embeds: Optional[torch.FloatTensor] = None, + use_cache: Optional[bool] = None, + cache_position: Optional[torch.LongTensor] = None, + **kwargs: Unpack[FlashAttentionKwargs], + ) -> Union[tuple, BaseModelOutputWithPast]: + if (input_ids is None) ^ (inputs_embeds is not None): + raise ValueError("You must specify exactly one of input_ids or inputs_embeds") + + # torch.jit.trace() doesn't support cache objects in the output + if use_cache and past_key_values is None and not torch.jit.is_tracing(): + past_key_values = DynamicCache(config=self.config) + + if inputs_embeds is None: + inputs_embeds = self.embed_tokens(input_ids) + + if cache_position is None: + past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 + cache_position = torch.arange( + past_seen_tokens, past_seen_tokens + inputs_embeds.shape[1], device=inputs_embeds.device + ) + + # the hard coded `3` is for temporal, height and width. + if position_ids is None: + position_ids = cache_position.view(1, 1, -1).expand(3, inputs_embeds.shape[0], -1) + elif position_ids.ndim == 2: + position_ids = position_ids[None, ...].expand(3, position_ids.shape[0], -1) + + # NOTE: we need to pass text position ids for packing. Qwen2-VL uses 3D positions + # where each dim indicates visual spatial positions for temporal/height/width grids. + # There are two scenarios when FA2-like packed masking might be activated. + # 1. User specifically passed packed `position_ids` and no attention mask. + # In this case we expect the useer to create correct position ids for all 3 grids + # and prepend text-only position ids to it. The final tensor will be [4, bs, seq-len] + # 2. User runs forward with no attention mask and no position ids. In this case, position ids + # are prepared by the model (`get_rope_index`) as `[4, bs, seq-len]` tensor. Text-only positions are + # prepended by us when creating positions so that the mask is constructed correctly. NOTE: failing to pass + # text-only positions will cause incorrect mask construction, do not change `prepare_input_for_generation` + if position_ids.ndim == 3 and position_ids.shape[0] == 4: + text_position_ids = position_ids[0] + position_ids = position_ids[1:] + else: + # If inputs are not packed (usual 3D positions), do not prepare mask from position_ids + text_position_ids = None + + mask_kwargs = { + "config": self.config, + "input_embeds": inputs_embeds, + "attention_mask": attention_mask, + "cache_position": cache_position, + "past_key_values": past_key_values, + "position_ids": text_position_ids, + } + # Create the masks + causal_mask = create_causal_mask(**mask_kwargs) + + hidden_states = inputs_embeds + position_embeddings = self.rotary_emb(hidden_states, position_ids=position_ids) + + for decoder_layer in self.layers: + layer_outputs = decoder_layer( + hidden_states, + attention_mask=causal_mask, + position_ids=position_ids, + past_key_values=past_key_values, + cache_position=cache_position, + position_embeddings=position_embeddings, + **kwargs, + ) + hidden_states = layer_outputs + + hidden_states = self.norm(hidden_states) + + return BaseModelOutputWithPast( + last_hidden_state=hidden_states, + past_key_values=past_key_values, + ) + + +@auto_docstring +class Glm46VModel(Glm46VPreTrainedModel): + base_model_prefix = "" + _checkpoint_conversion_mapping = {} + # Reference: fix gemma3 grad acc #37208 + accepts_loss_kwargs = False + config: Glm46VConfig + _no_split_modules = ["Glm46VTextDecoderLayer", "Glm46VVisionBlock"] + + def __init__(self, config): + super().__init__(config) + self.visual = Glm46VVisionModel._from_config(config.vision_config) + self.language_model = Glm46VTextModel._from_config(config.text_config) + self.rope_deltas = None # cache rope_deltas here + + # Initialize weights and apply final processing + self.post_init() + + def get_input_embeddings(self): + return self.language_model.get_input_embeddings() + + def set_input_embeddings(self, value): + self.language_model.set_input_embeddings(value) + + def set_decoder(self, decoder): + self.language_model = decoder + + def get_decoder(self): + return self.language_model + + def get_rope_index( + self, + input_ids: Optional[torch.LongTensor] = None, + image_grid_thw: Optional[torch.LongTensor] = None, + video_grid_thw: Optional[torch.LongTensor] = None, + attention_mask: Optional[torch.Tensor] = None, + ) -> tuple[torch.Tensor, torch.Tensor]: + """ + Calculate the 3D rope index based on image and video's temporal, height and width in LLM. + + Explanation: + Each embedding sequence contains vision embedding and text embedding or just contains text embedding. + + For pure text embedding sequence, the rotary position embedding has no difference with modern LLMs. + Examples: + input_ids: [T T T T T], here T is for text. + temporal position_ids: [0, 1, 2, 3, 4] + height position_ids: [0, 1, 2, 3, 4] + width position_ids: [0, 1, 2, 3, 4] + + For vision and text embedding sequence, we calculate 3D rotary position embedding for vision part + and 1D rotary position embedding for text part. + Examples: + Temporal (Time): 3 patches, representing different segments of the video in time. + Height: 2 patches, dividing each frame vertically. + Width: 2 patches, dividing each frame horizontally. + We also have some important parameters: + fps (Frames Per Second): The video's frame rate, set to 1. This means one frame is processed each second. + tokens_per_second: This is a crucial parameter. It dictates how many "time-steps" or "temporal tokens" are conceptually packed into a one-second interval of the video. In this case, we have 25 tokens per second. So each second of the video will be represented with 25 separate time points. It essentially defines the temporal granularity. + temporal_patch_size: The number of frames that compose one temporal patch. Here, it's 2 frames. + interval: The step size for the temporal position IDs, calculated as tokens_per_second * temporal_patch_size / fps. In this case, 25 * 2 / 1 = 50. This means that each temporal patch will be have a difference of 50 in the temporal position IDs. + input_ids: [V V V V V V V V V V V V T T T T T], here V is for vision. + vision temporal position_ids: [0, 0, 0, 0, 50, 50, 50, 50, 100, 100, 100, 100] + vision height position_ids: [0, 0, 1, 1, 0, 0, 1, 1, 0, 0, 1, 1] + vision width position_ids: [0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1] + text temporal position_ids: [101, 102, 103, 104, 105] + text height position_ids: [101, 102, 103, 104, 105] + text width position_ids: [101, 102, 103, 104, 105] + Here we calculate the text start position_ids as the max vision position_ids plus 1. + + Args: + input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): + Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide + it. + image_grid_thw (`torch.LongTensor` of shape `(num_images, 3)`, *optional*): + The temporal, height and width of feature shape of each image in LLM. + video_grid_thw (`torch.LongTensor` of shape `(num_videos, 3)`, *optional*): + The temporal, height and width of feature shape of each video in LLM. + attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): + Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: + + - 1 for tokens that are **not masked**, + - 0 for tokens that are **masked**. + + Returns: + position_ids (`torch.LongTensor` of shape `(3, batch_size, sequence_length)`) + mrope_position_deltas (`torch.Tensor` of shape `(batch_size)`) + """ + + spatial_merge_size = self.config.vision_config.spatial_merge_size + image_token_id = self.config.image_token_id + video_start_token_id = self.config.video_start_token_id + video_end_token_id = self.config.video_end_token_id + + mrope_position_deltas = [] + if input_ids is not None and (image_grid_thw is not None or video_grid_thw is not None): + total_input_ids = input_ids + if attention_mask is None: + attention_mask = torch.ones_like(total_input_ids) + position_ids = torch.ones( + 3, + input_ids.shape[0], + input_ids.shape[1], + dtype=input_ids.dtype, + device=input_ids.device, + ) + image_index, video_index = 0, 0 + video_group_index = 0 + attention_mask = attention_mask.to(total_input_ids.device) + for i, input_ids in enumerate(total_input_ids): + input_ids = input_ids[attention_mask[i] == 1] + input_tokens = input_ids.tolist() + + input_token_type = [] + video_check_flg = False + for token in input_tokens: + if token == video_start_token_id: + video_check_flg = True + elif token == video_end_token_id: + video_check_flg = False + + if token == image_token_id and not video_check_flg: + input_token_type.append("image") + elif token == image_token_id and video_check_flg: + input_token_type.append("video") + else: + input_token_type.append("text") + + input_type_group = [] + for key, group in itertools.groupby(enumerate(input_token_type), lambda x: x[1]): + group = list(group) + start_index = group[0][0] + end_index = group[-1][0] + 1 + input_type_group.append((key, start_index, end_index)) + + llm_pos_ids_list = [] + video_frame_num = 1 + for modality_type, start_idx, end_idx in input_type_group: + st_idx = llm_pos_ids_list[-1].max() + 1 if len(llm_pos_ids_list) > 0 else 0 + + if modality_type == "image": + t, h, w = ( + image_grid_thw[image_index][0], + image_grid_thw[image_index][1], + image_grid_thw[image_index][2], + ) + llm_grid_t, llm_grid_h, llm_grid_w = ( + t.item(), + h.item() // spatial_merge_size, + w.item() // spatial_merge_size, + ) + + t_index = torch.arange(llm_grid_t).view(-1, 1).expand(-1, llm_grid_h * llm_grid_w).flatten() + h_index = torch.arange(llm_grid_h).view(1, -1, 1).expand(llm_grid_t, -1, llm_grid_w).flatten() + w_index = torch.arange(llm_grid_w).view(1, 1, -1).expand(llm_grid_t, llm_grid_h, -1).flatten() + llm_pos_ids_list.append(torch.stack([t_index, h_index, w_index]) + st_idx) + + image_index += 1 + video_frame_num = 1 + + elif modality_type == "video": + t, h, w = ( + video_frame_num, + video_grid_thw[video_index][1], + video_grid_thw[video_index][2], + ) + + llm_grid_t, llm_grid_h, llm_grid_w = ( + t, + h.item() // spatial_merge_size, + w.item() // spatial_merge_size, + ) + + for t_idx in range(llm_grid_t): + t_index = torch.tensor(t_idx).view(-1, 1).expand(-1, llm_grid_h * llm_grid_w).flatten() + + h_index = torch.arange(llm_grid_h).view(1, -1, 1).expand(1, -1, llm_grid_w).flatten() + w_index = torch.arange(llm_grid_w).view(1, 1, -1).expand(1, llm_grid_h, -1).flatten() + llm_pos_ids_list.append(torch.stack([t_index, h_index, w_index]) + st_idx) + + video_group_index += 1 + + if video_group_index >= video_grid_thw[video_index][0]: + video_index += 1 + video_group_index = 0 + + video_frame_num += 1 + + else: + text_len = end_idx - start_idx + llm_pos_ids_list.append(torch.arange(text_len).view(1, -1).expand(3, -1) + st_idx) + + video_frame_num = 1 + + llm_positions = torch.cat(llm_pos_ids_list, dim=1).reshape(3, -1) + position_ids[..., i, attention_mask[i] == 1] = llm_positions.to(position_ids.device) + mrope_position_deltas.append(llm_positions.max() + 1 - len(total_input_ids[i])) + mrope_position_deltas = torch.tensor(mrope_position_deltas, device=input_ids.device).unsqueeze(1) + return position_ids, mrope_position_deltas + else: + if attention_mask is not None: + position_ids = attention_mask.long().cumsum(-1) - 1 + position_ids.masked_fill_(attention_mask == 0, 1) + position_ids = position_ids.unsqueeze(0).expand(3, -1, -1).to(attention_mask.device) + max_position_ids = position_ids.max(0, keepdim=False)[0].max(-1, keepdim=True)[0] + mrope_position_deltas = max_position_ids + 1 - attention_mask.shape[-1] + else: + position_ids = ( + torch.arange(input_ids.shape[1], device=input_ids.device) + .view(1, 1, -1) + .expand(3, input_ids.shape[0], -1) + ) + mrope_position_deltas = torch.zeros( + [input_ids.shape[0], 1], + device=input_ids.device, + dtype=input_ids.dtype, + ) + + return position_ids, mrope_position_deltas + + def get_video_features( + self, pixel_values_videos: torch.FloatTensor, video_grid_thw: Optional[torch.LongTensor] = None + ): + """ + Encodes videos into continuous embeddings that can be forwarded to the language model. + + Args: + pixel_values_videos (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`): + The tensors corresponding to the input videos. + video_grid_thw (`torch.LongTensor` of shape `(num_videos, 3)`, *optional*): + The temporal, height and width of feature shape of each video in LLM. + """ + pixel_values_videos = pixel_values_videos.type(self.visual.dtype) + # reshape video_grid_thw -> [b, 3] -> [1, h, w] * frames + temp_frames_hw = [] + for t, h, w in video_grid_thw: + repeated_row = torch.tensor([1, h.item(), w.item()]).unsqueeze(0).repeat(t, 1) + temp_frames_hw.append(repeated_row) + flattened_video_grid_thw = torch.cat(temp_frames_hw, dim=0) + video_embeds = self.visual(pixel_values_videos, grid_thw=flattened_video_grid_thw) + split_sizes = (video_grid_thw.prod(-1) // self.visual.spatial_merge_size**2).tolist() + video_embeds = torch.split(video_embeds, split_sizes) + return video_embeds + + def get_image_features(self, pixel_values: torch.FloatTensor, image_grid_thw: Optional[torch.LongTensor] = None): + """ + Encodes images into continuous embeddings that can be forwarded to the language model. + + Args: + pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`): + The tensors corresponding to the input images. + image_grid_thw (`torch.LongTensor` of shape `(num_images, 3)`, *optional*): + The temporal, height and width of feature shape of each image in LLM. + """ + pixel_values = pixel_values.type(self.visual.dtype) + image_embeds = self.visual(pixel_values, grid_thw=image_grid_thw) + split_sizes = (image_grid_thw.prod(-1) // self.visual.spatial_merge_size**2).tolist() + image_embeds = torch.split(image_embeds, split_sizes) + return image_embeds + + def get_placeholder_mask( + self, + input_ids: torch.LongTensor, + inputs_embeds: torch.FloatTensor, + image_features: Optional[torch.FloatTensor] = None, + video_features: Optional[torch.FloatTensor] = None, + ): + """ + Obtains multimodal placeholder mask from `input_ids` or `inputs_embeds`, and checks that the placeholder token count is + equal to the length of multimodal features. If the lengths are different, an error is raised. + """ + if input_ids is None: + special_image_mask = inputs_embeds == self.get_input_embeddings()( + torch.tensor(self.config.image_token_id, dtype=torch.long, device=inputs_embeds.device) + ) + special_image_mask = special_image_mask.all(-1) + special_video_mask = inputs_embeds == self.get_input_embeddings()( + torch.tensor(self.config.video_token_id, dtype=torch.long, device=inputs_embeds.device) + ) + special_video_mask = special_video_mask.all(-1) + else: + # GLM-4.1V and GLM-4.5V special_video_mask is special_image_mask + special_image_mask = input_ids == self.config.image_token_id + special_video_mask = input_ids == self.config.image_token_id + + n_image_tokens = special_image_mask.sum() + special_image_mask = special_image_mask.unsqueeze(-1).expand_as(inputs_embeds).to(inputs_embeds.device) + if image_features is not None and inputs_embeds[special_image_mask].numel() != image_features.numel(): + raise ValueError( + f"Image features and image tokens do not match: tokens: {n_image_tokens}, features {image_features.shape[0]}" + ) + + n_video_tokens = special_video_mask.sum() + special_video_mask = special_video_mask.unsqueeze(-1).expand_as(inputs_embeds).to(inputs_embeds.device) + if video_features is not None and inputs_embeds[special_video_mask].numel() != video_features.numel(): + raise ValueError( + f"Videos features and video tokens do not match: tokens: {n_video_tokens}, features {video_features.shape[0]}" + ) + + return special_image_mask, special_video_mask + + @auto_docstring + @can_return_tuple + def forward( + self, + input_ids: Optional[torch.LongTensor] = None, + attention_mask: Optional[torch.Tensor] = None, + position_ids: Optional[torch.LongTensor] = None, + past_key_values: Optional[Cache] = None, + inputs_embeds: Optional[torch.FloatTensor] = None, + pixel_values: Optional[torch.Tensor] = None, + pixel_values_videos: Optional[torch.FloatTensor] = None, + image_grid_thw: Optional[torch.LongTensor] = None, + video_grid_thw: Optional[torch.LongTensor] = None, + rope_deltas: Optional[torch.LongTensor] = None, + cache_position: Optional[torch.LongTensor] = None, + **kwargs: Unpack[TransformersKwargs], + ) -> Union[tuple, Glm46VModelOutputWithPast]: + r""" + image_grid_thw (`torch.LongTensor` of shape `(num_images, 3)`, *optional*): + The temporal, height and width of feature shape of each image in LLM. + video_grid_thw (`torch.LongTensor` of shape `(num_videos, 3)`, *optional*): + The temporal, height and width of feature shape of each video in LLM. + rope_deltas (`torch.LongTensor` of shape `(batch_size, )`, *optional*): + The rope index difference between sequence length and multimodal rope. + """ + if (input_ids is None) ^ (inputs_embeds is not None): + raise ValueError("You must specify exactly one of input_ids or inputs_embeds") + + if inputs_embeds is None: + inputs_embeds = self.get_input_embeddings()(input_ids) + + if pixel_values is not None: + image_embeds = self.get_image_features(pixel_values, image_grid_thw) + image_embeds = torch.cat(image_embeds, dim=0).to(inputs_embeds.device, inputs_embeds.dtype) + image_mask, _ = self.get_placeholder_mask(input_ids, inputs_embeds, image_features=image_embeds) + inputs_embeds = inputs_embeds.masked_scatter(image_mask, image_embeds) + + if pixel_values_videos is not None: + video_embeds = self.get_video_features(pixel_values_videos, video_grid_thw) + video_embeds = torch.cat(video_embeds, dim=0).to(inputs_embeds.device, inputs_embeds.dtype) + _, video_mask = self.get_placeholder_mask(input_ids, inputs_embeds, video_features=video_embeds) + inputs_embeds = inputs_embeds.masked_scatter(video_mask, video_embeds) + + if position_ids is None: + attention_mask_tensor = ( + attention_mask if not isinstance(attention_mask, dict) else attention_mask["full_attention"] + ) + if attention_mask_tensor is not None and attention_mask_tensor.ndim == 4: + attention_mask_tensor = torch.diagonal(attention_mask_tensor[:, 0], dim1=1, dim2=2) + # Only apply conversion for floating point tensors (inverted masks) + if attention_mask_tensor.dtype.is_floating_point: + attention_mask_tensor = attention_mask_tensor / torch.finfo(attention_mask_tensor.dtype).min + attention_mask_tensor = (1.0 - attention_mask_tensor).int() + + # Calculate RoPE index once per generation in the pre-fill stage only. + # When compiling, we can't check tensor values thus we check only input length + # It is safe to assume that `length!=1` means we're in pre-fill because compiled + # models currently cannot do asssisted decoding + prefill_compiled_stage = is_torchdynamo_compiling() and ( + (input_ids is not None and input_ids.shape[1] != 1) + or (inputs_embeds is not None and inputs_embeds.shape[1] != 1) + ) + prefill_noncompiled_stage = not is_torchdynamo_compiling() and ( + (cache_position is not None and cache_position[0] == 0) + or (past_key_values is None or past_key_values.get_seq_length() == 0) + ) + if (prefill_compiled_stage or prefill_noncompiled_stage) or self.rope_deltas is None: + position_ids, rope_deltas = self.get_rope_index( + input_ids, + image_grid_thw, + video_grid_thw, + attention_mask=attention_mask_tensor, + ) + self.rope_deltas = rope_deltas + # then use the prev pre-calculated rope-deltas to get the correct position ids + else: + batch_size, seq_length, _ = inputs_embeds.shape + delta = ( + (cache_position[0] + self.rope_deltas).to(inputs_embeds.device) + if cache_position is not None + else 0 + ) + position_ids = torch.arange(seq_length, device=inputs_embeds.device) + position_ids = position_ids.view(1, -1).expand(batch_size, -1) + if cache_position is not None: # otherwise `deltas` is an int `0` + delta = delta.repeat_interleave(batch_size // delta.shape[0], dim=0) + position_ids = position_ids.add(delta) + position_ids = position_ids.unsqueeze(0).expand(3, -1, -1) + + outputs = self.language_model( + input_ids=None, + position_ids=position_ids, + attention_mask=attention_mask, + past_key_values=past_key_values, + inputs_embeds=inputs_embeds, + cache_position=cache_position, + **kwargs, + ) + + return Glm46VModelOutputWithPast( + last_hidden_state=outputs.last_hidden_state, + past_key_values=outputs.past_key_values, + hidden_states=outputs.hidden_states, + attentions=outputs.attentions, + rope_deltas=self.rope_deltas, + ) + + +@dataclass +@auto_docstring( + custom_intro=""" + Base class for Glm46V causal language model (or autoregressive) outputs. + """ +) +class Glm46VCausalLMOutputWithPast(ModelOutput): + r""" + loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `labels` is provided): + Language modeling loss (for next-token prediction). + logits (`torch.FloatTensor` of shape `(batch_size, sequence_length, config.vocab_size)`): + Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax). + past_key_values (`Cache`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`): + It is a [`~cache_utils.Cache`] instance. For more details, see our [kv cache guide](https://huggingface.co/docs/transformers/en/kv_cache). + + Contains pre-computed hidden-states (key and values in the self-attention blocks) that can be used (see + `past_key_values` input) to speed up sequential decoding. + rope_deltas (`torch.LongTensor` of shape `(batch_size, )`, *optional*): + The rope index difference between sequence length and multimodal rope. + """ + + loss: Optional[torch.FloatTensor] = None + logits: Optional[torch.FloatTensor] = None + past_key_values: Optional[Cache] = None + hidden_states: Optional[tuple[torch.FloatTensor]] = None + attentions: Optional[tuple[torch.FloatTensor]] = None + rope_deltas: Optional[torch.LongTensor] = None + + +class Glm46VForConditionalGeneration(Glm46VPreTrainedModel, GenerationMixin): + _checkpoint_conversion_mapping = {} + _tied_weights_keys = ["lm_head.weight"] + # Reference: fix gemma3 grad acc #37208 + accepts_loss_kwargs = False + + def __init__(self, config): + super().__init__(config) + self.model = Glm46VModel(config) + self.lm_head = nn.Linear(config.text_config.hidden_size, config.text_config.vocab_size, bias=False) + + self.post_init() + + def get_input_embeddings(self): + return self.model.get_input_embeddings() + + def set_input_embeddings(self, value): + self.model.set_input_embeddings(value) + + def set_decoder(self, decoder): + self.model.set_decoder(decoder) + + def get_decoder(self): + return self.model.get_decoder() + + def get_video_features( + self, pixel_values_videos: torch.FloatTensor, video_grid_thw: Optional[torch.LongTensor] = None + ): + return self.model.get_video_features(pixel_values_videos, video_grid_thw) + + def get_image_features(self, pixel_values: torch.FloatTensor, image_grid_thw: Optional[torch.LongTensor] = None): + return self.model.get_image_features(pixel_values, image_grid_thw) + + # Make modules available through conditional class for BC + @property + def language_model(self): + return self.model.language_model + + @property + def visual(self): + return self.model.visual + + @can_return_tuple + @auto_docstring + def forward( + self, + input_ids: Optional[torch.LongTensor] = None, + attention_mask: Optional[torch.Tensor] = None, + position_ids: Optional[torch.LongTensor] = None, + past_key_values: Optional[Cache] = None, + inputs_embeds: Optional[torch.FloatTensor] = None, + labels: Optional[torch.LongTensor] = None, + pixel_values: Optional[torch.Tensor] = None, + pixel_values_videos: Optional[torch.FloatTensor] = None, + image_grid_thw: Optional[torch.LongTensor] = None, + video_grid_thw: Optional[torch.LongTensor] = None, + rope_deltas: Optional[torch.LongTensor] = None, + cache_position: Optional[torch.LongTensor] = None, + logits_to_keep: Union[int, torch.Tensor] = 0, + **kwargs: Unpack[TransformersKwargs], + ) -> Union[tuple, Glm46VCausalLMOutputWithPast]: + r""" + rope_deltas (`torch.LongTensor` of shape `(batch_size, )`, *optional*): + The rope index difference between sequence length and multimodal rope. + labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., + config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored + (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. + image_grid_thw (`torch.LongTensor` of shape `(num_images, 3)`, *optional*): + The temporal, height and width of feature shape of each image in LLM. + video_grid_thw (`torch.LongTensor` of shape `(num_videos, 3)`, *optional*): + The temporal, height and width of feature shape of each video in LLM. + + Example: + + ```python + >>> from PIL import Image + >>> import requests + >>> from transformers import AutoProcessor, Glm46VForConditionalGeneration + + >>> model = Glm46VForConditionalGeneration.from_pretrained("THUDM/GLM-4.1V-9B-Thinking") + >>> processor = AutoProcessor.from_pretrained("THUDM/GLM-4.1V-9B-Thinking") + + >>> messages = [ + { + "role": "user", + "content": [ + {"type": "image"}, + {"type": "text", "text": "What is shown in this image?"}, + ], + }, + ] + >>> url = "https://www.ilankelman.org/stopsigns/australia.jpg" + >>> image = Image.open(requests.get(url, stream=True).raw) + + >>> text = processor.apply_chat_template(messages, tokenize=False, add_generation_prompt=True) + >>> inputs = processor(text=[text], images=[image], vision_infos=[vision_infos]) + + >>> # Generate + >>> generate_ids = model.generate(inputs.input_ids, max_length=30) + >>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0] + "The image shows a street scene with a red stop sign in the foreground. In the background, there is a large red gate with Chinese characters ..." + ```""" + outputs = self.model( + input_ids=input_ids, + pixel_values=pixel_values, + pixel_values_videos=pixel_values_videos, + image_grid_thw=image_grid_thw, + video_grid_thw=video_grid_thw, + position_ids=position_ids, + attention_mask=attention_mask, + past_key_values=past_key_values, + inputs_embeds=inputs_embeds, + cache_position=cache_position, + **kwargs, + ) + + hidden_states = outputs[0] + + # Only compute necessary logits, and do not upcast them to float if we are not computing the loss + slice_indices = slice(-logits_to_keep, None) if isinstance(logits_to_keep, int) else logits_to_keep + logits = self.lm_head(hidden_states[:, slice_indices, :]) + + loss = None + if labels is not None: + loss = self.loss_function(logits=logits, labels=labels, vocab_size=self.config.text_config.vocab_size) + + return Glm46VCausalLMOutputWithPast( + loss=loss, + logits=logits, + past_key_values=outputs.past_key_values, + hidden_states=outputs.hidden_states, + attentions=outputs.attentions, + rope_deltas=outputs.rope_deltas, + ) + + def prepare_inputs_for_generation( + self, + input_ids, + past_key_values=None, + attention_mask=None, + inputs_embeds=None, + cache_position=None, + position_ids=None, + use_cache=True, + pixel_values=None, + pixel_values_videos=None, + image_grid_thw=None, + video_grid_thw=None, + **kwargs, + ): + # Overwritten -- in specific circumstances we don't want to forward image inputs to the model + + model_inputs = super().prepare_inputs_for_generation( + input_ids, + past_key_values=past_key_values, + attention_mask=attention_mask, + inputs_embeds=inputs_embeds, + cache_position=cache_position, + position_ids=position_ids, + pixel_values=pixel_values, + pixel_values_videos=pixel_values_videos, + image_grid_thw=image_grid_thw, + video_grid_thw=video_grid_thw, + use_cache=use_cache, + **kwargs, + ) + + # GLM-4.1V position_ids are prepareed with rope_deltas in forward + model_inputs["position_ids"] = None + + if cache_position[0] != 0: + model_inputs["pixel_values"] = None + model_inputs["pixel_values_videos"] = None + + return model_inputs + + def _get_image_nums_and_video_nums( + self, + input_ids: Optional[torch.LongTensor], + inputs_embeds: Optional[torch.Tensor] = None, + ) -> tuple[torch.Tensor, torch.Tensor]: + """ + Get the number of images and videos for each sample to calculate the separation length of the sample tensor. + These parameters are not passed through the processor to avoid unpredictable impacts from interface modifications. + + Args: + input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): + Indices of input sequence tokens in the vocabulary. + + Returns: + image_nums (`torch.LongTensor` of shape `(batch_size, num_images_sample)`) + video_nums (`torch.LongTensor` of shape `(batch_size, num_videos_sample)`) + """ + + if inputs_embeds is not None: + is_image = ( + inputs_embeds + == self.get_input_embeddings()( + torch.tensor(self.config.image_start_token_id, dtype=torch.long, device=inputs_embeds.device) + ) + )[..., 0] + is_video_start = ( + inputs_embeds + == self.get_input_embeddings()( + torch.tensor(self.config.video_start_token_id, dtype=torch.long, device=inputs_embeds.device) + ) + )[..., 0] + is_video_end = ( + inputs_embeds + == self.get_input_embeddings()( + torch.tensor(self.config.video_end_token_id, dtype=torch.long, device=inputs_embeds.device) + ) + )[..., 0] + else: + is_image = input_ids == self.config.image_start_token_id + is_video_start = input_ids == self.config.video_start_token_id + is_video_end = input_ids == self.config.video_end_token_id + + # Cumulative sum to track if we're inside a video span + # We'll assume well-formed video tags (i.e. matching starts and ends) + video_level = torch.cumsum(is_video_start.int() - is_video_end.int(), dim=1) + inside_video = video_level > 0 # shape (batch_size, seq_length) + + # Mask out image tokens that are inside video spans + standalone_images = is_image & (~inside_video) + + # Count per batch + image_counts = standalone_images.sum(dim=1) + video_counts = is_video_start.sum(dim=1) + + return image_counts, video_counts + + def _expand_inputs_for_generation( + self, + expand_size: int = 1, + is_encoder_decoder: bool = False, + input_ids: Optional[torch.LongTensor] = None, + **model_kwargs, + ) -> tuple[torch.LongTensor, dict[str, Any]]: + # Overwritten -- Support for expanding tensors without a batch size dimension + # e.g., pixel_values, image_grid_thw, pixel_values_videos, video_grid_thw, second_per_grid_t + # pixel_values.shape[0] is sum(seqlen_images for samples) + # image_grid_thw.shape[0] is sum(num_images for samples) + + if expand_size == 1: + return input_ids, model_kwargs + + visual_keys = ["pixel_values", "image_grid_thw", "pixel_values_videos", "video_grid_thw", "second_per_grid_ts"] + + def _expand_dict_for_generation_visual(dict_to_expand): + image_grid_thw = model_kwargs.get("image_grid_thw", None) + video_grid_thw = model_kwargs.get("video_grid_thw", None) + image_nums, video_nums = self._get_image_nums_and_video_nums( + input_ids, inputs_embeds=model_kwargs.get("inputs_embeds", None) + ) + + def _repeat_interleave_samples(x, lengths, repeat_times): + samples = torch.split(x, lengths) + repeat_args = [repeat_times] + [1] * (x.dim() - 1) + result = torch.cat([sample.repeat(*repeat_args) for sample in samples], dim=0) + return result + + for key in dict_to_expand: + if key == "pixel_values": + # split images into samples + samples = torch.split(image_grid_thw, list(image_nums)) + # compute the sequence length of images for each sample + lengths = [torch.prod(sample, dim=1).sum() for sample in samples] + dict_to_expand[key] = _repeat_interleave_samples( + dict_to_expand[key], lengths=lengths, repeat_times=expand_size + ) + elif key == "image_grid_thw": + # get the num of images for each sample + lengths = list(image_nums) + dict_to_expand[key] = _repeat_interleave_samples( + dict_to_expand[key], lengths=lengths, repeat_times=expand_size + ) + elif key == "pixel_values_videos": + samples = torch.split(video_grid_thw, list(video_nums)) + lengths = [torch.prod(sample, dim=1).sum() for sample in samples] + dict_to_expand[key] = _repeat_interleave_samples( + dict_to_expand[key], lengths=lengths, repeat_times=expand_size + ) + elif key == "video_grid_thw": + lengths = list(video_nums) + dict_to_expand[key] = _repeat_interleave_samples( + dict_to_expand[key], lengths=lengths, repeat_times=expand_size + ) + elif key == "second_per_grid_ts": + dict_to_expand[key] = _repeat_interleave_samples( + dict_to_expand[key], lengths=list(video_nums), repeat_times=expand_size + ) + return dict_to_expand + + def _expand_dict_for_generation(dict_to_expand): + for key in dict_to_expand: + if ( + key != "cache_position" + and dict_to_expand[key] is not None + and isinstance(dict_to_expand[key], torch.Tensor) + and key not in visual_keys + ): + dict_to_expand[key] = dict_to_expand[key].repeat_interleave(expand_size, dim=0) + return dict_to_expand + + model_kwargs = _expand_dict_for_generation_visual(model_kwargs) + + if input_ids is not None: + input_ids = input_ids.repeat_interleave(expand_size, dim=0) + + model_kwargs = _expand_dict_for_generation(model_kwargs) + + if is_encoder_decoder: + if model_kwargs.get("encoder_outputs") is None: + raise ValueError("If `is_encoder_decoder` is True, make sure that `encoder_outputs` is defined.") + model_kwargs["encoder_outputs"] = _expand_dict_for_generation(model_kwargs["encoder_outputs"]) + + return input_ids, model_kwargs + + +__all__ = ["Glm46VForConditionalGeneration", "Glm46VModel", "Glm46VPreTrainedModel", "Glm46VTextModel"] diff --git a/src/transformers/models/glm46v/modular_glm46v.py b/src/transformers/models/glm46v/modular_glm46v.py new file mode 100644 index 000000000000..6d20e69c3991 --- /dev/null +++ b/src/transformers/models/glm46v/modular_glm46v.py @@ -0,0 +1,141 @@ +# coding=utf-8 +# Copyright 2025 the HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from ..glm4v.configuration_glm4v import Glm4vConfig, Glm4vTextConfig, Glm4vVisionConfig +from ..glm4v.modeling_glm4v import ( + Glm4vCausalLMOutputWithPast, + Glm4vForConditionalGeneration, + Glm4VisionMlp, + Glm4vModel, + Glm4vModelOutputWithPast, + Glm4vPreTrainedModel, + Glm4vRMSNorm, + Glm4vTextAttention, + Glm4vTextDecoderLayer, + Glm4vTextMLP, + Glm4vTextModel, + Glm4vTextRotaryEmbedding, + Glm4vVisionAttention, + Glm4vVisionBlock, + Glm4vVisionEmbeddings, + Glm4vVisionModel, + Glm4vVisionPatchEmbed, + Glm4vVisionPatchMerger, + Glm4vVisionRotaryEmbedding, +) +from ..glm4v.processing_glm4v import Glm4vProcessor + + +class Glm46VVisionConfig(Glm4vVisionConfig): + pass + + +class Glm46VTextConfig(Glm4vTextConfig): + pass + + +class Glm46VConfig(Glm4vConfig): + pass + + +class Glm46VRMSNorm(Glm4vRMSNorm): + pass + + +class Glm4VisionMlp(Glm4VisionMlp): + pass + + +class Glm46VVisionPatchEmbed(Glm4vVisionPatchEmbed): + pass + + +class Glm46VVisionRotaryEmbedding(Glm4vVisionRotaryEmbedding): + pass + + +class Glm46VVisionPatchMerger(Glm4vVisionPatchMerger): + pass + + +class Glm46VVisionEmbeddings(Glm4vVisionEmbeddings): + pass + + +class Glm46VVisionAttention(Glm4vVisionAttention): + pass + + +class Glm46VVisionBlock(Glm4vVisionBlock): + pass + + +class Glm46VTextRotaryEmbedding(Glm4vTextRotaryEmbedding): + pass + + +class Glm46VTextAttention(Glm4vTextAttention): + pass + + +class Glm46VTextMLP(Glm4vTextMLP): + pass + + +class Glm46VTextDecoderLayer(Glm4vTextDecoderLayer): + pass + + +class Glm46VModelOutputWithPast(Glm4vModelOutputWithPast): + pass + + +class Glm46VPreTrainedModel(Glm4vPreTrainedModel): + pass + + +class Glm46VVisionModel(Glm4vVisionModel): + pass + + +class Glm46VTextModel(Glm4vTextModel): + pass + + +class Glm46VModel(Glm4vModel): + pass + + +class Glm46VCausalLMOutputWithPast(Glm4vCausalLMOutputWithPast): + pass + + +class Glm46VForConditionalGeneration(Glm4vForConditionalGeneration): + pass + + +class Glm46vProcessor(Glm4vProcessor): + + def replace_frame_token_id(self, timestamp_sec): + return f"<|begin_of_image|>{self.image_token}<|end_of_image|>{round(timestamp_sec)} seconds" + +__all__ = [ + "Glm46VConfig", + "Glm46VTextConfig", + "Glm46VForConditionalGeneration", + "Glm46VModel", + "Glm46VPreTrainedModel", + "Glm46VTextModel", +] diff --git a/src/transformers/models/glm46v/processing_glm46v.py b/src/transformers/models/glm46v/processing_glm46v.py new file mode 100644 index 000000000000..9c996a05c52a --- /dev/null +++ b/src/transformers/models/glm46v/processing_glm46v.py @@ -0,0 +1,282 @@ +# 🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨 +# This file was automatically generated from src/transformers/models/glm46v/modular_glm46v.py. +# Do NOT edit this file manually as any edits will be overwritten by the generation of +# the file from the modular. If any change should be done, please apply the change to the +# modular_glm46v.py file directly. One of our CI enforces this. +# 🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨 +# coding=utf-8 +# Copyright 2025 the HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Optional, Union + +import numpy as np + +from ...feature_extraction_utils import BatchFeature +from ...image_utils import ImageInput +from ...processing_utils import MultiModalData, ProcessingKwargs, ProcessorMixin, Unpack +from ...tokenization_utils_base import PreTokenizedInput, TextInput +from ...utils import logging +from ...video_utils import VideoInput + + +logger = logging.get_logger(__name__) + + +class Glm46VProcessorKwargs(ProcessingKwargs, total=False): + _defaults = { + "text_kwargs": { + "padding": False, + "return_token_type_ids": False, + "return_mm_token_type_ids": False, + }, + "videos_kwargs": {"return_metadata": True}, + } + + +class Glm46vProcessor(ProcessorMixin): + r""" + Constructs a GLM-4V processor which wraps a GLM-4V image processor and a GLM-4 tokenizer into a single processor. + [`~Glm46VProcessor.__call__`] and [`~Glm46VProcessor.decode`] for more information. + Args: + image_processor ([`Glm46VProcessor`], *optional*): + The image processor is a required input. + tokenizer ([`PreTrainedTokenizerFast`], *optional*): + The tokenizer is a required input. + video_processor ([`Glm46VVideoProcessor`], *optional*): + The video processor is a required input. + chat_template (`str`, *optional*): A Jinja template which will be used to convert lists of messages + in a chat into a tokenizable string. + """ + + attributes = ["image_processor", "tokenizer", "video_processor"] + image_processor_class = "AutoImageProcessor" + video_processor_class = "AutoVideoProcessor" + + tokenizer_class = ("PreTrainedTokenizer", "PreTrainedTokenizerFast") + + def __init__(self, image_processor=None, tokenizer=None, video_processor=None, chat_template=None, **kwargs): + self.image_token = "<|image|>" if not hasattr(tokenizer, "image_token") else tokenizer.image_token + self.video_token = "<|video|>" if not hasattr(tokenizer, "video_token") else tokenizer.video_token + self.image_token_id = ( + tokenizer.image_token_id + if getattr(tokenizer, "image_token_id", None) + else tokenizer.convert_tokens_to_ids(self.image_token) + ) + self.video_token_id = ( + tokenizer.video_token_id + if getattr(tokenizer, "video_token_id", None) + else tokenizer.convert_tokens_to_ids(self.video_token) + ) + super().__init__(image_processor, tokenizer, video_processor, chat_template=chat_template) + + def __call__( + self, + images: Optional[ImageInput] = None, + text: Union[TextInput, PreTokenizedInput, list[TextInput], list[PreTokenizedInput]] = None, + videos: Optional[VideoInput] = None, + **kwargs: Unpack[Glm46VProcessorKwargs], + ) -> BatchFeature: + """ + Main method to prepare for the model one or several sequences(s) and image(s). This method forwards the `text` + and `kwargs` arguments to PreTrainedTokenizerFast's [`~PreTrainedTokenizerFast.__call__`] if `text` is not `None` to encode + the text. + + Args: + images (`PIL.Image.Image`, `np.ndarray`, `torch.Tensor`, `List[PIL.Image.Image]`, `List[np.ndarray]`, `List[torch.Tensor]`): + The image or batch of images to be prepared. Each image can be a PIL image, NumPy array or PyTorch + tensor. Both channels-first and channels-last formats are supported. + text (`str`, `List[str]`, `List[List[str]]`): + The sequence or batch of sequences to be encoded. Each sequence can be a string or a list of strings + (pretokenized string). If the sequences are provided as list of strings (pretokenized), you must set + `is_split_into_words=True` (to lift the ambiguity with a batch of sequences). + videos (`np.ndarray`, `torch.Tensor`, `List[np.ndarray]`, `List[torch.Tensor]`): + The image or batch of videos to be prepared. Each video can be a 4D NumPy array or PyTorch + tensor, or a nested list of 3D frames. Both channels-first and channels-last formats are supported. + return_tensors (`str` or [`~utils.TensorType`], *optional*): + If set, will return tensors of a particular framework. Acceptable values are: + - `'pt'`: Return PyTorch `torch.Tensor` objects. + - `'np'`: Return NumPy `np.ndarray` objects. + + Returns: + [`BatchFeature`]: A [`BatchFeature`] with the following fields: + + - **input_ids** -- List of token ids to be fed to a model. Returned when `text` is not `None`. + - **attention_mask** -- List of indices specifying which tokens should be attended to by the model (when + `return_attention_mask=True` or if *"attention_mask"* is in `self.model_input_names` and if `text` is not + `None`). + - **pixel_values** -- Pixel values to be fed to a model. Returned when `images` is not `None`. + - **pixel_values_videos** -- Pixel values of videos to be fed to a model. Returned when `videos` is not `None`. + - **image_grid_thw** -- List of image 3D grid in LLM. Returned when `images` is not `None`. + - **video_grid_thw** -- List of video 3D grid in LLM. Returned when `videos` is not `None`. + """ + output_kwargs = self._merge_kwargs( + Glm46VProcessorKwargs, + tokenizer_init_kwargs=self.tokenizer.init_kwargs, + **kwargs, + ) + if images is not None: + image_inputs = self.image_processor(images=images, **output_kwargs["images_kwargs"]) + image_grid_thw = image_inputs["image_grid_thw"] + else: + image_inputs = {} + image_grid_thw = None + + if videos is not None: + videos_inputs = self.video_processor(videos=videos, **output_kwargs["videos_kwargs"]) + # If user has not requested video metadata, pop it + if "return_metadata" not in kwargs: + video_metadata = videos_inputs.pop("video_metadata") + else: + video_metadata = videos_inputs["video_metadata"] + video_grid_thw = videos_inputs["video_grid_thw"] + else: + videos_inputs = {} + video_grid_thw = None + + if not isinstance(text, list): + text = [text] + + text = text.copy() # below lines change text in-place + if image_grid_thw is not None: + merge_length = self.image_processor.merge_size**2 + index = 0 + for i in range(len(text)): + while self.image_token in text[i]: + num_image_tokens = image_grid_thw[index].prod() // merge_length + text[i] = text[i].replace(self.image_token, "<|placeholder|>" * num_image_tokens, 1) + index += 1 + text[i] = text[i].replace("<|placeholder|>", self.image_token) + + if video_grid_thw is not None: + merge_length = self.video_processor.merge_size**2 + video_index = 0 + for i in range(len(text)): + while self.video_token in text[i]: + num_frames = video_grid_thw[video_index][0] + video_structure = "" + + metadata = video_metadata[video_index] + if metadata.fps is None: + logger.warning_once( + "SmolVLM requires frame timestamps to construct prompts, but the `fps` of the input video could not be inferred. " + "Probably `video_metadata` was missing from inputs and you passed pre-sampled frames. " + "Defaulting to `fps=24`. Please provide `video_metadata` for more accurate results." + ) + metadata.fps = 24 if metadata.fps is None else metadata.fps + timestamps = metadata.timestamps[::2] # mrope + + unique_timestamps = [] + for idx in range(0, len(timestamps)): + unique_timestamps.append(timestamps[idx]) + + selected_timestamps = unique_timestamps[:num_frames] + while len(selected_timestamps) < num_frames: + selected_timestamps.append(selected_timestamps[-1] if selected_timestamps else 0) + + for frame_idx in range(num_frames): + timestamp_sec = selected_timestamps[frame_idx] + frame_structure = self.replace_frame_token_id(timestamp_sec) + video_structure += frame_structure + + text[i] = text[i].replace(self.video_token, video_structure, 1) + num_image_tokens = ( + video_grid_thw[video_index].prod() // merge_length // video_grid_thw[video_index][0] + ) + for frame_idx in range(num_frames): + if self.image_token in text[i]: + text[i] = text[i].replace(self.image_token, "<|placeholder|>" * num_image_tokens, 1) + + video_index += 1 + + text[i] = text[i].replace("<|placeholder|>", self.image_token) + return_tensors = output_kwargs["text_kwargs"].pop("return_tensors", None) + return_mm_token_type_ids = output_kwargs["text_kwargs"].pop("return_mm_token_type_ids", False) + text_inputs = self.tokenizer(text, **output_kwargs["text_kwargs"]) + self._check_special_mm_tokens(text, text_inputs, modalities=["image", "video"]) + + if return_mm_token_type_ids: + array_ids = np.array(text_inputs["input_ids"]) + mm_token_type_ids = np.zeros_like(text_inputs["input_ids"]) + mm_token_type_ids[array_ids == self.image_token_id] = 1 + text_inputs["mm_token_type_ids"] = mm_token_type_ids.tolist() + return BatchFeature(data={**text_inputs, **image_inputs, **videos_inputs}, tensor_type=return_tensors) + + def _get_num_multimodal_tokens(self, image_sizes=None, video_sizes=None, **kwargs): + """ + Computes the number of placeholder tokens needed for multimodal inputs with the given sizes. + Args: + image_sizes (`list[list[int]]`, *optional*): + The input sizes formatted as (height, width) per each image. + video_sizes (`list[list[int]]`, *optional*): + The input sizes formatted as (num_frames, height, width) per each video. + Returns: + `MultiModalData`: A `MultiModalData` object holding number of tokens per each of the provided + input modalities, along with other useful data. + """ + + vision_data = {} + if image_sizes is not None: + images_kwargs = Glm46VProcessorKwargs._defaults.get("images_kwargs", {}) + images_kwargs.update(kwargs) + merge_size = images_kwargs.get("merge_size", None) or self.image_processor.merge_size + + num_image_patches = [ + self.image_processor.get_number_of_image_patches(*image_size, images_kwargs) + for image_size in image_sizes + ] + num_image_tokens = [(num_patches // merge_size**2) for num_patches in num_image_patches] + vision_data.update({"num_image_tokens": num_image_tokens, "num_image_patches": num_image_patches}) + + if video_sizes is not None: + videos_kwargs = Glm46VProcessorKwargs._defaults.get("videos_kwargs", {}) + videos_kwargs.update(kwargs) + num_video_patches = [ + self.video_processor.get_number_of_video_patches(*video_size, videos_kwargs) + for video_size in video_sizes + ] + num_video_tokens = [(num_patches // merge_size**2) for num_patches in num_video_patches] + vision_data["num_video_tokens"] = num_video_tokens + + return MultiModalData(**vision_data) + + def post_process_image_text_to_text( + self, generated_outputs, skip_special_tokens=True, clean_up_tokenization_spaces=False, **kwargs + ): + """ + Post-process the output of the model to decode the text. + + Args: + generated_outputs (`torch.Tensor` or `np.ndarray`): + The output of the model `generate` function. The output is expected to be a tensor of shape `(batch_size, sequence_length)` + or `(sequence_length,)`. + skip_special_tokens (`bool`, *optional*, defaults to `True`): + Whether or not to remove special tokens in the output. Argument passed to the tokenizer's `batch_decode` method. + clean_up_tokenization_spaces (`bool`, *optional*, defaults to `False`): + Whether or not to clean up the tokenization spaces. Argument passed to the tokenizer's `batch_decode` method. + **kwargs: + Additional arguments to be passed to the tokenizer's `batch_decode method`. + + Returns: + `list[str]`: The decoded text. + """ + return self.tokenizer.batch_decode( + generated_outputs, + skip_special_tokens=skip_special_tokens, + clean_up_tokenization_spaces=clean_up_tokenization_spaces, + **kwargs, + ) + + def replace_frame_token_id(self, timestamp_sec): + return f"<|begin_of_image|>{self.image_token}<|end_of_image|>{round(timestamp_sec)} seconds" diff --git a/src/transformers/models/glm46v/video_processing_glm46v.py b/src/transformers/models/glm46v/video_processing_glm46v.py new file mode 100644 index 000000000000..ab1cf0cbfad1 --- /dev/null +++ b/src/transformers/models/glm46v/video_processing_glm46v.py @@ -0,0 +1,272 @@ +# coding=utf-8 +# Copyright 2025 The ZhipuAI Inc. team and HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""video processor class for GLM-4.6V.""" + +import math +from typing import Optional, Union + +import numpy as np +import torch + +from ...image_processing_utils import BatchFeature +from ...image_utils import ( + OPENAI_CLIP_MEAN, + OPENAI_CLIP_STD, + ChannelDimension, + PILImageResampling, + SizeDict, + get_image_size, +) +from ...processing_utils import Unpack, VideosKwargs +from ...utils import TensorType, add_start_docstrings +from ...video_processing_utils import BASE_VIDEO_PROCESSOR_DOCSTRING, BaseVideoProcessor +from ...video_utils import VideoMetadata, group_videos_by_shape, reorder_videos +from .image_processing_glm4v import smart_resize + + +class Glm4vVideoProcessorInitKwargs(VideosKwargs, total=False): + max_image_size: dict[str, int] + patch_size: int + temporal_patch_size: int + merge_size: int + max_duration: int + + +@add_start_docstrings( + "Constructs a fast GLM-4V image processor that dynamically resizes videos based on the original videos.", + BASE_VIDEO_PROCESSOR_DOCSTRING, + """ + patch_size (`int`, *optional*, defaults to 14): + The spacial patch size of the vision encoder. + temporal_patch_size (`int`, *optional*, defaults to 2): + The temporal patch size of the vision encoder. + merge_size (`int`, *optional*, defaults to 2): + The merge size of the vision encoder to llm encoder. + """, +) +class Glm46vVideoProcessor(BaseVideoProcessor): + resample = PILImageResampling.BICUBIC + size = {"shortest_edge": 112 * 112, "longest_edge": 28 * 28 * 2 * 30000} + max_image_size = {"longest_edge": 28 * 28 * 2 * 30000} + image_mean = OPENAI_CLIP_MEAN + image_std = OPENAI_CLIP_STD + do_resize = True + do_rescale = True + do_normalize = True + do_convert_rgb = True + do_sample_frames = True + patch_size = 14 + temporal_patch_size = 2 + max_duration = 300 + merge_size = 2 + valid_kwargs = Glm46vVideoProcessorInitKwargs + num_frames = 16 + fps = 2 + + model_input_names = ["pixel_values_videos", "video_grid_thw"] + + def __init__(self, **kwargs: Unpack[Glm46vVideoProcessorInitKwargs]): + super().__init__(**kwargs) + if self.size is not None and ( + self.size.get("shortest_edge", None) is None or self.size.get("longest_edge", None) is None + ): + raise ValueError("size must contain 'shortest_edge' and 'longest_edge' keys.") + + def _further_process_kwargs( + self, + size: Optional[SizeDict] = None, + **kwargs, + ) -> dict: + """ + Update kwargs that need further processing before being validated + Can be overridden by subclasses to customize the processing of kwargs. + """ + if size is not None and ("shortest_edge" not in size or "longest_edge" not in size): + raise ValueError("size must contain 'shortest_edge' and 'longest_edge' keys.") + + return super()._further_process_kwargs(size=size, **kwargs) + + def sample_frames( + self, + metadata: VideoMetadata, + fps: Optional[Union[int, float]] = None, + **kwargs, + ): + if metadata is None or getattr(metadata, "fps", None) is None: + raise ValueError( + "Asked to sample frames per second but no video metadata was provided which is required when sampling in Glm46v. " + "Please pass in `VideoMetadata` object or set `do_sample_frames=False`" + ) + + total_frames = metadata.total_num_frames + max_frame_idx = total_frames - 1 + duration = metadata.duration or round(max_frame_idx / metadata.fps) + 1 + + DYNAMIC_FPS_THRES = {30: 3, 300: 1, 2400: 0.5} + MAX_FRAME_COUNT_DYNAMIC = 640 + MAX_DURATION = 2400 + effective_duration = min(duration, MAX_DURATION) + if effective_duration <= 30: + target_fps = DYNAMIC_FPS_THRES[30] + elif effective_duration <= 300: + target_fps = DYNAMIC_FPS_THRES[300] + else: + target_fps = DYNAMIC_FPS_THRES[2400] + extract_t = int(effective_duration * target_fps * self.temporal_patch_size) + extract_t = min(extract_t, MAX_FRAME_COUNT_DYNAMIC) + + max_effective_frame = max_frame_idx + duration_per_frame = 1 / metadata.fps + timestamps = [i * duration_per_frame for i in range(total_frames)] + max_second = int(duration) + + if total_frames < extract_t: + frame_indices = np.linspace(0, total_frames - 1, extract_t, dtype=int).tolist() + else: + frame_indices = [] + current_second = 0 + inv_fps = 1 / (self.temporal_patch_size * target_fps) + for frame_index in range(total_frames): + if timestamps[frame_index] >= current_second: + current_second += inv_fps + frame_indices.append(frame_index) + if current_second >= max_second: + break + + if len(frame_indices) < 3: + frame_indices = np.linspace(0, total_frames - 1, extract_t, dtype=int).tolist() + + if len(frame_indices) < extract_t: + if len(frame_indices) < extract_t - 30: + print(f"Warning: Only {len(frame_indices)} frames extracted, but {extract_t} required. Padding.") + frame_indices = np.linspace(frame_indices[0], frame_indices[-1], extract_t, dtype=int).tolist() + elif len(frame_indices) > extract_t: + frame_indices = np.linspace(0, total_frames - 1, extract_t, dtype=int).tolist() + + if len(frame_indices) % self.temporal_patch_size != 0: + frame_indices.append(frame_indices[-1]) + + seen, uniq = set(), [] + for idx in frame_indices: + if idx not in seen: + seen.add(idx) + uniq.append(idx) + + if len(uniq) & 1: + uniq.append(uniq[-1]) + + return np.array(uniq) + + def _preprocess( + self, + videos: list[torch.Tensor], + do_convert_rgb: bool = True, + do_resize: bool = True, + size: Optional[SizeDict] = None, + interpolation: PILImageResampling = PILImageResampling.BICUBIC, + do_rescale: bool = True, + rescale_factor: float = 1 / 255.0, + do_normalize: bool = True, + image_mean: Optional[Union[float, list[float]]] = None, + image_std: Optional[Union[float, list[float]]] = None, + patch_size: Optional[int] = None, + temporal_patch_size: Optional[int] = None, + merge_size: Optional[int] = None, + return_tensors: Optional[Union[str, TensorType]] = None, + **kwargs, + ): + grouped_videos, grouped_videos_index = group_videos_by_shape(videos) + resized_videos_grouped = {} + + for shape, stacked_videos in grouped_videos.items(): + B, T, C, H, W = stacked_videos.shape + num_frames, height, width = T, H, W + if do_resize: + resized_height, resized_width = smart_resize( + num_frames=num_frames, + height=height, + width=width, + temporal_factor=temporal_patch_size, + factor=patch_size * merge_size, + min_pixels=size.shortest_edge, + max_pixels=size.longest_edge, + ) + stacked_videos = stacked_videos.view(B * T, C, H, W) + stacked_videos = self.resize( + stacked_videos, + size=SizeDict(height=resized_height, width=resized_width), + interpolation=interpolation, + ) + stacked_videos = stacked_videos.view(B, T, C, resized_height, resized_width) + resized_videos_grouped[shape] = stacked_videos + resized_videos = reorder_videos(resized_videos_grouped, grouped_videos_index) + + # Group videos by size for further processing + # Needed in case do_resize is False, or resize returns videos with different sizes + grouped_videos, grouped_videos_index = group_videos_by_shape(resized_videos) + processed_videos_grouped = {} + processed_grids = {} + for shape, stacked_videos in grouped_videos.items(): + resized_height, resized_width = get_image_size(stacked_videos[0], channel_dim=ChannelDimension.FIRST) + + # Fused rescale and normalize + stacked_videos = self.rescale_and_normalize( + stacked_videos, do_rescale, rescale_factor, do_normalize, image_mean, image_std + ) + patches = stacked_videos + + # Check that videos have `num_frames` divisible by `temporal_patch_size` + if patches.shape[1] % temporal_patch_size != 0: + repeats = patches[:, -1:].repeat(1, temporal_patch_size - 1, 1, 1, 1) + patches = torch.cat([patches, repeats], dim=1) + batch_size, grid_t, channel = patches.shape[:3] + grid_t = grid_t // temporal_patch_size + grid_h, grid_w = resized_height // patch_size, resized_width // patch_size + + patches = patches.view( + batch_size, + grid_t, + temporal_patch_size, + channel, + grid_h // merge_size, + merge_size, + patch_size, + grid_w // merge_size, + merge_size, + patch_size, + ) + patches = patches.permute(0, 1, 4, 7, 5, 8, 3, 2, 6, 9) + flatten_patches = patches.reshape( + batch_size, + grid_t * grid_h * grid_w, + channel * temporal_patch_size * patch_size * patch_size, + ) + + processed_videos_grouped[shape] = flatten_patches + processed_grids[shape] = [[grid_t, grid_h, grid_w]] * batch_size + + processed_videos = reorder_videos(processed_videos_grouped, grouped_videos_index) + processed_grids = reorder_videos(processed_grids, grouped_videos_index) + pixel_values_videos = torch.cat(processed_videos, dim=0) + video_grid_thw = torch.tensor(processed_grids) + data = { + "pixel_values_videos": pixel_values_videos, + "video_grid_thw": video_grid_thw, + } + + return BatchFeature(data=data, tensor_type=return_tensors) + + +__all__ = ["Glm46vVideoProcessor"] diff --git a/src/transformers/models/glm4v/modular_glm4v.py b/src/transformers/models/glm4v/modular_glm4v.py index 8ae513b63d44..99f67c1e728d 100644 --- a/src/transformers/models/glm4v/modular_glm4v.py +++ b/src/transformers/models/glm4v/modular_glm4v.py @@ -1658,7 +1658,7 @@ def __call__( for frame_idx in range(num_frames): timestamp_sec = selected_timestamps[frame_idx] - frame_structure = f"<|begin_of_image|>{self.image_token}<|end_of_image|>{int(timestamp_sec)}" + frame_structure = self.replace_frame_token_id(timestamp_sec) video_structure += frame_structure text[i] = text[i].replace(self.video_token, video_structure, 1) @@ -1684,6 +1684,8 @@ def __call__( text_inputs["mm_token_type_ids"] = mm_token_type_ids.tolist() return BatchFeature(data={**text_inputs, **image_inputs, **videos_inputs}, tensor_type=return_tensors) + def replace_frame_token_id(self, timestamp_sec): + return f"<|begin_of_image|>{self.image_token}<|end_of_image|>{int(timestamp_sec)}" __all__ = [ "Glm4vConfig", diff --git a/src/transformers/models/glm4v/processing_glm4v.py b/src/transformers/models/glm4v/processing_glm4v.py index e8f9c948c66d..8b170a41c05d 100644 --- a/src/transformers/models/glm4v/processing_glm4v.py +++ b/src/transformers/models/glm4v/processing_glm4v.py @@ -186,7 +186,7 @@ def __call__( for frame_idx in range(num_frames): timestamp_sec = selected_timestamps[frame_idx] - frame_structure = f"<|begin_of_image|>{self.image_token}<|end_of_image|>{int(timestamp_sec)}" + frame_structure = self.replace_frame_token_id(timestamp_sec) video_structure += frame_structure text[i] = text[i].replace(self.video_token, video_structure, 1) @@ -277,5 +277,8 @@ def post_process_image_text_to_text( **kwargs, ) + def replace_frame_token_id(self, timestamp_sec): + return f"<|begin_of_image|>{self.image_token}<|end_of_image|>{int(timestamp_sec)}" + __all__ = ["Glm4vProcessor"] diff --git a/tests/models/glm46v/__init__.py b/tests/models/glm46v/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/tests/models/glm46v/test_modeling_glm46v.py b/tests/models/glm46v/test_modeling_glm46v.py new file mode 100644 index 000000000000..427e7ea85a3a --- /dev/null +++ b/tests/models/glm46v/test_modeling_glm46v.py @@ -0,0 +1,586 @@ +# coding=utf-8 +# Copyright 2025 the HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Testing suite for the PyTorch GLM-4.6V model.""" + +import copy +import unittest + +from transformers import ( + AutoProcessor, + Glm46VConfig, + Glm46VForConditionalGeneration, + Glm46VModel, + is_torch_available, +) +from transformers.testing_utils import ( + Expectations, + cleanup, + require_deterministic_for_xpu, + require_flash_attn, + require_torch, + require_torch_gpu, + slow, + torch_device, +) + +from ...generation.test_utils import GenerationTesterMixin +from ...test_configuration_common import ConfigTester +from ...test_modeling_common import ( + ModelTesterMixin, + floats_tensor, + ids_tensor, +) + + +if is_torch_available(): + import torch + + +class Glm46VVisionText2TextModelTester: + def __init__( + self, + parent, + batch_size=3, + seq_length=7, + num_channels=3, + ignore_index=-100, + image_size=112, + video_start_token_id=3, + video_end_token_id=4, + image_start_token_id=5, + image_end_token_id=6, + image_token_id=7, + video_token_id=8, + is_training=True, + text_config={ + "vocab_size": 99, + "hidden_size": 16, + "intermediate_size": 22, + "num_hidden_layers": 2, + "num_attention_heads": 2, + "num_key_value_heads": 1, + "output_channels": 64, + "hidden_act": "silu", + "max_position_embeddings": 512, + "rope_parameters": {"type": "default", "mrope_section": [2, 1, 1]}, + "rope_theta": 10000, + "tie_word_embeddings": True, + "bos_token_id": 0, + "eos_token_id": 0, + "pad_token_id": 0, + }, + vision_config={ + "depth": 2, + "hidden_act": "silu", + "hidden_size": 48, + "out_hidden_size": 16, + "intermediate_size": 22, + "patch_size": 14, + "spatial_merge_size": 1, + "temporal_patch_size": 2, + }, + ): + self.parent = parent + self.ignore_index = ignore_index + self.bos_token_id = text_config["bos_token_id"] + self.eos_token_id = text_config["eos_token_id"] + self.pad_token_id = text_config["pad_token_id"] + self.video_start_token_id = video_start_token_id + self.video_end_token_id = video_end_token_id + self.image_start_token_id = image_start_token_id + self.image_end_token_id = image_end_token_id + self.image_token_id = image_token_id + self.video_token_id = video_token_id + self.text_config = text_config + self.vision_config = vision_config + self.batch_size = batch_size + self.num_channels = num_channels + self.image_size = image_size + self.is_training = is_training + self.hidden_size = text_config["hidden_size"] + self.num_hidden_layers = text_config["num_hidden_layers"] + self.num_attention_heads = text_config["num_attention_heads"] + self.vocab_size = text_config["vocab_size"] + self.num_image_tokens = 64 + self.seq_length = seq_length + self.num_image_tokens + + def get_config(self): + return Glm46VConfig( + text_config=self.text_config, + vision_config=self.vision_config, + image_token_id=self.image_token_id, + video_token_id=self.video_token_id, + video_start_token_id=self.video_start_token_id, + video_end_token_id=self.video_end_token_id, + image_start_token_id=self.image_start_token_id, + image_end_token_id=self.image_end_token_id, + ) + + def prepare_config_and_inputs(self): + config = self.get_config() + patch_size = config.vision_config.patch_size + temporal_patch_size = config.vision_config.temporal_patch_size + pixel_values = floats_tensor( + [ + self.batch_size * (self.image_size**2) // (patch_size**2), + self.num_channels * (patch_size**2) * temporal_patch_size, + ] + ) + + return config, pixel_values + + def prepare_config_and_inputs_for_common(self): + config_and_inputs = self.prepare_config_and_inputs() + config, pixel_values = config_and_inputs + input_ids = ids_tensor([self.batch_size, self.seq_length], self.vocab_size) + attention_mask = torch.ones(input_ids.shape, dtype=torch.long, device=torch_device) + + input_ids[input_ids == self.video_token_id] = self.pad_token_id + input_ids[input_ids == self.image_token_id] = self.pad_token_id + input_ids[input_ids == self.video_start_token_id] = self.pad_token_id + input_ids[input_ids == self.image_start_token_id] = self.pad_token_id + input_ids[input_ids == self.video_end_token_id] = self.pad_token_id + input_ids[input_ids == self.image_end_token_id] = self.pad_token_id + + input_ids[:, 0] = self.image_start_token_id + input_ids[:, 1 : 1 + self.num_image_tokens] = self.image_token_id + input_ids[:, 1 + self.num_image_tokens] = self.image_end_token_id + patch_size = config.vision_config.patch_size + patches_per_side = self.image_size // patch_size + + inputs_dict = { + "pixel_values": pixel_values, + "image_grid_thw": torch.tensor( + [[1, patches_per_side, patches_per_side]] * self.batch_size, device=torch_device + ), + "input_ids": input_ids, + "attention_mask": attention_mask, + } + return config, inputs_dict + + +@require_torch +class Glm46VModelTest(ModelTesterMixin, GenerationTesterMixin, unittest.TestCase): + all_model_classes = (Glm46VModel, Glm46VForConditionalGeneration) if is_torch_available() else () + + model_split_percents = [0.7, 0.9] # model too big to split at 0.5 + _is_composite = True + + def setUp(self): + self.model_tester = Glm46VVisionText2TextModelTester(self) + self.config_tester = ConfigTester(self, config_class=Glm46VConfig, has_text_modality=False) + + def test_config(self): + self.config_tester.run_common_tests() + + # GLM4V has images shaped as (bs*patch_len, dim) so we can't slice to batches in generate + def prepare_config_and_inputs_for_generate(self, batch_size=2): + config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() + + # We don't want a few model inputs in our model input dictionary for generation tests + input_keys_to_ignore = [ + # we don't want to mask attention heads + # we don't want encoder-decoder models to start from filled decoder ids + "decoder_input_ids", + "decoder_attention_mask", + # we'll set cache use in each test differently + "use_cache", + # Ignore labels if it is in the input dict + "labels", + # model-specific exceptions should overload/overwrite this function + ] + + # The diff from the general `prepare_config_and_inputs_for_generate` lies here + patch_size = config.vision_config.patch_size + filtered_image_length = batch_size * (self.model_tester.image_size**2) // (patch_size**2) + filtered_inputs_dict = { + k: v[:batch_size, ...] if isinstance(v, torch.Tensor) else v + for k, v in inputs_dict.items() + if k not in input_keys_to_ignore + } + filtered_inputs_dict["pixel_values"] = inputs_dict["pixel_values"][:filtered_image_length] + + # It is important set `eos_token_id` to `None` to avoid early stopping (would break for length-based checks) + text_gen_config = config.get_text_config(decoder=True) + if text_gen_config.eos_token_id is not None and text_gen_config.pad_token_id is None: + text_gen_config.pad_token_id = ( + text_gen_config.eos_token_id + if isinstance(text_gen_config.eos_token_id, int) + else text_gen_config.eos_token_id[0] + ) + text_gen_config.eos_token_id = None + text_gen_config.forced_eos_token_id = None + + return config, filtered_inputs_dict + + @unittest.skip(reason="No available kernels - not supported") + def test_sdpa_can_dispatch_on_flash(self): + pass + + @unittest.skip(reason="Size mismatch") + def test_multi_gpu_data_parallel_forward(self): + pass + + @unittest.skip("Error with compilation") + def test_generate_from_inputs_embeds_with_static_cache(self): + pass + + def test_inputs_embeds(self): + config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() + + for model_class in self.all_model_classes: + model = model_class(config) + model.to(torch_device) + model.eval() + + inputs = copy.deepcopy(self._prepare_for_class(inputs_dict, model_class)) + + input_ids = inputs["input_ids"] + del inputs["input_ids"] + del inputs["pixel_values"] + del inputs["image_grid_thw"] + + wte = model.get_input_embeddings() + inputs["inputs_embeds"] = wte(input_ids) + with torch.no_grad(): + model(**inputs)[0] + + def test_inputs_embeds_matches_input_ids(self): + config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() + + for model_class in self.all_model_classes: + model = model_class(config) + model.to(torch_device) + model.eval() + + inputs = self._prepare_for_class(inputs_dict, model_class) + input_ids = inputs["input_ids"] + del inputs["input_ids"] + del inputs["pixel_values"] + del inputs["image_grid_thw"] + + inputs_embeds = model.get_input_embeddings()(input_ids) + + with torch.no_grad(): + out_ids = model(input_ids=input_ids, **inputs)[0] + out_embeds = model(inputs_embeds=inputs_embeds, **inputs)[0] + torch.testing.assert_close(out_embeds, out_ids) + + +@require_torch +class Glm46VIntegrationTest(unittest.TestCase): + def setUp(self): + cleanup(torch_device, gc_collect=True) + + self.processor = AutoProcessor.from_pretrained("THUDM/GLM-4.1V-9B-Thinking") + self.message = [ + { + "role": "user", + "content": [ + { + "type": "image", + "url": "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/pipeline-cat-chonk.jpeg", + }, + {"type": "text", "text": "What kind of dog is this?"}, + ], + } + ] + self.message2 = [ + { + "role": "user", + "content": [ + { + "type": "image", + "url": "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/coco_sample.png", + }, + {"type": "text", "text": "What kind of dog is this?"}, + ], + } + ] + + def tearDown(self): + cleanup(torch_device, gc_collect=True) + + @slow + def test_small_model_integration_test(self): + model = Glm46VForConditionalGeneration.from_pretrained( + "THUDM/GLM-4.1V-9B-Thinking", dtype="auto", device_map="auto" + ) + + inputs = self.processor.apply_chat_template( + self.message, tokenize=True, add_generation_prompt=True, return_dict=True, return_tensors="pt" + ) + expected_input_ids = [151331, 151333, 151336, 198, 151339, 151343, 151343, 151343, 151343, 151343, 151343, 151343, 151343, 151343, 151343, 151343, 151343] # fmt: skip + assert expected_input_ids == inputs.input_ids[0].tolist()[:17] + + expected_pixel_slice = torch.tensor( + [ + [-0.0988, -0.0842, -0.0842], + [-0.5660, -0.5514, -0.4200], + [-0.0259, -0.0259, -0.0259], + [-0.1280, -0.0988, -0.2010], + [-0.4638, -0.5806, -0.6974], + [-1.2083, -1.2229, -1.2083], + ], + dtype=torch.float32, + device="cpu", + ) + assert torch.allclose(expected_pixel_slice, inputs.pixel_values[:6, :3], atol=3e-3) + + # verify generation + inputs = inputs.to(torch_device) + + # This model on the hub has `do_sample=True`. + torch.manual_seed(42) + + output = model.generate(**inputs, max_new_tokens=30) + EXPECTED_DECODED_TEXT = "\nWhat kind of dog is this?\nGot it, let's look at the image. The animal in the picture doesn't look like a dog; it's actually a cat. Specifically" + self.assertEqual( + self.processor.decode(output[0], skip_special_tokens=True), + EXPECTED_DECODED_TEXT, + ) + + @slow + def test_small_model_integration_test_batch(self): + model = Glm46VForConditionalGeneration.from_pretrained( + "THUDM/GLM-4.1V-9B-Thinking", dtype="auto", device_map="auto" + ) + batch_messages = [self.message] * 2 + inputs = self.processor.apply_chat_template( + batch_messages, tokenize=True, add_generation_prompt=True, return_dict=True, return_tensors="pt" + ).to(torch_device) + + # This model on the hub has `do_sample=True`. + torch.manual_seed(42) + + # it should not matter whether two images are the same size or not + output = model.generate(**inputs, max_new_tokens=30) + + EXPECTED_DECODED_TEXT = [ + "\nWhat kind of dog is this?\nGot it, let's look at the image. The animal in the picture doesn't look like a dog; it's actually a cat. Specifically", + "\nWhat kind of dog is this?\nGot it, let's look at the image. The animal in the picture has a stocky body, thick fur, and a face that's" + ] # fmt: skip + self.assertEqual( + self.processor.batch_decode(output, skip_special_tokens=True), + EXPECTED_DECODED_TEXT, + ) + + @slow + def test_small_model_integration_test_with_video(self): + processor = AutoProcessor.from_pretrained("THUDM/GLM-4.1V-9B-Thinking", max_image_size={"longest_edge": 50176}) + model = Glm46VForConditionalGeneration.from_pretrained( + "THUDM/GLM-4.1V-9B-Thinking", dtype=torch.float16, device_map="auto" + ) + questions = ["Describe this video."] + video_urls = ["https://huggingface.co/datasets/hf-internal-testing/fixtures_videos/resolve/main/tennis.mp4"] + messages = [ + [ + { + "role": "user", + "content": [ + { + "type": "video", + "video": video_url, + }, + {"type": "text", "text": question}, + ], + } + ] + for question, video_url in zip(questions, video_urls) + ] + inputs = processor.apply_chat_template( + messages, tokenize=True, add_generation_prompt=True, return_dict=True, return_tensors="pt", padding=True + ).to(torch_device) + + # This model on the hub has `do_sample=True`. + torch.manual_seed(42) + + output = model.generate(**inputs, max_new_tokens=30) + EXPECTED_DECODED_TEXT = ["\n012345Describe this video.\nGot it, let's analyze the video. First, the scene is an indoor tennis court. There are two players: one in a white shirt"] # fmt: skip + + self.assertEqual( + processor.batch_decode(output, skip_special_tokens=True), + EXPECTED_DECODED_TEXT, + ) + + @slow + @require_deterministic_for_xpu + def test_small_model_integration_test_expand(self): + model = Glm46VForConditionalGeneration.from_pretrained( + "THUDM/GLM-4.1V-9B-Thinking", dtype="auto", device_map="auto" + ) + inputs = self.processor.apply_chat_template( + self.message, tokenize=True, add_generation_prompt=True, return_dict=True, return_tensors="pt" + ).to(torch_device) + + # This model on the hub has `do_sample=True`. + torch.manual_seed(42) + + output = model.generate(**inputs, max_new_tokens=30, do_sample=False, num_beams=2, num_return_sequences=2) + + # fmt: off + EXPECTED_DECODED_TEXTS = Expectations( + { + + (None, None): ["\nWhat kind of dog is this?\nGot it, let's look at the image. The animal in the picture doesn't look like a dog; it's actually a cat. Specifically", + "\nWhat kind of dog is this?\nGot it, let's look at the image. The animal in the picture doesn't look like a dog; it's actually a cat, specifically" + ], + ("xpu", None): ["\nWhat kind of dog is this?\nGot it, let's look at the image. The animal in the picture is not a dog; it's a cat. Specifically, it looks", + "\nWhat kind of dog is this?\nGot it, let's look at the image. The animal in the picture is not a dog; it's a cat, specifically a Pallas" + ], + } + ) + # fmt: on + EXPECTED_DECODED_TEXT = EXPECTED_DECODED_TEXTS.get_expectation() + + decoded_text = self.processor.batch_decode(output, skip_special_tokens=True) + self.assertEqual(decoded_text, EXPECTED_DECODED_TEXT) + + @slow + def test_small_model_integration_test_batch_wo_image(self): + model = Glm46VForConditionalGeneration.from_pretrained( + "THUDM/GLM-4.1V-9B-Thinking", dtype="auto", device_map="auto" + ) + message_wo_image = [ + {"role": "user", "content": [{"type": "text", "text": "Who are you?"}]}, + ] + batched_messages = [self.message, message_wo_image] + inputs = self.processor.apply_chat_template( + batched_messages, + tokenize=True, + add_generation_prompt=True, + return_dict=True, + return_tensors="pt", + padding=True, + ).to(torch_device) + + # This model on the hub has `do_sample=True`. + torch.manual_seed(42) + + # it should not matter whether two images are the same size or not + output = model.generate(**inputs, max_new_tokens=30) + + EXPECTED_DECODED_TEXT = [ + "\nWhat kind of dog is this?\nGot it, let's look at the image. The animal in the picture doesn't look like a dog; it's actually a cat. Specifically", + "\nWho are you?\nGot it, let's look at the user's question: \"Who are you?\" This is a common question when someone is just starting a conversation" + ] # fmt: skip + self.assertEqual( + self.processor.batch_decode(output, skip_special_tokens=True), + EXPECTED_DECODED_TEXT, + ) + + @slow + def test_small_model_integration_test_batch_different_resolutions(self): + model = Glm46VForConditionalGeneration.from_pretrained( + "THUDM/GLM-4.1V-9B-Thinking", dtype="auto", device_map="auto" + ) + batched_messages = [self.message, self.message2] + inputs = self.processor.apply_chat_template( + batched_messages, + tokenize=True, + add_generation_prompt=True, + return_dict=True, + return_tensors="pt", + padding=True, + ).to(torch_device) + + # This model on the hub has `do_sample=True`. + torch.manual_seed(42) + + # it should not matter whether two images are the same size or not + output = model.generate(**inputs, max_new_tokens=30) + + EXPECTED_DECODED_TEXT = [ + "\nWhat kind of dog is this?\nGot it, let's look at the image. The animal in the picture doesn't look like a dog; it's actually a cat. Specifically", + "\nWhat kind of dog is this?\nGot it, let's look at the image. Wait, the animals here are cats, not dogs. The question is about a dog, but", + ] # fmt: skip + self.assertEqual( + self.processor.batch_decode(output, skip_special_tokens=True), + EXPECTED_DECODED_TEXT, + ) + + @slow + @require_flash_attn + @require_torch_gpu + def test_small_model_integration_test_batch_flashatt2(self): + model = Glm46VForConditionalGeneration.from_pretrained( + "THUDM/GLM-4.1V-9B-Thinking", + dtype=torch.bfloat16, + attn_implementation="flash_attention_2", + device_map="auto", + ) + batched_messages = [self.message, self.message2] + inputs = self.processor.apply_chat_template( + batched_messages, + tokenize=True, + add_generation_prompt=True, + return_dict=True, + return_tensors="pt", + padding=True, + ).to(torch_device) + + # This model on the hub has `do_sample=True`. + torch.manual_seed(42) + + # it should not matter whether two images are the same size or not + output = model.generate(**inputs, max_new_tokens=30) + + EXPECTED_DECODED_TEXT = [ + "\nWhat kind of dog is this?\nGot it, let's look at the image. The animal in the picture doesn't look like a dog. Wait, it's a cat,", + "\nWhat kind of dog is this?\nGot it, let's look at the image. Wait, the animals here are cats, not dogs. The question is about a dog, but" + ] # fmt: skip + self.assertEqual( + self.processor.batch_decode(output, skip_special_tokens=True), + EXPECTED_DECODED_TEXT, + ) + + @slow + @require_flash_attn + @require_torch_gpu + def test_small_model_integration_test_batch_wo_image_flashatt2(self): + model = Glm46VForConditionalGeneration.from_pretrained( + "THUDM/GLM-4.1V-9B-Thinking", + dtype=torch.bfloat16, + attn_implementation="flash_attention_2", + device_map="auto", + ) + message_wo_image = [ + {"role": "user", "content": [{"type": "text", "text": "Who are you?"}]}, + ] + batched_messages = [self.message, message_wo_image] + inputs = self.processor.apply_chat_template( + batched_messages, + tokenize=True, + add_generation_prompt=True, + return_dict=True, + return_tensors="pt", + padding=True, + ).to(torch_device) + + # This model on the hub has `do_sample=True`. + torch.manual_seed(42) + + # it should not matter whether two images are the same size or not + output = model.generate(**inputs, max_new_tokens=30) + + EXPECTED_DECODED_TEXT = [ + "\nWhat kind of dog is this?\nGot it, let's look at the image. The animal in the picture doesn't look like a dog; it's actually a cat. Specifically", + "\nWho are you?\nGot it, let's look at the user's question: \"Who are you?\" This is a common question when someone is just starting a conversation" + ] # fmt: skip + + self.assertEqual( + self.processor.batch_decode(output, skip_special_tokens=True), + EXPECTED_DECODED_TEXT, + ) diff --git a/tests/models/glm4v_moe/test_modeling_glm4v_moe.py b/tests/models/glm4v_moe/test_modeling_glm4v_moe.py index f291a868ab17..afae9ef118a8 100644 --- a/tests/models/glm4v_moe/test_modeling_glm4v_moe.py +++ b/tests/models/glm4v_moe/test_modeling_glm4v_moe.py @@ -11,7 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -"""Testing suite for the PyTorch GLM-4.1V model.""" +"""Testing suite for the PyTorch GLM-4.5V model.""" import copy import unittest From 621de8d39e9d8007e55c0e585865542f3f65a5b0 Mon Sep 17 00:00:00 2001 From: zRzRzRzRzRzRzR <2448370773@qq.com> Date: Thu, 6 Nov 2025 21:42:44 +0800 Subject: [PATCH 02/66] update --- src/transformers/models/__init__.py | 2 + .../models/auto/processing_auto.py | 1 + src/transformers/models/glm46v/__init__.py | 5 +- .../models/glm46v/modular_glm46v.py | 2 +- .../models/glm46v/processing_glm46v.py | 1 + .../models/glm46v/video_processing_glm46v.py | 4 +- .../models/glm4v/modular_glm4v.py | 1 + tests/models/glm46v/test_processor_glm46v.py | 279 +++++++++++++++ .../glm46v/test_video_processing_glm46v.py | 334 ++++++++++++++++++ 9 files changed, 622 insertions(+), 7 deletions(-) create mode 100644 tests/models/glm46v/test_processor_glm46v.py create mode 100644 tests/models/glm46v/test_video_processing_glm46v.py diff --git a/src/transformers/models/__init__.py b/src/transformers/models/__init__.py index 3b96d1a65691..08307922d0f5 100644 --- a/src/transformers/models/__init__.py +++ b/src/transformers/models/__init__.py @@ -141,6 +141,8 @@ from .git import * from .glm import * from .glm4 import * + from .glm4v import * + from .glm4v_moe import * from .glm46v import * from .glpn import * from .got_ocr2 import * diff --git a/src/transformers/models/auto/processing_auto.py b/src/transformers/models/auto/processing_auto.py index 5186b78b07e0..aeb8be8915b5 100644 --- a/src/transformers/models/auto/processing_auto.py +++ b/src/transformers/models/auto/processing_auto.py @@ -74,6 +74,7 @@ ("gemma3", "Gemma3Processor"), ("gemma3n", "Gemma3nProcessor"), ("git", "GitProcessor"), + ("glm46v", "Glm46vProcessor"), ("glm4v", "Glm4vProcessor"), ("glm4v_moe", "Glm4vProcessor"), ("got_ocr2", "GotOcr2Processor"), diff --git a/src/transformers/models/glm46v/__init__.py b/src/transformers/models/glm46v/__init__.py index ae27f605ed0d..9a25df25e1af 100644 --- a/src/transformers/models/glm46v/__init__.py +++ b/src/transformers/models/glm46v/__init__.py @@ -1,5 +1,4 @@ -# coding=utf-8 -# Copyright 2025 the HuggingFace Team. All rights reserved. +# Copyright 2025 The HuggingFace Team. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -12,7 +11,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - from typing import TYPE_CHECKING from ...utils import _LazyModule @@ -22,6 +20,7 @@ if TYPE_CHECKING: from .configuration_glm46v import * from .modeling_glm46v import * + from .processing_glm46v import * else: import sys diff --git a/src/transformers/models/glm46v/modular_glm46v.py b/src/transformers/models/glm46v/modular_glm46v.py index 6d20e69c3991..377ab80bf9db 100644 --- a/src/transformers/models/glm46v/modular_glm46v.py +++ b/src/transformers/models/glm46v/modular_glm46v.py @@ -127,10 +127,10 @@ class Glm46VForConditionalGeneration(Glm4vForConditionalGeneration): class Glm46vProcessor(Glm4vProcessor): - def replace_frame_token_id(self, timestamp_sec): return f"<|begin_of_image|>{self.image_token}<|end_of_image|>{round(timestamp_sec)} seconds" + __all__ = [ "Glm46VConfig", "Glm46VTextConfig", diff --git a/src/transformers/models/glm46v/processing_glm46v.py b/src/transformers/models/glm46v/processing_glm46v.py index 9c996a05c52a..fd8b57618c14 100644 --- a/src/transformers/models/glm46v/processing_glm46v.py +++ b/src/transformers/models/glm46v/processing_glm46v.py @@ -191,6 +191,7 @@ def __call__( video_structure += frame_structure text[i] = text[i].replace(self.video_token, video_structure, 1) + print(text[i]) num_image_tokens = ( video_grid_thw[video_index].prod() // merge_length // video_grid_thw[video_index][0] ) diff --git a/src/transformers/models/glm46v/video_processing_glm46v.py b/src/transformers/models/glm46v/video_processing_glm46v.py index ab1cf0cbfad1..95bc07f560ad 100644 --- a/src/transformers/models/glm46v/video_processing_glm46v.py +++ b/src/transformers/models/glm46v/video_processing_glm46v.py @@ -14,7 +14,6 @@ # limitations under the License. """video processor class for GLM-4.6V.""" -import math from typing import Optional, Union import numpy as np @@ -36,7 +35,7 @@ from .image_processing_glm4v import smart_resize -class Glm4vVideoProcessorInitKwargs(VideosKwargs, total=False): +class Glm46vVideoProcessorInitKwargs(VideosKwargs, total=False): max_image_size: dict[str, int] patch_size: int temporal_patch_size: int @@ -127,7 +126,6 @@ def sample_frames( extract_t = int(effective_duration * target_fps * self.temporal_patch_size) extract_t = min(extract_t, MAX_FRAME_COUNT_DYNAMIC) - max_effective_frame = max_frame_idx duration_per_frame = 1 / metadata.fps timestamps = [i * duration_per_frame for i in range(total_frames)] max_second = int(duration) diff --git a/src/transformers/models/glm4v/modular_glm4v.py b/src/transformers/models/glm4v/modular_glm4v.py index 99f67c1e728d..77def041a85d 100644 --- a/src/transformers/models/glm4v/modular_glm4v.py +++ b/src/transformers/models/glm4v/modular_glm4v.py @@ -1687,6 +1687,7 @@ def __call__( def replace_frame_token_id(self, timestamp_sec): return f"<|begin_of_image|>{self.image_token}<|end_of_image|>{int(timestamp_sec)}" + __all__ = [ "Glm4vConfig", "Glm4vTextConfig", diff --git a/tests/models/glm46v/test_processor_glm46v.py b/tests/models/glm46v/test_processor_glm46v.py new file mode 100644 index 000000000000..8695bba9886d --- /dev/null +++ b/tests/models/glm46v/test_processor_glm46v.py @@ -0,0 +1,279 @@ +# Copyright 2025 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import inspect +import shutil +import tempfile +import unittest + +import numpy as np + +from transformers import AutoProcessor +from transformers.testing_utils import require_av, require_torch, require_vision +from transformers.utils import is_torch_available, is_vision_available + +from ...test_processing_common import ProcessorTesterMixin, url_to_local_path + + +if is_vision_available(): + from transformers import Glm46vProcessor + +if is_torch_available(): + import torch + + +@require_vision +@require_torch +class Glm46vProcessorTest(ProcessorTesterMixin, unittest.TestCase): + processor_class = Glm46vProcessor + + @classmethod + def setUpClass(cls): + cls.tmpdirname = tempfile.mkdtemp() + processor = Glm46vProcessor.from_pretrained( + "THUDM/GLM-4.1V-9B-Thinking", patch_size=4, size={"shortest_edge": 12 * 12, "longest_edge": 18 * 18} + ) + processor.save_pretrained(cls.tmpdirname) + cls.image_token = processor.image_token + + def get_tokenizer(self, **kwargs): + return AutoProcessor.from_pretrained(self.tmpdirname, **kwargs).tokenizer + + def get_image_processor(self, **kwargs): + return AutoProcessor.from_pretrained(self.tmpdirname, **kwargs).image_processor + + def get_video_processor(self, **kwargs): + return AutoProcessor.from_pretrained(self.tmpdirname, **kwargs).video_processor + + def get_processor(self, **kwargs): + return AutoProcessor.from_pretrained(self.tmpdirname, **kwargs) + + @classmethod + def tearDownClass(cls): + shutil.rmtree(cls.tmpdirname, ignore_errors=True) + + @require_torch + @require_av + def _test_apply_chat_template( + self, + modality: str, + batch_size: int, + return_tensors: str, + input_name: str, + processor_name: str, + input_data: list[str], + ): + processor = self.get_processor() + if processor.chat_template is None: + self.skipTest("Processor has no chat template") + + if processor_name not in self.processor_class.attributes: + self.skipTest(f"{processor_name} attribute not present in {self.processor_class}") + + batch_messages = [ + [ + { + "role": "user", + "content": [{"type": "text", "text": "Describe this."}], + }, + ] + ] * batch_size + + # Test that jinja can be applied + formatted_prompt = processor.apply_chat_template(batch_messages, add_generation_prompt=True, tokenize=False) + self.assertEqual(len(formatted_prompt), batch_size) + + # Test that tokenizing with template and directly with `self.tokenizer` gives same output + formatted_prompt_tokenized = processor.apply_chat_template( + batch_messages, add_generation_prompt=True, tokenize=True, return_tensors=return_tensors + ) + add_special_tokens = True + if processor.tokenizer.bos_token is not None and formatted_prompt[0].startswith(processor.tokenizer.bos_token): + add_special_tokens = False + tok_output = processor.tokenizer( + formatted_prompt, return_tensors=return_tensors, add_special_tokens=add_special_tokens + ) + expected_output = tok_output.input_ids + self.assertListEqual(expected_output.tolist(), formatted_prompt_tokenized.tolist()) + + # Test that kwargs passed to processor's `__call__` are actually used + tokenized_prompt_100 = processor.apply_chat_template( + batch_messages, + add_generation_prompt=True, + tokenize=True, + padding="max_length", + truncation=True, + return_tensors=return_tensors, + max_length=100, + ) + self.assertEqual(len(tokenized_prompt_100[0]), 100) + + # Test that `return_dict=True` returns text related inputs in the dict + out_dict_text = processor.apply_chat_template( + batch_messages, + add_generation_prompt=True, + tokenize=True, + return_dict=True, + return_tensors=return_tensors, + ) + self.assertTrue(all(key in out_dict_text for key in ["input_ids", "attention_mask"])) + self.assertEqual(len(out_dict_text["input_ids"]), batch_size) + self.assertEqual(len(out_dict_text["attention_mask"]), batch_size) + + # Test that with modality URLs and `return_dict=True`, we get modality inputs in the dict + for idx, url in enumerate(input_data[:batch_size]): + batch_messages[idx][0]["content"] = [batch_messages[idx][0]["content"][0], {"type": modality, "url": url}] + + out_dict = processor.apply_chat_template( + batch_messages, + add_generation_prompt=True, + tokenize=True, + return_dict=True, + return_tensors=return_tensors, + fps=2 + if isinstance(input_data[0], str) + else None, # by default no more than 2 frames per second, otherwise too slow + do_sample_frames=bool(isinstance(input_data[0], str)), # don't sample frames if decoded video is used + ) + input_name = getattr(self, input_name) + self.assertTrue(input_name in out_dict) + self.assertEqual(len(out_dict["input_ids"]), batch_size) + self.assertEqual(len(out_dict["attention_mask"]), batch_size) + + if modality == "video": + # qwen pixels don't scale with bs same way as other models, calculate expected video token count based on video_grid_thw + expected_video_token_count = 0 + for thw in out_dict["video_grid_thw"]: + expected_video_token_count += thw[0] * thw[1] * thw[2] + mm_len = expected_video_token_count + else: + mm_len = batch_size * 4 + self.assertEqual(len(out_dict[input_name]), mm_len) + + return_tensor_to_type = {"pt": torch.Tensor, "np": np.ndarray, None: list} + for k in out_dict: + self.assertIsInstance(out_dict[k], return_tensor_to_type[return_tensors]) + + @require_av + def test_apply_chat_template_video_frame_sampling(self): + processor = self.get_processor() + if processor.chat_template is None: + self.skipTest("Processor has no chat template") + + signature = inspect.signature(processor.__call__) + if "videos" not in {*signature.parameters.keys()} or ( + signature.parameters.get("videos") is not None + and signature.parameters["videos"].annotation == inspect._empty + ): + self.skipTest("Processor doesn't accept videos at input") + + messages = [ + [ + { + "role": "user", + "content": [ + {"type": "video"}, + {"type": "text", "text": "What is shown in this video?"}, + ], + }, + ] + ] + + formatted_prompt = processor.apply_chat_template(messages, add_generation_prompt=True, tokenize=False) + self.assertEqual(len(formatted_prompt), 1) + + formatted_prompt_tokenized = processor.apply_chat_template(messages, add_generation_prompt=True, tokenize=True) + expected_output = processor.tokenizer(formatted_prompt, return_tensors=None).input_ids + self.assertListEqual(expected_output, formatted_prompt_tokenized) + + out_dict = processor.apply_chat_template(messages, add_generation_prompt=True, tokenize=True, return_dict=True) + self.assertListEqual(list(out_dict.keys()), ["input_ids", "attention_mask"]) + + # Add video URL for return dict and load with `num_frames` arg + messages[0][0]["content"][0] = { + "type": "video", + "url": url_to_local_path( + "https://huggingface.co/datasets/raushan-testing-hf/videos-test/resolve/main/tiny_video.mp4" + ), + } + + # Load with `video_fps` arg + video_fps = 10 + out_dict_with_video = processor.apply_chat_template( + messages, + add_generation_prompt=True, + tokenize=True, + return_dict=True, + video_fps=video_fps, + ) + self.assertTrue(self.videos_input_name in out_dict_with_video) + self.assertEqual(len(out_dict_with_video[self.videos_input_name]), 8) + + # Load the whole video + out_dict_with_video = processor.apply_chat_template( + messages, + add_generation_prompt=True, + tokenize=True, + return_dict=True, + do_sample_frames=False, + ) + self.assertTrue(self.videos_input_name in out_dict_with_video) + self.assertEqual(len(out_dict_with_video[self.videos_input_name]), 24) + + # Load video as a list of frames (i.e. images). NOTE: each frame should have same size + # because we assume they come from one video + messages[0][0]["content"][0] = { + "type": "video", + "url": [ + url_to_local_path( + "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/transformers/tasks/australia.jpg" + ), + url_to_local_path( + "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/transformers/tasks/australia.jpg" + ), + ], + } + out_dict_with_video = processor.apply_chat_template( + messages, + add_generation_prompt=True, + tokenize=True, + return_dict=True, + do_sample_frames=False, + ) + self.assertTrue(self.videos_input_name in out_dict_with_video) + self.assertEqual(len(out_dict_with_video[self.videos_input_name]), 4) + + # When the inputs are frame URLs/paths we expect that those are already + # sampled and will raise an error is asked to sample again. + with self.assertRaisesRegex( + ValueError, "Sampling frames from a list of images is not supported! Set `do_sample_frames=False`" + ): + out_dict_with_video = processor.apply_chat_template( + messages, + add_generation_prompt=True, + tokenize=True, + return_dict=True, + do_sample_frames=True, + ) + + def test_model_input_names(self): + processor = self.get_processor() + + text = self.prepare_text_inputs(modalities=["image", "video"]) + image_input = self.prepare_image_inputs() + video_inputs = self.prepare_video_inputs() + inputs_dict = {"text": text, "images": image_input, "videos": video_inputs} + inputs = processor(**inputs_dict, return_tensors="pt", do_sample_frames=False) + + self.assertSetEqual(set(inputs.keys()), set(processor.model_input_names)) diff --git a/tests/models/glm46v/test_video_processing_glm46v.py b/tests/models/glm46v/test_video_processing_glm46v.py new file mode 100644 index 000000000000..e15809ad8790 --- /dev/null +++ b/tests/models/glm46v/test_video_processing_glm46v.py @@ -0,0 +1,334 @@ +# coding=utf-8 +# Copyright 2025 HuggingFace Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest + +import numpy as np + +from transformers.image_utils import IMAGENET_STANDARD_MEAN, IMAGENET_STANDARD_STD +from transformers.testing_utils import require_torch, require_vision +from transformers.utils import is_torch_available, is_torchvision_available, is_vision_available + +from ...test_video_processing_common import VideoProcessingTestMixin, prepare_video_inputs + + +if is_torch_available(): + from PIL import Image + +if is_vision_available(): + if is_torchvision_available(): + from transformers import Glm46vVideoProcessor + from transformers.models.glm46v.video_processing_glm46v import smart_resize + + +class Glm46vVideoProcessingTester: + def __init__( + self, + parent, + batch_size=5, + num_frames=8, + num_channels=3, + min_resolution=30, + max_resolution=80, + temporal_patch_size=2, + patch_size=14, + merge_size=2, + do_resize=True, + size=None, + do_normalize=True, + image_mean=IMAGENET_STANDARD_MEAN, + image_std=IMAGENET_STANDARD_STD, + do_convert_rgb=True, + ): + size = size if size is not None else {"longest_edge": 20, "shortest_edge": 10} + self.parent = parent + self.batch_size = batch_size + self.num_frames = num_frames + self.num_channels = num_channels + self.min_resolution = min_resolution + self.max_resolution = max_resolution + self.do_resize = do_resize + self.size = size + self.do_normalize = do_normalize + self.image_mean = image_mean + self.image_std = image_std + self.do_convert_rgb = do_convert_rgb + self.temporal_patch_size = temporal_patch_size + self.patch_size = patch_size + self.merge_size = merge_size + + def prepare_video_processor_dict(self): + return { + "do_resize": self.do_resize, + "size": self.size, + "do_normalize": self.do_normalize, + "image_mean": self.image_mean, + "image_std": self.image_std, + "do_convert_rgb": self.do_convert_rgb, + "do_sample_frames": True, + } + + def prepare_video_metadata(self, videos): + video_metadata = [] + for video in videos: + if isinstance(video, list): + num_frames = len(video) + elif hasattr(video, "shape"): + if len(video.shape) == 4: # (T, H, W, C) + num_frames = video.shape[0] + else: + num_frames = 1 + else: + num_frames = self.num_frames + + metadata = { + "fps": 2, + "duration": num_frames / 2, + "total_num_frames": num_frames, + } + video_metadata.append(metadata) + return video_metadata + + def expected_output_video_shape(self, videos): + grid_t = self.num_frames // self.temporal_patch_size + hidden_dim = self.num_channels * self.temporal_patch_size * self.patch_size * self.patch_size + seq_len = 0 + for video in videos: + if isinstance(video, list) and isinstance(video[0], Image.Image): + video = np.stack([np.array(frame) for frame in video]) + elif hasattr(video, "shape"): + pass + else: + video = np.array(video) + + if hasattr(video, "shape") and len(video.shape) >= 3: + if len(video.shape) == 4: + t, height, width = video.shape[:3] + elif len(video.shape) == 3: + height, width = video.shape[:2] + t = 1 + else: + t, height, width = self.num_frames, self.min_resolution, self.min_resolution + else: + t, height, width = self.num_frames, self.min_resolution, self.min_resolution + + resized_height, resized_width = smart_resize( + t, + height, + width, + factor=self.patch_size * self.merge_size, + min_pixels=self.size["shortest_edge"], + max_pixels=self.size["longest_edge"], + ) + grid_h, grid_w = resized_height // self.patch_size, resized_width // self.patch_size + seq_len += grid_t * grid_h * grid_w + return [seq_len, hidden_dim] + + def prepare_video_inputs(self, equal_resolution=False, return_tensors="pil"): + videos = prepare_video_inputs( + batch_size=self.batch_size, + num_frames=self.num_frames, + num_channels=self.num_channels, + min_resolution=self.min_resolution, + max_resolution=self.max_resolution, + equal_resolution=equal_resolution, + return_tensors=return_tensors, + ) + return videos + + +@require_torch +@require_vision +class Glm46vVideoProcessingTest(VideoProcessingTestMixin, unittest.TestCase): + fast_video_processing_class = Glm46vVideoProcessor if is_torchvision_available() else None + input_name = "pixel_values_videos" + + def setUp(self): + super().setUp() + self.video_processor_tester = Glm46vVideoProcessingTester(self) + + @property + def video_processor_dict(self): + return self.video_processor_tester.prepare_video_processor_dict() + + def test_video_processor_from_dict_with_kwargs(self): + video_processor = self.fast_video_processing_class.from_dict(self.video_processor_dict) + self.assertEqual(video_processor.size, {"longest_edge": 20, "shortest_edge": 10}) + + video_processor = self.fast_video_processing_class.from_dict( + self.video_processor_dict, size={"longest_edge": 42, "shortest_edge": 42} + ) + self.assertEqual(video_processor.size, {"longest_edge": 42, "shortest_edge": 42}) + + def test_call_pil(self): + for video_processing_class in self.video_processor_list: + video_processing = video_processing_class(**self.video_processor_dict) + video_inputs = self.video_processor_tester.prepare_video_inputs( + equal_resolution=False, return_tensors="pil" + ) + + for video in video_inputs: + self.assertIsInstance(video[0], Image.Image) + + video_metadata = self.video_processor_tester.prepare_video_metadata(video_inputs) + encoded_videos = video_processing( + video_inputs[0], video_metadata=[video_metadata[0]], return_tensors="pt" + )[self.input_name] + expected_output_video_shape = self.video_processor_tester.expected_output_video_shape([video_inputs[0]]) + self.assertEqual(list(encoded_videos.shape), expected_output_video_shape) + encoded_videos = video_processing(video_inputs, video_metadata=video_metadata, return_tensors="pt")[ + self.input_name + ] + expected_output_video_shape = self.video_processor_tester.expected_output_video_shape(video_inputs) + self.assertEqual(list(encoded_videos.shape), expected_output_video_shape) + + def test_call_numpy(self): + for video_processing_class in self.video_processor_list: + video_processing = video_processing_class(**self.video_processor_dict) + video_inputs = self.video_processor_tester.prepare_video_inputs( + equal_resolution=False, return_tensors="np" + ) + + video_metadata = self.video_processor_tester.prepare_video_metadata(video_inputs) + encoded_videos = video_processing( + video_inputs[0], video_metadata=[video_metadata[0]], return_tensors="pt" + )[self.input_name] + expected_output_video_shape = self.video_processor_tester.expected_output_video_shape([video_inputs[0]]) + self.assertEqual(list(encoded_videos.shape), expected_output_video_shape) + + encoded_videos = video_processing(video_inputs, video_metadata=video_metadata, return_tensors="pt")[ + self.input_name + ] + expected_output_video_shape = self.video_processor_tester.expected_output_video_shape(video_inputs) + self.assertEqual(list(encoded_videos.shape), expected_output_video_shape) + + def test_call_pytorch(self): + for video_processing_class in self.video_processor_list: + video_processing = video_processing_class(**self.video_processor_dict) + video_inputs = self.video_processor_tester.prepare_video_inputs( + equal_resolution=False, return_tensors="pt" + ) + video_metadata = self.video_processor_tester.prepare_video_metadata(video_inputs) + encoded_videos = video_processing( + video_inputs[0], video_metadata=[video_metadata[0]], return_tensors="pt" + )[self.input_name] + expected_output_video_shape = self.video_processor_tester.expected_output_video_shape([video_inputs[0]]) + self.assertEqual(list(encoded_videos.shape), expected_output_video_shape) + encoded_videos = video_processing(video_inputs, video_metadata=video_metadata, return_tensors="pt")[ + self.input_name + ] + expected_output_video_shape = self.video_processor_tester.expected_output_video_shape(video_inputs) + self.assertEqual(list(encoded_videos.shape), expected_output_video_shape) + + @unittest.skip("Skip for now, the test needs adjustment for GLM-4.1V") + def test_call_numpy_4_channels(self): + for video_processing_class in self.video_processor_list: + # Test that can process videos which have an arbitrary number of channels + # Initialize video_processing + video_processor = video_processing_class(**self.video_processor_dict) + + # create random numpy tensors + self.video_processor_tester.num_channels = 4 + video_inputs = self.video_processor_tester.prepare_video_inputs( + equal_resolution=False, return_tensors="np" + ) + + # Test not batched input + encoded_videos = video_processor( + video_inputs[0], + return_tensors="pt", + input_data_format="channels_last", + image_mean=(0.0, 0.0, 0.0, 0.0), + image_std=(1.0, 1.0, 1.0, 1.0), + )[self.input_name] + expected_output_video_shape = self.video_processor_tester.expected_output_video_shape([video_inputs[0]]) + self.assertEqual(list(encoded_videos.shape), expected_output_video_shape) + + # Test batched + encoded_videos = video_processor( + video_inputs, + return_tensors="pt", + input_data_format="channels_last", + image_mean=(0.0, 0.0, 0.0, 0.0), + image_std=(1.0, 1.0, 1.0, 1.0), + )[self.input_name] + expected_output_video_shape = self.video_processor_tester.expected_output_video_shape(video_inputs) + self.assertEqual(list(encoded_videos.shape), expected_output_video_shape) + + def test_nested_input(self): + """Tests that the processor can work with nested list where each video is a list of arrays""" + for video_processing_class in self.video_processor_list: + video_processing = video_processing_class(**self.video_processor_dict) + video_inputs = self.video_processor_tester.prepare_video_inputs( + equal_resolution=False, return_tensors="np" + ) + + video_inputs_nested = [list(video) for video in video_inputs] + video_metadata = self.video_processor_tester.prepare_video_metadata(video_inputs) + + # Test not batched input + encoded_videos = video_processing( + video_inputs_nested[0], video_metadata=[video_metadata[0]], return_tensors="pt" + )[self.input_name] + expected_output_video_shape = self.video_processor_tester.expected_output_video_shape([video_inputs[0]]) + self.assertEqual(list(encoded_videos.shape), expected_output_video_shape) + + # Test batched + encoded_videos = video_processing(video_inputs_nested, video_metadata=video_metadata, return_tensors="pt")[ + self.input_name + ] + expected_output_video_shape = self.video_processor_tester.expected_output_video_shape(video_inputs) + self.assertEqual(list(encoded_videos.shape), expected_output_video_shape) + + def test_call_sample_frames(self): + for video_processing_class in self.video_processor_list: + video_processor_dict = self.video_processor_dict.copy() + video_processing = video_processing_class(**video_processor_dict) + + prev_num_frames = self.video_processor_tester.num_frames + self.video_processor_tester.num_frames = 8 + prev_min_resolution = getattr(self.video_processor_tester, "min_resolution", None) + prev_max_resolution = getattr(self.video_processor_tester, "max_resolution", None) + self.video_processor_tester.min_resolution = 56 + self.video_processor_tester.max_resolution = 112 + + video_inputs = self.video_processor_tester.prepare_video_inputs( + equal_resolution=False, + return_tensors="torch", + ) + + metadata = [[{"total_num_frames": 8, "fps": 4}]] + batched_metadata = metadata * len(video_inputs) + + encoded_videos = video_processing(video_inputs[0], return_tensors="pt", video_metadata=metadata)[ + self.input_name + ] + encoded_videos_batched = video_processing( + video_inputs, return_tensors="pt", video_metadata=batched_metadata + )[self.input_name] + + self.assertIsNotNone(encoded_videos) + self.assertIsNotNone(encoded_videos_batched) + self.assertEqual(len(encoded_videos.shape), 2) + self.assertEqual(len(encoded_videos_batched.shape), 2) + + with self.assertRaises(ValueError): + video_processing(video_inputs[0], return_tensors="pt")[self.input_name] + + self.video_processor_tester.num_frames = prev_num_frames + if prev_min_resolution is not None: + self.video_processor_tester.min_resolution = prev_min_resolution + if prev_max_resolution is not None: + self.video_processor_tester.max_resolution = prev_max_resolution From bf860ed5d514cda7e4dfd231140ed306f487a409 Mon Sep 17 00:00:00 2001 From: zRzRzRzRzRzRzR <2448370773@qq.com> Date: Thu, 6 Nov 2025 22:05:18 +0800 Subject: [PATCH 03/66] add --- src/transformers/models/auto/tokenization_auto.py | 1 + src/transformers/models/auto/video_processing_auto.py | 1 + 2 files changed, 2 insertions(+) diff --git a/src/transformers/models/auto/tokenization_auto.py b/src/transformers/models/auto/tokenization_auto.py index a861aee12c57..68840ade7f55 100644 --- a/src/transformers/models/auto/tokenization_auto.py +++ b/src/transformers/models/auto/tokenization_auto.py @@ -301,6 +301,7 @@ ("git", ("BertTokenizer", "BertTokenizerFast" if is_tokenizers_available() else None)), ("glm", (None, "PreTrainedTokenizerFast" if is_tokenizers_available() else None)), ("glm4", (None, "PreTrainedTokenizerFast" if is_tokenizers_available() else None)), + ("glm46v", (None, "PreTrainedTokenizerFast" if is_tokenizers_available() else None)), ("glm4_moe", (None, "PreTrainedTokenizerFast" if is_tokenizers_available() else None)), ("glm4v", (None, "PreTrainedTokenizerFast" if is_tokenizers_available() else None)), ("glm4v_moe", (None, "PreTrainedTokenizerFast" if is_tokenizers_available() else None)), diff --git a/src/transformers/models/auto/video_processing_auto.py b/src/transformers/models/auto/video_processing_auto.py index d9ac45af912f..2de6c79f9ead 100644 --- a/src/transformers/models/auto/video_processing_auto.py +++ b/src/transformers/models/auto/video_processing_auto.py @@ -45,6 +45,7 @@ else: VIDEO_PROCESSOR_MAPPING_NAMES = OrderedDict( [ + ("glm46v", "Glm46vVideoProcessor"), ("glm4v", "Glm4vVideoProcessor"), ("instructblip", "InstructBlipVideoVideoProcessor"), ("instructblipvideo", "InstructBlipVideoVideoProcessor"), From 480caa48d8d8d9a35eea71b5477b47fbac77879c Mon Sep 17 00:00:00 2001 From: zRzRzRzRzRzRzR <2448370773@qq.com> Date: Fri, 7 Nov 2025 11:13:47 +0400 Subject: [PATCH 04/66] Update video_processing_glm46v.py --- src/transformers/models/glm46v/video_processing_glm46v.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/transformers/models/glm46v/video_processing_glm46v.py b/src/transformers/models/glm46v/video_processing_glm46v.py index 95bc07f560ad..627f77b0aa2a 100644 --- a/src/transformers/models/glm46v/video_processing_glm46v.py +++ b/src/transformers/models/glm46v/video_processing_glm46v.py @@ -32,7 +32,7 @@ from ...utils import TensorType, add_start_docstrings from ...video_processing_utils import BASE_VIDEO_PROCESSOR_DOCSTRING, BaseVideoProcessor from ...video_utils import VideoMetadata, group_videos_by_shape, reorder_videos -from .image_processing_glm4v import smart_resize +from ..glm4v.image_processing_glm4v import smart_resize class Glm46vVideoProcessorInitKwargs(VideosKwargs, total=False): From fd7a3f06aa1d04b5ec742fbc684af60aeb404aea Mon Sep 17 00:00:00 2001 From: zRzRzRzRzRzRzR <2448370773@qq.com> Date: Fri, 7 Nov 2025 15:13:39 +0400 Subject: [PATCH 05/66] update doc --- docs/source/en/model_doc/glm46v.md | 11 ++++++++++- src/transformers/models/glm46v/modular_glm46v.py | 1 + src/transformers/models/glm46v/processing_glm46v.py | 4 +++- 3 files changed, 14 insertions(+), 2 deletions(-) diff --git a/docs/source/en/model_doc/glm46v.md b/docs/source/en/model_doc/glm46v.md index 2320c9072839..eff8f9e19ab7 100644 --- a/docs/source/en/model_doc/glm46v.md +++ b/docs/source/en/model_doc/glm46v.md @@ -65,4 +65,13 @@ The original code can be found [here](). ## Glm46VTextModel [[autodoc]] Glm46VTextModel - - forward \ No newline at end of file + - forward + +## Glm4vVideoProcessor + +[[autodoc]] Glm46vVideoProcessor + - preprocess + +## Glm46vProcessor + +[[autodoc]] Glm46vProcessor \ No newline at end of file diff --git a/src/transformers/models/glm46v/modular_glm46v.py b/src/transformers/models/glm46v/modular_glm46v.py index 377ab80bf9db..1f1504f172c9 100644 --- a/src/transformers/models/glm46v/modular_glm46v.py +++ b/src/transformers/models/glm46v/modular_glm46v.py @@ -138,4 +138,5 @@ def replace_frame_token_id(self, timestamp_sec): "Glm46VModel", "Glm46VPreTrainedModel", "Glm46VTextModel", + "Glm46vProcessor" ] diff --git a/src/transformers/models/glm46v/processing_glm46v.py b/src/transformers/models/glm46v/processing_glm46v.py index fd8b57618c14..b348d4952bae 100644 --- a/src/transformers/models/glm46v/processing_glm46v.py +++ b/src/transformers/models/glm46v/processing_glm46v.py @@ -191,7 +191,6 @@ def __call__( video_structure += frame_structure text[i] = text[i].replace(self.video_token, video_structure, 1) - print(text[i]) num_image_tokens = ( video_grid_thw[video_index].prod() // merge_length // video_grid_thw[video_index][0] ) @@ -281,3 +280,6 @@ def post_process_image_text_to_text( def replace_frame_token_id(self, timestamp_sec): return f"<|begin_of_image|>{self.image_token}<|end_of_image|>{round(timestamp_sec)} seconds" + + +__all__ = ["Glm46vProcessor"] From 0e1c22e4db2e997657ca243fc31d4e620093a865 Mon Sep 17 00:00:00 2001 From: zRzRzRzRzRzRzR <2448370773@qq.com> Date: Fri, 7 Nov 2025 15:17:34 +0400 Subject: [PATCH 06/66] Update modular_glm46v.py --- src/transformers/models/glm46v/modular_glm46v.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/transformers/models/glm46v/modular_glm46v.py b/src/transformers/models/glm46v/modular_glm46v.py index 1f1504f172c9..111de0a117b1 100644 --- a/src/transformers/models/glm46v/modular_glm46v.py +++ b/src/transformers/models/glm46v/modular_glm46v.py @@ -138,5 +138,5 @@ def replace_frame_token_id(self, timestamp_sec): "Glm46VModel", "Glm46VPreTrainedModel", "Glm46VTextModel", - "Glm46vProcessor" + "Glm46vProcessor", ] From 42d757f76f237fc67b80e33af8be51d06fe15689 Mon Sep 17 00:00:00 2001 From: zRzRzRzRzRzRzR <2448370773@qq.com> Date: Sun, 9 Nov 2025 16:07:46 +0400 Subject: [PATCH 07/66] 2 --- src/transformers/models/glm46v/modular_glm46v.py | 2 +- src/transformers/models/glm46v/processing_glm46v.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/transformers/models/glm46v/modular_glm46v.py b/src/transformers/models/glm46v/modular_glm46v.py index 111de0a117b1..ef3a3ce3bc21 100644 --- a/src/transformers/models/glm46v/modular_glm46v.py +++ b/src/transformers/models/glm46v/modular_glm46v.py @@ -128,7 +128,7 @@ class Glm46VForConditionalGeneration(Glm4vForConditionalGeneration): class Glm46vProcessor(Glm4vProcessor): def replace_frame_token_id(self, timestamp_sec): - return f"<|begin_of_image|>{self.image_token}<|end_of_image|>{round(timestamp_sec)} seconds" + return f"<|begin_of_image|>{self.image_token}<|end_of_image|>{timestamp_sec:.1f} seconds" __all__ = [ diff --git a/src/transformers/models/glm46v/processing_glm46v.py b/src/transformers/models/glm46v/processing_glm46v.py index b348d4952bae..2b2b1d10bdd7 100644 --- a/src/transformers/models/glm46v/processing_glm46v.py +++ b/src/transformers/models/glm46v/processing_glm46v.py @@ -279,7 +279,7 @@ def post_process_image_text_to_text( ) def replace_frame_token_id(self, timestamp_sec): - return f"<|begin_of_image|>{self.image_token}<|end_of_image|>{round(timestamp_sec)} seconds" + return f"<|begin_of_image|>{self.image_token}<|end_of_image|>{timestamp_sec:.1f} seconds" __all__ = ["Glm46vProcessor"] From 6f5aa1a87a08b3e10f7acd8e32333aa030d3ae97 Mon Sep 17 00:00:00 2001 From: zRzRzRzRzRzRzR <2448370773@qq.com> Date: Tue, 11 Nov 2025 11:03:00 +0800 Subject: [PATCH 08/66] Update processing_glm46v.py --- src/transformers/models/glm46v/processing_glm46v.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/transformers/models/glm46v/processing_glm46v.py b/src/transformers/models/glm46v/processing_glm46v.py index 2b2b1d10bdd7..5042b68eb134 100644 --- a/src/transformers/models/glm46v/processing_glm46v.py +++ b/src/transformers/models/glm46v/processing_glm46v.py @@ -136,7 +136,7 @@ def __call__( if videos is not None: videos_inputs = self.video_processor(videos=videos, **output_kwargs["videos_kwargs"]) # If user has not requested video metadata, pop it - if "return_metadata" not in kwargs: + if not kwargs.get("return_metadata"): video_metadata = videos_inputs.pop("video_metadata") else: video_metadata = videos_inputs["video_metadata"] From a43c606bbb77f79c0d92eb5eb6500336d23f9956 Mon Sep 17 00:00:00 2001 From: zRzRzRzRzRzRzR <2448370773@qq.com> Date: Tue, 11 Nov 2025 11:06:16 +0800 Subject: [PATCH 09/66] 21 --- src/transformers/models/glm46v/processing_glm46v.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/src/transformers/models/glm46v/processing_glm46v.py b/src/transformers/models/glm46v/processing_glm46v.py index 5042b68eb134..4f259249a12a 100644 --- a/src/transformers/models/glm46v/processing_glm46v.py +++ b/src/transformers/models/glm46v/processing_glm46v.py @@ -60,12 +60,6 @@ class Glm46vProcessor(ProcessorMixin): in a chat into a tokenizable string. """ - attributes = ["image_processor", "tokenizer", "video_processor"] - image_processor_class = "AutoImageProcessor" - video_processor_class = "AutoVideoProcessor" - - tokenizer_class = ("PreTrainedTokenizer", "PreTrainedTokenizerFast") - def __init__(self, image_processor=None, tokenizer=None, video_processor=None, chat_template=None, **kwargs): self.image_token = "<|image|>" if not hasattr(tokenizer, "image_token") else tokenizer.image_token self.video_token = "<|video|>" if not hasattr(tokenizer, "video_token") else tokenizer.video_token From a242652963199bba54d698cedc9945897fa6a444 Mon Sep 17 00:00:00 2001 From: zRzRzRzRzRzRzR <2448370773@qq.com> Date: Tue, 11 Nov 2025 12:12:02 +0800 Subject: [PATCH 10/66] Update check_repo.py --- utils/check_repo.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/utils/check_repo.py b/utils/check_repo.py index e01f72d74fc9..b6518d8e6b4f 100644 --- a/utils/check_repo.py +++ b/utils/check_repo.py @@ -94,8 +94,7 @@ "AriaTextModel", "Phi4MultimodalAudioModel", "Phi4MultimodalVisionModel", - "Glm4vVisionModel", - "Glm4vMoeVisionModel", + "Glm46VVisionModel", "EvollaSaProtPreTrainedModel", "BltLocalEncoder", # Building part of bigger (tested) model. Tested implicitly through BLTForCausalLM. "BltLocalDecoder", # Building part of bigger (tested) model. Tested implicitly through BLTForCausalLM. From ce596eb0cb2c27b03640a6bf81dea48a6802237f Mon Sep 17 00:00:00 2001 From: zRzRzRzRzRzRzR <2448370773@qq.com> Date: Tue, 11 Nov 2025 12:22:06 +0800 Subject: [PATCH 11/66] Update check_repo.py --- utils/check_repo.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/utils/check_repo.py b/utils/check_repo.py index b6518d8e6b4f..49995257da94 100644 --- a/utils/check_repo.py +++ b/utils/check_repo.py @@ -94,6 +94,8 @@ "AriaTextModel", "Phi4MultimodalAudioModel", "Phi4MultimodalVisionModel", + "Glm4vVisionModel", + "Glm4vMoeVisionModel", "Glm46VVisionModel", "EvollaSaProtPreTrainedModel", "BltLocalEncoder", # Building part of bigger (tested) model. Tested implicitly through BLTForCausalLM. From 559fcf8191bd106ac7e15ff8a1bca902b2c3022a Mon Sep 17 00:00:00 2001 From: zRzRzRzRzRzRzR <2448370773@qq.com> Date: Tue, 11 Nov 2025 12:32:58 +0800 Subject: [PATCH 12/66] Update test_processor_glm46v.py --- tests/models/glm46v/test_processor_glm46v.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/models/glm46v/test_processor_glm46v.py b/tests/models/glm46v/test_processor_glm46v.py index 8695bba9886d..f0b4a6c405bf 100644 --- a/tests/models/glm46v/test_processor_glm46v.py +++ b/tests/models/glm46v/test_processor_glm46v.py @@ -78,7 +78,7 @@ def _test_apply_chat_template( if processor.chat_template is None: self.skipTest("Processor has no chat template") - if processor_name not in self.processor_class.attributes: + if processor_name not in self.processor_class.get_attributes(): self.skipTest(f"{processor_name} attribute not present in {self.processor_class}") batch_messages = [ From 513c2cc7f33d9720caf1517ea3a3bb5fcd2d857b Mon Sep 17 00:00:00 2001 From: zRzRzRzRzRzRzR <2448370773@qq.com> Date: Tue, 11 Nov 2025 13:54:10 +0800 Subject: [PATCH 13/66] Update modeling_auto.py --- src/transformers/models/auto/modeling_auto.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/transformers/models/auto/modeling_auto.py b/src/transformers/models/auto/modeling_auto.py index 26ecdaad0bdf..b7c30bb48700 100644 --- a/src/transformers/models/auto/modeling_auto.py +++ b/src/transformers/models/auto/modeling_auto.py @@ -179,6 +179,7 @@ class _BaseModelWithGenerate(PreTrainedModel, GenerationMixin): ("glm4v_moe", "Glm4vMoeModel"), ("glm4v_moe_text", "Glm4vMoeTextModel"), ("glm4v_text", "Glm4vTextModel"), + ("glm46v_text", "Glm46VTextModel"), ("glpn", "GLPNModel"), ("got_ocr2", "GotOcr2Model"), ("gpt-sw3", "GPT2Model"), From d6e966e85f1bd7ab2a0e75f80abe78517e2805ba Mon Sep 17 00:00:00 2001 From: zRzRzRzRzRzRzR <2448370773@qq.com> Date: Tue, 11 Nov 2025 14:10:12 +0800 Subject: [PATCH 14/66] update --- src/transformers/models/auto/modeling_auto.py | 2 +- utils/check_repo.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/transformers/models/auto/modeling_auto.py b/src/transformers/models/auto/modeling_auto.py index b7c30bb48700..b41126d87c13 100644 --- a/src/transformers/models/auto/modeling_auto.py +++ b/src/transformers/models/auto/modeling_auto.py @@ -174,12 +174,12 @@ class _BaseModelWithGenerate(PreTrainedModel, GenerationMixin): ("glm", "GlmModel"), ("glm4", "Glm4Model"), ("glm46v", "Glm46VModel"), + ("glm46v_text", "Glm46VTextModel"), ("glm4_moe", "Glm4MoeModel"), ("glm4v", "Glm4vModel"), ("glm4v_moe", "Glm4vMoeModel"), ("glm4v_moe_text", "Glm4vMoeTextModel"), ("glm4v_text", "Glm4vTextModel"), - ("glm46v_text", "Glm46VTextModel"), ("glpn", "GLPNModel"), ("got_ocr2", "GotOcr2Model"), ("gpt-sw3", "GPT2Model"), diff --git a/utils/check_repo.py b/utils/check_repo.py index 49995257da94..a13cdba44afe 100644 --- a/utils/check_repo.py +++ b/utils/check_repo.py @@ -178,6 +178,7 @@ "Emu3VQVAE", # Building part of bigger (tested) model "Emu3TextModel", # Building part of bigger (tested) model "Glm4vTextModel", # Building part of bigger (tested) model + "Glm46VTextModel", # Building part of bigger (tested) model "Glm4vMoeTextModel", # Building part of bigger (tested) model "Qwen2VLTextModel", # Building part of bigger (tested) model "Qwen2_5_VLTextModel", # Building part of bigger (tested) model From 275ebfeef85cf6377f08b863a1a991a85b8cbe7f Mon Sep 17 00:00:00 2001 From: zRzRzRzRzRzRzR <2448370773@qq.com> Date: Tue, 11 Nov 2025 16:37:42 +0800 Subject: [PATCH 15/66] Update glm46v.md --- docs/source/en/model_doc/glm46v.md | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/docs/source/en/model_doc/glm46v.md b/docs/source/en/model_doc/glm46v.md index eff8f9e19ab7..328789ed5ea1 100644 --- a/docs/source/en/model_doc/glm46v.md +++ b/docs/source/en/model_doc/glm46v.md @@ -62,16 +62,21 @@ The original code can be found [here](). [[autodoc]] Glm46VPreTrainedModel - forward -## Glm46VTextModel - -[[autodoc]] Glm46VTextModel - - forward - -## Glm4vVideoProcessor +## Glm46vVideoProcessor [[autodoc]] Glm46vVideoProcessor - preprocess ## Glm46vProcessor -[[autodoc]] Glm46vProcessor \ No newline at end of file +[[autodoc]] Glm46vProcessor + +## Glm46VTextModel + +[[autodoc]] Glm46VTextModel + - forward + +## Glm46VForConditionalGeneration + +[[autodoc]] Glm46VForConditionalGeneration + - forward \ No newline at end of file From 69915648a293c4f9a35c1c8dc267b31f0aeec428 Mon Sep 17 00:00:00 2001 From: zRzRzRzRzRzRzR <2448370773@qq.com> Date: Tue, 11 Nov 2025 16:42:10 +0800 Subject: [PATCH 16/66] Update configuration_auto.py --- src/transformers/models/auto/configuration_auto.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/transformers/models/auto/configuration_auto.py b/src/transformers/models/auto/configuration_auto.py index 9801e36f410f..792de55b7e14 100644 --- a/src/transformers/models/auto/configuration_auto.py +++ b/src/transformers/models/auto/configuration_auto.py @@ -171,6 +171,7 @@ ("glm", "GlmConfig"), ("glm4", "Glm4Config"), ("glm46v", "Glm46VConfig"), + ("glm46v_text", "Glm46VTextModel"), ("glm4_moe", "Glm4MoeConfig"), ("glm4v", "Glm4vConfig"), ("glm4v_moe", "Glm4vMoeConfig"), From 3dff2160e19b57f903437e349f9d494bae243ce7 Mon Sep 17 00:00:00 2001 From: zRzRzRzRzRzRzR <2448370773@qq.com> Date: Tue, 11 Nov 2025 16:55:02 +0800 Subject: [PATCH 17/66] 2 --- src/transformers/models/auto/configuration_auto.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/transformers/models/auto/configuration_auto.py b/src/transformers/models/auto/configuration_auto.py index 792de55b7e14..5fee8f070d20 100644 --- a/src/transformers/models/auto/configuration_auto.py +++ b/src/transformers/models/auto/configuration_auto.py @@ -171,7 +171,7 @@ ("glm", "GlmConfig"), ("glm4", "Glm4Config"), ("glm46v", "Glm46VConfig"), - ("glm46v_text", "Glm46VTextModel"), + ("glm46v_text", "Glm46VTextConfig"), ("glm4_moe", "Glm4MoeConfig"), ("glm4v", "Glm4vConfig"), ("glm4v_moe", "Glm4vMoeConfig"), From f9546bdc87654007723505a7936d1d4a42e3474b Mon Sep 17 00:00:00 2001 From: zRzRzRzRzRzRzR <2448370773@qq.com> Date: Tue, 11 Nov 2025 17:17:14 +0800 Subject: [PATCH 18/66] update with glm46v import --- src/transformers/models/auto/configuration_auto.py | 2 ++ utils/models_to_deprecate.py | 1 + 2 files changed, 3 insertions(+) diff --git a/src/transformers/models/auto/configuration_auto.py b/src/transformers/models/auto/configuration_auto.py index 5fee8f070d20..0db88567983d 100644 --- a/src/transformers/models/auto/configuration_auto.py +++ b/src/transformers/models/auto/configuration_auto.py @@ -619,6 +619,7 @@ ("glm", "GLM"), ("glm4", "GLM4"), ("glm46v", "Glm46V"), + ("glm46v_text", "GLM46V"), ("glm4_moe", "Glm4MoE"), ("glm4v", "GLM4V"), ("glm4v_moe", "GLM4VMOE"), @@ -982,6 +983,7 @@ ("gemma3n_text", "gemma3n"), ("gemma3n_vision", "gemma3n"), ("glm4v_text", "glm4v"), + ("glm46v_text", "glm46v"), ("glm4v_moe_text", "glm4v_moe"), ("idefics3_vision", "idefics3"), ("siglip_vision_model", "siglip"), diff --git a/utils/models_to_deprecate.py b/utils/models_to_deprecate.py index c3c4eaf8e4d8..1cc31eea29e2 100644 --- a/utils/models_to_deprecate.py +++ b/utils/models_to_deprecate.py @@ -94,6 +94,7 @@ "gpt2": ["cpm", "dialogpt", "gpt-sw3", "megatron_gpt2"], "glm4v_moe": ["glm4v_moe_text"], "glm4v": ["glm4v_text"], + "glm46v": ["glm46v_text"], "idefics3": ["idefics3_vision"], "internvl": ["internvl_vision"], "layoutlmv2": ["layoutxlm"], From 6e5ae0325800ecc6c1f8fe5aac4e84ef019e1716 Mon Sep 17 00:00:00 2001 From: zRzRzRzRzRzRzR <2448370773@qq.com> Date: Wed, 12 Nov 2025 17:33:29 +0800 Subject: [PATCH 19/66] uppercase --- docs/source/en/model_doc/glm46v.md | 8 ++++---- src/transformers/models/auto/processing_auto.py | 2 +- .../models/auto/video_processing_auto.py | 2 +- src/transformers/models/glm46v/modular_glm46v.py | 4 ++-- src/transformers/models/glm46v/processing_glm46v.py | 4 ++-- .../models/glm46v/video_processing_glm46v.py | 12 ++++++------ tests/models/glm46v/test_processor_glm46v.py | 8 ++++---- tests/models/glm46v/test_video_processing_glm46v.py | 10 +++++----- 8 files changed, 25 insertions(+), 25 deletions(-) diff --git a/docs/source/en/model_doc/glm46v.md b/docs/source/en/model_doc/glm46v.md index 328789ed5ea1..8fc8865cb865 100644 --- a/docs/source/en/model_doc/glm46v.md +++ b/docs/source/en/model_doc/glm46v.md @@ -62,14 +62,14 @@ The original code can be found [here](). [[autodoc]] Glm46VPreTrainedModel - forward -## Glm46vVideoProcessor +## Glm46VVideoProcessor -[[autodoc]] Glm46vVideoProcessor +[[autodoc]] Glm46VVideoProcessor - preprocess -## Glm46vProcessor +## Glm46VProcessor -[[autodoc]] Glm46vProcessor +[[autodoc]] Glm46VProcessor ## Glm46VTextModel diff --git a/src/transformers/models/auto/processing_auto.py b/src/transformers/models/auto/processing_auto.py index 3e95cff9155e..9b9a47cf7bfd 100644 --- a/src/transformers/models/auto/processing_auto.py +++ b/src/transformers/models/auto/processing_auto.py @@ -74,7 +74,7 @@ ("gemma3", "Gemma3Processor"), ("gemma3n", "Gemma3nProcessor"), ("git", "GitProcessor"), - ("glm46v", "Glm46vProcessor"), + ("glm46v", "Glm46VProcessor"), ("glm4v", "Glm4vProcessor"), ("glm4v_moe", "Glm4vProcessor"), ("got_ocr2", "GotOcr2Processor"), diff --git a/src/transformers/models/auto/video_processing_auto.py b/src/transformers/models/auto/video_processing_auto.py index e0275d08f063..67798801bb54 100644 --- a/src/transformers/models/auto/video_processing_auto.py +++ b/src/transformers/models/auto/video_processing_auto.py @@ -45,7 +45,7 @@ else: VIDEO_PROCESSOR_MAPPING_NAMES = OrderedDict( [ - ("glm46v", "Glm46vVideoProcessor"), + ("glm46v", "Glm46VVideoProcessor"), ("glm4v", "Glm4vVideoProcessor"), ("instructblip", "InstructBlipVideoVideoProcessor"), ("instructblipvideo", "InstructBlipVideoVideoProcessor"), diff --git a/src/transformers/models/glm46v/modular_glm46v.py b/src/transformers/models/glm46v/modular_glm46v.py index ef3a3ce3bc21..6c6ca73ae22b 100644 --- a/src/transformers/models/glm46v/modular_glm46v.py +++ b/src/transformers/models/glm46v/modular_glm46v.py @@ -126,7 +126,7 @@ class Glm46VForConditionalGeneration(Glm4vForConditionalGeneration): pass -class Glm46vProcessor(Glm4vProcessor): +class Glm46VProcessor(Glm4vProcessor): def replace_frame_token_id(self, timestamp_sec): return f"<|begin_of_image|>{self.image_token}<|end_of_image|>{timestamp_sec:.1f} seconds" @@ -138,5 +138,5 @@ def replace_frame_token_id(self, timestamp_sec): "Glm46VModel", "Glm46VPreTrainedModel", "Glm46VTextModel", - "Glm46vProcessor", + "Glm46VProcessor", ] diff --git a/src/transformers/models/glm46v/processing_glm46v.py b/src/transformers/models/glm46v/processing_glm46v.py index 4f259249a12a..25a0aa34538b 100644 --- a/src/transformers/models/glm46v/processing_glm46v.py +++ b/src/transformers/models/glm46v/processing_glm46v.py @@ -45,7 +45,7 @@ class Glm46VProcessorKwargs(ProcessingKwargs, total=False): } -class Glm46vProcessor(ProcessorMixin): +class Glm46VProcessor(ProcessorMixin): r""" Constructs a GLM-4V processor which wraps a GLM-4V image processor and a GLM-4 tokenizer into a single processor. [`~Glm46VProcessor.__call__`] and [`~Glm46VProcessor.decode`] for more information. @@ -276,4 +276,4 @@ def replace_frame_token_id(self, timestamp_sec): return f"<|begin_of_image|>{self.image_token}<|end_of_image|>{timestamp_sec:.1f} seconds" -__all__ = ["Glm46vProcessor"] +__all__ = ["Glm46VProcessor"] diff --git a/src/transformers/models/glm46v/video_processing_glm46v.py b/src/transformers/models/glm46v/video_processing_glm46v.py index 627f77b0aa2a..0b5308a62b40 100644 --- a/src/transformers/models/glm46v/video_processing_glm46v.py +++ b/src/transformers/models/glm46v/video_processing_glm46v.py @@ -35,7 +35,7 @@ from ..glm4v.image_processing_glm4v import smart_resize -class Glm46vVideoProcessorInitKwargs(VideosKwargs, total=False): +class Glm46VVideoProcessorInitKwargs(VideosKwargs, total=False): max_image_size: dict[str, int] patch_size: int temporal_patch_size: int @@ -55,7 +55,7 @@ class Glm46vVideoProcessorInitKwargs(VideosKwargs, total=False): The merge size of the vision encoder to llm encoder. """, ) -class Glm46vVideoProcessor(BaseVideoProcessor): +class Glm46VVideoProcessor(BaseVideoProcessor): resample = PILImageResampling.BICUBIC size = {"shortest_edge": 112 * 112, "longest_edge": 28 * 28 * 2 * 30000} max_image_size = {"longest_edge": 28 * 28 * 2 * 30000} @@ -70,13 +70,13 @@ class Glm46vVideoProcessor(BaseVideoProcessor): temporal_patch_size = 2 max_duration = 300 merge_size = 2 - valid_kwargs = Glm46vVideoProcessorInitKwargs + valid_kwargs = Glm46VVideoProcessorInitKwargs num_frames = 16 fps = 2 model_input_names = ["pixel_values_videos", "video_grid_thw"] - def __init__(self, **kwargs: Unpack[Glm46vVideoProcessorInitKwargs]): + def __init__(self, **kwargs: Unpack[Glm46VVideoProcessorInitKwargs]): super().__init__(**kwargs) if self.size is not None and ( self.size.get("shortest_edge", None) is None or self.size.get("longest_edge", None) is None @@ -105,7 +105,7 @@ def sample_frames( ): if metadata is None or getattr(metadata, "fps", None) is None: raise ValueError( - "Asked to sample frames per second but no video metadata was provided which is required when sampling in Glm46v. " + "Asked to sample frames per second but no video metadata was provided which is required when sampling in Glm46V. " "Please pass in `VideoMetadata` object or set `do_sample_frames=False`" ) @@ -267,4 +267,4 @@ def _preprocess( return BatchFeature(data=data, tensor_type=return_tensors) -__all__ = ["Glm46vVideoProcessor"] +__all__ = ["Glm46VVideoProcessor"] diff --git a/tests/models/glm46v/test_processor_glm46v.py b/tests/models/glm46v/test_processor_glm46v.py index f0b4a6c405bf..4e9fb4449d67 100644 --- a/tests/models/glm46v/test_processor_glm46v.py +++ b/tests/models/glm46v/test_processor_glm46v.py @@ -27,7 +27,7 @@ if is_vision_available(): - from transformers import Glm46vProcessor + from transformers import Glm46VProcessor if is_torch_available(): import torch @@ -35,13 +35,13 @@ @require_vision @require_torch -class Glm46vProcessorTest(ProcessorTesterMixin, unittest.TestCase): - processor_class = Glm46vProcessor +class Glm46VProcessorTest(ProcessorTesterMixin, unittest.TestCase): + processor_class = Glm46VProcessor @classmethod def setUpClass(cls): cls.tmpdirname = tempfile.mkdtemp() - processor = Glm46vProcessor.from_pretrained( + processor = Glm46VProcessor.from_pretrained( "THUDM/GLM-4.1V-9B-Thinking", patch_size=4, size={"shortest_edge": 12 * 12, "longest_edge": 18 * 18} ) processor.save_pretrained(cls.tmpdirname) diff --git a/tests/models/glm46v/test_video_processing_glm46v.py b/tests/models/glm46v/test_video_processing_glm46v.py index e15809ad8790..fc62901a71be 100644 --- a/tests/models/glm46v/test_video_processing_glm46v.py +++ b/tests/models/glm46v/test_video_processing_glm46v.py @@ -29,11 +29,11 @@ if is_vision_available(): if is_torchvision_available(): - from transformers import Glm46vVideoProcessor + from transformers import Glm46VVideoProcessor from transformers.models.glm46v.video_processing_glm46v import smart_resize -class Glm46vVideoProcessingTester: +class Glm46VVideoProcessingTester: def __init__( self, parent, @@ -151,13 +151,13 @@ def prepare_video_inputs(self, equal_resolution=False, return_tensors="pil"): @require_torch @require_vision -class Glm46vVideoProcessingTest(VideoProcessingTestMixin, unittest.TestCase): - fast_video_processing_class = Glm46vVideoProcessor if is_torchvision_available() else None +class Glm46VVideoProcessingTest(VideoProcessingTestMixin, unittest.TestCase): + fast_video_processing_class = Glm46VVideoProcessor if is_torchvision_available() else None input_name = "pixel_values_videos" def setUp(self): super().setUp() - self.video_processor_tester = Glm46vVideoProcessingTester(self) + self.video_processor_tester = Glm46VVideoProcessingTester(self) @property def video_processor_dict(self): From 1e2535d3927ca3fb223425af2b681974ea5bc753 Mon Sep 17 00:00:00 2001 From: zRzRzRzRzRzRzR <2448370773@qq.com> Date: Wed, 12 Nov 2025 17:48:05 +0800 Subject: [PATCH 20/66] upload --- .../models/glm46v/configuration_glm46v.py | 194 +-- .../models/glm46v/modeling_glm46v.py | 1163 ++++++----------- .../models/glm46v/modular_glm46v.py | 89 +- 3 files changed, 498 insertions(+), 948 deletions(-) diff --git a/src/transformers/models/glm46v/configuration_glm46v.py b/src/transformers/models/glm46v/configuration_glm46v.py index 1dad5fdcbd60..ecdaf0ac075a 100644 --- a/src/transformers/models/glm46v/configuration_glm46v.py +++ b/src/transformers/models/glm46v/configuration_glm46v.py @@ -25,103 +25,6 @@ from ...modeling_rope_utils import RopeParameters, rope_config_validation, standardize_rope_params -class Glm46VVisionConfig(PreTrainedConfig): - r""" - This is the configuration class to store the configuration of a [`Glm46VVisionModel`]. It is used to instantiate an Glm46VVisionModel - model according to the specified arguments, defining the model architecture. Instantiating a configuration with the defaults will yield - a similar configuration to that of - GLM-4.1V-9B-Thinking [THUDM/GLM-4.1V-9B-Thinking](https://huggingface.co/THUDM/GLM-4.1V-9B-Thinking). - - Args: - hidden_size (`int`, *optional*, defaults to 1536): - Dimensionality of the encoder layers and the pooler layer. - depth (`int`, *optional*, defaults to 24): - Number of layers (depth) in the model. - attention_bias (`bool`, *optional*, defaults to `False`): - Whether to add a bias to the queries, keys and values. - intermediate_size (`int`, *optional*, defaults to 13696): - Dimensionality of the "intermediate" (i.e., feed-forward) layer in the Transformer encoder. - hidden_act (`str` or `function`, *optional*, defaults to `"selu"`): - The non-linear activation function (function or string) in the encoder and pooler. If string, `"gelu"`, - `"relu"`, `"selu"` and `"gelu_new"` are supported. - hidden_dropout_prob (`float`, *optional*, defaults to 0.0): - The dropout probability for all fully connected layers in the embeddings, encoder, and pooler. - attention_dropout (`float`, *optional*, defaults to 0.0): - Dropout probability for attention weights. - projection_dropout (`float`, *optional*, defaults to 0.0): - Dropout probability for the projection layer. - initializer_range (`float`, *optional*, defaults to 0.02): - The standard deviation of the truncated_normal_initializer for initializing all weight matrices. - image_size (`int` or `list[int]`, *optional*, defaults to `[336, 336]`): - The size (resolution) of each image. - patch_size (`int`, *optional*, defaults to `14`): - The size (resolution) of each patch. - num_channels (`int`, *optional*, defaults to 3): - The number of input channels. - out_hidden_size (`int`, *optional*, defaults to 4096): - The output hidden size of the vision model. - rms_norm_eps (`float`, *optional*, defaults to 1e-05): - The epsilon used by the rms normalization layers. - spatial_merge_size (`int`, *optional*, defaults to 2): - The size used for merging spatial dimensions. - temporal_patch_size (`int`, *optional*, defaults to 2): - The size used for patches along the temporal dimension. - Example: - - ```python - >>> from transformers import Glm46VVisionConfig, Glm46VVisionModel - - >>> # Initializing a Glm46VVisionConfig GLM-4.1V-9B style configuration - >>> configuration = Glm46VVisionConfig() - - >>> # Initializing a model (with random weights) from the GLM-4.1V-9B configuration - >>> model = Glm46VVisionModel(configuration) - - >>> # Accessing the model configuration - >>> configuration = model.config - ```""" - - model_type = "glm46v" - base_config_key = "vision_config" - - def __init__( - self, - depth=24, - hidden_size=1536, - hidden_act="silu", - attention_bias=False, - attention_dropout=0.0, - num_heads=12, - in_channels=3, - image_size=336, - patch_size=14, - rms_norm_eps=1e-05, - spatial_merge_size=2, - temporal_patch_size=2, - out_hidden_size=4096, - intermediate_size=13696, - initializer_range=0.02, - **kwargs, - ): - super().__init__(**kwargs) - - self.depth = depth - self.hidden_size = hidden_size - self.hidden_act = hidden_act - self.num_heads = num_heads - self.in_channels = in_channels - self.image_size = image_size - self.patch_size = patch_size - self.spatial_merge_size = spatial_merge_size - self.temporal_patch_size = temporal_patch_size - self.out_hidden_size = out_hidden_size - self.intermediate_size = intermediate_size - self.initializer_range = initializer_range - self.rms_norm_eps = rms_norm_eps - self.attention_bias = attention_bias - self.attention_dropout = attention_dropout - - class Glm46VTextConfig(PreTrainedConfig): r""" This is the configuration class to store the configuration of a [`Glm46VModel`]. It is used to instantiate a @@ -257,6 +160,103 @@ def __init__( super().__init__(tie_word_embeddings=tie_word_embeddings, **kwargs) +class Glm46VVisionConfig(PreTrainedConfig): + r""" + This is the configuration class to store the configuration of a [`Glm46VVisionModel`]. It is used to instantiate an Glm46VVisionModel + model according to the specified arguments, defining the model architecture. Instantiating a configuration with the defaults will yield + a similar configuration to that of + GLM-4.1V-9B-Thinking [THUDM/GLM-4.1V-9B-Thinking](https://huggingface.co/THUDM/GLM-4.1V-9B-Thinking). + + Args: + hidden_size (`int`, *optional*, defaults to 1536): + Dimensionality of the encoder layers and the pooler layer. + depth (`int`, *optional*, defaults to 24): + Number of layers (depth) in the model. + attention_bias (`bool`, *optional*, defaults to `False`): + Whether to add a bias to the queries, keys and values. + intermediate_size (`int`, *optional*, defaults to 13696): + Dimensionality of the "intermediate" (i.e., feed-forward) layer in the Transformer encoder. + hidden_act (`str` or `function`, *optional*, defaults to `"selu"`): + The non-linear activation function (function or string) in the encoder and pooler. If string, `"gelu"`, + `"relu"`, `"selu"` and `"gelu_new"` are supported. + hidden_dropout_prob (`float`, *optional*, defaults to 0.0): + The dropout probability for all fully connected layers in the embeddings, encoder, and pooler. + attention_dropout (`float`, *optional*, defaults to 0.0): + Dropout probability for attention weights. + projection_dropout (`float`, *optional*, defaults to 0.0): + Dropout probability for the projection layer. + initializer_range (`float`, *optional*, defaults to 0.02): + The standard deviation of the truncated_normal_initializer for initializing all weight matrices. + image_size (`int` or `list[int]`, *optional*, defaults to `[336, 336]`): + The size (resolution) of each image. + patch_size (`int`, *optional*, defaults to `14`): + The size (resolution) of each patch. + num_channels (`int`, *optional*, defaults to 3): + The number of input channels. + out_hidden_size (`int`, *optional*, defaults to 4096): + The output hidden size of the vision model. + rms_norm_eps (`float`, *optional*, defaults to 1e-05): + The epsilon used by the rms normalization layers. + spatial_merge_size (`int`, *optional*, defaults to 2): + The size used for merging spatial dimensions. + temporal_patch_size (`int`, *optional*, defaults to 2): + The size used for patches along the temporal dimension. + Example: + + ```python + >>> from transformers import Glm46VVisionConfig, Glm46VVisionModel + + >>> # Initializing a Glm46VVisionConfig GLM-4.1V-9B style configuration + >>> configuration = Glm46VVisionConfig() + + >>> # Initializing a model (with random weights) from the GLM-4.1V-9B configuration + >>> model = Glm46VVisionModel(configuration) + + >>> # Accessing the model configuration + >>> configuration = model.config + ```""" + + model_type = "glm46v" + base_config_key = "vision_config" + + def __init__( + self, + depth=24, + hidden_size=1536, + hidden_act="silu", + attention_bias=False, + attention_dropout=0.0, + num_heads=12, + in_channels=3, + image_size=336, + patch_size=14, + rms_norm_eps=1e-05, + spatial_merge_size=2, + temporal_patch_size=2, + out_hidden_size=4096, + intermediate_size=13696, + initializer_range=0.02, + **kwargs, + ): + super().__init__(**kwargs) + + self.depth = depth + self.hidden_size = hidden_size + self.hidden_act = hidden_act + self.num_heads = num_heads + self.in_channels = in_channels + self.image_size = image_size + self.patch_size = patch_size + self.spatial_merge_size = spatial_merge_size + self.temporal_patch_size = temporal_patch_size + self.out_hidden_size = out_hidden_size + self.intermediate_size = intermediate_size + self.initializer_range = initializer_range + self.rms_norm_eps = rms_norm_eps + self.attention_bias = attention_bias + self.attention_dropout = attention_dropout + + class Glm46VConfig(PreTrainedConfig): r""" This is the configuration class to store the configuration of a [`Glm46VModel`]. It is used to instantiate a diff --git a/src/transformers/models/glm46v/modeling_glm46v.py b/src/transformers/models/glm46v/modeling_glm46v.py index b0361e5dc55a..60138e10f9b9 100644 --- a/src/transformers/models/glm46v/modeling_glm46v.py +++ b/src/transformers/models/glm46v/modeling_glm46v.py @@ -22,7 +22,7 @@ import itertools from collections.abc import Callable from dataclasses import dataclass -from typing import Any, Optional, Union +from typing import Optional, Union import torch import torch.nn as nn @@ -31,7 +31,6 @@ from ...activations import ACT2FN from ...cache_utils import Cache, DynamicCache -from ...generation import GenerationMixin from ...integrations import use_kernel_forward_from_hub from ...masking_utils import create_causal_mask from ...modeling_flash_attention_utils import FlashAttentionKwargs @@ -66,179 +65,6 @@ def extra_repr(self): return f"{tuple(self.weight.shape)}, eps={self.variance_epsilon}" -class Glm4VisionMlp(nn.Module): - def __init__(self, config, bias: bool = False): - super().__init__() - self.hidden_size = config.hidden_size - self.intermediate_size = config.out_hidden_size - self.gate_proj = nn.Linear(self.hidden_size, self.intermediate_size, bias=bias) - self.up_proj = nn.Linear(self.hidden_size, self.intermediate_size, bias=bias) - self.down_proj = nn.Linear(self.intermediate_size, self.hidden_size, bias=bias) - self.act_fn = ACT2FN[config.hidden_act] - - def forward(self, hidden_state): - return self.down_proj(self.act_fn(self.gate_proj(hidden_state)) * self.up_proj(hidden_state)) - - -class Glm46VVisionPatchEmbed(nn.Module): - def __init__(self, config: Glm46VVisionConfig) -> None: - super().__init__() - self.patch_size = config.patch_size - self.temporal_patch_size = config.temporal_patch_size - self.in_channels = config.in_channels - self.embed_dim = config.hidden_size - - kernel_size = [self.temporal_patch_size, self.patch_size, self.patch_size] - self.proj = nn.Conv3d(self.in_channels, self.embed_dim, kernel_size=kernel_size, stride=kernel_size) - - def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: - target_dtype = self.proj.weight.dtype - hidden_states = hidden_states.view( - -1, self.in_channels, self.temporal_patch_size, self.patch_size, self.patch_size - ) - hidden_states = self.proj(hidden_states.to(dtype=target_dtype)).view(-1, self.embed_dim) - return hidden_states - - -class Glm46VVisionRotaryEmbedding(nn.Module): - inv_freq: torch.Tensor # fix linting for `register_buffer` - - def __init__(self, dim: int, theta: float = 10000.0) -> None: - super().__init__() - inv_freq = 1.0 / (theta ** (torch.arange(0, dim, 2, dtype=torch.float) / dim)) - self.register_buffer("inv_freq", inv_freq, persistent=False) - - def forward(self, seqlen: int) -> torch.Tensor: - seq = torch.arange(seqlen, device=self.inv_freq.device, dtype=self.inv_freq.dtype) - freqs = torch.outer(seq, self.inv_freq) - return freqs - - -class Glm46VVisionPatchMerger(nn.Module): - def __init__(self, dim: int, context_dim: int, hidden_act: str, bias: bool = False) -> None: - super().__init__() - self.proj = nn.Linear(dim, dim, bias=bias) - self.post_projection_norm = LayerNorm(dim) - self.gate_proj = nn.Linear(dim, context_dim, bias=bias) - self.up_proj = nn.Linear(dim, context_dim, bias=bias) - self.down_proj = nn.Linear(context_dim, dim, bias=bias) - self.act1 = nn.GELU() - self.act_fn = ACT2FN[hidden_act] - - def forward(self, hidden_state: torch.Tensor) -> torch.Tensor: - hidden_state = self.proj(hidden_state) - hidden_state = self.act1(self.post_projection_norm(hidden_state)) - return self.down_proj(self.act_fn(self.gate_proj(hidden_state)) * self.up_proj(hidden_state)) - - -class Glm46VVisionEmbeddings(nn.Module): - def __init__(self, config: Glm46VVisionConfig): - super().__init__() - self.config = config - self.embed_dim = config.hidden_size - self.image_size = config.image_size - self.patch_size = config.patch_size - - self.num_patches = (self.image_size // self.patch_size) ** 2 - self.num_positions = self.num_patches - self.position_embedding = nn.Embedding(self.num_positions, self.embed_dim) - self.register_buffer("position_ids", torch.arange(self.num_positions).expand((1, -1)), persistent=False) - - def forward(self, embeddings, lengths, image_shapes, h_coords, w_coords) -> torch.Tensor: - """ - Forward pass with integrated position encoding adaptation using 2D interpolation. - - Args: - embeddings: Input embeddings tensor - lengths (torch.Tensor): Sequence lengths for each image in the batch. - image_shapes (torch.Tensor): Tensor of shape [batch_size, 3] representing the image shapes (t, h, w). - h_coords (torch.Tensor): Tensor of shape [total_seq] representing the h coordinate for each patch. - w_coords (torch.Tensor): Tensor of shape [total_seq] representing the w coordinate for each patch. - - Returns: - torch.Tensor: Embeddings with adapted position encoding added. - """ - # Get position embedding parameters - pos_embed_weight = self.position_embedding.weight - hidden_size = pos_embed_weight.shape[1] - total_seq = h_coords.shape[0] - device = pos_embed_weight.device - - # Move coordinates to correct device - h_coords, w_coords = h_coords.to(device), w_coords.to(device) - - # Handle empty sequence case - if total_seq == 0: - adapted_pos_embed = torch.empty(0, hidden_size, device=device, dtype=pos_embed_weight.dtype) - else: - # Convert inputs to tensors if needed - if isinstance(lengths, list): - lengths = torch.tensor(lengths, device=device, dtype=torch.long) - if not isinstance(image_shapes, torch.Tensor): - image_shapes = torch.tensor(image_shapes, device=device, dtype=torch.long) - - # Prepare 2D position embedding - orig_size_sq = pos_embed_weight.shape[0] - orig_size = int(orig_size_sq**0.5) - pos_embed_2d = ( - pos_embed_weight.view(orig_size, orig_size, hidden_size) - .permute(2, 0, 1) - .unsqueeze(0) - .to(device=device, dtype=torch.float32) - ) - - # Calculate target dimensions for each patch - target_h = torch.cat([image_shapes[i, 1].repeat(lengths[i]) for i in range(len(lengths))]).to( - device=device, dtype=torch.float32 - ) - target_w = torch.cat([image_shapes[i, 2].repeat(lengths[i]) for i in range(len(lengths))]).to( - device=device, dtype=torch.float32 - ) - - # Normalize coordinates to [-1, 1] range for grid_sample - h_coords = h_coords.to(device=device, dtype=torch.float32) - w_coords = w_coords.to(device=device, dtype=torch.float32) - norm_w = ((w_coords + 0.5) / target_w) * 2 - 1 - norm_h = ((h_coords + 0.5) / target_h) * 2 - 1 - - # Create sampling grid - grid = torch.stack((norm_w, norm_h), dim=-1).unsqueeze(0).unsqueeze(2) - - # Perform bicubic interpolation - interpolated_embed_fp32 = F.grid_sample( - pos_embed_2d, grid, mode="bicubic", align_corners=False, padding_mode="border" - ) - - # Reshape and convert back to original dtype - adapted_pos_embed_fp32 = interpolated_embed_fp32.squeeze(0).squeeze(-1).permute(1, 0) - adapted_pos_embed = adapted_pos_embed_fp32.to(pos_embed_weight.dtype).to(embeddings.device) - - # Add adapted position encoding to embeddings - embeddings = embeddings + adapted_pos_embed - return embeddings - - -def rotate_half(x): - """Rotates half the hidden dims of the input.""" - x1 = x[..., : x.shape[-1] // 2] - x2 = x[..., x.shape[-1] // 2 :] - return torch.cat((-x2, x1), dim=-1) - - -def apply_rotary_pos_emb_vision( - q: torch.Tensor, k: torch.Tensor, cos: torch.Tensor, sin: torch.Tensor -) -> tuple[torch.Tensor, torch.Tensor]: - orig_q_dtype = q.dtype - orig_k_dtype = k.dtype - q, k = q.float(), k.float() - cos, sin = cos.unsqueeze(-2).float(), sin.unsqueeze(-2).float() - q_embed = (q * cos) + (rotate_half(q) * sin) - k_embed = (k * cos) + (rotate_half(k) * sin) - q_embed = q_embed.to(orig_q_dtype) - k_embed = k_embed.to(orig_k_dtype) - return q_embed, k_embed - - def repeat_kv(hidden_states: torch.Tensor, n_rep: int) -> torch.Tensor: """ This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch, @@ -277,217 +103,24 @@ def eager_attention_forward( return attn_output, attn_weights -class Glm46VVisionAttention(nn.Module): - def __init__(self, config: Glm46VVisionConfig) -> None: - super().__init__() - self.dim = config.hidden_size - self.num_heads = config.num_heads - self.head_dim = self.dim // self.num_heads - self.num_key_value_groups = 1 # needed for eager attention - self.qkv = nn.Linear(config.hidden_size, config.hidden_size * 3, bias=config.attention_bias) - self.proj = nn.Linear(config.hidden_size, config.hidden_size, bias=False) - self.scaling = self.head_dim**-0.5 - self.config = config - self.attention_dropout = config.attention_dropout - self.is_causal = False - - def forward( - self, - hidden_states: torch.Tensor, - cu_seqlens: torch.Tensor, - rotary_pos_emb: Optional[torch.Tensor] = None, - position_embeddings: Optional[tuple[torch.Tensor, torch.Tensor]] = None, - **kwargs, - ) -> torch.Tensor: - seq_length = hidden_states.shape[0] - query_states, key_states, value_states = ( - self.qkv(hidden_states).reshape(seq_length, 3, self.num_heads, -1).permute(1, 0, 2, 3).unbind(0) - ) - cos, sin = position_embeddings - query_states, key_states = apply_rotary_pos_emb_vision(query_states, key_states, cos, sin) +def rotate_half_llm(x): + """Rotates half the hidden dims of the input.""" + x1 = x[..., 0::2] + x2 = x[..., 1::2] + return torch.stack((-x2, x1), dim=-1).flatten(-2) - query_states = query_states.transpose(0, 1).unsqueeze(0) - key_states = key_states.transpose(0, 1).unsqueeze(0) - value_states = value_states.transpose(0, 1).unsqueeze(0) - attention_interface: Callable = eager_attention_forward - if self.config._attn_implementation != "eager": - attention_interface = ALL_ATTENTION_FUNCTIONS[self.config._attn_implementation] +def apply_multimodal_rotary_pos_emb(q, k, cos, sin, mrope_section, unsqueeze_dim=1): + """Applies Rotary Position Embedding with Multimodal Sections to the query and key tensors (https://qwenlm.github.io/blog/qwen2-vl/). - if self.config._attn_implementation == "flash_attention_2": - # Flash Attention 2: Use cu_seqlens for variable length attention - max_seqlen = (cu_seqlens[1:] - cu_seqlens[:-1]).max() - attn_output, _ = attention_interface( - self, - query_states, - key_states, - value_states, - attention_mask=None, - scaling=self.scaling, - dropout=0.0 if not self.training else self.attention_dropout, - cu_seq_lens_q=cu_seqlens, - cu_seq_lens_k=cu_seqlens, - max_length_q=max_seqlen, - max_length_k=max_seqlen, - is_causal=False, - **kwargs, - ) - else: - # Other implementations: Process each chunk separately - lengths = cu_seqlens[1:] - cu_seqlens[:-1] - splits = [ - torch.split(tensor, lengths.tolist(), dim=2) for tensor in (query_states, key_states, value_states) - ] - - attn_outputs = [ - attention_interface( - self, - q, - k, - v, - attention_mask=None, - scaling=self.scaling, - dropout=0.0 if not self.training else self.attention_dropout, - is_causal=False, - **kwargs, - )[0] - for q, k, v in zip(*splits) - ] - attn_output = torch.cat(attn_outputs, dim=1) - - attn_output = attn_output.reshape(seq_length, -1).contiguous() - attn_output = self.proj(attn_output) - return attn_output - - -class Glm46VisionMlp(nn.Module): - def __init__(self, config, bias: bool = False): - super().__init__() - self.hidden_size = config.hidden_size - self.intermediate_size = config.out_hidden_size - self.gate_proj = nn.Linear(self.hidden_size, self.intermediate_size, bias=bias) - self.up_proj = nn.Linear(self.hidden_size, self.intermediate_size, bias=bias) - self.down_proj = nn.Linear(self.intermediate_size, self.hidden_size, bias=bias) - self.act_fn = ACT2FN[config.hidden_act] - - def forward(self, hidden_state): - return self.down_proj(self.act_fn(self.gate_proj(hidden_state)) * self.up_proj(hidden_state)) - - -class Glm46VVisionBlock(GradientCheckpointingLayer): - def __init__(self, config) -> None: - super().__init__() - self.norm1 = Glm46VRMSNorm(config.hidden_size, eps=config.rms_norm_eps) - self.norm2 = Glm46VRMSNorm(config.hidden_size, eps=config.rms_norm_eps) - self.attn = Glm46VVisionAttention(config) - self.mlp = Glm46VisionMlp(config, bias=False) - - def forward( - self, - hidden_states: torch.Tensor, - cu_seqlens: torch.Tensor, - rotary_pos_emb: Optional[torch.Tensor] = None, - position_embeddings: Optional[tuple[torch.Tensor, torch.Tensor]] = None, - **kwargs, - ) -> torch.Tensor: - hidden_states = hidden_states + self.attn( - self.norm1(hidden_states), - cu_seqlens=cu_seqlens, - rotary_pos_emb=rotary_pos_emb, - position_embeddings=position_embeddings, - **kwargs, - ) - hidden_states = hidden_states + self.mlp(self.norm2(hidden_states)) - return hidden_states - - -class Glm46VTextRotaryEmbedding(nn.Module): - inv_freq: torch.Tensor # fix linting for `register_buffer` - - def __init__(self, config: Glm46VTextConfig, device=None): - super().__init__() - self.max_seq_len_cached = config.max_position_embeddings - self.original_max_seq_len = config.max_position_embeddings - - self.config = config - - self.rope_type = self.config.rope_parameters["rope_type"] - rope_init_fn: Callable = self.compute_default_rope_parameters - if self.rope_type != "default": - rope_init_fn = ROPE_INIT_FUNCTIONS[self.rope_type] - inv_freq, self.attention_scaling = rope_init_fn(self.config, device) - - self.register_buffer("inv_freq", inv_freq, persistent=False) - self.original_inv_freq = inv_freq - - @staticmethod - def compute_default_rope_parameters( - config: Optional[Glm46VTextConfig] = None, - device: Optional["torch.device"] = None, - seq_len: Optional[int] = None, - ) -> tuple["torch.Tensor", float]: - """ - Computes the inverse frequencies according to the original RoPE implementation - Args: - config ([`~transformers.PreTrainedConfig`]): - The model configuration. - device (`torch.device`): - The device to use for initialization of the inverse frequencies. - seq_len (`int`, *optional*): - The current sequence length. Unused for this type of RoPE. - Returns: - Tuple of (`torch.Tensor`, `float`), containing the inverse frequencies for the RoPE embeddings and the - post-processing scaling factor applied to the computed cos/sin (unused in this type of RoPE). - """ - base = config.rope_parameters["rope_theta"] - partial_rotary_factor = getattr(config, "partial_rotary_factor", 1.0) - head_dim = getattr(config, "head_dim", None) or config.hidden_size // config.num_attention_heads - dim = int(head_dim * partial_rotary_factor) - - attention_factor = 1.0 # Unused in this type of RoPE - - # Compute the inverse frequencies - inv_freq = 1.0 / ( - base ** (torch.arange(0, dim, 2, dtype=torch.int64).to(device=device, dtype=torch.float) / dim) - ) - return inv_freq, attention_factor - - @torch.no_grad() - @dynamic_rope_update # power user: used with advanced RoPE types (e.g. dynamic rope) - def forward(self, x, position_ids): - # In contrast to other models, GLM46V different position ids for the grids - # So we expand the inv_freq to shape (3, ...) - inv_freq_expanded = self.inv_freq[None, None, :, None].float().expand(3, position_ids.shape[1], -1, 1) - position_ids_expanded = position_ids[:, :, None, :].float() # shape (3, bs, 1, positions) - - device_type = x.device.type if isinstance(x.device.type, str) and x.device.type != "mps" else "cpu" - with torch.autocast(device_type=device_type, enabled=False): # Force float32 - freqs = (inv_freq_expanded.float() @ position_ids_expanded.float()).transpose(2, 3) - emb = torch.cat((freqs, freqs), dim=-1) - cos = emb.cos() * self.attention_scaling - sin = emb.sin() * self.attention_scaling - - return cos.to(dtype=x.dtype), sin.to(dtype=x.dtype) - - -def rotate_half_llm(x): - """Rotates half the hidden dims of the input.""" - x1 = x[..., 0::2] - x2 = x[..., 1::2] - return torch.stack((-x2, x1), dim=-1).flatten(-2) - - -def apply_multimodal_rotary_pos_emb(q, k, cos, sin, mrope_section, unsqueeze_dim=1): - """Applies Rotary Position Embedding with Multimodal Sections to the query and key tensors (https://qwenlm.github.io/blog/qwen2-vl/). - - Explanation: - Multimodal 3D rotary position embedding is an extension to 1D rotary position embedding. The input embedding - sequence contains vision (images / videos) embedding and text embedding or just contains text embedding. For - vision embedding part, we apply rotary position embedding on temporal, height and width dimension separately. - Here we split the channel dimension to 3 chunks for the temporal, height and width rotary position embedding. - For text embedding part, we just apply 1D rotary position embedding. The three rotary position index (temporal, - height and width) of text embedding is always the same, so the text embedding rotary position embedding has no - difference with modern LLMs. + Explanation: + Multimodal 3D rotary position embedding is an extension to 1D rotary position embedding. The input embedding + sequence contains vision (images / videos) embedding and text embedding or just contains text embedding. For + vision embedding part, we apply rotary position embedding on temporal, height and width dimension separately. + Here we split the channel dimension to 3 chunks for the temporal, height and width rotary position embedding. + For text embedding part, we just apply 1D rotary position embedding. The three rotary position index (temporal, + height and width) of text embedding is always the same, so the text embedding rotary position embedding has no + difference with modern LLMs. Args: q (`torch.Tensor`): The query tensor. @@ -678,30 +311,6 @@ def forward( return hidden_states -@dataclass -@auto_docstring( - custom_intro=""" - Base class for Llava outputs, with hidden states and attentions. - """ -) -class Glm46VModelOutputWithPast(ModelOutput): - r""" - past_key_values (`Cache`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`): - It is a [`~cache_utils.Cache`] instance. For more details, see our [kv cache guide](https://huggingface.co/docs/transformers/en/kv_cache). - - Contains pre-computed hidden-states (key and values in the self-attention blocks) that can be used (see - `past_key_values` input) to speed up sequential decoding. - rope_deltas (`torch.LongTensor` of shape `(batch_size, )`, *optional*): - The rope index difference between sequence length and multimodal rope. - """ - - last_hidden_state: Optional[torch.FloatTensor] = None - past_key_values: Optional[Cache] = None - hidden_states: Optional[tuple[torch.FloatTensor]] = None - attentions: Optional[tuple[torch.FloatTensor]] = None - rope_deltas: Optional[torch.LongTensor] = None - - @auto_docstring class Glm46VPreTrainedModel(PreTrainedModel): config: Glm46VConfig @@ -713,12 +322,295 @@ class Glm46VPreTrainedModel(PreTrainedModel): _supports_flash_attn = True _supports_sdpa = True - _can_compile_fullgraph = True - _supports_attention_backend = True - _can_record_outputs = { - "hidden_states": Glm46VTextDecoderLayer, - "attentions": Glm46VTextAttention, - } + _can_compile_fullgraph = True + _supports_attention_backend = True + _can_record_outputs = { + "hidden_states": Glm46VTextDecoderLayer, + "attentions": Glm46VTextAttention, + } + + +class Glm46VisionMlp(nn.Module): + def __init__(self, config, bias: bool = False): + super().__init__() + self.hidden_size = config.hidden_size + self.intermediate_size = config.out_hidden_size + self.gate_proj = nn.Linear(self.hidden_size, self.intermediate_size, bias=bias) + self.up_proj = nn.Linear(self.hidden_size, self.intermediate_size, bias=bias) + self.down_proj = nn.Linear(self.intermediate_size, self.hidden_size, bias=bias) + self.act_fn = ACT2FN[config.hidden_act] + + def forward(self, hidden_state): + return self.down_proj(self.act_fn(self.gate_proj(hidden_state)) * self.up_proj(hidden_state)) + + +class Glm46VVisionPatchEmbed(nn.Module): + def __init__(self, config: Glm46VVisionConfig) -> None: + super().__init__() + self.patch_size = config.patch_size + self.temporal_patch_size = config.temporal_patch_size + self.in_channels = config.in_channels + self.embed_dim = config.hidden_size + + kernel_size = [self.temporal_patch_size, self.patch_size, self.patch_size] + self.proj = nn.Conv3d(self.in_channels, self.embed_dim, kernel_size=kernel_size, stride=kernel_size) + + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: + target_dtype = self.proj.weight.dtype + hidden_states = hidden_states.view( + -1, self.in_channels, self.temporal_patch_size, self.patch_size, self.patch_size + ) + hidden_states = self.proj(hidden_states.to(dtype=target_dtype)).view(-1, self.embed_dim) + return hidden_states + + +class Glm46VVisionRotaryEmbedding(nn.Module): + inv_freq: torch.Tensor # fix linting for `register_buffer` + + def __init__(self, dim: int, theta: float = 10000.0) -> None: + super().__init__() + inv_freq = 1.0 / (theta ** (torch.arange(0, dim, 2, dtype=torch.float) / dim)) + self.register_buffer("inv_freq", inv_freq, persistent=False) + + def forward(self, seqlen: int) -> torch.Tensor: + seq = torch.arange(seqlen, device=self.inv_freq.device, dtype=self.inv_freq.dtype) + freqs = torch.outer(seq, self.inv_freq) + return freqs + + +class Glm46VVisionPatchMerger(nn.Module): + def __init__(self, dim: int, context_dim: int, hidden_act: str, bias: bool = False) -> None: + super().__init__() + self.proj = nn.Linear(dim, dim, bias=bias) + self.post_projection_norm = LayerNorm(dim) + self.gate_proj = nn.Linear(dim, context_dim, bias=bias) + self.up_proj = nn.Linear(dim, context_dim, bias=bias) + self.down_proj = nn.Linear(context_dim, dim, bias=bias) + self.act1 = nn.GELU() + self.act_fn = ACT2FN[hidden_act] + + def forward(self, hidden_state: torch.Tensor) -> torch.Tensor: + hidden_state = self.proj(hidden_state) + hidden_state = self.act1(self.post_projection_norm(hidden_state)) + return self.down_proj(self.act_fn(self.gate_proj(hidden_state)) * self.up_proj(hidden_state)) + + +class Glm46VVisionEmbeddings(nn.Module): + def __init__(self, config: Glm46VVisionConfig): + super().__init__() + self.config = config + self.embed_dim = config.hidden_size + self.image_size = config.image_size + self.patch_size = config.patch_size + + self.num_patches = (self.image_size // self.patch_size) ** 2 + self.num_positions = self.num_patches + self.position_embedding = nn.Embedding(self.num_positions, self.embed_dim) + self.register_buffer("position_ids", torch.arange(self.num_positions).expand((1, -1)), persistent=False) + + def forward(self, embeddings, lengths, image_shapes, h_coords, w_coords) -> torch.Tensor: + """ + Forward pass with integrated position encoding adaptation using 2D interpolation. + + Args: + embeddings: Input embeddings tensor + lengths (torch.Tensor): Sequence lengths for each image in the batch. + image_shapes (torch.Tensor): Tensor of shape [batch_size, 3] representing the image shapes (t, h, w). + h_coords (torch.Tensor): Tensor of shape [total_seq] representing the h coordinate for each patch. + w_coords (torch.Tensor): Tensor of shape [total_seq] representing the w coordinate for each patch. + + Returns: + torch.Tensor: Embeddings with adapted position encoding added. + """ + # Get position embedding parameters + pos_embed_weight = self.position_embedding.weight + hidden_size = pos_embed_weight.shape[1] + total_seq = h_coords.shape[0] + device = pos_embed_weight.device + + # Move coordinates to correct device + h_coords, w_coords = h_coords.to(device), w_coords.to(device) + + # Handle empty sequence case + if total_seq == 0: + adapted_pos_embed = torch.empty(0, hidden_size, device=device, dtype=pos_embed_weight.dtype) + else: + # Convert inputs to tensors if needed + if isinstance(lengths, list): + lengths = torch.tensor(lengths, device=device, dtype=torch.long) + if not isinstance(image_shapes, torch.Tensor): + image_shapes = torch.tensor(image_shapes, device=device, dtype=torch.long) + + # Prepare 2D position embedding + orig_size_sq = pos_embed_weight.shape[0] + orig_size = int(orig_size_sq**0.5) + pos_embed_2d = ( + pos_embed_weight.view(orig_size, orig_size, hidden_size) + .permute(2, 0, 1) + .unsqueeze(0) + .to(device=device, dtype=torch.float32) + ) + + # Calculate target dimensions for each patch + target_h = torch.cat([image_shapes[i, 1].repeat(lengths[i]) for i in range(len(lengths))]).to( + device=device, dtype=torch.float32 + ) + target_w = torch.cat([image_shapes[i, 2].repeat(lengths[i]) for i in range(len(lengths))]).to( + device=device, dtype=torch.float32 + ) + + # Normalize coordinates to [-1, 1] range for grid_sample + h_coords = h_coords.to(device=device, dtype=torch.float32) + w_coords = w_coords.to(device=device, dtype=torch.float32) + norm_w = ((w_coords + 0.5) / target_w) * 2 - 1 + norm_h = ((h_coords + 0.5) / target_h) * 2 - 1 + + # Create sampling grid + grid = torch.stack((norm_w, norm_h), dim=-1).unsqueeze(0).unsqueeze(2) + + # Perform bicubic interpolation + interpolated_embed_fp32 = F.grid_sample( + pos_embed_2d, grid, mode="bicubic", align_corners=False, padding_mode="border" + ) + + # Reshape and convert back to original dtype + adapted_pos_embed_fp32 = interpolated_embed_fp32.squeeze(0).squeeze(-1).permute(1, 0) + adapted_pos_embed = adapted_pos_embed_fp32.to(pos_embed_weight.dtype).to(embeddings.device) + + # Add adapted position encoding to embeddings + embeddings = embeddings + adapted_pos_embed + return embeddings + + +def rotate_half(x): + """Rotates half the hidden dims of the input.""" + x1 = x[..., : x.shape[-1] // 2] + x2 = x[..., x.shape[-1] // 2 :] + return torch.cat((-x2, x1), dim=-1) + + +def apply_rotary_pos_emb_vision( + q: torch.Tensor, k: torch.Tensor, cos: torch.Tensor, sin: torch.Tensor +) -> tuple[torch.Tensor, torch.Tensor]: + orig_q_dtype = q.dtype + orig_k_dtype = k.dtype + q, k = q.float(), k.float() + cos, sin = cos.unsqueeze(-2).float(), sin.unsqueeze(-2).float() + q_embed = (q * cos) + (rotate_half(q) * sin) + k_embed = (k * cos) + (rotate_half(k) * sin) + q_embed = q_embed.to(orig_q_dtype) + k_embed = k_embed.to(orig_k_dtype) + return q_embed, k_embed + + +class Glm46VVisionAttention(nn.Module): + def __init__(self, config: Glm46VVisionConfig) -> None: + super().__init__() + self.dim = config.hidden_size + self.num_heads = config.num_heads + self.head_dim = self.dim // self.num_heads + self.num_key_value_groups = 1 # needed for eager attention + self.qkv = nn.Linear(config.hidden_size, config.hidden_size * 3, bias=config.attention_bias) + self.proj = nn.Linear(config.hidden_size, config.hidden_size, bias=False) + self.scaling = self.head_dim**-0.5 + self.config = config + self.attention_dropout = config.attention_dropout + self.is_causal = False + + def forward( + self, + hidden_states: torch.Tensor, + cu_seqlens: torch.Tensor, + rotary_pos_emb: Optional[torch.Tensor] = None, + position_embeddings: Optional[tuple[torch.Tensor, torch.Tensor]] = None, + **kwargs, + ) -> torch.Tensor: + seq_length = hidden_states.shape[0] + query_states, key_states, value_states = ( + self.qkv(hidden_states).reshape(seq_length, 3, self.num_heads, -1).permute(1, 0, 2, 3).unbind(0) + ) + cos, sin = position_embeddings + query_states, key_states = apply_rotary_pos_emb_vision(query_states, key_states, cos, sin) + + query_states = query_states.transpose(0, 1).unsqueeze(0) + key_states = key_states.transpose(0, 1).unsqueeze(0) + value_states = value_states.transpose(0, 1).unsqueeze(0) + + attention_interface: Callable = eager_attention_forward + if self.config._attn_implementation != "eager": + attention_interface = ALL_ATTENTION_FUNCTIONS[self.config._attn_implementation] + + if self.config._attn_implementation == "flash_attention_2": + # Flash Attention 2: Use cu_seqlens for variable length attention + max_seqlen = (cu_seqlens[1:] - cu_seqlens[:-1]).max() + attn_output, _ = attention_interface( + self, + query_states, + key_states, + value_states, + attention_mask=None, + scaling=self.scaling, + dropout=0.0 if not self.training else self.attention_dropout, + cu_seq_lens_q=cu_seqlens, + cu_seq_lens_k=cu_seqlens, + max_length_q=max_seqlen, + max_length_k=max_seqlen, + is_causal=False, + **kwargs, + ) + else: + # Other implementations: Process each chunk separately + lengths = cu_seqlens[1:] - cu_seqlens[:-1] + splits = [ + torch.split(tensor, lengths.tolist(), dim=2) for tensor in (query_states, key_states, value_states) + ] + + attn_outputs = [ + attention_interface( + self, + q, + k, + v, + attention_mask=None, + scaling=self.scaling, + dropout=0.0 if not self.training else self.attention_dropout, + is_causal=False, + **kwargs, + )[0] + for q, k, v in zip(*splits) + ] + attn_output = torch.cat(attn_outputs, dim=1) + + attn_output = attn_output.reshape(seq_length, -1).contiguous() + attn_output = self.proj(attn_output) + return attn_output + + +class Glm46VVisionBlock(GradientCheckpointingLayer): + def __init__(self, config) -> None: + super().__init__() + self.norm1 = Glm46VRMSNorm(config.hidden_size, eps=config.rms_norm_eps) + self.norm2 = Glm46VRMSNorm(config.hidden_size, eps=config.rms_norm_eps) + self.attn = Glm46VVisionAttention(config) + self.mlp = Glm46VisionMlp(config, bias=False) + + def forward( + self, + hidden_states: torch.Tensor, + cu_seqlens: torch.Tensor, + rotary_pos_emb: Optional[torch.Tensor] = None, + position_embeddings: Optional[tuple[torch.Tensor, torch.Tensor]] = None, + **kwargs, + ) -> torch.Tensor: + hidden_states = hidden_states + self.attn( + self.norm1(hidden_states), + cu_seqlens=cu_seqlens, + rotary_pos_emb=rotary_pos_emb, + position_embeddings=position_embeddings, + **kwargs, + ) + hidden_states = hidden_states + self.mlp(self.norm2(hidden_states)) + return hidden_states class Glm46VVisionModel(Glm46VPreTrainedModel): @@ -832,6 +724,75 @@ def forward(self, hidden_states: torch.Tensor, grid_thw: torch.Tensor) -> torch. return hidden_states +class Glm46VTextRotaryEmbedding(nn.Module): + inv_freq: torch.Tensor # fix linting for `register_buffer` + + def __init__(self, config: Glm46VTextConfig, device=None): + super().__init__() + self.max_seq_len_cached = config.max_position_embeddings + self.original_max_seq_len = config.max_position_embeddings + + self.config = config + + self.rope_type = self.config.rope_parameters["rope_type"] + rope_init_fn: Callable = self.compute_default_rope_parameters + if self.rope_type != "default": + rope_init_fn = ROPE_INIT_FUNCTIONS[self.rope_type] + inv_freq, self.attention_scaling = rope_init_fn(self.config, device) + + self.register_buffer("inv_freq", inv_freq, persistent=False) + self.original_inv_freq = inv_freq + + @staticmethod + def compute_default_rope_parameters( + config: Optional[Glm46VTextConfig] = None, + device: Optional["torch.device"] = None, + seq_len: Optional[int] = None, + ) -> tuple["torch.Tensor", float]: + """ + Computes the inverse frequencies according to the original RoPE implementation + Args: + config ([`~transformers.PreTrainedConfig`]): + The model configuration. + device (`torch.device`): + The device to use for initialization of the inverse frequencies. + seq_len (`int`, *optional*): + The current sequence length. Unused for this type of RoPE. + Returns: + Tuple of (`torch.Tensor`, `float`), containing the inverse frequencies for the RoPE embeddings and the + post-processing scaling factor applied to the computed cos/sin (unused in this type of RoPE). + """ + base = config.rope_parameters["rope_theta"] + partial_rotary_factor = getattr(config, "partial_rotary_factor", 1.0) + head_dim = getattr(config, "head_dim", None) or config.hidden_size // config.num_attention_heads + dim = int(head_dim * partial_rotary_factor) + + attention_factor = 1.0 # Unused in this type of RoPE + + # Compute the inverse frequencies + inv_freq = 1.0 / ( + base ** (torch.arange(0, dim, 2, dtype=torch.int64).to(device=device, dtype=torch.float) / dim) + ) + return inv_freq, attention_factor + + @torch.no_grad() + @dynamic_rope_update # power user: used with advanced RoPE types (e.g. dynamic rope) + def forward(self, x, position_ids): + # In contrast to other models, GLM46V different position ids for the grids + # So we expand the inv_freq to shape (3, ...) + inv_freq_expanded = self.inv_freq[None, None, :, None].float().expand(3, position_ids.shape[1], -1, 1) + position_ids_expanded = position_ids[:, :, None, :].float() # shape (3, bs, 1, positions) + + device_type = x.device.type if isinstance(x.device.type, str) and x.device.type != "mps" else "cpu" + with torch.autocast(device_type=device_type, enabled=False): # Force float32 + freqs = (inv_freq_expanded.float() @ position_ids_expanded.float()).transpose(2, 3) + emb = torch.cat((freqs, freqs), dim=-1) + cos = emb.cos() * self.attention_scaling + sin = emb.sin() * self.attention_scaling + + return cos.to(dtype=x.dtype), sin.to(dtype=x.dtype) + + @auto_docstring class Glm46VTextModel(Glm46VPreTrainedModel): config: Glm46VTextConfig @@ -939,6 +900,30 @@ def forward( ) +@dataclass +@auto_docstring( + custom_intro=""" + Base class for Llava outputs, with hidden states and attentions. + """ +) +class Glm46VModelOutputWithPast(ModelOutput): + r""" + past_key_values (`Cache`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`): + It is a [`~cache_utils.Cache`] instance. For more details, see our [kv cache guide](https://huggingface.co/docs/transformers/en/kv_cache). + + Contains pre-computed hidden-states (key and values in the self-attention blocks) that can be used (see + `past_key_values` input) to speed up sequential decoding. + rope_deltas (`torch.LongTensor` of shape `(batch_size, )`, *optional*): + The rope index difference between sequence length and multimodal rope. + """ + + last_hidden_state: Optional[torch.FloatTensor] = None + past_key_values: Optional[Cache] = None + hidden_states: Optional[tuple[torch.FloatTensor]] = None + attentions: Optional[tuple[torch.FloatTensor]] = None + rope_deltas: Optional[torch.LongTensor] = None + + @auto_docstring class Glm46VModel(Glm46VPreTrainedModel): base_model_prefix = "" @@ -1348,354 +1333,4 @@ def forward( ) -@dataclass -@auto_docstring( - custom_intro=""" - Base class for Glm46V causal language model (or autoregressive) outputs. - """ -) -class Glm46VCausalLMOutputWithPast(ModelOutput): - r""" - loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `labels` is provided): - Language modeling loss (for next-token prediction). - logits (`torch.FloatTensor` of shape `(batch_size, sequence_length, config.vocab_size)`): - Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax). - past_key_values (`Cache`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`): - It is a [`~cache_utils.Cache`] instance. For more details, see our [kv cache guide](https://huggingface.co/docs/transformers/en/kv_cache). - - Contains pre-computed hidden-states (key and values in the self-attention blocks) that can be used (see - `past_key_values` input) to speed up sequential decoding. - rope_deltas (`torch.LongTensor` of shape `(batch_size, )`, *optional*): - The rope index difference between sequence length and multimodal rope. - """ - - loss: Optional[torch.FloatTensor] = None - logits: Optional[torch.FloatTensor] = None - past_key_values: Optional[Cache] = None - hidden_states: Optional[tuple[torch.FloatTensor]] = None - attentions: Optional[tuple[torch.FloatTensor]] = None - rope_deltas: Optional[torch.LongTensor] = None - - -class Glm46VForConditionalGeneration(Glm46VPreTrainedModel, GenerationMixin): - _checkpoint_conversion_mapping = {} - _tied_weights_keys = ["lm_head.weight"] - # Reference: fix gemma3 grad acc #37208 - accepts_loss_kwargs = False - - def __init__(self, config): - super().__init__(config) - self.model = Glm46VModel(config) - self.lm_head = nn.Linear(config.text_config.hidden_size, config.text_config.vocab_size, bias=False) - - self.post_init() - - def get_input_embeddings(self): - return self.model.get_input_embeddings() - - def set_input_embeddings(self, value): - self.model.set_input_embeddings(value) - - def set_decoder(self, decoder): - self.model.set_decoder(decoder) - - def get_decoder(self): - return self.model.get_decoder() - - def get_video_features( - self, pixel_values_videos: torch.FloatTensor, video_grid_thw: Optional[torch.LongTensor] = None - ): - return self.model.get_video_features(pixel_values_videos, video_grid_thw) - - def get_image_features(self, pixel_values: torch.FloatTensor, image_grid_thw: Optional[torch.LongTensor] = None): - return self.model.get_image_features(pixel_values, image_grid_thw) - - # Make modules available through conditional class for BC - @property - def language_model(self): - return self.model.language_model - - @property - def visual(self): - return self.model.visual - - @can_return_tuple - @auto_docstring - def forward( - self, - input_ids: Optional[torch.LongTensor] = None, - attention_mask: Optional[torch.Tensor] = None, - position_ids: Optional[torch.LongTensor] = None, - past_key_values: Optional[Cache] = None, - inputs_embeds: Optional[torch.FloatTensor] = None, - labels: Optional[torch.LongTensor] = None, - pixel_values: Optional[torch.Tensor] = None, - pixel_values_videos: Optional[torch.FloatTensor] = None, - image_grid_thw: Optional[torch.LongTensor] = None, - video_grid_thw: Optional[torch.LongTensor] = None, - rope_deltas: Optional[torch.LongTensor] = None, - cache_position: Optional[torch.LongTensor] = None, - logits_to_keep: Union[int, torch.Tensor] = 0, - **kwargs: Unpack[TransformersKwargs], - ) -> Union[tuple, Glm46VCausalLMOutputWithPast]: - r""" - rope_deltas (`torch.LongTensor` of shape `(batch_size, )`, *optional*): - The rope index difference between sequence length and multimodal rope. - labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., - config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored - (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. - image_grid_thw (`torch.LongTensor` of shape `(num_images, 3)`, *optional*): - The temporal, height and width of feature shape of each image in LLM. - video_grid_thw (`torch.LongTensor` of shape `(num_videos, 3)`, *optional*): - The temporal, height and width of feature shape of each video in LLM. - - Example: - - ```python - >>> from PIL import Image - >>> import requests - >>> from transformers import AutoProcessor, Glm46VForConditionalGeneration - - >>> model = Glm46VForConditionalGeneration.from_pretrained("THUDM/GLM-4.1V-9B-Thinking") - >>> processor = AutoProcessor.from_pretrained("THUDM/GLM-4.1V-9B-Thinking") - - >>> messages = [ - { - "role": "user", - "content": [ - {"type": "image"}, - {"type": "text", "text": "What is shown in this image?"}, - ], - }, - ] - >>> url = "https://www.ilankelman.org/stopsigns/australia.jpg" - >>> image = Image.open(requests.get(url, stream=True).raw) - - >>> text = processor.apply_chat_template(messages, tokenize=False, add_generation_prompt=True) - >>> inputs = processor(text=[text], images=[image], vision_infos=[vision_infos]) - - >>> # Generate - >>> generate_ids = model.generate(inputs.input_ids, max_length=30) - >>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0] - "The image shows a street scene with a red stop sign in the foreground. In the background, there is a large red gate with Chinese characters ..." - ```""" - outputs = self.model( - input_ids=input_ids, - pixel_values=pixel_values, - pixel_values_videos=pixel_values_videos, - image_grid_thw=image_grid_thw, - video_grid_thw=video_grid_thw, - position_ids=position_ids, - attention_mask=attention_mask, - past_key_values=past_key_values, - inputs_embeds=inputs_embeds, - cache_position=cache_position, - **kwargs, - ) - - hidden_states = outputs[0] - - # Only compute necessary logits, and do not upcast them to float if we are not computing the loss - slice_indices = slice(-logits_to_keep, None) if isinstance(logits_to_keep, int) else logits_to_keep - logits = self.lm_head(hidden_states[:, slice_indices, :]) - - loss = None - if labels is not None: - loss = self.loss_function(logits=logits, labels=labels, vocab_size=self.config.text_config.vocab_size) - - return Glm46VCausalLMOutputWithPast( - loss=loss, - logits=logits, - past_key_values=outputs.past_key_values, - hidden_states=outputs.hidden_states, - attentions=outputs.attentions, - rope_deltas=outputs.rope_deltas, - ) - - def prepare_inputs_for_generation( - self, - input_ids, - past_key_values=None, - attention_mask=None, - inputs_embeds=None, - cache_position=None, - position_ids=None, - use_cache=True, - pixel_values=None, - pixel_values_videos=None, - image_grid_thw=None, - video_grid_thw=None, - **kwargs, - ): - # Overwritten -- in specific circumstances we don't want to forward image inputs to the model - - model_inputs = super().prepare_inputs_for_generation( - input_ids, - past_key_values=past_key_values, - attention_mask=attention_mask, - inputs_embeds=inputs_embeds, - cache_position=cache_position, - position_ids=position_ids, - pixel_values=pixel_values, - pixel_values_videos=pixel_values_videos, - image_grid_thw=image_grid_thw, - video_grid_thw=video_grid_thw, - use_cache=use_cache, - **kwargs, - ) - - # GLM-4.1V position_ids are prepareed with rope_deltas in forward - model_inputs["position_ids"] = None - - if cache_position[0] != 0: - model_inputs["pixel_values"] = None - model_inputs["pixel_values_videos"] = None - - return model_inputs - - def _get_image_nums_and_video_nums( - self, - input_ids: Optional[torch.LongTensor], - inputs_embeds: Optional[torch.Tensor] = None, - ) -> tuple[torch.Tensor, torch.Tensor]: - """ - Get the number of images and videos for each sample to calculate the separation length of the sample tensor. - These parameters are not passed through the processor to avoid unpredictable impacts from interface modifications. - - Args: - input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): - Indices of input sequence tokens in the vocabulary. - - Returns: - image_nums (`torch.LongTensor` of shape `(batch_size, num_images_sample)`) - video_nums (`torch.LongTensor` of shape `(batch_size, num_videos_sample)`) - """ - - if inputs_embeds is not None: - is_image = ( - inputs_embeds - == self.get_input_embeddings()( - torch.tensor(self.config.image_start_token_id, dtype=torch.long, device=inputs_embeds.device) - ) - )[..., 0] - is_video_start = ( - inputs_embeds - == self.get_input_embeddings()( - torch.tensor(self.config.video_start_token_id, dtype=torch.long, device=inputs_embeds.device) - ) - )[..., 0] - is_video_end = ( - inputs_embeds - == self.get_input_embeddings()( - torch.tensor(self.config.video_end_token_id, dtype=torch.long, device=inputs_embeds.device) - ) - )[..., 0] - else: - is_image = input_ids == self.config.image_start_token_id - is_video_start = input_ids == self.config.video_start_token_id - is_video_end = input_ids == self.config.video_end_token_id - - # Cumulative sum to track if we're inside a video span - # We'll assume well-formed video tags (i.e. matching starts and ends) - video_level = torch.cumsum(is_video_start.int() - is_video_end.int(), dim=1) - inside_video = video_level > 0 # shape (batch_size, seq_length) - - # Mask out image tokens that are inside video spans - standalone_images = is_image & (~inside_video) - - # Count per batch - image_counts = standalone_images.sum(dim=1) - video_counts = is_video_start.sum(dim=1) - - return image_counts, video_counts - - def _expand_inputs_for_generation( - self, - expand_size: int = 1, - is_encoder_decoder: bool = False, - input_ids: Optional[torch.LongTensor] = None, - **model_kwargs, - ) -> tuple[torch.LongTensor, dict[str, Any]]: - # Overwritten -- Support for expanding tensors without a batch size dimension - # e.g., pixel_values, image_grid_thw, pixel_values_videos, video_grid_thw, second_per_grid_t - # pixel_values.shape[0] is sum(seqlen_images for samples) - # image_grid_thw.shape[0] is sum(num_images for samples) - - if expand_size == 1: - return input_ids, model_kwargs - - visual_keys = ["pixel_values", "image_grid_thw", "pixel_values_videos", "video_grid_thw", "second_per_grid_ts"] - - def _expand_dict_for_generation_visual(dict_to_expand): - image_grid_thw = model_kwargs.get("image_grid_thw", None) - video_grid_thw = model_kwargs.get("video_grid_thw", None) - image_nums, video_nums = self._get_image_nums_and_video_nums( - input_ids, inputs_embeds=model_kwargs.get("inputs_embeds", None) - ) - - def _repeat_interleave_samples(x, lengths, repeat_times): - samples = torch.split(x, lengths) - repeat_args = [repeat_times] + [1] * (x.dim() - 1) - result = torch.cat([sample.repeat(*repeat_args) for sample in samples], dim=0) - return result - - for key in dict_to_expand: - if key == "pixel_values": - # split images into samples - samples = torch.split(image_grid_thw, list(image_nums)) - # compute the sequence length of images for each sample - lengths = [torch.prod(sample, dim=1).sum() for sample in samples] - dict_to_expand[key] = _repeat_interleave_samples( - dict_to_expand[key], lengths=lengths, repeat_times=expand_size - ) - elif key == "image_grid_thw": - # get the num of images for each sample - lengths = list(image_nums) - dict_to_expand[key] = _repeat_interleave_samples( - dict_to_expand[key], lengths=lengths, repeat_times=expand_size - ) - elif key == "pixel_values_videos": - samples = torch.split(video_grid_thw, list(video_nums)) - lengths = [torch.prod(sample, dim=1).sum() for sample in samples] - dict_to_expand[key] = _repeat_interleave_samples( - dict_to_expand[key], lengths=lengths, repeat_times=expand_size - ) - elif key == "video_grid_thw": - lengths = list(video_nums) - dict_to_expand[key] = _repeat_interleave_samples( - dict_to_expand[key], lengths=lengths, repeat_times=expand_size - ) - elif key == "second_per_grid_ts": - dict_to_expand[key] = _repeat_interleave_samples( - dict_to_expand[key], lengths=list(video_nums), repeat_times=expand_size - ) - return dict_to_expand - - def _expand_dict_for_generation(dict_to_expand): - for key in dict_to_expand: - if ( - key != "cache_position" - and dict_to_expand[key] is not None - and isinstance(dict_to_expand[key], torch.Tensor) - and key not in visual_keys - ): - dict_to_expand[key] = dict_to_expand[key].repeat_interleave(expand_size, dim=0) - return dict_to_expand - - model_kwargs = _expand_dict_for_generation_visual(model_kwargs) - - if input_ids is not None: - input_ids = input_ids.repeat_interleave(expand_size, dim=0) - - model_kwargs = _expand_dict_for_generation(model_kwargs) - - if is_encoder_decoder: - if model_kwargs.get("encoder_outputs") is None: - raise ValueError("If `is_encoder_decoder` is True, make sure that `encoder_outputs` is defined.") - model_kwargs["encoder_outputs"] = _expand_dict_for_generation(model_kwargs["encoder_outputs"]) - - return input_ids, model_kwargs - - -__all__ = ["Glm46VForConditionalGeneration", "Glm46VModel", "Glm46VPreTrainedModel", "Glm46VTextModel"] +__all__ = ["Glm46VModel", "Glm46VPreTrainedModel", "Glm46VTextModel"] diff --git a/src/transformers/models/glm46v/modular_glm46v.py b/src/transformers/models/glm46v/modular_glm46v.py index 6c6ca73ae22b..05ee02126630 100644 --- a/src/transformers/models/glm46v/modular_glm46v.py +++ b/src/transformers/models/glm46v/modular_glm46v.py @@ -13,35 +13,11 @@ # See the License for the specific language governing permissions and # limitations under the License. -from ..glm4v.configuration_glm4v import Glm4vConfig, Glm4vTextConfig, Glm4vVisionConfig -from ..glm4v.modeling_glm4v import ( - Glm4vCausalLMOutputWithPast, - Glm4vForConditionalGeneration, - Glm4VisionMlp, - Glm4vModel, - Glm4vModelOutputWithPast, - Glm4vPreTrainedModel, - Glm4vRMSNorm, - Glm4vTextAttention, - Glm4vTextDecoderLayer, - Glm4vTextMLP, - Glm4vTextModel, - Glm4vTextRotaryEmbedding, - Glm4vVisionAttention, - Glm4vVisionBlock, - Glm4vVisionEmbeddings, - Glm4vVisionModel, - Glm4vVisionPatchEmbed, - Glm4vVisionPatchMerger, - Glm4vVisionRotaryEmbedding, -) +from ..glm4v.configuration_glm4v import Glm4vConfig, Glm4vTextConfig +from ..glm4v.modeling_glm4v import Glm4vModel, Glm4vPreTrainedModel, Glm4vTextModel, Glm4vVisionModel from ..glm4v.processing_glm4v import Glm4vProcessor -class Glm46VVisionConfig(Glm4vVisionConfig): - pass - - class Glm46VTextConfig(Glm4vTextConfig): pass @@ -50,58 +26,6 @@ class Glm46VConfig(Glm4vConfig): pass -class Glm46VRMSNorm(Glm4vRMSNorm): - pass - - -class Glm4VisionMlp(Glm4VisionMlp): - pass - - -class Glm46VVisionPatchEmbed(Glm4vVisionPatchEmbed): - pass - - -class Glm46VVisionRotaryEmbedding(Glm4vVisionRotaryEmbedding): - pass - - -class Glm46VVisionPatchMerger(Glm4vVisionPatchMerger): - pass - - -class Glm46VVisionEmbeddings(Glm4vVisionEmbeddings): - pass - - -class Glm46VVisionAttention(Glm4vVisionAttention): - pass - - -class Glm46VVisionBlock(Glm4vVisionBlock): - pass - - -class Glm46VTextRotaryEmbedding(Glm4vTextRotaryEmbedding): - pass - - -class Glm46VTextAttention(Glm4vTextAttention): - pass - - -class Glm46VTextMLP(Glm4vTextMLP): - pass - - -class Glm46VTextDecoderLayer(Glm4vTextDecoderLayer): - pass - - -class Glm46VModelOutputWithPast(Glm4vModelOutputWithPast): - pass - - class Glm46VPreTrainedModel(Glm4vPreTrainedModel): pass @@ -118,14 +42,6 @@ class Glm46VModel(Glm4vModel): pass -class Glm46VCausalLMOutputWithPast(Glm4vCausalLMOutputWithPast): - pass - - -class Glm46VForConditionalGeneration(Glm4vForConditionalGeneration): - pass - - class Glm46VProcessor(Glm4vProcessor): def replace_frame_token_id(self, timestamp_sec): return f"<|begin_of_image|>{self.image_token}<|end_of_image|>{timestamp_sec:.1f} seconds" @@ -134,7 +50,6 @@ def replace_frame_token_id(self, timestamp_sec): __all__ = [ "Glm46VConfig", "Glm46VTextConfig", - "Glm46VForConditionalGeneration", "Glm46VModel", "Glm46VPreTrainedModel", "Glm46VTextModel", From 8ae004f290027c9b6fb85f9a14ad5f56de7e8f83 Mon Sep 17 00:00:00 2001 From: zRzRzRzRzRzRzR <2448370773@qq.com> Date: Wed, 12 Nov 2025 17:49:01 +0800 Subject: [PATCH 21/66] upload --- .../models/glm46v/modeling_glm46v.py | 400 +++++++++--------- .../models/glm46v/modular_glm46v.py | 6 +- 2 files changed, 201 insertions(+), 205 deletions(-) diff --git a/src/transformers/models/glm46v/modeling_glm46v.py b/src/transformers/models/glm46v/modeling_glm46v.py index 60138e10f9b9..9ee84b1e254d 100644 --- a/src/transformers/models/glm46v/modeling_glm46v.py +++ b/src/transformers/models/glm46v/modeling_glm46v.py @@ -330,6 +330,182 @@ class Glm46VPreTrainedModel(PreTrainedModel): } +class Glm46VTextRotaryEmbedding(nn.Module): + inv_freq: torch.Tensor # fix linting for `register_buffer` + + def __init__(self, config: Glm46VTextConfig, device=None): + super().__init__() + self.max_seq_len_cached = config.max_position_embeddings + self.original_max_seq_len = config.max_position_embeddings + + self.config = config + + self.rope_type = self.config.rope_parameters["rope_type"] + rope_init_fn: Callable = self.compute_default_rope_parameters + if self.rope_type != "default": + rope_init_fn = ROPE_INIT_FUNCTIONS[self.rope_type] + inv_freq, self.attention_scaling = rope_init_fn(self.config, device) + + self.register_buffer("inv_freq", inv_freq, persistent=False) + self.original_inv_freq = inv_freq + + @staticmethod + def compute_default_rope_parameters( + config: Optional[Glm46VTextConfig] = None, + device: Optional["torch.device"] = None, + seq_len: Optional[int] = None, + ) -> tuple["torch.Tensor", float]: + """ + Computes the inverse frequencies according to the original RoPE implementation + Args: + config ([`~transformers.PreTrainedConfig`]): + The model configuration. + device (`torch.device`): + The device to use for initialization of the inverse frequencies. + seq_len (`int`, *optional*): + The current sequence length. Unused for this type of RoPE. + Returns: + Tuple of (`torch.Tensor`, `float`), containing the inverse frequencies for the RoPE embeddings and the + post-processing scaling factor applied to the computed cos/sin (unused in this type of RoPE). + """ + base = config.rope_parameters["rope_theta"] + partial_rotary_factor = getattr(config, "partial_rotary_factor", 1.0) + head_dim = getattr(config, "head_dim", None) or config.hidden_size // config.num_attention_heads + dim = int(head_dim * partial_rotary_factor) + + attention_factor = 1.0 # Unused in this type of RoPE + + # Compute the inverse frequencies + inv_freq = 1.0 / ( + base ** (torch.arange(0, dim, 2, dtype=torch.int64).to(device=device, dtype=torch.float) / dim) + ) + return inv_freq, attention_factor + + @torch.no_grad() + @dynamic_rope_update # power user: used with advanced RoPE types (e.g. dynamic rope) + def forward(self, x, position_ids): + # In contrast to other models, GLM46V different position ids for the grids + # So we expand the inv_freq to shape (3, ...) + inv_freq_expanded = self.inv_freq[None, None, :, None].float().expand(3, position_ids.shape[1], -1, 1) + position_ids_expanded = position_ids[:, :, None, :].float() # shape (3, bs, 1, positions) + + device_type = x.device.type if isinstance(x.device.type, str) and x.device.type != "mps" else "cpu" + with torch.autocast(device_type=device_type, enabled=False): # Force float32 + freqs = (inv_freq_expanded.float() @ position_ids_expanded.float()).transpose(2, 3) + emb = torch.cat((freqs, freqs), dim=-1) + cos = emb.cos() * self.attention_scaling + sin = emb.sin() * self.attention_scaling + + return cos.to(dtype=x.dtype), sin.to(dtype=x.dtype) + + +@auto_docstring +class Glm46VTextModel(Glm46VPreTrainedModel): + config: Glm46VTextConfig + input_modalities = "text" + + def __init__(self, config: Glm46VTextConfig): + super().__init__(config) + self.padding_idx = config.pad_token_id + self.vocab_size = config.vocab_size + + self.embed_tokens = nn.Embedding(config.vocab_size, config.hidden_size, self.padding_idx) + self.layers = nn.ModuleList( + [Glm46VTextDecoderLayer(config, layer_idx) for layer_idx in range(config.num_hidden_layers)] + ) + self.norm = Glm46VRMSNorm(config.hidden_size, eps=config.rms_norm_eps) + self.rotary_emb = Glm46VTextRotaryEmbedding(config=config) + + self.gradient_checkpointing = False + # Initialize weights and apply final processing + self.post_init() + + @auto_docstring + @check_model_inputs() + def forward( + self, + input_ids: Optional[torch.LongTensor] = None, + attention_mask: Optional[torch.Tensor] = None, + position_ids: Optional[torch.LongTensor] = None, + past_key_values: Optional[Cache] = None, + inputs_embeds: Optional[torch.FloatTensor] = None, + use_cache: Optional[bool] = None, + cache_position: Optional[torch.LongTensor] = None, + **kwargs: Unpack[FlashAttentionKwargs], + ) -> Union[tuple, BaseModelOutputWithPast]: + if (input_ids is None) ^ (inputs_embeds is not None): + raise ValueError("You must specify exactly one of input_ids or inputs_embeds") + + # torch.jit.trace() doesn't support cache objects in the output + if use_cache and past_key_values is None and not torch.jit.is_tracing(): + past_key_values = DynamicCache(config=self.config) + + if inputs_embeds is None: + inputs_embeds = self.embed_tokens(input_ids) + + if cache_position is None: + past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 + cache_position = torch.arange( + past_seen_tokens, past_seen_tokens + inputs_embeds.shape[1], device=inputs_embeds.device + ) + + # the hard coded `3` is for temporal, height and width. + if position_ids is None: + position_ids = cache_position.view(1, 1, -1).expand(3, inputs_embeds.shape[0], -1) + elif position_ids.ndim == 2: + position_ids = position_ids[None, ...].expand(3, position_ids.shape[0], -1) + + # NOTE: we need to pass text position ids for packing. Qwen2-VL uses 3D positions + # where each dim indicates visual spatial positions for temporal/height/width grids. + # There are two scenarios when FA2-like packed masking might be activated. + # 1. User specifically passed packed `position_ids` and no attention mask. + # In this case we expect the useer to create correct position ids for all 3 grids + # and prepend text-only position ids to it. The final tensor will be [4, bs, seq-len] + # 2. User runs forward with no attention mask and no position ids. In this case, position ids + # are prepared by the model (`get_rope_index`) as `[4, bs, seq-len]` tensor. Text-only positions are + # prepended by us when creating positions so that the mask is constructed correctly. NOTE: failing to pass + # text-only positions will cause incorrect mask construction, do not change `prepare_input_for_generation` + if position_ids.ndim == 3 and position_ids.shape[0] == 4: + text_position_ids = position_ids[0] + position_ids = position_ids[1:] + else: + # If inputs are not packed (usual 3D positions), do not prepare mask from position_ids + text_position_ids = None + + mask_kwargs = { + "config": self.config, + "input_embeds": inputs_embeds, + "attention_mask": attention_mask, + "cache_position": cache_position, + "past_key_values": past_key_values, + "position_ids": text_position_ids, + } + # Create the masks + causal_mask = create_causal_mask(**mask_kwargs) + + hidden_states = inputs_embeds + position_embeddings = self.rotary_emb(hidden_states, position_ids=position_ids) + + for decoder_layer in self.layers: + layer_outputs = decoder_layer( + hidden_states, + attention_mask=causal_mask, + position_ids=position_ids, + past_key_values=past_key_values, + cache_position=cache_position, + position_embeddings=position_embeddings, + **kwargs, + ) + hidden_states = layer_outputs + + hidden_states = self.norm(hidden_states) + + return BaseModelOutputWithPast( + last_hidden_state=hidden_states, + past_key_values=past_key_values, + ) + + class Glm46VisionMlp(nn.Module): def __init__(self, config, bias: bool = False): super().__init__() @@ -613,6 +789,30 @@ def forward( return hidden_states +@dataclass +@auto_docstring( + custom_intro=""" + Base class for Llava outputs, with hidden states and attentions. + """ +) +class Glm46VModelOutputWithPast(ModelOutput): + r""" + past_key_values (`Cache`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`): + It is a [`~cache_utils.Cache`] instance. For more details, see our [kv cache guide](https://huggingface.co/docs/transformers/en/kv_cache). + + Contains pre-computed hidden-states (key and values in the self-attention blocks) that can be used (see + `past_key_values` input) to speed up sequential decoding. + rope_deltas (`torch.LongTensor` of shape `(batch_size, )`, *optional*): + The rope index difference between sequence length and multimodal rope. + """ + + last_hidden_state: Optional[torch.FloatTensor] = None + past_key_values: Optional[Cache] = None + hidden_states: Optional[tuple[torch.FloatTensor]] = None + attentions: Optional[tuple[torch.FloatTensor]] = None + rope_deltas: Optional[torch.LongTensor] = None + + class Glm46VVisionModel(Glm46VPreTrainedModel): config: Glm46VVisionConfig input_modalities = ["image", "video"] @@ -724,206 +924,6 @@ def forward(self, hidden_states: torch.Tensor, grid_thw: torch.Tensor) -> torch. return hidden_states -class Glm46VTextRotaryEmbedding(nn.Module): - inv_freq: torch.Tensor # fix linting for `register_buffer` - - def __init__(self, config: Glm46VTextConfig, device=None): - super().__init__() - self.max_seq_len_cached = config.max_position_embeddings - self.original_max_seq_len = config.max_position_embeddings - - self.config = config - - self.rope_type = self.config.rope_parameters["rope_type"] - rope_init_fn: Callable = self.compute_default_rope_parameters - if self.rope_type != "default": - rope_init_fn = ROPE_INIT_FUNCTIONS[self.rope_type] - inv_freq, self.attention_scaling = rope_init_fn(self.config, device) - - self.register_buffer("inv_freq", inv_freq, persistent=False) - self.original_inv_freq = inv_freq - - @staticmethod - def compute_default_rope_parameters( - config: Optional[Glm46VTextConfig] = None, - device: Optional["torch.device"] = None, - seq_len: Optional[int] = None, - ) -> tuple["torch.Tensor", float]: - """ - Computes the inverse frequencies according to the original RoPE implementation - Args: - config ([`~transformers.PreTrainedConfig`]): - The model configuration. - device (`torch.device`): - The device to use for initialization of the inverse frequencies. - seq_len (`int`, *optional*): - The current sequence length. Unused for this type of RoPE. - Returns: - Tuple of (`torch.Tensor`, `float`), containing the inverse frequencies for the RoPE embeddings and the - post-processing scaling factor applied to the computed cos/sin (unused in this type of RoPE). - """ - base = config.rope_parameters["rope_theta"] - partial_rotary_factor = getattr(config, "partial_rotary_factor", 1.0) - head_dim = getattr(config, "head_dim", None) or config.hidden_size // config.num_attention_heads - dim = int(head_dim * partial_rotary_factor) - - attention_factor = 1.0 # Unused in this type of RoPE - - # Compute the inverse frequencies - inv_freq = 1.0 / ( - base ** (torch.arange(0, dim, 2, dtype=torch.int64).to(device=device, dtype=torch.float) / dim) - ) - return inv_freq, attention_factor - - @torch.no_grad() - @dynamic_rope_update # power user: used with advanced RoPE types (e.g. dynamic rope) - def forward(self, x, position_ids): - # In contrast to other models, GLM46V different position ids for the grids - # So we expand the inv_freq to shape (3, ...) - inv_freq_expanded = self.inv_freq[None, None, :, None].float().expand(3, position_ids.shape[1], -1, 1) - position_ids_expanded = position_ids[:, :, None, :].float() # shape (3, bs, 1, positions) - - device_type = x.device.type if isinstance(x.device.type, str) and x.device.type != "mps" else "cpu" - with torch.autocast(device_type=device_type, enabled=False): # Force float32 - freqs = (inv_freq_expanded.float() @ position_ids_expanded.float()).transpose(2, 3) - emb = torch.cat((freqs, freqs), dim=-1) - cos = emb.cos() * self.attention_scaling - sin = emb.sin() * self.attention_scaling - - return cos.to(dtype=x.dtype), sin.to(dtype=x.dtype) - - -@auto_docstring -class Glm46VTextModel(Glm46VPreTrainedModel): - config: Glm46VTextConfig - input_modalities = "text" - - def __init__(self, config: Glm46VTextConfig): - super().__init__(config) - self.padding_idx = config.pad_token_id - self.vocab_size = config.vocab_size - - self.embed_tokens = nn.Embedding(config.vocab_size, config.hidden_size, self.padding_idx) - self.layers = nn.ModuleList( - [Glm46VTextDecoderLayer(config, layer_idx) for layer_idx in range(config.num_hidden_layers)] - ) - self.norm = Glm46VRMSNorm(config.hidden_size, eps=config.rms_norm_eps) - self.rotary_emb = Glm46VTextRotaryEmbedding(config=config) - - self.gradient_checkpointing = False - # Initialize weights and apply final processing - self.post_init() - - @auto_docstring - @check_model_inputs() - def forward( - self, - input_ids: Optional[torch.LongTensor] = None, - attention_mask: Optional[torch.Tensor] = None, - position_ids: Optional[torch.LongTensor] = None, - past_key_values: Optional[Cache] = None, - inputs_embeds: Optional[torch.FloatTensor] = None, - use_cache: Optional[bool] = None, - cache_position: Optional[torch.LongTensor] = None, - **kwargs: Unpack[FlashAttentionKwargs], - ) -> Union[tuple, BaseModelOutputWithPast]: - if (input_ids is None) ^ (inputs_embeds is not None): - raise ValueError("You must specify exactly one of input_ids or inputs_embeds") - - # torch.jit.trace() doesn't support cache objects in the output - if use_cache and past_key_values is None and not torch.jit.is_tracing(): - past_key_values = DynamicCache(config=self.config) - - if inputs_embeds is None: - inputs_embeds = self.embed_tokens(input_ids) - - if cache_position is None: - past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 - cache_position = torch.arange( - past_seen_tokens, past_seen_tokens + inputs_embeds.shape[1], device=inputs_embeds.device - ) - - # the hard coded `3` is for temporal, height and width. - if position_ids is None: - position_ids = cache_position.view(1, 1, -1).expand(3, inputs_embeds.shape[0], -1) - elif position_ids.ndim == 2: - position_ids = position_ids[None, ...].expand(3, position_ids.shape[0], -1) - - # NOTE: we need to pass text position ids for packing. Qwen2-VL uses 3D positions - # where each dim indicates visual spatial positions for temporal/height/width grids. - # There are two scenarios when FA2-like packed masking might be activated. - # 1. User specifically passed packed `position_ids` and no attention mask. - # In this case we expect the useer to create correct position ids for all 3 grids - # and prepend text-only position ids to it. The final tensor will be [4, bs, seq-len] - # 2. User runs forward with no attention mask and no position ids. In this case, position ids - # are prepared by the model (`get_rope_index`) as `[4, bs, seq-len]` tensor. Text-only positions are - # prepended by us when creating positions so that the mask is constructed correctly. NOTE: failing to pass - # text-only positions will cause incorrect mask construction, do not change `prepare_input_for_generation` - if position_ids.ndim == 3 and position_ids.shape[0] == 4: - text_position_ids = position_ids[0] - position_ids = position_ids[1:] - else: - # If inputs are not packed (usual 3D positions), do not prepare mask from position_ids - text_position_ids = None - - mask_kwargs = { - "config": self.config, - "input_embeds": inputs_embeds, - "attention_mask": attention_mask, - "cache_position": cache_position, - "past_key_values": past_key_values, - "position_ids": text_position_ids, - } - # Create the masks - causal_mask = create_causal_mask(**mask_kwargs) - - hidden_states = inputs_embeds - position_embeddings = self.rotary_emb(hidden_states, position_ids=position_ids) - - for decoder_layer in self.layers: - layer_outputs = decoder_layer( - hidden_states, - attention_mask=causal_mask, - position_ids=position_ids, - past_key_values=past_key_values, - cache_position=cache_position, - position_embeddings=position_embeddings, - **kwargs, - ) - hidden_states = layer_outputs - - hidden_states = self.norm(hidden_states) - - return BaseModelOutputWithPast( - last_hidden_state=hidden_states, - past_key_values=past_key_values, - ) - - -@dataclass -@auto_docstring( - custom_intro=""" - Base class for Llava outputs, with hidden states and attentions. - """ -) -class Glm46VModelOutputWithPast(ModelOutput): - r""" - past_key_values (`Cache`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`): - It is a [`~cache_utils.Cache`] instance. For more details, see our [kv cache guide](https://huggingface.co/docs/transformers/en/kv_cache). - - Contains pre-computed hidden-states (key and values in the self-attention blocks) that can be used (see - `past_key_values` input) to speed up sequential decoding. - rope_deltas (`torch.LongTensor` of shape `(batch_size, )`, *optional*): - The rope index difference between sequence length and multimodal rope. - """ - - last_hidden_state: Optional[torch.FloatTensor] = None - past_key_values: Optional[Cache] = None - hidden_states: Optional[tuple[torch.FloatTensor]] = None - attentions: Optional[tuple[torch.FloatTensor]] = None - rope_deltas: Optional[torch.LongTensor] = None - - @auto_docstring class Glm46VModel(Glm46VPreTrainedModel): base_model_prefix = "" diff --git a/src/transformers/models/glm46v/modular_glm46v.py b/src/transformers/models/glm46v/modular_glm46v.py index 05ee02126630..a586e8c79c1c 100644 --- a/src/transformers/models/glm46v/modular_glm46v.py +++ b/src/transformers/models/glm46v/modular_glm46v.py @@ -14,7 +14,7 @@ # limitations under the License. from ..glm4v.configuration_glm4v import Glm4vConfig, Glm4vTextConfig -from ..glm4v.modeling_glm4v import Glm4vModel, Glm4vPreTrainedModel, Glm4vTextModel, Glm4vVisionModel +from ..glm4v.modeling_glm4v import Glm4vModel, Glm4vPreTrainedModel, Glm4vTextModel from ..glm4v.processing_glm4v import Glm4vProcessor @@ -30,10 +30,6 @@ class Glm46VPreTrainedModel(Glm4vPreTrainedModel): pass -class Glm46VVisionModel(Glm4vVisionModel): - pass - - class Glm46VTextModel(Glm4vTextModel): pass From c1425c3753188497658b2d88a76eaab60adfc45c Mon Sep 17 00:00:00 2001 From: zRzRzRzRzRzRzR <2448370773@qq.com> Date: Wed, 12 Nov 2025 17:59:07 +0800 Subject: [PATCH 22/66] upload with modular --- .../models/glm46v/modular_glm46v.py | 63 +++++++++++++++++++ .../models/glm46v/video_processing_glm46v.py | 61 +++++++++--------- 2 files changed, 93 insertions(+), 31 deletions(-) diff --git a/src/transformers/models/glm46v/modular_glm46v.py b/src/transformers/models/glm46v/modular_glm46v.py index a586e8c79c1c..5c1725764738 100644 --- a/src/transformers/models/glm46v/modular_glm46v.py +++ b/src/transformers/models/glm46v/modular_glm46v.py @@ -13,9 +13,15 @@ # See the License for the specific language governing permissions and # limitations under the License. +from typing import Optional, Union + +import numpy as np + +from ...video_utils import VideoMetadata from ..glm4v.configuration_glm4v import Glm4vConfig, Glm4vTextConfig from ..glm4v.modeling_glm4v import Glm4vModel, Glm4vPreTrainedModel, Glm4vTextModel from ..glm4v.processing_glm4v import Glm4vProcessor +from ..glm4v.video_processing_glm4v import Glm4vVideoProcessor class Glm46VTextConfig(Glm4vTextConfig): @@ -43,6 +49,62 @@ def replace_frame_token_id(self, timestamp_sec): return f"<|begin_of_image|>{self.image_token}<|end_of_image|>{timestamp_sec:.1f} seconds" +class Glm46VVideoProcessor(Glm4vVideoProcessor): + def sample_frames( + self, + metadata: VideoMetadata, + fps: Optional[Union[int, float]] = None, + **kwargs, + ): + if metadata is None or getattr(metadata, "fps", None) is None: + raise ValueError( + "Asked to sample frames per second but no video metadata was provided which is required when sampling in Glm46V. " + "Please pass in `VideoMetadata` object or set `do_sample_frames=False`" + ) + + total_frames = metadata.total_num_frames + max_frame_idx = total_frames - 1 + duration = metadata.duration or round(max_frame_idx / metadata.fps) + 1 + + DYNAMIC_FPS_THRES = {30: 3, 300: 1, 2400: 0.5} + MAX_FRAME_COUNT_DYNAMIC = 640 + MAX_DURATION = 2400 + effective_duration = min(duration, MAX_DURATION) + if effective_duration <= 30: + target_fps = DYNAMIC_FPS_THRES[30] + elif effective_duration <= 300: + target_fps = DYNAMIC_FPS_THRES[300] + else: + target_fps = DYNAMIC_FPS_THRES[2400] + extract_t = int(effective_duration * target_fps * self.temporal_patch_size) + extract_t = min(extract_t, MAX_FRAME_COUNT_DYNAMIC) + + duration_per_frame = 1 / metadata.fps + timestamps = [i * duration_per_frame for i in range(total_frames)] + max_second = int(duration) + + frame_indices = [] + current_second = 0 + inv_fps = 1 / (self.temporal_patch_size * target_fps) + for frame_index in range(total_frames): + if timestamps[frame_index] >= current_second: + current_second += inv_fps + frame_indices.append(frame_index) + if current_second >= max_second: + break + + if len(frame_indices) > extract_t: + frame_indices = np.linspace(0, total_frames - 1, extract_t, dtype=int).tolist() + + seen, uniq = set(), [] + for idx in frame_indices: + if idx not in seen: + seen.add(idx) + uniq.append(idx) + + return np.array(uniq) + + __all__ = [ "Glm46VConfig", "Glm46VTextConfig", @@ -50,4 +112,5 @@ def replace_frame_token_id(self, timestamp_sec): "Glm46VPreTrainedModel", "Glm46VTextModel", "Glm46VProcessor", + "Glm46VVideoProcessor", ] diff --git a/src/transformers/models/glm46v/video_processing_glm46v.py b/src/transformers/models/glm46v/video_processing_glm46v.py index 0b5308a62b40..4008117be89d 100644 --- a/src/transformers/models/glm46v/video_processing_glm46v.py +++ b/src/transformers/models/glm46v/video_processing_glm46v.py @@ -1,5 +1,11 @@ +# 🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨 +# This file was automatically generated from src/transformers/models/glm46v/modular_glm46v.py. +# Do NOT edit this file manually as any edits will be overwritten by the generation of +# the file from the modular. If any change should be done, please apply the change to the +# modular_glm46v.py file directly. One of our CI enforces this. +# 🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨 # coding=utf-8 -# Copyright 2025 The ZhipuAI Inc. team and HuggingFace Inc. team. All rights reserved. +# Copyright 2025 the HuggingFace Team. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -12,14 +18,13 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -"""video processor class for GLM-4.6V.""" from typing import Optional, Union import numpy as np import torch -from ...image_processing_utils import BatchFeature +from ...feature_extraction_utils import BatchFeature from ...image_utils import ( OPENAI_CLIP_MEAN, OPENAI_CLIP_STD, @@ -32,7 +37,7 @@ from ...utils import TensorType, add_start_docstrings from ...video_processing_utils import BASE_VIDEO_PROCESSOR_DOCSTRING, BaseVideoProcessor from ...video_utils import VideoMetadata, group_videos_by_shape, reorder_videos -from ..glm4v.image_processing_glm4v import smart_resize +from .image_processing_glm46v import smart_resize class Glm46VVideoProcessorInitKwargs(VideosKwargs, total=False): @@ -103,6 +108,16 @@ def sample_frames( fps: Optional[Union[int, float]] = None, **kwargs, ): + """ + Args: + metadata (`VideoMetadata`): + Metadata of the video containing information about total duration, fps and total number of frames. + fps (`int` or `float`, *optional*): + Target frames to sample per second. Defaults to `self.fps`. + Returns: + np.ndarray: + Indices to sample video frames. + """ if metadata is None or getattr(metadata, "fps", None) is None: raise ValueError( "Asked to sample frames per second but no video metadata was provided which is required when sampling in Glm46V. " @@ -130,31 +145,18 @@ def sample_frames( timestamps = [i * duration_per_frame for i in range(total_frames)] max_second = int(duration) - if total_frames < extract_t: + frame_indices = [] + current_second = 0 + inv_fps = 1 / (self.temporal_patch_size * target_fps) + for frame_index in range(total_frames): + if timestamps[frame_index] >= current_second: + current_second += inv_fps + frame_indices.append(frame_index) + if current_second >= max_second: + break + + if len(frame_indices) > extract_t: frame_indices = np.linspace(0, total_frames - 1, extract_t, dtype=int).tolist() - else: - frame_indices = [] - current_second = 0 - inv_fps = 1 / (self.temporal_patch_size * target_fps) - for frame_index in range(total_frames): - if timestamps[frame_index] >= current_second: - current_second += inv_fps - frame_indices.append(frame_index) - if current_second >= max_second: - break - - if len(frame_indices) < 3: - frame_indices = np.linspace(0, total_frames - 1, extract_t, dtype=int).tolist() - - if len(frame_indices) < extract_t: - if len(frame_indices) < extract_t - 30: - print(f"Warning: Only {len(frame_indices)} frames extracted, but {extract_t} required. Padding.") - frame_indices = np.linspace(frame_indices[0], frame_indices[-1], extract_t, dtype=int).tolist() - elif len(frame_indices) > extract_t: - frame_indices = np.linspace(0, total_frames - 1, extract_t, dtype=int).tolist() - - if len(frame_indices) % self.temporal_patch_size != 0: - frame_indices.append(frame_indices[-1]) seen, uniq = set(), [] for idx in frame_indices: @@ -162,9 +164,6 @@ def sample_frames( seen.add(idx) uniq.append(idx) - if len(uniq) & 1: - uniq.append(uniq[-1]) - return np.array(uniq) def _preprocess( From 9e184af9b5d9a8cf47bd04fa49614955c5f57ba8 Mon Sep 17 00:00:00 2001 From: zRzRzRzRzRzRzR <2448370773@qq.com> Date: Wed, 12 Nov 2025 18:00:30 +0800 Subject: [PATCH 23/66] 1 --- src/transformers/models/glm46v/modular_glm46v.py | 3 +++ src/transformers/models/glm46v/video_processing_glm46v.py | 3 +++ 2 files changed, 6 insertions(+) diff --git a/src/transformers/models/glm46v/modular_glm46v.py b/src/transformers/models/glm46v/modular_glm46v.py index 5c1725764738..8f3390165d05 100644 --- a/src/transformers/models/glm46v/modular_glm46v.py +++ b/src/transformers/models/glm46v/modular_glm46v.py @@ -102,6 +102,9 @@ def sample_frames( seen.add(idx) uniq.append(idx) + if len(uniq) & 1: + uniq.append(uniq[-1]) + return np.array(uniq) diff --git a/src/transformers/models/glm46v/video_processing_glm46v.py b/src/transformers/models/glm46v/video_processing_glm46v.py index 4008117be89d..6074345f52cf 100644 --- a/src/transformers/models/glm46v/video_processing_glm46v.py +++ b/src/transformers/models/glm46v/video_processing_glm46v.py @@ -164,6 +164,9 @@ def sample_frames( seen.add(idx) uniq.append(idx) + if len(uniq) & 1: + uniq.append(uniq[-1]) + return np.array(uniq) def _preprocess( From 0bd1a50b2c88e6af96da7d939c9d274213408f4f Mon Sep 17 00:00:00 2001 From: zRzRzRzRzRzRzR <2448370773@qq.com> Date: Wed, 12 Nov 2025 18:50:08 +0800 Subject: [PATCH 24/66] - --- .../models/glm46v/modular_glm46v.py | 31 ++++++++++++------- .../models/glm46v/video_processing_glm46v.py | 29 +++++++++++------ 2 files changed, 39 insertions(+), 21 deletions(-) diff --git a/src/transformers/models/glm46v/modular_glm46v.py b/src/transformers/models/glm46v/modular_glm46v.py index 8f3390165d05..9b8dce4f048b 100644 --- a/src/transformers/models/glm46v/modular_glm46v.py +++ b/src/transformers/models/glm46v/modular_glm46v.py @@ -83,17 +83,26 @@ def sample_frames( timestamps = [i * duration_per_frame for i in range(total_frames)] max_second = int(duration) - frame_indices = [] - current_second = 0 - inv_fps = 1 / (self.temporal_patch_size * target_fps) - for frame_index in range(total_frames): - if timestamps[frame_index] >= current_second: - current_second += inv_fps - frame_indices.append(frame_index) - if current_second >= max_second: - break - - if len(frame_indices) > extract_t: + if total_frames < extract_t: + frame_indices = np.linspace(0, total_frames - 1, extract_t, dtype=int).tolist() + else: + frame_indices = [] + current_second = 0 + inv_fps = 1 / (self.temporal_patch_size * target_fps) + for frame_index in range(total_frames): + if timestamps[frame_index] >= current_second: + current_second += inv_fps + frame_indices.append(frame_index) + if current_second >= max_second: + break + + if len(frame_indices) < extract_t: + if len(frame_indices) == 0: + start, end = 0, max(total_frames - 1, 0) + else: + start, end = frame_indices[0], frame_indices[-1] + frame_indices = np.linspace(start, end, extract_t, dtype=int).tolist() + elif len(frame_indices) > extract_t: frame_indices = np.linspace(0, total_frames - 1, extract_t, dtype=int).tolist() seen, uniq = set(), [] diff --git a/src/transformers/models/glm46v/video_processing_glm46v.py b/src/transformers/models/glm46v/video_processing_glm46v.py index 6074345f52cf..ab99312385da 100644 --- a/src/transformers/models/glm46v/video_processing_glm46v.py +++ b/src/transformers/models/glm46v/video_processing_glm46v.py @@ -145,17 +145,26 @@ def sample_frames( timestamps = [i * duration_per_frame for i in range(total_frames)] max_second = int(duration) - frame_indices = [] - current_second = 0 - inv_fps = 1 / (self.temporal_patch_size * target_fps) - for frame_index in range(total_frames): - if timestamps[frame_index] >= current_second: - current_second += inv_fps - frame_indices.append(frame_index) - if current_second >= max_second: - break + if total_frames < extract_t: + frame_indices = np.linspace(0, total_frames - 1, extract_t, dtype=int).tolist() + else: + frame_indices = [] + current_second = 0 + inv_fps = 1 / (self.temporal_patch_size * target_fps) + for frame_index in range(total_frames): + if timestamps[frame_index] >= current_second: + current_second += inv_fps + frame_indices.append(frame_index) + if current_second >= max_second: + break - if len(frame_indices) > extract_t: + if len(frame_indices) < extract_t: + if len(frame_indices) == 0: + start, end = 0, max(total_frames - 1, 0) + else: + start, end = frame_indices[0], frame_indices[-1] + frame_indices = np.linspace(start, end, extract_t, dtype=int).tolist() + elif len(frame_indices) > extract_t: frame_indices = np.linspace(0, total_frames - 1, extract_t, dtype=int).tolist() seen, uniq = set(), [] From 1f74fa796e12bdd83ac2e69e079f21849199c5e0 Mon Sep 17 00:00:00 2001 From: zRzRzRzRzRzRzR <2448370773@qq.com> Date: Wed, 12 Nov 2025 18:56:12 +0800 Subject: [PATCH 25/66] update --- .../models/glm46v/image_processing_glm46v.py | 481 ++++++++++++++++++ .../glm46v/image_processing_glm46v_fast.py | 183 +++++++ .../models/glm46v/modeling_glm46v.py | 2 +- .../models/glm46v/modular_glm46v.py | 12 + .../models/glm46v/processing_glm46v.py | 2 +- .../models/glm46v/video_processing_glm46v.py | 2 +- 6 files changed, 679 insertions(+), 3 deletions(-) create mode 100644 src/transformers/models/glm46v/image_processing_glm46v.py create mode 100644 src/transformers/models/glm46v/image_processing_glm46v_fast.py diff --git a/src/transformers/models/glm46v/image_processing_glm46v.py b/src/transformers/models/glm46v/image_processing_glm46v.py new file mode 100644 index 000000000000..bb0f46f8b052 --- /dev/null +++ b/src/transformers/models/glm46v/image_processing_glm46v.py @@ -0,0 +1,481 @@ +# 🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨 +# This file was automatically generated from src/transformers/models/glm46v/modular_glm46v.py. +# Do NOT edit this file manually as any edits will be overwritten by the generation of +# the file from the modular. If any change should be done, please apply the change to the +# modular_glm46v.py file directly. One of our CI enforces this. +# 🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨 +# coding=utf-8 +# Copyright 2025 the HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import math +from typing import Optional, Union + +import numpy as np + +from ...image_processing_utils import BaseImageProcessor, BatchFeature +from ...image_transforms import convert_to_rgb, resize, to_channel_dimension_format +from ...image_utils import ( + OPENAI_CLIP_MEAN, + OPENAI_CLIP_STD, + ChannelDimension, + ImageInput, + PILImageResampling, + get_image_size, + infer_channel_dimension_format, + is_scaled_image, + make_flat_list_of_images, + to_numpy_array, + valid_images, + validate_preprocess_arguments, +) +from ...processing_utils import ImagesKwargs +from ...utils import TensorType, logging +from ...video_utils import VideoInput + + +logger = logging.get_logger(__name__) + + +class Glm46VImageProcessorKwargs(ImagesKwargs, total=False): + """ + patch_size (`int`, *optional*, defaults to 14): + The spatial patch size of the vision encoder. + temporal_patch_size (`int`, *optional*, defaults to 2): + The temporal patch size of the vision encoder. + merge_size (`int`, *optional*, defaults to 2): + The merge size of the vision encoder to llm encoder. + """ + + patch_size: int + temporal_patch_size: int + merge_size: int + + +def smart_resize( + num_frames: int, + height: int, + width: int, + temporal_factor: int = 2, + factor: int = 28, + min_pixels: int = 112 * 112, + max_pixels: int = 14 * 14 * 2 * 2 * 2 * 6144, +): + if num_frames < temporal_factor: + raise ValueError(f"t:{num_frames} must be larger than temporal_factor:{temporal_factor}") + if height < factor or width < factor: + raise ValueError(f"height:{height} or width:{width} must be larger than factor:{factor}") + elif max(height, width) / min(height, width) > 200: + raise ValueError( + f"absolute aspect ratio must be smaller than 200, got {max(height, width) / min(height, width)}" + ) + h_bar = round(height / factor) * factor + w_bar = round(width / factor) * factor + t_bar = round(num_frames / temporal_factor) * temporal_factor + + if t_bar * h_bar * w_bar > max_pixels: + beta = math.sqrt((num_frames * height * width) / max_pixels) + h_bar = max(factor, math.floor(height / beta / factor) * factor) + w_bar = max(factor, math.floor(width / beta / factor) * factor) + elif t_bar * h_bar * w_bar < min_pixels: + beta = math.sqrt(min_pixels / (num_frames * height * width)) + h_bar = math.ceil(height * beta / factor) * factor + w_bar = math.ceil(width * beta / factor) * factor + + return h_bar, w_bar + + +class Glm46VImageProcessor(BaseImageProcessor): + r""" + Constructs a GLM-4V image processor that dynamically resizes images based on the original images. + + Args: + do_resize (`bool`, *optional*, defaults to `True`): + Whether to resize the image's (height, width) dimensions. + size (`Dict[str, int]` *optional*, defaults to `{"shortest_edge": 112 * 112, "longest_edge": 28 * 28 * 15000}`): + Size of the image's `(height, width)` dimensions after resizing. Can be overridden by the `size` parameter + in the `preprocess` method. Available options are: + - `{"height": int, "width": int}`: The image will be resized to the exact size `(height, width)`. + Do NOT keep the aspect ratio. + - `{"shortest_edge": int, "longest_edge": int}`: The image will be resized to a maximum size respecting + the aspect ratio and keeping the shortest edge less or equal to `shortest_edge` and the longest edge + less or equal to `longest_edge`. + - `{"max_height": int, "max_width": int}`: The image will be resized to the maximum size respecting the + aspect ratio and keeping the height less or equal to `max_height` and the width less or equal to + `max_width`. + resample (`PILImageResampling`, *optional*, defaults to `Resampling.BICUBIC`): + Resampling filter to use when resizing the image. + do_rescale (`bool`, *optional*, defaults to `True`): + Whether to rescale the image by the specified scale `rescale_factor`. + rescale_factor (`int` or `float`, *optional*, defaults to `1/255`): + Scale factor to use if rescaling the image. + do_normalize (`bool`, *optional*, defaults to `True`): + Whether to normalize the image. + image_mean (`float` or `List[float]`, *optional*, defaults to `[0.48145466, 0.4578275, 0.40821073]`): + Mean to use if normalizing the image. This is a float or list of floats for each channel in the image. + image_std (`float` or `List[float]`, *optional*, defaults to `[0.26862954, 0.26130258, 0.27577711]`): + Standard deviation to use if normalizing the image. This is a float or list of floats for each channel in the image. + do_convert_rgb (`bool`, *optional*, defaults to `True`): + Whether to convert the image to RGB. + patch_size (`int`, *optional*, defaults to 14): + The spatial patch size of the vision encoder. + temporal_patch_size (`int`, *optional*, defaults to 2): + The temporal patch size of the vision encoder. + merge_size (`int`, *optional*, defaults to 2): + The merge size of the vision encoder to llm encoder. + """ + + model_input_names = ["pixel_values", "image_grid_thw"] + valid_kwargs = Glm46VImageProcessorKwargs + + def __init__( + self, + do_resize: bool = True, + size: Optional[dict[str, int]] = None, + resample: PILImageResampling = PILImageResampling.BICUBIC, + do_rescale: bool = True, + rescale_factor: Union[int, float] = 1 / 255, + do_normalize: bool = True, + image_mean: Optional[Union[float, list[float]]] = None, + image_std: Optional[Union[float, list[float]]] = None, + do_convert_rgb: bool = True, + patch_size: int = 14, + temporal_patch_size: int = 2, + merge_size: int = 2, + **kwargs, + ) -> None: + super().__init__(**kwargs) + if size is not None and ("shortest_edge" not in size or "longest_edge" not in size): + raise ValueError("size must contain 'shortest_edge' and 'longest_edge' keys.") + elif size is None: + size = {"shortest_edge": 112 * 112, "longest_edge": 28 * 28 * 15000} + self.size = size + + self.do_resize = do_resize + self.resample = resample + self.do_rescale = do_rescale + self.rescale_factor = rescale_factor + self.do_normalize = do_normalize + self.image_mean = image_mean if image_mean is not None else OPENAI_CLIP_MEAN + self.image_std = image_std if image_std is not None else OPENAI_CLIP_STD + + self.patch_size = patch_size + self.temporal_patch_size = temporal_patch_size + self.merge_size = merge_size + self.do_convert_rgb = do_convert_rgb + + def _preprocess( + self, + images: Union[ImageInput, VideoInput], + do_resize: Optional[bool] = None, + size: Optional[dict[str, int]] = None, + resample: Optional[PILImageResampling] = None, + do_rescale: Optional[bool] = None, + rescale_factor: Optional[float] = None, + do_normalize: Optional[bool] = None, + image_mean: Optional[Union[float, list[float]]] = None, + image_std: Optional[Union[float, list[float]]] = None, + patch_size: Optional[int] = None, + temporal_patch_size: Optional[int] = None, + merge_size: Optional[int] = None, + do_convert_rgb: Optional[bool] = None, + data_format: Optional[ChannelDimension] = ChannelDimension.FIRST, + input_data_format: Optional[Union[str, ChannelDimension]] = None, + ): + """ + Preprocess an image or batch of images. Copy of the `preprocess` method from `CLIPImageProcessor`. + + Args: + images (`ImageInput`): + Image or batch of images to preprocess. Expects pixel values ranging from 0 to 255. If pixel values range from 0 to 1, set `do_rescale=False`. + vision_info (`List[Dict]`, *optional*): + Optional list of dictionaries containing additional information about vision inputs. + do_resize (`bool`, *optional*, defaults to `self.do_resize`): + Whether to resize the image. + size (`Dict[str, int]`, *optional*, defaults to `self.size`): + Size of the image after resizing. `shortest_edge` and `longest_edge` keys must be present. + resample (`PILImageResampling`, *optional*, defaults to `self.resample`): + Resampling filter to use if resizing the image. This can be one of the `PILImageResampling` enums. + do_rescale (`bool`, *optional*, defaults to `self.do_rescale`): + Whether to rescale the image. + rescale_factor (`float`, *optional*, defaults to `self.rescale_factor`): + Scale factor to use if rescaling the image. + do_normalize (`bool`, *optional*, defaults to `self.do_normalize`): + Whether to normalize the image. + image_mean (`float` or `List[float]`, *optional*, defaults to `self.image_mean`): + Mean to use if normalizing the image. Can be a float or a list of floats corresponding to the number of channels in the image. + image_std (`float` or `List[float]`, *optional*, defaults to `self.image_std`): + Standard deviation to use if normalizing the image. Can be a float or a list of floats corresponding to the number of channels in the image. + patch_size (`int`, *optional*, defaults to `self.patch_size`): + The spatial patch size of the vision encoder. + temporal_patch_size (`int`, *optional*, defaults to `self.temporal_patch_size`): + The temporal patch size of the vision encoder. + merge_size (`int`, *optional*, defaults to `self.merge_size`): + The merge size of the vision encoder to llm encoder. + do_convert_rgb (`bool`, *optional*, defaults to `self.do_convert_rgb`): + Whether to convert the image to RGB. + data_format (`ChannelDimension`, *optional*, defaults to `ChannelDimension.FIRST`): + The channel dimension format for the output image. Can be one of: + - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. + - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. + - Unset: Use the channel dimension format of the input image. + input_data_format (`ChannelDimension` or `str`, *optional*): + The channel dimension format for the input image. Can be one of: + - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. + - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. + - `"none"` or `ChannelDimension.NONE`: image in (height, width) format. - `"none"` or `ChannelDimension.NONE`: image in (height, width) format. + """ + images = make_flat_list_of_images(images) + + if do_convert_rgb: + images = [convert_to_rgb(image) for image in images] + + # All transformations expect numpy arrays. + images = [to_numpy_array(image) for image in images] + + if do_rescale and is_scaled_image(images[0]): + logger.warning_once( + "It looks like you are trying to rescale already rescaled images. If the input" + " images have pixel values between 0 and 1, set `do_rescale=False` to avoid rescaling them again." + ) + if input_data_format is None: + # We assume that all images have the same channel dimension format. + input_data_format = infer_channel_dimension_format(images[0]) + + height, width = get_image_size(images[0], channel_dim=input_data_format) + resized_height, resized_width = height, width + processed_images = [] + for image in images: + if do_resize: + resized_height, resized_width = smart_resize( + num_frames=temporal_patch_size, + height=height, + width=width, + temporal_factor=temporal_patch_size, + factor=patch_size * merge_size, + min_pixels=size["shortest_edge"], + max_pixels=size["longest_edge"], + ) + image = resize( + image, size=(resized_height, resized_width), resample=resample, input_data_format=input_data_format + ) + + if do_rescale: + image = self.rescale(image, scale=rescale_factor, input_data_format=input_data_format) + + if do_normalize: + image = self.normalize( + image=image, mean=image_mean, std=image_std, input_data_format=input_data_format + ) + + image = to_channel_dimension_format(image, data_format, input_channel_dim=input_data_format) + processed_images.append(image) + + patches = np.array(processed_images) + if data_format == ChannelDimension.LAST: + patches = patches.transpose(0, 3, 1, 2) + if patches.shape[0] % temporal_patch_size != 0: + repeats = np.repeat( + patches[-1][np.newaxis], temporal_patch_size - (patches.shape[0] % temporal_patch_size), axis=0 + ) + patches = np.concatenate([patches, repeats], axis=0) + channel = patches.shape[1] + grid_t = patches.shape[0] // temporal_patch_size + grid_h, grid_w = resized_height // patch_size, resized_width // patch_size + patches = patches.reshape( + grid_t, + temporal_patch_size, + channel, + grid_h // merge_size, + merge_size, + patch_size, + grid_w // merge_size, + merge_size, + patch_size, + ) + patches = patches.transpose(0, 3, 6, 4, 7, 2, 1, 5, 8) + flatten_patches = patches.reshape( + grid_t * grid_h * grid_w, channel * temporal_patch_size * patch_size * patch_size + ) + + return flatten_patches, (grid_t, grid_h, grid_w) + + def preprocess( + self, + images: ImageInput, + do_resize: Optional[bool] = None, + size: Optional[dict[str, int]] = None, + resample: Optional[PILImageResampling] = None, + do_rescale: Optional[bool] = None, + rescale_factor: Optional[float] = None, + do_normalize: Optional[bool] = None, + image_mean: Optional[Union[float, list[float]]] = None, + image_std: Optional[Union[float, list[float]]] = None, + patch_size: Optional[int] = None, + temporal_patch_size: Optional[int] = None, + merge_size: Optional[int] = None, + do_convert_rgb: Optional[bool] = None, + return_tensors: Optional[Union[str, TensorType]] = None, + data_format: Optional[ChannelDimension] = ChannelDimension.FIRST, + input_data_format: Optional[Union[str, ChannelDimension]] = None, + ): + """ + Args: + images (`ImageInput`): + Image to preprocess. Expects a single or batch of images with pixel values ranging from 0 to 255. If + passing in images with pixel values between 0 and 1, set `do_rescale=False`. + do_resize (`bool`, *optional*, defaults to `self.do_resize`): + Whether to resize the image. + size (`Dict[str, int]`, *optional*, defaults to `self.size`): + Size of the image after resizing. Shortest edge of the image is resized to size["shortest_edge"], with + the longest edge resized to keep the input aspect ratio. + resample (`int`, *optional*, defaults to `self.resample`): + Resampling filter to use if resizing the image. This can be one of the enum `PILImageResampling`. Only + has an effect if `do_resize` is set to `True`. + do_rescale (`bool`, *optional*, defaults to `self.do_rescale`): + Whether to rescale the image. + rescale_factor (`float`, *optional*, defaults to `self.rescale_factor`): + Rescale factor to rescale the image by if `do_rescale` is set to `True`. + do_normalize (`bool`, *optional*, defaults to `self.do_normalize`): + Whether to normalize the image. + image_mean (`float` or `List[float]`, *optional*, defaults to `self.image_mean`): + Image mean to use for normalization. Only has an effect if `do_normalize` is set to `True`. + image_std (`float` or `List[float]`, *optional*, defaults to `self.image_std`): + Image standard deviation to use for normalization. Only has an effect if `do_normalize` is set to + `True`. + The max pixels of the image to resize the image. + patch_size (`int`, *optional*, defaults to `self.patch_size`): + The spatial patch size of the vision encoder. + temporal_patch_size (`int`, *optional*, defaults to `self.temporal_patch_size`): + The temporal patch size of the vision encoder. + merge_size (`int`, *optional*, defaults to `self.merge_size`): + The merge size of the vision encoder to llm encoder. + do_convert_rgb (`bool`, *optional*, defaults to `self.do_convert_rgb`): + Whether to convert the image to RGB. + return_tensors (`str` or `TensorType`, *optional*): + The type of tensors to return. Can be one of: + - Unset: Return a list of `np.ndarray`. + - `TensorType.PYTORCH` or `'pt'`: Return a batch of type `torch.Tensor`. + - `TensorType.NUMPY` or `'np'`: Return a batch of type `np.ndarray`. + data_format (`ChannelDimension` or `str`, *optional*, defaults to `ChannelDimension.FIRST`): + The channel dimension format for the output image. Can be one of: + - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. + - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. + - Unset: Use the channel dimension format of the input image. + input_data_format (`ChannelDimension` or `str`, *optional*): + The channel dimension format for the input image. If unset, the channel dimension format is inferred + from the input image. Can be one of: + - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. + - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. + - `"none"` or `ChannelDimension.NONE`: image in (height, width) format. + + """ + # Try to use config values if set, otherwise fallback to global defaults + size = size if size is not None else self.size + if size is not None and ("shortest_edge" not in size or "longest_edge" not in size): + raise ValueError("size must contain 'shortest_edge' and 'longest_edge' keys.") + elif size is None: + size = {"shortest_edge": 112 * 112, "longest_edge": 28 * 28 * 15000} + + do_resize = do_resize if do_resize is not None else self.do_resize + resample = resample if resample is not None else self.resample + do_rescale = do_rescale if do_rescale is not None else self.do_rescale + rescale_factor = rescale_factor if rescale_factor is not None else self.rescale_factor + do_normalize = do_normalize if do_normalize is not None else self.do_normalize + image_mean = image_mean if image_mean is not None else self.image_mean + image_std = image_std if image_std is not None else self.image_std + patch_size = patch_size if patch_size is not None else self.patch_size + temporal_patch_size = temporal_patch_size if temporal_patch_size is not None else self.temporal_patch_size + merge_size = merge_size if merge_size is not None else self.merge_size + do_convert_rgb = do_convert_rgb if do_convert_rgb is not None else self.do_convert_rgb + + if images is not None: + images = self.fetch_images(images) + images = make_flat_list_of_images(images) + + if images is not None and not valid_images(images): + raise ValueError("Invalid image type. Must be of type PIL.Image.Image, numpy.ndarray, or torch.Tensor") + + validate_preprocess_arguments( + rescale_factor=rescale_factor, + do_normalize=do_normalize, + image_mean=image_mean, + image_std=image_std, + do_resize=do_resize, + size=size, + resample=resample, + ) + + data = {} + if images is not None: + pixel_values, vision_grid_thws = [], [] + for image in images: + patches, image_grid_thw = self._preprocess( + image, + do_resize=do_resize, + size=size, + resample=resample, + do_rescale=do_rescale, + rescale_factor=rescale_factor, + do_normalize=do_normalize, + image_mean=image_mean, + image_std=image_std, + patch_size=patch_size, + temporal_patch_size=temporal_patch_size, + merge_size=merge_size, + data_format=data_format, + do_convert_rgb=do_convert_rgb, + input_data_format=input_data_format, + ) + pixel_values.extend(patches) + vision_grid_thws.append(image_grid_thw) + pixel_values = np.array(pixel_values) + vision_grid_thws = np.array(vision_grid_thws) + data.update({"pixel_values": pixel_values, "image_grid_thw": vision_grid_thws}) + + return BatchFeature(data=data, tensor_type=return_tensors) + + def get_number_of_image_patches(self, height: int, width: int, images_kwargs=None): + """ + A utility that returns number of image patches for a given image size. + + Args: + height (`int`): + Height of the input image. + width (`int`): + Width of the input image. + images_kwargs (`dict`, *optional*) + Any kwargs to override defaults of the image processor. + Returns: + `int`: Number of image patches per image. + """ + patch_size = images_kwargs.get("patch_size", self.patch_size) + merge_size = images_kwargs.get("merge_size", self.merge_size) + size = images_kwargs.get("size", {"shortest_edge": 112 * 112, "longest_edge": 28 * 28 * 15000}) + + factor = patch_size * merge_size + resized_height, resized_width = smart_resize( + num_frames=self.temporal_patch_size, + height=height, + width=width, + factor=factor, + min_pixels=size["shortest_edge"], + max_pixels=size["longest_edge"], + temporal_factor=self.temporal_patch_size, + ) + grid_h, grid_w = resized_height // patch_size, resized_width // patch_size + return grid_h * grid_w + + +__all__ = ["Glm46VImageProcessor"] diff --git a/src/transformers/models/glm46v/image_processing_glm46v_fast.py b/src/transformers/models/glm46v/image_processing_glm46v_fast.py new file mode 100644 index 000000000000..f86b78153783 --- /dev/null +++ b/src/transformers/models/glm46v/image_processing_glm46v_fast.py @@ -0,0 +1,183 @@ +# 🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨 +# This file was automatically generated from src/transformers/models/glm46v/modular_glm46v.py. +# Do NOT edit this file manually as any edits will be overwritten by the generation of +# the file from the modular. If any change should be done, please apply the change to the +# modular_glm46v.py file directly. One of our CI enforces this. +# 🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨 +# coding=utf-8 +# Copyright 2025 the HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Optional, Union + +import torch +from torchvision.transforms.v2 import functional as F + +from ...image_processing_utils import BatchFeature +from ...image_processing_utils_fast import BaseImageProcessorFast, group_images_by_shape, reorder_images +from ...image_utils import OPENAI_CLIP_MEAN, OPENAI_CLIP_STD, ImageInput, PILImageResampling, SizeDict +from ...processing_utils import Unpack +from ...utils import TensorType, auto_docstring +from .image_processing_glm46v import Glm46VImageProcessorKwargs, smart_resize + + +@auto_docstring +class Glm46VImageProcessorFast(BaseImageProcessorFast): + do_resize = True + resample = PILImageResampling.BICUBIC + size = {"shortest_edge": 112 * 112, "longest_edge": 28 * 28 * 15000} + do_rescale = True + do_normalize = True + image_mean = OPENAI_CLIP_MEAN + image_std = OPENAI_CLIP_STD + do_convert_rgb = True + patch_size = 14 + temporal_patch_size = 2 + merge_size = 2 + valid_kwargs = Glm46VImageProcessorKwargs + model_input_names = ["pixel_values", "image_grid_thw"] + + def __init__(self, **kwargs: Unpack[Glm46VImageProcessorKwargs]): + super().__init__(**kwargs) + if self.size is not None and ( + self.size.get("shortest_edge", None) is None or self.size.get("longest_edge", None) is None + ): + raise ValueError("size must contain 'shortest_edge' and 'longest_edge' keys.") + + def _further_process_kwargs( + self, + size: Optional[SizeDict] = None, + **kwargs, + ) -> dict: + """ + Update kwargs that need further processing before being validated + Can be overridden by subclasses to customize the processing of kwargs. + """ + if size is not None and ("shortest_edge" not in size or "longest_edge" not in size): + raise ValueError("size must contain 'shortest_edge' and 'longest_edge' keys.") + + return super()._further_process_kwargs(size=size, **kwargs) + + def _preprocess( + self, + images: list["torch.Tensor"], + do_resize: bool, + size: SizeDict, + interpolation: Optional["F.InterpolationMode"], + do_rescale: bool, + rescale_factor: float, + do_normalize: bool, + image_mean: Optional[Union[float, list[float]]], + image_std: Optional[Union[float, list[float]]], + patch_size: int, + temporal_patch_size: int, + merge_size: int, + disable_grouping: Optional[bool], + return_tensors: Optional[Union[str, TensorType]], + **kwargs, + ) -> BatchFeature: + """ + Preprocess an image or batch of images. Copy of the `preprocess` method from `CLIPImageProcessor`. + """ + + grouped_images, grouped_images_index = group_images_by_shape(images, disable_grouping=disable_grouping) + resized_images_grouped = {} + for shape, stacked_images in grouped_images.items(): + height, width = stacked_images.shape[-2:] + if do_resize: + resized_height, resized_width = smart_resize( + num_frames=temporal_patch_size, + height=height, + width=width, + temporal_factor=temporal_patch_size, + factor=patch_size * merge_size, + min_pixels=size.shortest_edge, + max_pixels=size.longest_edge, + ) + stacked_images = self.resize( + stacked_images, + size=SizeDict(height=resized_height, width=resized_width), + interpolation=interpolation, + ) + resized_images_grouped[shape] = stacked_images + + resized_images = reorder_images(resized_images_grouped, grouped_images_index) + + grouped_images, grouped_images_index = group_images_by_shape(resized_images, disable_grouping=disable_grouping) + processed_images_grouped = {} + processed_grids = {} + + for shape, stacked_images in grouped_images.items(): + resized_height, resized_width = stacked_images.shape[-2:] + + patches = self.rescale_and_normalize( + stacked_images, do_rescale, rescale_factor, do_normalize, image_mean, image_std + ) + if patches.ndim == 4: # (B, C, H, W) + patches = patches.unsqueeze(1) # (B, T=1, C, H, W) + + if patches.shape[1] % temporal_patch_size != 0: + repeats = patches[:, -1:].repeat( + 1, temporal_patch_size - (patches.shape[1] % temporal_patch_size), 1, 1, 1 + ) + patches = torch.cat([patches, repeats], dim=1) + + batch_size, t_len, channel = patches.shape[:3] + grid_t = t_len // temporal_patch_size + grid_h, grid_w = resized_height // patch_size, resized_width // patch_size + + patches = patches.view( + batch_size, + grid_t, + temporal_patch_size, + channel, + grid_h // merge_size, + merge_size, + patch_size, + grid_w // merge_size, + merge_size, + patch_size, + ) + # (B, grid_t, gh, gw, mh, mw, C, tp, ph, pw) + patches = patches.permute(0, 1, 4, 7, 5, 8, 3, 2, 6, 9) + + flatten_patches = patches.reshape( + batch_size, + grid_t * grid_h * grid_w, + channel * temporal_patch_size * patch_size * patch_size, + ) + + processed_images_grouped[shape] = flatten_patches + processed_grids[shape] = [[grid_t, grid_h, grid_w]] * batch_size + + processed_images = reorder_images(processed_images_grouped, grouped_images_index) + processed_grids = reorder_images(processed_grids, grouped_images_index) + + pixel_values = torch.cat(processed_images, dim=0) + image_grid_thw = torch.tensor(processed_grids) + + return BatchFeature( + data={"pixel_values": pixel_values, "image_grid_thw": image_grid_thw}, tensor_type=return_tensors + ) + + @auto_docstring + def preprocess( + self, + images: ImageInput, + **kwargs: Unpack[Glm46VImageProcessorKwargs], + ) -> BatchFeature: + return super().preprocess(images, **kwargs) + + +__all__ = ["Glm46VImageProcessorFast"] diff --git a/src/transformers/models/glm46v/modeling_glm46v.py b/src/transformers/models/glm46v/modeling_glm46v.py index 9ee84b1e254d..9c45f8ff6f2e 100644 --- a/src/transformers/models/glm46v/modeling_glm46v.py +++ b/src/transformers/models/glm46v/modeling_glm46v.py @@ -26,8 +26,8 @@ import torch import torch.nn as nn -import torch.nn.functional as F from torch.nn import LayerNorm +from torchvision.transforms.v2 import functional as F from ...activations import ACT2FN from ...cache_utils import Cache, DynamicCache diff --git a/src/transformers/models/glm46v/modular_glm46v.py b/src/transformers/models/glm46v/modular_glm46v.py index 9b8dce4f048b..ad57aa32b899 100644 --- a/src/transformers/models/glm46v/modular_glm46v.py +++ b/src/transformers/models/glm46v/modular_glm46v.py @@ -19,6 +19,8 @@ from ...video_utils import VideoMetadata from ..glm4v.configuration_glm4v import Glm4vConfig, Glm4vTextConfig +from ..glm4v.image_processing_glm4v import Glm4vImageProcessor +from ..glm4v.image_processing_glm4v_fast import Glm4vImageProcessorFast from ..glm4v.modeling_glm4v import Glm4vModel, Glm4vPreTrainedModel, Glm4vTextModel from ..glm4v.processing_glm4v import Glm4vProcessor from ..glm4v.video_processing_glm4v import Glm4vVideoProcessor @@ -49,6 +51,14 @@ def replace_frame_token_id(self, timestamp_sec): return f"<|begin_of_image|>{self.image_token}<|end_of_image|>{timestamp_sec:.1f} seconds" +class Glm46VImageProcessor(Glm4vImageProcessor): + pass + + +class Glm46VImageProcessorFast(Glm4vImageProcessorFast): + pass + + class Glm46VVideoProcessor(Glm4vVideoProcessor): def sample_frames( self, @@ -124,5 +134,7 @@ def sample_frames( "Glm46VPreTrainedModel", "Glm46VTextModel", "Glm46VProcessor", + "Glm46VImageProcessor", + "Glm46VImageProcessorFast", "Glm46VVideoProcessor", ] diff --git a/src/transformers/models/glm46v/processing_glm46v.py b/src/transformers/models/glm46v/processing_glm46v.py index 25a0aa34538b..ce6a17d3178b 100644 --- a/src/transformers/models/glm46v/processing_glm46v.py +++ b/src/transformers/models/glm46v/processing_glm46v.py @@ -23,7 +23,7 @@ import numpy as np -from ...feature_extraction_utils import BatchFeature +from ...image_processing_utils import BatchFeature from ...image_utils import ImageInput from ...processing_utils import MultiModalData, ProcessingKwargs, ProcessorMixin, Unpack from ...tokenization_utils_base import PreTokenizedInput, TextInput diff --git a/src/transformers/models/glm46v/video_processing_glm46v.py b/src/transformers/models/glm46v/video_processing_glm46v.py index ab99312385da..f8e6d248502b 100644 --- a/src/transformers/models/glm46v/video_processing_glm46v.py +++ b/src/transformers/models/glm46v/video_processing_glm46v.py @@ -24,7 +24,7 @@ import numpy as np import torch -from ...feature_extraction_utils import BatchFeature +from ...image_processing_utils import BatchFeature from ...image_utils import ( OPENAI_CLIP_MEAN, OPENAI_CLIP_STD, From b9c8484562f80fc906ef0d97b40c2dbb5fdea646 Mon Sep 17 00:00:00 2001 From: zRzRzRzRzRzRzR <2448370773@qq.com> Date: Wed, 12 Nov 2025 20:51:06 +0800 Subject: [PATCH 26/66] 1 --- docs/source/en/model_doc/glm46v.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/docs/source/en/model_doc/glm46v.md b/docs/source/en/model_doc/glm46v.md index 8fc8865cb865..d48a0105e773 100644 --- a/docs/source/en/model_doc/glm46v.md +++ b/docs/source/en/model_doc/glm46v.md @@ -62,6 +62,16 @@ The original code can be found [here](). [[autodoc]] Glm46VPreTrainedModel - forward +## Glm46VImageProcessor + +[[autodoc]] Glm46VImageProcessor + - preprocess + +## Glm46VImageProcessorFast + +[[autodoc]] Glm46VImageProcessorFast + - preprocess + ## Glm46VVideoProcessor [[autodoc]] Glm46VVideoProcessor From 7c92ad722727050ee87e54c9be120071efa9566f Mon Sep 17 00:00:00 2001 From: zRzRzRzRzRzRzR <2448370773@qq.com> Date: Wed, 12 Nov 2025 21:29:37 +0800 Subject: [PATCH 27/66] 2 --- docs/source/en/model_doc/glm46v.md | 5 ----- 1 file changed, 5 deletions(-) diff --git a/docs/source/en/model_doc/glm46v.md b/docs/source/en/model_doc/glm46v.md index d48a0105e773..ad992f0211f3 100644 --- a/docs/source/en/model_doc/glm46v.md +++ b/docs/source/en/model_doc/glm46v.md @@ -84,9 +84,4 @@ The original code can be found [here](). ## Glm46VTextModel [[autodoc]] Glm46VTextModel - - forward - -## Glm46VForConditionalGeneration - -[[autodoc]] Glm46VForConditionalGeneration - forward \ No newline at end of file From 5552ff2ec10731cb376b0fc5e4b04a2d407540df Mon Sep 17 00:00:00 2001 From: zRzRzRzRzRzRzR <2448370773@qq.com> Date: Wed, 12 Nov 2025 21:34:00 +0800 Subject: [PATCH 28/66] 1 --- docs/source/en/model_doc/glm46v.md | 76 +--- .../models/glm46v/modeling_glm46v.py | 355 +++++++++++++++++- .../models/glm46v/modular_glm46v.py | 7 +- 3 files changed, 375 insertions(+), 63 deletions(-) diff --git a/docs/source/en/model_doc/glm46v.md b/docs/source/en/model_doc/glm46v.md index ad992f0211f3..a8032634fcd8 100644 --- a/docs/source/en/model_doc/glm46v.md +++ b/docs/source/en/model_doc/glm46v.md @@ -1,44 +1,4 @@ - - - -# Glm46V - -## Overview - -The Glm46V model was proposed in []() by . - - -The abstract from the paper is the following: - - - -Tips: - - - -This model was contributed by [INSERT YOUR HF USERNAME HERE](https://huggingface.co/). -The original code can be found [here](). - -## Usage examples - - +# GLM-4.6V ## Glm46VConfig @@ -48,33 +8,19 @@ The original code can be found [here](). [[autodoc]] Glm46VTextConfig -## Glm46VForConditionalGeneration - -[[autodoc]] Glm46VForConditionalGeneration - -## Glm46VModel - -[[autodoc]] Glm46VModel - - forward - -## Glm46VPreTrainedModel - -[[autodoc]] Glm46VPreTrainedModel - - forward - ## Glm46VImageProcessor [[autodoc]] Glm46VImageProcessor - preprocess -## Glm46VImageProcessorFast +## Glm46VVideoProcessor -[[autodoc]] Glm46VImageProcessorFast +[[autodoc]] Glm46VVideoProcessor - preprocess -## Glm46VVideoProcessor +## Glm46VImageProcessorFast -[[autodoc]] Glm46VVideoProcessor +[[autodoc]] Glm46VImageProcessorFast - preprocess ## Glm46VProcessor @@ -84,4 +30,14 @@ The original code can be found [here](). ## Glm46VTextModel [[autodoc]] Glm46VTextModel - - forward \ No newline at end of file + - forward + +## Glm46VModel + +[[autodoc]] Glm46VModel + - forward + +## Glm46VForConditionalGeneration + +[[autodoc]] Glm46VForConditionalGeneration + - forward diff --git a/src/transformers/models/glm46v/modeling_glm46v.py b/src/transformers/models/glm46v/modeling_glm46v.py index 9c45f8ff6f2e..d8bc732bb2ca 100644 --- a/src/transformers/models/glm46v/modeling_glm46v.py +++ b/src/transformers/models/glm46v/modeling_glm46v.py @@ -22,7 +22,7 @@ import itertools from collections.abc import Callable from dataclasses import dataclass -from typing import Optional, Union +from typing import Any, Optional, Union import torch import torch.nn as nn @@ -31,6 +31,7 @@ from ...activations import ACT2FN from ...cache_utils import Cache, DynamicCache +from ...generation import GenerationMixin from ...integrations import use_kernel_forward_from_hub from ...masking_utils import create_causal_mask from ...modeling_flash_attention_utils import FlashAttentionKwargs @@ -506,6 +507,356 @@ def forward( ) +@dataclass +@auto_docstring( + custom_intro=""" + Base class for Glm46V causal language model (or autoregressive) outputs. + """ +) +class Glm46VCausalLMOutputWithPast(ModelOutput): + r""" + loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `labels` is provided): + Language modeling loss (for next-token prediction). + logits (`torch.FloatTensor` of shape `(batch_size, sequence_length, config.vocab_size)`): + Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax). + past_key_values (`Cache`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`): + It is a [`~cache_utils.Cache`] instance. For more details, see our [kv cache guide](https://huggingface.co/docs/transformers/en/kv_cache). + + Contains pre-computed hidden-states (key and values in the self-attention blocks) that can be used (see + `past_key_values` input) to speed up sequential decoding. + rope_deltas (`torch.LongTensor` of shape `(batch_size, )`, *optional*): + The rope index difference between sequence length and multimodal rope. + """ + + loss: Optional[torch.FloatTensor] = None + logits: Optional[torch.FloatTensor] = None + past_key_values: Optional[Cache] = None + hidden_states: Optional[tuple[torch.FloatTensor]] = None + attentions: Optional[tuple[torch.FloatTensor]] = None + rope_deltas: Optional[torch.LongTensor] = None + + +class Glm46VForConditionalGeneration(Glm46VPreTrainedModel, GenerationMixin): + _checkpoint_conversion_mapping = {} + _tied_weights_keys = ["lm_head.weight"] + # Reference: fix gemma3 grad acc #37208 + accepts_loss_kwargs = False + + def __init__(self, config): + super().__init__(config) + self.model = Glm46VModel(config) + self.lm_head = nn.Linear(config.text_config.hidden_size, config.text_config.vocab_size, bias=False) + + self.post_init() + + def get_input_embeddings(self): + return self.model.get_input_embeddings() + + def set_input_embeddings(self, value): + self.model.set_input_embeddings(value) + + def set_decoder(self, decoder): + self.model.set_decoder(decoder) + + def get_decoder(self): + return self.model.get_decoder() + + def get_video_features( + self, pixel_values_videos: torch.FloatTensor, video_grid_thw: Optional[torch.LongTensor] = None + ): + return self.model.get_video_features(pixel_values_videos, video_grid_thw) + + def get_image_features(self, pixel_values: torch.FloatTensor, image_grid_thw: Optional[torch.LongTensor] = None): + return self.model.get_image_features(pixel_values, image_grid_thw) + + # Make modules available through conditional class for BC + @property + def language_model(self): + return self.model.language_model + + @property + def visual(self): + return self.model.visual + + @can_return_tuple + @auto_docstring + def forward( + self, + input_ids: Optional[torch.LongTensor] = None, + attention_mask: Optional[torch.Tensor] = None, + position_ids: Optional[torch.LongTensor] = None, + past_key_values: Optional[Cache] = None, + inputs_embeds: Optional[torch.FloatTensor] = None, + labels: Optional[torch.LongTensor] = None, + pixel_values: Optional[torch.Tensor] = None, + pixel_values_videos: Optional[torch.FloatTensor] = None, + image_grid_thw: Optional[torch.LongTensor] = None, + video_grid_thw: Optional[torch.LongTensor] = None, + rope_deltas: Optional[torch.LongTensor] = None, + cache_position: Optional[torch.LongTensor] = None, + logits_to_keep: Union[int, torch.Tensor] = 0, + **kwargs: Unpack[TransformersKwargs], + ) -> Union[tuple, Glm46VCausalLMOutputWithPast]: + r""" + rope_deltas (`torch.LongTensor` of shape `(batch_size, )`, *optional*): + The rope index difference between sequence length and multimodal rope. + labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., + config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored + (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. + image_grid_thw (`torch.LongTensor` of shape `(num_images, 3)`, *optional*): + The temporal, height and width of feature shape of each image in LLM. + video_grid_thw (`torch.LongTensor` of shape `(num_videos, 3)`, *optional*): + The temporal, height and width of feature shape of each video in LLM. + + Example: + + ```python + >>> from PIL import Image + >>> import requests + >>> from transformers import AutoProcessor, Glm46VForConditionalGeneration + + >>> model = Glm46VForConditionalGeneration.from_pretrained("THUDM/GLM-4.1V-9B-Thinking") + >>> processor = AutoProcessor.from_pretrained("THUDM/GLM-4.1V-9B-Thinking") + + >>> messages = [ + { + "role": "user", + "content": [ + {"type": "image"}, + {"type": "text", "text": "What is shown in this image?"}, + ], + }, + ] + >>> url = "https://www.ilankelman.org/stopsigns/australia.jpg" + >>> image = Image.open(requests.get(url, stream=True).raw) + + >>> text = processor.apply_chat_template(messages, tokenize=False, add_generation_prompt=True) + >>> inputs = processor(text=[text], images=[image], vision_infos=[vision_infos]) + + >>> # Generate + >>> generate_ids = model.generate(inputs.input_ids, max_length=30) + >>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0] + "The image shows a street scene with a red stop sign in the foreground. In the background, there is a large red gate with Chinese characters ..." + ```""" + outputs = self.model( + input_ids=input_ids, + pixel_values=pixel_values, + pixel_values_videos=pixel_values_videos, + image_grid_thw=image_grid_thw, + video_grid_thw=video_grid_thw, + position_ids=position_ids, + attention_mask=attention_mask, + past_key_values=past_key_values, + inputs_embeds=inputs_embeds, + cache_position=cache_position, + **kwargs, + ) + + hidden_states = outputs[0] + + # Only compute necessary logits, and do not upcast them to float if we are not computing the loss + slice_indices = slice(-logits_to_keep, None) if isinstance(logits_to_keep, int) else logits_to_keep + logits = self.lm_head(hidden_states[:, slice_indices, :]) + + loss = None + if labels is not None: + loss = self.loss_function(logits=logits, labels=labels, vocab_size=self.config.text_config.vocab_size) + + return Glm46VCausalLMOutputWithPast( + loss=loss, + logits=logits, + past_key_values=outputs.past_key_values, + hidden_states=outputs.hidden_states, + attentions=outputs.attentions, + rope_deltas=outputs.rope_deltas, + ) + + def prepare_inputs_for_generation( + self, + input_ids, + past_key_values=None, + attention_mask=None, + inputs_embeds=None, + cache_position=None, + position_ids=None, + use_cache=True, + pixel_values=None, + pixel_values_videos=None, + image_grid_thw=None, + video_grid_thw=None, + **kwargs, + ): + # Overwritten -- in specific circumstances we don't want to forward image inputs to the model + + model_inputs = super().prepare_inputs_for_generation( + input_ids, + past_key_values=past_key_values, + attention_mask=attention_mask, + inputs_embeds=inputs_embeds, + cache_position=cache_position, + position_ids=position_ids, + pixel_values=pixel_values, + pixel_values_videos=pixel_values_videos, + image_grid_thw=image_grid_thw, + video_grid_thw=video_grid_thw, + use_cache=use_cache, + **kwargs, + ) + + # GLM-4.1V position_ids are prepareed with rope_deltas in forward + model_inputs["position_ids"] = None + + if cache_position[0] != 0: + model_inputs["pixel_values"] = None + model_inputs["pixel_values_videos"] = None + + return model_inputs + + def _get_image_nums_and_video_nums( + self, + input_ids: Optional[torch.LongTensor], + inputs_embeds: Optional[torch.Tensor] = None, + ) -> tuple[torch.Tensor, torch.Tensor]: + """ + Get the number of images and videos for each sample to calculate the separation length of the sample tensor. + These parameters are not passed through the processor to avoid unpredictable impacts from interface modifications. + + Args: + input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): + Indices of input sequence tokens in the vocabulary. + + Returns: + image_nums (`torch.LongTensor` of shape `(batch_size, num_images_sample)`) + video_nums (`torch.LongTensor` of shape `(batch_size, num_videos_sample)`) + """ + + if inputs_embeds is not None: + is_image = ( + inputs_embeds + == self.get_input_embeddings()( + torch.tensor(self.config.image_start_token_id, dtype=torch.long, device=inputs_embeds.device) + ) + )[..., 0] + is_video_start = ( + inputs_embeds + == self.get_input_embeddings()( + torch.tensor(self.config.video_start_token_id, dtype=torch.long, device=inputs_embeds.device) + ) + )[..., 0] + is_video_end = ( + inputs_embeds + == self.get_input_embeddings()( + torch.tensor(self.config.video_end_token_id, dtype=torch.long, device=inputs_embeds.device) + ) + )[..., 0] + else: + is_image = input_ids == self.config.image_start_token_id + is_video_start = input_ids == self.config.video_start_token_id + is_video_end = input_ids == self.config.video_end_token_id + + # Cumulative sum to track if we're inside a video span + # We'll assume well-formed video tags (i.e. matching starts and ends) + video_level = torch.cumsum(is_video_start.int() - is_video_end.int(), dim=1) + inside_video = video_level > 0 # shape (batch_size, seq_length) + + # Mask out image tokens that are inside video spans + standalone_images = is_image & (~inside_video) + + # Count per batch + image_counts = standalone_images.sum(dim=1) + video_counts = is_video_start.sum(dim=1) + + return image_counts, video_counts + + def _expand_inputs_for_generation( + self, + expand_size: int = 1, + is_encoder_decoder: bool = False, + input_ids: Optional[torch.LongTensor] = None, + **model_kwargs, + ) -> tuple[torch.LongTensor, dict[str, Any]]: + # Overwritten -- Support for expanding tensors without a batch size dimension + # e.g., pixel_values, image_grid_thw, pixel_values_videos, video_grid_thw, second_per_grid_t + # pixel_values.shape[0] is sum(seqlen_images for samples) + # image_grid_thw.shape[0] is sum(num_images for samples) + + if expand_size == 1: + return input_ids, model_kwargs + + visual_keys = ["pixel_values", "image_grid_thw", "pixel_values_videos", "video_grid_thw", "second_per_grid_ts"] + + def _expand_dict_for_generation_visual(dict_to_expand): + image_grid_thw = model_kwargs.get("image_grid_thw", None) + video_grid_thw = model_kwargs.get("video_grid_thw", None) + image_nums, video_nums = self._get_image_nums_and_video_nums( + input_ids, inputs_embeds=model_kwargs.get("inputs_embeds", None) + ) + + def _repeat_interleave_samples(x, lengths, repeat_times): + samples = torch.split(x, lengths) + repeat_args = [repeat_times] + [1] * (x.dim() - 1) + result = torch.cat([sample.repeat(*repeat_args) for sample in samples], dim=0) + return result + + for key in dict_to_expand: + if key == "pixel_values": + # split images into samples + samples = torch.split(image_grid_thw, list(image_nums)) + # compute the sequence length of images for each sample + lengths = [torch.prod(sample, dim=1).sum() for sample in samples] + dict_to_expand[key] = _repeat_interleave_samples( + dict_to_expand[key], lengths=lengths, repeat_times=expand_size + ) + elif key == "image_grid_thw": + # get the num of images for each sample + lengths = list(image_nums) + dict_to_expand[key] = _repeat_interleave_samples( + dict_to_expand[key], lengths=lengths, repeat_times=expand_size + ) + elif key == "pixel_values_videos": + samples = torch.split(video_grid_thw, list(video_nums)) + lengths = [torch.prod(sample, dim=1).sum() for sample in samples] + dict_to_expand[key] = _repeat_interleave_samples( + dict_to_expand[key], lengths=lengths, repeat_times=expand_size + ) + elif key == "video_grid_thw": + lengths = list(video_nums) + dict_to_expand[key] = _repeat_interleave_samples( + dict_to_expand[key], lengths=lengths, repeat_times=expand_size + ) + elif key == "second_per_grid_ts": + dict_to_expand[key] = _repeat_interleave_samples( + dict_to_expand[key], lengths=list(video_nums), repeat_times=expand_size + ) + return dict_to_expand + + def _expand_dict_for_generation(dict_to_expand): + for key in dict_to_expand: + if ( + key != "cache_position" + and dict_to_expand[key] is not None + and isinstance(dict_to_expand[key], torch.Tensor) + and key not in visual_keys + ): + dict_to_expand[key] = dict_to_expand[key].repeat_interleave(expand_size, dim=0) + return dict_to_expand + + model_kwargs = _expand_dict_for_generation_visual(model_kwargs) + + if input_ids is not None: + input_ids = input_ids.repeat_interleave(expand_size, dim=0) + + model_kwargs = _expand_dict_for_generation(model_kwargs) + + if is_encoder_decoder: + if model_kwargs.get("encoder_outputs") is None: + raise ValueError("If `is_encoder_decoder` is True, make sure that `encoder_outputs` is defined.") + model_kwargs["encoder_outputs"] = _expand_dict_for_generation(model_kwargs["encoder_outputs"]) + + return input_ids, model_kwargs + + class Glm46VisionMlp(nn.Module): def __init__(self, config, bias: bool = False): super().__init__() @@ -1333,4 +1684,4 @@ def forward( ) -__all__ = ["Glm46VModel", "Glm46VPreTrainedModel", "Glm46VTextModel"] +__all__ = ["Glm46VModel", "Glm4vForConditionalGeneration", "Glm46VPreTrainedModel", "Glm46VTextModel"] diff --git a/src/transformers/models/glm46v/modular_glm46v.py b/src/transformers/models/glm46v/modular_glm46v.py index ad57aa32b899..087f34e95c9f 100644 --- a/src/transformers/models/glm46v/modular_glm46v.py +++ b/src/transformers/models/glm46v/modular_glm46v.py @@ -21,7 +21,7 @@ from ..glm4v.configuration_glm4v import Glm4vConfig, Glm4vTextConfig from ..glm4v.image_processing_glm4v import Glm4vImageProcessor from ..glm4v.image_processing_glm4v_fast import Glm4vImageProcessorFast -from ..glm4v.modeling_glm4v import Glm4vModel, Glm4vPreTrainedModel, Glm4vTextModel +from ..glm4v.modeling_glm4v import Glm4vForConditionalGeneration, Glm4vModel, Glm4vPreTrainedModel, Glm4vTextModel from ..glm4v.processing_glm4v import Glm4vProcessor from ..glm4v.video_processing_glm4v import Glm4vVideoProcessor @@ -42,6 +42,10 @@ class Glm46VTextModel(Glm4vTextModel): pass +class Glm46VForConditionalGeneration(Glm4vForConditionalGeneration): + pass + + class Glm46VModel(Glm4vModel): pass @@ -131,6 +135,7 @@ def sample_frames( "Glm46VConfig", "Glm46VTextConfig", "Glm46VModel", + "Glm4vForConditionalGeneration", "Glm46VPreTrainedModel", "Glm46VTextModel", "Glm46VProcessor", From f7bfc34fbe40374ca41d9a99832dc3b351477d77 Mon Sep 17 00:00:00 2001 From: zRzRzRzRzRzRzR <2448370773@qq.com> Date: Wed, 12 Nov 2025 21:35:56 +0800 Subject: [PATCH 29/66] 2 --- src/transformers/models/glm46v/modeling_glm46v.py | 2 +- src/transformers/models/glm46v/modular_glm46v.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/transformers/models/glm46v/modeling_glm46v.py b/src/transformers/models/glm46v/modeling_glm46v.py index d8bc732bb2ca..4e9fb07276f5 100644 --- a/src/transformers/models/glm46v/modeling_glm46v.py +++ b/src/transformers/models/glm46v/modeling_glm46v.py @@ -1684,4 +1684,4 @@ def forward( ) -__all__ = ["Glm46VModel", "Glm4vForConditionalGeneration", "Glm46VPreTrainedModel", "Glm46VTextModel"] +__all__ = ["Glm46VModel", "Glm46VForConditionalGeneration", "Glm46VPreTrainedModel", "Glm46VTextModel"] diff --git a/src/transformers/models/glm46v/modular_glm46v.py b/src/transformers/models/glm46v/modular_glm46v.py index 087f34e95c9f..787310a51ef6 100644 --- a/src/transformers/models/glm46v/modular_glm46v.py +++ b/src/transformers/models/glm46v/modular_glm46v.py @@ -135,7 +135,7 @@ def sample_frames( "Glm46VConfig", "Glm46VTextConfig", "Glm46VModel", - "Glm4vForConditionalGeneration", + "Glm46VForConditionalGeneration", "Glm46VPreTrainedModel", "Glm46VTextModel", "Glm46VProcessor", From 0376e223c802dd172c2185e3cdf20aa806964e01 Mon Sep 17 00:00:00 2001 From: zRzRzRzRzRzRzR <2448370773@qq.com> Date: Wed, 12 Nov 2025 21:48:59 +0800 Subject: [PATCH 30/66] 2 --- .../models/glm46v/modeling_glm46v.py | 832 +----------------- .../models/glm46v/modular_glm46v.py | 2 +- 2 files changed, 4 insertions(+), 830 deletions(-) diff --git a/src/transformers/models/glm46v/modeling_glm46v.py b/src/transformers/models/glm46v/modeling_glm46v.py index 4e9fb07276f5..15ab3b36132d 100644 --- a/src/transformers/models/glm46v/modeling_glm46v.py +++ b/src/transformers/models/glm46v/modeling_glm46v.py @@ -19,15 +19,12 @@ # See the License for the specific language governing permissions and # limitations under the License. -import itertools from collections.abc import Callable from dataclasses import dataclass from typing import Any, Optional, Union import torch import torch.nn as nn -from torch.nn import LayerNorm -from torchvision.transforms.v2 import functional as F from ...activations import ACT2FN from ...cache_utils import Cache, DynamicCache @@ -40,9 +37,9 @@ from ...modeling_rope_utils import ROPE_INIT_FUNCTIONS, dynamic_rope_update from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel from ...processing_utils import Unpack -from ...utils import TransformersKwargs, auto_docstring, can_return_tuple, is_torchdynamo_compiling +from ...utils import TransformersKwargs, auto_docstring, can_return_tuple from ...utils.generic import check_model_inputs -from .configuration_glm46v import Glm46VConfig, Glm46VTextConfig, Glm46VVisionConfig +from .configuration_glm46v import Glm46VConfig, Glm46VTextConfig @use_kernel_forward_from_hub("RMSNorm") @@ -857,831 +854,8 @@ def _expand_dict_for_generation(dict_to_expand): return input_ids, model_kwargs -class Glm46VisionMlp(nn.Module): - def __init__(self, config, bias: bool = False): - super().__init__() - self.hidden_size = config.hidden_size - self.intermediate_size = config.out_hidden_size - self.gate_proj = nn.Linear(self.hidden_size, self.intermediate_size, bias=bias) - self.up_proj = nn.Linear(self.hidden_size, self.intermediate_size, bias=bias) - self.down_proj = nn.Linear(self.intermediate_size, self.hidden_size, bias=bias) - self.act_fn = ACT2FN[config.hidden_act] - - def forward(self, hidden_state): - return self.down_proj(self.act_fn(self.gate_proj(hidden_state)) * self.up_proj(hidden_state)) - - -class Glm46VVisionPatchEmbed(nn.Module): - def __init__(self, config: Glm46VVisionConfig) -> None: - super().__init__() - self.patch_size = config.patch_size - self.temporal_patch_size = config.temporal_patch_size - self.in_channels = config.in_channels - self.embed_dim = config.hidden_size - - kernel_size = [self.temporal_patch_size, self.patch_size, self.patch_size] - self.proj = nn.Conv3d(self.in_channels, self.embed_dim, kernel_size=kernel_size, stride=kernel_size) - - def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: - target_dtype = self.proj.weight.dtype - hidden_states = hidden_states.view( - -1, self.in_channels, self.temporal_patch_size, self.patch_size, self.patch_size - ) - hidden_states = self.proj(hidden_states.to(dtype=target_dtype)).view(-1, self.embed_dim) - return hidden_states - - -class Glm46VVisionRotaryEmbedding(nn.Module): - inv_freq: torch.Tensor # fix linting for `register_buffer` - - def __init__(self, dim: int, theta: float = 10000.0) -> None: - super().__init__() - inv_freq = 1.0 / (theta ** (torch.arange(0, dim, 2, dtype=torch.float) / dim)) - self.register_buffer("inv_freq", inv_freq, persistent=False) - - def forward(self, seqlen: int) -> torch.Tensor: - seq = torch.arange(seqlen, device=self.inv_freq.device, dtype=self.inv_freq.dtype) - freqs = torch.outer(seq, self.inv_freq) - return freqs - - -class Glm46VVisionPatchMerger(nn.Module): - def __init__(self, dim: int, context_dim: int, hidden_act: str, bias: bool = False) -> None: - super().__init__() - self.proj = nn.Linear(dim, dim, bias=bias) - self.post_projection_norm = LayerNorm(dim) - self.gate_proj = nn.Linear(dim, context_dim, bias=bias) - self.up_proj = nn.Linear(dim, context_dim, bias=bias) - self.down_proj = nn.Linear(context_dim, dim, bias=bias) - self.act1 = nn.GELU() - self.act_fn = ACT2FN[hidden_act] - - def forward(self, hidden_state: torch.Tensor) -> torch.Tensor: - hidden_state = self.proj(hidden_state) - hidden_state = self.act1(self.post_projection_norm(hidden_state)) - return self.down_proj(self.act_fn(self.gate_proj(hidden_state)) * self.up_proj(hidden_state)) - - -class Glm46VVisionEmbeddings(nn.Module): - def __init__(self, config: Glm46VVisionConfig): - super().__init__() - self.config = config - self.embed_dim = config.hidden_size - self.image_size = config.image_size - self.patch_size = config.patch_size - - self.num_patches = (self.image_size // self.patch_size) ** 2 - self.num_positions = self.num_patches - self.position_embedding = nn.Embedding(self.num_positions, self.embed_dim) - self.register_buffer("position_ids", torch.arange(self.num_positions).expand((1, -1)), persistent=False) - - def forward(self, embeddings, lengths, image_shapes, h_coords, w_coords) -> torch.Tensor: - """ - Forward pass with integrated position encoding adaptation using 2D interpolation. - - Args: - embeddings: Input embeddings tensor - lengths (torch.Tensor): Sequence lengths for each image in the batch. - image_shapes (torch.Tensor): Tensor of shape [batch_size, 3] representing the image shapes (t, h, w). - h_coords (torch.Tensor): Tensor of shape [total_seq] representing the h coordinate for each patch. - w_coords (torch.Tensor): Tensor of shape [total_seq] representing the w coordinate for each patch. - - Returns: - torch.Tensor: Embeddings with adapted position encoding added. - """ - # Get position embedding parameters - pos_embed_weight = self.position_embedding.weight - hidden_size = pos_embed_weight.shape[1] - total_seq = h_coords.shape[0] - device = pos_embed_weight.device - - # Move coordinates to correct device - h_coords, w_coords = h_coords.to(device), w_coords.to(device) - - # Handle empty sequence case - if total_seq == 0: - adapted_pos_embed = torch.empty(0, hidden_size, device=device, dtype=pos_embed_weight.dtype) - else: - # Convert inputs to tensors if needed - if isinstance(lengths, list): - lengths = torch.tensor(lengths, device=device, dtype=torch.long) - if not isinstance(image_shapes, torch.Tensor): - image_shapes = torch.tensor(image_shapes, device=device, dtype=torch.long) - - # Prepare 2D position embedding - orig_size_sq = pos_embed_weight.shape[0] - orig_size = int(orig_size_sq**0.5) - pos_embed_2d = ( - pos_embed_weight.view(orig_size, orig_size, hidden_size) - .permute(2, 0, 1) - .unsqueeze(0) - .to(device=device, dtype=torch.float32) - ) - - # Calculate target dimensions for each patch - target_h = torch.cat([image_shapes[i, 1].repeat(lengths[i]) for i in range(len(lengths))]).to( - device=device, dtype=torch.float32 - ) - target_w = torch.cat([image_shapes[i, 2].repeat(lengths[i]) for i in range(len(lengths))]).to( - device=device, dtype=torch.float32 - ) - - # Normalize coordinates to [-1, 1] range for grid_sample - h_coords = h_coords.to(device=device, dtype=torch.float32) - w_coords = w_coords.to(device=device, dtype=torch.float32) - norm_w = ((w_coords + 0.5) / target_w) * 2 - 1 - norm_h = ((h_coords + 0.5) / target_h) * 2 - 1 - - # Create sampling grid - grid = torch.stack((norm_w, norm_h), dim=-1).unsqueeze(0).unsqueeze(2) - - # Perform bicubic interpolation - interpolated_embed_fp32 = F.grid_sample( - pos_embed_2d, grid, mode="bicubic", align_corners=False, padding_mode="border" - ) - - # Reshape and convert back to original dtype - adapted_pos_embed_fp32 = interpolated_embed_fp32.squeeze(0).squeeze(-1).permute(1, 0) - adapted_pos_embed = adapted_pos_embed_fp32.to(pos_embed_weight.dtype).to(embeddings.device) - - # Add adapted position encoding to embeddings - embeddings = embeddings + adapted_pos_embed - return embeddings - - -def rotate_half(x): - """Rotates half the hidden dims of the input.""" - x1 = x[..., : x.shape[-1] // 2] - x2 = x[..., x.shape[-1] // 2 :] - return torch.cat((-x2, x1), dim=-1) - - -def apply_rotary_pos_emb_vision( - q: torch.Tensor, k: torch.Tensor, cos: torch.Tensor, sin: torch.Tensor -) -> tuple[torch.Tensor, torch.Tensor]: - orig_q_dtype = q.dtype - orig_k_dtype = k.dtype - q, k = q.float(), k.float() - cos, sin = cos.unsqueeze(-2).float(), sin.unsqueeze(-2).float() - q_embed = (q * cos) + (rotate_half(q) * sin) - k_embed = (k * cos) + (rotate_half(k) * sin) - q_embed = q_embed.to(orig_q_dtype) - k_embed = k_embed.to(orig_k_dtype) - return q_embed, k_embed - - -class Glm46VVisionAttention(nn.Module): - def __init__(self, config: Glm46VVisionConfig) -> None: - super().__init__() - self.dim = config.hidden_size - self.num_heads = config.num_heads - self.head_dim = self.dim // self.num_heads - self.num_key_value_groups = 1 # needed for eager attention - self.qkv = nn.Linear(config.hidden_size, config.hidden_size * 3, bias=config.attention_bias) - self.proj = nn.Linear(config.hidden_size, config.hidden_size, bias=False) - self.scaling = self.head_dim**-0.5 - self.config = config - self.attention_dropout = config.attention_dropout - self.is_causal = False - - def forward( - self, - hidden_states: torch.Tensor, - cu_seqlens: torch.Tensor, - rotary_pos_emb: Optional[torch.Tensor] = None, - position_embeddings: Optional[tuple[torch.Tensor, torch.Tensor]] = None, - **kwargs, - ) -> torch.Tensor: - seq_length = hidden_states.shape[0] - query_states, key_states, value_states = ( - self.qkv(hidden_states).reshape(seq_length, 3, self.num_heads, -1).permute(1, 0, 2, 3).unbind(0) - ) - cos, sin = position_embeddings - query_states, key_states = apply_rotary_pos_emb_vision(query_states, key_states, cos, sin) - - query_states = query_states.transpose(0, 1).unsqueeze(0) - key_states = key_states.transpose(0, 1).unsqueeze(0) - value_states = value_states.transpose(0, 1).unsqueeze(0) - - attention_interface: Callable = eager_attention_forward - if self.config._attn_implementation != "eager": - attention_interface = ALL_ATTENTION_FUNCTIONS[self.config._attn_implementation] - - if self.config._attn_implementation == "flash_attention_2": - # Flash Attention 2: Use cu_seqlens for variable length attention - max_seqlen = (cu_seqlens[1:] - cu_seqlens[:-1]).max() - attn_output, _ = attention_interface( - self, - query_states, - key_states, - value_states, - attention_mask=None, - scaling=self.scaling, - dropout=0.0 if not self.training else self.attention_dropout, - cu_seq_lens_q=cu_seqlens, - cu_seq_lens_k=cu_seqlens, - max_length_q=max_seqlen, - max_length_k=max_seqlen, - is_causal=False, - **kwargs, - ) - else: - # Other implementations: Process each chunk separately - lengths = cu_seqlens[1:] - cu_seqlens[:-1] - splits = [ - torch.split(tensor, lengths.tolist(), dim=2) for tensor in (query_states, key_states, value_states) - ] - - attn_outputs = [ - attention_interface( - self, - q, - k, - v, - attention_mask=None, - scaling=self.scaling, - dropout=0.0 if not self.training else self.attention_dropout, - is_causal=False, - **kwargs, - )[0] - for q, k, v in zip(*splits) - ] - attn_output = torch.cat(attn_outputs, dim=1) - - attn_output = attn_output.reshape(seq_length, -1).contiguous() - attn_output = self.proj(attn_output) - return attn_output - - -class Glm46VVisionBlock(GradientCheckpointingLayer): - def __init__(self, config) -> None: - super().__init__() - self.norm1 = Glm46VRMSNorm(config.hidden_size, eps=config.rms_norm_eps) - self.norm2 = Glm46VRMSNorm(config.hidden_size, eps=config.rms_norm_eps) - self.attn = Glm46VVisionAttention(config) - self.mlp = Glm46VisionMlp(config, bias=False) - - def forward( - self, - hidden_states: torch.Tensor, - cu_seqlens: torch.Tensor, - rotary_pos_emb: Optional[torch.Tensor] = None, - position_embeddings: Optional[tuple[torch.Tensor, torch.Tensor]] = None, - **kwargs, - ) -> torch.Tensor: - hidden_states = hidden_states + self.attn( - self.norm1(hidden_states), - cu_seqlens=cu_seqlens, - rotary_pos_emb=rotary_pos_emb, - position_embeddings=position_embeddings, - **kwargs, - ) - hidden_states = hidden_states + self.mlp(self.norm2(hidden_states)) - return hidden_states - - -@dataclass -@auto_docstring( - custom_intro=""" - Base class for Llava outputs, with hidden states and attentions. - """ -) -class Glm46VModelOutputWithPast(ModelOutput): - r""" - past_key_values (`Cache`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`): - It is a [`~cache_utils.Cache`] instance. For more details, see our [kv cache guide](https://huggingface.co/docs/transformers/en/kv_cache). - - Contains pre-computed hidden-states (key and values in the self-attention blocks) that can be used (see - `past_key_values` input) to speed up sequential decoding. - rope_deltas (`torch.LongTensor` of shape `(batch_size, )`, *optional*): - The rope index difference between sequence length and multimodal rope. - """ - - last_hidden_state: Optional[torch.FloatTensor] = None - past_key_values: Optional[Cache] = None - hidden_states: Optional[tuple[torch.FloatTensor]] = None - attentions: Optional[tuple[torch.FloatTensor]] = None - rope_deltas: Optional[torch.LongTensor] = None - - -class Glm46VVisionModel(Glm46VPreTrainedModel): - config: Glm46VVisionConfig - input_modalities = ["image", "video"] - _no_split_modules = ["Glm46VVisionBlock"] - - def __init__(self, config) -> None: - super().__init__(config) - self.spatial_merge_size = config.spatial_merge_size - self.patch_size = config.patch_size - - self.embeddings = Glm46VVisionEmbeddings(config) - self.patch_embed = Glm46VVisionPatchEmbed(config) - - head_dim = config.hidden_size // config.num_heads - self.rotary_pos_emb = Glm46VVisionRotaryEmbedding(head_dim // 2) - - self.blocks = nn.ModuleList([Glm46VVisionBlock(config) for _ in range(config.depth)]) - self.merger = Glm46VVisionPatchMerger( - dim=config.out_hidden_size, context_dim=config.intermediate_size, hidden_act=config.hidden_act - ) - - self.post_conv_layernorm = Glm46VRMSNorm(config.hidden_size, eps=config.rms_norm_eps) - self.downsample = nn.Conv2d( - in_channels=config.hidden_size, - out_channels=config.out_hidden_size, - kernel_size=config.spatial_merge_size, - stride=config.spatial_merge_size, - ) - self.post_layernorm = Glm46VRMSNorm(config.hidden_size, eps=config.rms_norm_eps) - - self.gradient_checkpointing = False - self.post_init() - - def rot_pos_emb(self, grid_thw): - pos_ids = [] - for t, h, w in grid_thw: - hpos_ids = torch.arange(h).unsqueeze(1).expand(-1, w) - hpos_ids = hpos_ids.reshape( - h // self.spatial_merge_size, - self.spatial_merge_size, - w // self.spatial_merge_size, - self.spatial_merge_size, - ) - hpos_ids = hpos_ids.permute(0, 2, 1, 3) - hpos_ids = hpos_ids.flatten() - - wpos_ids = torch.arange(w).unsqueeze(0).expand(h, -1) - wpos_ids = wpos_ids.reshape( - h // self.spatial_merge_size, - self.spatial_merge_size, - w // self.spatial_merge_size, - self.spatial_merge_size, - ) - wpos_ids = wpos_ids.permute(0, 2, 1, 3) - wpos_ids = wpos_ids.flatten() - pos_ids.append(torch.stack([hpos_ids, wpos_ids], dim=-1).repeat(t, 1)) - pos_ids = torch.cat(pos_ids, dim=0) - max_grid_size = grid_thw[:, 1:].max() - rotary_pos_emb_full = self.rotary_pos_emb(max_grid_size) - rotary_pos_emb = rotary_pos_emb_full[pos_ids].flatten(1) - return rotary_pos_emb, pos_ids - - def forward(self, hidden_states: torch.Tensor, grid_thw: torch.Tensor) -> torch.Tensor: - """ - Args: - hidden_states (`torch.Tensor` of shape `(seq_len, hidden_size)`): - The final hidden states of the model. - grid_thw (`torch.Tensor` of shape `(num_images_or_videos, 3)`): - The temporal, height and width of feature shape of each image in LLM. - - Returns: - `torch.Tensor`: hidden_states. - """ - hidden_states = self.patch_embed(hidden_states) - hidden_states = self.post_conv_layernorm(hidden_states) - - rotary_pos_emb, image_type_ids = self.rot_pos_emb(grid_thw) - emb = torch.cat((rotary_pos_emb, rotary_pos_emb), dim=-1) - position_embeddings = (emb.cos(), emb.sin()) - - cu_seqlens = torch.repeat_interleave(grid_thw[:, 1] * grid_thw[:, 2], grid_thw[:, 0]).cumsum( - dim=0, - # Select dtype based on the following factors: - # - FA2 requires that cu_seqlens_q must have dtype int32 - # - torch.onnx.export requires that cu_seqlens_q must have same dtype as grid_thw - # See https://github.com/huggingface/transformers/pull/34852 for more information - dtype=grid_thw.dtype if torch.jit.is_tracing() else torch.int32, - ) - cu_seqlens = F.pad(cu_seqlens, (1, 0), value=0) - seqlens = (cu_seqlens[1:] - cu_seqlens[:-1]).tolist() - hidden_states = self.embeddings(hidden_states, seqlens, grid_thw, image_type_ids[:, 0], image_type_ids[:, 1]) - - for blk in self.blocks: - hidden_states = blk( - hidden_states, - cu_seqlens=cu_seqlens, - position_embeddings=position_embeddings, - ) - - hidden_states = self.post_layernorm(hidden_states) - - hidden_states = hidden_states.view( - -1, self.spatial_merge_size, self.spatial_merge_size, hidden_states.shape[-1] - ) - hidden_states = hidden_states.permute(0, 3, 1, 2) - hidden_states = self.downsample(hidden_states).view(-1, self.config.out_hidden_size) - - hidden_states = self.merger(hidden_states) - return hidden_states - - -@auto_docstring class Glm46VModel(Glm46VPreTrainedModel): - base_model_prefix = "" - _checkpoint_conversion_mapping = {} - # Reference: fix gemma3 grad acc #37208 - accepts_loss_kwargs = False - config: Glm46VConfig - _no_split_modules = ["Glm46VTextDecoderLayer", "Glm46VVisionBlock"] - - def __init__(self, config): - super().__init__(config) - self.visual = Glm46VVisionModel._from_config(config.vision_config) - self.language_model = Glm46VTextModel._from_config(config.text_config) - self.rope_deltas = None # cache rope_deltas here - - # Initialize weights and apply final processing - self.post_init() - - def get_input_embeddings(self): - return self.language_model.get_input_embeddings() - - def set_input_embeddings(self, value): - self.language_model.set_input_embeddings(value) - - def set_decoder(self, decoder): - self.language_model = decoder - - def get_decoder(self): - return self.language_model - - def get_rope_index( - self, - input_ids: Optional[torch.LongTensor] = None, - image_grid_thw: Optional[torch.LongTensor] = None, - video_grid_thw: Optional[torch.LongTensor] = None, - attention_mask: Optional[torch.Tensor] = None, - ) -> tuple[torch.Tensor, torch.Tensor]: - """ - Calculate the 3D rope index based on image and video's temporal, height and width in LLM. - - Explanation: - Each embedding sequence contains vision embedding and text embedding or just contains text embedding. - - For pure text embedding sequence, the rotary position embedding has no difference with modern LLMs. - Examples: - input_ids: [T T T T T], here T is for text. - temporal position_ids: [0, 1, 2, 3, 4] - height position_ids: [0, 1, 2, 3, 4] - width position_ids: [0, 1, 2, 3, 4] - - For vision and text embedding sequence, we calculate 3D rotary position embedding for vision part - and 1D rotary position embedding for text part. - Examples: - Temporal (Time): 3 patches, representing different segments of the video in time. - Height: 2 patches, dividing each frame vertically. - Width: 2 patches, dividing each frame horizontally. - We also have some important parameters: - fps (Frames Per Second): The video's frame rate, set to 1. This means one frame is processed each second. - tokens_per_second: This is a crucial parameter. It dictates how many "time-steps" or "temporal tokens" are conceptually packed into a one-second interval of the video. In this case, we have 25 tokens per second. So each second of the video will be represented with 25 separate time points. It essentially defines the temporal granularity. - temporal_patch_size: The number of frames that compose one temporal patch. Here, it's 2 frames. - interval: The step size for the temporal position IDs, calculated as tokens_per_second * temporal_patch_size / fps. In this case, 25 * 2 / 1 = 50. This means that each temporal patch will be have a difference of 50 in the temporal position IDs. - input_ids: [V V V V V V V V V V V V T T T T T], here V is for vision. - vision temporal position_ids: [0, 0, 0, 0, 50, 50, 50, 50, 100, 100, 100, 100] - vision height position_ids: [0, 0, 1, 1, 0, 0, 1, 1, 0, 0, 1, 1] - vision width position_ids: [0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1] - text temporal position_ids: [101, 102, 103, 104, 105] - text height position_ids: [101, 102, 103, 104, 105] - text width position_ids: [101, 102, 103, 104, 105] - Here we calculate the text start position_ids as the max vision position_ids plus 1. - - Args: - input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): - Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide - it. - image_grid_thw (`torch.LongTensor` of shape `(num_images, 3)`, *optional*): - The temporal, height and width of feature shape of each image in LLM. - video_grid_thw (`torch.LongTensor` of shape `(num_videos, 3)`, *optional*): - The temporal, height and width of feature shape of each video in LLM. - attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - Returns: - position_ids (`torch.LongTensor` of shape `(3, batch_size, sequence_length)`) - mrope_position_deltas (`torch.Tensor` of shape `(batch_size)`) - """ - - spatial_merge_size = self.config.vision_config.spatial_merge_size - image_token_id = self.config.image_token_id - video_start_token_id = self.config.video_start_token_id - video_end_token_id = self.config.video_end_token_id - - mrope_position_deltas = [] - if input_ids is not None and (image_grid_thw is not None or video_grid_thw is not None): - total_input_ids = input_ids - if attention_mask is None: - attention_mask = torch.ones_like(total_input_ids) - position_ids = torch.ones( - 3, - input_ids.shape[0], - input_ids.shape[1], - dtype=input_ids.dtype, - device=input_ids.device, - ) - image_index, video_index = 0, 0 - video_group_index = 0 - attention_mask = attention_mask.to(total_input_ids.device) - for i, input_ids in enumerate(total_input_ids): - input_ids = input_ids[attention_mask[i] == 1] - input_tokens = input_ids.tolist() - - input_token_type = [] - video_check_flg = False - for token in input_tokens: - if token == video_start_token_id: - video_check_flg = True - elif token == video_end_token_id: - video_check_flg = False - - if token == image_token_id and not video_check_flg: - input_token_type.append("image") - elif token == image_token_id and video_check_flg: - input_token_type.append("video") - else: - input_token_type.append("text") - - input_type_group = [] - for key, group in itertools.groupby(enumerate(input_token_type), lambda x: x[1]): - group = list(group) - start_index = group[0][0] - end_index = group[-1][0] + 1 - input_type_group.append((key, start_index, end_index)) - - llm_pos_ids_list = [] - video_frame_num = 1 - for modality_type, start_idx, end_idx in input_type_group: - st_idx = llm_pos_ids_list[-1].max() + 1 if len(llm_pos_ids_list) > 0 else 0 - - if modality_type == "image": - t, h, w = ( - image_grid_thw[image_index][0], - image_grid_thw[image_index][1], - image_grid_thw[image_index][2], - ) - llm_grid_t, llm_grid_h, llm_grid_w = ( - t.item(), - h.item() // spatial_merge_size, - w.item() // spatial_merge_size, - ) - - t_index = torch.arange(llm_grid_t).view(-1, 1).expand(-1, llm_grid_h * llm_grid_w).flatten() - h_index = torch.arange(llm_grid_h).view(1, -1, 1).expand(llm_grid_t, -1, llm_grid_w).flatten() - w_index = torch.arange(llm_grid_w).view(1, 1, -1).expand(llm_grid_t, llm_grid_h, -1).flatten() - llm_pos_ids_list.append(torch.stack([t_index, h_index, w_index]) + st_idx) - - image_index += 1 - video_frame_num = 1 - - elif modality_type == "video": - t, h, w = ( - video_frame_num, - video_grid_thw[video_index][1], - video_grid_thw[video_index][2], - ) - - llm_grid_t, llm_grid_h, llm_grid_w = ( - t, - h.item() // spatial_merge_size, - w.item() // spatial_merge_size, - ) - - for t_idx in range(llm_grid_t): - t_index = torch.tensor(t_idx).view(-1, 1).expand(-1, llm_grid_h * llm_grid_w).flatten() - - h_index = torch.arange(llm_grid_h).view(1, -1, 1).expand(1, -1, llm_grid_w).flatten() - w_index = torch.arange(llm_grid_w).view(1, 1, -1).expand(1, llm_grid_h, -1).flatten() - llm_pos_ids_list.append(torch.stack([t_index, h_index, w_index]) + st_idx) - - video_group_index += 1 - - if video_group_index >= video_grid_thw[video_index][0]: - video_index += 1 - video_group_index = 0 - - video_frame_num += 1 - - else: - text_len = end_idx - start_idx - llm_pos_ids_list.append(torch.arange(text_len).view(1, -1).expand(3, -1) + st_idx) - - video_frame_num = 1 - - llm_positions = torch.cat(llm_pos_ids_list, dim=1).reshape(3, -1) - position_ids[..., i, attention_mask[i] == 1] = llm_positions.to(position_ids.device) - mrope_position_deltas.append(llm_positions.max() + 1 - len(total_input_ids[i])) - mrope_position_deltas = torch.tensor(mrope_position_deltas, device=input_ids.device).unsqueeze(1) - return position_ids, mrope_position_deltas - else: - if attention_mask is not None: - position_ids = attention_mask.long().cumsum(-1) - 1 - position_ids.masked_fill_(attention_mask == 0, 1) - position_ids = position_ids.unsqueeze(0).expand(3, -1, -1).to(attention_mask.device) - max_position_ids = position_ids.max(0, keepdim=False)[0].max(-1, keepdim=True)[0] - mrope_position_deltas = max_position_ids + 1 - attention_mask.shape[-1] - else: - position_ids = ( - torch.arange(input_ids.shape[1], device=input_ids.device) - .view(1, 1, -1) - .expand(3, input_ids.shape[0], -1) - ) - mrope_position_deltas = torch.zeros( - [input_ids.shape[0], 1], - device=input_ids.device, - dtype=input_ids.dtype, - ) - - return position_ids, mrope_position_deltas - - def get_video_features( - self, pixel_values_videos: torch.FloatTensor, video_grid_thw: Optional[torch.LongTensor] = None - ): - """ - Encodes videos into continuous embeddings that can be forwarded to the language model. - - Args: - pixel_values_videos (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`): - The tensors corresponding to the input videos. - video_grid_thw (`torch.LongTensor` of shape `(num_videos, 3)`, *optional*): - The temporal, height and width of feature shape of each video in LLM. - """ - pixel_values_videos = pixel_values_videos.type(self.visual.dtype) - # reshape video_grid_thw -> [b, 3] -> [1, h, w] * frames - temp_frames_hw = [] - for t, h, w in video_grid_thw: - repeated_row = torch.tensor([1, h.item(), w.item()]).unsqueeze(0).repeat(t, 1) - temp_frames_hw.append(repeated_row) - flattened_video_grid_thw = torch.cat(temp_frames_hw, dim=0) - video_embeds = self.visual(pixel_values_videos, grid_thw=flattened_video_grid_thw) - split_sizes = (video_grid_thw.prod(-1) // self.visual.spatial_merge_size**2).tolist() - video_embeds = torch.split(video_embeds, split_sizes) - return video_embeds - - def get_image_features(self, pixel_values: torch.FloatTensor, image_grid_thw: Optional[torch.LongTensor] = None): - """ - Encodes images into continuous embeddings that can be forwarded to the language model. - - Args: - pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`): - The tensors corresponding to the input images. - image_grid_thw (`torch.LongTensor` of shape `(num_images, 3)`, *optional*): - The temporal, height and width of feature shape of each image in LLM. - """ - pixel_values = pixel_values.type(self.visual.dtype) - image_embeds = self.visual(pixel_values, grid_thw=image_grid_thw) - split_sizes = (image_grid_thw.prod(-1) // self.visual.spatial_merge_size**2).tolist() - image_embeds = torch.split(image_embeds, split_sizes) - return image_embeds - - def get_placeholder_mask( - self, - input_ids: torch.LongTensor, - inputs_embeds: torch.FloatTensor, - image_features: Optional[torch.FloatTensor] = None, - video_features: Optional[torch.FloatTensor] = None, - ): - """ - Obtains multimodal placeholder mask from `input_ids` or `inputs_embeds`, and checks that the placeholder token count is - equal to the length of multimodal features. If the lengths are different, an error is raised. - """ - if input_ids is None: - special_image_mask = inputs_embeds == self.get_input_embeddings()( - torch.tensor(self.config.image_token_id, dtype=torch.long, device=inputs_embeds.device) - ) - special_image_mask = special_image_mask.all(-1) - special_video_mask = inputs_embeds == self.get_input_embeddings()( - torch.tensor(self.config.video_token_id, dtype=torch.long, device=inputs_embeds.device) - ) - special_video_mask = special_video_mask.all(-1) - else: - # GLM-4.1V and GLM-4.5V special_video_mask is special_image_mask - special_image_mask = input_ids == self.config.image_token_id - special_video_mask = input_ids == self.config.image_token_id - - n_image_tokens = special_image_mask.sum() - special_image_mask = special_image_mask.unsqueeze(-1).expand_as(inputs_embeds).to(inputs_embeds.device) - if image_features is not None and inputs_embeds[special_image_mask].numel() != image_features.numel(): - raise ValueError( - f"Image features and image tokens do not match: tokens: {n_image_tokens}, features {image_features.shape[0]}" - ) - - n_video_tokens = special_video_mask.sum() - special_video_mask = special_video_mask.unsqueeze(-1).expand_as(inputs_embeds).to(inputs_embeds.device) - if video_features is not None and inputs_embeds[special_video_mask].numel() != video_features.numel(): - raise ValueError( - f"Videos features and video tokens do not match: tokens: {n_video_tokens}, features {video_features.shape[0]}" - ) - - return special_image_mask, special_video_mask - - @auto_docstring - @can_return_tuple - def forward( - self, - input_ids: Optional[torch.LongTensor] = None, - attention_mask: Optional[torch.Tensor] = None, - position_ids: Optional[torch.LongTensor] = None, - past_key_values: Optional[Cache] = None, - inputs_embeds: Optional[torch.FloatTensor] = None, - pixel_values: Optional[torch.Tensor] = None, - pixel_values_videos: Optional[torch.FloatTensor] = None, - image_grid_thw: Optional[torch.LongTensor] = None, - video_grid_thw: Optional[torch.LongTensor] = None, - rope_deltas: Optional[torch.LongTensor] = None, - cache_position: Optional[torch.LongTensor] = None, - **kwargs: Unpack[TransformersKwargs], - ) -> Union[tuple, Glm46VModelOutputWithPast]: - r""" - image_grid_thw (`torch.LongTensor` of shape `(num_images, 3)`, *optional*): - The temporal, height and width of feature shape of each image in LLM. - video_grid_thw (`torch.LongTensor` of shape `(num_videos, 3)`, *optional*): - The temporal, height and width of feature shape of each video in LLM. - rope_deltas (`torch.LongTensor` of shape `(batch_size, )`, *optional*): - The rope index difference between sequence length and multimodal rope. - """ - if (input_ids is None) ^ (inputs_embeds is not None): - raise ValueError("You must specify exactly one of input_ids or inputs_embeds") - - if inputs_embeds is None: - inputs_embeds = self.get_input_embeddings()(input_ids) - - if pixel_values is not None: - image_embeds = self.get_image_features(pixel_values, image_grid_thw) - image_embeds = torch.cat(image_embeds, dim=0).to(inputs_embeds.device, inputs_embeds.dtype) - image_mask, _ = self.get_placeholder_mask(input_ids, inputs_embeds, image_features=image_embeds) - inputs_embeds = inputs_embeds.masked_scatter(image_mask, image_embeds) - - if pixel_values_videos is not None: - video_embeds = self.get_video_features(pixel_values_videos, video_grid_thw) - video_embeds = torch.cat(video_embeds, dim=0).to(inputs_embeds.device, inputs_embeds.dtype) - _, video_mask = self.get_placeholder_mask(input_ids, inputs_embeds, video_features=video_embeds) - inputs_embeds = inputs_embeds.masked_scatter(video_mask, video_embeds) - - if position_ids is None: - attention_mask_tensor = ( - attention_mask if not isinstance(attention_mask, dict) else attention_mask["full_attention"] - ) - if attention_mask_tensor is not None and attention_mask_tensor.ndim == 4: - attention_mask_tensor = torch.diagonal(attention_mask_tensor[:, 0], dim1=1, dim2=2) - # Only apply conversion for floating point tensors (inverted masks) - if attention_mask_tensor.dtype.is_floating_point: - attention_mask_tensor = attention_mask_tensor / torch.finfo(attention_mask_tensor.dtype).min - attention_mask_tensor = (1.0 - attention_mask_tensor).int() - - # Calculate RoPE index once per generation in the pre-fill stage only. - # When compiling, we can't check tensor values thus we check only input length - # It is safe to assume that `length!=1` means we're in pre-fill because compiled - # models currently cannot do asssisted decoding - prefill_compiled_stage = is_torchdynamo_compiling() and ( - (input_ids is not None and input_ids.shape[1] != 1) - or (inputs_embeds is not None and inputs_embeds.shape[1] != 1) - ) - prefill_noncompiled_stage = not is_torchdynamo_compiling() and ( - (cache_position is not None and cache_position[0] == 0) - or (past_key_values is None or past_key_values.get_seq_length() == 0) - ) - if (prefill_compiled_stage or prefill_noncompiled_stage) or self.rope_deltas is None: - position_ids, rope_deltas = self.get_rope_index( - input_ids, - image_grid_thw, - video_grid_thw, - attention_mask=attention_mask_tensor, - ) - self.rope_deltas = rope_deltas - # then use the prev pre-calculated rope-deltas to get the correct position ids - else: - batch_size, seq_length, _ = inputs_embeds.shape - delta = ( - (cache_position[0] + self.rope_deltas).to(inputs_embeds.device) - if cache_position is not None - else 0 - ) - position_ids = torch.arange(seq_length, device=inputs_embeds.device) - position_ids = position_ids.view(1, -1).expand(batch_size, -1) - if cache_position is not None: # otherwise `deltas` is an int `0` - delta = delta.repeat_interleave(batch_size // delta.shape[0], dim=0) - position_ids = position_ids.add(delta) - position_ids = position_ids.unsqueeze(0).expand(3, -1, -1) - - outputs = self.language_model( - input_ids=None, - position_ids=position_ids, - attention_mask=attention_mask, - past_key_values=past_key_values, - inputs_embeds=inputs_embeds, - cache_position=cache_position, - **kwargs, - ) - - return Glm46VModelOutputWithPast( - last_hidden_state=outputs.last_hidden_state, - past_key_values=outputs.past_key_values, - hidden_states=outputs.hidden_states, - attentions=outputs.attentions, - rope_deltas=self.rope_deltas, - ) + pass __all__ = ["Glm46VModel", "Glm46VForConditionalGeneration", "Glm46VPreTrainedModel", "Glm46VTextModel"] diff --git a/src/transformers/models/glm46v/modular_glm46v.py b/src/transformers/models/glm46v/modular_glm46v.py index 787310a51ef6..373caea2160b 100644 --- a/src/transformers/models/glm46v/modular_glm46v.py +++ b/src/transformers/models/glm46v/modular_glm46v.py @@ -46,7 +46,7 @@ class Glm46VForConditionalGeneration(Glm4vForConditionalGeneration): pass -class Glm46VModel(Glm4vModel): +class Glm46VModel(Glm46VPreTrainedModel): pass From d03061d9fe748ca56b0fddb722bedd25befefbb3 Mon Sep 17 00:00:00 2001 From: zRzRzRzRzRzRzR <2448370773@qq.com> Date: Wed, 12 Nov 2025 21:52:40 +0800 Subject: [PATCH 31/66] 1 --- src/transformers/models/glm46v/modular_glm46v.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/transformers/models/glm46v/modular_glm46v.py b/src/transformers/models/glm46v/modular_glm46v.py index 373caea2160b..057e3ebefa93 100644 --- a/src/transformers/models/glm46v/modular_glm46v.py +++ b/src/transformers/models/glm46v/modular_glm46v.py @@ -21,7 +21,7 @@ from ..glm4v.configuration_glm4v import Glm4vConfig, Glm4vTextConfig from ..glm4v.image_processing_glm4v import Glm4vImageProcessor from ..glm4v.image_processing_glm4v_fast import Glm4vImageProcessorFast -from ..glm4v.modeling_glm4v import Glm4vForConditionalGeneration, Glm4vModel, Glm4vPreTrainedModel, Glm4vTextModel +from ..glm4v.modeling_glm4v import Glm4vForConditionalGeneration, Glm4vPreTrainedModel, Glm4vTextModel from ..glm4v.processing_glm4v import Glm4vProcessor from ..glm4v.video_processing_glm4v import Glm4vVideoProcessor From c2033b45a3351f514ae938e5d5406a521b85a137 Mon Sep 17 00:00:00 2001 From: zRzRzRzRzRzRzR <2448370773@qq.com> Date: Thu, 13 Nov 2025 00:17:33 +0800 Subject: [PATCH 32/66] update config --- .../models/glm46v/modeling_glm46v.py | 406 +++++++++++++++++- .../models/glm46v/modular_glm46v.py | 10 +- 2 files changed, 413 insertions(+), 3 deletions(-) diff --git a/src/transformers/models/glm46v/modeling_glm46v.py b/src/transformers/models/glm46v/modeling_glm46v.py index 15ab3b36132d..831698fbf3ea 100644 --- a/src/transformers/models/glm46v/modeling_glm46v.py +++ b/src/transformers/models/glm46v/modeling_glm46v.py @@ -25,6 +25,8 @@ import torch import torch.nn as nn +from torch.nn import LayerNorm +from torchvision.transforms.v2 import functional as F from ...activations import ACT2FN from ...cache_utils import Cache, DynamicCache @@ -39,7 +41,7 @@ from ...processing_utils import Unpack from ...utils import TransformersKwargs, auto_docstring, can_return_tuple from ...utils.generic import check_model_inputs -from .configuration_glm46v import Glm46VConfig, Glm46VTextConfig +from .configuration_glm46v import Glm46VConfig, Glm46VTextConfig, Glm46VVisionConfig @use_kernel_forward_from_hub("RMSNorm") @@ -328,6 +330,400 @@ class Glm46VPreTrainedModel(PreTrainedModel): } +class Glm46VisionMlp(nn.Module): + def __init__(self, config, bias: bool = False): + super().__init__() + self.hidden_size = config.hidden_size + self.intermediate_size = config.out_hidden_size + self.gate_proj = nn.Linear(self.hidden_size, self.intermediate_size, bias=bias) + self.up_proj = nn.Linear(self.hidden_size, self.intermediate_size, bias=bias) + self.down_proj = nn.Linear(self.intermediate_size, self.hidden_size, bias=bias) + self.act_fn = ACT2FN[config.hidden_act] + + def forward(self, hidden_state): + return self.down_proj(self.act_fn(self.gate_proj(hidden_state)) * self.up_proj(hidden_state)) + + +class Glm46VVisionPatchEmbed(nn.Module): + def __init__(self, config: Glm46VVisionConfig) -> None: + super().__init__() + self.patch_size = config.patch_size + self.temporal_patch_size = config.temporal_patch_size + self.in_channels = config.in_channels + self.embed_dim = config.hidden_size + + kernel_size = [self.temporal_patch_size, self.patch_size, self.patch_size] + self.proj = nn.Conv3d(self.in_channels, self.embed_dim, kernel_size=kernel_size, stride=kernel_size) + + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: + target_dtype = self.proj.weight.dtype + hidden_states = hidden_states.view( + -1, self.in_channels, self.temporal_patch_size, self.patch_size, self.patch_size + ) + hidden_states = self.proj(hidden_states.to(dtype=target_dtype)).view(-1, self.embed_dim) + return hidden_states + + +class Glm46VVisionRotaryEmbedding(nn.Module): + inv_freq: torch.Tensor # fix linting for `register_buffer` + + def __init__(self, dim: int, theta: float = 10000.0) -> None: + super().__init__() + inv_freq = 1.0 / (theta ** (torch.arange(0, dim, 2, dtype=torch.float) / dim)) + self.register_buffer("inv_freq", inv_freq, persistent=False) + + def forward(self, seqlen: int) -> torch.Tensor: + seq = torch.arange(seqlen, device=self.inv_freq.device, dtype=self.inv_freq.dtype) + freqs = torch.outer(seq, self.inv_freq) + return freqs + + +class Glm46VVisionPatchMerger(nn.Module): + def __init__(self, dim: int, context_dim: int, hidden_act: str, bias: bool = False) -> None: + super().__init__() + self.proj = nn.Linear(dim, dim, bias=bias) + self.post_projection_norm = LayerNorm(dim) + self.gate_proj = nn.Linear(dim, context_dim, bias=bias) + self.up_proj = nn.Linear(dim, context_dim, bias=bias) + self.down_proj = nn.Linear(context_dim, dim, bias=bias) + self.act1 = nn.GELU() + self.act_fn = ACT2FN[hidden_act] + + def forward(self, hidden_state: torch.Tensor) -> torch.Tensor: + hidden_state = self.proj(hidden_state) + hidden_state = self.act1(self.post_projection_norm(hidden_state)) + return self.down_proj(self.act_fn(self.gate_proj(hidden_state)) * self.up_proj(hidden_state)) + + +class Glm46VVisionEmbeddings(nn.Module): + def __init__(self, config: Glm46VVisionConfig): + super().__init__() + self.config = config + self.embed_dim = config.hidden_size + self.image_size = config.image_size + self.patch_size = config.patch_size + + self.num_patches = (self.image_size // self.patch_size) ** 2 + self.num_positions = self.num_patches + self.position_embedding = nn.Embedding(self.num_positions, self.embed_dim) + self.register_buffer("position_ids", torch.arange(self.num_positions).expand((1, -1)), persistent=False) + + def forward(self, embeddings, lengths, image_shapes, h_coords, w_coords) -> torch.Tensor: + """ + Forward pass with integrated position encoding adaptation using 2D interpolation. + + Args: + embeddings: Input embeddings tensor + lengths (torch.Tensor): Sequence lengths for each image in the batch. + image_shapes (torch.Tensor): Tensor of shape [batch_size, 3] representing the image shapes (t, h, w). + h_coords (torch.Tensor): Tensor of shape [total_seq] representing the h coordinate for each patch. + w_coords (torch.Tensor): Tensor of shape [total_seq] representing the w coordinate for each patch. + + Returns: + torch.Tensor: Embeddings with adapted position encoding added. + """ + # Get position embedding parameters + pos_embed_weight = self.position_embedding.weight + hidden_size = pos_embed_weight.shape[1] + total_seq = h_coords.shape[0] + device = pos_embed_weight.device + + # Move coordinates to correct device + h_coords, w_coords = h_coords.to(device), w_coords.to(device) + + # Handle empty sequence case + if total_seq == 0: + adapted_pos_embed = torch.empty(0, hidden_size, device=device, dtype=pos_embed_weight.dtype) + else: + # Convert inputs to tensors if needed + if isinstance(lengths, list): + lengths = torch.tensor(lengths, device=device, dtype=torch.long) + if not isinstance(image_shapes, torch.Tensor): + image_shapes = torch.tensor(image_shapes, device=device, dtype=torch.long) + + # Prepare 2D position embedding + orig_size_sq = pos_embed_weight.shape[0] + orig_size = int(orig_size_sq**0.5) + pos_embed_2d = ( + pos_embed_weight.view(orig_size, orig_size, hidden_size) + .permute(2, 0, 1) + .unsqueeze(0) + .to(device=device, dtype=torch.float32) + ) + + # Calculate target dimensions for each patch + target_h = torch.cat([image_shapes[i, 1].repeat(lengths[i]) for i in range(len(lengths))]).to( + device=device, dtype=torch.float32 + ) + target_w = torch.cat([image_shapes[i, 2].repeat(lengths[i]) for i in range(len(lengths))]).to( + device=device, dtype=torch.float32 + ) + + # Normalize coordinates to [-1, 1] range for grid_sample + h_coords = h_coords.to(device=device, dtype=torch.float32) + w_coords = w_coords.to(device=device, dtype=torch.float32) + norm_w = ((w_coords + 0.5) / target_w) * 2 - 1 + norm_h = ((h_coords + 0.5) / target_h) * 2 - 1 + + # Create sampling grid + grid = torch.stack((norm_w, norm_h), dim=-1).unsqueeze(0).unsqueeze(2) + + # Perform bicubic interpolation + interpolated_embed_fp32 = F.grid_sample( + pos_embed_2d, grid, mode="bicubic", align_corners=False, padding_mode="border" + ) + + # Reshape and convert back to original dtype + adapted_pos_embed_fp32 = interpolated_embed_fp32.squeeze(0).squeeze(-1).permute(1, 0) + adapted_pos_embed = adapted_pos_embed_fp32.to(pos_embed_weight.dtype).to(embeddings.device) + + # Add adapted position encoding to embeddings + embeddings = embeddings + adapted_pos_embed + return embeddings + + +def rotate_half(x): + """Rotates half the hidden dims of the input.""" + x1 = x[..., : x.shape[-1] // 2] + x2 = x[..., x.shape[-1] // 2 :] + return torch.cat((-x2, x1), dim=-1) + + +def apply_rotary_pos_emb_vision( + q: torch.Tensor, k: torch.Tensor, cos: torch.Tensor, sin: torch.Tensor +) -> tuple[torch.Tensor, torch.Tensor]: + orig_q_dtype = q.dtype + orig_k_dtype = k.dtype + q, k = q.float(), k.float() + cos, sin = cos.unsqueeze(-2).float(), sin.unsqueeze(-2).float() + q_embed = (q * cos) + (rotate_half(q) * sin) + k_embed = (k * cos) + (rotate_half(k) * sin) + q_embed = q_embed.to(orig_q_dtype) + k_embed = k_embed.to(orig_k_dtype) + return q_embed, k_embed + + +class Glm46VVisionAttention(nn.Module): + def __init__(self, config: Glm46VVisionConfig) -> None: + super().__init__() + self.dim = config.hidden_size + self.num_heads = config.num_heads + self.head_dim = self.dim // self.num_heads + self.num_key_value_groups = 1 # needed for eager attention + self.qkv = nn.Linear(config.hidden_size, config.hidden_size * 3, bias=config.attention_bias) + self.proj = nn.Linear(config.hidden_size, config.hidden_size, bias=False) + self.scaling = self.head_dim**-0.5 + self.config = config + self.attention_dropout = config.attention_dropout + self.is_causal = False + + def forward( + self, + hidden_states: torch.Tensor, + cu_seqlens: torch.Tensor, + rotary_pos_emb: Optional[torch.Tensor] = None, + position_embeddings: Optional[tuple[torch.Tensor, torch.Tensor]] = None, + **kwargs, + ) -> torch.Tensor: + seq_length = hidden_states.shape[0] + query_states, key_states, value_states = ( + self.qkv(hidden_states).reshape(seq_length, 3, self.num_heads, -1).permute(1, 0, 2, 3).unbind(0) + ) + cos, sin = position_embeddings + query_states, key_states = apply_rotary_pos_emb_vision(query_states, key_states, cos, sin) + + query_states = query_states.transpose(0, 1).unsqueeze(0) + key_states = key_states.transpose(0, 1).unsqueeze(0) + value_states = value_states.transpose(0, 1).unsqueeze(0) + + attention_interface: Callable = eager_attention_forward + if self.config._attn_implementation != "eager": + attention_interface = ALL_ATTENTION_FUNCTIONS[self.config._attn_implementation] + + if self.config._attn_implementation == "flash_attention_2": + # Flash Attention 2: Use cu_seqlens for variable length attention + max_seqlen = (cu_seqlens[1:] - cu_seqlens[:-1]).max() + attn_output, _ = attention_interface( + self, + query_states, + key_states, + value_states, + attention_mask=None, + scaling=self.scaling, + dropout=0.0 if not self.training else self.attention_dropout, + cu_seq_lens_q=cu_seqlens, + cu_seq_lens_k=cu_seqlens, + max_length_q=max_seqlen, + max_length_k=max_seqlen, + is_causal=False, + **kwargs, + ) + else: + # Other implementations: Process each chunk separately + lengths = cu_seqlens[1:] - cu_seqlens[:-1] + splits = [ + torch.split(tensor, lengths.tolist(), dim=2) for tensor in (query_states, key_states, value_states) + ] + + attn_outputs = [ + attention_interface( + self, + q, + k, + v, + attention_mask=None, + scaling=self.scaling, + dropout=0.0 if not self.training else self.attention_dropout, + is_causal=False, + **kwargs, + )[0] + for q, k, v in zip(*splits) + ] + attn_output = torch.cat(attn_outputs, dim=1) + + attn_output = attn_output.reshape(seq_length, -1).contiguous() + attn_output = self.proj(attn_output) + return attn_output + + +class Glm46VVisionBlock(GradientCheckpointingLayer): + def __init__(self, config) -> None: + super().__init__() + self.norm1 = Glm46VRMSNorm(config.hidden_size, eps=config.rms_norm_eps) + self.norm2 = Glm46VRMSNorm(config.hidden_size, eps=config.rms_norm_eps) + self.attn = Glm46VVisionAttention(config) + self.mlp = Glm46VisionMlp(config, bias=False) + + def forward( + self, + hidden_states: torch.Tensor, + cu_seqlens: torch.Tensor, + rotary_pos_emb: Optional[torch.Tensor] = None, + position_embeddings: Optional[tuple[torch.Tensor, torch.Tensor]] = None, + **kwargs, + ) -> torch.Tensor: + hidden_states = hidden_states + self.attn( + self.norm1(hidden_states), + cu_seqlens=cu_seqlens, + rotary_pos_emb=rotary_pos_emb, + position_embeddings=position_embeddings, + **kwargs, + ) + hidden_states = hidden_states + self.mlp(self.norm2(hidden_states)) + return hidden_states + + +class Glm46VVisionModel(Glm46VPreTrainedModel): + config: Glm46VVisionConfig + input_modalities = ["image", "video"] + _no_split_modules = ["Glm46VVisionBlock"] + + def __init__(self, config) -> None: + super().__init__(config) + self.spatial_merge_size = config.spatial_merge_size + self.patch_size = config.patch_size + + self.embeddings = Glm46VVisionEmbeddings(config) + self.patch_embed = Glm46VVisionPatchEmbed(config) + + head_dim = config.hidden_size // config.num_heads + self.rotary_pos_emb = Glm46VVisionRotaryEmbedding(head_dim // 2) + + self.blocks = nn.ModuleList([Glm46VVisionBlock(config) for _ in range(config.depth)]) + self.merger = Glm46VVisionPatchMerger( + dim=config.out_hidden_size, context_dim=config.intermediate_size, hidden_act=config.hidden_act + ) + + self.post_conv_layernorm = Glm46VRMSNorm(config.hidden_size, eps=config.rms_norm_eps) + self.downsample = nn.Conv2d( + in_channels=config.hidden_size, + out_channels=config.out_hidden_size, + kernel_size=config.spatial_merge_size, + stride=config.spatial_merge_size, + ) + self.post_layernorm = Glm46VRMSNorm(config.hidden_size, eps=config.rms_norm_eps) + + self.gradient_checkpointing = False + self.post_init() + + def rot_pos_emb(self, grid_thw): + pos_ids = [] + for t, h, w in grid_thw: + hpos_ids = torch.arange(h).unsqueeze(1).expand(-1, w) + hpos_ids = hpos_ids.reshape( + h // self.spatial_merge_size, + self.spatial_merge_size, + w // self.spatial_merge_size, + self.spatial_merge_size, + ) + hpos_ids = hpos_ids.permute(0, 2, 1, 3) + hpos_ids = hpos_ids.flatten() + + wpos_ids = torch.arange(w).unsqueeze(0).expand(h, -1) + wpos_ids = wpos_ids.reshape( + h // self.spatial_merge_size, + self.spatial_merge_size, + w // self.spatial_merge_size, + self.spatial_merge_size, + ) + wpos_ids = wpos_ids.permute(0, 2, 1, 3) + wpos_ids = wpos_ids.flatten() + pos_ids.append(torch.stack([hpos_ids, wpos_ids], dim=-1).repeat(t, 1)) + pos_ids = torch.cat(pos_ids, dim=0) + max_grid_size = grid_thw[:, 1:].max() + rotary_pos_emb_full = self.rotary_pos_emb(max_grid_size) + rotary_pos_emb = rotary_pos_emb_full[pos_ids].flatten(1) + return rotary_pos_emb, pos_ids + + def forward(self, hidden_states: torch.Tensor, grid_thw: torch.Tensor) -> torch.Tensor: + """ + Args: + hidden_states (`torch.Tensor` of shape `(seq_len, hidden_size)`): + The final hidden states of the model. + grid_thw (`torch.Tensor` of shape `(num_images_or_videos, 3)`): + The temporal, height and width of feature shape of each image in LLM. + + Returns: + `torch.Tensor`: hidden_states. + """ + hidden_states = self.patch_embed(hidden_states) + hidden_states = self.post_conv_layernorm(hidden_states) + + rotary_pos_emb, image_type_ids = self.rot_pos_emb(grid_thw) + emb = torch.cat((rotary_pos_emb, rotary_pos_emb), dim=-1) + position_embeddings = (emb.cos(), emb.sin()) + + cu_seqlens = torch.repeat_interleave(grid_thw[:, 1] * grid_thw[:, 2], grid_thw[:, 0]).cumsum( + dim=0, + # Select dtype based on the following factors: + # - FA2 requires that cu_seqlens_q must have dtype int32 + # - torch.onnx.export requires that cu_seqlens_q must have same dtype as grid_thw + # See https://github.com/huggingface/transformers/pull/34852 for more information + dtype=grid_thw.dtype if torch.jit.is_tracing() else torch.int32, + ) + cu_seqlens = F.pad(cu_seqlens, (1, 0), value=0) + seqlens = (cu_seqlens[1:] - cu_seqlens[:-1]).tolist() + hidden_states = self.embeddings(hidden_states, seqlens, grid_thw, image_type_ids[:, 0], image_type_ids[:, 1]) + + for blk in self.blocks: + hidden_states = blk( + hidden_states, + cu_seqlens=cu_seqlens, + position_embeddings=position_embeddings, + ) + + hidden_states = self.post_layernorm(hidden_states) + + hidden_states = hidden_states.view( + -1, self.spatial_merge_size, self.spatial_merge_size, hidden_states.shape[-1] + ) + hidden_states = hidden_states.permute(0, 3, 1, 2) + hidden_states = self.downsample(hidden_states).view(-1, self.config.out_hidden_size) + + hidden_states = self.merger(hidden_states) + return hidden_states + + class Glm46VTextRotaryEmbedding(nn.Module): inv_freq: torch.Tensor # fix linting for `register_buffer` @@ -858,4 +1254,10 @@ class Glm46VModel(Glm46VPreTrainedModel): pass -__all__ = ["Glm46VModel", "Glm46VForConditionalGeneration", "Glm46VPreTrainedModel", "Glm46VTextModel"] +__all__ = [ + "Glm46VModel", + "Glm46VForConditionalGeneration", + "Glm46VPreTrainedModel", + "Glm46VVisionModel", + "Glm46VTextModel", +] diff --git a/src/transformers/models/glm46v/modular_glm46v.py b/src/transformers/models/glm46v/modular_glm46v.py index 057e3ebefa93..53ae90b97c96 100644 --- a/src/transformers/models/glm46v/modular_glm46v.py +++ b/src/transformers/models/glm46v/modular_glm46v.py @@ -21,7 +21,12 @@ from ..glm4v.configuration_glm4v import Glm4vConfig, Glm4vTextConfig from ..glm4v.image_processing_glm4v import Glm4vImageProcessor from ..glm4v.image_processing_glm4v_fast import Glm4vImageProcessorFast -from ..glm4v.modeling_glm4v import Glm4vForConditionalGeneration, Glm4vPreTrainedModel, Glm4vTextModel +from ..glm4v.modeling_glm4v import ( + Glm4vForConditionalGeneration, + Glm4vPreTrainedModel, + Glm4vTextModel, + Glm4vVisionModel, +) from ..glm4v.processing_glm4v import Glm4vProcessor from ..glm4v.video_processing_glm4v import Glm4vVideoProcessor @@ -37,6 +42,8 @@ class Glm46VConfig(Glm4vConfig): class Glm46VPreTrainedModel(Glm4vPreTrainedModel): pass +class Glm46VVisionModel(Glm4vVisionModel): + pass class Glm46VTextModel(Glm4vTextModel): pass @@ -137,6 +144,7 @@ def sample_frames( "Glm46VModel", "Glm46VForConditionalGeneration", "Glm46VPreTrainedModel", + "Glm46VVisionModel", "Glm46VTextModel", "Glm46VProcessor", "Glm46VImageProcessor", From 79eef09feb71fa5193ee5d1b057dce109e79e5bb Mon Sep 17 00:00:00 2001 From: zRzRzRzRzRzRzR <2448370773@qq.com> Date: Thu, 13 Nov 2025 00:20:17 +0800 Subject: [PATCH 33/66] 1 --- src/transformers/models/glm46v/modular_glm46v.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/transformers/models/glm46v/modular_glm46v.py b/src/transformers/models/glm46v/modular_glm46v.py index 53ae90b97c96..3a557843c50f 100644 --- a/src/transformers/models/glm46v/modular_glm46v.py +++ b/src/transformers/models/glm46v/modular_glm46v.py @@ -42,9 +42,11 @@ class Glm46VConfig(Glm4vConfig): class Glm46VPreTrainedModel(Glm4vPreTrainedModel): pass + class Glm46VVisionModel(Glm4vVisionModel): pass + class Glm46VTextModel(Glm4vTextModel): pass From 514dec8b87781d5c732937e53d37110b1343c0e7 Mon Sep 17 00:00:00 2001 From: zRzRzRzRzRzRzR <2448370773@qq.com> Date: Thu, 13 Nov 2025 00:25:34 +0800 Subject: [PATCH 34/66] update as automoel --- .../models/glm46v/modeling_glm46v.py | 825 ++++++++---------- .../models/glm46v/modular_glm46v.py | 21 +- 2 files changed, 367 insertions(+), 479 deletions(-) diff --git a/src/transformers/models/glm46v/modeling_glm46v.py b/src/transformers/models/glm46v/modeling_glm46v.py index 831698fbf3ea..4096b4c7bb22 100644 --- a/src/transformers/models/glm46v/modeling_glm46v.py +++ b/src/transformers/models/glm46v/modeling_glm46v.py @@ -19,9 +19,10 @@ # See the License for the specific language governing permissions and # limitations under the License. +import itertools from collections.abc import Callable from dataclasses import dataclass -from typing import Any, Optional, Union +from typing import Optional, Union import torch import torch.nn as nn @@ -29,18 +30,15 @@ from torchvision.transforms.v2 import functional as F from ...activations import ACT2FN -from ...cache_utils import Cache, DynamicCache -from ...generation import GenerationMixin +from ...cache_utils import Cache from ...integrations import use_kernel_forward_from_hub -from ...masking_utils import create_causal_mask from ...modeling_flash_attention_utils import FlashAttentionKwargs from ...modeling_layers import GradientCheckpointingLayer -from ...modeling_outputs import BaseModelOutputWithPast, ModelOutput -from ...modeling_rope_utils import ROPE_INIT_FUNCTIONS, dynamic_rope_update +from ...modeling_outputs import ModelOutput from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel from ...processing_utils import Unpack -from ...utils import TransformersKwargs, auto_docstring, can_return_tuple -from ...utils.generic import check_model_inputs +from ...utils import TransformersKwargs, auto_docstring, can_return_tuple, is_torchdynamo_compiling +from ..auto.modeling_auto import AutoModel from .configuration_glm46v import Glm46VConfig, Glm46VTextConfig, Glm46VVisionConfig @@ -724,194 +722,14 @@ def forward(self, hidden_states: torch.Tensor, grid_thw: torch.Tensor) -> torch. return hidden_states -class Glm46VTextRotaryEmbedding(nn.Module): - inv_freq: torch.Tensor # fix linting for `register_buffer` - - def __init__(self, config: Glm46VTextConfig, device=None): - super().__init__() - self.max_seq_len_cached = config.max_position_embeddings - self.original_max_seq_len = config.max_position_embeddings - - self.config = config - - self.rope_type = self.config.rope_parameters["rope_type"] - rope_init_fn: Callable = self.compute_default_rope_parameters - if self.rope_type != "default": - rope_init_fn = ROPE_INIT_FUNCTIONS[self.rope_type] - inv_freq, self.attention_scaling = rope_init_fn(self.config, device) - - self.register_buffer("inv_freq", inv_freq, persistent=False) - self.original_inv_freq = inv_freq - - @staticmethod - def compute_default_rope_parameters( - config: Optional[Glm46VTextConfig] = None, - device: Optional["torch.device"] = None, - seq_len: Optional[int] = None, - ) -> tuple["torch.Tensor", float]: - """ - Computes the inverse frequencies according to the original RoPE implementation - Args: - config ([`~transformers.PreTrainedConfig`]): - The model configuration. - device (`torch.device`): - The device to use for initialization of the inverse frequencies. - seq_len (`int`, *optional*): - The current sequence length. Unused for this type of RoPE. - Returns: - Tuple of (`torch.Tensor`, `float`), containing the inverse frequencies for the RoPE embeddings and the - post-processing scaling factor applied to the computed cos/sin (unused in this type of RoPE). - """ - base = config.rope_parameters["rope_theta"] - partial_rotary_factor = getattr(config, "partial_rotary_factor", 1.0) - head_dim = getattr(config, "head_dim", None) or config.hidden_size // config.num_attention_heads - dim = int(head_dim * partial_rotary_factor) - - attention_factor = 1.0 # Unused in this type of RoPE - - # Compute the inverse frequencies - inv_freq = 1.0 / ( - base ** (torch.arange(0, dim, 2, dtype=torch.int64).to(device=device, dtype=torch.float) / dim) - ) - return inv_freq, attention_factor - - @torch.no_grad() - @dynamic_rope_update # power user: used with advanced RoPE types (e.g. dynamic rope) - def forward(self, x, position_ids): - # In contrast to other models, GLM46V different position ids for the grids - # So we expand the inv_freq to shape (3, ...) - inv_freq_expanded = self.inv_freq[None, None, :, None].float().expand(3, position_ids.shape[1], -1, 1) - position_ids_expanded = position_ids[:, :, None, :].float() # shape (3, bs, 1, positions) - - device_type = x.device.type if isinstance(x.device.type, str) and x.device.type != "mps" else "cpu" - with torch.autocast(device_type=device_type, enabled=False): # Force float32 - freqs = (inv_freq_expanded.float() @ position_ids_expanded.float()).transpose(2, 3) - emb = torch.cat((freqs, freqs), dim=-1) - cos = emb.cos() * self.attention_scaling - sin = emb.sin() * self.attention_scaling - - return cos.to(dtype=x.dtype), sin.to(dtype=x.dtype) - - -@auto_docstring -class Glm46VTextModel(Glm46VPreTrainedModel): - config: Glm46VTextConfig - input_modalities = "text" - - def __init__(self, config: Glm46VTextConfig): - super().__init__(config) - self.padding_idx = config.pad_token_id - self.vocab_size = config.vocab_size - - self.embed_tokens = nn.Embedding(config.vocab_size, config.hidden_size, self.padding_idx) - self.layers = nn.ModuleList( - [Glm46VTextDecoderLayer(config, layer_idx) for layer_idx in range(config.num_hidden_layers)] - ) - self.norm = Glm46VRMSNorm(config.hidden_size, eps=config.rms_norm_eps) - self.rotary_emb = Glm46VTextRotaryEmbedding(config=config) - - self.gradient_checkpointing = False - # Initialize weights and apply final processing - self.post_init() - - @auto_docstring - @check_model_inputs() - def forward( - self, - input_ids: Optional[torch.LongTensor] = None, - attention_mask: Optional[torch.Tensor] = None, - position_ids: Optional[torch.LongTensor] = None, - past_key_values: Optional[Cache] = None, - inputs_embeds: Optional[torch.FloatTensor] = None, - use_cache: Optional[bool] = None, - cache_position: Optional[torch.LongTensor] = None, - **kwargs: Unpack[FlashAttentionKwargs], - ) -> Union[tuple, BaseModelOutputWithPast]: - if (input_ids is None) ^ (inputs_embeds is not None): - raise ValueError("You must specify exactly one of input_ids or inputs_embeds") - - # torch.jit.trace() doesn't support cache objects in the output - if use_cache and past_key_values is None and not torch.jit.is_tracing(): - past_key_values = DynamicCache(config=self.config) - - if inputs_embeds is None: - inputs_embeds = self.embed_tokens(input_ids) - - if cache_position is None: - past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 - cache_position = torch.arange( - past_seen_tokens, past_seen_tokens + inputs_embeds.shape[1], device=inputs_embeds.device - ) - - # the hard coded `3` is for temporal, height and width. - if position_ids is None: - position_ids = cache_position.view(1, 1, -1).expand(3, inputs_embeds.shape[0], -1) - elif position_ids.ndim == 2: - position_ids = position_ids[None, ...].expand(3, position_ids.shape[0], -1) - - # NOTE: we need to pass text position ids for packing. Qwen2-VL uses 3D positions - # where each dim indicates visual spatial positions for temporal/height/width grids. - # There are two scenarios when FA2-like packed masking might be activated. - # 1. User specifically passed packed `position_ids` and no attention mask. - # In this case we expect the useer to create correct position ids for all 3 grids - # and prepend text-only position ids to it. The final tensor will be [4, bs, seq-len] - # 2. User runs forward with no attention mask and no position ids. In this case, position ids - # are prepared by the model (`get_rope_index`) as `[4, bs, seq-len]` tensor. Text-only positions are - # prepended by us when creating positions so that the mask is constructed correctly. NOTE: failing to pass - # text-only positions will cause incorrect mask construction, do not change `prepare_input_for_generation` - if position_ids.ndim == 3 and position_ids.shape[0] == 4: - text_position_ids = position_ids[0] - position_ids = position_ids[1:] - else: - # If inputs are not packed (usual 3D positions), do not prepare mask from position_ids - text_position_ids = None - - mask_kwargs = { - "config": self.config, - "input_embeds": inputs_embeds, - "attention_mask": attention_mask, - "cache_position": cache_position, - "past_key_values": past_key_values, - "position_ids": text_position_ids, - } - # Create the masks - causal_mask = create_causal_mask(**mask_kwargs) - - hidden_states = inputs_embeds - position_embeddings = self.rotary_emb(hidden_states, position_ids=position_ids) - - for decoder_layer in self.layers: - layer_outputs = decoder_layer( - hidden_states, - attention_mask=causal_mask, - position_ids=position_ids, - past_key_values=past_key_values, - cache_position=cache_position, - position_embeddings=position_embeddings, - **kwargs, - ) - hidden_states = layer_outputs - - hidden_states = self.norm(hidden_states) - - return BaseModelOutputWithPast( - last_hidden_state=hidden_states, - past_key_values=past_key_values, - ) - - @dataclass @auto_docstring( custom_intro=""" - Base class for Glm46V causal language model (or autoregressive) outputs. + Base class for Llava outputs, with hidden states and attentions. """ ) -class Glm46VCausalLMOutputWithPast(ModelOutput): +class Glm46VModelOutputWithPast(ModelOutput): r""" - loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `labels` is provided): - Language modeling loss (for next-token prediction). - logits (`torch.FloatTensor` of shape `(batch_size, sequence_length, config.vocab_size)`): - Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax). past_key_values (`Cache`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`): It is a [`~cache_utils.Cache`] instance. For more details, see our [kv cache guide](https://huggingface.co/docs/transformers/en/kv_cache). @@ -921,58 +739,316 @@ class Glm46VCausalLMOutputWithPast(ModelOutput): The rope index difference between sequence length and multimodal rope. """ - loss: Optional[torch.FloatTensor] = None - logits: Optional[torch.FloatTensor] = None + last_hidden_state: Optional[torch.FloatTensor] = None past_key_values: Optional[Cache] = None hidden_states: Optional[tuple[torch.FloatTensor]] = None attentions: Optional[tuple[torch.FloatTensor]] = None rope_deltas: Optional[torch.LongTensor] = None -class Glm46VForConditionalGeneration(Glm46VPreTrainedModel, GenerationMixin): +@auto_docstring +class Glm46VModel(Glm46VPreTrainedModel): + base_model_prefix = "" _checkpoint_conversion_mapping = {} - _tied_weights_keys = ["lm_head.weight"] # Reference: fix gemma3 grad acc #37208 accepts_loss_kwargs = False + config: Glm46VConfig + _no_split_modules = ["Glm46VTextDecoderLayer", "Glm46VVisionBlock"] - def __init__(self, config): + def __init__(self, config: Glm46VConfig): super().__init__(config) - self.model = Glm46VModel(config) - self.lm_head = nn.Linear(config.text_config.hidden_size, config.text_config.vocab_size, bias=False) + self.visual = AutoModel.from_config(config.vision_config) + self.language_model = AutoModel.from_config(config.text_config) + self.rope_deltas = None # cache rope_deltas here + # Initialize weights and apply final processing self.post_init() def get_input_embeddings(self): - return self.model.get_input_embeddings() + return self.language_model.get_input_embeddings() def set_input_embeddings(self, value): - self.model.set_input_embeddings(value) + self.language_model.set_input_embeddings(value) def set_decoder(self, decoder): - self.model.set_decoder(decoder) + self.language_model = decoder def get_decoder(self): - return self.model.get_decoder() + return self.language_model + + def get_rope_index( + self, + input_ids: Optional[torch.LongTensor] = None, + image_grid_thw: Optional[torch.LongTensor] = None, + video_grid_thw: Optional[torch.LongTensor] = None, + attention_mask: Optional[torch.Tensor] = None, + ) -> tuple[torch.Tensor, torch.Tensor]: + """ + Calculate the 3D rope index based on image and video's temporal, height and width in LLM. + + Explanation: + Each embedding sequence contains vision embedding and text embedding or just contains text embedding. + + For pure text embedding sequence, the rotary position embedding has no difference with modern LLMs. + Examples: + input_ids: [T T T T T], here T is for text. + temporal position_ids: [0, 1, 2, 3, 4] + height position_ids: [0, 1, 2, 3, 4] + width position_ids: [0, 1, 2, 3, 4] + + For vision and text embedding sequence, we calculate 3D rotary position embedding for vision part + and 1D rotary position embedding for text part. + Examples: + Temporal (Time): 3 patches, representing different segments of the video in time. + Height: 2 patches, dividing each frame vertically. + Width: 2 patches, dividing each frame horizontally. + We also have some important parameters: + fps (Frames Per Second): The video's frame rate, set to 1. This means one frame is processed each second. + tokens_per_second: This is a crucial parameter. It dictates how many "time-steps" or "temporal tokens" are conceptually packed into a one-second interval of the video. In this case, we have 25 tokens per second. So each second of the video will be represented with 25 separate time points. It essentially defines the temporal granularity. + temporal_patch_size: The number of frames that compose one temporal patch. Here, it's 2 frames. + interval: The step size for the temporal position IDs, calculated as tokens_per_second * temporal_patch_size / fps. In this case, 25 * 2 / 1 = 50. This means that each temporal patch will be have a difference of 50 in the temporal position IDs. + input_ids: [V V V V V V V V V V V V T T T T T], here V is for vision. + vision temporal position_ids: [0, 0, 0, 0, 50, 50, 50, 50, 100, 100, 100, 100] + vision height position_ids: [0, 0, 1, 1, 0, 0, 1, 1, 0, 0, 1, 1] + vision width position_ids: [0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1] + text temporal position_ids: [101, 102, 103, 104, 105] + text height position_ids: [101, 102, 103, 104, 105] + text width position_ids: [101, 102, 103, 104, 105] + Here we calculate the text start position_ids as the max vision position_ids plus 1. + + Args: + input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): + Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide + it. + image_grid_thw (`torch.LongTensor` of shape `(num_images, 3)`, *optional*): + The temporal, height and width of feature shape of each image in LLM. + video_grid_thw (`torch.LongTensor` of shape `(num_videos, 3)`, *optional*): + The temporal, height and width of feature shape of each video in LLM. + attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): + Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: + + - 1 for tokens that are **not masked**, + - 0 for tokens that are **masked**. + + Returns: + position_ids (`torch.LongTensor` of shape `(3, batch_size, sequence_length)`) + mrope_position_deltas (`torch.Tensor` of shape `(batch_size)`) + """ + + spatial_merge_size = self.config.vision_config.spatial_merge_size + image_token_id = self.config.image_token_id + video_start_token_id = self.config.video_start_token_id + video_end_token_id = self.config.video_end_token_id + + mrope_position_deltas = [] + if input_ids is not None and (image_grid_thw is not None or video_grid_thw is not None): + total_input_ids = input_ids + if attention_mask is None: + attention_mask = torch.ones_like(total_input_ids) + position_ids = torch.ones( + 3, + input_ids.shape[0], + input_ids.shape[1], + dtype=input_ids.dtype, + device=input_ids.device, + ) + image_index, video_index = 0, 0 + video_group_index = 0 + attention_mask = attention_mask.to(total_input_ids.device) + for i, input_ids in enumerate(total_input_ids): + input_ids = input_ids[attention_mask[i] == 1] + input_tokens = input_ids.tolist() + + input_token_type = [] + video_check_flg = False + for token in input_tokens: + if token == video_start_token_id: + video_check_flg = True + elif token == video_end_token_id: + video_check_flg = False + + if token == image_token_id and not video_check_flg: + input_token_type.append("image") + elif token == image_token_id and video_check_flg: + input_token_type.append("video") + else: + input_token_type.append("text") + + input_type_group = [] + for key, group in itertools.groupby(enumerate(input_token_type), lambda x: x[1]): + group = list(group) + start_index = group[0][0] + end_index = group[-1][0] + 1 + input_type_group.append((key, start_index, end_index)) + + llm_pos_ids_list = [] + video_frame_num = 1 + for modality_type, start_idx, end_idx in input_type_group: + st_idx = llm_pos_ids_list[-1].max() + 1 if len(llm_pos_ids_list) > 0 else 0 + + if modality_type == "image": + t, h, w = ( + image_grid_thw[image_index][0], + image_grid_thw[image_index][1], + image_grid_thw[image_index][2], + ) + llm_grid_t, llm_grid_h, llm_grid_w = ( + t.item(), + h.item() // spatial_merge_size, + w.item() // spatial_merge_size, + ) + + t_index = torch.arange(llm_grid_t).view(-1, 1).expand(-1, llm_grid_h * llm_grid_w).flatten() + h_index = torch.arange(llm_grid_h).view(1, -1, 1).expand(llm_grid_t, -1, llm_grid_w).flatten() + w_index = torch.arange(llm_grid_w).view(1, 1, -1).expand(llm_grid_t, llm_grid_h, -1).flatten() + llm_pos_ids_list.append(torch.stack([t_index, h_index, w_index]) + st_idx) + + image_index += 1 + video_frame_num = 1 + + elif modality_type == "video": + t, h, w = ( + video_frame_num, + video_grid_thw[video_index][1], + video_grid_thw[video_index][2], + ) + + llm_grid_t, llm_grid_h, llm_grid_w = ( + t, + h.item() // spatial_merge_size, + w.item() // spatial_merge_size, + ) + + for t_idx in range(llm_grid_t): + t_index = torch.tensor(t_idx).view(-1, 1).expand(-1, llm_grid_h * llm_grid_w).flatten() + + h_index = torch.arange(llm_grid_h).view(1, -1, 1).expand(1, -1, llm_grid_w).flatten() + w_index = torch.arange(llm_grid_w).view(1, 1, -1).expand(1, llm_grid_h, -1).flatten() + llm_pos_ids_list.append(torch.stack([t_index, h_index, w_index]) + st_idx) + + video_group_index += 1 + + if video_group_index >= video_grid_thw[video_index][0]: + video_index += 1 + video_group_index = 0 + + video_frame_num += 1 + + else: + text_len = end_idx - start_idx + llm_pos_ids_list.append(torch.arange(text_len).view(1, -1).expand(3, -1) + st_idx) + + video_frame_num = 1 + + llm_positions = torch.cat(llm_pos_ids_list, dim=1).reshape(3, -1) + position_ids[..., i, attention_mask[i] == 1] = llm_positions.to(position_ids.device) + mrope_position_deltas.append(llm_positions.max() + 1 - len(total_input_ids[i])) + mrope_position_deltas = torch.tensor(mrope_position_deltas, device=input_ids.device).unsqueeze(1) + return position_ids, mrope_position_deltas + else: + if attention_mask is not None: + position_ids = attention_mask.long().cumsum(-1) - 1 + position_ids.masked_fill_(attention_mask == 0, 1) + position_ids = position_ids.unsqueeze(0).expand(3, -1, -1).to(attention_mask.device) + max_position_ids = position_ids.max(0, keepdim=False)[0].max(-1, keepdim=True)[0] + mrope_position_deltas = max_position_ids + 1 - attention_mask.shape[-1] + else: + position_ids = ( + torch.arange(input_ids.shape[1], device=input_ids.device) + .view(1, 1, -1) + .expand(3, input_ids.shape[0], -1) + ) + mrope_position_deltas = torch.zeros( + [input_ids.shape[0], 1], + device=input_ids.device, + dtype=input_ids.dtype, + ) + + return position_ids, mrope_position_deltas def get_video_features( self, pixel_values_videos: torch.FloatTensor, video_grid_thw: Optional[torch.LongTensor] = None ): - return self.model.get_video_features(pixel_values_videos, video_grid_thw) + """ + Encodes videos into continuous embeddings that can be forwarded to the language model. + + Args: + pixel_values_videos (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`): + The tensors corresponding to the input videos. + video_grid_thw (`torch.LongTensor` of shape `(num_videos, 3)`, *optional*): + The temporal, height and width of feature shape of each video in LLM. + """ + pixel_values_videos = pixel_values_videos.type(self.visual.dtype) + # reshape video_grid_thw -> [b, 3] -> [1, h, w] * frames + temp_frames_hw = [] + for t, h, w in video_grid_thw: + repeated_row = torch.tensor([1, h.item(), w.item()]).unsqueeze(0).repeat(t, 1) + temp_frames_hw.append(repeated_row) + flattened_video_grid_thw = torch.cat(temp_frames_hw, dim=0) + video_embeds = self.visual(pixel_values_videos, grid_thw=flattened_video_grid_thw) + split_sizes = (video_grid_thw.prod(-1) // self.visual.spatial_merge_size**2).tolist() + video_embeds = torch.split(video_embeds, split_sizes) + return video_embeds def get_image_features(self, pixel_values: torch.FloatTensor, image_grid_thw: Optional[torch.LongTensor] = None): - return self.model.get_image_features(pixel_values, image_grid_thw) + """ + Encodes images into continuous embeddings that can be forwarded to the language model. + + Args: + pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`): + The tensors corresponding to the input images. + image_grid_thw (`torch.LongTensor` of shape `(num_images, 3)`, *optional*): + The temporal, height and width of feature shape of each image in LLM. + """ + pixel_values = pixel_values.type(self.visual.dtype) + image_embeds = self.visual(pixel_values, grid_thw=image_grid_thw) + split_sizes = (image_grid_thw.prod(-1) // self.visual.spatial_merge_size**2).tolist() + image_embeds = torch.split(image_embeds, split_sizes) + return image_embeds - # Make modules available through conditional class for BC - @property - def language_model(self): - return self.model.language_model + def get_placeholder_mask( + self, + input_ids: torch.LongTensor, + inputs_embeds: torch.FloatTensor, + image_features: Optional[torch.FloatTensor] = None, + video_features: Optional[torch.FloatTensor] = None, + ): + """ + Obtains multimodal placeholder mask from `input_ids` or `inputs_embeds`, and checks that the placeholder token count is + equal to the length of multimodal features. If the lengths are different, an error is raised. + """ + if input_ids is None: + special_image_mask = inputs_embeds == self.get_input_embeddings()( + torch.tensor(self.config.image_token_id, dtype=torch.long, device=inputs_embeds.device) + ) + special_image_mask = special_image_mask.all(-1) + special_video_mask = inputs_embeds == self.get_input_embeddings()( + torch.tensor(self.config.video_token_id, dtype=torch.long, device=inputs_embeds.device) + ) + special_video_mask = special_video_mask.all(-1) + else: + # GLM-4.1V and GLM-4.5V special_video_mask is special_image_mask + special_image_mask = input_ids == self.config.image_token_id + special_video_mask = input_ids == self.config.image_token_id + + n_image_tokens = special_image_mask.sum() + special_image_mask = special_image_mask.unsqueeze(-1).expand_as(inputs_embeds).to(inputs_embeds.device) + if image_features is not None and inputs_embeds[special_image_mask].numel() != image_features.numel(): + raise ValueError( + f"Image features and image tokens do not match: tokens: {n_image_tokens}, features {image_features.shape[0]}" + ) - @property - def visual(self): - return self.model.visual + n_video_tokens = special_video_mask.sum() + special_video_mask = special_video_mask.unsqueeze(-1).expand_as(inputs_embeds).to(inputs_embeds.device) + if video_features is not None and inputs_embeds[special_video_mask].numel() != video_features.numel(): + raise ValueError( + f"Videos features and video tokens do not match: tokens: {n_video_tokens}, features {video_features.shape[0]}" + ) + + return special_image_mask, special_video_mask - @can_return_tuple @auto_docstring + @can_return_tuple def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -980,64 +1056,88 @@ def forward( position_ids: Optional[torch.LongTensor] = None, past_key_values: Optional[Cache] = None, inputs_embeds: Optional[torch.FloatTensor] = None, - labels: Optional[torch.LongTensor] = None, pixel_values: Optional[torch.Tensor] = None, pixel_values_videos: Optional[torch.FloatTensor] = None, image_grid_thw: Optional[torch.LongTensor] = None, video_grid_thw: Optional[torch.LongTensor] = None, rope_deltas: Optional[torch.LongTensor] = None, cache_position: Optional[torch.LongTensor] = None, - logits_to_keep: Union[int, torch.Tensor] = 0, **kwargs: Unpack[TransformersKwargs], - ) -> Union[tuple, Glm46VCausalLMOutputWithPast]: + ) -> Union[tuple, Glm46VModelOutputWithPast]: r""" - rope_deltas (`torch.LongTensor` of shape `(batch_size, )`, *optional*): - The rope index difference between sequence length and multimodal rope. - labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., - config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored - (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. image_grid_thw (`torch.LongTensor` of shape `(num_images, 3)`, *optional*): The temporal, height and width of feature shape of each image in LLM. video_grid_thw (`torch.LongTensor` of shape `(num_videos, 3)`, *optional*): The temporal, height and width of feature shape of each video in LLM. + rope_deltas (`torch.LongTensor` of shape `(batch_size, )`, *optional*): + The rope index difference between sequence length and multimodal rope. + """ + if (input_ids is None) ^ (inputs_embeds is not None): + raise ValueError("You must specify exactly one of input_ids or inputs_embeds") + + if inputs_embeds is None: + inputs_embeds = self.get_input_embeddings()(input_ids) + + if pixel_values is not None: + image_embeds = self.get_image_features(pixel_values, image_grid_thw) + image_embeds = torch.cat(image_embeds, dim=0).to(inputs_embeds.device, inputs_embeds.dtype) + image_mask, _ = self.get_placeholder_mask(input_ids, inputs_embeds, image_features=image_embeds) + inputs_embeds = inputs_embeds.masked_scatter(image_mask, image_embeds) - Example: - - ```python - >>> from PIL import Image - >>> import requests - >>> from transformers import AutoProcessor, Glm46VForConditionalGeneration - - >>> model = Glm46VForConditionalGeneration.from_pretrained("THUDM/GLM-4.1V-9B-Thinking") - >>> processor = AutoProcessor.from_pretrained("THUDM/GLM-4.1V-9B-Thinking") - - >>> messages = [ - { - "role": "user", - "content": [ - {"type": "image"}, - {"type": "text", "text": "What is shown in this image?"}, - ], - }, - ] - >>> url = "https://www.ilankelman.org/stopsigns/australia.jpg" - >>> image = Image.open(requests.get(url, stream=True).raw) - - >>> text = processor.apply_chat_template(messages, tokenize=False, add_generation_prompt=True) - >>> inputs = processor(text=[text], images=[image], vision_infos=[vision_infos]) - - >>> # Generate - >>> generate_ids = model.generate(inputs.input_ids, max_length=30) - >>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0] - "The image shows a street scene with a red stop sign in the foreground. In the background, there is a large red gate with Chinese characters ..." - ```""" - outputs = self.model( - input_ids=input_ids, - pixel_values=pixel_values, - pixel_values_videos=pixel_values_videos, - image_grid_thw=image_grid_thw, - video_grid_thw=video_grid_thw, + if pixel_values_videos is not None: + video_embeds = self.get_video_features(pixel_values_videos, video_grid_thw) + video_embeds = torch.cat(video_embeds, dim=0).to(inputs_embeds.device, inputs_embeds.dtype) + _, video_mask = self.get_placeholder_mask(input_ids, inputs_embeds, video_features=video_embeds) + inputs_embeds = inputs_embeds.masked_scatter(video_mask, video_embeds) + + if position_ids is None: + attention_mask_tensor = ( + attention_mask if not isinstance(attention_mask, dict) else attention_mask["full_attention"] + ) + if attention_mask_tensor is not None and attention_mask_tensor.ndim == 4: + attention_mask_tensor = torch.diagonal(attention_mask_tensor[:, 0], dim1=1, dim2=2) + # Only apply conversion for floating point tensors (inverted masks) + if attention_mask_tensor.dtype.is_floating_point: + attention_mask_tensor = attention_mask_tensor / torch.finfo(attention_mask_tensor.dtype).min + attention_mask_tensor = (1.0 - attention_mask_tensor).int() + + # Calculate RoPE index once per generation in the pre-fill stage only. + # When compiling, we can't check tensor values thus we check only input length + # It is safe to assume that `length!=1` means we're in pre-fill because compiled + # models currently cannot do asssisted decoding + prefill_compiled_stage = is_torchdynamo_compiling() and ( + (input_ids is not None and input_ids.shape[1] != 1) + or (inputs_embeds is not None and inputs_embeds.shape[1] != 1) + ) + prefill_noncompiled_stage = not is_torchdynamo_compiling() and ( + (cache_position is not None and cache_position[0] == 0) + or (past_key_values is None or past_key_values.get_seq_length() == 0) + ) + if (prefill_compiled_stage or prefill_noncompiled_stage) or self.rope_deltas is None: + position_ids, rope_deltas = self.get_rope_index( + input_ids, + image_grid_thw, + video_grid_thw, + attention_mask=attention_mask_tensor, + ) + self.rope_deltas = rope_deltas + # then use the prev pre-calculated rope-deltas to get the correct position ids + else: + batch_size, seq_length, _ = inputs_embeds.shape + delta = ( + (cache_position[0] + self.rope_deltas).to(inputs_embeds.device) + if cache_position is not None + else 0 + ) + position_ids = torch.arange(seq_length, device=inputs_embeds.device) + position_ids = position_ids.view(1, -1).expand(batch_size, -1) + if cache_position is not None: # otherwise `deltas` is an int `0` + delta = delta.repeat_interleave(batch_size // delta.shape[0], dim=0) + position_ids = position_ids.add(delta) + position_ids = position_ids.unsqueeze(0).expand(3, -1, -1) + + outputs = self.language_model( + input_ids=None, position_ids=position_ids, attention_mask=attention_mask, past_key_values=past_key_values, @@ -1046,218 +1146,13 @@ def forward( **kwargs, ) - hidden_states = outputs[0] - - # Only compute necessary logits, and do not upcast them to float if we are not computing the loss - slice_indices = slice(-logits_to_keep, None) if isinstance(logits_to_keep, int) else logits_to_keep - logits = self.lm_head(hidden_states[:, slice_indices, :]) - - loss = None - if labels is not None: - loss = self.loss_function(logits=logits, labels=labels, vocab_size=self.config.text_config.vocab_size) - - return Glm46VCausalLMOutputWithPast( - loss=loss, - logits=logits, + return Glm46VModelOutputWithPast( + last_hidden_state=outputs.last_hidden_state, past_key_values=outputs.past_key_values, hidden_states=outputs.hidden_states, attentions=outputs.attentions, - rope_deltas=outputs.rope_deltas, + rope_deltas=self.rope_deltas, ) - def prepare_inputs_for_generation( - self, - input_ids, - past_key_values=None, - attention_mask=None, - inputs_embeds=None, - cache_position=None, - position_ids=None, - use_cache=True, - pixel_values=None, - pixel_values_videos=None, - image_grid_thw=None, - video_grid_thw=None, - **kwargs, - ): - # Overwritten -- in specific circumstances we don't want to forward image inputs to the model - - model_inputs = super().prepare_inputs_for_generation( - input_ids, - past_key_values=past_key_values, - attention_mask=attention_mask, - inputs_embeds=inputs_embeds, - cache_position=cache_position, - position_ids=position_ids, - pixel_values=pixel_values, - pixel_values_videos=pixel_values_videos, - image_grid_thw=image_grid_thw, - video_grid_thw=video_grid_thw, - use_cache=use_cache, - **kwargs, - ) - - # GLM-4.1V position_ids are prepareed with rope_deltas in forward - model_inputs["position_ids"] = None - - if cache_position[0] != 0: - model_inputs["pixel_values"] = None - model_inputs["pixel_values_videos"] = None - - return model_inputs - - def _get_image_nums_and_video_nums( - self, - input_ids: Optional[torch.LongTensor], - inputs_embeds: Optional[torch.Tensor] = None, - ) -> tuple[torch.Tensor, torch.Tensor]: - """ - Get the number of images and videos for each sample to calculate the separation length of the sample tensor. - These parameters are not passed through the processor to avoid unpredictable impacts from interface modifications. - - Args: - input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): - Indices of input sequence tokens in the vocabulary. - - Returns: - image_nums (`torch.LongTensor` of shape `(batch_size, num_images_sample)`) - video_nums (`torch.LongTensor` of shape `(batch_size, num_videos_sample)`) - """ - - if inputs_embeds is not None: - is_image = ( - inputs_embeds - == self.get_input_embeddings()( - torch.tensor(self.config.image_start_token_id, dtype=torch.long, device=inputs_embeds.device) - ) - )[..., 0] - is_video_start = ( - inputs_embeds - == self.get_input_embeddings()( - torch.tensor(self.config.video_start_token_id, dtype=torch.long, device=inputs_embeds.device) - ) - )[..., 0] - is_video_end = ( - inputs_embeds - == self.get_input_embeddings()( - torch.tensor(self.config.video_end_token_id, dtype=torch.long, device=inputs_embeds.device) - ) - )[..., 0] - else: - is_image = input_ids == self.config.image_start_token_id - is_video_start = input_ids == self.config.video_start_token_id - is_video_end = input_ids == self.config.video_end_token_id - - # Cumulative sum to track if we're inside a video span - # We'll assume well-formed video tags (i.e. matching starts and ends) - video_level = torch.cumsum(is_video_start.int() - is_video_end.int(), dim=1) - inside_video = video_level > 0 # shape (batch_size, seq_length) - - # Mask out image tokens that are inside video spans - standalone_images = is_image & (~inside_video) - - # Count per batch - image_counts = standalone_images.sum(dim=1) - video_counts = is_video_start.sum(dim=1) - - return image_counts, video_counts - - def _expand_inputs_for_generation( - self, - expand_size: int = 1, - is_encoder_decoder: bool = False, - input_ids: Optional[torch.LongTensor] = None, - **model_kwargs, - ) -> tuple[torch.LongTensor, dict[str, Any]]: - # Overwritten -- Support for expanding tensors without a batch size dimension - # e.g., pixel_values, image_grid_thw, pixel_values_videos, video_grid_thw, second_per_grid_t - # pixel_values.shape[0] is sum(seqlen_images for samples) - # image_grid_thw.shape[0] is sum(num_images for samples) - - if expand_size == 1: - return input_ids, model_kwargs - - visual_keys = ["pixel_values", "image_grid_thw", "pixel_values_videos", "video_grid_thw", "second_per_grid_ts"] - - def _expand_dict_for_generation_visual(dict_to_expand): - image_grid_thw = model_kwargs.get("image_grid_thw", None) - video_grid_thw = model_kwargs.get("video_grid_thw", None) - image_nums, video_nums = self._get_image_nums_and_video_nums( - input_ids, inputs_embeds=model_kwargs.get("inputs_embeds", None) - ) - - def _repeat_interleave_samples(x, lengths, repeat_times): - samples = torch.split(x, lengths) - repeat_args = [repeat_times] + [1] * (x.dim() - 1) - result = torch.cat([sample.repeat(*repeat_args) for sample in samples], dim=0) - return result - - for key in dict_to_expand: - if key == "pixel_values": - # split images into samples - samples = torch.split(image_grid_thw, list(image_nums)) - # compute the sequence length of images for each sample - lengths = [torch.prod(sample, dim=1).sum() for sample in samples] - dict_to_expand[key] = _repeat_interleave_samples( - dict_to_expand[key], lengths=lengths, repeat_times=expand_size - ) - elif key == "image_grid_thw": - # get the num of images for each sample - lengths = list(image_nums) - dict_to_expand[key] = _repeat_interleave_samples( - dict_to_expand[key], lengths=lengths, repeat_times=expand_size - ) - elif key == "pixel_values_videos": - samples = torch.split(video_grid_thw, list(video_nums)) - lengths = [torch.prod(sample, dim=1).sum() for sample in samples] - dict_to_expand[key] = _repeat_interleave_samples( - dict_to_expand[key], lengths=lengths, repeat_times=expand_size - ) - elif key == "video_grid_thw": - lengths = list(video_nums) - dict_to_expand[key] = _repeat_interleave_samples( - dict_to_expand[key], lengths=lengths, repeat_times=expand_size - ) - elif key == "second_per_grid_ts": - dict_to_expand[key] = _repeat_interleave_samples( - dict_to_expand[key], lengths=list(video_nums), repeat_times=expand_size - ) - return dict_to_expand - - def _expand_dict_for_generation(dict_to_expand): - for key in dict_to_expand: - if ( - key != "cache_position" - and dict_to_expand[key] is not None - and isinstance(dict_to_expand[key], torch.Tensor) - and key not in visual_keys - ): - dict_to_expand[key] = dict_to_expand[key].repeat_interleave(expand_size, dim=0) - return dict_to_expand - - model_kwargs = _expand_dict_for_generation_visual(model_kwargs) - - if input_ids is not None: - input_ids = input_ids.repeat_interleave(expand_size, dim=0) - - model_kwargs = _expand_dict_for_generation(model_kwargs) - - if is_encoder_decoder: - if model_kwargs.get("encoder_outputs") is None: - raise ValueError("If `is_encoder_decoder` is True, make sure that `encoder_outputs` is defined.") - model_kwargs["encoder_outputs"] = _expand_dict_for_generation(model_kwargs["encoder_outputs"]) - - return input_ids, model_kwargs - - -class Glm46VModel(Glm46VPreTrainedModel): - pass - -__all__ = [ - "Glm46VModel", - "Glm46VForConditionalGeneration", - "Glm46VPreTrainedModel", - "Glm46VVisionModel", - "Glm46VTextModel", -] +__all__ = ["Glm46VModel", "Glm46VPreTrainedModel", "Glm46VVisionModel"] diff --git a/src/transformers/models/glm46v/modular_glm46v.py b/src/transformers/models/glm46v/modular_glm46v.py index 3a557843c50f..b669de86b4fb 100644 --- a/src/transformers/models/glm46v/modular_glm46v.py +++ b/src/transformers/models/glm46v/modular_glm46v.py @@ -18,13 +18,13 @@ import numpy as np from ...video_utils import VideoMetadata +from ..auto.modeling_auto import AutoModel from ..glm4v.configuration_glm4v import Glm4vConfig, Glm4vTextConfig from ..glm4v.image_processing_glm4v import Glm4vImageProcessor from ..glm4v.image_processing_glm4v_fast import Glm4vImageProcessorFast from ..glm4v.modeling_glm4v import ( - Glm4vForConditionalGeneration, + Glm4vModel, Glm4vPreTrainedModel, - Glm4vTextModel, Glm4vVisionModel, ) from ..glm4v.processing_glm4v import Glm4vProcessor @@ -47,16 +47,11 @@ class Glm46VVisionModel(Glm4vVisionModel): pass -class Glm46VTextModel(Glm4vTextModel): - pass - - -class Glm46VForConditionalGeneration(Glm4vForConditionalGeneration): - pass - - -class Glm46VModel(Glm46VPreTrainedModel): - pass +class Glm46VModel(Glm4vModel): + def __init__(self, config: Glm46VConfig): + super().__init__(config) + self.visual = AutoModel.from_config(config.vision_config) + self.language_model = AutoModel.from_config(config.text_config) class Glm46VProcessor(Glm4vProcessor): @@ -144,10 +139,8 @@ def sample_frames( "Glm46VConfig", "Glm46VTextConfig", "Glm46VModel", - "Glm46VForConditionalGeneration", "Glm46VPreTrainedModel", "Glm46VVisionModel", - "Glm46VTextModel", "Glm46VProcessor", "Glm46VImageProcessor", "Glm46VImageProcessorFast", From 3ceff09316c5b87688c004f07047c6b26e751ec2 Mon Sep 17 00:00:00 2001 From: zRzRzRzRzRzRzR <2448370773@qq.com> Date: Thu, 13 Nov 2025 00:32:51 +0800 Subject: [PATCH 35/66] 1 --- docs/source/en/model_doc/glm46v.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/source/en/model_doc/glm46v.md b/docs/source/en/model_doc/glm46v.md index a8032634fcd8..a09a8538adbf 100644 --- a/docs/source/en/model_doc/glm46v.md +++ b/docs/source/en/model_doc/glm46v.md @@ -27,6 +27,10 @@ [[autodoc]] Glm46VProcessor +## Glm46VVisionModel + +[[autodoc]] Glm46VVisionModel + ## Glm46VTextModel [[autodoc]] Glm46VTextModel From 63dafb1f43500e717122417a0e4440273382b82d Mon Sep 17 00:00:00 2001 From: zRzRzRzRzRzRzR <2448370773@qq.com> Date: Thu, 13 Nov 2025 00:56:51 +0800 Subject: [PATCH 36/66] try remove --- docs/source/en/model_doc/glm46v.md | 13 - src/transformers/models/auto/modeling_auto.py | 2 - .../models/glm46v/modeling_glm46v.py | 400 +----------------- .../models/glm46v/modular_glm46v.py | 11 +- utils/check_repo.py | 2 - 5 files changed, 3 insertions(+), 425 deletions(-) diff --git a/docs/source/en/model_doc/glm46v.md b/docs/source/en/model_doc/glm46v.md index a09a8538adbf..c6fd6873f2ca 100644 --- a/docs/source/en/model_doc/glm46v.md +++ b/docs/source/en/model_doc/glm46v.md @@ -27,21 +27,8 @@ [[autodoc]] Glm46VProcessor -## Glm46VVisionModel - -[[autodoc]] Glm46VVisionModel - -## Glm46VTextModel - -[[autodoc]] Glm46VTextModel - - forward - ## Glm46VModel [[autodoc]] Glm46VModel - forward -## Glm46VForConditionalGeneration - -[[autodoc]] Glm46VForConditionalGeneration - - forward diff --git a/src/transformers/models/auto/modeling_auto.py b/src/transformers/models/auto/modeling_auto.py index 2e3af16af509..6a0800fbe435 100644 --- a/src/transformers/models/auto/modeling_auto.py +++ b/src/transformers/models/auto/modeling_auto.py @@ -176,7 +176,6 @@ class _BaseModelWithGenerate(PreTrainedModel, GenerationMixin): ("glm", "GlmModel"), ("glm4", "Glm4Model"), ("glm46v", "Glm46VModel"), - ("glm46v_text", "Glm46VTextModel"), ("glm4_moe", "Glm4MoeModel"), ("glm4v", "Glm4vModel"), ("glm4v_moe", "Glm4vMoeModel"), @@ -1034,7 +1033,6 @@ class _BaseModelWithGenerate(PreTrainedModel, GenerationMixin): ("gemma3", "Gemma3ForConditionalGeneration"), ("gemma3n", "Gemma3nForConditionalGeneration"), ("git", "GitForCausalLM"), - ("glm46v", "Glm46VForConditionalGeneration"), ("glm4v", "Glm4vForConditionalGeneration"), ("glm4v_moe", "Glm4vMoeForConditionalGeneration"), ("got_ocr2", "GotOcr2ForConditionalGeneration"), diff --git a/src/transformers/models/glm46v/modeling_glm46v.py b/src/transformers/models/glm46v/modeling_glm46v.py index 4096b4c7bb22..8c1fe0621ce4 100644 --- a/src/transformers/models/glm46v/modeling_glm46v.py +++ b/src/transformers/models/glm46v/modeling_glm46v.py @@ -26,8 +26,6 @@ import torch import torch.nn as nn -from torch.nn import LayerNorm -from torchvision.transforms.v2 import functional as F from ...activations import ACT2FN from ...cache_utils import Cache @@ -39,7 +37,7 @@ from ...processing_utils import Unpack from ...utils import TransformersKwargs, auto_docstring, can_return_tuple, is_torchdynamo_compiling from ..auto.modeling_auto import AutoModel -from .configuration_glm46v import Glm46VConfig, Glm46VTextConfig, Glm46VVisionConfig +from .configuration_glm46v import Glm46VConfig, Glm46VTextConfig @use_kernel_forward_from_hub("RMSNorm") @@ -328,400 +326,6 @@ class Glm46VPreTrainedModel(PreTrainedModel): } -class Glm46VisionMlp(nn.Module): - def __init__(self, config, bias: bool = False): - super().__init__() - self.hidden_size = config.hidden_size - self.intermediate_size = config.out_hidden_size - self.gate_proj = nn.Linear(self.hidden_size, self.intermediate_size, bias=bias) - self.up_proj = nn.Linear(self.hidden_size, self.intermediate_size, bias=bias) - self.down_proj = nn.Linear(self.intermediate_size, self.hidden_size, bias=bias) - self.act_fn = ACT2FN[config.hidden_act] - - def forward(self, hidden_state): - return self.down_proj(self.act_fn(self.gate_proj(hidden_state)) * self.up_proj(hidden_state)) - - -class Glm46VVisionPatchEmbed(nn.Module): - def __init__(self, config: Glm46VVisionConfig) -> None: - super().__init__() - self.patch_size = config.patch_size - self.temporal_patch_size = config.temporal_patch_size - self.in_channels = config.in_channels - self.embed_dim = config.hidden_size - - kernel_size = [self.temporal_patch_size, self.patch_size, self.patch_size] - self.proj = nn.Conv3d(self.in_channels, self.embed_dim, kernel_size=kernel_size, stride=kernel_size) - - def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: - target_dtype = self.proj.weight.dtype - hidden_states = hidden_states.view( - -1, self.in_channels, self.temporal_patch_size, self.patch_size, self.patch_size - ) - hidden_states = self.proj(hidden_states.to(dtype=target_dtype)).view(-1, self.embed_dim) - return hidden_states - - -class Glm46VVisionRotaryEmbedding(nn.Module): - inv_freq: torch.Tensor # fix linting for `register_buffer` - - def __init__(self, dim: int, theta: float = 10000.0) -> None: - super().__init__() - inv_freq = 1.0 / (theta ** (torch.arange(0, dim, 2, dtype=torch.float) / dim)) - self.register_buffer("inv_freq", inv_freq, persistent=False) - - def forward(self, seqlen: int) -> torch.Tensor: - seq = torch.arange(seqlen, device=self.inv_freq.device, dtype=self.inv_freq.dtype) - freqs = torch.outer(seq, self.inv_freq) - return freqs - - -class Glm46VVisionPatchMerger(nn.Module): - def __init__(self, dim: int, context_dim: int, hidden_act: str, bias: bool = False) -> None: - super().__init__() - self.proj = nn.Linear(dim, dim, bias=bias) - self.post_projection_norm = LayerNorm(dim) - self.gate_proj = nn.Linear(dim, context_dim, bias=bias) - self.up_proj = nn.Linear(dim, context_dim, bias=bias) - self.down_proj = nn.Linear(context_dim, dim, bias=bias) - self.act1 = nn.GELU() - self.act_fn = ACT2FN[hidden_act] - - def forward(self, hidden_state: torch.Tensor) -> torch.Tensor: - hidden_state = self.proj(hidden_state) - hidden_state = self.act1(self.post_projection_norm(hidden_state)) - return self.down_proj(self.act_fn(self.gate_proj(hidden_state)) * self.up_proj(hidden_state)) - - -class Glm46VVisionEmbeddings(nn.Module): - def __init__(self, config: Glm46VVisionConfig): - super().__init__() - self.config = config - self.embed_dim = config.hidden_size - self.image_size = config.image_size - self.patch_size = config.patch_size - - self.num_patches = (self.image_size // self.patch_size) ** 2 - self.num_positions = self.num_patches - self.position_embedding = nn.Embedding(self.num_positions, self.embed_dim) - self.register_buffer("position_ids", torch.arange(self.num_positions).expand((1, -1)), persistent=False) - - def forward(self, embeddings, lengths, image_shapes, h_coords, w_coords) -> torch.Tensor: - """ - Forward pass with integrated position encoding adaptation using 2D interpolation. - - Args: - embeddings: Input embeddings tensor - lengths (torch.Tensor): Sequence lengths for each image in the batch. - image_shapes (torch.Tensor): Tensor of shape [batch_size, 3] representing the image shapes (t, h, w). - h_coords (torch.Tensor): Tensor of shape [total_seq] representing the h coordinate for each patch. - w_coords (torch.Tensor): Tensor of shape [total_seq] representing the w coordinate for each patch. - - Returns: - torch.Tensor: Embeddings with adapted position encoding added. - """ - # Get position embedding parameters - pos_embed_weight = self.position_embedding.weight - hidden_size = pos_embed_weight.shape[1] - total_seq = h_coords.shape[0] - device = pos_embed_weight.device - - # Move coordinates to correct device - h_coords, w_coords = h_coords.to(device), w_coords.to(device) - - # Handle empty sequence case - if total_seq == 0: - adapted_pos_embed = torch.empty(0, hidden_size, device=device, dtype=pos_embed_weight.dtype) - else: - # Convert inputs to tensors if needed - if isinstance(lengths, list): - lengths = torch.tensor(lengths, device=device, dtype=torch.long) - if not isinstance(image_shapes, torch.Tensor): - image_shapes = torch.tensor(image_shapes, device=device, dtype=torch.long) - - # Prepare 2D position embedding - orig_size_sq = pos_embed_weight.shape[0] - orig_size = int(orig_size_sq**0.5) - pos_embed_2d = ( - pos_embed_weight.view(orig_size, orig_size, hidden_size) - .permute(2, 0, 1) - .unsqueeze(0) - .to(device=device, dtype=torch.float32) - ) - - # Calculate target dimensions for each patch - target_h = torch.cat([image_shapes[i, 1].repeat(lengths[i]) for i in range(len(lengths))]).to( - device=device, dtype=torch.float32 - ) - target_w = torch.cat([image_shapes[i, 2].repeat(lengths[i]) for i in range(len(lengths))]).to( - device=device, dtype=torch.float32 - ) - - # Normalize coordinates to [-1, 1] range for grid_sample - h_coords = h_coords.to(device=device, dtype=torch.float32) - w_coords = w_coords.to(device=device, dtype=torch.float32) - norm_w = ((w_coords + 0.5) / target_w) * 2 - 1 - norm_h = ((h_coords + 0.5) / target_h) * 2 - 1 - - # Create sampling grid - grid = torch.stack((norm_w, norm_h), dim=-1).unsqueeze(0).unsqueeze(2) - - # Perform bicubic interpolation - interpolated_embed_fp32 = F.grid_sample( - pos_embed_2d, grid, mode="bicubic", align_corners=False, padding_mode="border" - ) - - # Reshape and convert back to original dtype - adapted_pos_embed_fp32 = interpolated_embed_fp32.squeeze(0).squeeze(-1).permute(1, 0) - adapted_pos_embed = adapted_pos_embed_fp32.to(pos_embed_weight.dtype).to(embeddings.device) - - # Add adapted position encoding to embeddings - embeddings = embeddings + adapted_pos_embed - return embeddings - - -def rotate_half(x): - """Rotates half the hidden dims of the input.""" - x1 = x[..., : x.shape[-1] // 2] - x2 = x[..., x.shape[-1] // 2 :] - return torch.cat((-x2, x1), dim=-1) - - -def apply_rotary_pos_emb_vision( - q: torch.Tensor, k: torch.Tensor, cos: torch.Tensor, sin: torch.Tensor -) -> tuple[torch.Tensor, torch.Tensor]: - orig_q_dtype = q.dtype - orig_k_dtype = k.dtype - q, k = q.float(), k.float() - cos, sin = cos.unsqueeze(-2).float(), sin.unsqueeze(-2).float() - q_embed = (q * cos) + (rotate_half(q) * sin) - k_embed = (k * cos) + (rotate_half(k) * sin) - q_embed = q_embed.to(orig_q_dtype) - k_embed = k_embed.to(orig_k_dtype) - return q_embed, k_embed - - -class Glm46VVisionAttention(nn.Module): - def __init__(self, config: Glm46VVisionConfig) -> None: - super().__init__() - self.dim = config.hidden_size - self.num_heads = config.num_heads - self.head_dim = self.dim // self.num_heads - self.num_key_value_groups = 1 # needed for eager attention - self.qkv = nn.Linear(config.hidden_size, config.hidden_size * 3, bias=config.attention_bias) - self.proj = nn.Linear(config.hidden_size, config.hidden_size, bias=False) - self.scaling = self.head_dim**-0.5 - self.config = config - self.attention_dropout = config.attention_dropout - self.is_causal = False - - def forward( - self, - hidden_states: torch.Tensor, - cu_seqlens: torch.Tensor, - rotary_pos_emb: Optional[torch.Tensor] = None, - position_embeddings: Optional[tuple[torch.Tensor, torch.Tensor]] = None, - **kwargs, - ) -> torch.Tensor: - seq_length = hidden_states.shape[0] - query_states, key_states, value_states = ( - self.qkv(hidden_states).reshape(seq_length, 3, self.num_heads, -1).permute(1, 0, 2, 3).unbind(0) - ) - cos, sin = position_embeddings - query_states, key_states = apply_rotary_pos_emb_vision(query_states, key_states, cos, sin) - - query_states = query_states.transpose(0, 1).unsqueeze(0) - key_states = key_states.transpose(0, 1).unsqueeze(0) - value_states = value_states.transpose(0, 1).unsqueeze(0) - - attention_interface: Callable = eager_attention_forward - if self.config._attn_implementation != "eager": - attention_interface = ALL_ATTENTION_FUNCTIONS[self.config._attn_implementation] - - if self.config._attn_implementation == "flash_attention_2": - # Flash Attention 2: Use cu_seqlens for variable length attention - max_seqlen = (cu_seqlens[1:] - cu_seqlens[:-1]).max() - attn_output, _ = attention_interface( - self, - query_states, - key_states, - value_states, - attention_mask=None, - scaling=self.scaling, - dropout=0.0 if not self.training else self.attention_dropout, - cu_seq_lens_q=cu_seqlens, - cu_seq_lens_k=cu_seqlens, - max_length_q=max_seqlen, - max_length_k=max_seqlen, - is_causal=False, - **kwargs, - ) - else: - # Other implementations: Process each chunk separately - lengths = cu_seqlens[1:] - cu_seqlens[:-1] - splits = [ - torch.split(tensor, lengths.tolist(), dim=2) for tensor in (query_states, key_states, value_states) - ] - - attn_outputs = [ - attention_interface( - self, - q, - k, - v, - attention_mask=None, - scaling=self.scaling, - dropout=0.0 if not self.training else self.attention_dropout, - is_causal=False, - **kwargs, - )[0] - for q, k, v in zip(*splits) - ] - attn_output = torch.cat(attn_outputs, dim=1) - - attn_output = attn_output.reshape(seq_length, -1).contiguous() - attn_output = self.proj(attn_output) - return attn_output - - -class Glm46VVisionBlock(GradientCheckpointingLayer): - def __init__(self, config) -> None: - super().__init__() - self.norm1 = Glm46VRMSNorm(config.hidden_size, eps=config.rms_norm_eps) - self.norm2 = Glm46VRMSNorm(config.hidden_size, eps=config.rms_norm_eps) - self.attn = Glm46VVisionAttention(config) - self.mlp = Glm46VisionMlp(config, bias=False) - - def forward( - self, - hidden_states: torch.Tensor, - cu_seqlens: torch.Tensor, - rotary_pos_emb: Optional[torch.Tensor] = None, - position_embeddings: Optional[tuple[torch.Tensor, torch.Tensor]] = None, - **kwargs, - ) -> torch.Tensor: - hidden_states = hidden_states + self.attn( - self.norm1(hidden_states), - cu_seqlens=cu_seqlens, - rotary_pos_emb=rotary_pos_emb, - position_embeddings=position_embeddings, - **kwargs, - ) - hidden_states = hidden_states + self.mlp(self.norm2(hidden_states)) - return hidden_states - - -class Glm46VVisionModel(Glm46VPreTrainedModel): - config: Glm46VVisionConfig - input_modalities = ["image", "video"] - _no_split_modules = ["Glm46VVisionBlock"] - - def __init__(self, config) -> None: - super().__init__(config) - self.spatial_merge_size = config.spatial_merge_size - self.patch_size = config.patch_size - - self.embeddings = Glm46VVisionEmbeddings(config) - self.patch_embed = Glm46VVisionPatchEmbed(config) - - head_dim = config.hidden_size // config.num_heads - self.rotary_pos_emb = Glm46VVisionRotaryEmbedding(head_dim // 2) - - self.blocks = nn.ModuleList([Glm46VVisionBlock(config) for _ in range(config.depth)]) - self.merger = Glm46VVisionPatchMerger( - dim=config.out_hidden_size, context_dim=config.intermediate_size, hidden_act=config.hidden_act - ) - - self.post_conv_layernorm = Glm46VRMSNorm(config.hidden_size, eps=config.rms_norm_eps) - self.downsample = nn.Conv2d( - in_channels=config.hidden_size, - out_channels=config.out_hidden_size, - kernel_size=config.spatial_merge_size, - stride=config.spatial_merge_size, - ) - self.post_layernorm = Glm46VRMSNorm(config.hidden_size, eps=config.rms_norm_eps) - - self.gradient_checkpointing = False - self.post_init() - - def rot_pos_emb(self, grid_thw): - pos_ids = [] - for t, h, w in grid_thw: - hpos_ids = torch.arange(h).unsqueeze(1).expand(-1, w) - hpos_ids = hpos_ids.reshape( - h // self.spatial_merge_size, - self.spatial_merge_size, - w // self.spatial_merge_size, - self.spatial_merge_size, - ) - hpos_ids = hpos_ids.permute(0, 2, 1, 3) - hpos_ids = hpos_ids.flatten() - - wpos_ids = torch.arange(w).unsqueeze(0).expand(h, -1) - wpos_ids = wpos_ids.reshape( - h // self.spatial_merge_size, - self.spatial_merge_size, - w // self.spatial_merge_size, - self.spatial_merge_size, - ) - wpos_ids = wpos_ids.permute(0, 2, 1, 3) - wpos_ids = wpos_ids.flatten() - pos_ids.append(torch.stack([hpos_ids, wpos_ids], dim=-1).repeat(t, 1)) - pos_ids = torch.cat(pos_ids, dim=0) - max_grid_size = grid_thw[:, 1:].max() - rotary_pos_emb_full = self.rotary_pos_emb(max_grid_size) - rotary_pos_emb = rotary_pos_emb_full[pos_ids].flatten(1) - return rotary_pos_emb, pos_ids - - def forward(self, hidden_states: torch.Tensor, grid_thw: torch.Tensor) -> torch.Tensor: - """ - Args: - hidden_states (`torch.Tensor` of shape `(seq_len, hidden_size)`): - The final hidden states of the model. - grid_thw (`torch.Tensor` of shape `(num_images_or_videos, 3)`): - The temporal, height and width of feature shape of each image in LLM. - - Returns: - `torch.Tensor`: hidden_states. - """ - hidden_states = self.patch_embed(hidden_states) - hidden_states = self.post_conv_layernorm(hidden_states) - - rotary_pos_emb, image_type_ids = self.rot_pos_emb(grid_thw) - emb = torch.cat((rotary_pos_emb, rotary_pos_emb), dim=-1) - position_embeddings = (emb.cos(), emb.sin()) - - cu_seqlens = torch.repeat_interleave(grid_thw[:, 1] * grid_thw[:, 2], grid_thw[:, 0]).cumsum( - dim=0, - # Select dtype based on the following factors: - # - FA2 requires that cu_seqlens_q must have dtype int32 - # - torch.onnx.export requires that cu_seqlens_q must have same dtype as grid_thw - # See https://github.com/huggingface/transformers/pull/34852 for more information - dtype=grid_thw.dtype if torch.jit.is_tracing() else torch.int32, - ) - cu_seqlens = F.pad(cu_seqlens, (1, 0), value=0) - seqlens = (cu_seqlens[1:] - cu_seqlens[:-1]).tolist() - hidden_states = self.embeddings(hidden_states, seqlens, grid_thw, image_type_ids[:, 0], image_type_ids[:, 1]) - - for blk in self.blocks: - hidden_states = blk( - hidden_states, - cu_seqlens=cu_seqlens, - position_embeddings=position_embeddings, - ) - - hidden_states = self.post_layernorm(hidden_states) - - hidden_states = hidden_states.view( - -1, self.spatial_merge_size, self.spatial_merge_size, hidden_states.shape[-1] - ) - hidden_states = hidden_states.permute(0, 3, 1, 2) - hidden_states = self.downsample(hidden_states).view(-1, self.config.out_hidden_size) - - hidden_states = self.merger(hidden_states) - return hidden_states - - @dataclass @auto_docstring( custom_intro=""" @@ -1155,4 +759,4 @@ def forward( ) -__all__ = ["Glm46VModel", "Glm46VPreTrainedModel", "Glm46VVisionModel"] +__all__ = ["Glm46VModel", "Glm46VPreTrainedModel"] diff --git a/src/transformers/models/glm46v/modular_glm46v.py b/src/transformers/models/glm46v/modular_glm46v.py index b669de86b4fb..fc200a02f610 100644 --- a/src/transformers/models/glm46v/modular_glm46v.py +++ b/src/transformers/models/glm46v/modular_glm46v.py @@ -22,11 +22,7 @@ from ..glm4v.configuration_glm4v import Glm4vConfig, Glm4vTextConfig from ..glm4v.image_processing_glm4v import Glm4vImageProcessor from ..glm4v.image_processing_glm4v_fast import Glm4vImageProcessorFast -from ..glm4v.modeling_glm4v import ( - Glm4vModel, - Glm4vPreTrainedModel, - Glm4vVisionModel, -) +from ..glm4v.modeling_glm4v import Glm4vModel, Glm4vPreTrainedModel from ..glm4v.processing_glm4v import Glm4vProcessor from ..glm4v.video_processing_glm4v import Glm4vVideoProcessor @@ -43,10 +39,6 @@ class Glm46VPreTrainedModel(Glm4vPreTrainedModel): pass -class Glm46VVisionModel(Glm4vVisionModel): - pass - - class Glm46VModel(Glm4vModel): def __init__(self, config: Glm46VConfig): super().__init__(config) @@ -140,7 +132,6 @@ def sample_frames( "Glm46VTextConfig", "Glm46VModel", "Glm46VPreTrainedModel", - "Glm46VVisionModel", "Glm46VProcessor", "Glm46VImageProcessor", "Glm46VImageProcessorFast", diff --git a/utils/check_repo.py b/utils/check_repo.py index da81ab8b5601..762b50e12ceb 100644 --- a/utils/check_repo.py +++ b/utils/check_repo.py @@ -98,7 +98,6 @@ "Phi4MultimodalVisionModel", "Glm4vVisionModel", "Glm4vMoeVisionModel", - "Glm46VVisionModel", "EvollaSaProtPreTrainedModel", "BltLocalEncoder", # Building part of bigger (tested) model. Tested implicitly through BLTForCausalLM. "BltLocalDecoder", # Building part of bigger (tested) model. Tested implicitly through BLTForCausalLM. @@ -180,7 +179,6 @@ "Emu3VQVAE", # Building part of bigger (tested) model "Emu3TextModel", # Building part of bigger (tested) model "Glm4vTextModel", # Building part of bigger (tested) model - "Glm46VTextModel", # Building part of bigger (tested) model "Glm4vMoeTextModel", # Building part of bigger (tested) model "Qwen2VLTextModel", # Building part of bigger (tested) model "Qwen2_5_VLTextModel", # Building part of bigger (tested) model From 900c335007a1d1f5145ca718329fc223b6bcfac8 Mon Sep 17 00:00:00 2001 From: zRzRzRzRzRzRzR <2448370773@qq.com> Date: Thu, 13 Nov 2025 01:12:25 +0800 Subject: [PATCH 37/66] delete --- .../models/auto/configuration_auto.py | 2 - .../models/glm46v/modeling_glm46v.py | 291 +----------------- .../models/glm46v/modular_glm46v.py | 20 +- tests/models/glm46v/test_modeling_glm46v.py | 20 +- 4 files changed, 24 insertions(+), 309 deletions(-) diff --git a/src/transformers/models/auto/configuration_auto.py b/src/transformers/models/auto/configuration_auto.py index 9aec777596fd..9bf657444249 100644 --- a/src/transformers/models/auto/configuration_auto.py +++ b/src/transformers/models/auto/configuration_auto.py @@ -172,8 +172,6 @@ ("git", "GitConfig"), ("glm", "GlmConfig"), ("glm4", "Glm4Config"), - ("glm46v", "Glm46VConfig"), - ("glm46v_text", "Glm46VTextConfig"), ("glm4_moe", "Glm4MoeConfig"), ("glm4v", "Glm4vConfig"), ("glm4v_moe", "Glm4vMoeConfig"), diff --git a/src/transformers/models/glm46v/modeling_glm46v.py b/src/transformers/models/glm46v/modeling_glm46v.py index 8c1fe0621ce4..5c5154866fcf 100644 --- a/src/transformers/models/glm46v/modeling_glm46v.py +++ b/src/transformers/models/glm46v/modeling_glm46v.py @@ -20,310 +20,33 @@ # limitations under the License. import itertools -from collections.abc import Callable from dataclasses import dataclass from typing import Optional, Union import torch -import torch.nn as nn -from ...activations import ACT2FN from ...cache_utils import Cache -from ...integrations import use_kernel_forward_from_hub -from ...modeling_flash_attention_utils import FlashAttentionKwargs -from ...modeling_layers import GradientCheckpointingLayer from ...modeling_outputs import ModelOutput -from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel +from ...modeling_utils import PreTrainedModel from ...processing_utils import Unpack from ...utils import TransformersKwargs, auto_docstring, can_return_tuple, is_torchdynamo_compiling from ..auto.modeling_auto import AutoModel -from .configuration_glm46v import Glm46VConfig, Glm46VTextConfig - - -@use_kernel_forward_from_hub("RMSNorm") -class Glm46VRMSNorm(nn.Module): - def __init__(self, hidden_size, eps=1e-6): - """ - Glm46VRMSNorm is equivalent to T5LayerNorm - """ - super().__init__() - self.weight = nn.Parameter(torch.ones(hidden_size)) - self.variance_epsilon = eps - - def forward(self, hidden_states): - input_dtype = hidden_states.dtype - hidden_states = hidden_states.to(torch.float32) - variance = hidden_states.pow(2).mean(-1, keepdim=True) - hidden_states = hidden_states * torch.rsqrt(variance + self.variance_epsilon) - return self.weight * hidden_states.to(input_dtype) - - def extra_repr(self): - return f"{tuple(self.weight.shape)}, eps={self.variance_epsilon}" - - -def repeat_kv(hidden_states: torch.Tensor, n_rep: int) -> torch.Tensor: - """ - This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch, - num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim) - """ - batch, num_key_value_heads, slen, head_dim = hidden_states.shape - if n_rep == 1: - return hidden_states - hidden_states = hidden_states[:, :, None, :, :].expand(batch, num_key_value_heads, n_rep, slen, head_dim) - return hidden_states.reshape(batch, num_key_value_heads * n_rep, slen, head_dim) - - -def eager_attention_forward( - module: nn.Module, - query: torch.Tensor, - key: torch.Tensor, - value: torch.Tensor, - attention_mask: Optional[torch.Tensor], - scaling: float, - dropout: float = 0.0, - **kwargs: Unpack[TransformersKwargs], -): - key_states = repeat_kv(key, module.num_key_value_groups) - value_states = repeat_kv(value, module.num_key_value_groups) - - attn_weights = torch.matmul(query, key_states.transpose(2, 3)) * scaling - if attention_mask is not None: - causal_mask = attention_mask[:, :, :, : key_states.shape[-2]] - attn_weights = attn_weights + causal_mask - - attn_weights = nn.functional.softmax(attn_weights, dim=-1, dtype=torch.float32).to(query.dtype) - attn_weights = nn.functional.dropout(attn_weights, p=dropout, training=module.training) - attn_output = torch.matmul(attn_weights, value_states) - attn_output = attn_output.transpose(1, 2).contiguous() - - return attn_output, attn_weights - - -def rotate_half_llm(x): - """Rotates half the hidden dims of the input.""" - x1 = x[..., 0::2] - x2 = x[..., 1::2] - return torch.stack((-x2, x1), dim=-1).flatten(-2) - - -def apply_multimodal_rotary_pos_emb(q, k, cos, sin, mrope_section, unsqueeze_dim=1): - """Applies Rotary Position Embedding with Multimodal Sections to the query and key tensors (https://qwenlm.github.io/blog/qwen2-vl/). - - Explanation: - Multimodal 3D rotary position embedding is an extension to 1D rotary position embedding. The input embedding - sequence contains vision (images / videos) embedding and text embedding or just contains text embedding. For - vision embedding part, we apply rotary position embedding on temporal, height and width dimension separately. - Here we split the channel dimension to 3 chunks for the temporal, height and width rotary position embedding. - For text embedding part, we just apply 1D rotary position embedding. The three rotary position index (temporal, - height and width) of text embedding is always the same, so the text embedding rotary position embedding has no - difference with modern LLMs. - - Args: - q (`torch.Tensor`): The query tensor. - k (`torch.Tensor`): The key tensor. - cos (`torch.Tensor`): The cosine part of the rotary embedding. - sin (`torch.Tensor`): The sine part of the rotary embedding. - mrope_section(`List(int)`): - Multimodal rope section is for channel dimension of temporal, height and width in rope calculation. - unsqueeze_dim (`int`, *optional*, defaults to 1): - The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and - sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note - that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and - k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes - cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have - the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2. - Returns: - `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding. - """ - mrope_section = mrope_section * 2 - cos = torch.cat([m[i % 3] for i, m in enumerate(cos.split(mrope_section, dim=-1))], dim=-1).unsqueeze( - unsqueeze_dim - ) - sin = torch.cat([m[i % 3] for i, m in enumerate(sin.split(mrope_section, dim=-1))], dim=-1).unsqueeze( - unsqueeze_dim - ) - - # Interleave them instead of usual shape - cos = cos[..., : cos.shape[-1] // 2].repeat_interleave(2, dim=-1) - sin = sin[..., : sin.shape[-1] // 2].repeat_interleave(2, dim=-1) - - # Keep half or full tensor for later concatenation - rotary_dim = cos.shape[-1] - q_rot, q_pass = q[..., :rotary_dim], q[..., rotary_dim:] - k_rot, k_pass = k[..., :rotary_dim], k[..., rotary_dim:] - - # Apply rotary embeddings on the first half or full tensor - q_embed = (q_rot * cos) + (rotate_half_llm(q_rot) * sin) - k_embed = (k_rot * cos) + (rotate_half_llm(k_rot) * sin) - - # Concatenate back to full shape - q_embed = torch.cat([q_embed, q_pass], dim=-1) - k_embed = torch.cat([k_embed, k_pass], dim=-1) - - return q_embed, k_embed - - -class Glm46VTextAttention(nn.Module): - """ - Multi-headed attention from 'Attention Is All You Need' paper. - and "Generating Long Sequences with Sparse Transformers". - """ - - def __init__(self, config: Glm46VTextConfig, layer_idx: Optional[int] = None): - super().__init__() - self.config = config - self.layer_idx = layer_idx - - self.hidden_size = config.hidden_size - self.num_heads = config.num_attention_heads - self.head_dim = self.hidden_size // self.num_heads - self.num_key_value_heads = config.num_key_value_heads - self.num_key_value_groups = self.num_heads // self.num_key_value_heads - self.is_causal = True - self.attention_dropout = config.attention_dropout - self.rope_parameters = config.rope_parameters - self.scaling = self.head_dim**-0.5 - - self.q_proj = nn.Linear(self.hidden_size, self.num_heads * self.head_dim, bias=True) - self.k_proj = nn.Linear(self.hidden_size, self.num_key_value_heads * self.head_dim, bias=True) - self.v_proj = nn.Linear(self.hidden_size, self.num_key_value_heads * self.head_dim, bias=True) - self.o_proj = nn.Linear(self.num_heads * self.head_dim, self.hidden_size, bias=False) - - def forward( - self, - hidden_states: torch.Tensor, - position_embeddings: Optional[tuple[torch.Tensor, torch.Tensor]] = None, - attention_mask: Optional[torch.Tensor] = None, - past_key_values: Optional[Cache] = None, - cache_position: Optional[torch.LongTensor] = None, - **kwargs: Unpack[FlashAttentionKwargs], - ) -> tuple[torch.Tensor, Optional[torch.Tensor], Optional[tuple[torch.Tensor]]]: - bsz, q_len, _ = hidden_states.size() - - query_states = self.q_proj(hidden_states) - key_states = self.k_proj(hidden_states) - value_states = self.v_proj(hidden_states) - - query_states = query_states.view(bsz, q_len, -1, self.head_dim).transpose(1, 2) - key_states = key_states.view(bsz, q_len, -1, self.head_dim).transpose(1, 2) - value_states = value_states.view(bsz, q_len, -1, self.head_dim).transpose(1, 2) - - cos, sin = position_embeddings - query_states, key_states = apply_multimodal_rotary_pos_emb( # diff with Llama - query_states, key_states, cos, sin, self.rope_parameters["mrope_section"] - ) - - if past_key_values is not None: - cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position} # Specific to RoPE models - key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx, cache_kwargs) - - attention_interface: Callable = eager_attention_forward - if self.config._attn_implementation != "eager": - attention_interface = ALL_ATTENTION_FUNCTIONS[self.config._attn_implementation] - - attn_output, attn_weights = attention_interface( - self, - query_states, - key_states, - value_states, - attention_mask, - dropout=0.0 if not self.training else self.attention_dropout, - scaling=self.scaling, - **kwargs, - ) - - attn_output = attn_output.reshape(bsz, q_len, -1).contiguous() - attn_output = self.o_proj(attn_output) - return attn_output, attn_weights - - -class Glm46VTextMLP(nn.Module): - def __init__(self, config): - super().__init__() - - self.config = config - self.gate_up_proj = nn.Linear(config.hidden_size, 2 * config.intermediate_size, bias=False) - self.down_proj = nn.Linear(config.intermediate_size, config.hidden_size, bias=False) - self.activation_fn = ACT2FN[config.hidden_act] - - def forward(self, hidden_states: torch.FloatTensor) -> torch.FloatTensor: - up_states = self.gate_up_proj(hidden_states) - - gate, up_states = up_states.chunk(2, dim=-1) - up_states = up_states * self.activation_fn(gate) - - return self.down_proj(up_states) - - -class Glm46VTextDecoderLayer(GradientCheckpointingLayer): - def __init__(self, config: Glm46VTextConfig, layer_idx: int): - super().__init__() - self.hidden_size = config.hidden_size - self.self_attn = Glm46VTextAttention(config, layer_idx) - self.mlp = Glm46VTextMLP(config) - self.input_layernorm = Glm46VRMSNorm(config.hidden_size, eps=config.rms_norm_eps) - self.post_attention_layernorm = Glm46VRMSNorm(config.hidden_size, eps=config.rms_norm_eps) - self.post_self_attn_layernorm = Glm46VRMSNorm(config.hidden_size, eps=config.rms_norm_eps) - self.post_mlp_layernorm = Glm46VRMSNorm(config.hidden_size, eps=config.rms_norm_eps) - - @auto_docstring - def forward( - self, - hidden_states: torch.Tensor, - position_embeddings: Optional[tuple[torch.Tensor, torch.Tensor]] = None, - attention_mask: Optional[torch.Tensor] = None, - position_ids: Optional[torch.LongTensor] = None, - past_key_values: Optional[Cache] = None, - use_cache: Optional[bool] = False, - cache_position: Optional[torch.LongTensor] = None, - **kwargs, - ) -> tuple[torch.FloatTensor, Optional[tuple[torch.FloatTensor, torch.FloatTensor]]]: - residual = hidden_states - - hidden_states = self.input_layernorm(hidden_states) - - # Self Attention - hidden_states, _ = self.self_attn( - hidden_states=hidden_states, - position_embeddings=position_embeddings, - attention_mask=attention_mask, - position_ids=position_ids, - past_key_values=past_key_values, - use_cache=use_cache, - cache_position=cache_position, - **kwargs, - ) - - hidden_states = self.post_self_attn_layernorm(hidden_states) - hidden_states = residual + hidden_states - - # Fully Connected - residual = hidden_states - hidden_states = self.post_attention_layernorm(hidden_states) - hidden_states = self.mlp(hidden_states) - hidden_states = self.post_mlp_layernorm(hidden_states) - hidden_states = residual + hidden_states - - return hidden_states @auto_docstring class Glm46VPreTrainedModel(PreTrainedModel): - config: Glm46VConfig + config: None base_model_prefix = "model" input_modalities = ["image", "video", "text"] supports_gradient_checkpointing = True - _no_split_modules = ["Glm46VTextDecoderLayer", "Glm46VVisionBlock"] + _no_split_modules = None _skip_keys_device_placement = "past_key_values" _supports_flash_attn = True _supports_sdpa = True _can_compile_fullgraph = True _supports_attention_backend = True - _can_record_outputs = { - "hidden_states": Glm46VTextDecoderLayer, - "attentions": Glm46VTextAttention, - } + _can_record_outputs = None @dataclass @@ -356,10 +79,10 @@ class Glm46VModel(Glm46VPreTrainedModel): _checkpoint_conversion_mapping = {} # Reference: fix gemma3 grad acc #37208 accepts_loss_kwargs = False - config: Glm46VConfig - _no_split_modules = ["Glm46VTextDecoderLayer", "Glm46VVisionBlock"] + config: None + _no_split_modules = None - def __init__(self, config: Glm46VConfig): + def __init__(self, config): super().__init__(config) self.visual = AutoModel.from_config(config.vision_config) self.language_model = AutoModel.from_config(config.text_config) diff --git a/src/transformers/models/glm46v/modular_glm46v.py b/src/transformers/models/glm46v/modular_glm46v.py index fc200a02f610..f0d535f3029f 100644 --- a/src/transformers/models/glm46v/modular_glm46v.py +++ b/src/transformers/models/glm46v/modular_glm46v.py @@ -19,7 +19,6 @@ from ...video_utils import VideoMetadata from ..auto.modeling_auto import AutoModel -from ..glm4v.configuration_glm4v import Glm4vConfig, Glm4vTextConfig from ..glm4v.image_processing_glm4v import Glm4vImageProcessor from ..glm4v.image_processing_glm4v_fast import Glm4vImageProcessorFast from ..glm4v.modeling_glm4v import Glm4vModel, Glm4vPreTrainedModel @@ -27,20 +26,17 @@ from ..glm4v.video_processing_glm4v import Glm4vVideoProcessor -class Glm46VTextConfig(Glm4vTextConfig): - pass - - -class Glm46VConfig(Glm4vConfig): - pass - - class Glm46VPreTrainedModel(Glm4vPreTrainedModel): - pass + config: None + _can_record_outputs = None + _no_split_modules = None class Glm46VModel(Glm4vModel): - def __init__(self, config: Glm46VConfig): + _no_split_modules = None + config: None + + def __init__(self, config): super().__init__(config) self.visual = AutoModel.from_config(config.vision_config) self.language_model = AutoModel.from_config(config.text_config) @@ -128,8 +124,6 @@ def sample_frames( __all__ = [ - "Glm46VConfig", - "Glm46VTextConfig", "Glm46VModel", "Glm46VPreTrainedModel", "Glm46VProcessor", diff --git a/tests/models/glm46v/test_modeling_glm46v.py b/tests/models/glm46v/test_modeling_glm46v.py index 427e7ea85a3a..8169a7643734 100644 --- a/tests/models/glm46v/test_modeling_glm46v.py +++ b/tests/models/glm46v/test_modeling_glm46v.py @@ -20,9 +20,9 @@ from transformers import ( AutoProcessor, Glm46VConfig, - Glm46VForConditionalGeneration, Glm46VModel, is_torch_available, + AutoModel ) from transformers.testing_utils import ( Expectations, @@ -173,7 +173,7 @@ def prepare_config_and_inputs_for_common(self): @require_torch class Glm46VModelTest(ModelTesterMixin, GenerationTesterMixin, unittest.TestCase): - all_model_classes = (Glm46VModel, Glm46VForConditionalGeneration) if is_torch_available() else () + all_model_classes = (Glm46VModel) if is_torch_available() else () model_split_percents = [0.7, 0.9] # model too big to split at 0.5 _is_composite = True @@ -315,7 +315,7 @@ def tearDown(self): @slow def test_small_model_integration_test(self): - model = Glm46VForConditionalGeneration.from_pretrained( + model = AutoModel.from_pretrained( "THUDM/GLM-4.1V-9B-Thinking", dtype="auto", device_map="auto" ) @@ -354,7 +354,7 @@ def test_small_model_integration_test(self): @slow def test_small_model_integration_test_batch(self): - model = Glm46VForConditionalGeneration.from_pretrained( + model = AutoModel.from_pretrained( "THUDM/GLM-4.1V-9B-Thinking", dtype="auto", device_map="auto" ) batch_messages = [self.message] * 2 @@ -380,7 +380,7 @@ def test_small_model_integration_test_batch(self): @slow def test_small_model_integration_test_with_video(self): processor = AutoProcessor.from_pretrained("THUDM/GLM-4.1V-9B-Thinking", max_image_size={"longest_edge": 50176}) - model = Glm46VForConditionalGeneration.from_pretrained( + model = AutoModel.from_pretrained( "THUDM/GLM-4.1V-9B-Thinking", dtype=torch.float16, device_map="auto" ) questions = ["Describe this video."] @@ -418,7 +418,7 @@ def test_small_model_integration_test_with_video(self): @slow @require_deterministic_for_xpu def test_small_model_integration_test_expand(self): - model = Glm46VForConditionalGeneration.from_pretrained( + model = AutoModel.from_pretrained( "THUDM/GLM-4.1V-9B-Thinking", dtype="auto", device_map="auto" ) inputs = self.processor.apply_chat_template( @@ -450,7 +450,7 @@ def test_small_model_integration_test_expand(self): @slow def test_small_model_integration_test_batch_wo_image(self): - model = Glm46VForConditionalGeneration.from_pretrained( + model = AutoModel.from_pretrained( "THUDM/GLM-4.1V-9B-Thinking", dtype="auto", device_map="auto" ) message_wo_image = [ @@ -483,7 +483,7 @@ def test_small_model_integration_test_batch_wo_image(self): @slow def test_small_model_integration_test_batch_different_resolutions(self): - model = Glm46VForConditionalGeneration.from_pretrained( + model = AutoModel.from_pretrained( "THUDM/GLM-4.1V-9B-Thinking", dtype="auto", device_map="auto" ) batched_messages = [self.message, self.message2] @@ -515,7 +515,7 @@ def test_small_model_integration_test_batch_different_resolutions(self): @require_flash_attn @require_torch_gpu def test_small_model_integration_test_batch_flashatt2(self): - model = Glm46VForConditionalGeneration.from_pretrained( + model = AutoModel.from_pretrained( "THUDM/GLM-4.1V-9B-Thinking", dtype=torch.bfloat16, attn_implementation="flash_attention_2", @@ -550,7 +550,7 @@ def test_small_model_integration_test_batch_flashatt2(self): @require_flash_attn @require_torch_gpu def test_small_model_integration_test_batch_wo_image_flashatt2(self): - model = Glm46VForConditionalGeneration.from_pretrained( + model = AutoModel.from_pretrained( "THUDM/GLM-4.1V-9B-Thinking", dtype=torch.bfloat16, attn_implementation="flash_attention_2", From 1c8905d7d8b650cc35f714fcd80088c6ba284574 Mon Sep 17 00:00:00 2001 From: zRzRzRzRzRzRzR <2448370773@qq.com> Date: Thu, 13 Nov 2025 01:15:52 +0800 Subject: [PATCH 38/66] delete --- docs/source/en/model_doc/glm46v.md | 4 - .../models/auto/configuration_auto.py | 1 + .../models/glm46v/configuration_glm46v.py | 239 +----------------- .../models/glm46v/modular_glm46v.py | 5 + tests/models/glm46v/test_modeling_glm46v.py | 32 +-- 5 files changed, 14 insertions(+), 267 deletions(-) diff --git a/docs/source/en/model_doc/glm46v.md b/docs/source/en/model_doc/glm46v.md index c6fd6873f2ca..6d1a1af5cc03 100644 --- a/docs/source/en/model_doc/glm46v.md +++ b/docs/source/en/model_doc/glm46v.md @@ -4,10 +4,6 @@ [[autodoc]] Glm46VConfig -## Glm46VTextConfig - -[[autodoc]] Glm46VTextConfig - ## Glm46VImageProcessor [[autodoc]] Glm46VImageProcessor diff --git a/src/transformers/models/auto/configuration_auto.py b/src/transformers/models/auto/configuration_auto.py index 9bf657444249..d3b08a150017 100644 --- a/src/transformers/models/auto/configuration_auto.py +++ b/src/transformers/models/auto/configuration_auto.py @@ -172,6 +172,7 @@ ("git", "GitConfig"), ("glm", "GlmConfig"), ("glm4", "Glm4Config"), + ("glm46v", "Glm46VConfig"), ("glm4_moe", "Glm4MoeConfig"), ("glm4v", "Glm4vConfig"), ("glm4v_moe", "Glm4vMoeConfig"), diff --git a/src/transformers/models/glm46v/configuration_glm46v.py b/src/transformers/models/glm46v/configuration_glm46v.py index ecdaf0ac075a..50f42ad44552 100644 --- a/src/transformers/models/glm46v/configuration_glm46v.py +++ b/src/transformers/models/glm46v/configuration_glm46v.py @@ -19,242 +19,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -from typing import Optional from ...configuration_utils import PreTrainedConfig -from ...modeling_rope_utils import RopeParameters, rope_config_validation, standardize_rope_params - - -class Glm46VTextConfig(PreTrainedConfig): - r""" - This is the configuration class to store the configuration of a [`Glm46VModel`]. It is used to instantiate a - GLM-4.1V model according to the specified arguments, defining the model architecture. Instantiating a - configuration with the defaults will yield a similar configuration to that of - GLM-4.1V-9B-Thinking [THUDM/GLM-4.1V-9B-Thinking](https://huggingface.co/THUDM/GLM-4.1V-9B-Thinking). - - Configuration objects inherit from [`PreTrainedConfig`] and can be used to control the model outputs. Read the - documentation from [`PreTrainedConfig`] for more information. - - Args: - vocab_size (`int`, *optional*, defaults to 151552): - Vocabulary size of the Glm46V model. Defines the number of different tokens that can be represented by the - `inputs_ids` passed when calling [`Glm46VModel`] - hidden_size (`int`, *optional*, defaults to 4096): - Dimension of the hidden representations. - intermediate_size (`int`, *optional*, defaults to 13696): - Dimension of the MLP representations. - num_hidden_layers (`int`, *optional*, defaults to 40): - Number of hidden layers in the Transformer encoder. - num_attention_heads (`int`, *optional*, defaults to 32): - Number of attention heads for each attention layer in the Transformer encoder. - num_key_value_heads (`int`, *optional*, defaults to 2): - This is the number of key_value heads that should be used to implement Grouped Query Attention. If - `num_key_value_heads=num_attention_heads`, the model will use Multi Head Attention (MHA), if - `num_key_value_heads=1` the model will use Multi Query Attention (MQA) otherwise GQA is used. When - converting a multi-head checkpoint to a GQA checkpoint, each group key and value head should be constructed - by meanpooling all the original heads within that group. For more details checkout [this - paper](https://huggingface.co/papers/2305.13245). If it is not specified, will default to `32`. - hidden_act (`str` or `function`, *optional*, defaults to `"silu"`): - The non-linear activation function (function or string) in the decoder. - max_position_embeddings (`int`, *optional*, defaults to 32768): - The maximum sequence length that this model might ever be used with. - initializer_range (`float`, *optional*, defaults to 0.02): - The standard deviation of the truncated_normal_initializer for initializing all weight matrices. - rms_norm_eps (`float`, *optional*, defaults to 1e-05): - The epsilon used by the rms normalization layers. - use_cache (`bool`, *optional*, defaults to `True`): - Whether or not the model should return the last key/values attentions (not used by all models). Only - relevant if `config.is_decoder=True`. - tie_word_embeddings (`bool`, *optional*, defaults to `False`): - Whether the model's input and output word embeddings should be tied. - attention_dropout (`float`, *optional*, defaults to 0.0): - The dropout ratio for the attention probabilities. - rope_parameters (`RopeParameters`, *optional*): - Dictionary containing the configuration parameters for the RoPE embeddings. The dictionaty should contain - a value for `rope_theta` and optionally parameters used for scaling in case you want to use RoPE - with longer `max_position_embeddings`. - image_token_id (`int`, *optional*): - Token index used as placeholder for image embeddings. - video_token_id (`int`, *optional*): - Token index used as placeholder for video embeddings. - - ```python - >>> from transformers import Glm46VTextModel, Glm46VConfig - - >>> # Initializing a GLM-4.1V style configuration - >>> configuration = Glm46VConfig() - - >>> # Initializing a model from the GLM-4.1V style configuration - >>> model = Glm46VTextModel(configuration) - - >>> # Accessing the model configuration - >>> configuration = model.config - ```""" - - model_type = "glm46v_text" - base_config_key = "text_config" - keys_to_ignore_at_inference = ["past_key_values"] - # Default tensor parallel plan for base model `Glm46V` - base_model_tp_plan = { - "layers.*.self_attn.q_proj": "colwise", - "layers.*.self_attn.k_proj": "colwise", - "layers.*.self_attn.v_proj": "colwise", - "layers.*.self_attn.o_proj": "rowwise", - "layers.*.mlp.gate_up_proj": "colwise_rep", # we need to replicate here due to the `chunk` operation - "layers.*.mlp.down_proj": "rowwise_rep", # we need to replicate here due to the `chunk` operation - } - base_model_pp_plan = { - "embed_tokens": (["input_ids"], ["inputs_embeds"]), - "layers": (["hidden_states", "attention_mask"], ["hidden_states"]), - "norm": (["hidden_states"], ["hidden_states"]), - } - - def __init__( - self, - vocab_size: Optional[int] = 151552, - hidden_size: Optional[int] = 4096, - intermediate_size: Optional[int] = 13696, - num_hidden_layers: Optional[int] = 40, - num_attention_heads: Optional[int] = 32, - num_key_value_heads: Optional[int] = 2, - hidden_act: Optional[str] = "silu", - max_position_embeddings: Optional[int] = 32768, - initializer_range: Optional[float] = 0.02, - rms_norm_eps: Optional[int] = 1e-05, - use_cache: Optional[bool] = True, - tie_word_embeddings: Optional[bool] = False, - attention_dropout: Optional[float] = 0.0, - rope_parameters: Optional[RopeParameters | dict[str, RopeParameters]] = None, - image_token_id: Optional[int] = None, - video_token_id: Optional[int] = None, - **kwargs, - ): - self.vocab_size = vocab_size - self.max_position_embeddings = max_position_embeddings - self.hidden_size = hidden_size - self.intermediate_size = intermediate_size - self.num_hidden_layers = num_hidden_layers - self.num_attention_heads = num_attention_heads - - # for backward compatibility - if num_key_value_heads is None: - num_key_value_heads = num_attention_heads - - self.num_key_value_heads = num_key_value_heads - self.hidden_act = hidden_act - self.initializer_range = initializer_range - self.rms_norm_eps = rms_norm_eps - self.use_cache = use_cache - self.attention_dropout = attention_dropout - # Try to set `rope_scaling` if available, otherwise use `rope_parameters` - rope_scaling = kwargs.pop("rope_scaling", None) - self.rope_parameters = rope_scaling or rope_parameters - - # Validate the correctness of rotary position embeddings parameters - rope_theta = kwargs.get("rope_theta", 10000.0) - standardize_rope_params(self, rope_theta=rope_theta) - rope_config_validation(self, ignore_keys={"mrope_section"}) - self.image_token_id = image_token_id - self.video_token_id = video_token_id - - super().__init__(tie_word_embeddings=tie_word_embeddings, **kwargs) - - -class Glm46VVisionConfig(PreTrainedConfig): - r""" - This is the configuration class to store the configuration of a [`Glm46VVisionModel`]. It is used to instantiate an Glm46VVisionModel - model according to the specified arguments, defining the model architecture. Instantiating a configuration with the defaults will yield - a similar configuration to that of - GLM-4.1V-9B-Thinking [THUDM/GLM-4.1V-9B-Thinking](https://huggingface.co/THUDM/GLM-4.1V-9B-Thinking). - - Args: - hidden_size (`int`, *optional*, defaults to 1536): - Dimensionality of the encoder layers and the pooler layer. - depth (`int`, *optional*, defaults to 24): - Number of layers (depth) in the model. - attention_bias (`bool`, *optional*, defaults to `False`): - Whether to add a bias to the queries, keys and values. - intermediate_size (`int`, *optional*, defaults to 13696): - Dimensionality of the "intermediate" (i.e., feed-forward) layer in the Transformer encoder. - hidden_act (`str` or `function`, *optional*, defaults to `"selu"`): - The non-linear activation function (function or string) in the encoder and pooler. If string, `"gelu"`, - `"relu"`, `"selu"` and `"gelu_new"` are supported. - hidden_dropout_prob (`float`, *optional*, defaults to 0.0): - The dropout probability for all fully connected layers in the embeddings, encoder, and pooler. - attention_dropout (`float`, *optional*, defaults to 0.0): - Dropout probability for attention weights. - projection_dropout (`float`, *optional*, defaults to 0.0): - Dropout probability for the projection layer. - initializer_range (`float`, *optional*, defaults to 0.02): - The standard deviation of the truncated_normal_initializer for initializing all weight matrices. - image_size (`int` or `list[int]`, *optional*, defaults to `[336, 336]`): - The size (resolution) of each image. - patch_size (`int`, *optional*, defaults to `14`): - The size (resolution) of each patch. - num_channels (`int`, *optional*, defaults to 3): - The number of input channels. - out_hidden_size (`int`, *optional*, defaults to 4096): - The output hidden size of the vision model. - rms_norm_eps (`float`, *optional*, defaults to 1e-05): - The epsilon used by the rms normalization layers. - spatial_merge_size (`int`, *optional*, defaults to 2): - The size used for merging spatial dimensions. - temporal_patch_size (`int`, *optional*, defaults to 2): - The size used for patches along the temporal dimension. - Example: - - ```python - >>> from transformers import Glm46VVisionConfig, Glm46VVisionModel - - >>> # Initializing a Glm46VVisionConfig GLM-4.1V-9B style configuration - >>> configuration = Glm46VVisionConfig() - - >>> # Initializing a model (with random weights) from the GLM-4.1V-9B configuration - >>> model = Glm46VVisionModel(configuration) - - >>> # Accessing the model configuration - >>> configuration = model.config - ```""" - - model_type = "glm46v" - base_config_key = "vision_config" - - def __init__( - self, - depth=24, - hidden_size=1536, - hidden_act="silu", - attention_bias=False, - attention_dropout=0.0, - num_heads=12, - in_channels=3, - image_size=336, - patch_size=14, - rms_norm_eps=1e-05, - spatial_merge_size=2, - temporal_patch_size=2, - out_hidden_size=4096, - intermediate_size=13696, - initializer_range=0.02, - **kwargs, - ): - super().__init__(**kwargs) - - self.depth = depth - self.hidden_size = hidden_size - self.hidden_act = hidden_act - self.num_heads = num_heads - self.in_channels = in_channels - self.image_size = image_size - self.patch_size = patch_size - self.spatial_merge_size = spatial_merge_size - self.temporal_patch_size = temporal_patch_size - self.out_hidden_size = out_hidden_size - self.intermediate_size = intermediate_size - self.initializer_range = initializer_range - self.rms_norm_eps = rms_norm_eps - self.attention_bias = attention_bias - self.attention_dropout = attention_dropout class Glm46VConfig(PreTrainedConfig): @@ -300,7 +66,7 @@ class Glm46VConfig(PreTrainedConfig): ```""" model_type = "glm46v" - sub_configs = {"vision_config": Glm46VVisionConfig, "text_config": Glm46VTextConfig} + sub_configs = None keys_to_ignore_at_inference = ["past_key_values"] def __init__( @@ -333,6 +99,3 @@ def __init__( self.image_end_token_id = image_end_token_id super().__init__(**kwargs) - - -__all__ = ["Glm46VConfig", "Glm46VTextConfig"] diff --git a/src/transformers/models/glm46v/modular_glm46v.py b/src/transformers/models/glm46v/modular_glm46v.py index f0d535f3029f..e1619afd16c4 100644 --- a/src/transformers/models/glm46v/modular_glm46v.py +++ b/src/transformers/models/glm46v/modular_glm46v.py @@ -19,6 +19,7 @@ from ...video_utils import VideoMetadata from ..auto.modeling_auto import AutoModel +from ..glm4v.configuration_glm4v import Glm4vConfig from ..glm4v.image_processing_glm4v import Glm4vImageProcessor from ..glm4v.image_processing_glm4v_fast import Glm4vImageProcessorFast from ..glm4v.modeling_glm4v import Glm4vModel, Glm4vPreTrainedModel @@ -26,6 +27,10 @@ from ..glm4v.video_processing_glm4v import Glm4vVideoProcessor +class Glm46VConfig(Glm4vConfig): + sub_configs = None + + class Glm46VPreTrainedModel(Glm4vPreTrainedModel): config: None _can_record_outputs = None diff --git a/tests/models/glm46v/test_modeling_glm46v.py b/tests/models/glm46v/test_modeling_glm46v.py index 8169a7643734..eeb9752eed95 100644 --- a/tests/models/glm46v/test_modeling_glm46v.py +++ b/tests/models/glm46v/test_modeling_glm46v.py @@ -17,13 +17,7 @@ import copy import unittest -from transformers import ( - AutoProcessor, - Glm46VConfig, - Glm46VModel, - is_torch_available, - AutoModel -) +from transformers import AutoModel, AutoProcessor, Glm46VConfig, Glm46VModel, is_torch_available from transformers.testing_utils import ( Expectations, cleanup, @@ -315,9 +309,7 @@ def tearDown(self): @slow def test_small_model_integration_test(self): - model = AutoModel.from_pretrained( - "THUDM/GLM-4.1V-9B-Thinking", dtype="auto", device_map="auto" - ) + model = AutoModel.from_pretrained("THUDM/GLM-4.1V-9B-Thinking", dtype="auto", device_map="auto") inputs = self.processor.apply_chat_template( self.message, tokenize=True, add_generation_prompt=True, return_dict=True, return_tensors="pt" @@ -354,9 +346,7 @@ def test_small_model_integration_test(self): @slow def test_small_model_integration_test_batch(self): - model = AutoModel.from_pretrained( - "THUDM/GLM-4.1V-9B-Thinking", dtype="auto", device_map="auto" - ) + model = AutoModel.from_pretrained("THUDM/GLM-4.1V-9B-Thinking", dtype="auto", device_map="auto") batch_messages = [self.message] * 2 inputs = self.processor.apply_chat_template( batch_messages, tokenize=True, add_generation_prompt=True, return_dict=True, return_tensors="pt" @@ -380,9 +370,7 @@ def test_small_model_integration_test_batch(self): @slow def test_small_model_integration_test_with_video(self): processor = AutoProcessor.from_pretrained("THUDM/GLM-4.1V-9B-Thinking", max_image_size={"longest_edge": 50176}) - model = AutoModel.from_pretrained( - "THUDM/GLM-4.1V-9B-Thinking", dtype=torch.float16, device_map="auto" - ) + model = AutoModel.from_pretrained("THUDM/GLM-4.1V-9B-Thinking", dtype=torch.float16, device_map="auto") questions = ["Describe this video."] video_urls = ["https://huggingface.co/datasets/hf-internal-testing/fixtures_videos/resolve/main/tennis.mp4"] messages = [ @@ -418,9 +406,7 @@ def test_small_model_integration_test_with_video(self): @slow @require_deterministic_for_xpu def test_small_model_integration_test_expand(self): - model = AutoModel.from_pretrained( - "THUDM/GLM-4.1V-9B-Thinking", dtype="auto", device_map="auto" - ) + model = AutoModel.from_pretrained("THUDM/GLM-4.1V-9B-Thinking", dtype="auto", device_map="auto") inputs = self.processor.apply_chat_template( self.message, tokenize=True, add_generation_prompt=True, return_dict=True, return_tensors="pt" ).to(torch_device) @@ -450,9 +436,7 @@ def test_small_model_integration_test_expand(self): @slow def test_small_model_integration_test_batch_wo_image(self): - model = AutoModel.from_pretrained( - "THUDM/GLM-4.1V-9B-Thinking", dtype="auto", device_map="auto" - ) + model = AutoModel.from_pretrained("THUDM/GLM-4.1V-9B-Thinking", dtype="auto", device_map="auto") message_wo_image = [ {"role": "user", "content": [{"type": "text", "text": "Who are you?"}]}, ] @@ -483,9 +467,7 @@ def test_small_model_integration_test_batch_wo_image(self): @slow def test_small_model_integration_test_batch_different_resolutions(self): - model = AutoModel.from_pretrained( - "THUDM/GLM-4.1V-9B-Thinking", dtype="auto", device_map="auto" - ) + model = AutoModel.from_pretrained("THUDM/GLM-4.1V-9B-Thinking", dtype="auto", device_map="auto") batched_messages = [self.message, self.message2] inputs = self.processor.apply_chat_template( batched_messages, From 9c2d854fc81fdf8f061297cb7a01221974be86a5 Mon Sep 17 00:00:00 2001 From: zRzRzRzRzRzRzR <2448370773@qq.com> Date: Thu, 13 Nov 2025 01:20:19 +0800 Subject: [PATCH 39/66] test --- src/transformers/models/glm46v/configuration_glm46v.py | 3 +++ src/transformers/models/glm46v/modeling_glm46v.py | 5 +++-- src/transformers/models/glm46v/modular_glm46v.py | 3 +-- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/src/transformers/models/glm46v/configuration_glm46v.py b/src/transformers/models/glm46v/configuration_glm46v.py index 50f42ad44552..c9049607ac2c 100644 --- a/src/transformers/models/glm46v/configuration_glm46v.py +++ b/src/transformers/models/glm46v/configuration_glm46v.py @@ -99,3 +99,6 @@ def __init__( self.image_end_token_id = image_end_token_id super().__init__(**kwargs) + + +__all__ = ["Glm46VConfig"] diff --git a/src/transformers/models/glm46v/modeling_glm46v.py b/src/transformers/models/glm46v/modeling_glm46v.py index 5c5154866fcf..8e0cb7a9b34f 100644 --- a/src/transformers/models/glm46v/modeling_glm46v.py +++ b/src/transformers/models/glm46v/modeling_glm46v.py @@ -31,11 +31,12 @@ from ...processing_utils import Unpack from ...utils import TransformersKwargs, auto_docstring, can_return_tuple, is_torchdynamo_compiling from ..auto.modeling_auto import AutoModel +from .configuration_glm46v import Glm46VConfig @auto_docstring class Glm46VPreTrainedModel(PreTrainedModel): - config: None + config: Glm46VConfig base_model_prefix = "model" input_modalities = ["image", "video", "text"] supports_gradient_checkpointing = True @@ -79,7 +80,7 @@ class Glm46VModel(Glm46VPreTrainedModel): _checkpoint_conversion_mapping = {} # Reference: fix gemma3 grad acc #37208 accepts_loss_kwargs = False - config: None + config: Glm46VConfig _no_split_modules = None def __init__(self, config): diff --git a/src/transformers/models/glm46v/modular_glm46v.py b/src/transformers/models/glm46v/modular_glm46v.py index e1619afd16c4..d4eb3c3edd51 100644 --- a/src/transformers/models/glm46v/modular_glm46v.py +++ b/src/transformers/models/glm46v/modular_glm46v.py @@ -32,14 +32,12 @@ class Glm46VConfig(Glm4vConfig): class Glm46VPreTrainedModel(Glm4vPreTrainedModel): - config: None _can_record_outputs = None _no_split_modules = None class Glm46VModel(Glm4vModel): _no_split_modules = None - config: None def __init__(self, config): super().__init__(config) @@ -130,6 +128,7 @@ def sample_frames( __all__ = [ "Glm46VModel", + "Glm46VConfig", "Glm46VPreTrainedModel", "Glm46VProcessor", "Glm46VImageProcessor", From beedf506a55cb778f0944aaf514bcfdbba205ea0 Mon Sep 17 00:00:00 2001 From: zRzRzRzRzRzRzR <2448370773@qq.com> Date: Thu, 13 Nov 2025 13:58:25 +0800 Subject: [PATCH 40/66] update --- .../models/glm46v/configuration_glm46v.py | 20 +++++------ .../models/glm46v/modeling_glm46v.py | 2 +- .../models/glm46v/modular_glm46v.py | 35 +++++++++++++++++-- 3 files changed, 44 insertions(+), 13 deletions(-) diff --git a/src/transformers/models/glm46v/configuration_glm46v.py b/src/transformers/models/glm46v/configuration_glm46v.py index c9049607ac2c..7ac309b52bf7 100644 --- a/src/transformers/models/glm46v/configuration_glm46v.py +++ b/src/transformers/models/glm46v/configuration_glm46v.py @@ -19,8 +19,8 @@ # See the License for the specific language governing permissions and # limitations under the License. - from ...configuration_utils import PreTrainedConfig +from ..auto import CONFIG_MAPPING, AutoConfig class Glm46VConfig(PreTrainedConfig): @@ -66,7 +66,7 @@ class Glm46VConfig(PreTrainedConfig): ```""" model_type = "glm46v" - sub_configs = None + sub_configs = {"text_config": AutoConfig, "vision_config": AutoConfig} keys_to_ignore_at_inference = ["past_key_values"] def __init__( @@ -77,19 +77,21 @@ def __init__( video_token_id=151344, image_start_token_id=151339, image_end_token_id=151340, - video_start_token_id=151341, - video_end_token_id=151342, + video_start_token_id=151361, + video_end_token_id=151362, **kwargs, ): if isinstance(vision_config, dict): - self.vision_config = self.sub_configs["vision_config"](**vision_config) + vision_config["model_type"] = vision_config.get("model_type", "glm4v") + self.vision_config = CONFIG_MAPPING["vision_config"](**vision_config) elif vision_config is None: - self.vision_config = self.sub_configs["vision_config"]() + self.vision_config = CONFIG_MAPPING["glm4v"]() if isinstance(text_config, dict): - self.text_config = self.sub_configs["text_config"](**text_config) + text_config["model_type"] = text_config.get("model_type", "glm4v_text") + self.text_config = CONFIG_MAPPING[text_config["model_type"]](**text_config) elif text_config is None: - self.text_config = self.sub_configs["text_config"](**kwargs) + self.text_config = CONFIG_MAPPING["glm4v_text"]() self.image_token_id = image_token_id self.video_token_id = video_token_id @@ -98,7 +100,5 @@ def __init__( self.image_start_token_id = image_start_token_id self.image_end_token_id = image_end_token_id - super().__init__(**kwargs) - __all__ = ["Glm46VConfig"] diff --git a/src/transformers/models/glm46v/modeling_glm46v.py b/src/transformers/models/glm46v/modeling_glm46v.py index 8e0cb7a9b34f..2684b8441e42 100644 --- a/src/transformers/models/glm46v/modeling_glm46v.py +++ b/src/transformers/models/glm46v/modeling_glm46v.py @@ -30,7 +30,7 @@ from ...modeling_utils import PreTrainedModel from ...processing_utils import Unpack from ...utils import TransformersKwargs, auto_docstring, can_return_tuple, is_torchdynamo_compiling -from ..auto.modeling_auto import AutoModel +from ..auto import AutoModel from .configuration_glm46v import Glm46VConfig diff --git a/src/transformers/models/glm46v/modular_glm46v.py b/src/transformers/models/glm46v/modular_glm46v.py index d4eb3c3edd51..9a5f53815189 100644 --- a/src/transformers/models/glm46v/modular_glm46v.py +++ b/src/transformers/models/glm46v/modular_glm46v.py @@ -18,7 +18,7 @@ import numpy as np from ...video_utils import VideoMetadata -from ..auto.modeling_auto import AutoModel +from ..auto import CONFIG_MAPPING, AutoConfig, AutoModel from ..glm4v.configuration_glm4v import Glm4vConfig from ..glm4v.image_processing_glm4v import Glm4vImageProcessor from ..glm4v.image_processing_glm4v_fast import Glm4vImageProcessorFast @@ -28,7 +28,38 @@ class Glm46VConfig(Glm4vConfig): - sub_configs = None + sub_configs = {"text_config": AutoConfig, "vision_config": AutoConfig} + + def __init__( + self, + text_config=None, + vision_config=None, + image_token_id=151343, + video_token_id=151344, + image_start_token_id=151339, + image_end_token_id=151340, + video_start_token_id=151361, + video_end_token_id=151362, + **kwargs, + ): + if isinstance(vision_config, dict): + vision_config["model_type"] = vision_config.get("model_type", "glm4v") + self.vision_config = CONFIG_MAPPING["vision_config"](**vision_config) + elif vision_config is None: + self.vision_config = CONFIG_MAPPING["glm4v"]() + + if isinstance(text_config, dict): + text_config["model_type"] = text_config.get("model_type", "glm4v_text") + self.text_config = CONFIG_MAPPING[text_config["model_type"]](**text_config) + elif text_config is None: + self.text_config = CONFIG_MAPPING["glm4v_text"]() + + self.image_token_id = image_token_id + self.video_token_id = video_token_id + self.video_start_token_id = video_start_token_id + self.video_end_token_id = video_end_token_id + self.image_start_token_id = image_start_token_id + self.image_end_token_id = image_end_token_id class Glm46VPreTrainedModel(Glm4vPreTrainedModel): From efe6495b06a54f186e9b45c84f7b107f082aaac1 Mon Sep 17 00:00:00 2001 From: zRzRzRzRzRzRzR <2448370773@qq.com> Date: Thu, 13 Nov 2025 14:48:33 +0800 Subject: [PATCH 41/66] 1 --- .../models/glm46v/configuration_glm46v.py | 4 +- .../models/glm46v/modeling_glm46v.py | 356 +++++++++++++++++- .../models/glm46v/modular_glm46v.py | 13 +- 3 files changed, 365 insertions(+), 8 deletions(-) diff --git a/src/transformers/models/glm46v/configuration_glm46v.py b/src/transformers/models/glm46v/configuration_glm46v.py index 7ac309b52bf7..729ce03478c3 100644 --- a/src/transformers/models/glm46v/configuration_glm46v.py +++ b/src/transformers/models/glm46v/configuration_glm46v.py @@ -77,8 +77,8 @@ def __init__( video_token_id=151344, image_start_token_id=151339, image_end_token_id=151340, - video_start_token_id=151361, - video_end_token_id=151362, + video_start_token_id=151341, + video_end_token_id=151342, **kwargs, ): if isinstance(vision_config, dict): diff --git a/src/transformers/models/glm46v/modeling_glm46v.py b/src/transformers/models/glm46v/modeling_glm46v.py index 2684b8441e42..9a94755d0a9e 100644 --- a/src/transformers/models/glm46v/modeling_glm46v.py +++ b/src/transformers/models/glm46v/modeling_glm46v.py @@ -21,11 +21,13 @@ import itertools from dataclasses import dataclass -from typing import Optional, Union +from typing import Any, Optional, Union import torch +import torch.nn as nn from ...cache_utils import Cache +from ...generation import GenerationMixin from ...modeling_outputs import ModelOutput from ...modeling_utils import PreTrainedModel from ...processing_utils import Unpack @@ -483,4 +485,354 @@ def forward( ) -__all__ = ["Glm46VModel", "Glm46VPreTrainedModel"] +@dataclass +@auto_docstring( + custom_intro=""" + Base class for Glm46V causal language model (or autoregressive) outputs. + """ +) +class Glm46VCausalLMOutputWithPast(ModelOutput): + r""" + loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `labels` is provided): + Language modeling loss (for next-token prediction). + logits (`torch.FloatTensor` of shape `(batch_size, sequence_length, config.vocab_size)`): + Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax). + past_key_values (`Cache`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`): + It is a [`~cache_utils.Cache`] instance. For more details, see our [kv cache guide](https://huggingface.co/docs/transformers/en/kv_cache). + + Contains pre-computed hidden-states (key and values in the self-attention blocks) that can be used (see + `past_key_values` input) to speed up sequential decoding. + rope_deltas (`torch.LongTensor` of shape `(batch_size, )`, *optional*): + The rope index difference between sequence length and multimodal rope. + """ + + loss: Optional[torch.FloatTensor] = None + logits: Optional[torch.FloatTensor] = None + past_key_values: Optional[Cache] = None + hidden_states: Optional[tuple[torch.FloatTensor]] = None + attentions: Optional[tuple[torch.FloatTensor]] = None + rope_deltas: Optional[torch.LongTensor] = None + + +class Glm46VForConditionalGeneration(Glm46VPreTrainedModel, GenerationMixin): + _checkpoint_conversion_mapping = {} + _tied_weights_keys = ["lm_head.weight"] + # Reference: fix gemma3 grad acc #37208 + accepts_loss_kwargs = False + + def __init__(self, config): + super().__init__(config) + self.model = Glm46VModel(config) + self.lm_head = nn.Linear(config.text_config.hidden_size, config.text_config.vocab_size, bias=False) + + self.post_init() + + def get_input_embeddings(self): + return self.model.get_input_embeddings() + + def set_input_embeddings(self, value): + self.model.set_input_embeddings(value) + + def set_decoder(self, decoder): + self.model.set_decoder(decoder) + + def get_decoder(self): + return self.model.get_decoder() + + def get_video_features( + self, pixel_values_videos: torch.FloatTensor, video_grid_thw: Optional[torch.LongTensor] = None + ): + return self.model.get_video_features(pixel_values_videos, video_grid_thw) + + def get_image_features(self, pixel_values: torch.FloatTensor, image_grid_thw: Optional[torch.LongTensor] = None): + return self.model.get_image_features(pixel_values, image_grid_thw) + + # Make modules available through conditional class for BC + @property + def language_model(self): + return self.model.language_model + + @property + def visual(self): + return self.model.visual + + @can_return_tuple + @auto_docstring + def forward( + self, + input_ids: Optional[torch.LongTensor] = None, + attention_mask: Optional[torch.Tensor] = None, + position_ids: Optional[torch.LongTensor] = None, + past_key_values: Optional[Cache] = None, + inputs_embeds: Optional[torch.FloatTensor] = None, + labels: Optional[torch.LongTensor] = None, + pixel_values: Optional[torch.Tensor] = None, + pixel_values_videos: Optional[torch.FloatTensor] = None, + image_grid_thw: Optional[torch.LongTensor] = None, + video_grid_thw: Optional[torch.LongTensor] = None, + rope_deltas: Optional[torch.LongTensor] = None, + cache_position: Optional[torch.LongTensor] = None, + logits_to_keep: Union[int, torch.Tensor] = 0, + **kwargs: Unpack[TransformersKwargs], + ) -> Union[tuple, Glm46VCausalLMOutputWithPast]: + r""" + rope_deltas (`torch.LongTensor` of shape `(batch_size, )`, *optional*): + The rope index difference between sequence length and multimodal rope. + labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., + config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored + (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. + image_grid_thw (`torch.LongTensor` of shape `(num_images, 3)`, *optional*): + The temporal, height and width of feature shape of each image in LLM. + video_grid_thw (`torch.LongTensor` of shape `(num_videos, 3)`, *optional*): + The temporal, height and width of feature shape of each video in LLM. + + Example: + + ```python + >>> from PIL import Image + >>> import requests + >>> from transformers import AutoProcessor, Glm46VForConditionalGeneration + + >>> model = Glm46VForConditionalGeneration.from_pretrained("THUDM/GLM-4.1V-9B-Thinking") + >>> processor = AutoProcessor.from_pretrained("THUDM/GLM-4.1V-9B-Thinking") + + >>> messages = [ + { + "role": "user", + "content": [ + {"type": "image"}, + {"type": "text", "text": "What is shown in this image?"}, + ], + }, + ] + >>> url = "https://www.ilankelman.org/stopsigns/australia.jpg" + >>> image = Image.open(requests.get(url, stream=True).raw) + + >>> text = processor.apply_chat_template(messages, tokenize=False, add_generation_prompt=True) + >>> inputs = processor(text=[text], images=[image], vision_infos=[vision_infos]) + + >>> # Generate + >>> generate_ids = model.generate(inputs.input_ids, max_length=30) + >>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0] + "The image shows a street scene with a red stop sign in the foreground. In the background, there is a large red gate with Chinese characters ..." + ```""" + outputs = self.model( + input_ids=input_ids, + pixel_values=pixel_values, + pixel_values_videos=pixel_values_videos, + image_grid_thw=image_grid_thw, + video_grid_thw=video_grid_thw, + position_ids=position_ids, + attention_mask=attention_mask, + past_key_values=past_key_values, + inputs_embeds=inputs_embeds, + cache_position=cache_position, + **kwargs, + ) + + hidden_states = outputs[0] + + # Only compute necessary logits, and do not upcast them to float if we are not computing the loss + slice_indices = slice(-logits_to_keep, None) if isinstance(logits_to_keep, int) else logits_to_keep + logits = self.lm_head(hidden_states[:, slice_indices, :]) + + loss = None + if labels is not None: + loss = self.loss_function(logits=logits, labels=labels, vocab_size=self.config.text_config.vocab_size) + + return Glm46VCausalLMOutputWithPast( + loss=loss, + logits=logits, + past_key_values=outputs.past_key_values, + hidden_states=outputs.hidden_states, + attentions=outputs.attentions, + rope_deltas=outputs.rope_deltas, + ) + + def prepare_inputs_for_generation( + self, + input_ids, + past_key_values=None, + attention_mask=None, + inputs_embeds=None, + cache_position=None, + position_ids=None, + use_cache=True, + pixel_values=None, + pixel_values_videos=None, + image_grid_thw=None, + video_grid_thw=None, + **kwargs, + ): + # Overwritten -- in specific circumstances we don't want to forward image inputs to the model + + model_inputs = super().prepare_inputs_for_generation( + input_ids, + past_key_values=past_key_values, + attention_mask=attention_mask, + inputs_embeds=inputs_embeds, + cache_position=cache_position, + position_ids=position_ids, + pixel_values=pixel_values, + pixel_values_videos=pixel_values_videos, + image_grid_thw=image_grid_thw, + video_grid_thw=video_grid_thw, + use_cache=use_cache, + **kwargs, + ) + + # GLM-4.1V position_ids are prepareed with rope_deltas in forward + model_inputs["position_ids"] = None + + if cache_position[0] != 0: + model_inputs["pixel_values"] = None + model_inputs["pixel_values_videos"] = None + + return model_inputs + + def _get_image_nums_and_video_nums( + self, + input_ids: Optional[torch.LongTensor], + inputs_embeds: Optional[torch.Tensor] = None, + ) -> tuple[torch.Tensor, torch.Tensor]: + """ + Get the number of images and videos for each sample to calculate the separation length of the sample tensor. + These parameters are not passed through the processor to avoid unpredictable impacts from interface modifications. + + Args: + input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): + Indices of input sequence tokens in the vocabulary. + + Returns: + image_nums (`torch.LongTensor` of shape `(batch_size, num_images_sample)`) + video_nums (`torch.LongTensor` of shape `(batch_size, num_videos_sample)`) + """ + + if inputs_embeds is not None: + is_image = ( + inputs_embeds + == self.get_input_embeddings()( + torch.tensor(self.config.image_start_token_id, dtype=torch.long, device=inputs_embeds.device) + ) + )[..., 0] + is_video_start = ( + inputs_embeds + == self.get_input_embeddings()( + torch.tensor(self.config.video_start_token_id, dtype=torch.long, device=inputs_embeds.device) + ) + )[..., 0] + is_video_end = ( + inputs_embeds + == self.get_input_embeddings()( + torch.tensor(self.config.video_end_token_id, dtype=torch.long, device=inputs_embeds.device) + ) + )[..., 0] + else: + is_image = input_ids == self.config.image_start_token_id + is_video_start = input_ids == self.config.video_start_token_id + is_video_end = input_ids == self.config.video_end_token_id + + # Cumulative sum to track if we're inside a video span + # We'll assume well-formed video tags (i.e. matching starts and ends) + video_level = torch.cumsum(is_video_start.int() - is_video_end.int(), dim=1) + inside_video = video_level > 0 # shape (batch_size, seq_length) + + # Mask out image tokens that are inside video spans + standalone_images = is_image & (~inside_video) + + # Count per batch + image_counts = standalone_images.sum(dim=1) + video_counts = is_video_start.sum(dim=1) + + return image_counts, video_counts + + def _expand_inputs_for_generation( + self, + expand_size: int = 1, + is_encoder_decoder: bool = False, + input_ids: Optional[torch.LongTensor] = None, + **model_kwargs, + ) -> tuple[torch.LongTensor, dict[str, Any]]: + # Overwritten -- Support for expanding tensors without a batch size dimension + # e.g., pixel_values, image_grid_thw, pixel_values_videos, video_grid_thw, second_per_grid_t + # pixel_values.shape[0] is sum(seqlen_images for samples) + # image_grid_thw.shape[0] is sum(num_images for samples) + + if expand_size == 1: + return input_ids, model_kwargs + + visual_keys = ["pixel_values", "image_grid_thw", "pixel_values_videos", "video_grid_thw", "second_per_grid_ts"] + + def _expand_dict_for_generation_visual(dict_to_expand): + image_grid_thw = model_kwargs.get("image_grid_thw", None) + video_grid_thw = model_kwargs.get("video_grid_thw", None) + image_nums, video_nums = self._get_image_nums_and_video_nums( + input_ids, inputs_embeds=model_kwargs.get("inputs_embeds", None) + ) + + def _repeat_interleave_samples(x, lengths, repeat_times): + samples = torch.split(x, lengths) + repeat_args = [repeat_times] + [1] * (x.dim() - 1) + result = torch.cat([sample.repeat(*repeat_args) for sample in samples], dim=0) + return result + + for key in dict_to_expand: + if key == "pixel_values": + # split images into samples + samples = torch.split(image_grid_thw, list(image_nums)) + # compute the sequence length of images for each sample + lengths = [torch.prod(sample, dim=1).sum() for sample in samples] + dict_to_expand[key] = _repeat_interleave_samples( + dict_to_expand[key], lengths=lengths, repeat_times=expand_size + ) + elif key == "image_grid_thw": + # get the num of images for each sample + lengths = list(image_nums) + dict_to_expand[key] = _repeat_interleave_samples( + dict_to_expand[key], lengths=lengths, repeat_times=expand_size + ) + elif key == "pixel_values_videos": + samples = torch.split(video_grid_thw, list(video_nums)) + lengths = [torch.prod(sample, dim=1).sum() for sample in samples] + dict_to_expand[key] = _repeat_interleave_samples( + dict_to_expand[key], lengths=lengths, repeat_times=expand_size + ) + elif key == "video_grid_thw": + lengths = list(video_nums) + dict_to_expand[key] = _repeat_interleave_samples( + dict_to_expand[key], lengths=lengths, repeat_times=expand_size + ) + elif key == "second_per_grid_ts": + dict_to_expand[key] = _repeat_interleave_samples( + dict_to_expand[key], lengths=list(video_nums), repeat_times=expand_size + ) + return dict_to_expand + + def _expand_dict_for_generation(dict_to_expand): + for key in dict_to_expand: + if ( + key != "cache_position" + and dict_to_expand[key] is not None + and isinstance(dict_to_expand[key], torch.Tensor) + and key not in visual_keys + ): + dict_to_expand[key] = dict_to_expand[key].repeat_interleave(expand_size, dim=0) + return dict_to_expand + + model_kwargs = _expand_dict_for_generation_visual(model_kwargs) + + if input_ids is not None: + input_ids = input_ids.repeat_interleave(expand_size, dim=0) + + model_kwargs = _expand_dict_for_generation(model_kwargs) + + if is_encoder_decoder: + if model_kwargs.get("encoder_outputs") is None: + raise ValueError("If `is_encoder_decoder` is True, make sure that `encoder_outputs` is defined.") + model_kwargs["encoder_outputs"] = _expand_dict_for_generation(model_kwargs["encoder_outputs"]) + + return input_ids, model_kwargs + + +__all__ = ["Glm46VModel", "Glm46VPreTrainedModel", "Glm46VForConditionalGeneration"] diff --git a/src/transformers/models/glm46v/modular_glm46v.py b/src/transformers/models/glm46v/modular_glm46v.py index 9a5f53815189..c7ecffe208ba 100644 --- a/src/transformers/models/glm46v/modular_glm46v.py +++ b/src/transformers/models/glm46v/modular_glm46v.py @@ -22,7 +22,7 @@ from ..glm4v.configuration_glm4v import Glm4vConfig from ..glm4v.image_processing_glm4v import Glm4vImageProcessor from ..glm4v.image_processing_glm4v_fast import Glm4vImageProcessorFast -from ..glm4v.modeling_glm4v import Glm4vModel, Glm4vPreTrainedModel +from ..glm4v.modeling_glm4v import Glm4vModel, Glm4vPreTrainedModel, Glm4vForConditionalGeneration from ..glm4v.processing_glm4v import Glm4vProcessor from ..glm4v.video_processing_glm4v import Glm4vVideoProcessor @@ -38,8 +38,8 @@ def __init__( video_token_id=151344, image_start_token_id=151339, image_end_token_id=151340, - video_start_token_id=151361, - video_end_token_id=151362, + video_start_token_id=151341, + video_end_token_id=151342, **kwargs, ): if isinstance(vision_config, dict): @@ -76,6 +76,10 @@ def __init__(self, config): self.language_model = AutoModel.from_config(config.text_config) +class Glm46VForConditionalGeneration(Glm4vForConditionalGeneration): + pass + + class Glm46VProcessor(Glm4vProcessor): def replace_frame_token_id(self, timestamp_sec): return f"<|begin_of_image|>{self.image_token}<|end_of_image|>{timestamp_sec:.1f} seconds" @@ -158,9 +162,10 @@ def sample_frames( __all__ = [ - "Glm46VModel", "Glm46VConfig", + "Glm46VModel", "Glm46VPreTrainedModel", + "Glm46VForConditionalGeneration", "Glm46VProcessor", "Glm46VImageProcessor", "Glm46VImageProcessorFast", From c811264e520951ede3f8ba01cccd2973017f51c0 Mon Sep 17 00:00:00 2001 From: zRzRzRzRzRzRzR <2448370773@qq.com> Date: Thu, 13 Nov 2025 14:52:58 +0800 Subject: [PATCH 42/66] Update modular_glm46v.py --- src/transformers/models/glm46v/modular_glm46v.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/transformers/models/glm46v/modular_glm46v.py b/src/transformers/models/glm46v/modular_glm46v.py index c7ecffe208ba..efe60cea4722 100644 --- a/src/transformers/models/glm46v/modular_glm46v.py +++ b/src/transformers/models/glm46v/modular_glm46v.py @@ -22,7 +22,7 @@ from ..glm4v.configuration_glm4v import Glm4vConfig from ..glm4v.image_processing_glm4v import Glm4vImageProcessor from ..glm4v.image_processing_glm4v_fast import Glm4vImageProcessorFast -from ..glm4v.modeling_glm4v import Glm4vModel, Glm4vPreTrainedModel, Glm4vForConditionalGeneration +from ..glm4v.modeling_glm4v import Glm4vForConditionalGeneration, Glm4vModel, Glm4vPreTrainedModel from ..glm4v.processing_glm4v import Glm4vProcessor from ..glm4v.video_processing_glm4v import Glm4vVideoProcessor From dd4dc1f3da9eb3a9a6739b37150f27887d7c3a46 Mon Sep 17 00:00:00 2001 From: zRzRzRzRzRzRzR <2448370773@qq.com> Date: Thu, 13 Nov 2025 15:00:48 +0800 Subject: [PATCH 43/66] Update test_modeling_glm46v.py --- tests/models/glm46v/test_modeling_glm46v.py | 41 +++++++++++++++------ 1 file changed, 29 insertions(+), 12 deletions(-) diff --git a/tests/models/glm46v/test_modeling_glm46v.py b/tests/models/glm46v/test_modeling_glm46v.py index eeb9752eed95..6034a3dbde3d 100644 --- a/tests/models/glm46v/test_modeling_glm46v.py +++ b/tests/models/glm46v/test_modeling_glm46v.py @@ -1,5 +1,4 @@ -# coding=utf-8 -# Copyright 2025 the HuggingFace Team. All rights reserved. +# Copyright 2025 The HuggingFace Inc. team. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -17,7 +16,13 @@ import copy import unittest -from transformers import AutoModel, AutoProcessor, Glm46VConfig, Glm46VModel, is_torch_available +from transformers import ( + AutoProcessor, + Glm46VConfig, + Glm46VForConditionalGeneration, + Glm46VModel, + is_torch_available, +) from transformers.testing_utils import ( Expectations, cleanup, @@ -167,7 +172,7 @@ def prepare_config_and_inputs_for_common(self): @require_torch class Glm46VModelTest(ModelTesterMixin, GenerationTesterMixin, unittest.TestCase): - all_model_classes = (Glm46VModel) if is_torch_available() else () + all_model_classes = (Glm46VModel, Glm46VForConditionalGeneration) if is_torch_available() else () model_split_percents = [0.7, 0.9] # model too big to split at 0.5 _is_composite = True @@ -309,7 +314,9 @@ def tearDown(self): @slow def test_small_model_integration_test(self): - model = AutoModel.from_pretrained("THUDM/GLM-4.1V-9B-Thinking", dtype="auto", device_map="auto") + model = Glm46VForConditionalGeneration.from_pretrained( + "THUDM/GLM-4.1V-9B-Thinking", dtype="auto", device_map="auto" + ) inputs = self.processor.apply_chat_template( self.message, tokenize=True, add_generation_prompt=True, return_dict=True, return_tensors="pt" @@ -346,7 +353,9 @@ def test_small_model_integration_test(self): @slow def test_small_model_integration_test_batch(self): - model = AutoModel.from_pretrained("THUDM/GLM-4.1V-9B-Thinking", dtype="auto", device_map="auto") + model = Glm46VForConditionalGeneration.from_pretrained( + "THUDM/GLM-4.1V-9B-Thinking", dtype="auto", device_map="auto" + ) batch_messages = [self.message] * 2 inputs = self.processor.apply_chat_template( batch_messages, tokenize=True, add_generation_prompt=True, return_dict=True, return_tensors="pt" @@ -370,7 +379,9 @@ def test_small_model_integration_test_batch(self): @slow def test_small_model_integration_test_with_video(self): processor = AutoProcessor.from_pretrained("THUDM/GLM-4.1V-9B-Thinking", max_image_size={"longest_edge": 50176}) - model = AutoModel.from_pretrained("THUDM/GLM-4.1V-9B-Thinking", dtype=torch.float16, device_map="auto") + model = Glm46VForConditionalGeneration.from_pretrained( + "THUDM/GLM-4.1V-9B-Thinking", dtype=torch.float16, device_map="auto" + ) questions = ["Describe this video."] video_urls = ["https://huggingface.co/datasets/hf-internal-testing/fixtures_videos/resolve/main/tennis.mp4"] messages = [ @@ -406,7 +417,9 @@ def test_small_model_integration_test_with_video(self): @slow @require_deterministic_for_xpu def test_small_model_integration_test_expand(self): - model = AutoModel.from_pretrained("THUDM/GLM-4.1V-9B-Thinking", dtype="auto", device_map="auto") + model = Glm46VForConditionalGeneration.from_pretrained( + "THUDM/GLM-4.1V-9B-Thinking", dtype="auto", device_map="auto" + ) inputs = self.processor.apply_chat_template( self.message, tokenize=True, add_generation_prompt=True, return_dict=True, return_tensors="pt" ).to(torch_device) @@ -436,7 +449,9 @@ def test_small_model_integration_test_expand(self): @slow def test_small_model_integration_test_batch_wo_image(self): - model = AutoModel.from_pretrained("THUDM/GLM-4.1V-9B-Thinking", dtype="auto", device_map="auto") + model = Glm46VForConditionalGeneration.from_pretrained( + "THUDM/GLM-4.1V-9B-Thinking", dtype="auto", device_map="auto" + ) message_wo_image = [ {"role": "user", "content": [{"type": "text", "text": "Who are you?"}]}, ] @@ -467,7 +482,9 @@ def test_small_model_integration_test_batch_wo_image(self): @slow def test_small_model_integration_test_batch_different_resolutions(self): - model = AutoModel.from_pretrained("THUDM/GLM-4.1V-9B-Thinking", dtype="auto", device_map="auto") + model = Glm46VForConditionalGeneration.from_pretrained( + "THUDM/GLM-4.1V-9B-Thinking", dtype="auto", device_map="auto" + ) batched_messages = [self.message, self.message2] inputs = self.processor.apply_chat_template( batched_messages, @@ -497,7 +514,7 @@ def test_small_model_integration_test_batch_different_resolutions(self): @require_flash_attn @require_torch_gpu def test_small_model_integration_test_batch_flashatt2(self): - model = AutoModel.from_pretrained( + model = Glm46VForConditionalGeneration.from_pretrained( "THUDM/GLM-4.1V-9B-Thinking", dtype=torch.bfloat16, attn_implementation="flash_attention_2", @@ -532,7 +549,7 @@ def test_small_model_integration_test_batch_flashatt2(self): @require_flash_attn @require_torch_gpu def test_small_model_integration_test_batch_wo_image_flashatt2(self): - model = AutoModel.from_pretrained( + model = Glm46VForConditionalGeneration.from_pretrained( "THUDM/GLM-4.1V-9B-Thinking", dtype=torch.bfloat16, attn_implementation="flash_attention_2", From b79487d6f1c5d511293b32115e4e2d9b67560dcd Mon Sep 17 00:00:00 2001 From: zRzRzRzRzRzRzR <2448370773@qq.com> Date: Thu, 13 Nov 2025 15:19:00 +0800 Subject: [PATCH 44/66] update 1513 --- docs/source/en/model_doc/glm46v.md | 4 ++++ src/transformers/models/auto/modeling_auto.py | 1 + 2 files changed, 5 insertions(+) diff --git a/docs/source/en/model_doc/glm46v.md b/docs/source/en/model_doc/glm46v.md index 6d1a1af5cc03..6e099d7aaef6 100644 --- a/docs/source/en/model_doc/glm46v.md +++ b/docs/source/en/model_doc/glm46v.md @@ -28,3 +28,7 @@ [[autodoc]] Glm46VModel - forward +## Glm46VForConditionalGeneration + +[[autodoc]] Glm46VForConditionalGeneration + - forward diff --git a/src/transformers/models/auto/modeling_auto.py b/src/transformers/models/auto/modeling_auto.py index 6a0800fbe435..b9606471632e 100644 --- a/src/transformers/models/auto/modeling_auto.py +++ b/src/transformers/models/auto/modeling_auto.py @@ -1033,6 +1033,7 @@ class _BaseModelWithGenerate(PreTrainedModel, GenerationMixin): ("gemma3", "Gemma3ForConditionalGeneration"), ("gemma3n", "Gemma3nForConditionalGeneration"), ("git", "GitForCausalLM"), + ("glm46v", "Glm46VForConditionalGeneration"), ("glm4v", "Glm4vForConditionalGeneration"), ("glm4v_moe", "Glm4vMoeForConditionalGeneration"), ("got_ocr2", "GotOcr2ForConditionalGeneration"), From e5b4a6de077fdffc3b7242dc553d0f2b3b25c9f5 Mon Sep 17 00:00:00 2001 From: zRzRzRzRzRzRzR <2448370773@qq.com> Date: Thu, 13 Nov 2025 15:34:58 +0800 Subject: [PATCH 45/66] 1 --- src/transformers/models/glm46v/configuration_glm46v.py | 2 +- src/transformers/models/glm46v/modular_glm46v.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/transformers/models/glm46v/configuration_glm46v.py b/src/transformers/models/glm46v/configuration_glm46v.py index 729ce03478c3..c0991997c0ba 100644 --- a/src/transformers/models/glm46v/configuration_glm46v.py +++ b/src/transformers/models/glm46v/configuration_glm46v.py @@ -83,7 +83,7 @@ def __init__( ): if isinstance(vision_config, dict): vision_config["model_type"] = vision_config.get("model_type", "glm4v") - self.vision_config = CONFIG_MAPPING["vision_config"](**vision_config) + self.vision_config = CONFIG_MAPPING[vision_config["model_type"]](**vision_config) elif vision_config is None: self.vision_config = CONFIG_MAPPING["glm4v"]() diff --git a/src/transformers/models/glm46v/modular_glm46v.py b/src/transformers/models/glm46v/modular_glm46v.py index efe60cea4722..c056e1e10ba0 100644 --- a/src/transformers/models/glm46v/modular_glm46v.py +++ b/src/transformers/models/glm46v/modular_glm46v.py @@ -44,7 +44,7 @@ def __init__( ): if isinstance(vision_config, dict): vision_config["model_type"] = vision_config.get("model_type", "glm4v") - self.vision_config = CONFIG_MAPPING["vision_config"](**vision_config) + self.vision_config = CONFIG_MAPPING[vision_config["model_type"]](**vision_config) elif vision_config is None: self.vision_config = CONFIG_MAPPING["glm4v"]() From 325216e91ea79bf01111d9a51317e2487260d159 Mon Sep 17 00:00:00 2001 From: zRzRzRzRzRzRzR <2448370773@qq.com> Date: Thu, 13 Nov 2025 16:04:41 +0800 Subject: [PATCH 46/66] use PreTrainedConfig --- .../models/glm46v/configuration_glm46v.py | 45 +------------------ .../models/glm46v/modular_glm46v.py | 5 ++- 2 files changed, 5 insertions(+), 45 deletions(-) diff --git a/src/transformers/models/glm46v/configuration_glm46v.py b/src/transformers/models/glm46v/configuration_glm46v.py index c0991997c0ba..91defa63c4e2 100644 --- a/src/transformers/models/glm46v/configuration_glm46v.py +++ b/src/transformers/models/glm46v/configuration_glm46v.py @@ -24,50 +24,7 @@ class Glm46VConfig(PreTrainedConfig): - r""" - This is the configuration class to store the configuration of a [`Glm46VModel`]. It is used to instantiate a - GLM-4.1V model according to the specified arguments, defining the model architecture. Instantiating a - configuration with the defaults will yield a similar configuration to that of - GLM-4.1V-9B-Thinking [THUDM/GLM-4.1V-9B-Thinking](https://huggingface.co/THUDM/GLM-4.1V-9B-Thinking). - - Configuration objects inherit from [`PreTrainedConfig`] and can be used to control the model outputs. Read the - documentation from [`PreTrainedConfig`] for more information. - - - Args: - text_config (`Union[PreTrainedConfig, dict]`, *optional*, defaults to `Glm46VTextConfig`): - The config object or dictionary of the text backbone. - vision_config (`Union[PreTrainedConfig, dict]`, *optional*, defaults to `Glm46VVisionConfig`): - The config object or dictionary of the vision backbone. - image_token_id (`int`, *optional*, defaults to 151343): - The image token index to encode the image prompt. - video_token_id (`int`, *optional*, defaults to 151344): - The video token index to encode the image prompt. - image_start_token_id (`int`, *optional*, defaults to 151339): - The image start token index to encode the start of image. - image_end_token_id (`int`, *optional*, defaults to 151340): - The image end token index to encode the end of image. - video_start_token_id (`int`, *optional*, defaults to 151341): - The video start token index to encode the start of video. - video_end_token_id (`int`, *optional*, defaults to 151342): - The video end token index to encode the end of video. - - ```python - >>> from transformers import Glm46VForConditionalGeneration, Glm46VConfig - - >>> # Initializing a GLM-4.1V style configuration - >>> configuration = Glm46VConfig() - - >>> # Initializing a model from the GLM-4.1V style configuration - >>> model = Glm46VForConditionalGeneration(configuration) - - >>> # Accessing the model configuration - >>> configuration = model.config - ```""" - - model_type = "glm46v" sub_configs = {"text_config": AutoConfig, "vision_config": AutoConfig} - keys_to_ignore_at_inference = ["past_key_values"] def __init__( self, @@ -100,5 +57,7 @@ def __init__( self.image_start_token_id = image_start_token_id self.image_end_token_id = image_end_token_id + super().__init__(**kwargs) + __all__ = ["Glm46VConfig"] diff --git a/src/transformers/models/glm46v/modular_glm46v.py b/src/transformers/models/glm46v/modular_glm46v.py index c056e1e10ba0..825b10f774b7 100644 --- a/src/transformers/models/glm46v/modular_glm46v.py +++ b/src/transformers/models/glm46v/modular_glm46v.py @@ -19,7 +19,7 @@ from ...video_utils import VideoMetadata from ..auto import CONFIG_MAPPING, AutoConfig, AutoModel -from ..glm4v.configuration_glm4v import Glm4vConfig +from ...configuration_utils import PreTrainedConfig from ..glm4v.image_processing_glm4v import Glm4vImageProcessor from ..glm4v.image_processing_glm4v_fast import Glm4vImageProcessorFast from ..glm4v.modeling_glm4v import Glm4vForConditionalGeneration, Glm4vModel, Glm4vPreTrainedModel @@ -27,7 +27,7 @@ from ..glm4v.video_processing_glm4v import Glm4vVideoProcessor -class Glm46VConfig(Glm4vConfig): +class Glm46VConfig(PreTrainedConfig): sub_configs = {"text_config": AutoConfig, "vision_config": AutoConfig} def __init__( @@ -61,6 +61,7 @@ def __init__( self.image_start_token_id = image_start_token_id self.image_end_token_id = image_end_token_id + super().__init__(**kwargs) class Glm46VPreTrainedModel(Glm4vPreTrainedModel): _can_record_outputs = None From b08437d40b3b237fd3bbce4f310a64d7d1af94e7 Mon Sep 17 00:00:00 2001 From: zRzRzRzRzRzRzR <2448370773@qq.com> Date: Thu, 13 Nov 2025 16:08:58 +0800 Subject: [PATCH 47/66] Update modular_glm46v.py --- src/transformers/models/glm46v/modular_glm46v.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/transformers/models/glm46v/modular_glm46v.py b/src/transformers/models/glm46v/modular_glm46v.py index 825b10f774b7..a2e1067cf4a6 100644 --- a/src/transformers/models/glm46v/modular_glm46v.py +++ b/src/transformers/models/glm46v/modular_glm46v.py @@ -17,9 +17,9 @@ import numpy as np +from ...configuration_utils import PreTrainedConfig from ...video_utils import VideoMetadata from ..auto import CONFIG_MAPPING, AutoConfig, AutoModel -from ...configuration_utils import PreTrainedConfig from ..glm4v.image_processing_glm4v import Glm4vImageProcessor from ..glm4v.image_processing_glm4v_fast import Glm4vImageProcessorFast from ..glm4v.modeling_glm4v import Glm4vForConditionalGeneration, Glm4vModel, Glm4vPreTrainedModel @@ -63,6 +63,7 @@ def __init__( super().__init__(**kwargs) + class Glm46VPreTrainedModel(Glm4vPreTrainedModel): _can_record_outputs = None _no_split_modules = None From 87f18873bcfe827b594ceb2b3116ac419e531860 Mon Sep 17 00:00:00 2001 From: zRzRzRzRzRzRzR <2448370773@qq.com> Date: Thu, 13 Nov 2025 16:17:43 +0800 Subject: [PATCH 48/66] Update configuration_glm46v.py --- src/transformers/models/glm46v/configuration_glm46v.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/transformers/models/glm46v/configuration_glm46v.py b/src/transformers/models/glm46v/configuration_glm46v.py index 91defa63c4e2..4904a9bfd1a8 100644 --- a/src/transformers/models/glm46v/configuration_glm46v.py +++ b/src/transformers/models/glm46v/configuration_glm46v.py @@ -19,6 +19,7 @@ # See the License for the specific language governing permissions and # limitations under the License. + from ...configuration_utils import PreTrainedConfig from ..auto import CONFIG_MAPPING, AutoConfig From ed88670634ab675c9b40ebf67040f9685809b6af Mon Sep 17 00:00:00 2001 From: zRzRzRzRzRzRzR <2448370773@qq.com> Date: Thu, 13 Nov 2025 17:19:34 +0800 Subject: [PATCH 49/66] model_type = "glm46v" --- src/transformers/models/glm46v/configuration_glm46v.py | 2 ++ src/transformers/models/glm46v/modular_glm46v.py | 2 ++ 2 files changed, 4 insertions(+) diff --git a/src/transformers/models/glm46v/configuration_glm46v.py b/src/transformers/models/glm46v/configuration_glm46v.py index 4904a9bfd1a8..259bd372e593 100644 --- a/src/transformers/models/glm46v/configuration_glm46v.py +++ b/src/transformers/models/glm46v/configuration_glm46v.py @@ -25,7 +25,9 @@ class Glm46VConfig(PreTrainedConfig): + model_type = "glm46v" sub_configs = {"text_config": AutoConfig, "vision_config": AutoConfig} + keys_to_ignore_at_inference = ["past_key_values"] def __init__( self, diff --git a/src/transformers/models/glm46v/modular_glm46v.py b/src/transformers/models/glm46v/modular_glm46v.py index a2e1067cf4a6..ed37821848f4 100644 --- a/src/transformers/models/glm46v/modular_glm46v.py +++ b/src/transformers/models/glm46v/modular_glm46v.py @@ -28,7 +28,9 @@ class Glm46VConfig(PreTrainedConfig): + model_type = "glm46v" sub_configs = {"text_config": AutoConfig, "vision_config": AutoConfig} + keys_to_ignore_at_inference = ["past_key_values"] def __init__( self, From 25b36eb386c773fdf4490d71d6b7355b5afe8282 Mon Sep 17 00:00:00 2001 From: zRzRzRzRzRzRzR <2448370773@qq.com> Date: Thu, 13 Nov 2025 17:26:33 +0800 Subject: [PATCH 50/66] remove glm46v_text --- src/transformers/models/auto/configuration_auto.py | 2 -- utils/models_to_deprecate.py | 1 - 2 files changed, 3 deletions(-) diff --git a/src/transformers/models/auto/configuration_auto.py b/src/transformers/models/auto/configuration_auto.py index d3b08a150017..e8e3ddf69f31 100644 --- a/src/transformers/models/auto/configuration_auto.py +++ b/src/transformers/models/auto/configuration_auto.py @@ -622,7 +622,6 @@ ("glm", "GLM"), ("glm4", "GLM4"), ("glm46v", "Glm46V"), - ("glm46v_text", "GLM46V"), ("glm4_moe", "Glm4MoE"), ("glm4v", "GLM4V"), ("glm4v_moe", "GLM4VMOE"), @@ -987,7 +986,6 @@ ("gemma3n_text", "gemma3n"), ("gemma3n_vision", "gemma3n"), ("glm4v_text", "glm4v"), - ("glm46v_text", "glm46v"), ("glm4v_moe_text", "glm4v_moe"), ("idefics3_vision", "idefics3"), ("siglip_vision_model", "siglip"), diff --git a/utils/models_to_deprecate.py b/utils/models_to_deprecate.py index 1cc31eea29e2..c3c4eaf8e4d8 100644 --- a/utils/models_to_deprecate.py +++ b/utils/models_to_deprecate.py @@ -94,7 +94,6 @@ "gpt2": ["cpm", "dialogpt", "gpt-sw3", "megatron_gpt2"], "glm4v_moe": ["glm4v_moe_text"], "glm4v": ["glm4v_text"], - "glm46v": ["glm46v_text"], "idefics3": ["idefics3_vision"], "internvl": ["internvl_vision"], "layoutlmv2": ["layoutxlm"], From 733d27b1f92985f20e986526badb87579ea499bc Mon Sep 17 00:00:00 2001 From: zRzRzRzRzRzRzR <2448370773@qq.com> Date: Thu, 13 Nov 2025 17:28:34 +0800 Subject: [PATCH 51/66] Update image_processing_auto.py --- src/transformers/models/auto/image_processing_auto.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/transformers/models/auto/image_processing_auto.py b/src/transformers/models/auto/image_processing_auto.py index 5ed14d4f3b1d..7739ecc7961b 100644 --- a/src/transformers/models/auto/image_processing_auto.py +++ b/src/transformers/models/auto/image_processing_auto.py @@ -108,6 +108,7 @@ ("gemma3", ("Gemma3ImageProcessor", "Gemma3ImageProcessorFast")), ("gemma3n", ("SiglipImageProcessor", "SiglipImageProcessorFast")), ("git", ("CLIPImageProcessor", "CLIPImageProcessorFast")), + ("glm46v", ("Glm46VImageProcessor", "Glm46VImageProcessorFast")), ("glm4v", ("Glm4vImageProcessor", "Glm4vImageProcessorFast")), ("glpn", ("GLPNImageProcessor", "GLPNImageProcessorFast")), ("got_ocr2", ("GotOcr2ImageProcessor", "GotOcr2ImageProcessorFast")), From ee29a2b19d03f888cc699cfd2d7ab5c77d0a2802 Mon Sep 17 00:00:00 2001 From: zRzRzRzRzRzRzR <2448370773@qq.com> Date: Thu, 13 Nov 2025 17:30:40 +0800 Subject: [PATCH 52/66] 1 --- src/transformers/models/glm46v/__init__.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/transformers/models/glm46v/__init__.py b/src/transformers/models/glm46v/__init__.py index 9a25df25e1af..fd8354648ce7 100644 --- a/src/transformers/models/glm46v/__init__.py +++ b/src/transformers/models/glm46v/__init__.py @@ -21,6 +21,9 @@ from .configuration_glm46v import * from .modeling_glm46v import * from .processing_glm46v import * + from .image_processing_glm46v import * + from .image_processing_glm46v_fast import * + from .video_processing_glm46v import * else: import sys From 6dac0c47c2e261b93e731d9e8fc931686448b760 Mon Sep 17 00:00:00 2001 From: zRzRzRzRzRzRzR <2448370773@qq.com> Date: Thu, 13 Nov 2025 17:34:59 +0800 Subject: [PATCH 53/66] update readme --- src/transformers/models/glm46v/__init__.py | 4 +- .../models/glm46v/configuration_glm46v.py | 42 ++++++++++++++++++- .../models/glm46v/modular_glm46v.py | 42 ++++++++++++++++++- 3 files changed, 82 insertions(+), 6 deletions(-) diff --git a/src/transformers/models/glm46v/__init__.py b/src/transformers/models/glm46v/__init__.py index fd8354648ce7..c5bb0c40f19d 100644 --- a/src/transformers/models/glm46v/__init__.py +++ b/src/transformers/models/glm46v/__init__.py @@ -19,10 +19,10 @@ if TYPE_CHECKING: from .configuration_glm46v import * - from .modeling_glm46v import * - from .processing_glm46v import * from .image_processing_glm46v import * from .image_processing_glm46v_fast import * + from .modeling_glm46v import * + from .processing_glm46v import * from .video_processing_glm46v import * else: import sys diff --git a/src/transformers/models/glm46v/configuration_glm46v.py b/src/transformers/models/glm46v/configuration_glm46v.py index 259bd372e593..1a1f615a700c 100644 --- a/src/transformers/models/glm46v/configuration_glm46v.py +++ b/src/transformers/models/glm46v/configuration_glm46v.py @@ -25,6 +25,44 @@ class Glm46VConfig(PreTrainedConfig): + r""" + This is the configuration class to store the configuration of a [`Glm4vModel`]. It is used to instantiate a + GLM-4.6V model according to the specified arguments, defining the model architecture. + + Configuration objects inherit from [`PreTrainedConfig`] and can be used to control the model outputs. Read the + documentation from [`PreTrainedConfig`] for more information. + + Args: + text_config (`Union[PreTrainedConfig, dict]`, *optional*, defaults to `Glm4vTextConfig`): + The config object or dictionary of the text backbone. + vision_config (`Union[PreTrainedConfig, dict]`, *optional*, defaults to `Glm4vVisionConfig`): + The config object or dictionary of the vision backbone. + image_token_id (`int`, *optional*, defaults to 151343): + The image token index to encode the image prompt. + video_token_id (`int`, *optional*, defaults to 151344): + The video token index to encode the image prompt. + image_start_token_id (`int`, *optional*, defaults to 151339): + The image start token index to encode the start of image. + image_end_token_id (`int`, *optional*, defaults to 151340): + The image end token index to encode the end of image. + video_start_token_id (`int`, *optional*, defaults to 151361): + The video start token index to encode the start of video. + video_end_token_id (`int`, *optional*, defaults to 151362): + The video end token index to encode the end of video. + + ```python + >>> from transformers import Glm46VForConditionalGeneration, Glm46VConfig + + >>> # Initializing a GLM-4.6V style configuration + >>> configuration = Glm46VConfig() + + >>> # Initializing a model from the GLM-4.6V style configuration + >>> model = Glm4vForConditionalGeneration(configuration) + + >>> # Accessing the model configuration + >>> configuration = model.config + ```""" + model_type = "glm46v" sub_configs = {"text_config": AutoConfig, "vision_config": AutoConfig} keys_to_ignore_at_inference = ["past_key_values"] @@ -37,8 +75,8 @@ def __init__( video_token_id=151344, image_start_token_id=151339, image_end_token_id=151340, - video_start_token_id=151341, - video_end_token_id=151342, + video_start_token_id=151361, + video_end_token_id=151362, **kwargs, ): if isinstance(vision_config, dict): diff --git a/src/transformers/models/glm46v/modular_glm46v.py b/src/transformers/models/glm46v/modular_glm46v.py index ed37821848f4..7903a2dae5a2 100644 --- a/src/transformers/models/glm46v/modular_glm46v.py +++ b/src/transformers/models/glm46v/modular_glm46v.py @@ -28,6 +28,44 @@ class Glm46VConfig(PreTrainedConfig): + r""" + This is the configuration class to store the configuration of a [`Glm4vModel`]. It is used to instantiate a + GLM-4.6V model according to the specified arguments, defining the model architecture. + + Configuration objects inherit from [`PreTrainedConfig`] and can be used to control the model outputs. Read the + documentation from [`PreTrainedConfig`] for more information. + + Args: + text_config (`Union[PreTrainedConfig, dict]`, *optional*, defaults to `Glm4vTextConfig`): + The config object or dictionary of the text backbone. + vision_config (`Union[PreTrainedConfig, dict]`, *optional*, defaults to `Glm4vVisionConfig`): + The config object or dictionary of the vision backbone. + image_token_id (`int`, *optional*, defaults to 151343): + The image token index to encode the image prompt. + video_token_id (`int`, *optional*, defaults to 151344): + The video token index to encode the image prompt. + image_start_token_id (`int`, *optional*, defaults to 151339): + The image start token index to encode the start of image. + image_end_token_id (`int`, *optional*, defaults to 151340): + The image end token index to encode the end of image. + video_start_token_id (`int`, *optional*, defaults to 151361): + The video start token index to encode the start of video. + video_end_token_id (`int`, *optional*, defaults to 151362): + The video end token index to encode the end of video. + + ```python + >>> from transformers import Glm46VForConditionalGeneration, Glm46VConfig + + >>> # Initializing a GLM-4.6V style configuration + >>> configuration = Glm46VConfig() + + >>> # Initializing a model from the GLM-4.6V style configuration + >>> model = Glm4vForConditionalGeneration(configuration) + + >>> # Accessing the model configuration + >>> configuration = model.config + ```""" + model_type = "glm46v" sub_configs = {"text_config": AutoConfig, "vision_config": AutoConfig} keys_to_ignore_at_inference = ["past_key_values"] @@ -40,8 +78,8 @@ def __init__( video_token_id=151344, image_start_token_id=151339, image_end_token_id=151340, - video_start_token_id=151341, - video_end_token_id=151342, + video_start_token_id=151361, + video_end_token_id=151362, **kwargs, ): if isinstance(vision_config, dict): From 3910b8a4779cabbe6a07e87c148df9385453e855 Mon Sep 17 00:00:00 2001 From: zRzRzRzRzRzRzR <2448370773@qq.com> Date: Thu, 13 Nov 2025 17:40:23 +0800 Subject: [PATCH 54/66] GLM-4.6V --- src/transformers/models/glm46v/configuration_glm46v.py | 4 +++- src/transformers/models/glm46v/modular_glm46v.py | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/src/transformers/models/glm46v/configuration_glm46v.py b/src/transformers/models/glm46v/configuration_glm46v.py index 1a1f615a700c..2d427fd76cc5 100644 --- a/src/transformers/models/glm46v/configuration_glm46v.py +++ b/src/transformers/models/glm46v/configuration_glm46v.py @@ -27,7 +27,9 @@ class Glm46VConfig(PreTrainedConfig): r""" This is the configuration class to store the configuration of a [`Glm4vModel`]. It is used to instantiate a - GLM-4.6V model according to the specified arguments, defining the model architecture. + GLM-4.6V model according to the specified arguments, defining the model architecture. Instantiating a + configuration with the defaults will yield a similar configuration to that of + GLM-4.1V-9B-Thinking [zai-org/GLM-4.1V-9B-Thinking](https://huggingface.co/zai-org/GLM-4.1V-9B-Thinking). Configuration objects inherit from [`PreTrainedConfig`] and can be used to control the model outputs. Read the documentation from [`PreTrainedConfig`] for more information. diff --git a/src/transformers/models/glm46v/modular_glm46v.py b/src/transformers/models/glm46v/modular_glm46v.py index 7903a2dae5a2..81cc4264c14b 100644 --- a/src/transformers/models/glm46v/modular_glm46v.py +++ b/src/transformers/models/glm46v/modular_glm46v.py @@ -30,7 +30,9 @@ class Glm46VConfig(PreTrainedConfig): r""" This is the configuration class to store the configuration of a [`Glm4vModel`]. It is used to instantiate a - GLM-4.6V model according to the specified arguments, defining the model architecture. + GLM-4.6V model according to the specified arguments, defining the model architecture. Instantiating a + configuration with the defaults will yield a similar configuration to that of + GLM-4.1V-9B-Thinking [zai-org/GLM-4.1V-9B-Thinking](https://huggingface.co/zai-org/GLM-4.1V-9B-Thinking). Configuration objects inherit from [`PreTrainedConfig`] and can be used to control the model outputs. Read the documentation from [`PreTrainedConfig`] for more information. From 7c160f0bc5dee2b618b81af65164dfabbf561637 Mon Sep 17 00:00:00 2001 From: zRzRzRzRzRzRzR <2448370773@qq.com> Date: Thu, 13 Nov 2025 18:57:31 +0800 Subject: [PATCH 55/66] update --- src/transformers/models/auto/configuration_auto.py | 6 ++++++ src/transformers/models/auto/modeling_auto.py | 2 ++ src/transformers/models/glm46v/configuration_glm46v.py | 4 ++-- src/transformers/models/glm46v/modular_glm46v.py | 4 ++-- src/transformers/models/glm4v/__init__.py | 2 ++ src/transformers/models/glm4v/configuration_glm4v.py | 2 +- .../models/glm4v/convert_glm4v_mgt_weights_to_hf.py | 1 + src/transformers/models/glm4v/modular_glm4v.py | 2 +- .../models/glm4v_moe/configuration_glm4v_moe.py | 2 +- utils/models_to_deprecate.py | 4 ++-- 10 files changed, 20 insertions(+), 9 deletions(-) diff --git a/src/transformers/models/auto/configuration_auto.py b/src/transformers/models/auto/configuration_auto.py index e8e3ddf69f31..e17a41263504 100644 --- a/src/transformers/models/auto/configuration_auto.py +++ b/src/transformers/models/auto/configuration_auto.py @@ -177,7 +177,9 @@ ("glm4v", "Glm4vConfig"), ("glm4v_moe", "Glm4vMoeConfig"), ("glm4v_moe_text", "Glm4vMoeTextConfig"), + ("glm4v_moe_vision", "Glm4vMoeVisionConfig"), ("glm4v_text", "Glm4vTextConfig"), + ("glm4v_vision", "Glm4vVisionConfig"), ("glpn", "GLPNConfig"), ("got_ocr2", "GotOcr2Config"), ("gpt-sw3", "GPT2Config"), @@ -626,7 +628,9 @@ ("glm4v", "GLM4V"), ("glm4v_moe", "GLM4VMOE"), ("glm4v_moe_text", "GLM4VMOE"), + ("glm4v_moe_vision", "Glm4vMoeVisionModel"), ("glm4v_text", "GLM4V"), + ("glm4v_vision", "Glm4vVisionModel"), ("glpn", "GLPN"), ("got_ocr2", "GOT-OCR2"), ("gpt-sw3", "GPT-Sw3"), @@ -985,6 +989,8 @@ ("gemma3n_audio", "gemma3n"), ("gemma3n_text", "gemma3n"), ("gemma3n_vision", "gemma3n"), + ("glm4v_vision", "glm4v"), + ("glm4v_moe_vision", "glm4v_moe"), ("glm4v_text", "glm4v"), ("glm4v_moe_text", "glm4v_moe"), ("idefics3_vision", "idefics3"), diff --git a/src/transformers/models/auto/modeling_auto.py b/src/transformers/models/auto/modeling_auto.py index b9606471632e..acdf6c9db280 100644 --- a/src/transformers/models/auto/modeling_auto.py +++ b/src/transformers/models/auto/modeling_auto.py @@ -180,7 +180,9 @@ class _BaseModelWithGenerate(PreTrainedModel, GenerationMixin): ("glm4v", "Glm4vModel"), ("glm4v_moe", "Glm4vMoeModel"), ("glm4v_moe_text", "Glm4vMoeTextModel"), + ("glm4v_moe_vision", "Glm4vMoeVisionModel"), ("glm4v_text", "Glm4vTextModel"), + ("glm4v_vision", "Glm4vVisionModel"), ("glpn", "GLPNModel"), ("got_ocr2", "GotOcr2Model"), ("gpt-sw3", "GPT2Model"), diff --git a/src/transformers/models/glm46v/configuration_glm46v.py b/src/transformers/models/glm46v/configuration_glm46v.py index 2d427fd76cc5..125019940f13 100644 --- a/src/transformers/models/glm46v/configuration_glm46v.py +++ b/src/transformers/models/glm46v/configuration_glm46v.py @@ -82,10 +82,10 @@ def __init__( **kwargs, ): if isinstance(vision_config, dict): - vision_config["model_type"] = vision_config.get("model_type", "glm4v") + vision_config["model_type"] = vision_config.get("model_type", "glm4v_vision") self.vision_config = CONFIG_MAPPING[vision_config["model_type"]](**vision_config) elif vision_config is None: - self.vision_config = CONFIG_MAPPING["glm4v"]() + self.vision_config = CONFIG_MAPPING["glm4v_vision"]() if isinstance(text_config, dict): text_config["model_type"] = text_config.get("model_type", "glm4v_text") diff --git a/src/transformers/models/glm46v/modular_glm46v.py b/src/transformers/models/glm46v/modular_glm46v.py index 81cc4264c14b..0bb86b0aacf7 100644 --- a/src/transformers/models/glm46v/modular_glm46v.py +++ b/src/transformers/models/glm46v/modular_glm46v.py @@ -85,10 +85,10 @@ def __init__( **kwargs, ): if isinstance(vision_config, dict): - vision_config["model_type"] = vision_config.get("model_type", "glm4v") + vision_config["model_type"] = vision_config.get("model_type", "glm4v_vision") self.vision_config = CONFIG_MAPPING[vision_config["model_type"]](**vision_config) elif vision_config is None: - self.vision_config = CONFIG_MAPPING["glm4v"]() + self.vision_config = CONFIG_MAPPING["glm4v_vision"]() if isinstance(text_config, dict): text_config["model_type"] = text_config.get("model_type", "glm4v_text") diff --git a/src/transformers/models/glm4v/__init__.py b/src/transformers/models/glm4v/__init__.py index 4216c137fbe2..126b3a768789 100644 --- a/src/transformers/models/glm4v/__init__.py +++ b/src/transformers/models/glm4v/__init__.py @@ -21,6 +21,8 @@ from .configuration_glm4v import * from .modeling_glm4v import * from .processing_glm4v import * + from .image_processing_glm4v import * + from .video_processing_glm4v import * else: import sys diff --git a/src/transformers/models/glm4v/configuration_glm4v.py b/src/transformers/models/glm4v/configuration_glm4v.py index c8f2ef75ca71..28240695f1f5 100644 --- a/src/transformers/models/glm4v/configuration_glm4v.py +++ b/src/transformers/models/glm4v/configuration_glm4v.py @@ -80,7 +80,7 @@ class Glm4vVisionConfig(PreTrainedConfig): >>> configuration = model.config ```""" - model_type = "glm4v" + model_type = "glm4v_vision" base_config_key = "vision_config" def __init__( diff --git a/src/transformers/models/glm4v/convert_glm4v_mgt_weights_to_hf.py b/src/transformers/models/glm4v/convert_glm4v_mgt_weights_to_hf.py index 722dab5759fc..fb57f66a9ae0 100644 --- a/src/transformers/models/glm4v/convert_glm4v_mgt_weights_to_hf.py +++ b/src/transformers/models/glm4v/convert_glm4v_mgt_weights_to_hf.py @@ -710,6 +710,7 @@ def offset_layer(x, offset=llm_layer_offset): if "vision_config" in model_config: vision_config = { + "model_type": "glm4v_vision", "hidden_size": model_config["vision_config"].get("hidden_size", 1536), "depth": model_config["vision_config"].get("num_layers", 24), "num_heads": model_config["vision_config"].get("num_attention_heads", 12), diff --git a/src/transformers/models/glm4v/modular_glm4v.py b/src/transformers/models/glm4v/modular_glm4v.py index 2d96b8fd2f84..d2d0e9acfedd 100644 --- a/src/transformers/models/glm4v/modular_glm4v.py +++ b/src/transformers/models/glm4v/modular_glm4v.py @@ -117,7 +117,7 @@ class Glm4vVisionConfig(PreTrainedConfig): >>> configuration = model.config ```""" - model_type = "glm4v" + model_type = "glm4v_vision" base_config_key = "vision_config" def __init__( diff --git a/src/transformers/models/glm4v_moe/configuration_glm4v_moe.py b/src/transformers/models/glm4v_moe/configuration_glm4v_moe.py index 05a9a58089dd..d0b1cab8f7ba 100644 --- a/src/transformers/models/glm4v_moe/configuration_glm4v_moe.py +++ b/src/transformers/models/glm4v_moe/configuration_glm4v_moe.py @@ -80,7 +80,7 @@ class Glm4vMoeVisionConfig(PreTrainedConfig): >>> configuration = model.config ```""" - model_type = "glm4v_moe" + model_type = "glm4v_moe_vision" base_config_key = "vision_config" def __init__( diff --git a/utils/models_to_deprecate.py b/utils/models_to_deprecate.py index c3c4eaf8e4d8..14565904d7e3 100644 --- a/utils/models_to_deprecate.py +++ b/utils/models_to_deprecate.py @@ -92,8 +92,8 @@ "gemma3": ["gemma3_text"], "gemma3n": ["gemma3n_audio", "gemma3n_text", "gemma3n_vision"], "gpt2": ["cpm", "dialogpt", "gpt-sw3", "megatron_gpt2"], - "glm4v_moe": ["glm4v_moe_text"], - "glm4v": ["glm4v_text"], + "glm4v_moe": ["glm4v_moe_text", "glm4v_moe_vision"], + "glm4v": ["glm4v_text", "glm4v_vision"], "idefics3": ["idefics3_vision"], "internvl": ["internvl_vision"], "layoutlmv2": ["layoutxlm"], From b0654d9f4a6daddd6cb615cdba1cc60ae941fc72 Mon Sep 17 00:00:00 2001 From: zRzRzRzRzRzRzR <2448370773@qq.com> Date: Thu, 13 Nov 2025 19:02:56 +0800 Subject: [PATCH 56/66] update --- src/transformers/models/glm4v_moe/configuration_glm4v_moe.py | 2 +- .../models/glm4v_moe/convert_glm4v_moe_mgt_weights_to_hf.py | 2 +- src/transformers/models/glm4v_moe/modular_glm4v_moe.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/transformers/models/glm4v_moe/configuration_glm4v_moe.py b/src/transformers/models/glm4v_moe/configuration_glm4v_moe.py index d0b1cab8f7ba..12a059f46b6e 100644 --- a/src/transformers/models/glm4v_moe/configuration_glm4v_moe.py +++ b/src/transformers/models/glm4v_moe/configuration_glm4v_moe.py @@ -206,7 +206,7 @@ class Glm4vMoeTextConfig(PreTrainedConfig): >>> configuration = model.config ```""" - model_type = "Glm4vMoe_text" + model_type = "glm4v_moe_text" keys_to_ignore_at_inference = ["past_key_values"] # Default tensor parallel plan for base model `Glm4vMoe` base_model_tp_plan = { diff --git a/src/transformers/models/glm4v_moe/convert_glm4v_moe_mgt_weights_to_hf.py b/src/transformers/models/glm4v_moe/convert_glm4v_moe_mgt_weights_to_hf.py index 1949966d2738..d8b08716b6c4 100644 --- a/src/transformers/models/glm4v_moe/convert_glm4v_moe_mgt_weights_to_hf.py +++ b/src/transformers/models/glm4v_moe/convert_glm4v_moe_mgt_weights_to_hf.py @@ -713,7 +713,7 @@ def offset_layer(x, offset=llm_layer_offset): if "vision_config" in model_config: vision_config = { - "model_type": "glm4v_moe", + "model_type": "glm4v_moe_vision", "hidden_size": model_config["vision_config"].get("hidden_size", 1536), "depth": model_config["vision_config"].get("num_layers", 24), "num_heads": model_config["vision_config"].get("num_attention_heads", 12), diff --git a/src/transformers/models/glm4v_moe/modular_glm4v_moe.py b/src/transformers/models/glm4v_moe/modular_glm4v_moe.py index 9e7557c9ecf5..7f4ec9c21912 100644 --- a/src/transformers/models/glm4v_moe/modular_glm4v_moe.py +++ b/src/transformers/models/glm4v_moe/modular_glm4v_moe.py @@ -150,7 +150,7 @@ class Glm4vMoeTextConfig(Glm4MoeConfig): >>> configuration = model.config ```""" - model_type = "Glm4vMoe_text" + model_type = "glm4v_moe_text" base_config_key = "text_config" keys_to_ignore_at_inference = ["past_key_values"] # Default tensor parallel plan for base model `Glm4vMoe` From b94cc13b3e973b8bbf7f9c152c87b832a412e802 Mon Sep 17 00:00:00 2001 From: zRzRzRzRzRzRzR <2448370773@qq.com> Date: Thu, 13 Nov 2025 19:12:13 +0800 Subject: [PATCH 57/66] Update __init__.py --- src/transformers/models/glm4v/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/transformers/models/glm4v/__init__.py b/src/transformers/models/glm4v/__init__.py index 126b3a768789..65d56c9a6467 100644 --- a/src/transformers/models/glm4v/__init__.py +++ b/src/transformers/models/glm4v/__init__.py @@ -19,9 +19,9 @@ if TYPE_CHECKING: from .configuration_glm4v import * + from .image_processing_glm4v import * from .modeling_glm4v import * from .processing_glm4v import * - from .image_processing_glm4v import * from .video_processing_glm4v import * else: import sys From ca310eee6d662f1e19495eeb8bcfdb2131a8a1ea Mon Sep 17 00:00:00 2001 From: zRzRzRzRzRzRzR <2448370773@qq.com> Date: Thu, 13 Nov 2025 19:24:51 +0800 Subject: [PATCH 58/66] update --- .../models/glm4v/configuration_glm4v.py | 2 +- .../models/glm4v/modeling_glm4v.py | 2 +- .../models/glm4v/modular_glm4v.py | 2 + .../glm4v_moe/configuration_glm4v_moe.py | 2 +- .../models/glm4v_moe/modeling_glm4v_moe.py | 227 +++++++++--------- .../models/glm4v_moe/modular_glm4v_moe.py | 10 +- 6 files changed, 131 insertions(+), 114 deletions(-) diff --git a/src/transformers/models/glm4v/configuration_glm4v.py b/src/transformers/models/glm4v/configuration_glm4v.py index 28240695f1f5..716feae9a47a 100644 --- a/src/transformers/models/glm4v/configuration_glm4v.py +++ b/src/transformers/models/glm4v/configuration_glm4v.py @@ -334,4 +334,4 @@ def __init__( super().__init__(**kwargs) -__all__ = ["Glm4vConfig", "Glm4vTextConfig"] +__all__ = ["Glm4vConfig", "Glm4vTextConfig", "Glm4vVisionConfig"] diff --git a/src/transformers/models/glm4v/modeling_glm4v.py b/src/transformers/models/glm4v/modeling_glm4v.py index 147e18b7e78e..fc5222e2d852 100644 --- a/src/transformers/models/glm4v/modeling_glm4v.py +++ b/src/transformers/models/glm4v/modeling_glm4v.py @@ -1683,4 +1683,4 @@ def _expand_dict_for_generation(dict_to_expand): return input_ids, model_kwargs -__all__ = ["Glm4vForConditionalGeneration", "Glm4vModel", "Glm4vPreTrainedModel", "Glm4vTextModel"] +__all__ = ["Glm4vForConditionalGeneration", "Glm4vModel", "Glm4vPreTrainedModel", "Glm4vTextModel", "Glm4vVisionModel"] diff --git a/src/transformers/models/glm4v/modular_glm4v.py b/src/transformers/models/glm4v/modular_glm4v.py index d2d0e9acfedd..f02b58a6b7e1 100644 --- a/src/transformers/models/glm4v/modular_glm4v.py +++ b/src/transformers/models/glm4v/modular_glm4v.py @@ -1689,9 +1689,11 @@ def replace_frame_token_id(self, timestamp_sec): __all__ = [ "Glm4vConfig", "Glm4vTextConfig", + "Glm4vVisionConfig", "Glm4vForConditionalGeneration", "Glm4vModel", "Glm4vPreTrainedModel", "Glm4vProcessor", "Glm4vTextModel", + "Glm4vVisionModel", ] diff --git a/src/transformers/models/glm4v_moe/configuration_glm4v_moe.py b/src/transformers/models/glm4v_moe/configuration_glm4v_moe.py index 12a059f46b6e..c6ab05619400 100644 --- a/src/transformers/models/glm4v_moe/configuration_glm4v_moe.py +++ b/src/transformers/models/glm4v_moe/configuration_glm4v_moe.py @@ -373,4 +373,4 @@ def __init__( super().__init__(**kwargs) -__all__ = ["Glm4vMoeConfig", "Glm4vMoeTextConfig"] +__all__ = ["Glm4vMoeConfig", "Glm4vMoeTextConfig", "Glm4vMoeVisionConfig"] diff --git a/src/transformers/models/glm4v_moe/modeling_glm4v_moe.py b/src/transformers/models/glm4v_moe/modeling_glm4v_moe.py index 7afb2e0b1463..e243eda50236 100644 --- a/src/transformers/models/glm4v_moe/modeling_glm4v_moe.py +++ b/src/transformers/models/glm4v_moe/modeling_glm4v_moe.py @@ -583,115 +583,6 @@ class Glm4vMoeCausalLMOutputWithPast(ModelOutput): aux_loss: Optional[torch.FloatTensor] = None -@auto_docstring -class Glm4vMoeTextModel(Glm4vMoePreTrainedModel): - config: Glm4vMoeTextConfig - input_modalities = "text" - - def __init__(self, config: Glm4vMoeTextConfig): - super().__init__(config) - self.padding_idx = config.pad_token_id - self.vocab_size = config.vocab_size - - self.embed_tokens = nn.Embedding(config.vocab_size, config.hidden_size, self.padding_idx) - self.layers = nn.ModuleList( - [Glm4vMoeTextDecoderLayer(config, layer_idx) for layer_idx in range(config.num_hidden_layers)] - ) - self.norm = Glm4vMoeRMSNorm(config.hidden_size, eps=config.rms_norm_eps) - self.rotary_emb = Glm4vMoeTextRotaryEmbedding(config=config) - - self.gradient_checkpointing = False - # Initialize weights and apply final processing - self.post_init() - - @auto_docstring - @check_model_inputs() - def forward( - self, - input_ids: Optional[torch.LongTensor] = None, - attention_mask: Optional[torch.Tensor] = None, - position_ids: Optional[torch.LongTensor] = None, - past_key_values: Optional[Cache] = None, - inputs_embeds: Optional[torch.FloatTensor] = None, - use_cache: Optional[bool] = None, - cache_position: Optional[torch.LongTensor] = None, - **kwargs: Unpack[FlashAttentionKwargs], - ) -> MoeModelOutputWithPast: - if (input_ids is None) ^ (inputs_embeds is not None): - raise ValueError("You must specify exactly one of input_ids or inputs_embeds") - - # torch.jit.trace() doesn't support cache objects in the output - if use_cache and past_key_values is None and not torch.jit.is_tracing(): - past_key_values = DynamicCache(config=self.config) - - if inputs_embeds is None: - inputs_embeds = self.embed_tokens(input_ids) - - if cache_position is None: - past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 - cache_position = torch.arange( - past_seen_tokens, past_seen_tokens + inputs_embeds.shape[1], device=inputs_embeds.device - ) - - # the hard coded `3` is for temporal, height and width. - if position_ids is None: - position_ids = cache_position.view(1, 1, -1).expand(3, inputs_embeds.shape[0], -1) - elif position_ids.ndim == 2: - position_ids = position_ids[None, ...].expand(3, position_ids.shape[0], -1) - - # NOTE: we need to pass text position ids for packing. Qwen2-VL uses 3D positions - # where each dim indicates visual spatial positions for temporal/height/width grids. - # There are two scenarios when FA2-like packed masking might be activated. - # 1. User specifically passed packed `position_ids` and no attention mask. - # In this case we expect the useer to create correct position ids for all 3 grids - # and prepend text-only position ids to it. The final tensor will be [4, bs, seq-len] - # 2. User runs forward with no attention mask and no position ids. In this case, position ids - # are prepared by the model (`get_rope_index`) as `[4, bs, seq-len]` tensor. Text-only positions are - # prepended by us when creating positions so that the mask is constructed correctly. NOTE: failing to pass - # text-only positions will cause incorrect mask construction, do not change `prepare_input_for_generation` - if position_ids.ndim == 3 and position_ids.shape[0] == 4: - text_position_ids = position_ids[0] - position_ids = position_ids[1:] - else: - # If inputs are not packed (usual 3D positions), do not prepare mask from position_ids - text_position_ids = None - - mask_kwargs = { - "config": self.config, - "input_embeds": inputs_embeds, - "attention_mask": attention_mask, - "cache_position": cache_position, - "past_key_values": past_key_values, - "position_ids": text_position_ids, - } - # Create the masks - causal_mask = create_causal_mask(**mask_kwargs) - - hidden_states = inputs_embeds - - # create position embeddings to be shared across the decoder layers - position_embeddings = self.rotary_emb(hidden_states, position_ids) - - for i, decoder_layer in enumerate(self.layers[: self.config.num_hidden_layers]): - layer_outputs = decoder_layer( - hidden_states, - position_embeddings=position_embeddings, - attention_mask=causal_mask, - position_ids=position_ids, - past_key_values=past_key_values, - cache_position=cache_position, - **kwargs, - ) - hidden_states = layer_outputs - - hidden_states = self.norm(hidden_states) - - return MoeModelOutputWithPast( - last_hidden_state=hidden_states, - past_key_values=past_key_values, - ) - - class Glm4vMoeisionMlp(nn.Module): def __init__(self, config, bias: bool = False): super().__init__() @@ -968,6 +859,7 @@ def forward( return hidden_states +@auto_docstring class Glm4vMoeVisionModel(Glm4vMoePreTrainedModel): config: Glm4vMoeVisionConfig input_modalities = ["image", "video"] @@ -1079,6 +971,115 @@ def forward(self, hidden_states: torch.Tensor, grid_thw: torch.Tensor) -> torch. return hidden_states +@auto_docstring +class Glm4vMoeTextModel(Glm4vMoePreTrainedModel): + config: Glm4vMoeTextConfig + input_modalities = "text" + + def __init__(self, config: Glm4vMoeTextConfig): + super().__init__(config) + self.padding_idx = config.pad_token_id + self.vocab_size = config.vocab_size + + self.embed_tokens = nn.Embedding(config.vocab_size, config.hidden_size, self.padding_idx) + self.layers = nn.ModuleList( + [Glm4vMoeTextDecoderLayer(config, layer_idx) for layer_idx in range(config.num_hidden_layers)] + ) + self.norm = Glm4vMoeRMSNorm(config.hidden_size, eps=config.rms_norm_eps) + self.rotary_emb = Glm4vMoeTextRotaryEmbedding(config=config) + + self.gradient_checkpointing = False + # Initialize weights and apply final processing + self.post_init() + + @auto_docstring + @check_model_inputs() + def forward( + self, + input_ids: Optional[torch.LongTensor] = None, + attention_mask: Optional[torch.Tensor] = None, + position_ids: Optional[torch.LongTensor] = None, + past_key_values: Optional[Cache] = None, + inputs_embeds: Optional[torch.FloatTensor] = None, + use_cache: Optional[bool] = None, + cache_position: Optional[torch.LongTensor] = None, + **kwargs: Unpack[FlashAttentionKwargs], + ) -> MoeModelOutputWithPast: + if (input_ids is None) ^ (inputs_embeds is not None): + raise ValueError("You must specify exactly one of input_ids or inputs_embeds") + + # torch.jit.trace() doesn't support cache objects in the output + if use_cache and past_key_values is None and not torch.jit.is_tracing(): + past_key_values = DynamicCache(config=self.config) + + if inputs_embeds is None: + inputs_embeds = self.embed_tokens(input_ids) + + if cache_position is None: + past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 + cache_position = torch.arange( + past_seen_tokens, past_seen_tokens + inputs_embeds.shape[1], device=inputs_embeds.device + ) + + # the hard coded `3` is for temporal, height and width. + if position_ids is None: + position_ids = cache_position.view(1, 1, -1).expand(3, inputs_embeds.shape[0], -1) + elif position_ids.ndim == 2: + position_ids = position_ids[None, ...].expand(3, position_ids.shape[0], -1) + + # NOTE: we need to pass text position ids for packing. Qwen2-VL uses 3D positions + # where each dim indicates visual spatial positions for temporal/height/width grids. + # There are two scenarios when FA2-like packed masking might be activated. + # 1. User specifically passed packed `position_ids` and no attention mask. + # In this case we expect the useer to create correct position ids for all 3 grids + # and prepend text-only position ids to it. The final tensor will be [4, bs, seq-len] + # 2. User runs forward with no attention mask and no position ids. In this case, position ids + # are prepared by the model (`get_rope_index`) as `[4, bs, seq-len]` tensor. Text-only positions are + # prepended by us when creating positions so that the mask is constructed correctly. NOTE: failing to pass + # text-only positions will cause incorrect mask construction, do not change `prepare_input_for_generation` + if position_ids.ndim == 3 and position_ids.shape[0] == 4: + text_position_ids = position_ids[0] + position_ids = position_ids[1:] + else: + # If inputs are not packed (usual 3D positions), do not prepare mask from position_ids + text_position_ids = None + + mask_kwargs = { + "config": self.config, + "input_embeds": inputs_embeds, + "attention_mask": attention_mask, + "cache_position": cache_position, + "past_key_values": past_key_values, + "position_ids": text_position_ids, + } + # Create the masks + causal_mask = create_causal_mask(**mask_kwargs) + + hidden_states = inputs_embeds + + # create position embeddings to be shared across the decoder layers + position_embeddings = self.rotary_emb(hidden_states, position_ids) + + for i, decoder_layer in enumerate(self.layers[: self.config.num_hidden_layers]): + layer_outputs = decoder_layer( + hidden_states, + position_embeddings=position_embeddings, + attention_mask=causal_mask, + position_ids=position_ids, + past_key_values=past_key_values, + cache_position=cache_position, + **kwargs, + ) + hidden_states = layer_outputs + + hidden_states = self.norm(hidden_states) + + return MoeModelOutputWithPast( + last_hidden_state=hidden_states, + past_key_values=past_key_values, + ) + + @auto_docstring class Glm4vMoeModel(Glm4vMoePreTrainedModel): base_model_prefix = "" @@ -1903,4 +1904,10 @@ def _expand_dict_for_generation(dict_to_expand): return input_ids, model_kwargs -__all__ = ["Glm4vMoeForConditionalGeneration", "Glm4vMoeModel", "Glm4vMoePreTrainedModel", "Glm4vMoeTextModel"] +__all__ = [ + "Glm4vMoeForConditionalGeneration", + "Glm4vMoeModel", + "Glm4vMoePreTrainedModel", + "Glm4vMoeTextModel", + "Glm4vMoeVisionModel", +] diff --git a/src/transformers/models/glm4v_moe/modular_glm4v_moe.py b/src/transformers/models/glm4v_moe/modular_glm4v_moe.py index 7f4ec9c21912..120499275444 100644 --- a/src/transformers/models/glm4v_moe/modular_glm4v_moe.py +++ b/src/transformers/models/glm4v_moe/modular_glm4v_moe.py @@ -45,6 +45,7 @@ Glm4vForConditionalGeneration, Glm4vTextModel, Glm4vTextRotaryEmbedding, + Glm4vVisionModel, rotate_half, ) from ..qwen3_vl_moe.modeling_qwen3_vl_moe import ( @@ -489,6 +490,11 @@ class Glm4vMoeCausalLMOutputWithPast(Qwen3VLMoeCausalLMOutputWithPast): pass +@auto_docstring +class Glm4vMoeVisionModel(Glm4vVisionModel): + pass + + @auto_docstring class Glm4vMoeTextModel(Glm4vTextModel): def forward( @@ -646,8 +652,10 @@ def forward( __all__ = [ "Glm4vMoeConfig", "Glm4vMoeTextConfig", + "Glm4vMoeVisionConfig", "Glm4vMoeForConditionalGeneration", "Glm4vMoeModel", # noqa: F822 "Glm4vMoePreTrainedModel", - "Glm4vMoeTextModel", # noqa: F822 + "Glm4vMoeTextModel", + "Glm4vMoeVisionModel", ] From 92fa57b5cf3584627ef6455312e9a251cadf6926 Mon Sep 17 00:00:00 2001 From: zRzRzRzRzRzRzR <2448370773@qq.com> Date: Thu, 13 Nov 2025 19:34:10 +0800 Subject: [PATCH 59/66] update doc --- .../models/glm46v/modeling_glm46v.py | 2 + .../models/glm4v/configuration_glm4v.py | 62 +++++++++--------- .../models/glm4v/modeling_glm4v.py | 2 + .../models/glm4v/modular_glm4v.py | 64 +++++++++---------- .../glm4v_moe/configuration_glm4v_moe.py | 62 +++++++++--------- .../models/glm4v_moe/modeling_glm4v_moe.py | 2 + 6 files changed, 95 insertions(+), 99 deletions(-) diff --git a/src/transformers/models/glm46v/modeling_glm46v.py b/src/transformers/models/glm46v/modeling_glm46v.py index 9a94755d0a9e..e1306f148005 100644 --- a/src/transformers/models/glm46v/modeling_glm46v.py +++ b/src/transformers/models/glm46v/modeling_glm46v.py @@ -586,6 +586,8 @@ def forward( The temporal, height and width of feature shape of each image in LLM. video_grid_thw (`torch.LongTensor` of shape `(num_videos, 3)`, *optional*): The temporal, height and width of feature shape of each video in LLM. + rope_deltas (`torch.LongTensor` of shape `(batch_size, )`, *optional*): + The rope index difference between sequence length and multimodal rope. Example: diff --git a/src/transformers/models/glm4v/configuration_glm4v.py b/src/transformers/models/glm4v/configuration_glm4v.py index 716feae9a47a..7370a80b52f2 100644 --- a/src/transformers/models/glm4v/configuration_glm4v.py +++ b/src/transformers/models/glm4v/configuration_glm4v.py @@ -32,39 +32,35 @@ class Glm4vVisionConfig(PreTrainedConfig): GLM-4.1V-9B-Thinking [THUDM/GLM-4.1V-9B-Thinking](https://huggingface.co/THUDM/GLM-4.1V-9B-Thinking). Args: - hidden_size (`int`, *optional*, defaults to 1536): - Dimensionality of the encoder layers and the pooler layer. - depth (`int`, *optional*, defaults to 24): - Number of layers (depth) in the model. - attention_bias (`bool`, *optional*, defaults to `False`): - Whether to add a bias to the queries, keys and values. - intermediate_size (`int`, *optional*, defaults to 13696): - Dimensionality of the "intermediate" (i.e., feed-forward) layer in the Transformer encoder. - hidden_act (`str` or `function`, *optional*, defaults to `"selu"`): - The non-linear activation function (function or string) in the encoder and pooler. If string, `"gelu"`, - `"relu"`, `"selu"` and `"gelu_new"` are supported. - hidden_dropout_prob (`float`, *optional*, defaults to 0.0): - The dropout probability for all fully connected layers in the embeddings, encoder, and pooler. - attention_dropout (`float`, *optional*, defaults to 0.0): - Dropout probability for attention weights. - projection_dropout (`float`, *optional*, defaults to 0.0): - Dropout probability for the projection layer. - initializer_range (`float`, *optional*, defaults to 0.02): - The standard deviation of the truncated_normal_initializer for initializing all weight matrices. - image_size (`int` or `list[int]`, *optional*, defaults to `[336, 336]`): - The size (resolution) of each image. - patch_size (`int`, *optional*, defaults to `14`): - The size (resolution) of each patch. - num_channels (`int`, *optional*, defaults to 3): - The number of input channels. - out_hidden_size (`int`, *optional*, defaults to 4096): - The output hidden size of the vision model. - rms_norm_eps (`float`, *optional*, defaults to 1e-05): - The epsilon used by the rms normalization layers. - spatial_merge_size (`int`, *optional*, defaults to 2): - The size used for merging spatial dimensions. - temporal_patch_size (`int`, *optional*, defaults to 2): - The size used for patches along the temporal dimension. + depth (`int`, *optional*, defaults to 24): + Number of layers (depth) in the model. + hidden_size (`int`, *optional*, defaults to 1536): + Dimensionality of the encoder layers and the pooler layer. + hidden_act (`str` or `function`, *optional*, defaults to `"silu"`): + The non-linear activation function (function or string) in the encoder and pooler. If string, `"gelu"`, + `"relu"`, `"selu"` and `"gelu_new"` are supported. + attention_bias (`bool`, *optional*, defaults to `False`): + Whether to add a bias to the queries, keys and values. + attention_dropout (`float`, *optional*, defaults to 0.0): + Dropout probability for attention weights. + num_heads (``, *optional*, defaults to 12): + in_channels (``, *optional*, defaults to 3): + image_size (`int` or `list[int]`, *optional*, defaults to 336): + The size (resolution) of each image. + patch_size (`int`, *optional*, defaults to 14): + The size (resolution) of each patch. + rms_norm_eps (`float`, *optional*, defaults to 1e-05): + The epsilon used by the rms normalization layers. + spatial_merge_size (`int`, *optional*, defaults to 2): + The size used for merging spatial dimensions. + temporal_patch_size (`int`, *optional*, defaults to 2): + The size used for patches along the temporal dimension. + out_hidden_size (`int`, *optional*, defaults to 4096): + The output hidden size of the vision model. + intermediate_size (`int`, *optional*, defaults to 13696): + Dimensionality of the "intermediate" (i.e., feed-forward) layer in the Transformer encoder. + initializer_range (`float`, *optional*, defaults to 0.02): + The standard deviation of the truncated_normal_initializer for initializing all weight matrices. Example: ```python diff --git a/src/transformers/models/glm4v/modeling_glm4v.py b/src/transformers/models/glm4v/modeling_glm4v.py index fc5222e2d852..c73c51369894 100644 --- a/src/transformers/models/glm4v/modeling_glm4v.py +++ b/src/transformers/models/glm4v/modeling_glm4v.py @@ -1434,6 +1434,8 @@ def forward( The temporal, height and width of feature shape of each image in LLM. video_grid_thw (`torch.LongTensor` of shape `(num_videos, 3)`, *optional*): The temporal, height and width of feature shape of each video in LLM. + rope_deltas (`torch.LongTensor` of shape `(batch_size, )`, *optional*): + The rope index difference between sequence length and multimodal rope. Example: diff --git a/src/transformers/models/glm4v/modular_glm4v.py b/src/transformers/models/glm4v/modular_glm4v.py index f02b58a6b7e1..3a89c889af76 100644 --- a/src/transformers/models/glm4v/modular_glm4v.py +++ b/src/transformers/models/glm4v/modular_glm4v.py @@ -69,39 +69,35 @@ class Glm4vVisionConfig(PreTrainedConfig): GLM-4.1V-9B-Thinking [THUDM/GLM-4.1V-9B-Thinking](https://huggingface.co/THUDM/GLM-4.1V-9B-Thinking). Args: - hidden_size (`int`, *optional*, defaults to 1536): - Dimensionality of the encoder layers and the pooler layer. - depth (`int`, *optional*, defaults to 24): - Number of layers (depth) in the model. - attention_bias (`bool`, *optional*, defaults to `False`): - Whether to add a bias to the queries, keys and values. - intermediate_size (`int`, *optional*, defaults to 13696): - Dimensionality of the "intermediate" (i.e., feed-forward) layer in the Transformer encoder. - hidden_act (`str` or `function`, *optional*, defaults to `"selu"`): - The non-linear activation function (function or string) in the encoder and pooler. If string, `"gelu"`, - `"relu"`, `"selu"` and `"gelu_new"` are supported. - hidden_dropout_prob (`float`, *optional*, defaults to 0.0): - The dropout probability for all fully connected layers in the embeddings, encoder, and pooler. - attention_dropout (`float`, *optional*, defaults to 0.0): - Dropout probability for attention weights. - projection_dropout (`float`, *optional*, defaults to 0.0): - Dropout probability for the projection layer. - initializer_range (`float`, *optional*, defaults to 0.02): - The standard deviation of the truncated_normal_initializer for initializing all weight matrices. - image_size (`int` or `list[int]`, *optional*, defaults to `[336, 336]`): - The size (resolution) of each image. - patch_size (`int`, *optional*, defaults to `14`): - The size (resolution) of each patch. - num_channels (`int`, *optional*, defaults to 3): - The number of input channels. - out_hidden_size (`int`, *optional*, defaults to 4096): - The output hidden size of the vision model. - rms_norm_eps (`float`, *optional*, defaults to 1e-05): - The epsilon used by the rms normalization layers. - spatial_merge_size (`int`, *optional*, defaults to 2): - The size used for merging spatial dimensions. - temporal_patch_size (`int`, *optional*, defaults to 2): - The size used for patches along the temporal dimension. + depth (`int`, *optional*, defaults to 24): + Number of layers (depth) in the model. + hidden_size (`int`, *optional*, defaults to 1536): + Dimensionality of the encoder layers and the pooler layer. + hidden_act (`str` or `function`, *optional*, defaults to `"silu"`): + The non-linear activation function (function or string) in the encoder and pooler. If string, `"gelu"`, + `"relu"`, `"selu"` and `"gelu_new"` are supported. + attention_bias (`bool`, *optional*, defaults to `False`): + Whether to add a bias to the queries, keys and values. + attention_dropout (`float`, *optional*, defaults to 0.0): + Dropout probability for attention weights. + num_heads (``, *optional*, defaults to 12): + in_channels (``, *optional*, defaults to 3): + image_size (`int` or `list[int]`, *optional*, defaults to 336): + The size (resolution) of each image. + patch_size (`int`, *optional*, defaults to 14): + The size (resolution) of each patch. + rms_norm_eps (`float`, *optional*, defaults to 1e-05): + The epsilon used by the rms normalization layers. + spatial_merge_size (`int`, *optional*, defaults to 2): + The size used for merging spatial dimensions. + temporal_patch_size (`int`, *optional*, defaults to 2): + The size used for patches along the temporal dimension. + out_hidden_size (`int`, *optional*, defaults to 4096): + The output hidden size of the vision model. + intermediate_size (`int`, *optional*, defaults to 13696): + Dimensionality of the "intermediate" (i.e., feed-forward) layer in the Transformer encoder. + initializer_range (`float`, *optional*, defaults to 0.02): + The standard deviation of the truncated_normal_initializer for initializing all weight matrices. Example: ```python @@ -1357,6 +1353,8 @@ def forward( The temporal, height and width of feature shape of each image in LLM. video_grid_thw (`torch.LongTensor` of shape `(num_videos, 3)`, *optional*): The temporal, height and width of feature shape of each video in LLM. + rope_deltas (`torch.LongTensor` of shape `(batch_size, )`, *optional*): + The rope index difference between sequence length and multimodal rope. Example: diff --git a/src/transformers/models/glm4v_moe/configuration_glm4v_moe.py b/src/transformers/models/glm4v_moe/configuration_glm4v_moe.py index c6ab05619400..ac39012bef69 100644 --- a/src/transformers/models/glm4v_moe/configuration_glm4v_moe.py +++ b/src/transformers/models/glm4v_moe/configuration_glm4v_moe.py @@ -32,39 +32,35 @@ class Glm4vMoeVisionConfig(PreTrainedConfig): GLM-4.1V-9B-Thinking [THUDM/GLM-4.1V-9B-Thinking](https://huggingface.co/THUDM/GLM-4.1V-9B-Thinking). Args: - hidden_size (`int`, *optional*, defaults to 1536): - Dimensionality of the encoder layers and the pooler layer. - depth (`int`, *optional*, defaults to 24): - Number of layers (depth) in the model. - attention_bias (`bool`, *optional*, defaults to `False`): - Whether to add a bias to the queries, keys and values. - intermediate_size (`int`, *optional*, defaults to 13696): - Dimensionality of the "intermediate" (i.e., feed-forward) layer in the Transformer encoder. - hidden_act (`str` or `function`, *optional*, defaults to `"selu"`): - The non-linear activation function (function or string) in the encoder and pooler. If string, `"gelu"`, - `"relu"`, `"selu"` and `"gelu_new"` are supported. - hidden_dropout_prob (`float`, *optional*, defaults to 0.0): - The dropout probability for all fully connected layers in the embeddings, encoder, and pooler. - attention_dropout (`float`, *optional*, defaults to 0.0): - Dropout probability for attention weights. - projection_dropout (`float`, *optional*, defaults to 0.0): - Dropout probability for the projection layer. - initializer_range (`float`, *optional*, defaults to 0.02): - The standard deviation of the truncated_normal_initializer for initializing all weight matrices. - image_size (`int` or `list[int]`, *optional*, defaults to `[336, 336]`): - The size (resolution) of each image. - patch_size (`int`, *optional*, defaults to `14`): - The size (resolution) of each patch. - num_channels (`int`, *optional*, defaults to 3): - The number of input channels. - out_hidden_size (`int`, *optional*, defaults to 4096): - The output hidden size of the vision model. - rms_norm_eps (`float`, *optional*, defaults to 1e-05): - The epsilon used by the rms normalization layers. - spatial_merge_size (`int`, *optional*, defaults to 2): - The size used for merging spatial dimensions. - temporal_patch_size (`int`, *optional*, defaults to 2): - The size used for patches along the temporal dimension. + depth (`int`, *optional*, defaults to 24): + Number of layers (depth) in the model. + hidden_size (`int`, *optional*, defaults to 1536): + Dimensionality of the encoder layers and the pooler layer. + hidden_act (`str` or `function`, *optional*, defaults to `"silu"`): + The non-linear activation function (function or string) in the encoder and pooler. If string, `"gelu"`, + `"relu"`, `"selu"` and `"gelu_new"` are supported. + attention_bias (`bool`, *optional*, defaults to `False`): + Whether to add a bias to the queries, keys and values. + attention_dropout (`float`, *optional*, defaults to 0.0): + Dropout probability for attention weights. + num_heads (``, *optional*, defaults to 12): + in_channels (``, *optional*, defaults to 3): + image_size (`int` or `list[int]`, *optional*, defaults to 336): + The size (resolution) of each image. + patch_size (`int`, *optional*, defaults to 14): + The size (resolution) of each patch. + rms_norm_eps (`float`, *optional*, defaults to 1e-05): + The epsilon used by the rms normalization layers. + spatial_merge_size (`int`, *optional*, defaults to 2): + The size used for merging spatial dimensions. + temporal_patch_size (`int`, *optional*, defaults to 2): + The size used for patches along the temporal dimension. + out_hidden_size (`int`, *optional*, defaults to 4096): + The output hidden size of the vision model. + intermediate_size (`int`, *optional*, defaults to 13696): + Dimensionality of the "intermediate" (i.e., feed-forward) layer in the Transformer encoder. + initializer_range (`float`, *optional*, defaults to 0.02): + The standard deviation of the truncated_normal_initializer for initializing all weight matrices. Example: ```python diff --git a/src/transformers/models/glm4v_moe/modeling_glm4v_moe.py b/src/transformers/models/glm4v_moe/modeling_glm4v_moe.py index e243eda50236..9e7bea994f09 100644 --- a/src/transformers/models/glm4v_moe/modeling_glm4v_moe.py +++ b/src/transformers/models/glm4v_moe/modeling_glm4v_moe.py @@ -1642,6 +1642,8 @@ def forward( The temporal, height and width of feature shape of each image in LLM. video_grid_thw (`torch.LongTensor` of shape `(num_videos, 3)`, *optional*): The temporal, height and width of feature shape of each video in LLM. + rope_deltas (`torch.LongTensor` of shape `(batch_size, )`, *optional*): + The rope index difference between sequence length and multimodal rope. Example: From c9f260c31a46680e932af5cea079805906f064ac Mon Sep 17 00:00:00 2001 From: zRzRzRzRzRzRzR <2448370773@qq.com> Date: Thu, 13 Nov 2025 19:38:48 +0800 Subject: [PATCH 60/66] Update check_docstrings.py --- utils/check_docstrings.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/utils/check_docstrings.py b/utils/check_docstrings.py index 1e57ee66479a..3a957c2589cf 100644 --- a/utils/check_docstrings.py +++ b/utils/check_docstrings.py @@ -215,6 +215,8 @@ "GPTSanJapaneseConfig", "GitConfig", "GitVisionConfig", + "Glm4vVisionConfig", + "Glm4vMoeVisionConfig", "GraphormerConfig", "GroupViTTextConfig", "GroupViTVisionConfig", From 5ca314432f6a647b0c3fe52bfeed790ca122c909 Mon Sep 17 00:00:00 2001 From: zRzRzRzRzRzRzR <2448370773@qq.com> Date: Thu, 13 Nov 2025 19:44:17 +0800 Subject: [PATCH 61/66] update doc --- docs/source/en/model_doc/glm4v.md | 10 ++++++++++ docs/source/en/model_doc/glm4v_moe.md | 14 ++++++++++++-- 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/docs/source/en/model_doc/glm4v.md b/docs/source/en/model_doc/glm4v.md index 874b71435b48..98f6d8e761c5 100644 --- a/docs/source/en/model_doc/glm4v.md +++ b/docs/source/en/model_doc/glm4v.md @@ -170,6 +170,11 @@ print(output_text) [[autodoc]] Glm4vConfig + +## Glm4vVisionConfig + +[[autodoc]] Glm4vVisionConfig + ## Glm4vTextConfig [[autodoc]] Glm4vTextConfig @@ -193,6 +198,11 @@ print(output_text) [[autodoc]] Glm4vProcessor +## Glm4vVisionModel + +[[autodoc]] Glm4vVisionModel + - forward + ## Glm4vTextModel [[autodoc]] Glm4vTextModel diff --git a/docs/source/en/model_doc/glm4v_moe.md b/docs/source/en/model_doc/glm4v_moe.md index c814fdb5becd..8fab75298dfc 100644 --- a/docs/source/en/model_doc/glm4v_moe.md +++ b/docs/source/en/model_doc/glm4v_moe.md @@ -22,7 +22,7 @@ rendered properly in your Markdown viewer. SDPA -# Glm4vMoe +# Glm4vMoeMoe ## Overview @@ -48,10 +48,20 @@ The model also introduces a **Thinking Mode** switch, allowing users to balance [[autodoc]] Glm4vMoeConfig + +## Glm4vMoeVisionConfig + +[[autodoc]] Glm4vMoeVisionConfig + ## Glm4vMoeTextConfig [[autodoc]] Glm4vMoeTextConfig +## Glm4vMoeVisionModel + +[[autodoc]] Glm4vMoeVisionModel + - forward + ## Glm4vMoeTextModel [[autodoc]] Glm4vMoeTextModel @@ -65,4 +75,4 @@ The model also introduces a **Thinking Mode** switch, allowing users to balance ## Glm4vMoeForConditionalGeneration [[autodoc]] Glm4vMoeForConditionalGeneration - - forward + - forward \ No newline at end of file From 6d325c35b9cc038f2526cd035907c4b833a8aed1 Mon Sep 17 00:00:00 2001 From: Arthur Date: Fri, 14 Nov 2025 11:59:11 +0100 Subject: [PATCH 62/66] fix copies for tied weight keys! --- src/transformers/models/glm46v/modeling_glm46v.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/transformers/models/glm46v/modeling_glm46v.py b/src/transformers/models/glm46v/modeling_glm46v.py index e1306f148005..e830121da3a0 100644 --- a/src/transformers/models/glm46v/modeling_glm46v.py +++ b/src/transformers/models/glm46v/modeling_glm46v.py @@ -516,7 +516,7 @@ class Glm46VCausalLMOutputWithPast(ModelOutput): class Glm46VForConditionalGeneration(Glm46VPreTrainedModel, GenerationMixin): _checkpoint_conversion_mapping = {} - _tied_weights_keys = ["lm_head.weight"] + _tied_weights_keys = {"lm_head.weight": "model.language_model.embed_tokens.weight"} # Reference: fix gemma3 grad acc #37208 accepts_loss_kwargs = False From fef665ec0bedb7ac8b1527d896915d1bff94195c Mon Sep 17 00:00:00 2001 From: Arthur Date: Fri, 14 Nov 2025 12:01:28 +0100 Subject: [PATCH 63/66] more fixup --- src/transformers/models/glm46v/modeling_glm46v.py | 2 -- src/transformers/models/glm4v/modeling_glm4v.py | 2 -- src/transformers/models/glm4v_moe/modeling_glm4v_moe.py | 2 -- 3 files changed, 6 deletions(-) diff --git a/src/transformers/models/glm46v/modeling_glm46v.py b/src/transformers/models/glm46v/modeling_glm46v.py index e830121da3a0..71dd38f2076b 100644 --- a/src/transformers/models/glm46v/modeling_glm46v.py +++ b/src/transformers/models/glm46v/modeling_glm46v.py @@ -576,8 +576,6 @@ def forward( **kwargs: Unpack[TransformersKwargs], ) -> Union[tuple, Glm46VCausalLMOutputWithPast]: r""" - rope_deltas (`torch.LongTensor` of shape `(batch_size, )`, *optional*): - The rope index difference between sequence length and multimodal rope. labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored diff --git a/src/transformers/models/glm4v/modeling_glm4v.py b/src/transformers/models/glm4v/modeling_glm4v.py index 1f81e3326477..2b18c9fdfd6e 100644 --- a/src/transformers/models/glm4v/modeling_glm4v.py +++ b/src/transformers/models/glm4v/modeling_glm4v.py @@ -1424,8 +1424,6 @@ def forward( **kwargs: Unpack[TransformersKwargs], ) -> Union[tuple, Glm4vCausalLMOutputWithPast]: r""" - rope_deltas (`torch.LongTensor` of shape `(batch_size, )`, *optional*): - The rope index difference between sequence length and multimodal rope. labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored diff --git a/src/transformers/models/glm4v_moe/modeling_glm4v_moe.py b/src/transformers/models/glm4v_moe/modeling_glm4v_moe.py index 85160ec9b8c7..d10dacfef6d7 100644 --- a/src/transformers/models/glm4v_moe/modeling_glm4v_moe.py +++ b/src/transformers/models/glm4v_moe/modeling_glm4v_moe.py @@ -1649,8 +1649,6 @@ def forward( The temporal, height and width of feature shape of each image in LLM. video_grid_thw (`torch.LongTensor` of shape `(num_videos, 3)`, *optional*): The temporal, height and width of feature shape of each video in LLM. - rope_deltas (`torch.LongTensor` of shape `(batch_size, )`, *optional*): - The rope index difference between sequence length and multimodal rope. Example: From 0170755f4989e37bfaa190f972fd09590d73e33f Mon Sep 17 00:00:00 2001 From: Arthur Date: Sat, 15 Nov 2025 08:59:20 +0100 Subject: [PATCH 64/66] fix copies? --- src/transformers/models/glm46v/modeling_glm46v.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/transformers/models/glm46v/modeling_glm46v.py b/src/transformers/models/glm46v/modeling_glm46v.py index 71dd38f2076b..8a440f72fab7 100644 --- a/src/transformers/models/glm46v/modeling_glm46v.py +++ b/src/transformers/models/glm46v/modeling_glm46v.py @@ -570,7 +570,6 @@ def forward( pixel_values_videos: Optional[torch.FloatTensor] = None, image_grid_thw: Optional[torch.LongTensor] = None, video_grid_thw: Optional[torch.LongTensor] = None, - rope_deltas: Optional[torch.LongTensor] = None, cache_position: Optional[torch.LongTensor] = None, logits_to_keep: Union[int, torch.Tensor] = 0, **kwargs: Unpack[TransformersKwargs], From 54a2196603efdcc330256e8c3ebe788e17bd2a88 Mon Sep 17 00:00:00 2001 From: Arthur Date: Sat, 15 Nov 2025 08:59:46 +0100 Subject: [PATCH 65/66] more fix copies --- src/transformers/models/glm46v/modeling_glm46v.py | 4 ++-- src/transformers/models/glm4v/modeling_glm4v.py | 4 ++-- src/transformers/models/glm4v_moe/modeling_glm4v_moe.py | 2 ++ 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/transformers/models/glm46v/modeling_glm46v.py b/src/transformers/models/glm46v/modeling_glm46v.py index 8a440f72fab7..5bc1cc3ee059 100644 --- a/src/transformers/models/glm46v/modeling_glm46v.py +++ b/src/transformers/models/glm46v/modeling_glm46v.py @@ -575,6 +575,8 @@ def forward( **kwargs: Unpack[TransformersKwargs], ) -> Union[tuple, Glm46VCausalLMOutputWithPast]: r""" + rope_deltas (`torch.LongTensor` of shape `(batch_size, )`, *optional*): + The rope index difference between sequence length and multimodal rope. labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored @@ -583,8 +585,6 @@ def forward( The temporal, height and width of feature shape of each image in LLM. video_grid_thw (`torch.LongTensor` of shape `(num_videos, 3)`, *optional*): The temporal, height and width of feature shape of each video in LLM. - rope_deltas (`torch.LongTensor` of shape `(batch_size, )`, *optional*): - The rope index difference between sequence length and multimodal rope. Example: diff --git a/src/transformers/models/glm4v/modeling_glm4v.py b/src/transformers/models/glm4v/modeling_glm4v.py index d08aefc8aee8..eace363381a9 100644 --- a/src/transformers/models/glm4v/modeling_glm4v.py +++ b/src/transformers/models/glm4v/modeling_glm4v.py @@ -1423,6 +1423,8 @@ def forward( **kwargs: Unpack[TransformersKwargs], ) -> Union[tuple, Glm4vCausalLMOutputWithPast]: r""" + rope_deltas (`torch.LongTensor` of shape `(batch_size, )`, *optional*): + The rope index difference between sequence length and multimodal rope. labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored @@ -1431,8 +1433,6 @@ def forward( The temporal, height and width of feature shape of each image in LLM. video_grid_thw (`torch.LongTensor` of shape `(num_videos, 3)`, *optional*): The temporal, height and width of feature shape of each video in LLM. - rope_deltas (`torch.LongTensor` of shape `(batch_size, )`, *optional*): - The rope index difference between sequence length and multimodal rope. Example: diff --git a/src/transformers/models/glm4v_moe/modeling_glm4v_moe.py b/src/transformers/models/glm4v_moe/modeling_glm4v_moe.py index 631505562bc6..d26852c3f1c6 100644 --- a/src/transformers/models/glm4v_moe/modeling_glm4v_moe.py +++ b/src/transformers/models/glm4v_moe/modeling_glm4v_moe.py @@ -1640,6 +1640,8 @@ def forward( **kwargs: Unpack[TransformersKwargs], ) -> Union[tuple, Glm4vMoeCausalLMOutputWithPast]: r""" + rope_deltas (`torch.LongTensor` of shape `(batch_size, )`, *optional*): + The rope index difference between sequence length and multimodal rope. labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored From 0fd529add7a605b7847da112044c03c2fba2bc07 Mon Sep 17 00:00:00 2001 From: Arthur Date: Sat, 15 Nov 2025 09:37:11 +0100 Subject: [PATCH 66/66] Up --- src/transformers/models/glm46v/modeling_glm46v.py | 2 -- src/transformers/models/glm4v/modeling_glm4v.py | 2 -- src/transformers/models/glm4v/modular_glm4v.py | 2 -- src/transformers/models/glm4v_moe/modeling_glm4v_moe.py | 2 -- 4 files changed, 8 deletions(-) diff --git a/src/transformers/models/glm46v/modeling_glm46v.py b/src/transformers/models/glm46v/modeling_glm46v.py index 5bc1cc3ee059..7fc18482c4da 100644 --- a/src/transformers/models/glm46v/modeling_glm46v.py +++ b/src/transformers/models/glm46v/modeling_glm46v.py @@ -575,8 +575,6 @@ def forward( **kwargs: Unpack[TransformersKwargs], ) -> Union[tuple, Glm46VCausalLMOutputWithPast]: r""" - rope_deltas (`torch.LongTensor` of shape `(batch_size, )`, *optional*): - The rope index difference between sequence length and multimodal rope. labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored diff --git a/src/transformers/models/glm4v/modeling_glm4v.py b/src/transformers/models/glm4v/modeling_glm4v.py index eace363381a9..47ad72ac96ce 100644 --- a/src/transformers/models/glm4v/modeling_glm4v.py +++ b/src/transformers/models/glm4v/modeling_glm4v.py @@ -1423,8 +1423,6 @@ def forward( **kwargs: Unpack[TransformersKwargs], ) -> Union[tuple, Glm4vCausalLMOutputWithPast]: r""" - rope_deltas (`torch.LongTensor` of shape `(batch_size, )`, *optional*): - The rope index difference between sequence length and multimodal rope. labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored diff --git a/src/transformers/models/glm4v/modular_glm4v.py b/src/transformers/models/glm4v/modular_glm4v.py index 85cd3ed44085..2df8b6f9d04a 100644 --- a/src/transformers/models/glm4v/modular_glm4v.py +++ b/src/transformers/models/glm4v/modular_glm4v.py @@ -1350,8 +1350,6 @@ def forward( The temporal, height and width of feature shape of each image in LLM. video_grid_thw (`torch.LongTensor` of shape `(num_videos, 3)`, *optional*): The temporal, height and width of feature shape of each video in LLM. - rope_deltas (`torch.LongTensor` of shape `(batch_size, )`, *optional*): - The rope index difference between sequence length and multimodal rope. Example: diff --git a/src/transformers/models/glm4v_moe/modeling_glm4v_moe.py b/src/transformers/models/glm4v_moe/modeling_glm4v_moe.py index d26852c3f1c6..631505562bc6 100644 --- a/src/transformers/models/glm4v_moe/modeling_glm4v_moe.py +++ b/src/transformers/models/glm4v_moe/modeling_glm4v_moe.py @@ -1640,8 +1640,6 @@ def forward( **kwargs: Unpack[TransformersKwargs], ) -> Union[tuple, Glm4vMoeCausalLMOutputWithPast]: r""" - rope_deltas (`torch.LongTensor` of shape `(batch_size, )`, *optional*): - The rope index difference between sequence length and multimodal rope. labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored