feat: add gme-Qwen2-VL-7B-Instruct (#153)

* feat: add gme-Qwen2-VL-7B-Instruct

* fix: revert vllm engine to 084v for gme qwen2vl

Author: yanbasic

src/emd/models/embeddings/qwen.py

@@ -1,18 +1,19 @@
 from .. import Model
-from ..engines import vllm_qwen3_engin091
+from ..engines import vllm_qwen3_engin091, vllm_gme_qwen2vl_engine091, vllm_gme_qwen2vl_engine084_compat
 from ..services import sagemaker_service,local_service,ecs_service
 from ..frameworks import fastapi_framework
 from ..instances import (
     g5dxlarge_instance,
     g5d2xlarge_instance,
     g5d4xlarge_instance,
     g5d8xlarge_instance,
+    g5d12xlarge_instance,
     g5d16xlarge_instance,
     local_instance
 )
 from emd.models.utils.constants import ModelType
 from emd.models import ModelSeries
-from ..model_series import QWEN3_SERIES
+from ..model_series import QWEN3_SERIES, GME_SERIES
 
 
 Model.register(
@@ -104,3 +105,33 @@
         model_series=QWEN3_SERIES
     )
 )
+
+Model.register(
+    dict(
+        model_id = "gme-Qwen2-VL-7B-Instruct",
+        supported_engines=[vllm_gme_qwen2vl_engine084_compat],
+        supported_instances=[
+            g5d4xlarge_instance,
+            g5d8xlarge_instance,
+            g5d12xlarge_instance,
+            g5d16xlarge_instance,
+            local_instance
+        ],
+        supported_services=[
+            sagemaker_service,
+            ecs_service,
+            local_service
+        ],
+        supported_frameworks=[
+            fastapi_framework
+        ],
+        allow_china_region=True,
+        huggingface_model_id="Alibaba-NLP/gme-Qwen2-VL-7B-Instruct",
+        modelscope_model_id="Alibaba-NLP/gme-Qwen2-VL-7B-Instruct",
+        require_huggingface_token=False,
+        application_scenario="Multimodal RAG, image-text retrieval, visual search",
+        description="General Multimodal Embedding model based on Qwen2-VL architecture, supporting text, image, and image-text pair inputs for unified multimodal representation learning and retrieval tasks. Uses vLLM v0.8.4 for transformers compatibility.",
+        model_type=ModelType.EMBEDDING,
+        model_series=GME_SERIES
+    )
+)

src/emd/models/engines.py

@@ -183,6 +183,22 @@ class KtransformersEngine(OpenAICompitableEngine):
             "default_cli_args": " --max_num_seq 30 --disable-log-stats --trust-remote-code --task embed"
 })
 
+vllm_gme_qwen2vl_engine091 = VllmEngine(**{
+            **vllm_embedding_engine091.model_dump(),
+            "environment_variables": "export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True",
+            "default_cli_args": " --max_num_seq 20 --disable-log-stats --trust-remote-code --task embed --limit-mm-per-prompt image=10 --gpu_memory_utilization 0.8",
+            "description": "VLLM engine for GME multimodal embedding models based on Qwen2-VL"
+})
+
+# GME-compatible engine with transformers 4.51.3
+vllm_gme_qwen2vl_engine084_compat = VllmEngine(**{
+            **vllm_embedding_engine091.model_dump(),
+            "engine_dockerfile_config": {"VERSION":"v0.8.4"},
+            "environment_variables": "export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True",
+            "default_cli_args": " --max_num_seq 20 --disable-log-stats --trust-remote-code --task embed --limit-mm-per-prompt image=10 --gpu_memory_utilization 0.8",
+            "description": "VLLM engine v0.8.4 for GME multimodal embedding models with compatible transformers version"
+})
+
 
 vllm_qwen2vl72b_engine064 = VllmEngine(**{
              **vllm_engine064.model_dump(),

src/emd/models/model_series.py

@@ -151,3 +151,9 @@
     description="Baichuan Intelligent Technology.",
     reference_link="https://github.com/baichuan-inc"
 )
+
+GME_SERIES = ModelSeries(
+    model_series_name=ModelSeriesType.GME,
+    description="General Multimodal Embedding (GME) models based on Qwen2-VL architecture, designed for unified multimodal representation learning supporting text, image, and image-text pair inputs for retrieval and search applications.",
+    reference_link="https://huggingface.co/Alibaba-NLP/gme-Qwen2-VL-7B-Instruct"
+)

src/emd/models/utils/constants.py

@@ -227,6 +227,7 @@ class ModelSeriesType(ConstantBase):
     BCE = "bce"
     COMFYUI = "comfyui"
     QWEN2VL = "qwen2vl"
+    GME = "gme"
     AGENT = "agent"
     INTERNVL25 = "internvl2.5"
     LLAMA = "llama"