Fix TasksManager._TRANSFORMERS_TASKS_TO_MODEL_LOADERS (#1501)

* Fix TasksManager._TRANSFORMERS_TASKS_TO_MODEL_LOADERS

* remove quantizer set task

* add test with transformers v4.46

* temporary fix

* fix

* style

* remove 4.46 tests

* remove video-text-to-text task

* only change task if has remote code

* add comment

* Update optimum/intel/openvino/quantization.py

Co-authored-by: Nikita Savelyev <[email protected]>

---------

Co-authored-by: Nikita Savelyev <[email protected]>

Author: echarlaix

optimum/exporters/openvino/__main__.py

@@ -394,8 +394,16 @@ class StoreAttr(object):
         if library_name == "open_clip":
             model = _OpenClipForZeroShotImageClassification.from_pretrained(model_name_or_path, cache_dir=cache_dir)
         else:
+            # remote code models like phi3_v internvl2, minicpmv, internvl2, nanollava, maira2 should be loaded using AutoModelForCausalLM and not AutoModelForImageTextToText
+            # TODO: use config.auto_map to load remote code models instead (for other models we can directly use config.architectures)
+            task_model_loading = task
+            if library_name == "transformers":
+                has_remote_code = hasattr(config, "auto_map")
+                if has_remote_code and trust_remote_code and task == "image-text-to-text":
+                    task_model_loading = "text-generation"
+
             model = TasksManager.get_model_from_task(
-                task,
+                task_model_loading,
                 model_name_or_path,
                 subfolder=subfolder,
                 revision=revision,

optimum/exporters/openvino/model_configs.py

@@ -202,12 +202,6 @@ def init_model_configs():
         "AutoModelForImageTextToText",
     )
 
-    TasksManager._TRANSFORMERS_TASKS_TO_MODEL_LOADERS[
-        "image-text-to-text"
-    ] = TasksManager._TRANSFORMERS_TASKS_TO_MODEL_LOADERS["text-generation"]
-
-    TasksManager._TRANSFORMERS_TASKS_TO_MODEL_LOADERS["video-text-to-text"] = "AutoModelForVision2Seq"
-
     if is_diffusers_available() and "fill" not in TasksManager._DIFFUSERS_TASKS_TO_MODEL_LOADERS:
         TasksManager._DIFFUSERS_TASKS_TO_MODEL_LOADERS["fill"] = "FluxFillPipeline"
         TasksManager._DIFFUSERS_TASKS_TO_MODEL_MAPPINGS["fill"] = {"flux": "FluxFillPipeline"}
@@ -1698,9 +1692,7 @@ class LlavaNextVideoConfigBehavior(str, enum.Enum):
     TEXT_EMBEDDINGS = "text_embeddings"
 
 
-@register_in_tasks_manager(
-    "llava_next_video", *["image-text-to-text", "video-text-to-text"], library_name="transformers"
-)
+@register_in_tasks_manager("llava_next_video", *["image-text-to-text"], library_name="transformers")
 class LlavaNextVideoOpenVINOConfig(LlavaOpenVINOConfig):
     MIN_TRANSFORMERS_VERSION = "4.42.0"
     SUPPORTED_BEHAVIORS = [model_type.value for model_type in LlavaNextVideoConfigBehavior]
@@ -3301,11 +3293,7 @@ class Qwen2VLConfigBehavior(str, enum.Enum):
     TEXT_EMBEDDINGS = "text_embeddings"
 
 
-@register_in_tasks_manager(
-    "qwen2_vl",
-    *["image-text-to-text", "video-text-to-text"],
-    library_name="transformers",
-)
+@register_in_tasks_manager("qwen2_vl", *["image-text-to-text"], library_name="transformers")
 class Qwen2VLOpenVINOConfig(BaseVLMOpenVINOConfig):
     SUPPORTED_BEHAVIORS = [model_type.value for model_type in Qwen2VLConfigBehavior]
     NORMALIZED_CONFIG_CLASS = NormalizedVisionConfig
@@ -3436,11 +3424,7 @@ def outputs(self) -> Dict[str, Dict[int, str]]:
         return {}
 
 
-@register_in_tasks_manager(
-    "qwen2_5_vl",
-    *["image-text-to-text", "video-text-to-text"],
-    library_name="transformers",
-)
+@register_in_tasks_manager("qwen2_5_vl", *["image-text-to-text"], library_name="transformers")
 class Qwen2_5_VLOpenVINOConfig(Qwen2VLOpenVINOConfig):
     MIN_TRANSFORMERS_VERSION = "4.49.0"
 
@@ -3784,7 +3768,7 @@ def generate(self, input_name: str, framework: str = "pt", int_dtype: str = "int
         return super().generate(input_name, framework, int_dtype, float_dtype)
 
 
-@register_in_tasks_manager("idefics3", *["image-text-to-text", "video-text-to-text"], library_name="transformers")
+@register_in_tasks_manager("idefics3", *["image-text-to-text"], library_name="transformers")
 class Idefics3OpenVINOConfig(BaseVLMOpenVINOConfig):
     DUMMY_INPUT_GENERATOR_CLASSES = (DummyVisionInputGenerator, DummyVisionPositionIdsInputGenerator)
     MIN_TRANSFORMERS_VERSION = "4.46.0"
@@ -3843,7 +3827,7 @@ def get_model_for_behavior(self, model, behavior: Union[str, VLMConfigBehavior])
             return text_embedding
 
 
-@register_in_tasks_manager("smolvlm", *["image-text-to-text", "video-text-to-text"], library_name="transformers")
+@register_in_tasks_manager("smolvlm", *["image-text-to-text"], library_name="transformers")
 class SmolVLMOpenVINOConfig(Idefics3OpenVINOConfig):
     MIN_TRANSFORMERS_VERSION = "4.50.0"
 

optimum/intel/openvino/quantization.py

@@ -40,11 +40,9 @@
 from transformers.pytorch_utils import Conv1D
 from transformers.utils import is_accelerate_available
 
-from optimum.exporters.tasks import TasksManager
 from optimum.quantization_base import OptimumQuantizer
 from optimum.utils.logging import warn_once
 
-from ..utils.constant import _TASK_ALIASES
 from ..utils.import_utils import (
     DATASETS_IMPORT_ERROR,
     _nncf_version,
@@ -1142,15 +1140,20 @@ def __init__(self, model: OVModel, task: Optional[str] = None, seed: int = 42, *
         Args:
             model (`OVModel`):
                 The [OVModel](https://huggingface.co/docs/optimum-intel/en/openvino/reference) to quantize.
-            task (`str`, defaults to None):
-                The task defining the model topology used for the ONNX export.
             seed (`int`, defaults to 42):
                 The random seed to use when shuffling the calibration dataset.
         """
         super().__init__()
         self.model = model
-        self.task = task
         self.dataset_builder = OVCalibrationDatasetBuilder(model, seed)
+        self._task = task
+        if self._task is not None:
+            logger.warning("The `task` argument is ignored and will be removed in optimum-intel v1.27")
+
+    @property
+    def task(self) -> Dict[str, Union[openvino.Model, openvino.runtime.CompiledModel]]:
+        logger.warning("The `task` attribute is deprecated and will be removed in v1.27.")
+        return self._task
 
     @classmethod
     def from_pretrained(cls, model: OVModel, **kwargs):
@@ -1196,7 +1199,7 @@ def quantize(
         >>> from optimum.intel import OVQuantizer, OVModelForCausalLM
         >>> from transformers import AutoModelForCausalLM
         >>> model = AutoModelForCausalLM.from_pretrained("databricks/dolly-v2-3b")
-        >>> quantizer = OVQuantizer.from_pretrained(model, task="text-generation")
+        >>> quantizer = OVQuantizer.from_pretrained(model)
         >>> ov_config = OVConfig(quantization_config=OVWeightQuantizationConfig())
         >>> quantizer.quantize(ov_config=ov_config, save_directory="./quantized_model")
         >>> optimized_model = OVModelForCausalLM.from_pretrained("./quantized_model")
@@ -1208,7 +1211,7 @@ def quantize(
         >>> model = OVModelForSequenceClassification.from_pretrained("distilbert-base-uncased-finetuned-sst-2-english", export=True)
         >>> # or
         >>> model = AutoModelForSequenceClassification.from_pretrained("distilbert-base-uncased-finetuned-sst-2-english")
-        >>> quantizer = OVQuantizer.from_pretrained(model, task="text-classification")
+        >>> quantizer = OVQuantizer.from_pretrained(model)
         >>> ov_config = OVConfig(quantization_config=OVQuantizationConfig())
         >>> quantizer.quantize(calibration_dataset=dataset, ov_config=ov_config, save_directory="./quantized_model")
         >>> optimized_model = OVModelForSequenceClassification.from_pretrained("./quantized_model")
@@ -1454,22 +1457,6 @@ def _save_pretrained(model: openvino.Model, output_path: str):
         compress_quantize_weights_transformation(model)
         openvino.save_model(model, output_path, compress_to_fp16=False)
 
-    def _set_task(self):
-        if self.task is None:
-            self.task = TasksManager.infer_task_from_model(self.model.config._name_or_path)
-            if self.task is None:
-                raise ValueError(
-                    "The task defining the model topology could not be extracted and needs to be specified for the ONNX export."
-                )
-
-        self.task = _TASK_ALIASES.get(self.task, self.task)
-
-        if self.task == "text2text-generation":
-            raise ValueError("Seq2Seq models are currently not supported for post-training static quantization.")
-
-        if self.task == "image-to-text":
-            raise ValueError("Image2Text models are currently not supported for post-training static quantization.")
-
     def get_calibration_dataset(
         self,
         dataset_name: str,