| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | |
| 4 | import logging |
| 5 | |
| 6 | from vllm.config import VllmConfig |
| 7 | from vllm.plugins import IO_PROCESSOR_PLUGINS_GROUP, load_plugins_by_group |
| 8 | from vllm.plugins.io_processors.interface import IOProcessor |
| 9 | from vllm.renderers import BaseRenderer |
| 10 | from vllm.utils.import_utils import resolve_obj_by_qualname |
| 11 | |
| 12 | logger = logging.getLogger(__name__) |
| 13 | |
| 14 | |
| 15 | def has_io_processor( |
| 16 | vllm_config: VllmConfig, |
| 17 | plugin_from_init: str | None = None, |
| 18 | ): |
| 19 | if plugin_from_init: |
| 20 | model_plugin = plugin_from_init |
| 21 | else: |
| 22 | # A plugin can be specified via the model config |
| 23 | # Retrieve the model specific plugin if available |
| 24 | # This is using a custom field in the hf_config for the model |
| 25 | hf_config = vllm_config.model_config.hf_config.to_dict() |
| 26 | config_plugin = hf_config.get("io_processor_plugin") |
| 27 | model_plugin = config_plugin |
| 28 | |
| 29 | return model_plugin is not None |
| 30 | |
| 31 | |
| 32 | def get_io_processor( |
| 33 | vllm_config: VllmConfig, |
| 34 | renderer: BaseRenderer, |
| 35 | plugin_from_init: str | None = None, |
| 36 | ) -> IOProcessor | None: |
| 37 | # Input.Output processors are loaded as plugins under the |
| 38 | # 'vllm.io_processor_plugins' group. Similar to platform |
| 39 | # plugins, these plugins register a function that returns the class |
| 40 | # name for the processor to install. |
| 41 | |
| 42 | if plugin_from_init: |
| 43 | model_plugin = plugin_from_init |
| 44 | else: |
| 45 | # A plugin can be specified via the model config |
| 46 | # Retrieve the model specific plugin if available |
| 47 | # This is using a custom field in the hf_config for the model |
| 48 | hf_config = vllm_config.model_config.hf_config.to_dict() |
| 49 | config_plugin = hf_config.get("io_processor_plugin") |
| 50 | model_plugin = config_plugin |
| 51 | |
| 52 | if model_plugin is None: |
| 53 | logger.debug("No IOProcessor plugins requested by the model") |
| 54 | return None |
| 55 | |
| 56 | logger.debug("IOProcessor plugin to be loaded %s", model_plugin) |
| 57 | |
| 58 | # Load all installed plugin in the group |
| 59 | multimodal_data_processor_plugins = load_plugins_by_group( |
| 60 | IO_PROCESSOR_PLUGINS_GROUP |
| 61 | ) |
| 62 | |
| 63 | loadable_plugins = {} |
| 64 | for name, func in multimodal_data_processor_plugins.items(): |
| 65 | try: |
| 66 | assert callable(func) |
| 67 | processor_cls_qualname = func() |
| 68 | if processor_cls_qualname is not None: |
| 69 | loadable_plugins[name] = processor_cls_qualname |
| 70 | except Exception: |
| 71 | logger.warning("Failed to load plugin %s.", name, exc_info=True) |
| 72 | |
| 73 | num_available_plugins = len(loadable_plugins.keys()) |
| 74 | if num_available_plugins == 0: |
| 75 | raise ValueError( |
| 76 | f"No IOProcessor plugins installed but one is required ({model_plugin})." |
| 77 | ) |
| 78 | |
| 79 | if model_plugin not in loadable_plugins: |
| 80 | raise ValueError( |
| 81 | f"The model requires the '{model_plugin}' IO Processor plugin " |
| 82 | "but it is not installed. " |
| 83 | f"Available plugins: {list(loadable_plugins.keys())}" |
| 84 | ) |
| 85 | |
| 86 | activated_plugin_cls = resolve_obj_by_qualname(loadable_plugins[model_plugin]) |
| 87 | |
| 88 | return activated_plugin_cls(vllm_config, renderer) |