@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 41060c6e08
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 15, 02:15 UTC

[Core] Add Support for Default Modality Specific LoRAs [generate / chat completions] (#19126)

Signed-off-by: Alex-Brooks <Alex.Brooks@ibm.com>

Alex Brooks committed Jul 10, 2025 at 14:09 UTC 41060c6e085c48a20c8b23a0032cfab050f3f06d
9 files changed +482 -5
docs/features/lora.md
+77
@@ -272,3 +272,80 @@ The new format of `--lora-modules` is mainly to support the display of parent mo
272 ]
273 }
274 ```
275 +
276 +## Default LoRA Models For Multimodal Models
277 +
278 +Some models, e.g., [Granite Speech](https://huggingface.co/ibm-granite/granite-speech-3.3-8b) and [Phi-4-multimodal-instruct](https://huggingface.co/microsoft/Phi-4-multimodal-instruct) multimodal, contain LoRA adapter(s) that are expected to always be applied when a given modality is present. This can be a bit tedious to manage with the above approaches, as it requires the user to send the `LoRARequest` (offline) or to filter requests between the base model and LoRA model (server) depending on the content of the request's multimodal data.
279 +
280 +To this end, we allow registration of default multimodal LoRAs to handle this automatically, where users can map each modality to a LoRA adapter to automatically apply it when the corresponding inputs are present. Note that currently, we only allow one LoRA per prompt; if several modalities are provided, each of which are registered to a given modality, none of them will be applied.
281 +
282 +Example usage for offline inference:
283 +
284 +```python
285 +from transformers import AutoTokenizer
286 +from vllm import LLM, SamplingParams
287 +from vllm.assets.audio import AudioAsset
288 +
289 +model_id = "ibm-granite/granite-speech-3.3-2b"
290 +tokenizer = AutoTokenizer.from_pretrained(model_id)
291 +
292 +def get_prompt(question: str, has_audio: bool):
293 + """Build the input prompt to send to vLLM."""
294 + if has_audio:
295 + question = f"<|audio|>{question}"
296 + chat = [
297 + {
298 + "role": "user",
299 + "content": question
300 + }
301 + ]
302 + return tokenizer.apply_chat_template(chat, tokenize=False)
303 +
304 +
305 +model = LLM(
306 + model=model_id,
307 + enable_lora=True,
308 + max_lora_rank=64,
309 + max_model_len=2048,
310 + limit_mm_per_prompt={"audio": 1},
311 + # Will always pass a `LoRARequest` with the `model_id`
312 + # whenever audio is contained in the request data.
313 + default_mm_loras = {"audio": model_id},
314 + enforce_eager=True,
315 +)
316 +
317 +question = "can you transcribe the speech into a written format?"
318 +prompt_with_audio = get_prompt(
319 + question=question,
320 + has_audio=True,
321 +)
322 +audio = AudioAsset("mary_had_lamb").audio_and_sample_rate
323 +
324 +inputs = {
325 + "prompt": prompt_with_audio,
326 + "multi_modal_data": {
327 + "audio": audio,
328 + }
329 +}
330 +
331 +
332 +outputs = model.generate(
333 + inputs,
334 + sampling_params=SamplingParams(
335 + temperature=0.2,
336 + max_tokens=64,
337 + ),
338 +)
339 +```
340 +
341 +You can also pass a json dictionary of `--default-mm-loras` mapping modalities to LoRA model IDs. For example, when starting the server:
342 +
343 +```bash
344 +vllm serve ibm-granite/granite-speech-3.3-2b \
345 + --max-model-len 2048 \
346 + --enable-lora \
347 + --default-mm-loras '{"audio":"ibm-granite/granite-speech-3.3-2b"}' \
348 + --max-lora-rank 64
349 +```
350 +
351 +Note: Default multimodal LoRAs are currently only available for `.generate` and chat completions.
tests/entrypoints/openai/test_default_mm_loras.py new
+107
@@ -0,0 +1,107 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +
4 +import os
5 +
6 +import openai # use the official client for correctness check
7 +import pytest
8 +import pytest_asyncio
9 +from huggingface_hub import snapshot_download
10 +
11 +from ...conftest import AudioTestAssets
12 +from ...utils import RemoteOpenAIServer
13 +
14 +# NOTE - the tests in this module are currently analogous to test_chat, but are
15 +# separated to avoid OOM killing due to module-scoped servers, since we
16 +# need a multimodal model for these tests.
17 +
18 +# Contains a modality specific lora alongside the base model
19 +MULTIMODAL_MODEL_NAME = snapshot_download(
20 + "microsoft/Phi-4-multimodal-instruct")
21 +AUDIO_LORA_PATH = os.path.join(MULTIMODAL_MODEL_NAME, "speech-lora")
22 +
23 +ACTIVE_MM_LORA_RESPONSE = "Spoken text: The first words I spoke in the original chronograph, a little piece of practical poetry. Mary had a little lamb, it slept with quite a snow, and everywhere that Mary went, the lamb was sure to go." # noqa: E501
24 +
25 +
26 +@pytest.fixture(scope="module")
27 +def monkeypatch_module():
28 + from _pytest.monkeypatch import MonkeyPatch
29 + mpatch = MonkeyPatch()
30 + yield mpatch
31 + mpatch.undo()
32 +
33 +
34 +@pytest.fixture(scope="module", params=[False, True])
35 +def multimodal_server(request, monkeypatch_module): # noqa: F811
36 +
37 + use_v1 = request.param
38 + monkeypatch_module.setenv('VLLM_USE_V1', '1' if use_v1 else '0')
39 +
40 + args = [
41 + # use half precision for speed and memory savings in CI environment
42 + "--dtype",
43 + "half",
44 + "--max-model-len",
45 + "12800",
46 + "--enforce-eager",
47 + # lora config below
48 + "--enable-lora",
49 + "--lora-modules",
50 + f"speech={AUDIO_LORA_PATH}",
51 + "--max-lora-rank",
52 + "320",
53 + "--max-num-seqs",
54 + "2",
55 + "--trust-remote-code",
56 + "--gpu-memory-utilization",
57 + "0.8",
58 + "--default-mm-loras",
59 + f"{{\"audio\": \"{AUDIO_LORA_PATH}\"}}",
60 + ]
61 +
62 + with RemoteOpenAIServer(MULTIMODAL_MODEL_NAME, args) as remote_server:
63 + yield remote_server
64 +
65 +
66 +@pytest_asyncio.fixture
67 +async def multi_modal_client(multimodal_server):
68 + async with multimodal_server.get_async_client() as async_client:
69 + yield async_client
70 +
71 +
72 +@pytest.mark.asyncio
73 +@pytest.mark.parametrize(
74 + # base model with default lora should give the same response as lora model
75 + "model_name",
76 + [MULTIMODAL_MODEL_NAME, "speech"],
77 +)
78 +async def test_default_mm_lora_chat_completions(
79 + model_name: str,
80 + multi_modal_client: openai.AsyncOpenAI,
81 + audio_assets: AudioTestAssets,
82 +):
83 + messages = [{
84 + "role":
85 + "user",
86 + "content": [{
87 + "type": "text",
88 + "text": "Can you transcribe this audio?",
89 + }, {
90 + "type": "audio_url",
91 + "audio_url": {
92 + "url": audio_assets[0].url
93 + },
94 + }]
95 + }]
96 +
97 + chat_completion = await multi_modal_client.chat.completions.create(
98 + model=model_name,
99 + messages=messages,
100 + max_completion_tokens=128,
101 + temperature=0.0)
102 +
103 + assert len(chat_completion.choices) > 0
104 +
105 + message = chat_completion.choices[0].message
106 + assert message.content is not None and len(message.content) >= 0
107 + assert message.content == ACTIVE_MM_LORA_RESPONSE
tests/lora/test_default_mm_loras.py new
+118
@@ -0,0 +1,118 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +"""
4 +Tests for applying default registered multimodal loras.
5 +"""
6 +
7 +import os
8 +
9 +from huggingface_hub import snapshot_download
10 +
11 +from vllm.lora.request import LoRARequest
12 +
13 +from ..conftest import AudioTestAssets, VllmRunner
14 +
15 +MODEL_PATH = snapshot_download("microsoft/Phi-4-multimodal-instruct")
16 +AUDIO_LORA_PATH = os.path.join(MODEL_PATH, "speech-lora")
17 +IMAGE_LORA_PATH = os.path.join(MODEL_PATH, "vision-lora")
18 +
19 +AUDIO_PROMPT = "<|user|><|audio_1|>Can you transcribe this audio?<|end|><|assistant|>" # noqa: E501
20 +
21 +# Responses are greedy decoded; we just check the end of
22 +# the generated text. If the lora is inactive, this model
23 +# generates commentary on the transcription.
24 +RESPONSE_SUFFIX_WITH_LORA = "Spoken text: The first words I spoke in the original chronograph, a little piece of practical poetry. Mary had a little lamb, it slept with quite a snow, and everywhere that Mary went, the lamb was sure to go." # noqa: E501
25 +RESPONSE_SUFFIX_WITHOUT_LORA = "Certainly! Here is the transcription of the audio you provided:\n\nThe first words I spoke in the original phonograph record: A little piece of practical poetry. Mary had a little lamb; its fleece was white as snow, and everywhere that Mary went, the lamb was sure to go." # noqa: E501
26 +
27 +VLLM_RUNNER_BASE_KWARGS = {
28 + "model_name": MODEL_PATH,
29 + "dtype": "half",
30 + "enable_lora": "True",
31 + "max_num_seqs": 2,
32 + "max_lora_rank": 320,
33 + "max_model_len": 12800,
34 + "gpu_memory_utilization": 0.8,
35 + "limit_mm_per_prompt": {
36 + "audio": 1
37 + },
38 + "enforce_eager": True,
39 +}
40 +
41 +
42 +def run_test(vllm_runner, audio_assets, lora_request, expected_suffix,
43 + **kwargs):
44 + inputs = [([AUDIO_PROMPT], [audio_assets[0].audio_and_sample_rate[0]])]
45 +
46 + # Apply any additional kwargs as overrides to the base kwargs
47 + vllm_runner_kwargs = {**VLLM_RUNNER_BASE_KWARGS, **kwargs}
48 +
49 + with vllm_runner(**vllm_runner_kwargs) as vllm_model:
50 + vllm_outputs_with_default_lora = [
51 + vllm_model.generate_greedy(
52 + prompts,
53 + max_tokens=128,
54 + audios=audios,
55 + lora_request=lora_request,
56 + ) for prompts, audios in inputs
57 + ]
58 +
59 + assert vllm_outputs_with_default_lora[-1][-1][-1].endswith(
60 + expected_suffix)
61 +
62 +
63 +def test_active_default_mm_lora(
64 + vllm_runner: type[VllmRunner],
65 + audio_assets: AudioTestAssets,
66 +):
67 + """Ensure that we can use the default audio lora."""
68 + run_test(
69 + vllm_runner,
70 + audio_assets,
71 + lora_request=None,
72 + default_mm_loras={"audio": AUDIO_LORA_PATH},
73 + expected_suffix=RESPONSE_SUFFIX_WITH_LORA,
74 + )
75 +
76 +
77 +def test_inactive_default_mm_lora(
78 + vllm_runner: type[VllmRunner],
79 + audio_assets: AudioTestAssets,
80 +):
81 + """Ensure that modalities are filtered properly."""
82 + # Default image lora won't be active since we only pass audio
83 + run_test(
84 + vllm_runner,
85 + audio_assets,
86 + lora_request=None,
87 + default_mm_loras={"image": IMAGE_LORA_PATH},
88 + expected_suffix=RESPONSE_SUFFIX_WITHOUT_LORA,
89 + )
90 +
91 +
92 +def test_default_mm_lora_succeeds_with_redundant_lora_request(
93 + vllm_runner: type[VllmRunner],
94 + audio_assets: AudioTestAssets,
95 +):
96 + """Ensure that redundantly providing the lora works."""
97 + run_test(
98 + vllm_runner,
99 + audio_assets,
100 + lora_request=LoRARequest("audio", 1, AUDIO_LORA_PATH),
101 + default_mm_loras={"audio": AUDIO_LORA_PATH},
102 + expected_suffix=RESPONSE_SUFFIX_WITH_LORA,
103 + )
104 +
105 +
106 +def test_default_mm_lora_fails_with_overridden_lora_request(
107 + vllm_runner: type[VllmRunner],
108 + audio_assets: AudioTestAssets,
109 +):
110 + """Ensure that if the lora_request conflicts with default_mm_loras,
111 + we use the lora_request."""
112 + run_test(
113 + vllm_runner,
114 + audio_assets,
115 + lora_request=LoRARequest("speech", 2, AUDIO_LORA_PATH),
116 + default_mm_loras={"audio": IMAGE_LORA_PATH},
117 + expected_suffix=RESPONSE_SUFFIX_WITH_LORA,
118 + )
vllm/config.py
+11
@@ -33,6 +33,7 @@ import vllm.envs as envs
33 from vllm import version
34 from vllm.compilation.inductor_pass import CallableInductorPass, InductorPass
35 from vllm.logger import init_logger
36 +from vllm.model_executor.layers.quantization import QuantizationMethods
37 from vllm.platforms import current_platform
38 from vllm.transformers_utils.config import (
39 ConfigFormat, get_config, get_hf_image_processor_config,
@@ -2989,6 +2990,16 @@ class LoRAConfig:
2990 trained with those scaling factors to be used at the same time. If not
2991 specified, only adapters trained with the base model scaling factor are
2992 allowed."""
2993 + default_mm_loras: Optional[dict[str, str]] = None
2994 + """Dictionary mapping specific modalities to LoRA model paths; this field
2995 + is only applicable to multimodal models and should be leveraged when a
2996 + model always expects a LoRA to be active when a given modality is present.
2997 + Note that currently, if a request provides multiple additional
2998 + modalities, each of which have their own LoRA, we do NOT apply
2999 + default_mm_loras because we currently only support one lora adapter
3000 + per prompt. When run in offline mode, the lora IDs for n modalities
3001 + will be automatically assigned to 1-n with the names of the modalities
3002 + in alphabetic order."""
3003 bias_enabled: bool = False
3004 """Enable bias for LoRA adapters."""
3005
vllm/engine/arg_utils.py
+10
@@ -395,6 +395,8 @@ class EngineArgs:
395 enable_lora_bias: bool = LoRAConfig.bias_enabled
396 max_loras: int = LoRAConfig.max_loras
397 max_lora_rank: int = LoRAConfig.max_lora_rank
398 + default_mm_loras: Optional[Dict[str, str]] = \
399 + LoRAConfig.default_mm_loras
400 fully_sharded_loras: bool = LoRAConfig.fully_sharded_loras
401 max_cpu_loras: Optional[int] = LoRAConfig.max_cpu_loras
402 lora_dtype: Optional[Union[str, torch.dtype]] = LoRAConfig.lora_dtype
@@ -807,6 +809,8 @@ class EngineArgs:
809 **lora_kwargs["max_cpu_loras"])
810 lora_group.add_argument("--fully-sharded-loras",
811 **lora_kwargs["fully_sharded_loras"])
812 + lora_group.add_argument("--default-mm-loras",
813 + **lora_kwargs["default_mm_loras"])
814
815 # PromptAdapter related configs
816 prompt_adapter_kwargs = get_kwargs(PromptAdapterConfig)
@@ -1284,10 +1288,16 @@ class EngineArgs:
1288 disable_hybrid_kv_cache_manager,
1289 )
1290
1291 + if not model_config.is_multimodal_model and self.default_mm_loras:
1292 + raise ValueError(
1293 + "Default modality-specific LoRA(s) were provided for a "
1294 + "non multimodal model")
1295 +
1296 lora_config = LoRAConfig(
1297 bias_enabled=self.enable_lora_bias,
1298 max_lora_rank=self.max_lora_rank,
1299 max_loras=self.max_loras,
1300 + default_mm_loras=self.default_mm_loras,
1301 fully_sharded_loras=self.fully_sharded_loras,
1302 lora_extra_vocab_size=self.lora_extra_vocab_size,
1303 long_lora_scaling_factors=self.long_lora_scaling_factors,
vllm/entrypoints/llm.py
+81
@@ -499,6 +499,10 @@ class LLM:
499 _validate_truncation_size(self.llm_engine.model_config.max_model_len,
500 truncate_prompt_tokens, tokenization_kwargs)
501
502 + # Add any modality specific loras to the corresponding prompts
503 + lora_request = self._get_modality_specific_lora_reqs(
504 + parsed_prompts, lora_request)
505 +
506 self._validate_and_add_requests(
507 prompts=parsed_prompts,
508 params=sampling_params,
@@ -513,6 +517,83 @@ class LLM:
517 outputs = self._run_engine(use_tqdm=use_tqdm)
518 return self.engine_class.validate_outputs(outputs, RequestOutput)
519
520 + def _get_modality_specific_lora_reqs(
521 + self, parsed_prompts: Union[PromptType, Sequence[PromptType]],
522 + lora_request: Optional[Union[list[LoRARequest], LoRARequest]]):
523 + # Grab the lora config off the vllm config on the engine,
524 + # since this is the same for both v0 & v1.
525 + lora_config = self.llm_engine.vllm_config.lora_config
526 +
527 + # If there's no lora config / default_mm_loras, or the model
528 + # isn't multimodal, leave the lora as is.
529 + if (lora_config is None
530 + or not self.llm_engine.model_config.is_multimodal_model
531 + or (lora_config and lora_config.default_mm_loras is None)):
532 + return lora_request
533 +
534 + if not isinstance(parsed_prompts, Sequence):
535 + parsed_prompts = [parsed_prompts]
536 +
537 + optional_loras = ([lora_request] * len(parsed_prompts)
538 + if not isinstance(lora_request, Sequence) else
539 + lora_request)
540 +
541 + return [
542 + self._resolve_single_prompt_mm_lora(
543 + parsed_prompt,
544 + opt_lora_req,
545 + lora_config.default_mm_loras,
546 + ) for parsed_prompt, opt_lora_req in zip(parsed_prompts,
547 + optional_loras)
548 + ]
549 +
550 + def _resolve_single_prompt_mm_lora(self, parsed_prompt: PromptType,
551 + lora_request: Optional[LoRARequest],
552 + default_mm_loras: Optional[dict[str,
553 + str]]):
554 + if (not default_mm_loras or not isinstance(parsed_prompt, dict)
555 + or "multi_modal_data" not in parsed_prompt):
556 + return lora_request
557 +
558 + parsed_prompt = cast(Union[TextPrompt, TokensPrompt], parsed_prompt)
559 +
560 + intersection = set(
561 + parsed_prompt["multi_modal_data"].keys()).intersection(
562 + default_mm_loras.keys())
563 + if not intersection:
564 + return lora_request
565 + if len(intersection) > 1:
566 + # TODO: Would be nice to be able to have multiple loras per prompt
567 + logger.warning(
568 + "Multiple modality specific loras were registered and would be"
569 + " used by a single prompt consuming several modalities; "
570 + " currently we only support one lora per request; as such,"
571 + " lora(s) registered with modalities: %s"
572 + " will be skipped", intersection)
573 + return lora_request
574 +
575 + # Build the LoRA request; the ID of the default mm lora is the
576 + # index of the modality name sorted alphabetically + 1.
577 + modality_name = intersection.pop()
578 + modality_lora_path = default_mm_loras[modality_name]
579 + modality_lora_id = sorted(default_mm_loras).index(modality_name) + 1
580 +
581 + # If we have a collision, warn if there is a collision,
582 + # but always send the explicitly provided request.
583 + if lora_request:
584 + if lora_request.lora_int_id != modality_lora_id:
585 + logger.warning(
586 + "A modality with a registered lora and a lora_request "
587 + "with a different ID were provided; falling back to the "
588 + "lora_request as we only apply one LoRARequest per prompt")
589 + return lora_request
590 +
591 + return LoRARequest(
592 + modality_name,
593 + modality_lora_id,
594 + modality_lora_path,
595 + )
596 +
597 def collective_rpc(self,
598 method: Union[str, Callable[..., _R]],
599 timeout: Optional[float] = None,
vllm/entrypoints/openai/api_server.py
+19 -1
@@ -87,6 +87,7 @@ from vllm.entrypoints.openai.serving_completion import OpenAIServingCompletion
87 from vllm.entrypoints.openai.serving_embedding import OpenAIServingEmbedding
88 from vllm.entrypoints.openai.serving_engine import OpenAIServing
89 from vllm.entrypoints.openai.serving_models import (BaseModelPath,
90 + LoRAModulePath,
91 OpenAIServingModels)
92 from vllm.entrypoints.openai.serving_pooling import OpenAIServingPooling
93 from vllm.entrypoints.openai.serving_responses import OpenAIServingResponses
@@ -1481,11 +1482,28 @@ async def init_app_state(
1482 "This discrepancy may lead to performance degradation.",
1483 resolved_chat_template, args.model)
1484
1485 + # Merge default_mm_loras into the static lora_modules
1486 + default_mm_loras = (vllm_config.lora_config.default_mm_loras
1487 + if vllm_config.lora_config is not None else {})
1488 +
1489 + lora_modules = args.lora_modules
1490 + if default_mm_loras:
1491 + default_mm_lora_paths = [
1492 + LoRAModulePath(
1493 + name=modality,
1494 + path=lora_path,
1495 + ) for modality, lora_path in default_mm_loras.items()
1496 + ]
1497 + if args.lora_modules is None:
1498 + lora_modules = default_mm_lora_paths
1499 + else:
1500 + lora_modules += default_mm_lora_paths
1501 +
1502 state.openai_serving_models = OpenAIServingModels(
1503 engine_client=engine_client,
1504 model_config=model_config,
1505 base_model_paths=base_model_paths,
1488 - lora_modules=args.lora_modules,
1506 + lora_modules=lora_modules,
1507 prompt_adapters=args.prompt_adapters,
1508 )
1509 await state.openai_serving_models.init_static_loras()
vllm/entrypoints/openai/serving_chat.py
+2 -1
@@ -153,7 +153,8 @@ class OpenAIServingChat(OpenAIServing):
153 (
154 lora_request,
155 prompt_adapter_request,
156 - ) = self._maybe_get_adapters(request)
156 + ) = self._maybe_get_adapters(request,
157 + supports_default_mm_loras=True)
158
159 model_name = self._get_model_name(request.model, lora_request)
160
vllm/entrypoints/openai/serving_engine.py
+57 -3
@@ -458,20 +458,74 @@ class OpenAIServing:
458 err_type="NotFoundError",
459 status_code=HTTPStatus.NOT_FOUND)
460
461 + def _get_active_default_mm_loras(
462 + self, request: AnyRequest) -> Optional[LoRARequest]:
463 + """Determine if there are any active default multimodal loras."""
464 + # TODO: Currently this is only enabled for chat completions
465 + # to be better aligned with only being enabled for .generate
466 + # when run offline. It would be nice to support additional
467 + # tasks types in the future.
468 + message_types = self._get_message_types(request)
469 + default_mm_loras = set()
470 +
471 + for lora in self.models.lora_requests.values():
472 + # Best effort match for default multimodal lora adapters;
473 + # There is probably a better way to do this, but currently
474 + # this matches against the set of 'types' in any content lists
475 + # up until '_', e.g., to match audio_url -> audio
476 + if lora.lora_name in message_types:
477 + default_mm_loras.add(lora)
478 +
479 + # Currently only support default modality specific loras if
480 + # we have exactly one lora matched on the request.
481 + if len(default_mm_loras) == 1:
482 + return default_mm_loras.pop()
483 + return None
484 +
485 def _maybe_get_adapters(
462 - self, request: AnyRequest
486 + self,
487 + request: AnyRequest,
488 + supports_default_mm_loras: bool = False,
489 ) -> Union[tuple[None, None], tuple[LoRARequest, None], tuple[
490 None, PromptAdapterRequest]]:
465 - if self._is_model_supported(request.model):
466 - return None, None
491 +
492 if request.model in self.models.lora_requests:
493 return self.models.lora_requests[request.model], None
494 +
495 + # Currently only support default modality specific loras
496 + # if we have exactly one lora matched on the request.
497 + if supports_default_mm_loras:
498 + default_mm_lora = self._get_active_default_mm_loras(request)
499 + if default_mm_lora is not None:
500 + return default_mm_lora, None
501 +
502 + if self._is_model_supported(request.model):
503 + return None, None
504 +
505 for prompt_adapter in self.models.prompt_adapter_requests:
506 if request.model == prompt_adapter.prompt_adapter_name:
507 return None, prompt_adapter
508 # if _check_model has been called earlier, this will be unreachable
509 raise ValueError(f"The model `{request.model}` does not exist.")
510
511 + def _get_message_types(self, request: AnyRequest) -> set[str]:
512 + """Retrieve the set of types from message content dicts up
513 + until `_`; we use this to match potential multimodal data
514 + with default per modality loras.
515 + """
516 + message_types: set[str] = set()
517 +
518 + if not hasattr(request, "messages"):
519 + return message_types
520 +
521 + for message in request.messages:
522 + if (isinstance(message, dict) and "content" in message
523 + and isinstance(message["content"], list)):
524 + for content_dict in message["content"]:
525 + if "type" in content_dict:
526 + message_types.add(content_dict["type"].split("_")[0])
527 + return message_types
528 +
529 async def _normalize_prompt_text_to_input(
530 self,
531 request: AnyRequest,