A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 7, 02:45 UTC
[bugfix] add api process rank in default multimodal request (#36150)
Signed-off-by: fake0fan <645327136@qq.com> Signed-off-by: Chenguang ZHENG <645327136@qq.com>
Chenguang Zheng committed
Mar 6, 2026 at 20:00 UTC
fcb73f306ccedb07ff33e3e3696018f66ccd40ea
7 files changed
+46
-9
tests/entrypoints/openai/test_chat_error.py
+7
-1
@@ -59,16 +59,22 @@ class MockModelConfig:
59
return self.diff_sampling_param or {}
60
61
62
+@dataclass
63
+class MockParallelConfig:
64
+ _api_process_rank: int = 0
65
+
66
+
67
@dataclass
68
class MockVllmConfig:
69
model_config: MockModelConfig
70
+ parallel_config: MockParallelConfig
71
72
73
def _build_renderer(model_config: MockModelConfig):
74
_, tokenizer_name, _, kwargs = tokenizer_args_from_config(model_config)
75
76
return HfRenderer.from_config(
71
- MockVllmConfig(model_config),
77
+ MockVllmConfig(model_config, parallel_config=MockParallelConfig()),
78
tokenizer_kwargs={**kwargs, "tokenizer_name": tokenizer_name},
79
)
80
tests/entrypoints/openai/test_completion_error.py
+7
-1
@@ -58,9 +58,15 @@ class MockModelConfig:
58
return self.diff_sampling_param or {}
59
60
61
+@dataclass
62
+class MockParallelConfig:
63
+ _api_process_rank: int = 0
64
+
65
+
66
@dataclass
67
class MockVllmConfig:
68
model_config: MockModelConfig
69
+ parallel_config: MockParallelConfig
70
71
72
def _build_serving_completion(engine: AsyncLLM) -> OpenAIServingCompletion:
@@ -79,7 +85,7 @@ def _build_renderer(model_config: MockModelConfig):
85
_, tokenizer_name, _, kwargs = tokenizer_args_from_config(model_config)
86
87
return HfRenderer.from_config(
82
- MockVllmConfig(model_config),
88
+ MockVllmConfig(model_config, parallel_config=MockParallelConfig()),
89
tokenizer_kwargs={**kwargs, "tokenizer_name": tokenizer_name},
90
)
91
tests/entrypoints/openai/test_lora_resolvers.py
+7
-1
@@ -58,9 +58,15 @@ class MockModelConfig:
58
return self.diff_sampling_param or {}
59
60
61
+@dataclass
62
+class MockParallelConfig:
63
+ _api_process_rank: int = 0
64
+
65
+
66
@dataclass
67
class MockVllmConfig:
68
model_config: MockModelConfig
69
+ parallel_config: MockParallelConfig
70
71
72
class MockLoRAResolver(LoRAResolver):
@@ -97,7 +103,7 @@ def _build_renderer(model_config: MockModelConfig):
103
_, tokenizer_name, _, kwargs = tokenizer_args_from_config(model_config)
104
105
return HfRenderer.from_config(
100
- MockVllmConfig(model_config),
106
+ MockVllmConfig(model_config, parallel_config=MockParallelConfig()),
107
tokenizer_kwargs={**kwargs, "tokenizer_name": tokenizer_name},
108
)
109
tests/entrypoints/openai/test_serving_chat.py
+9
-3
@@ -537,16 +537,22 @@ class MockModelConfig:
537
return self.diff_sampling_param or {}
538
539
540
+@dataclass
541
+class MockParallelConfig:
542
+ _api_process_rank: int = 0
543
+
544
+
545
@dataclass
546
class MockVllmConfig:
547
model_config: MockModelConfig
548
+ parallel_config: MockParallelConfig
549
550
551
def _build_renderer(model_config: MockModelConfig):
552
_, tokenizer_name, _, kwargs = tokenizer_args_from_config(model_config)
553
554
return HfRenderer.from_config(
549
- MockVllmConfig(model_config),
555
+ MockVllmConfig(model_config, parallel_config=MockParallelConfig()),
556
tokenizer_kwargs={**kwargs, "tokenizer_name": tokenizer_name},
557
)
558
@@ -797,7 +803,7 @@ async def test_serving_chat_mistral_token_ids_prompt_is_validated():
803
804
mock_tokenizer = MagicMock(spec=MistralTokenizer)
805
mock_renderer = MistralRenderer(
800
- MockVllmConfig(mock_engine.model_config),
806
+ MockVllmConfig(mock_engine.model_config, parallel_config=MockParallelConfig()),
807
tokenizer=mock_tokenizer,
808
)
809
# Force the Mistral chat template renderer to return token IDs.
@@ -837,7 +843,7 @@ async def test_serving_chat_mistral_token_ids_prompt_too_long_is_rejected():
843
844
mock_tokenizer = MagicMock(spec=MistralTokenizer)
845
mock_renderer = MistralRenderer(
840
- MockVllmConfig(mock_engine.model_config),
846
+ MockVllmConfig(mock_engine.model_config, parallel_config=MockParallelConfig()),
847
tokenizer=mock_tokenizer,
848
)
849
# prompt_token_ids length == max_model_len should be rejected for
tests/renderers/test_completions.py
+7
-1
@@ -41,9 +41,15 @@ class MockModelConfig:
41
is_multimodal_model: bool = False
42
43
44
+@dataclass
45
+class MockParallelConfig:
46
+ _api_process_rank: int = 0
47
+
48
+
49
@dataclass
50
class MockVllmConfig:
51
model_config: MockModelConfig
52
+ parallel_config: MockParallelConfig
53
54
55
@dataclass
@@ -78,7 +84,7 @@ def _build_renderer(
84
_, tokenizer_name, _, kwargs = tokenizer_args_from_config(model_config)
85
86
renderer = HfRenderer(
81
- MockVllmConfig(model_config),
87
+ MockVllmConfig(model_config, parallel_config=MockParallelConfig()),
88
tokenizer=(
89
None
90
if model_config.skip_tokenizer_init
tests/renderers/test_mistral.py
+7
-1
@@ -39,9 +39,15 @@ class MockModelConfig:
39
is_multimodal_model: bool = False
40
41
42
+@dataclass
43
+class MockParallelConfig:
44
+ _api_process_rank: int = 0
45
+
46
+
47
@dataclass
48
class MockVllmConfig:
49
model_config: MockModelConfig
50
+ parallel_config: MockParallelConfig
51
52
53
@pytest.mark.asyncio
@@ -57,7 +63,7 @@ async def test_async_mistral_tokenizer_does_not_block_event_loop():
63
mock_tokenizer = Mock(spec=MistralTokenizer)
64
mock_tokenizer.apply_chat_template = mocked_apply_chat_template
65
mock_renderer = MistralRenderer(
60
- MockVllmConfig(mock_model_config),
66
+ MockVllmConfig(mock_model_config, parallel_config=MockParallelConfig()),
67
tokenizer=mock_tokenizer,
68
)
69
vllm/renderers/base.py
+2
-1
@@ -75,6 +75,7 @@ class BaseRenderer(ABC, Generic[_T]):
75
76
self.config = config
77
self.model_config = config.model_config
78
+ self.api_process_rank = config.parallel_config._api_process_rank
79
80
self.tokenizer = tokenizer
81
@@ -539,7 +540,7 @@ class BaseRenderer(ABC, Generic[_T]):
540
from vllm.multimodal.parse import parse_mm_uuids
541
from vllm.multimodal.processing import ProcessorInputs as MMProcessorInputs
542
542
- mm_req_id = f"renderer-mm-{self._mm_req_counter.inc(1)}"
543
+ mm_req_id = f"renderer{self.api_process_rank}-mm-{self._mm_req_counter.inc(1)}"
544
545
mm_processor = self.get_mm_processor()
546