[Security] Fix chat template resource-exhaustion DoS (GHSA-4hhp-h66f-… (#50300)
Signed-off-by: jperezde <jperezde@redhat.com> Signed-off-by: Juan Pérez de Algaba <jperezde@redhat.com> Co-authored-by: Cursor <cursoragent@cursor.com>
Juan Pérez de Algaba committed
Sep 30, 2026 at 17:32 UTC
cff08b461e086523cee8bba5bd3c66d302625134
8 files changed
+230
-9
rust/Cargo.toml
+1
-1
@@ -63,7 +63,7 @@ itertools = "0.14.0"
63
libc = "0.2.177"
64
llm-multimodal = { git = "https://github.com/smg-project/llm-multimodal", rev = "f0985ef65967615db2c79279aa07818499301bfd", default-features = false, features = ["native-tls"] }
65
mimalloc = "0.1.52"
66
-minijinja = { version = "2.24", features = ["unstable_machinery", "json", "builtins", "loader", "loop_controls", "preserve_order"] }
66
+minijinja = { version = "2.24", features = ["unstable_machinery", "json", "builtins", "loader", "loop_controls", "preserve_order", "fuel"] }
67
minijinja-contrib = { version = "2.24", features = ["pycompat"] }
68
native-tls-vendored = { package = "native-tls", version = "0.2.18", features = ["vendored"] }
69
ndarray = { version = "0.17", features = ["serde"] }
rust/src/chat/src/lib.rs
+6
-1
@@ -180,7 +180,12 @@ impl ChatRequestProcessor {
180
async fn prepare_text_request(&self, request: ChatRequest) -> Result<TextRequest> {
181
// Stamp before rendering so render and tokenize count toward TTFT/e2e.
182
let arrival_time = vllm_llm::current_unix_timestamp_secs();
183
- let mut rendered = self.backend.chat_renderer().render(&request)?;
183
+ let renderer = self.backend.chat_renderer();
184
+ let (mut rendered, request) = tokio::task::spawn_blocking(move || {
185
+ renderer.render(&request).map(|rendered| (rendered, request))
186
+ })
187
+ .await
188
+ .map_err(|e| Error::ChatTemplate(format!("render task join error: {e}")))??;
189
let chat_template_kwargs = std::mem::take(&mut rendered.effective_template_kwargs);
190
let (prompt, mm_features) = self.finalize_rendered_prompt(&request, rendered).await?;
191
Ok(TextRequest {
rust/src/chat/src/renderer/hf/template.rs
+7
@@ -29,6 +29,11 @@ use crate::renderer::hf::{TemplateMessage, TemplateTool};
29
30
type Result<T> = std::result::Result<T, TemplateError>;
31
32
+/// Default fuel budget for template rendering. Prevents unbounded loops and
33
+/// string operations from exhausting CPU. Each template instruction consumes
34
+/// one unit of fuel; when depleted, rendering aborts with an error.
35
+const DEFAULT_TEMPLATE_FUEL: u64 = 500_000;
36
+
37
/// Build a pre-configured environment with the given template string.
38
fn build_environment(template: String) -> Result<Environment<'static>> {
39
let mut env = Environment::new();
@@ -36,6 +41,8 @@ fn build_environment(template: String) -> Result<Environment<'static>> {
41
env.set_trim_blocks(true);
42
env.set_lstrip_blocks(true);
43
44
+ env.set_fuel(Some(DEFAULT_TEMPLATE_FUEL));
45
+
46
env.add_template_owned("chat".to_owned(), template)?;
47
48
env.set_unknown_method_callback(minijinja_contrib::pycompat::unknown_method_callback);
tests/renderers/test_executor_replace.py
new
+126
@@ -0,0 +1,126 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+import asyncio
4
+from dataclasses import dataclass
5
+from typing import Any
6
+
7
+import pytest
8
+import torch
9
+
10
+from vllm.renderers.base import _SwappableExecutor
11
+from vllm.renderers.hf import HfRenderer
12
+from vllm.renderers.params import TokenizeParams
13
+from vllm.utils.async_utils import make_async
14
+
15
+MODEL_NAME = "openai-community/gpt2"
16
+
17
+
18
+@dataclass
19
+class MockHFConfig:
20
+ model_type: str = "any"
21
+
22
+
23
+@dataclass
24
+class MockModelConfig:
25
+ runner_type = "generate"
26
+ model: str = MODEL_NAME
27
+ tokenizer: str = MODEL_NAME
28
+ trust_remote_code: bool = False
29
+ tokenizer_revision = None
30
+ tokenizer_mode = "auto"
31
+ hf_config = MockHFConfig()
32
+ encoder_config: dict[str, Any] | None = None
33
+ enable_prompt_embeds: bool = False
34
+ skip_tokenizer_init: bool = False
35
+ is_encoder_decoder: bool = False
36
+ is_multimodal_model: bool = False
37
+ renderer_num_workers: int = 1
38
+ hidden_size: int = 768
39
+ dtype: torch.dtype = torch.float32
40
+
41
+ def get_hidden_size(self) -> int:
42
+ return self.hidden_size
43
+
44
+
45
+@dataclass
46
+class MockParallelConfig:
47
+ _api_process_rank: int = 0
48
+
49
+
50
+@dataclass
51
+class MockVllmConfig:
52
+ model_config: MockModelConfig
53
+ parallel_config: MockParallelConfig
54
+
55
+
56
+@dataclass
57
+class DummyTokenizer:
58
+ truncation_side: str = "left"
59
+ max_chars_per_token: int = 1
60
+ is_fast: bool = False
61
+
62
+ def decode(self, tokens: list[int], **kwargs):
63
+ return str(tokens)
64
+
65
+ def encode(self, text: str, **kwargs):
66
+ return list(range(len(text)))
67
+
68
+ def __call__(self, text: str, **kwargs):
69
+ return {"input_ids": self.encode(text, **kwargs)}
70
+
71
+
72
+def _build_renderer() -> HfRenderer:
73
+ return HfRenderer(
74
+ MockVllmConfig(MockModelConfig(), parallel_config=MockParallelConfig()),
75
+ tokenizer=DummyTokenizer(),
76
+ )
77
+
78
+
79
+def test_swappable_executor_keeps_make_async_wrappers_alive():
80
+ pool = _SwappableExecutor(max_workers=1)
81
+ async_add = make_async(lambda x: x + 1, executor=pool)
82
+
83
+ async def _run():
84
+ assert await async_add(1) == 2
85
+ old_inner = pool._inner
86
+ pool.replace_inner()
87
+
88
+ with pytest.raises(RuntimeError, match="cannot schedule new futures"):
89
+ old_inner.submit(lambda: None)
90
+
91
+ assert await async_add(40) == 41
92
+
93
+ try:
94
+ asyncio.run(_run())
95
+ finally:
96
+ pool.shutdown(wait=False)
97
+
98
+
99
+def test_replace_executor_does_not_break_tokenize_or_decode():
100
+ renderer = _build_renderer()
101
+ executor = renderer._executor
102
+ old_inner = executor._inner
103
+
104
+ async def _run():
105
+ assert await renderer._tokenize_prompt_async(
106
+ {"prompt": "ab"},
107
+ TokenizeParams(max_total_tokens=100),
108
+ )
109
+ renderer._replace_executor()
110
+ assert renderer._executor is executor
111
+ assert executor._inner is not old_inner
112
+
113
+ with pytest.raises(RuntimeError, match="cannot schedule new futures"):
114
+ old_inner.submit(lambda: None)
115
+
116
+ tokenized = await renderer._tokenize_prompt_async(
117
+ {"prompt": "abc"},
118
+ TokenizeParams(max_total_tokens=100),
119
+ )
120
+ assert tokenized["prompt_token_ids"] == [0, 1, 2]
121
+ assert await renderer._async_tokenizer_decode([1, 2]) == "[1, 2]"
122
+
123
+ try:
124
+ asyncio.run(_run())
125
+ finally:
126
+ renderer.shutdown()
vllm/entrypoints/openai/responses/serving.py
+10
@@ -334,6 +334,16 @@ class OpenAIServingResponses(GenerateBaseServing):
334
if maybe_validation_error is not None:
335
return maybe_validation_error
336
337
+ maybe_template_error = self.online_renderer.validate_chat_template(
338
+ request_chat_template=None,
339
+ chat_template_kwargs=request.chat_template_kwargs,
340
+ trust_request_chat_template=(
341
+ self.online_renderer.trust_request_chat_template
342
+ ),
343
+ )
344
+ if maybe_template_error is not None:
345
+ return maybe_template_error
346
+
347
self._preflight()
348
349
if request.store and not self.enable_store:
vllm/envs.py
+6
@@ -25,6 +25,7 @@ if TYPE_CHECKING:
25
CUDA_VISIBLE_DEVICES: str | None = None
26
VLLM_ENGINE_ITERATION_TIMEOUT_S: int = 60
27
VLLM_ENGINE_READY_TIMEOUT_S: int = 600
28
+ VLLM_CHAT_TEMPLATE_RENDER_TIMEOUT: float = 30.0
29
VLLM_API_KEY: str | None = None
30
VLLM_DEBUG_LOG_API_SERVER_RESPONSE: bool = False
31
S3_ACCESS_KEY_ID: str | None = None
@@ -812,6 +813,11 @@ environment_variables: dict[str, Callable[[], Any]] = {
813
"VLLM_ENGINE_READY_TIMEOUT_S": lambda: int(
814
os.environ.get("VLLM_ENGINE_READY_TIMEOUT_S", "600")
815
),
816
+ # Maximum wall-clock seconds allowed for a single chat template render.
817
+ # Set to 0 to disable the timeout.
818
+ "VLLM_CHAT_TEMPLATE_RENDER_TIMEOUT": lambda: float(
819
+ os.environ.get("VLLM_CHAT_TEMPLATE_RENDER_TIMEOUT", "30")
820
+ ),
821
# API key for vLLM API server
822
"VLLM_API_KEY": lambda: os.environ.get("VLLM_API_KEY", None),
823
# Whether to log responses from API Server for debugging
vllm/renderers/base.py
+27
-2
@@ -5,7 +5,7 @@ import time
5
import weakref
6
from abc import ABC, abstractmethod
7
from collections.abc import Mapping, Sequence
8
-from concurrent.futures import Future, ThreadPoolExecutor
8
+from concurrent.futures import Executor, Future, ThreadPoolExecutor
9
from contextlib import ExitStack
10
from dataclasses import replace
11
from functools import cached_property
@@ -77,6 +77,31 @@ logger = init_logger(__name__)
77
_T = TypeVar("_T", bound=TokenizerLike, default=TokenizerLike)
78
79
80
+class _SwappableExecutor(Executor):
81
+ """Executor whose inner pool can be replaced without changing identity.
82
+
83
+ ``make_async`` captures the executor object at wrap time. Replacing
84
+ ``self._executor`` with a new ``ThreadPoolExecutor`` would leave those
85
+ wrappers (tokenize, decode, pooling, derender) bound to a shutdown pool.
86
+ """
87
+
88
+ def __init__(self, max_workers: int) -> None:
89
+ super().__init__()
90
+ self._max_workers = max_workers
91
+ self._inner = ThreadPoolExecutor(max_workers=max_workers)
92
+
93
+ def submit(self, fn, /, *args, **kwargs):
94
+ return self._inner.submit(fn, *args, **kwargs)
95
+
96
+ def shutdown(self, wait: bool = True, *, cancel_futures: bool = False) -> None:
97
+ self._inner.shutdown(wait=wait, cancel_futures=cancel_futures)
98
+
99
+ def replace_inner(self) -> None:
100
+ old = self._inner
101
+ old.shutdown(wait=False)
102
+ self._inner = ThreadPoolExecutor(max_workers=self._max_workers)
103
+
104
+
105
class BaseRenderer(ABC, Generic[_T]):
106
def __init__(self, config: "VllmConfig", tokenizer: _T | None) -> None:
107
super().__init__()
@@ -115,7 +140,7 @@ class BaseRenderer(ABC, Generic[_T]):
140
# multimodal processor receives a deep-copied tokenizer (see #36557)
141
# so it is safe to run tokenization and MM preprocessing concurrently.
142
pool_workers = config.model_config.renderer_num_workers
118
- self._executor = ThreadPoolExecutor(max_workers=pool_workers)
143
+ self._executor = _SwappableExecutor(max_workers=pool_workers)
144
self._resources.callback(self._executor.shutdown, wait=False)
145
146
# Separate single-worker executor so tokenization never queues behind
vllm/renderers/hf.py
+47
-5
@@ -2,6 +2,7 @@
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
from __future__ import annotations
4
5
+import asyncio
6
import copy
7
import inspect
8
import itertools
@@ -20,6 +21,7 @@ import jinja2.sandbox
21
import torch
22
from typing_extensions import override
23
24
+from vllm import envs
25
from vllm.entrypoints.chat_utils import (
26
PROMPT_EMBEDS_PLACEHOLDER_TOKEN,
27
ChatTemplateResolutionError,
@@ -969,6 +971,44 @@ class HfRenderer(BaseRenderer[HfTokenizer]):
971
self.tokenizer, config.model_config.renderer_num_workers + 1
972
)
973
974
+ def _replace_executor(self) -> None:
975
+ """Replace the inner render pool after a stuck worker.
976
+
977
+ Called after a render timeout so that the stuck thread (which CPython
978
+ cannot interrupt) does not permanently block all subsequent renders.
979
+ The executor *object* is kept so ``make_async`` wrappers bound at
980
+ init (tokenize, decode, pooling, derender) keep submitting work.
981
+ The old inner pool is abandoned with wait=False — its thread will
982
+ eventually finish or be reaped on process exit.
983
+ """
984
+ self._executor.replace_inner()
985
+ logger.warning(
986
+ "Chat template render timed out — executor pool replaced "
987
+ "(%d worker(s)). The stuck thread may continue consuming "
988
+ "CPU until the process exits.",
989
+ self.model_config.renderer_num_workers,
990
+ )
991
+
992
+ async def _render_with_timeout(self, coro):
993
+ """Run a template-rendering coroutine with a timeout.
994
+
995
+ On timeout, replaces the executor so subsequent renders are not
996
+ permanently blocked by the stuck thread.
997
+ """
998
+ timeout = envs.VLLM_CHAT_TEMPLATE_RENDER_TIMEOUT
999
+ if timeout <= 0:
1000
+ return await coro
1001
+ try:
1002
+ return await asyncio.wait_for(coro, timeout=timeout)
1003
+ except asyncio.TimeoutError:
1004
+ self._replace_executor()
1005
+ raise TimeoutError(
1006
+ f"Chat template rendering timed out after "
1007
+ f"{timeout}s. This may indicate a malicious or "
1008
+ f"excessively complex template. Adjust "
1009
+ f"VLLM_CHAT_TEMPLATE_RENDER_TIMEOUT to override."
1010
+ ) from None
1011
+
1012
def _can_produce_offsets(self) -> bool:
1013
# Only fast tokenizers expose offset_mapping.
1014
return self.tokenizer is not None and self.tokenizer.is_fast
@@ -1124,11 +1164,13 @@ class HfRenderer(BaseRenderer[HfTokenizer]):
1164
logger.warning_once(_TOKENIZE_OVERRIDE_WARNING)
1165
chat_template_kwargs["tokenize"] = True
1166
1127
- prompt_raw = await self._apply_chat_template_async(
1128
- model_config,
1129
- tokenizer,
1130
- conversation,
1131
- **chat_template_kwargs,
1167
+ prompt_raw = await self._render_with_timeout(
1168
+ self._apply_chat_template_async(
1169
+ model_config,
1170
+ tokenizer,
1171
+ conversation,
1172
+ **chat_template_kwargs,
1173
+ )
1174
)
1175
1176
# NOTE: use_unified_vision_chunk is currently specific to Kimi-K2.5