@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 84e23d103d
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 11, 11:15 UTC

additional protection for CVE-2025-62164 (#30649)

Signed-off-by: Wenqi Glantz <wglantz@nvidia.com>

Wenqi Glantz committed Dec 14, 2025 at 22:07 UTC 84e23d103d3483f944780d0d42bcf0993fd27e3a
5 files changed +510 -15
tests/entrypoints/openai/test_sparse_tensor_validation.py new
+342
@@ -0,0 +1,342 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +"""
4 +Sparse tensor validation in embedding APIs.
5 +
6 +Tests verify that malicious sparse tensors are rejected before they can trigger
7 +out-of-bounds memory writes during to_dense() operations.
8 +"""
9 +
10 +import base64
11 +import io
12 +
13 +import pytest
14 +import torch
15 +
16 +from vllm.entrypoints.renderer import CompletionRenderer
17 +from vllm.multimodal.audio import AudioEmbeddingMediaIO
18 +from vllm.multimodal.image import ImageEmbeddingMediaIO
19 +
20 +
21 +def _encode_tensor(tensor: torch.Tensor) -> bytes:
22 + """Helper to encode a tensor as base64 bytes."""
23 + buffer = io.BytesIO()
24 + torch.save(tensor, buffer)
25 + buffer.seek(0)
26 + return base64.b64encode(buffer.read())
27 +
28 +
29 +def _create_malicious_sparse_tensor() -> torch.Tensor:
30 + """
31 + Create a malicious sparse COO tensor with out-of-bounds indices.
32 +
33 + This tensor has indices that point beyond the declared shape, which would
34 + cause an out-of-bounds write when converted to dense format without
35 + validation.
36 + """
37 + # Create a 3x3 sparse tensor but with indices pointing to (10, 10)
38 + indices = torch.tensor([[10], [10]]) # Out of bounds for 3x3 shape
39 + values = torch.tensor([1.0])
40 + shape = (3, 3)
41 +
42 + # Create sparse tensor (this will be invalid)
43 + sparse_tensor = torch.sparse_coo_tensor(indices, values, shape, dtype=torch.float32)
44 + return sparse_tensor
45 +
46 +
47 +def _create_valid_sparse_tensor() -> torch.Tensor:
48 + """Create a valid sparse COO tensor for baseline testing."""
49 + indices = torch.tensor([[0, 1, 2], [0, 1, 2]])
50 + values = torch.tensor([1.0, 2.0, 3.0])
51 + shape = (3, 3)
52 +
53 + sparse_tensor = torch.sparse_coo_tensor(indices, values, shape, dtype=torch.float32)
54 + return sparse_tensor
55 +
56 +
57 +def _create_valid_dense_tensor() -> torch.Tensor:
58 + """Create a valid dense tensor for baseline testing."""
59 + return torch.randn(10, 768, dtype=torch.float32) # (seq_len, hidden_size)
60 +
61 +
62 +class TestPromptEmbedsValidation:
63 + """Test sparse tensor validation in prompt embeddings (Completions API)."""
64 +
65 + def test_valid_dense_tensor_accepted(self, model_config):
66 + """Baseline: Valid dense tensors should work normally."""
67 + renderer = CompletionRenderer(model_config)
68 +
69 + valid_tensor = _create_valid_dense_tensor()
70 + encoded = _encode_tensor(valid_tensor)
71 +
72 + # Should not raise any exception
73 + result = renderer.load_prompt_embeds(encoded)
74 + assert len(result) == 1
75 + assert result[0]["prompt_embeds"].shape == valid_tensor.shape
76 +
77 + def test_valid_sparse_tensor_accepted(self):
78 + """Baseline: Valid sparse tensors should load successfully."""
79 + io_handler = ImageEmbeddingMediaIO()
80 +
81 + valid_sparse = _create_valid_sparse_tensor()
82 + encoded = _encode_tensor(valid_sparse)
83 +
84 + # Should not raise any exception (sparse tensors remain sparse)
85 + result = io_handler.load_base64("", encoded.decode("utf-8"))
86 + assert result.shape == valid_sparse.shape
87 +
88 + def test_malicious_sparse_tensor_rejected(self, model_config):
89 + """Security: Malicious sparse tensors should be rejected."""
90 + renderer = CompletionRenderer(model_config)
91 +
92 + malicious_tensor = _create_malicious_sparse_tensor()
93 + encoded = _encode_tensor(malicious_tensor)
94 +
95 + # Should raise RuntimeError due to invalid sparse tensor
96 + with pytest.raises((RuntimeError, ValueError)) as exc_info:
97 + renderer.load_prompt_embeds(encoded)
98 +
99 + # Error should indicate sparse tensor validation failure
100 + error_msg = str(exc_info.value).lower()
101 + assert "sparse" in error_msg or "index" in error_msg or "bounds" in error_msg
102 +
103 + def test_extremely_large_indices_rejected(self, model_config):
104 + """Security: Sparse tensors with extremely large indices should be rejected."""
105 + renderer = CompletionRenderer(model_config)
106 +
107 + # Create tensor with indices far beyond reasonable bounds
108 + indices = torch.tensor([[999999], [999999]])
109 + values = torch.tensor([1.0])
110 + shape = (10, 10)
111 +
112 + malicious_tensor = torch.sparse_coo_tensor(
113 + indices, values, shape, dtype=torch.float32
114 + )
115 + encoded = _encode_tensor(malicious_tensor)
116 +
117 + with pytest.raises((RuntimeError, ValueError)):
118 + renderer.load_prompt_embeds(encoded)
119 +
120 + def test_negative_indices_rejected(self, model_config):
121 + """Security: Sparse tensors with negative indices should be rejected."""
122 + renderer = CompletionRenderer(model_config)
123 +
124 + # Create tensor with negative indices
125 + indices = torch.tensor([[-1], [-1]])
126 + values = torch.tensor([1.0])
127 + shape = (10, 10)
128 +
129 + malicious_tensor = torch.sparse_coo_tensor(
130 + indices, values, shape, dtype=torch.float32
131 + )
132 + encoded = _encode_tensor(malicious_tensor)
133 +
134 + with pytest.raises((RuntimeError, ValueError)):
135 + renderer.load_prompt_embeds(encoded)
136 +
137 +
138 +class TestImageEmbedsValidation:
139 + """Test sparse tensor validation in image embeddings (Chat API)."""
140 +
141 + def test_valid_dense_tensor_accepted(self):
142 + """Baseline: Valid dense tensors should work normally."""
143 + io_handler = ImageEmbeddingMediaIO()
144 +
145 + valid_tensor = _create_valid_dense_tensor()
146 + encoded = _encode_tensor(valid_tensor)
147 +
148 + # Should not raise any exception
149 + result = io_handler.load_base64("", encoded.decode("utf-8"))
150 + assert result.shape == valid_tensor.shape
151 +
152 + def test_valid_sparse_tensor_accepted(self):
153 + """Baseline: Valid sparse tensors should load successfully."""
154 + io_handler = AudioEmbeddingMediaIO()
155 +
156 + valid_sparse = _create_valid_sparse_tensor()
157 + encoded = _encode_tensor(valid_sparse)
158 +
159 + # Should not raise any exception (sparse tensors remain sparse)
160 + result = io_handler.load_base64("", encoded.decode("utf-8"))
161 + assert result.shape == valid_sparse.shape
162 +
163 + def test_malicious_sparse_tensor_rejected(self):
164 + """Security: Malicious sparse tensors should be rejected."""
165 + io_handler = ImageEmbeddingMediaIO()
166 +
167 + malicious_tensor = _create_malicious_sparse_tensor()
168 + encoded = _encode_tensor(malicious_tensor)
169 +
170 + # Should raise RuntimeError due to invalid sparse tensor
171 + with pytest.raises((RuntimeError, ValueError)) as exc_info:
172 + io_handler.load_base64("", encoded.decode("utf-8"))
173 +
174 + error_msg = str(exc_info.value).lower()
175 + assert "sparse" in error_msg or "index" in error_msg or "bounds" in error_msg
176 +
177 + def test_load_bytes_validates(self):
178 + """Security: Validation should also work for load_bytes method."""
179 + io_handler = ImageEmbeddingMediaIO()
180 +
181 + malicious_tensor = _create_malicious_sparse_tensor()
182 + buffer = io.BytesIO()
183 + torch.save(malicious_tensor, buffer)
184 + buffer.seek(0)
185 +
186 + with pytest.raises((RuntimeError, ValueError)):
187 + io_handler.load_bytes(buffer.read())
188 +
189 +
190 +class TestAudioEmbedsValidation:
191 + """Test sparse tensor validation in audio embeddings (Chat API)."""
192 +
193 + def test_valid_dense_tensor_accepted(self):
194 + """Baseline: Valid dense tensors should work normally."""
195 + io_handler = AudioEmbeddingMediaIO()
196 +
197 + valid_tensor = _create_valid_dense_tensor()
198 + encoded = _encode_tensor(valid_tensor)
199 +
200 + # Should not raise any exception
201 + result = io_handler.load_base64("", encoded.decode("utf-8"))
202 + assert result.shape == valid_tensor.shape
203 +
204 + def test_valid_sparse_tensor_accepted(self):
205 + """Baseline: Valid sparse tensors should be converted successfully."""
206 + io_handler = AudioEmbeddingMediaIO()
207 +
208 + valid_sparse = _create_valid_sparse_tensor()
209 + encoded = _encode_tensor(valid_sparse)
210 +
211 + # Should not raise any exception
212 + result = io_handler.load_base64("", encoded.decode("utf-8"))
213 + assert result.is_sparse is False
214 +
215 + def test_malicious_sparse_tensor_rejected(self):
216 + """Security: Malicious sparse tensors should be rejected."""
217 + io_handler = AudioEmbeddingMediaIO()
218 +
219 + malicious_tensor = _create_malicious_sparse_tensor()
220 + encoded = _encode_tensor(malicious_tensor)
221 +
222 + # Should raise RuntimeError due to invalid sparse tensor
223 + with pytest.raises((RuntimeError, ValueError)) as exc_info:
224 + io_handler.load_base64("", encoded.decode("utf-8"))
225 +
226 + error_msg = str(exc_info.value).lower()
227 + assert "sparse" in error_msg or "index" in error_msg or "bounds" in error_msg
228 +
229 + def test_load_bytes_validates(self):
230 + """Security: Validation should also work for load_bytes method."""
231 + io_handler = AudioEmbeddingMediaIO()
232 +
233 + malicious_tensor = _create_malicious_sparse_tensor()
234 + buffer = io.BytesIO()
235 + torch.save(malicious_tensor, buffer)
236 + buffer.seek(0)
237 +
238 + with pytest.raises((RuntimeError, ValueError)):
239 + io_handler.load_bytes(buffer.read())
240 +
241 +
242 +class TestSparseTensorValidationIntegration:
243 + """
244 + These tests verify the complete attack chain is blocked at all entry points.
245 + """
246 +
247 + def test_attack_scenario_completions_api(self, model_config):
248 + """
249 + Simulate a complete attack through the Completions API.
250 +
251 + Attack scenario:
252 + 1. Attacker crafts malicious sparse tensor
253 + 2. Encodes it as base64
254 + 3. Sends to /v1/completions with prompt_embeds parameter
255 + 4. Server should reject before memory corruption occurs
256 + """
257 + renderer = CompletionRenderer(model_config)
258 +
259 + # Step 1-2: Attacker creates malicious payload
260 + attack_payload = _encode_tensor(_create_malicious_sparse_tensor())
261 +
262 + # Step 3-4: Server processes and should reject
263 + with pytest.raises((RuntimeError, ValueError)):
264 + renderer.load_prompt_embeds(attack_payload)
265 +
266 + def test_attack_scenario_chat_api_image(self):
267 + """
268 + Simulate attack through Chat API with image_embeds.
269 +
270 + Verifies the image embeddings path is protected.
271 + """
272 + io_handler = ImageEmbeddingMediaIO()
273 + attack_payload = _encode_tensor(_create_malicious_sparse_tensor())
274 +
275 + with pytest.raises((RuntimeError, ValueError)):
276 + io_handler.load_base64("", attack_payload.decode("utf-8"))
277 +
278 + def test_attack_scenario_chat_api_audio(self):
279 + """
280 + Simulate attack through Chat API with audio_embeds.
281 +
282 + Verifies the audio embeddings path is protected.
283 + """
284 + io_handler = AudioEmbeddingMediaIO()
285 + attack_payload = _encode_tensor(_create_malicious_sparse_tensor())
286 +
287 + with pytest.raises((RuntimeError, ValueError)):
288 + io_handler.load_base64("", attack_payload.decode("utf-8"))
289 +
290 + def test_multiple_valid_embeddings_in_batch(self, model_config):
291 + """
292 + Regression test: Multiple valid embeddings should still work.
293 +
294 + Ensures the fix doesn't break legitimate batch processing.
295 + """
296 + renderer = CompletionRenderer(model_config)
297 +
298 + valid_tensors = [
299 + _encode_tensor(_create_valid_dense_tensor()),
300 + _encode_tensor(_create_valid_dense_tensor()),
301 + _encode_tensor(_create_valid_dense_tensor()),
302 + ]
303 +
304 + # Should process all without error
305 + result = renderer.load_prompt_embeds(valid_tensors)
306 + assert len(result) == 3
307 +
308 + def test_mixed_valid_and_malicious_rejected(self, model_config):
309 + """
310 + Security: Batch with one malicious tensor should be rejected.
311 +
312 + Even if most tensors are valid, a single malicious one should
313 + cause rejection of the entire batch.
314 + """
315 + renderer = CompletionRenderer(model_config)
316 +
317 + mixed_batch = [
318 + _encode_tensor(_create_valid_dense_tensor()),
319 + _encode_tensor(_create_malicious_sparse_tensor()), # Malicious
320 + _encode_tensor(_create_valid_dense_tensor()),
321 + ]
322 +
323 + # Should fail on the malicious tensor
324 + with pytest.raises((RuntimeError, ValueError)):
325 + renderer.load_prompt_embeds(mixed_batch)
326 +
327 +
328 +# Pytest fixtures
329 +@pytest.fixture
330 +def model_config():
331 + """Mock ModelConfig for testing."""
332 + from vllm.config import ModelConfig
333 +
334 + return ModelConfig(
335 + model="facebook/opt-125m",
336 + tokenizer="facebook/opt-125m",
337 + tokenizer_mode="auto",
338 + trust_remote_code=False,
339 + dtype="float32",
340 + seed=0,
341 + enable_prompt_embeds=True, # Required for prompt embeds tests
342 + )
tests/multimodal/test_sparse_tensor_validation_unit.py new
+134
@@ -0,0 +1,134 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +"""
4 +Unit tests for sparse tensor validation.
5 +
6 +Simple, fast unit tests that can run without server fixtures.
7 +Run with: pytest tests/multimodal/test_sparse_tensor_validation_unit.py -v
8 +"""
9 +
10 +import io
11 +
12 +import pytest
13 +import torch
14 +
15 +
16 +class TestSparseTensorValidationContextManager:
17 + """Test that torch.sparse.check_sparse_tensor_invariants() works as expected."""
18 +
19 + def test_valid_sparse_tensor_passes(self):
20 + """Valid sparse tensors should pass validation."""
21 + indices = torch.tensor([[0, 1], [0, 1]])
22 + values = torch.tensor([1.0, 2.0])
23 + shape = (2, 2)
24 +
25 + with torch.sparse.check_sparse_tensor_invariants():
26 + tensor = torch.sparse_coo_tensor(indices, values, shape)
27 + dense = tensor.to_dense()
28 +
29 + assert dense.shape == shape
30 +
31 + def test_out_of_bounds_indices_rejected(self):
32 + """Sparse tensors with out-of-bounds indices should be rejected."""
33 + indices = torch.tensor([[5], [5]]) # Out of bounds for 2x2
34 + values = torch.tensor([1.0])
35 + shape = (2, 2)
36 +
37 + with pytest.raises(RuntimeError) as exc_info: # noqa: SIM117
38 + with torch.sparse.check_sparse_tensor_invariants():
39 + tensor = torch.sparse_coo_tensor(indices, values, shape)
40 + tensor.to_dense()
41 +
42 + assert (
43 + "index" in str(exc_info.value).lower()
44 + or "bound" in str(exc_info.value).lower()
45 + )
46 +
47 + def test_negative_indices_rejected(self):
48 + """Sparse tensors with negative indices should be rejected."""
49 + indices = torch.tensor([[-1], [0]])
50 + values = torch.tensor([1.0])
51 + shape = (2, 2)
52 +
53 + with pytest.raises(RuntimeError): # noqa: SIM117
54 + with torch.sparse.check_sparse_tensor_invariants():
55 + tensor = torch.sparse_coo_tensor(indices, values, shape)
56 + tensor.to_dense()
57 +
58 + def test_without_context_manager_allows_invalid(self):
59 + """
60 + WITHOUT validation, invalid tensors may not immediately error.
61 +
62 + This demonstrates the vulnerability: PyTorch 2.8.0+ doesn't validate
63 + by default, which can lead to memory corruption.
64 + """
65 + indices = torch.tensor([[100], [100]]) # Way out of bounds
66 + values = torch.tensor([1.0])
67 + shape = (2, 2)
68 +
69 + # Without validation context, this might create an invalid tensor
70 + # (actual behavior depends on PyTorch version)
71 + tensor = torch.sparse_coo_tensor(indices, values, shape)
72 +
73 + # The tensor object is created, but it's invalid
74 + assert tensor.is_sparse
75 +
76 +
77 +class TestTorchLoadWithValidation:
78 + """Test torch.load() with sparse tensor validation."""
79 +
80 + def test_load_valid_sparse_tensor_with_validation(self):
81 + """Valid sparse tensors should load successfully with validation."""
82 + # Create and save a valid sparse tensor
83 + indices = torch.tensor([[0, 1], [0, 1]])
84 + values = torch.tensor([1.0, 2.0])
85 + tensor = torch.sparse_coo_tensor(indices, values, (2, 2))
86 +
87 + buffer = io.BytesIO()
88 + torch.save(tensor, buffer)
89 + buffer.seek(0)
90 +
91 + # Load with validation
92 + with torch.sparse.check_sparse_tensor_invariants():
93 + loaded = torch.load(buffer, weights_only=True)
94 + dense = loaded.to_dense()
95 +
96 + assert dense.shape == (2, 2)
97 +
98 + def test_load_invalid_sparse_tensor_rejected(self):
99 + """Invalid sparse tensors should be caught when loaded with validation."""
100 + # Create an invalid sparse tensor (out of bounds)
101 + indices = torch.tensor([[10], [10]])
102 + values = torch.tensor([1.0])
103 + tensor = torch.sparse_coo_tensor(indices, values, (2, 2))
104 +
105 + buffer = io.BytesIO()
106 + torch.save(tensor, buffer)
107 + buffer.seek(0)
108 +
109 + # Load with validation - should fail on to_dense()
110 + with pytest.raises(RuntimeError): # noqa: SIM117
111 + with torch.sparse.check_sparse_tensor_invariants():
112 + loaded = torch.load(buffer, weights_only=True)
113 + loaded.to_dense()
114 +
115 + def test_load_dense_tensor_unaffected(self):
116 + """Dense tensors should work normally with the validation context."""
117 + # Create and save a dense tensor
118 + tensor = torch.randn(10, 20)
119 +
120 + buffer = io.BytesIO()
121 + torch.save(tensor, buffer)
122 + buffer.seek(0)
123 +
124 + # Load with validation (should have no effect on dense tensors)
125 + with torch.sparse.check_sparse_tensor_invariants():
126 + loaded = torch.load(buffer, weights_only=True)
127 +
128 + assert loaded.shape == (10, 20)
129 + assert not loaded.is_sparse
130 +
131 +
132 +if __name__ == "__main__":
133 + # Allow running directly for quick testing
134 + pytest.main([__file__, "-v", "--tb=short"])
vllm/entrypoints/renderer.py
+14 -11
@@ -167,17 +167,20 @@ class BaseRenderer(ABC):
167 )
168
169 def _load_and_validate_embed(embed: bytes) -> EmbedsPrompt:
170 - tensor = torch.load(
171 - io.BytesIO(pybase64.b64decode(embed, validate=True)),
172 - weights_only=True,
173 - map_location=torch.device("cpu"),
174 - )
175 - assert isinstance(tensor, torch.Tensor) and tensor.dtype in (
176 - torch.float32,
177 - torch.bfloat16,
178 - torch.float16,
179 - )
180 - tensor = tensor.to_dense()
170 + # Enable sparse tensor integrity checks to prevent out-of-bounds
171 + # writes from maliciously crafted tensors
172 + with torch.sparse.check_sparse_tensor_invariants():
173 + tensor = torch.load(
174 + io.BytesIO(pybase64.b64decode(embed, validate=True)),
175 + weights_only=True,
176 + map_location=torch.device("cpu"),
177 + )
178 + assert isinstance(tensor, torch.Tensor) and tensor.dtype in (
179 + torch.float32,
180 + torch.bfloat16,
181 + torch.float16,
182 + )
183 + tensor = tensor.to_dense()
184 if tensor.dim() > 2:
185 tensor = tensor.squeeze(0)
186 assert tensor.dim() == 2
vllm/multimodal/audio.py
+10 -2
@@ -127,13 +127,21 @@ class AudioEmbeddingMediaIO(MediaIO[torch.Tensor]):
127
128 def load_bytes(self, data: bytes) -> torch.Tensor:
129 buffer = BytesIO(data)
130 - return torch.load(buffer, weights_only=True)
130 + # Enable sparse tensor integrity checks to prevent out-of-bounds
131 + # writes from maliciously crafted tensors
132 + with torch.sparse.check_sparse_tensor_invariants():
133 + tensor = torch.load(buffer, weights_only=True)
134 + return tensor.to_dense()
135
136 def load_base64(self, media_type: str, data: str) -> torch.Tensor:
137 return self.load_bytes(pybase64.b64decode(data, validate=True))
138
139 def load_file(self, filepath: Path) -> torch.Tensor:
136 - return torch.load(filepath, weights_only=True)
140 + # Enable sparse tensor integrity checks to prevent out-of-bounds
141 + # writes from maliciously crafted tensors
142 + with torch.sparse.check_sparse_tensor_invariants():
143 + tensor = torch.load(filepath, weights_only=True)
144 + return tensor.to_dense()
145
146 def encode_base64(self, media: torch.Tensor) -> str:
147 return tensor2base64(media)
vllm/multimodal/image.py
+10 -2
@@ -122,13 +122,21 @@ class ImageEmbeddingMediaIO(MediaIO[torch.Tensor]):
122
123 def load_bytes(self, data: bytes) -> torch.Tensor:
124 buffer = BytesIO(data)
125 - return torch.load(buffer, weights_only=True)
125 + # Enable sparse tensor integrity checks to prevent out-of-bounds
126 + # writes from maliciously crafted tensors
127 + with torch.sparse.check_sparse_tensor_invariants():
128 + tensor = torch.load(buffer, weights_only=True)
129 + return tensor.to_dense()
130
131 def load_base64(self, media_type: str, data: str) -> torch.Tensor:
132 return self.load_bytes(pybase64.b64decode(data, validate=True))
133
134 def load_file(self, filepath: Path) -> torch.Tensor:
131 - return torch.load(filepath, weights_only=True)
135 + # Enable sparse tensor integrity checks to prevent out-of-bounds
136 + # writes from maliciously crafted tensors
137 + with torch.sparse.check_sparse_tensor_invariants():
138 + tensor = torch.load(filepath, weights_only=True)
139 + return tensor.to_dense()
140
141 def encode_base64(self, media: torch.Tensor) -> str:
142 return pybase64.b64encode(media.numpy()).decode("utf-8")