@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 8baf85e4e9
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 10:45 UTC

[Doc] Compatibility matrix for mutual exclusive features (#8512)

Signed-off-by: Wallas Santos <wallashss@ibm.com>

Wallas Henrique committed Oct 11, 2024 at 15:18 UTC 8baf85e4e9355611532e361a5cd4d458bc8fe1fe
13 files changed +467
docs/source/index.rst
+1
@@ -86,6 +86,7 @@ Documentation
86 serving/usage_stats
87 serving/integrations
88 serving/tensorizer
89 + serving/compatibility_matrix
90 serving/faq
91
92 .. toctree::
docs/source/models/performance.rst
+2
@@ -22,6 +22,8 @@ If you frequently encounter preemptions from the vLLM engine, consider the follo
22
23 You can also monitor the number of preemption requests through Prometheus metrics exposed by the vLLM. Additionally, you can log the cumulative number of preemption requests by setting disable_log_stats=False.
24
25 +.. _chunked-prefill:
26 +
27 Chunked Prefill
28 ---------------
29 vLLM supports an experimental feature chunked prefill. Chunked prefill allows to chunk large prefills into smaller chunks and batch them together with decode requests.
docs/source/serving/compatibility_matrix.rst new
+427
@@ -0,0 +1,427 @@
1 +.. _compatibility_matrix:
2 +
3 +Compatibility Matrix
4 +====================
5 +
6 +The tables below show mutually exclusive features and the support on some hardware.
7 +
8 +.. note::
9 +
10 + Check the '✗' with links to see tracking issue for unsupported feature/hardware combination.
11 +
12 +Feature x Feature
13 +-----------------
14 +
15 +
16 +.. raw:: html
17 +
18 + <style>
19 + /* Make smaller to try to improve readability */
20 + td {
21 + font-size: 0.8rem;
22 + text-align: center;
23 + }
24 +
25 + th {
26 + text-align: center;
27 + font-size: 0.8rem;
28 + }
29 + </style>
30 +
31 +.. list-table::
32 + :header-rows: 1
33 + :widths: auto
34 +
35 + * - Feature
36 + - :ref:`CP <chunked-prefill>`
37 + - :ref:`APC <apc>`
38 + - :ref:`LoRA <lora>`
39 + - :abbr:`prmpt adptr (Prompt Adapter)`
40 + - :ref:`SD <spec_decode>`
41 + - CUDA graph
42 + - :abbr:`enc-dec (Encoder-Decoder Models)`
43 + - :abbr:`logP (Logprobs)`
44 + - :abbr:`prmpt logP (Prompt Logprobs)`
45 + - :abbr:`async output (Async Output Processing)`
46 + - multi-step
47 + - :abbr:`MM (Multimodal)`
48 + - best-of
49 + - beam-search
50 + - :abbr:`guided dec (Guided Decoding)`
51 + * - :ref:`CP <chunked-prefill>`
52 + -
53 + -
54 + -
55 + -
56 + -
57 + -
58 + -
59 + -
60 + -
61 + -
62 + -
63 + -
64 + -
65 + -
66 + -
67 + * - :ref:`APC <apc>`
68 + - ✅
69 + -
70 + -
71 + -
72 + -
73 + -
74 + -
75 + -
76 + -
77 + -
78 + -
79 + -
80 + -
81 + -
82 + -
83 + * - :ref:`LoRA <lora>`
84 + - `✗ <https://github.com/vllm-project/vllm/pull/9057>`__
85 + - ✅
86 + -
87 + -
88 + -
89 + -
90 + -
91 + -
92 + -
93 + -
94 + -
95 + -
96 + -
97 + -
98 + -
99 + * - :abbr:`prmpt adptr (Prompt Adapter)`
100 + - ✅
101 + - ✅
102 + - ✅
103 + -
104 + -
105 + -
106 + -
107 + -
108 + -
109 + -
110 + -
111 + -
112 + -
113 + -
114 + -
115 + * - :ref:`SD <spec_decode>`
116 + - ✗
117 + - ✅
118 + - ✗
119 + - ✅
120 + -
121 + -
122 + -
123 + -
124 + -
125 + -
126 + -
127 + -
128 + -
129 + -
130 + -
131 + * - CUDA graph
132 + - ✅
133 + - ✅
134 + - ✅
135 + - ✅
136 + - ✅
137 + -
138 + -
139 + -
140 + -
141 + -
142 + -
143 + -
144 + -
145 + -
146 + -
147 + * - :abbr:`enc-dec (Encoder-Decoder Models)`
148 + - ✗
149 + - `✗ <https://github.com/vllm-project/vllm/issues/7366>`__
150 + - ✗
151 + - ✗
152 + - `✗ <https://github.com/vllm-project/vllm/issues/7366>`__
153 + - ✅
154 + -
155 + -
156 + -
157 + -
158 + -
159 + -
160 + -
161 + -
162 + -
163 + * - :abbr:`logP (Logprobs)`
164 + - ✅
165 + - ✅
166 + - ✅
167 + - ✅
168 + - ✅
169 + - ✅
170 + - ✅
171 + -
172 + -
173 + -
174 + -
175 + -
176 + -
177 + -
178 + -
179 + * - :abbr:`prmpt logP (Prompt Logprobs)`
180 + - ✅
181 + - ✅
182 + - ✅
183 + - ✅
184 + - `✗ <https://github.com/vllm-project/vllm/pull/8199>`__
185 + - ✅
186 + - ✅
187 + - ✅
188 + -
189 + -
190 + -
191 + -
192 + -
193 + -
194 + -
195 + * - :abbr:`async output (Async Output Processing)`
196 + - ✅
197 + - ✅
198 + - ✅
199 + - ✅
200 + - ✗
201 + - ✅
202 + - ✗
203 + - ✅
204 + - ✅
205 + -
206 + -
207 + -
208 + -
209 + -
210 + -
211 + * - multi-step
212 + - ✗
213 + - ✅
214 + - ✗
215 + - ✅
216 + - ✗
217 + - ✅
218 + - ✗
219 + - ✅
220 + - `✗ <https://github.com/vllm-project/vllm/issues/8198>`__
221 + - ✅
222 + -
223 + -
224 + -
225 + -
226 + -
227 + * - :abbr:`MM (Multimodal)`
228 + - `✗ <https://github.com/vllm-project/vllm/pull/8346>`__
229 + - `✗ <https://github.com/vllm-project/vllm/pull/8348>`__
230 + - `✗ <https://github.com/vllm-project/vllm/pull/7199>`__
231 + - ?
232 + - ?
233 + - ✅
234 + - ✗
235 + - ✅
236 + - ✅
237 + - ✅
238 + - ?
239 + -
240 + -
241 + -
242 + -
243 + * - best-of
244 + - ✅
245 + - ✅
246 + - ✅
247 + - ✅
248 + - `✗ <https://github.com/vllm-project/vllm/issues/6137>`__
249 + - ✅
250 + - ✅
251 + - ✅
252 + - ✅
253 + - ?
254 + - `✗ <https://github.com/vllm-project/vllm/issues/7968>`__
255 + - ✅
256 + -
257 + -
258 + -
259 + * - beam-search
260 + - ✅
261 + - ✅
262 + - ✅
263 + - ✅
264 + - `✗ <https://github.com/vllm-project/vllm/issues/6137>`__
265 + - ✅
266 + - ✅
267 + - ✅
268 + - ✅
269 + - ?
270 + - `✗ <https://github.com/vllm-project/vllm/issues/7968>`__
271 + - ?
272 + - ✅
273 + -
274 + -
275 + * - :abbr:`guided dec (Guided Decoding)`
276 + - ✅
277 + - ✅
278 + - ?
279 + - ?
280 + - ✅
281 + - ✅
282 + - ?
283 + - ✅
284 + - ✅
285 + - ✅
286 + - ✗
287 + - ?
288 + - ✅
289 + - ✅
290 + -
291 +
292 +
293 +Feature x Hardware
294 +^^^^^^^^^^^^^^^^^^
295 +
296 +.. list-table::
297 + :header-rows: 1
298 + :widths: auto
299 +
300 + * - Feature
301 + - Volta
302 + - Turing
303 + - Ampere
304 + - Ada
305 + - Hopper
306 + - CPU
307 + - AMD
308 + * - :ref:`CP <chunked-prefill>`
309 + - `✗ <https://github.com/vllm-project/vllm/issues/2729>`__
310 + - ✅
311 + - ✅
312 + - ✅
313 + - ✅
314 + - ✗
315 + - ✅
316 + * - :ref:`APC <apc>`
317 + - `✗ <https://github.com/vllm-project/vllm/issues/3687>`__
318 + - ✅
319 + - ✅
320 + - ✅
321 + - ✅
322 + - ✗
323 + - ✅
324 + * - :ref:`LoRA <lora>`
325 + - ✅
326 + - ✅
327 + - ✅
328 + - ✅
329 + - ✅
330 + - `✗ <https://github.com/vllm-project/vllm/pull/4830>`__
331 + - ✅
332 + * - :abbr:`prmpt adptr (Prompt Adapter)`
333 + - ✅
334 + - ✅
335 + - ✅
336 + - ✅
337 + - ✅
338 + - `✗ <https://github.com/vllm-project/vllm/issues/8475>`__
339 + - ✅
340 + * - :ref:`SD <spec_decode>`
341 + - ✅
342 + - ✅
343 + - ✅
344 + - ✅
345 + - ✅
346 + - ✅
347 + - ✅
348 + * - CUDA graph
349 + - ✅
350 + - ✅
351 + - ✅
352 + - ✅
353 + - ✅
354 + - ✗
355 + - ✅
356 + * - :abbr:`enc-dec (Encoder-Decoder Models)`
357 + - ✅
358 + - ✅
359 + - ✅
360 + - ✅
361 + - ✅
362 + - `✗ <https://github.com/vllm-project/vllm/blob/a84e598e2125960d3b4f716b78863f24ac562947/vllm/worker/cpu_model_runner.py#L125>`__
363 + - ✗
364 + * - :abbr:`logP (Logprobs)`
365 + - ✅
366 + - ✅
367 + - ✅
368 + - ✅
369 + - ✅
370 + - ✅
371 + - ✅
372 + * - :abbr:`prmpt logP (Prompt Logprobs)`
373 + - ✅
374 + - ✅
375 + - ✅
376 + - ✅
377 + - ✅
378 + - ✅
379 + - ✅
380 + * - :abbr:`async output (Async Output Processing)`
381 + - ✅
382 + - ✅
383 + - ✅
384 + - ✅
385 + - ✅
386 + - ✗
387 + - ✗
388 + * - multi-step
389 + - ✅
390 + - ✅
391 + - ✅
392 + - ✅
393 + - ✅
394 + - `✗ <https://github.com/vllm-project/vllm/issues/8477>`__
395 + - ✅
396 + * - :abbr:`MM (Multimodal)`
397 + - ✅
398 + - ✅
399 + - ✅
400 + - ✅
401 + - ✅
402 + - ✅
403 + - ✅
404 + * - best-of
405 + - ✅
406 + - ✅
407 + - ✅
408 + - ✅
409 + - ✅
410 + - ✅
411 + - ✅
412 + * - beam-search
413 + - ✅
414 + - ✅
415 + - ✅
416 + - ✅
417 + - ✅
418 + - ✅
419 + - ✅
420 + * - :abbr:`guided dec (Guided Decoding)`
421 + - ✅
422 + - ✅
423 + - ✅
424 + - ✅
425 + - ✅
426 + - ✅
427 + - ✅
vllm/attention/backends/rocm_flash_attn.py
+2
@@ -420,6 +420,8 @@ class ROCmFlashAttentionImpl(AttentionImpl):
420 Returns:
421 shape = [num_tokens, num_heads * head_size]
422 """
423 + # Reminder: Please update docs/source/serving/compatibility_matrix.rst
424 + # If the feature combo become valid
425 if attn_type != AttentionType.DECODER:
426 raise NotImplementedError("Encoder self-attention and "
427 "encoder/decoder cross-attention "
vllm/config.py
+10
@@ -359,6 +359,8 @@ class ModelConfig:
359 self.use_async_output_proc = False
360 return
361
362 + # Reminder: Please update docs/source/serving/compatibility_matrix.rst
363 + # If the feature combo become valid
364 if device_config.device_type not in ("cuda", "tpu"):
365 logger.warning(
366 "Async output processing is only supported for CUDA or TPU. "
@@ -372,6 +374,8 @@ class ModelConfig:
374 self.use_async_output_proc = False
375 return
376
377 + # Reminder: Please update docs/source/serving/compatibility_matrix.rst
378 + # If the feature combo become valid
379 if device_config.device_type == "cuda" and self.enforce_eager:
380 logger.warning(
381 "To see benefits of async output processing, enable CUDA "
@@ -385,6 +389,8 @@ class ModelConfig:
389 if self.embedding_mode:
390 self.use_async_output_proc = False
391
392 + # Reminder: Please update docs/source/serving/compatibility_matrix.rst
393 + # If the feature combo become valid
394 if speculative_config:
395 logger.warning("Async output processing is not supported with"
396 " speculative decoding currently.")
@@ -1200,6 +1206,8 @@ class SpeculativeConfig:
1206 "speculative decoding is > 1, but got "
1207 f"{speculative_disable_by_batch_size=}")
1208
1209 + # Reminder: Please update docs/source/serving/compatibility_matrix.rst
1210 + # If the feature combo become valid
1211 if enable_chunked_prefill:
1212 raise ValueError(
1213 "Speculative decoding and chunked prefill are "
@@ -1561,6 +1569,8 @@ class LoRAConfig:
1569 model_config.quantization)
1570
1571 def verify_with_scheduler_config(self, scheduler_config: SchedulerConfig):
1572 + # Reminder: Please update docs/source/serving/compatibility_matrix.rst
1573 + # If the feature combo become valid
1574 if scheduler_config.chunked_prefill_enabled:
1575 raise ValueError("LoRA is not supported with chunked prefill yet.")
1576
vllm/engine/arg_utils.py
+2
@@ -1000,6 +1000,8 @@ class EngineArgs:
1000 disable_logprobs=self.disable_logprobs_during_spec_decoding,
1001 )
1002
1003 + # Reminder: Please update docs/source/serving/compatibility_matrix.rst
1004 + # If the feature combo become valid
1005 if self.num_scheduler_steps > 1:
1006 if speculative_config is not None:
1007 raise ValueError("Speculative decoding is not supported with "
vllm/engine/output_processor/multi_step.py
+2
@@ -62,6 +62,8 @@ class MultiStepOutputProcessor(SequenceGroupOutputProcessor):
62 @staticmethod
63 @functools.lru_cache()
64 def _log_prompt_logprob_unsupported_warning_once():
65 + # Reminder: Please update docs/source/serving/compatibility_matrix.rst
66 + # If the feature combo become valid
67 logger.warning(
68 "Prompt logprob is not supported by multi step workers. "
69 "(e.g., speculative decode uses multi step workers).")
vllm/executor/cpu_executor.py
+8
@@ -28,6 +28,8 @@ class CPUExecutor(ExecutorBase):
28
29 def _init_executor(self) -> None:
30 assert self.device_config.device_type == "cpu"
31 + # Reminder: Please update docs/source/serving/compatibility_matrix.rst
32 + # If the feature combo become valid
33 assert self.lora_config is None, "cpu backend doesn't support LoRA"
34
35 #
@@ -324,6 +326,8 @@ def _verify_and_get_model_config(config: ModelConfig) -> ModelConfig:
326 if config.dtype == torch.float16:
327 logger.warning("float16 is not supported on CPU, casting to bfloat16.")
328 config.dtype = torch.bfloat16
329 + # Reminder: Please update docs/source/serving/compatibility_matrix.rst
330 + # If the feature combo become valid
331 if not config.enforce_eager:
332 logger.warning(
333 "CUDA graph is not supported on CPU, fallback to the eager "
@@ -334,6 +338,8 @@ def _verify_and_get_model_config(config: ModelConfig) -> ModelConfig:
338
339 def _verify_and_get_scheduler_config(
340 config: SchedulerConfig) -> SchedulerConfig:
341 + # Reminder: Please update docs/source/serving/compatibility_matrix.rst
342 + # If the feature combo become valid
343 if config.chunked_prefill_enabled:
344 logger.warning("Chunked prefill is not supported on CPU, disable it.")
345 config.chunked_prefill_enabled = False
@@ -342,6 +348,8 @@ def _verify_and_get_scheduler_config(
348
349
350 def _verify_and_get_cache_config(config: CacheConfig) -> CacheConfig:
351 + # Reminder: Please update docs/source/serving/compatibility_matrix.rst
352 + # If the feature combo become valid
353 if config.enable_prefix_caching:
354 logger.warning("Prefix caching is not supported on CPU, disable it.")
355 config.enable_prefix_caching = False
vllm/inputs/preprocess.py
+2
@@ -310,6 +310,8 @@ class InputPreprocessor:
310 encoder_prompt, encoder_prompt_ids, encoder_mm_data, _ = encoder_comps
311 decoder_prompt, decoder_prompt_ids, decoder_mm_data, _ = decoder_comps
312
313 + # Reminder: Please update docs/source/serving/compatibility_matrix.rst
314 + # If the feature combo become valid
315 if decoder_mm_data is not None:
316 raise ValueError(
317 "Multi-modality decoder inputs of encoder-decoder models are "
vllm/spec_decode/spec_decode_worker.py
+2
@@ -87,6 +87,8 @@ def create_spec_worker(*args, **kwargs) -> "SpecDecodeWorker":
87 return spec_decode_worker
88
89
90 +# Reminder: Please update docs/source/serving/compatibility_matrix.rst
91 +# If the feature combo become valid
92 class SpecDecodeWorker(LoraNotSupportedWorkerBase):
93 """Worker which implements speculative decoding.
94
vllm/utils.py
+3
@@ -41,6 +41,9 @@ logger = init_logger(__name__)
41
42 # Exception strings for non-implemented encoder/decoder scenarios
43
44 +# Reminder: Please update docs/source/serving/compatibility_matrix.rst
45 +# If the feature combo become valid
46 +
47 STR_NOT_IMPL_ENC_DEC_SWA = \
48 "Sliding window attention for encoder/decoder models " + \
49 "is not currently supported."
vllm/worker/multi_step_model_runner.py
+3
@@ -816,6 +816,9 @@ def _pythonize_sampler_output(
816
817 for sgdx, (seq_group,
818 sample_result) in enumerate(zip(seq_groups, samples_list)):
819 + # Reminder: Please update docs/source/serving/compatibility_matrix.rst
820 + # If the feature combo become valid
821 + # (Check for Guided Decoding)
822 if seq_group.sampling_params.logits_processors:
823 assert len(seq_group.sampling_params.logits_processors) == 0, (
824 "Logits Processors are not supported in multi-step decoding")
vllm/worker/utils.py
+3
@@ -13,6 +13,9 @@ def assert_enc_dec_mr_supported_scenario(
13 a supported scenario.
14 '''
15
16 + # Reminder: Please update docs/source/serving/compatibility_matrix.rst
17 + # If the feature combo become valid
18 +
19 if enc_dec_mr.cache_config.enable_prefix_caching:
20 raise NotImplementedError(
21 STR_NOT_IMPL_ENC_DEC_ERR_STRS['STR_NOT_IMPL_ENC_DEC_PREFIX_CACHE'])