A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 10:45 UTC
[Doc] Compatibility matrix for mutual exclusive features (#8512)
Signed-off-by: Wallas Santos <wallashss@ibm.com>
Wallas Henrique committed
Oct 11, 2024 at 15:18 UTC
8baf85e4e9355611532e361a5cd4d458bc8fe1fe
13 files changed
+467
docs/source/index.rst
+1
@@ -86,6 +86,7 @@ Documentation
86
serving/usage_stats
87
serving/integrations
88
serving/tensorizer
89
+ serving/compatibility_matrix
90
serving/faq
91
92
.. toctree::
docs/source/models/performance.rst
+2
@@ -22,6 +22,8 @@ If you frequently encounter preemptions from the vLLM engine, consider the follo
22
23
You can also monitor the number of preemption requests through Prometheus metrics exposed by the vLLM. Additionally, you can log the cumulative number of preemption requests by setting disable_log_stats=False.
24
25
+.. _chunked-prefill:
26
+
27
Chunked Prefill
28
---------------
29
vLLM supports an experimental feature chunked prefill. Chunked prefill allows to chunk large prefills into smaller chunks and batch them together with decode requests.
docs/source/serving/compatibility_matrix.rst
new
+427
@@ -0,0 +1,427 @@
1
+.. _compatibility_matrix:
2
+
3
+Compatibility Matrix
4
+====================
5
+
6
+The tables below show mutually exclusive features and the support on some hardware.
7
+
8
+.. note::
9
+
10
+ Check the '✗' with links to see tracking issue for unsupported feature/hardware combination.
11
+
12
+Feature x Feature
13
+-----------------
14
+
15
+
16
+.. raw:: html
17
+
18
+ <style>
19
+ /* Make smaller to try to improve readability */
20
+ td {
21
+ font-size: 0.8rem;
22
+ text-align: center;
23
+ }
24
+
25
+ th {
26
+ text-align: center;
27
+ font-size: 0.8rem;
28
+ }
29
+ </style>
30
+
31
+.. list-table::
32
+ :header-rows: 1
33
+ :widths: auto
34
+
35
+ * - Feature
36
+ - :ref:`CP <chunked-prefill>`
37
+ - :ref:`APC <apc>`
38
+ - :ref:`LoRA <lora>`
39
+ - :abbr:`prmpt adptr (Prompt Adapter)`
40
+ - :ref:`SD <spec_decode>`
41
+ - CUDA graph
42
+ - :abbr:`enc-dec (Encoder-Decoder Models)`
43
+ - :abbr:`logP (Logprobs)`
44
+ - :abbr:`prmpt logP (Prompt Logprobs)`
45
+ - :abbr:`async output (Async Output Processing)`
46
+ - multi-step
47
+ - :abbr:`MM (Multimodal)`
48
+ - best-of
49
+ - beam-search
50
+ - :abbr:`guided dec (Guided Decoding)`
51
+ * - :ref:`CP <chunked-prefill>`
52
+ -
53
+ -
54
+ -
55
+ -
56
+ -
57
+ -
58
+ -
59
+ -
60
+ -
61
+ -
62
+ -
63
+ -
64
+ -
65
+ -
66
+ -
67
+ * - :ref:`APC <apc>`
68
+ - ✅
69
+ -
70
+ -
71
+ -
72
+ -
73
+ -
74
+ -
75
+ -
76
+ -
77
+ -
78
+ -
79
+ -
80
+ -
81
+ -
82
+ -
83
+ * - :ref:`LoRA <lora>`
84
+ - `✗ <https://github.com/vllm-project/vllm/pull/9057>`__
85
+ - ✅
86
+ -
87
+ -
88
+ -
89
+ -
90
+ -
91
+ -
92
+ -
93
+ -
94
+ -
95
+ -
96
+ -
97
+ -
98
+ -
99
+ * - :abbr:`prmpt adptr (Prompt Adapter)`
100
+ - ✅
101
+ - ✅
102
+ - ✅
103
+ -
104
+ -
105
+ -
106
+ -
107
+ -
108
+ -
109
+ -
110
+ -
111
+ -
112
+ -
113
+ -
114
+ -
115
+ * - :ref:`SD <spec_decode>`
116
+ - ✗
117
+ - ✅
118
+ - ✗
119
+ - ✅
120
+ -
121
+ -
122
+ -
123
+ -
124
+ -
125
+ -
126
+ -
127
+ -
128
+ -
129
+ -
130
+ -
131
+ * - CUDA graph
132
+ - ✅
133
+ - ✅
134
+ - ✅
135
+ - ✅
136
+ - ✅
137
+ -
138
+ -
139
+ -
140
+ -
141
+ -
142
+ -
143
+ -
144
+ -
145
+ -
146
+ -
147
+ * - :abbr:`enc-dec (Encoder-Decoder Models)`
148
+ - ✗
149
+ - `✗ <https://github.com/vllm-project/vllm/issues/7366>`__
150
+ - ✗
151
+ - ✗
152
+ - `✗ <https://github.com/vllm-project/vllm/issues/7366>`__
153
+ - ✅
154
+ -
155
+ -
156
+ -
157
+ -
158
+ -
159
+ -
160
+ -
161
+ -
162
+ -
163
+ * - :abbr:`logP (Logprobs)`
164
+ - ✅
165
+ - ✅
166
+ - ✅
167
+ - ✅
168
+ - ✅
169
+ - ✅
170
+ - ✅
171
+ -
172
+ -
173
+ -
174
+ -
175
+ -
176
+ -
177
+ -
178
+ -
179
+ * - :abbr:`prmpt logP (Prompt Logprobs)`
180
+ - ✅
181
+ - ✅
182
+ - ✅
183
+ - ✅
184
+ - `✗ <https://github.com/vllm-project/vllm/pull/8199>`__
185
+ - ✅
186
+ - ✅
187
+ - ✅
188
+ -
189
+ -
190
+ -
191
+ -
192
+ -
193
+ -
194
+ -
195
+ * - :abbr:`async output (Async Output Processing)`
196
+ - ✅
197
+ - ✅
198
+ - ✅
199
+ - ✅
200
+ - ✗
201
+ - ✅
202
+ - ✗
203
+ - ✅
204
+ - ✅
205
+ -
206
+ -
207
+ -
208
+ -
209
+ -
210
+ -
211
+ * - multi-step
212
+ - ✗
213
+ - ✅
214
+ - ✗
215
+ - ✅
216
+ - ✗
217
+ - ✅
218
+ - ✗
219
+ - ✅
220
+ - `✗ <https://github.com/vllm-project/vllm/issues/8198>`__
221
+ - ✅
222
+ -
223
+ -
224
+ -
225
+ -
226
+ -
227
+ * - :abbr:`MM (Multimodal)`
228
+ - `✗ <https://github.com/vllm-project/vllm/pull/8346>`__
229
+ - `✗ <https://github.com/vllm-project/vllm/pull/8348>`__
230
+ - `✗ <https://github.com/vllm-project/vllm/pull/7199>`__
231
+ - ?
232
+ - ?
233
+ - ✅
234
+ - ✗
235
+ - ✅
236
+ - ✅
237
+ - ✅
238
+ - ?
239
+ -
240
+ -
241
+ -
242
+ -
243
+ * - best-of
244
+ - ✅
245
+ - ✅
246
+ - ✅
247
+ - ✅
248
+ - `✗ <https://github.com/vllm-project/vllm/issues/6137>`__
249
+ - ✅
250
+ - ✅
251
+ - ✅
252
+ - ✅
253
+ - ?
254
+ - `✗ <https://github.com/vllm-project/vllm/issues/7968>`__
255
+ - ✅
256
+ -
257
+ -
258
+ -
259
+ * - beam-search
260
+ - ✅
261
+ - ✅
262
+ - ✅
263
+ - ✅
264
+ - `✗ <https://github.com/vllm-project/vllm/issues/6137>`__
265
+ - ✅
266
+ - ✅
267
+ - ✅
268
+ - ✅
269
+ - ?
270
+ - `✗ <https://github.com/vllm-project/vllm/issues/7968>`__
271
+ - ?
272
+ - ✅
273
+ -
274
+ -
275
+ * - :abbr:`guided dec (Guided Decoding)`
276
+ - ✅
277
+ - ✅
278
+ - ?
279
+ - ?
280
+ - ✅
281
+ - ✅
282
+ - ?
283
+ - ✅
284
+ - ✅
285
+ - ✅
286
+ - ✗
287
+ - ?
288
+ - ✅
289
+ - ✅
290
+ -
291
+
292
+
293
+Feature x Hardware
294
+^^^^^^^^^^^^^^^^^^
295
+
296
+.. list-table::
297
+ :header-rows: 1
298
+ :widths: auto
299
+
300
+ * - Feature
301
+ - Volta
302
+ - Turing
303
+ - Ampere
304
+ - Ada
305
+ - Hopper
306
+ - CPU
307
+ - AMD
308
+ * - :ref:`CP <chunked-prefill>`
309
+ - `✗ <https://github.com/vllm-project/vllm/issues/2729>`__
310
+ - ✅
311
+ - ✅
312
+ - ✅
313
+ - ✅
314
+ - ✗
315
+ - ✅
316
+ * - :ref:`APC <apc>`
317
+ - `✗ <https://github.com/vllm-project/vllm/issues/3687>`__
318
+ - ✅
319
+ - ✅
320
+ - ✅
321
+ - ✅
322
+ - ✗
323
+ - ✅
324
+ * - :ref:`LoRA <lora>`
325
+ - ✅
326
+ - ✅
327
+ - ✅
328
+ - ✅
329
+ - ✅
330
+ - `✗ <https://github.com/vllm-project/vllm/pull/4830>`__
331
+ - ✅
332
+ * - :abbr:`prmpt adptr (Prompt Adapter)`
333
+ - ✅
334
+ - ✅
335
+ - ✅
336
+ - ✅
337
+ - ✅
338
+ - `✗ <https://github.com/vllm-project/vllm/issues/8475>`__
339
+ - ✅
340
+ * - :ref:`SD <spec_decode>`
341
+ - ✅
342
+ - ✅
343
+ - ✅
344
+ - ✅
345
+ - ✅
346
+ - ✅
347
+ - ✅
348
+ * - CUDA graph
349
+ - ✅
350
+ - ✅
351
+ - ✅
352
+ - ✅
353
+ - ✅
354
+ - ✗
355
+ - ✅
356
+ * - :abbr:`enc-dec (Encoder-Decoder Models)`
357
+ - ✅
358
+ - ✅
359
+ - ✅
360
+ - ✅
361
+ - ✅
362
+ - `✗ <https://github.com/vllm-project/vllm/blob/a84e598e2125960d3b4f716b78863f24ac562947/vllm/worker/cpu_model_runner.py#L125>`__
363
+ - ✗
364
+ * - :abbr:`logP (Logprobs)`
365
+ - ✅
366
+ - ✅
367
+ - ✅
368
+ - ✅
369
+ - ✅
370
+ - ✅
371
+ - ✅
372
+ * - :abbr:`prmpt logP (Prompt Logprobs)`
373
+ - ✅
374
+ - ✅
375
+ - ✅
376
+ - ✅
377
+ - ✅
378
+ - ✅
379
+ - ✅
380
+ * - :abbr:`async output (Async Output Processing)`
381
+ - ✅
382
+ - ✅
383
+ - ✅
384
+ - ✅
385
+ - ✅
386
+ - ✗
387
+ - ✗
388
+ * - multi-step
389
+ - ✅
390
+ - ✅
391
+ - ✅
392
+ - ✅
393
+ - ✅
394
+ - `✗ <https://github.com/vllm-project/vllm/issues/8477>`__
395
+ - ✅
396
+ * - :abbr:`MM (Multimodal)`
397
+ - ✅
398
+ - ✅
399
+ - ✅
400
+ - ✅
401
+ - ✅
402
+ - ✅
403
+ - ✅
404
+ * - best-of
405
+ - ✅
406
+ - ✅
407
+ - ✅
408
+ - ✅
409
+ - ✅
410
+ - ✅
411
+ - ✅
412
+ * - beam-search
413
+ - ✅
414
+ - ✅
415
+ - ✅
416
+ - ✅
417
+ - ✅
418
+ - ✅
419
+ - ✅
420
+ * - :abbr:`guided dec (Guided Decoding)`
421
+ - ✅
422
+ - ✅
423
+ - ✅
424
+ - ✅
425
+ - ✅
426
+ - ✅
427
+ - ✅
vllm/attention/backends/rocm_flash_attn.py
+2
@@ -420,6 +420,8 @@ class ROCmFlashAttentionImpl(AttentionImpl):
420
Returns:
421
shape = [num_tokens, num_heads * head_size]
422
"""
423
+ # Reminder: Please update docs/source/serving/compatibility_matrix.rst
424
+ # If the feature combo become valid
425
if attn_type != AttentionType.DECODER:
426
raise NotImplementedError("Encoder self-attention and "
427
"encoder/decoder cross-attention "
vllm/config.py
+10
@@ -359,6 +359,8 @@ class ModelConfig:
359
self.use_async_output_proc = False
360
return
361
362
+ # Reminder: Please update docs/source/serving/compatibility_matrix.rst
363
+ # If the feature combo become valid
364
if device_config.device_type not in ("cuda", "tpu"):
365
logger.warning(
366
"Async output processing is only supported for CUDA or TPU. "
@@ -372,6 +374,8 @@ class ModelConfig:
374
self.use_async_output_proc = False
375
return
376
377
+ # Reminder: Please update docs/source/serving/compatibility_matrix.rst
378
+ # If the feature combo become valid
379
if device_config.device_type == "cuda" and self.enforce_eager:
380
logger.warning(
381
"To see benefits of async output processing, enable CUDA "
@@ -385,6 +389,8 @@ class ModelConfig:
389
if self.embedding_mode:
390
self.use_async_output_proc = False
391
392
+ # Reminder: Please update docs/source/serving/compatibility_matrix.rst
393
+ # If the feature combo become valid
394
if speculative_config:
395
logger.warning("Async output processing is not supported with"
396
" speculative decoding currently.")
@@ -1200,6 +1206,8 @@ class SpeculativeConfig:
1206
"speculative decoding is > 1, but got "
1207
f"{speculative_disable_by_batch_size=}")
1208
1209
+ # Reminder: Please update docs/source/serving/compatibility_matrix.rst
1210
+ # If the feature combo become valid
1211
if enable_chunked_prefill:
1212
raise ValueError(
1213
"Speculative decoding and chunked prefill are "
@@ -1561,6 +1569,8 @@ class LoRAConfig:
1569
model_config.quantization)
1570
1571
def verify_with_scheduler_config(self, scheduler_config: SchedulerConfig):
1572
+ # Reminder: Please update docs/source/serving/compatibility_matrix.rst
1573
+ # If the feature combo become valid
1574
if scheduler_config.chunked_prefill_enabled:
1575
raise ValueError("LoRA is not supported with chunked prefill yet.")
1576
vllm/engine/arg_utils.py
+2
@@ -1000,6 +1000,8 @@ class EngineArgs:
1000
disable_logprobs=self.disable_logprobs_during_spec_decoding,
1001
)
1002
1003
+ # Reminder: Please update docs/source/serving/compatibility_matrix.rst
1004
+ # If the feature combo become valid
1005
if self.num_scheduler_steps > 1:
1006
if speculative_config is not None:
1007
raise ValueError("Speculative decoding is not supported with "
vllm/engine/output_processor/multi_step.py
+2
@@ -62,6 +62,8 @@ class MultiStepOutputProcessor(SequenceGroupOutputProcessor):
62
@staticmethod
63
@functools.lru_cache()
64
def _log_prompt_logprob_unsupported_warning_once():
65
+ # Reminder: Please update docs/source/serving/compatibility_matrix.rst
66
+ # If the feature combo become valid
67
logger.warning(
68
"Prompt logprob is not supported by multi step workers. "
69
"(e.g., speculative decode uses multi step workers).")
vllm/executor/cpu_executor.py
+8
@@ -28,6 +28,8 @@ class CPUExecutor(ExecutorBase):
28
29
def _init_executor(self) -> None:
30
assert self.device_config.device_type == "cpu"
31
+ # Reminder: Please update docs/source/serving/compatibility_matrix.rst
32
+ # If the feature combo become valid
33
assert self.lora_config is None, "cpu backend doesn't support LoRA"
34
35
#
@@ -324,6 +326,8 @@ def _verify_and_get_model_config(config: ModelConfig) -> ModelConfig:
326
if config.dtype == torch.float16:
327
logger.warning("float16 is not supported on CPU, casting to bfloat16.")
328
config.dtype = torch.bfloat16
329
+ # Reminder: Please update docs/source/serving/compatibility_matrix.rst
330
+ # If the feature combo become valid
331
if not config.enforce_eager:
332
logger.warning(
333
"CUDA graph is not supported on CPU, fallback to the eager "
@@ -334,6 +338,8 @@ def _verify_and_get_model_config(config: ModelConfig) -> ModelConfig:
338
339
def _verify_and_get_scheduler_config(
340
config: SchedulerConfig) -> SchedulerConfig:
341
+ # Reminder: Please update docs/source/serving/compatibility_matrix.rst
342
+ # If the feature combo become valid
343
if config.chunked_prefill_enabled:
344
logger.warning("Chunked prefill is not supported on CPU, disable it.")
345
config.chunked_prefill_enabled = False
@@ -342,6 +348,8 @@ def _verify_and_get_scheduler_config(
348
349
350
def _verify_and_get_cache_config(config: CacheConfig) -> CacheConfig:
351
+ # Reminder: Please update docs/source/serving/compatibility_matrix.rst
352
+ # If the feature combo become valid
353
if config.enable_prefix_caching:
354
logger.warning("Prefix caching is not supported on CPU, disable it.")
355
config.enable_prefix_caching = False
vllm/inputs/preprocess.py
+2
@@ -310,6 +310,8 @@ class InputPreprocessor:
310
encoder_prompt, encoder_prompt_ids, encoder_mm_data, _ = encoder_comps
311
decoder_prompt, decoder_prompt_ids, decoder_mm_data, _ = decoder_comps
312
313
+ # Reminder: Please update docs/source/serving/compatibility_matrix.rst
314
+ # If the feature combo become valid
315
if decoder_mm_data is not None:
316
raise ValueError(
317
"Multi-modality decoder inputs of encoder-decoder models are "
vllm/spec_decode/spec_decode_worker.py
+2
@@ -87,6 +87,8 @@ def create_spec_worker(*args, **kwargs) -> "SpecDecodeWorker":
87
return spec_decode_worker
88
89
90
+# Reminder: Please update docs/source/serving/compatibility_matrix.rst
91
+# If the feature combo become valid
92
class SpecDecodeWorker(LoraNotSupportedWorkerBase):
93
"""Worker which implements speculative decoding.
94
vllm/utils.py
+3
@@ -41,6 +41,9 @@ logger = init_logger(__name__)
41
42
# Exception strings for non-implemented encoder/decoder scenarios
43
44
+# Reminder: Please update docs/source/serving/compatibility_matrix.rst
45
+# If the feature combo become valid
46
+
47
STR_NOT_IMPL_ENC_DEC_SWA = \
48
"Sliding window attention for encoder/decoder models " + \
49
"is not currently supported."
vllm/worker/multi_step_model_runner.py
+3
@@ -816,6 +816,9 @@ def _pythonize_sampler_output(
816
817
for sgdx, (seq_group,
818
sample_result) in enumerate(zip(seq_groups, samples_list)):
819
+ # Reminder: Please update docs/source/serving/compatibility_matrix.rst
820
+ # If the feature combo become valid
821
+ # (Check for Guided Decoding)
822
if seq_group.sampling_params.logits_processors:
823
assert len(seq_group.sampling_params.logits_processors) == 0, (
824
"Logits Processors are not supported in multi-step decoding")
vllm/worker/utils.py
+3
@@ -13,6 +13,9 @@ def assert_enc_dec_mr_supported_scenario(
13
a supported scenario.
14
'''
15
16
+ # Reminder: Please update docs/source/serving/compatibility_matrix.rst
17
+ # If the feature combo become valid
18
+
19
if enc_dec_mr.cache_config.enable_prefix_caching:
20
raise NotImplementedError(
21
STR_NOT_IMPL_ENC_DEC_ERR_STRS['STR_NOT_IMPL_ENC_DEC_PREFIX_CACHE'])