A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 07:00 UTC
[Bugfix] Enforce no chunked prefill for embedding models (#10470)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
Cyrus Leung committed
Nov 20, 2024 at 13:12 UTC
b4be5a8adba95020187ae3cb43a7db7eef20c0ff
2 files changed
+60
-15
docs/source/serving/compatibility_matrix.rst
+55
-14
@@ -39,12 +39,13 @@ Feature x Feature
39
- :abbr:`prmpt adptr (Prompt Adapter)`
40
- :ref:`SD <spec_decode>`
41
- CUDA graph
42
+ - :abbr:`emd (Embedding Models)`
43
- :abbr:`enc-dec (Encoder-Decoder Models)`
44
- :abbr:`logP (Logprobs)`
45
- :abbr:`prmpt logP (Prompt Logprobs)`
46
- :abbr:`async output (Async Output Processing)`
47
- multi-step
47
- - :abbr:`MM (Multimodal)`
48
+ - :abbr:`mm (Multimodal)`
49
- best-of
50
- beam-search
51
- :abbr:`guided dec (Guided Decoding)`
@@ -64,6 +65,7 @@ Feature x Feature
65
-
66
-
67
-
68
+ -
69
* - :ref:`APC <apc>`
70
- ✅
71
-
@@ -80,6 +82,7 @@ Feature x Feature
82
-
83
-
84
-
85
+ -
86
* - :ref:`LoRA <lora>`
87
- `✗ <https://github.com/vllm-project/vllm/pull/9057>`__
88
- ✅
@@ -96,6 +99,7 @@ Feature x Feature
99
-
100
-
101
-
102
+ -
103
* - :abbr:`prmpt adptr (Prompt Adapter)`
104
- ✅
105
- ✅
@@ -112,6 +116,7 @@ Feature x Feature
116
-
117
-
118
-
119
+ -
120
* - :ref:`SD <spec_decode>`
121
- ✗
122
- ✅
@@ -128,6 +133,7 @@ Feature x Feature
133
-
134
-
135
-
136
+ -
137
* - CUDA graph
138
- ✅
139
- ✅
@@ -144,6 +150,24 @@ Feature x Feature
150
-
151
-
152
-
153
+ -
154
+ * - :abbr:`emd (Embedding Models)`
155
+ - ✗
156
+ - ✗
157
+ - ✗
158
+ - ✗
159
+ - ✗
160
+ - ✗
161
+ -
162
+ -
163
+ -
164
+ -
165
+ -
166
+ -
167
+ -
168
+ -
169
+ -
170
+ -
171
* - :abbr:`enc-dec (Encoder-Decoder Models)`
172
- ✗
173
- `✗ <https://github.com/vllm-project/vllm/issues/7366>`__
@@ -151,6 +175,7 @@ Feature x Feature
175
- ✗
176
- `✗ <https://github.com/vllm-project/vllm/issues/7366>`__
177
- ✅
178
+ - ✅
179
-
180
-
181
-
@@ -166,7 +191,8 @@ Feature x Feature
191
- ✅
192
- ✅
193
- ✅
169
- - ✅
194
+ - ✅
195
+ - ✗
196
- ✅
197
-
198
-
@@ -183,7 +209,8 @@ Feature x Feature
209
- ✅
210
- `✗ <https://github.com/vllm-project/vllm/pull/8199>`__
211
- ✅
186
- - ✅
212
+ - ✗
213
+ - ✅
214
- ✅
215
-
216
-
@@ -199,6 +226,7 @@ Feature x Feature
226
- ✅
227
- ✗
228
- ✅
229
+ - ✗
230
- ✗
231
- ✅
232
- ✅
@@ -215,6 +243,7 @@ Feature x Feature
243
- ✅
244
- ✗
245
- ✅
246
+ - ✗
247
- ✗
248
- ✅
249
- `✗ <https://github.com/vllm-project/vllm/issues/8198>`__
@@ -224,14 +253,15 @@ Feature x Feature
253
-
254
-
255
-
227
- * - :abbr:`MM (Multimodal)`
228
- - `✗ <https://github.com/vllm-project/vllm/pull/8346>`__
256
+ * - :abbr:`mm (Multimodal)`
257
+ - ✅
258
- `✗ <https://github.com/vllm-project/vllm/pull/8348>`__
259
- `✗ <https://github.com/vllm-project/vllm/pull/7199>`__
260
- ?
261
- ?
262
- ✅
234
- - ✗
263
+ - ✅
264
+ - ✅
265
- ✅
266
- ✅
267
- ✅
@@ -247,6 +277,7 @@ Feature x Feature
277
- ✅
278
- `✗ <https://github.com/vllm-project/vllm/issues/6137>`__
279
- ✅
280
+ - ✗
281
- ✅
282
- ✅
283
- ✅
@@ -263,6 +294,7 @@ Feature x Feature
294
- ✅
295
- `✗ <https://github.com/vllm-project/vllm/issues/6137>`__
296
- ✅
297
+ - ✗
298
- ✅
299
- ✅
300
- ✅
@@ -279,6 +311,7 @@ Feature x Feature
311
- ?
312
- ✅
313
- ✅
314
+ - ✗
315
- ?
316
- ✅
317
- ✅
@@ -353,6 +386,14 @@ Feature x Hardware
386
- ✅
387
- ✗
388
- ✅
389
+ * - :abbr:`emd (Embedding Models)`
390
+ - ✅
391
+ - ✅
392
+ - ✅
393
+ - ✅
394
+ - ✅
395
+ - ✅
396
+ - ✗
397
* - :abbr:`enc-dec (Encoder-Decoder Models)`
398
- ✅
399
- ✅
@@ -361,7 +402,7 @@ Feature x Hardware
402
- ✅
403
- ✅
404
- ✗
364
- * - :abbr:`logP (Logprobs)`
405
+ * - :abbr:`mm (Multimodal)`
406
- ✅
407
- ✅
408
- ✅
@@ -369,7 +410,7 @@ Feature x Hardware
410
- ✅
411
- ✅
412
- ✅
372
- * - :abbr:`prmpt logP (Prompt Logprobs)`
413
+ * - :abbr:`logP (Logprobs)`
414
- ✅
415
- ✅
416
- ✅
@@ -377,29 +418,29 @@ Feature x Hardware
418
- ✅
419
- ✅
420
- ✅
380
- * - :abbr:`async output (Async Output Processing)`
421
+ * - :abbr:`prmpt logP (Prompt Logprobs)`
422
- ✅
423
- ✅
424
- ✅
425
- ✅
426
- ✅
386
- - ✗
387
- - ✗
388
- * - multi-step
427
- ✅
428
- ✅
429
+ * - :abbr:`async output (Async Output Processing)`
430
- ✅
431
- ✅
432
- ✅
394
- - `✗ <https://github.com/vllm-project/vllm/issues/8477>`__
433
- ✅
396
- * - :abbr:`MM (Multimodal)`
434
- ✅
435
+ - ✗
436
+ - ✗
437
+ * - multi-step
438
- ✅
439
- ✅
440
- ✅
441
- ✅
442
- ✅
443
+ - `✗ <https://github.com/vllm-project/vllm/issues/8477>`__
444
- ✅
445
* - best-of
446
- ✅
vllm/engine/arg_utils.py
+5
-1
@@ -1014,7 +1014,8 @@ class EngineArgs:
1014
use_spec_decode = self.speculative_model is not None
1015
if (is_gpu and not use_sliding_window and not use_spec_decode
1016
and not self.enable_lora
1017
- and not self.enable_prompt_adapter):
1017
+ and not self.enable_prompt_adapter
1018
+ and model_config.task != "embedding"):
1019
self.enable_chunked_prefill = True
1020
logger.warning(
1021
"Chunked prefill is enabled by default for models with "
@@ -1031,6 +1032,9 @@ class EngineArgs:
1032
"errors during the initial memory profiling phase, or result "
1033
"in low performance due to small KV cache space. Consider "
1034
"setting --max-model-len to a smaller value.", max_model_len)
1035
+ elif self.enable_chunked_prefill and model_config.task == "embedding":
1036
+ msg = "Chunked prefill is not supported for embedding models"
1037
+ raise ValueError(msg)
1038
1039
speculative_config = SpeculativeConfig.maybe_create_spec_config(
1040
target_model_config=model_config,