A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[Deprecation] Remove `prompt_token_ids` arg fallback in `LLM.generate` and `LLM.embed` (#18800)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
Cyrus Leung committed
Aug 22, 2025 at 10:56 UTC
8896eb72ebe90be6f1b83f32b3d3d0c379db4f82
24 files changed
+116
-467
.buildkite/lm-eval-harness/run-lm-eval-gsm-hf-baseline.sh
+1
-1
@@ -2,7 +2,7 @@
2
# We can use this script to compute baseline accuracy on GSM for transformers.
3
#
4
# Make sure you have lm-eval-harness installed:
5
-# pip install lm-eval==0.4.4
5
+# pip install git+https://github.com/EleutherAI/lm-evaluation-harness.git@206b7722158f58c35b7ffcd53b035fdbdda5126d#egg=lm-eval[api]
6
7
usage() {
8
echo``
.buildkite/lm-eval-harness/run-lm-eval-gsm-vllm-baseline.sh
+1
-1
@@ -3,7 +3,7 @@
3
# We use this for fp8, which HF does not support.
4
#
5
# Make sure you have lm-eval-harness installed:
6
-# pip install lm-eval==0.4.4
6
+# pip install git+https://github.com/EleutherAI/lm-evaluation-harness.git@206b7722158f58c35b7ffcd53b035fdbdda5126d#egg=lm-eval[api]
7
8
usage() {
9
echo``
docker/Dockerfile.rocm
+1
-1
@@ -71,7 +71,7 @@ COPY --from=build_vllm ${COMMON_WORKDIR}/vllm /vllm-workspace
71
RUN cd /vllm-workspace \
72
&& rm -rf vllm \
73
&& python3 -m pip install -e tests/vllm_test_utils \
74
- && python3 -m pip install lm-eval[api]==0.4.4 \
74
+ && python3 -m pip install git+https://github.com/EleutherAI/lm-evaluation-harness.git@206b7722158f58c35b7ffcd53b035fdbdda5126d#egg=lm-eval[api] \
75
&& python3 -m pip install pytest-shard
76
77
# -----------------------
docs/features/quantization/fp8.md
+1
-1
@@ -79,7 +79,7 @@ Since simple RTN does not require data for weight quantization and the activatio
79
Install `vllm` and `lm-evaluation-harness` for evaluation:
80
81
```bash
82
-pip install vllm lm-eval==0.4.4
82
+pip install vllm git+https://github.com/EleutherAI/lm-evaluation-harness.git@206b7722158f58c35b7ffcd53b035fdbdda5126d#egg=lm-eval[api]
83
```
84
85
Load and run the model in `vllm`:
docs/features/quantization/int4.md
+1
-1
@@ -18,7 +18,7 @@ pip install llmcompressor
18
Additionally, install `vllm` and `lm-evaluation-harness` for evaluation:
19
20
```bash
21
-pip install vllm lm-eval==0.4.4
21
+pip install vllm git+https://github.com/EleutherAI/lm-evaluation-harness.git@206b7722158f58c35b7ffcd53b035fdbdda5126d#egg=lm-eval[api]
22
```
23
24
## Quantization Process
docs/features/quantization/int8.md
+1
-1
@@ -19,7 +19,7 @@ pip install llmcompressor
19
Additionally, install `vllm` and `lm-evaluation-harness` for evaluation:
20
21
```bash
22
-pip install vllm lm-eval==0.4.4
22
+pip install vllm git+https://github.com/EleutherAI/lm-evaluation-harness.git@206b7722158f58c35b7ffcd53b035fdbdda5126d#egg=lm-eval[api]
23
```
24
25
## Quantization Process
docs/features/quantization/quark.md
+1
-1
@@ -20,7 +20,7 @@ for more installation details.
20
Additionally, install `vllm` and `lm-evaluation-harness` for evaluation:
21
22
```bash
23
-pip install vllm lm-eval==0.4.4
23
+pip install vllm git+https://github.com/EleutherAI/lm-evaluation-harness.git@206b7722158f58c35b7ffcd53b035fdbdda5126d#egg=lm-eval[api]
24
```
25
26
## Quantization Process
examples/offline_inference/spec_decode.py
+3
-1
@@ -5,6 +5,7 @@ from transformers import AutoTokenizer
5
6
from vllm import LLM, SamplingParams
7
from vllm.benchmarks.datasets import add_dataset_parser, get_samples
8
+from vllm.inputs import TokensPrompt
9
from vllm.v1.metrics.reader import Counter, Vector
10
11
try:
@@ -137,7 +138,8 @@ def main():
138
sampling_params = SamplingParams(temperature=args.temp, max_tokens=args.output_len)
139
if not args.custom_mm_prompts:
140
outputs = llm.generate(
140
- prompt_token_ids=prompt_ids, sampling_params=sampling_params
141
+ TokensPrompt(prompt_token_ids=prompt_ids),
142
+ sampling_params=sampling_params,
143
)
144
else:
145
outputs = llm.chat(prompts, sampling_params=sampling_params)
examples/offline_inference/structured_outputs.py
+1
-1
@@ -85,7 +85,7 @@ def format_output(title: str, output: str):
85
86
87
def generate_output(prompt: str, sampling_params: SamplingParams, llm: LLM):
88
- outputs = llm.generate(prompts=prompt, sampling_params=sampling_params)
88
+ outputs = llm.generate(prompt, sampling_params=sampling_params)
89
return outputs[0].outputs[0].text
90
91
requirements/nightly_torch_test.txt
+1
-1
@@ -27,7 +27,7 @@ mistral_common[image,audio] >= 1.8.2 # required for voxtral test
27
num2words # required for smolvlm test
28
opencv-python-headless >= 4.11.0 # required for video test
29
datamodel_code_generator # required for minicpm3 test
30
-lm-eval[api]==0.4.8 # required for model evaluation test
30
+lm-eval[api] @ git+https://github.com/EleutherAI/lm-evaluation-harness.git@206b7722158f58c35b7ffcd53b035fdbdda5126d # required for model evaluation test
31
mteb>=1.38.11, <2 # required for mteb test
32
transformers==4.52.4
33
tokenizers==0.21.1
requirements/test.in
+2
-1
@@ -32,7 +32,8 @@ num2words # required for smolvlm test
32
open_clip_torch==2.32.0 # Required for nemotron_vl test
33
opencv-python-headless >= 4.11.0 # required for video test
34
datamodel_code_generator # required for minicpm3 test
35
-lm-eval[api]==0.4.8 # required for model evaluation test
35
+# TODO: Use lm-eval[api]==0.4.10 once released
36
+lm-eval[api] @ git+https://github.com/EleutherAI/lm-evaluation-harness.git@206b7722158f58c35b7ffcd53b035fdbdda5126d # required for model evaluation test
37
mteb[bm25s]>=1.38.11, <2 # required for mteb test
38
transformers==4.55.2
39
tokenizers==0.21.1
requirements/test.txt
+1
-1
@@ -408,7 +408,7 @@ lightning-utilities==0.14.3
408
# torchmetrics
409
llvmlite==0.44.0
410
# via numba
411
-lm-eval==0.4.8
411
+lm-eval @ git+https://github.com/EleutherAI/lm-evaluation-harness.git@206b7722158f58c35b7ffcd53b035fdbdda5126d
412
# via -r requirements/test.in
413
lxml==5.3.0
414
# via
tests/entrypoints/llm/test_chat.py
+6
-9
@@ -18,10 +18,9 @@ def text_llm():
18
enforce_eager=True,
19
seed=0)
20
21
- with llm.deprecate_legacy_api():
22
- yield weakref.proxy(llm)
21
+ yield weakref.proxy(llm)
22
24
- del llm
23
+ del llm
24
25
cleanup_dist_env_and_memory()
26
@@ -88,10 +87,9 @@ def vision_llm():
87
seed=0,
88
)
89
91
- with llm.deprecate_legacy_api():
92
- yield weakref.proxy(llm)
90
+ yield weakref.proxy(llm)
91
94
- del llm
92
+ del llm
93
94
cleanup_dist_env_and_memory()
95
@@ -158,10 +156,9 @@ def thinking_llm():
156
seed=0,
157
)
158
161
- with llm.deprecate_legacy_api():
162
- yield weakref.proxy(llm)
159
+ yield weakref.proxy(llm)
160
164
- del llm
161
+ del llm
162
163
cleanup_dist_env_and_memory()
164
tests/entrypoints/llm/test_classify.py
+2
-3
@@ -35,10 +35,9 @@ def llm():
35
enforce_eager=True,
36
seed=0)
37
38
- with llm.deprecate_legacy_api():
39
- yield weakref.proxy(llm)
38
+ yield weakref.proxy(llm)
39
41
- del llm
40
+ del llm
41
42
cleanup_dist_env_and_memory()
43
tests/entrypoints/llm/test_embedding.py
+2
-3
@@ -26,10 +26,9 @@ def llm():
26
enforce_eager=True,
27
seed=0)
28
29
- with llm.deprecate_legacy_api():
30
- yield weakref.proxy(llm)
29
+ yield weakref.proxy(llm)
30
32
- del llm
31
+ del llm
32
33
cleanup_dist_env_and_memory()
34
tests/entrypoints/llm/test_encode.py
+3
-49
@@ -5,11 +5,9 @@ import weakref
5
6
import pytest
7
8
-from vllm import LLM, PoolingParams, PoolingRequestOutput
8
+from vllm import LLM, PoolingParams
9
from vllm.distributed import cleanup_dist_env_and_memory
10
11
-from ...models.utils import check_embeddings_close
12
-
11
MODEL_NAME = "intfloat/multilingual-e5-small"
12
13
PROMPTS = [
@@ -48,57 +46,13 @@ def llm():
46
enforce_eager=True,
47
seed=0)
48
51
- with llm.deprecate_legacy_api():
52
- yield weakref.proxy(llm)
49
+ yield weakref.proxy(llm)
50
54
- del llm
51
+ del llm
52
53
cleanup_dist_env_and_memory()
54
55
59
-def assert_outputs_match(o1: list[PoolingRequestOutput],
60
- o2: list[PoolingRequestOutput]):
61
- check_embeddings_close(
62
- embeddings_0_lst=[o.outputs.data for o in o1],
63
- embeddings_1_lst=[o.outputs.data for o in o2],
64
- name_0="hf",
65
- name_1="vllm",
66
- tol=1e-2,
67
- )
68
-
69
-
70
-@pytest.mark.skip_global_cleanup
71
-@pytest.mark.parametrize('prompt_token_ids', TOKEN_IDS)
72
-def test_v1_v2_api_consistency_single_prompt_tokens(llm: LLM,
73
- prompt_token_ids):
74
- pooling_params = PoolingParams()
75
-
76
- with pytest.warns(DeprecationWarning, match="'prompt_token_ids'"):
77
- v1_output = llm.encode(prompt_token_ids=prompt_token_ids,
78
- pooling_params=pooling_params)
79
-
80
- v2_output = llm.encode({"prompt_token_ids": prompt_token_ids},
81
- pooling_params=pooling_params)
82
- assert_outputs_match(v1_output, v2_output)
83
-
84
-
85
-@pytest.mark.skip_global_cleanup
86
-def test_v1_v2_api_consistency_multi_prompt_tokens(llm: LLM):
87
- pooling_params = PoolingParams()
88
-
89
- with pytest.warns(DeprecationWarning, match="'prompt_token_ids'"):
90
- v1_output = llm.encode(prompt_token_ids=TOKEN_IDS,
91
- pooling_params=pooling_params)
92
-
93
- v2_output = llm.encode(
94
- [{
95
- "prompt_token_ids": p
96
- } for p in TOKEN_IDS],
97
- pooling_params=pooling_params,
98
- )
99
- assert_outputs_match(v1_output, v2_output)
100
-
101
-
56
@pytest.mark.skip_global_cleanup
57
def test_multiple_pooling_params(llm: LLM):
58
pooling_params = [
tests/entrypoints/llm/test_generate.py
+3
-40
@@ -5,7 +5,7 @@ import weakref
5
6
import pytest
7
8
-from vllm import LLM, RequestOutput, SamplingParams
8
+from vllm import LLM, SamplingParams
9
from vllm.distributed import cleanup_dist_env_and_memory
10
11
MODEL_NAME = "distilbert/distilgpt2"
@@ -41,50 +41,13 @@ def llm():
41
gpu_memory_utilization=0.10,
42
enforce_eager=True)
43
44
- with llm.deprecate_legacy_api():
45
- yield weakref.proxy(llm)
44
+ yield weakref.proxy(llm)
45
47
- del llm
46
+ del llm
47
48
cleanup_dist_env_and_memory()
49
50
52
-def assert_outputs_equal(o1: list[RequestOutput], o2: list[RequestOutput]):
53
- assert [o.outputs for o in o1] == [o.outputs for o in o2]
54
-
55
-
56
-@pytest.mark.skip_global_cleanup
57
-@pytest.mark.parametrize('prompt_token_ids', TOKEN_IDS)
58
-def test_v1_v2_api_consistency_single_prompt_tokens(llm: LLM,
59
- prompt_token_ids):
60
- sampling_params = SamplingParams(temperature=0.0, top_p=1.0)
61
-
62
- with pytest.warns(DeprecationWarning, match="'prompt_token_ids'"):
63
- v1_output = llm.generate(prompt_token_ids=prompt_token_ids,
64
- sampling_params=sampling_params)
65
-
66
- v2_output = llm.generate({"prompt_token_ids": prompt_token_ids},
67
- sampling_params=sampling_params)
68
- assert_outputs_equal(v1_output, v2_output)
69
-
70
-
71
-@pytest.mark.skip_global_cleanup
72
-def test_v1_v2_api_consistency_multi_prompt_tokens(llm: LLM):
73
- sampling_params = SamplingParams(temperature=0.0, top_p=1.0)
74
-
75
- with pytest.warns(DeprecationWarning, match="'prompt_token_ids'"):
76
- v1_output = llm.generate(prompt_token_ids=TOKEN_IDS,
77
- sampling_params=sampling_params)
78
-
79
- v2_output = llm.generate(
80
- [{
81
- "prompt_token_ids": p
82
- } for p in TOKEN_IDS],
83
- sampling_params=sampling_params,
84
- )
85
- assert_outputs_equal(v1_output, v2_output)
86
-
87
-
51
@pytest.mark.skip_global_cleanup
52
def test_multiple_sampling_params(llm: LLM):
53
sampling_params = [
tests/entrypoints/llm/test_generate_multiple_loras.py
+2
-3
@@ -48,10 +48,9 @@ def llm(request, monkeypatch_module):
48
max_num_seqs=128,
49
enforce_eager=True)
50
51
- with llm.deprecate_legacy_api():
52
- yield weakref.proxy(llm)
51
+ yield weakref.proxy(llm)
52
54
- del llm
53
+ del llm
54
55
cleanup_dist_env_and_memory()
56
tests/entrypoints/llm/test_reward.py
+2
-3
@@ -36,10 +36,9 @@ def llm():
36
trust_remote_code=True,
37
seed=0)
38
39
- with llm.deprecate_legacy_api():
40
- yield weakref.proxy(llm)
39
+ yield weakref.proxy(llm)
40
42
- del llm
41
+ del llm
42
43
cleanup_dist_env_and_memory()
44
tests/entrypoints/llm/test_score.py
+2
-3
@@ -33,10 +33,9 @@ def llm():
33
enforce_eager=True,
34
seed=0)
35
36
- with llm.deprecate_legacy_api():
37
- yield weakref.proxy(llm)
36
+ yield weakref.proxy(llm)
37
39
- del llm
38
+ del llm
39
40
cleanup_dist_env_and_memory()
41
tests/quantization/test_fp8.py
+2
-4
@@ -38,8 +38,7 @@ def test_model_load_and_run(vllm_runner, model_id: str, force_marlin: bool,
38
with vllm_runner(model_id) as llm:
39
# note: this does not test accuracy, just that we can run through
40
# see lm-eval tests for accuracy
41
- outputs = llm.generate_greedy(prompts=["Hello my name is"],
42
- max_tokens=10)
41
+ outputs = llm.generate_greedy(["Hello my name is"], max_tokens=10)
42
print(outputs[0][1])
43
44
@@ -90,8 +89,7 @@ def test_kv_cache_model_load_and_run(vllm_runner, model_id: str,
89
90
# note: this does not test accuracy, just that we can run through
91
# see lm-eval tests for accuracy
93
- outputs = llm.generate_greedy(prompts=["Hello my name is"],
94
- max_tokens=10)
92
+ outputs = llm.generate_greedy(["Hello my name is"], max_tokens=10)
93
print(outputs[0][1])
94
95
tests/quantization/test_lm_head.py
+1
-1
@@ -46,5 +46,5 @@ def test_lm_head(
46
vllm_model.apply_model(check_model)
47
48
print(
49
- vllm_model.generate_greedy(prompts=["Hello my name is"],
49
+ vllm_model.generate_greedy(["Hello my name is"],
50
max_tokens=10)[0][1])
tests/v1/entrypoints/llm/test_struct_output_generate.py
+55
-49
@@ -127,13 +127,15 @@ def test_structured_output(
127
temperature=1.0,
128
max_tokens=4096,
129
guided_decoding=GuidedDecodingParams(json=sample_json_schema))
130
- outputs = llm.generate(prompts=[
131
- (f"Give an example JSON for an employee profile that fits this "
132
- f"schema. Make the response as short as possible. Schema: "
133
- f"{sample_json_schema}")
134
- ] * 2,
135
- sampling_params=sampling_params,
136
- use_tqdm=True)
130
+
131
+ prompt = ("Give an example JSON for an employee profile that fits this "
132
+ "schema. Make the response as short as possible. Schema: "
133
+ f"{sample_json_schema}")
134
+ outputs = llm.generate(
135
+ [prompt] * 2,
136
+ sampling_params=sampling_params,
137
+ use_tqdm=True,
138
+ )
139
140
assert outputs is not None
141
@@ -191,20 +193,24 @@ def test_structured_output(
193
with pytest.raises(ValueError,
194
match="The provided JSON schema contains features "
195
"not supported by xgrammar."):
196
+
197
+ prompt = (f"Give an example JSON for an employee profile that "
198
+ f"fits this schema: {unsupported_json_schema}. "
199
+ f"Make the response as short as possible.")
200
llm.generate(
195
- prompts=[(f"Give an example JSON for an employee profile that "
196
- f"fits this schema: {unsupported_json_schema}. "
197
- f"Make the response as short as possible.")] * 2,
201
+ [prompt] * 2,
202
sampling_params=sampling_params,
199
- use_tqdm=True)
203
+ use_tqdm=True,
204
+ )
205
else:
201
- outputs = llm.generate(prompts=(
202
- "Give an example JSON object for a grade "
203
- "that fits this schema: "
204
- f"{unsupported_json_schema}. Make the response as short as "
205
- "possible."),
206
- sampling_params=sampling_params,
207
- use_tqdm=True)
206
+ prompt = (f"Give an example JSON object for a grade that "
207
+ f"fits this schema: {unsupported_json_schema}. "
208
+ f"Make the response as short as possible.")
209
+ outputs = llm.generate(
210
+ prompt,
211
+ sampling_params=sampling_params,
212
+ use_tqdm=True,
213
+ )
214
assert outputs is not None
215
for output in outputs:
216
assert output is not None
@@ -227,10 +233,9 @@ def test_structured_output(
233
max_tokens=1000,
234
guided_decoding=GuidedDecodingParams(grammar=sample_sql_ebnf))
235
outputs = llm.generate(
230
- prompts=(
231
- "Generate a sql statement that selects col_1 from "
232
- "table_1 where it is equal to 1. Make the response as short as "
233
- "possible."),
236
+ ("Generate a sql statement that selects col_1 from "
237
+ "table_1 where it is equal to 1. Make the response as short as "
238
+ "possible."),
239
sampling_params=sampling_params,
240
use_tqdm=True,
241
)
@@ -261,10 +266,9 @@ def test_structured_output(
266
max_tokens=1000,
267
guided_decoding=GuidedDecodingParams(grammar=sample_sql_lark))
268
outputs = llm.generate(
264
- prompts=(
265
- "Generate a sql statement that selects col_1 from "
266
- "table_1 where it is equal to 1. Make the response as short as "
267
- "possible."),
269
+ ("Generate a sql statement that selects col_1 from "
270
+ "table_1 where it is equal to 1. Make the response as short as "
271
+ "possible."),
272
sampling_params=sampling_params,
273
use_tqdm=True,
274
)
@@ -301,7 +305,6 @@ def test_structured_output(
305
guided_decoding=GuidedDecodingParams(grammar="not a grammar"))
306
with pytest.raises(ValueError, match="Failed to convert the grammar "):
307
llm.generate(
304
- prompts=
308
("Generate a sql statement that selects col_1 from "
309
"table_1 where it is equal to 1. Make the response as short "
310
"as possible."),
@@ -316,11 +319,11 @@ def test_structured_output(
319
temperature=0.8,
320
top_p=0.95,
321
guided_decoding=GuidedDecodingParams(regex=sample_regex))
322
+
323
+ prompt = (f"Give an example IPv4 address with this regex: {sample_regex}. "
324
+ f"Make the response as short as possible.")
325
outputs = llm.generate(
320
- prompts=[
321
- (f"Give an example IPv4 address with this regex: {sample_regex}. "
322
- f"Make the response as short as possible.")
323
- ] * 2,
326
+ [prompt] * 2,
327
sampling_params=sampling_params,
328
use_tqdm=True,
329
)
@@ -343,11 +346,13 @@ def test_structured_output(
346
temperature=0.8,
347
top_p=0.95,
348
guided_decoding=GuidedDecodingParams(choice=sample_guided_choice))
349
+
350
outputs = llm.generate(
347
- prompts=("The best language for type-safe systems programming is "
348
- "(Make the response as short as possible.) "),
351
+ ("The best language for type-safe systems programming is "
352
+ "(Make the response as short as possible.) "),
353
sampling_params=sampling_params,
350
- use_tqdm=True)
354
+ use_tqdm=True,
355
+ )
356
assert outputs is not None
357
for output in outputs:
358
assert output is not None
@@ -367,12 +372,14 @@ def test_structured_output(
372
temperature=1.0,
373
max_tokens=1000,
374
guided_decoding=GuidedDecodingParams(json=json_schema))
370
- outputs = llm.generate(prompts=(
371
- "Generate a JSON with the brand, model and car_type of the most "
372
- "iconic car from the 90's. Make the response as short as "
373
- "possible."),
374
- sampling_params=sampling_params,
375
- use_tqdm=True)
375
+
376
+ outputs = llm.generate(
377
+ ("Generate a JSON with the brand, model and car_type of the most "
378
+ "iconic car from the 90's. Make the response as short as "
379
+ "possible."),
380
+ sampling_params=sampling_params,
381
+ use_tqdm=True,
382
+ )
383
384
assert outputs is not None
385
@@ -411,10 +418,11 @@ def test_structured_output(
418
guided_decoding=GuidedDecodingParams(json=json_schema))
419
420
outputs = llm.generate(
414
- prompts=("Generate a description of a frog using 50 characters. "
415
- "Make the response as short as possible."),
421
+ ("Generate a description of a frog using 50 characters. "
422
+ "Make the response as short as possible."),
423
sampling_params=sampling_params,
417
- use_tqdm=True)
424
+ use_tqdm=True,
425
+ )
426
427
assert outputs is not None
428
@@ -498,7 +506,7 @@ Make the response as short as possible.
506
"""
507
508
# Change this once other backends support structural_tag
501
- outputs = llm.generate(prompts=prompt,
509
+ outputs = llm.generate(prompt,
510
sampling_params=sampling_params,
511
use_tqdm=True)
512
assert outputs is not None
@@ -639,15 +647,13 @@ def test_structured_output_auto_mode(
647
f"{unsupported_json_schema}. Make the response as short as possible.")
648
# This would fail with the default of "xgrammar", but in "auto"
649
# we will handle fallback automatically.
642
- outputs = llm.generate(prompts=prompts,
650
+ outputs = llm.generate(prompts,
651
sampling_params=sampling_params,
652
use_tqdm=True)
653
# Make sure `auto` backend handling doesn't mess up sampling_params
654
# and that we can reuse it without error.
655
outputs.extend(
648
- llm.generate(prompts=prompts,
649
- sampling_params=sampling_params,
650
- use_tqdm=True))
656
+ llm.generate(prompts, sampling_params=sampling_params, use_tqdm=True))
657
658
assert outputs is not None
659
for output in outputs:
@@ -705,7 +711,7 @@ def test_guidance_no_additional_properties(monkeypatch: pytest.MonkeyPatch):
711
max_tokens=256,
712
guided_decoding=guided_params)
713
708
- outputs = llm.generate(prompts=prompt, sampling_params=sampling_params)
714
+ outputs = llm.generate(prompt, sampling_params=sampling_params)
715
assert outputs is not None
716
generated_text = outputs[0].outputs[0].text
717
assert generated_text is not None
vllm/entrypoints/llm.py
+21
-288
@@ -3,15 +3,13 @@
3
4
import itertools
5
from collections.abc import Sequence
6
-from contextlib import contextmanager
7
-from typing import (TYPE_CHECKING, Any, Callable, ClassVar, Optional, Union,
8
- cast, overload)
6
+from typing import TYPE_CHECKING, Any, Callable, Optional, Union, cast
7
8
import cloudpickle
9
import torch.nn as nn
10
from pydantic import ValidationError
11
from tqdm.auto import tqdm
14
-from typing_extensions import TypeVar, deprecated
12
+from typing_extensions import TypeVar
13
14
import vllm.envs as envs
15
from vllm.beam_search import (BeamSearchInstance, BeamSearchOutput,
@@ -40,7 +38,6 @@ from vllm.entrypoints.score_utils import (ScoreContentPartParam,
38
from vllm.entrypoints.utils import (_validate_truncation_size,
39
log_non_default_args)
40
from vllm.inputs import PromptType, SingletonPrompt, TextPrompt, TokensPrompt
43
-from vllm.inputs.parse import parse_and_batch_prompt
41
from vllm.logger import init_logger
42
from vllm.lora.request import LoRARequest
43
from vllm.model_executor.layers.quantization import QuantizationMethods
@@ -54,7 +51,7 @@ from vllm.tasks import PoolingTask
51
from vllm.transformers_utils.tokenizer import (AnyTokenizer, MistralTokenizer,
52
get_cached_tokenizer)
53
from vllm.usage.usage_lib import UsageContext
57
-from vllm.utils import Counter, Device, deprecate_kwargs, is_list_of
54
+from vllm.utils import Counter, Device, is_list_of
55
from vllm.v1.sample.logits_processor import LogitsProcessor
56
57
if TYPE_CHECKING:
@@ -157,18 +154,6 @@ class LLM:
154
serving, use the [AsyncLLMEngine][vllm.AsyncLLMEngine] class instead.
155
"""
156
160
- DEPRECATE_LEGACY: ClassVar[bool] = True
161
- """A flag to toggle whether to deprecate the legacy generate/encode API."""
162
-
163
- @classmethod
164
- @contextmanager
165
- def deprecate_legacy_api(cls):
166
- cls.DEPRECATE_LEGACY = True
167
-
168
- yield
169
-
170
- cls.DEPRECATE_LEGACY = False
171
-
157
def __init__(
158
self,
159
model: str,
@@ -325,99 +310,14 @@ class LLM:
310
return SamplingParams.from_optional(**self.default_sampling_params)
311
return SamplingParams()
312
328
- @overload
313
def generate(
314
self,
315
prompts: Union[PromptType, Sequence[PromptType]],
332
- /,
316
sampling_params: Optional[Union[SamplingParams,
317
Sequence[SamplingParams]]] = None,
318
*,
319
use_tqdm: Union[bool, Callable[..., tqdm]] = True,
320
lora_request: Optional[Union[list[LoRARequest], LoRARequest]] = None,
338
- ) -> list[RequestOutput]:
339
- ...
340
-
341
- @overload # LEGACY: single (prompt + optional token ids)
342
- @deprecated("'prompt_token_ids' will become part of 'prompts'")
343
- def generate(
344
- self,
345
- prompts: str,
346
- sampling_params: Optional[Union[SamplingParams,
347
- list[SamplingParams]]] = None,
348
- prompt_token_ids: Optional[list[int]] = None,
349
- use_tqdm: Union[bool, Callable[..., tqdm]] = True,
350
- lora_request: Optional[Union[list[LoRARequest], LoRARequest]] = None,
351
- ) -> list[RequestOutput]:
352
- ...
353
-
354
- @overload # LEGACY: multi (prompt + optional token ids)
355
- @deprecated("'prompt_token_ids' will become part of 'prompts'")
356
- def generate(
357
- self,
358
- prompts: list[str],
359
- sampling_params: Optional[Union[SamplingParams,
360
- list[SamplingParams]]] = None,
361
- prompt_token_ids: Optional[list[list[int]]] = None,
362
- use_tqdm: Union[bool, Callable[..., tqdm]] = True,
363
- lora_request: Optional[Union[list[LoRARequest], LoRARequest]] = None,
364
- ) -> list[RequestOutput]:
365
- ...
366
-
367
- @overload # LEGACY: single (token ids + optional prompt)
368
- @deprecated("'prompt_token_ids' will become part of 'prompts'")
369
- def generate(
370
- self,
371
- prompts: Optional[str] = None,
372
- sampling_params: Optional[Union[SamplingParams,
373
- list[SamplingParams]]] = None,
374
- *,
375
- prompt_token_ids: list[int],
376
- use_tqdm: Union[bool, Callable[..., tqdm]] = True,
377
- lora_request: Optional[Union[list[LoRARequest], LoRARequest]] = None,
378
- ) -> list[RequestOutput]:
379
- ...
380
-
381
- @overload # LEGACY: multi (token ids + optional prompt)
382
- @deprecated("'prompt_token_ids' will become part of 'prompts'")
383
- def generate(
384
- self,
385
- prompts: Optional[list[str]] = None,
386
- sampling_params: Optional[Union[SamplingParams,
387
- list[SamplingParams]]] = None,
388
- *,
389
- prompt_token_ids: list[list[int]],
390
- use_tqdm: Union[bool, Callable[..., tqdm]] = True,
391
- lora_request: Optional[Union[list[LoRARequest], LoRARequest]] = None,
392
- ) -> list[RequestOutput]:
393
- ...
394
-
395
- @overload # LEGACY: single or multi token ids [pos-only]
396
- @deprecated("'prompt_token_ids' will become part of 'prompts'")
397
- def generate(
398
- self,
399
- prompts: None,
400
- sampling_params: None,
401
- prompt_token_ids: Union[list[int], list[list[int]]],
402
- use_tqdm: Union[bool, Callable[..., tqdm]] = True,
403
- lora_request: Optional[Union[list[LoRARequest], LoRARequest]] = None,
404
- ) -> list[RequestOutput]:
405
- ...
406
-
407
- @deprecate_kwargs(
408
- "prompt_token_ids",
409
- is_deprecated=lambda: LLM.DEPRECATE_LEGACY,
410
- additional_message="Please use the 'prompts' parameter instead.",
411
- )
412
- def generate(
413
- self,
414
- prompts: Union[Union[PromptType, Sequence[PromptType]],
415
- Optional[Union[str, list[str]]]] = None,
416
- sampling_params: Optional[Union[SamplingParams,
417
- Sequence[SamplingParams]]] = None,
418
- prompt_token_ids: Optional[Union[list[int], list[list[int]]]] = None,
419
- use_tqdm: Union[bool, Callable[..., tqdm]] = True,
420
- lora_request: Optional[Union[list[LoRARequest], LoRARequest]] = None,
321
priority: Optional[list[int]] = None,
322
) -> list[RequestOutput]:
323
"""Generates the completions for the input prompts.
@@ -460,15 +360,6 @@ class LLM:
360
"Try passing `--runner generate` to use the model as a "
361
"generative model.")
362
463
- if prompt_token_ids is not None:
464
- parsed_prompts = self._convert_v1_inputs(
465
- prompts=cast(Optional[Union[str, list[str]]], prompts),
466
- prompt_token_ids=prompt_token_ids,
467
- )
468
- else:
469
- parsed_prompts = cast(Union[PromptType, Sequence[PromptType]],
470
- prompts)
471
-
363
if sampling_params is None:
364
# Use default sampling params.
365
sampling_params = self.get_default_sampling_params()
@@ -483,10 +374,10 @@ class LLM:
374
375
# Add any modality specific loras to the corresponding prompts
376
lora_request = self._get_modality_specific_lora_reqs(
486
- parsed_prompts, lora_request)
377
+ prompts, lora_request)
378
379
self._validate_and_add_requests(
489
- prompts=parsed_prompts,
380
+ prompts=prompts,
381
params=sampling_params,
382
use_tqdm=use_tqdm,
383
lora_request=lora_request,
@@ -498,7 +389,7 @@ class LLM:
389
return self.engine_class.validate_outputs(outputs, RequestOutput)
390
391
def _get_modality_specific_lora_reqs(
501
- self, parsed_prompts: Union[PromptType, Sequence[PromptType]],
392
+ self, prompts: Union[PromptType, Sequence[PromptType]],
393
lora_request: Optional[Union[list[LoRARequest], LoRARequest]]):
394
# Grab the lora config off the vllm config on the engine,
395
# since this is the same for both v0 & v1.
@@ -511,35 +402,33 @@ class LLM:
402
or (lora_config and lora_config.default_mm_loras is None)):
403
return lora_request
404
514
- if not isinstance(parsed_prompts, Sequence):
515
- parsed_prompts = [parsed_prompts]
405
+ if not isinstance(prompts, Sequence):
406
+ prompts = [prompts]
407
517
- optional_loras = ([lora_request] * len(parsed_prompts)
408
+ optional_loras = ([lora_request] * len(prompts)
409
if not isinstance(lora_request, Sequence) else
410
lora_request)
411
412
return [
413
self._resolve_single_prompt_mm_lora(
523
- parsed_prompt,
414
+ prompt,
415
opt_lora_req,
416
lora_config.default_mm_loras,
526
- ) for parsed_prompt, opt_lora_req in zip(parsed_prompts,
527
- optional_loras)
417
+ ) for prompt, opt_lora_req in zip(prompts, optional_loras)
418
]
419
530
- def _resolve_single_prompt_mm_lora(self, parsed_prompt: PromptType,
420
+ def _resolve_single_prompt_mm_lora(self, prompt: PromptType,
421
lora_request: Optional[LoRARequest],
422
default_mm_loras: Optional[dict[str,
423
str]]):
534
- if (not default_mm_loras or not isinstance(parsed_prompt, dict)
535
- or "multi_modal_data" not in parsed_prompt):
424
+ if (not default_mm_loras or not isinstance(prompt, dict)
425
+ or "multi_modal_data" not in prompt):
426
return lora_request
427
538
- parsed_prompt = cast(Union[TextPrompt, TokensPrompt], parsed_prompt)
428
+ prompt = cast(Union[TextPrompt, TokensPrompt], prompt)
429
540
- intersection = set(
541
- parsed_prompt["multi_modal_data"].keys()).intersection(
542
- default_mm_loras.keys())
430
+ intersection = set(prompt["multi_modal_data"].keys()) \
431
+ .intersection(default_mm_loras.keys())
432
if not intersection:
433
return lora_request
434
if len(intersection) > 1:
@@ -933,120 +822,17 @@ class LLM:
822
lora_request=lora_request,
823
)
824
936
- @overload
825
def encode(
826
self,
827
prompts: Union[PromptType, Sequence[PromptType]],
940
- /,
941
- pooling_params: Optional[Union[PoolingParams,
942
- Sequence[PoolingParams]]] = None,
943
- *,
944
- truncate_prompt_tokens: Optional[int] = None,
945
- use_tqdm: Union[bool, Callable[..., tqdm]] = True,
946
- lora_request: Optional[Union[list[LoRARequest], LoRARequest]] = None,
947
- pooling_task: PoolingTask = "encode",
948
- tokenization_kwargs: Optional[dict[str, Any]] = None,
949
- ) -> list[PoolingRequestOutput]:
950
- ...
951
-
952
- @overload # LEGACY: single (prompt + optional token ids)
953
- @deprecated("'prompt_token_ids' will become part of 'prompts'")
954
- def encode(
955
- self,
956
- prompts: str,
957
- pooling_params: Optional[Union[PoolingParams,
958
- Sequence[PoolingParams]]] = None,
959
- prompt_token_ids: Optional[list[int]] = None,
960
- truncate_prompt_tokens: Optional[int] = None,
961
- use_tqdm: Union[bool, Callable[..., tqdm]] = True,
962
- lora_request: Optional[Union[list[LoRARequest], LoRARequest]] = None,
963
- pooling_task: PoolingTask = "encode",
964
- tokenization_kwargs: Optional[dict[str, Any]] = None,
965
- ) -> list[PoolingRequestOutput]:
966
- ...
967
-
968
- @overload # LEGACY: multi (prompt + optional token ids)
969
- @deprecated("'prompt_token_ids' will become part of 'prompts'")
970
- def encode(
971
- self,
972
- prompts: list[str],
973
- pooling_params: Optional[Union[PoolingParams,
974
- Sequence[PoolingParams]]] = None,
975
- prompt_token_ids: Optional[list[list[int]]] = None,
976
- truncate_prompt_tokens: Optional[int] = None,
977
- use_tqdm: Union[bool, Callable[..., tqdm]] = True,
978
- lora_request: Optional[Union[list[LoRARequest], LoRARequest]] = None,
979
- pooling_task: PoolingTask = "encode",
980
- tokenization_kwargs: Optional[dict[str, Any]] = None,
981
- ) -> list[PoolingRequestOutput]:
982
- ...
983
-
984
- @overload # LEGACY: single (token ids + optional prompt)
985
- @deprecated("'prompt_token_ids' will become part of 'prompts'")
986
- def encode(
987
- self,
988
- prompts: Optional[str] = None,
989
- pooling_params: Optional[Union[PoolingParams,
990
- Sequence[PoolingParams]]] = None,
991
- *,
992
- prompt_token_ids: list[int],
993
- truncate_prompt_tokens: Optional[int] = None,
994
- use_tqdm: Union[bool, Callable[..., tqdm]] = True,
995
- lora_request: Optional[Union[list[LoRARequest], LoRARequest]] = None,
996
- pooling_task: PoolingTask = "encode",
997
- tokenization_kwargs: Optional[dict[str, Any]] = None,
998
- ) -> list[PoolingRequestOutput]:
999
- ...
1000
-
1001
- @overload # LEGACY: multi (token ids + optional prompt)
1002
- @deprecated("'prompt_token_ids' will become part of 'prompts'")
1003
- def encode(
1004
- self,
1005
- prompts: Optional[list[str]] = None,
828
pooling_params: Optional[Union[PoolingParams,
829
Sequence[PoolingParams]]] = None,
830
*,
1009
- prompt_token_ids: list[list[int]],
831
truncate_prompt_tokens: Optional[int] = None,
832
use_tqdm: Union[bool, Callable[..., tqdm]] = True,
833
lora_request: Optional[Union[list[LoRARequest], LoRARequest]] = None,
834
pooling_task: PoolingTask = "encode",
835
tokenization_kwargs: Optional[dict[str, Any]] = None,
1015
- ) -> list[PoolingRequestOutput]:
1016
- ...
1017
-
1018
- @overload # LEGACY: single or multi token ids [pos-only]
1019
- @deprecated("'prompt_token_ids' will become part of 'prompts'")
1020
- def encode(
1021
- self,
1022
- prompts: None,
1023
- pooling_params: None,
1024
- prompt_token_ids: Union[list[int], list[list[int]]],
1025
- truncate_prompt_tokens: Optional[int] = None,
1026
- use_tqdm: Union[bool, Callable[..., tqdm]] = True,
1027
- lora_request: Optional[Union[list[LoRARequest], LoRARequest]] = None,
1028
- pooling_task: PoolingTask = "encode",
1029
- tokenization_kwargs: Optional[dict[str, Any]] = None,
1030
- ) -> list[PoolingRequestOutput]:
1031
- ...
1032
-
1033
- @deprecate_kwargs(
1034
- "prompt_token_ids",
1035
- is_deprecated=lambda: LLM.DEPRECATE_LEGACY,
1036
- additional_message="Please use the 'prompts' parameter instead.",
1037
- )
1038
- def encode(
1039
- self,
1040
- prompts: Union[Union[PromptType, Sequence[PromptType]],
1041
- Optional[Union[str, list[str]]]] = None,
1042
- pooling_params: Optional[Union[PoolingParams,
1043
- Sequence[PoolingParams]]] = None,
1044
- prompt_token_ids: Optional[Union[list[int], list[list[int]]]] = None,
1045
- truncate_prompt_tokens: Optional[int] = None,
1046
- use_tqdm: Union[bool, Callable[..., tqdm]] = True,
1047
- lora_request: Optional[Union[list[LoRARequest], LoRARequest]] = None,
1048
- pooling_task: Optional[PoolingTask] = None,
1049
- tokenization_kwargs: Optional[dict[str, Any]] = None,
836
) -> list[PoolingRequestOutput]:
837
"""Apply pooling to the hidden states corresponding to the input
838
prompts.
@@ -1108,15 +894,6 @@ class LLM:
894
raise ValueError(
895
f"pooling_task must be one of {self.supported_tasks}.")
896
1111
- if prompt_token_ids is not None:
1112
- parsed_prompts = self._convert_v1_inputs(
1113
- prompts=cast(Optional[Union[str, list[str]]], prompts),
1114
- prompt_token_ids=prompt_token_ids,
1115
- )
1116
- else:
1117
- parsed_prompts = cast(Union[PromptType, Sequence[PromptType]],
1118
- prompts)
1119
-
897
if pooling_params is None:
898
# Use default pooling params.
899
pooling_params = PoolingParams()
@@ -1134,7 +911,7 @@ class LLM:
911
tokenization_kwargs)
912
913
self._validate_and_add_requests(
1137
- prompts=parsed_prompts,
914
+ prompts=prompts,
915
params=pooling_params,
916
use_tqdm=use_tqdm,
917
lora_request=lora_request,
@@ -1148,7 +925,6 @@ class LLM:
925
def embed(
926
self,
927
prompts: Union[PromptType, Sequence[PromptType]],
1151
- /,
928
*,
929
truncate_prompt_tokens: Optional[int] = None,
930
use_tqdm: Union[bool, Callable[..., tqdm]] = True,
@@ -1198,7 +974,6 @@ class LLM:
974
def classify(
975
self,
976
prompts: Union[PromptType, Sequence[PromptType]],
1201
- /,
977
*,
978
use_tqdm: Union[bool, Callable[..., tqdm]] = True,
979
pooling_params: Optional[Union[PoolingParams,
@@ -1348,7 +1123,7 @@ class LLM:
1123
_validate_truncation_size(model_config.max_model_len,
1124
truncate_prompt_tokens, tokenization_kwargs)
1125
1351
- parsed_prompts = []
1126
+ prompts = list[PromptType]()
1127
1128
input_pairs = [(t1, t2) for t1, t2 in zip(data_1, data_2)]
1129
@@ -1372,10 +1147,10 @@ class LLM:
1147
else:
1148
pooling_params_list.append(pooling_params)
1149
1375
- parsed_prompts.append(engine_prompt)
1150
+ prompts.append(engine_prompt)
1151
1152
self._validate_and_add_requests(
1378
- prompts=parsed_prompts,
1153
+ prompts=prompts,
1154
params=pooling_params_list,
1155
use_tqdm=use_tqdm,
1156
lora_request=lora_request,
@@ -1585,48 +1360,6 @@ class LLM:
1360
assert isinstance(self.llm_engine, V1LLMEngine)
1361
return self.llm_engine.get_metrics()
1362
1588
- # LEGACY
1589
- def _convert_v1_inputs(
1590
- self,
1591
- prompts: Optional[Union[str, list[str]]],
1592
- prompt_token_ids: Optional[Union[list[int], list[list[int]]]],
1593
- ):
1594
- # skip_tokenizer_init is now checked in engine
1595
-
1596
- if prompts is None and prompt_token_ids is None:
1597
- raise ValueError(
1598
- "Either prompts or prompt_token_ids must be provided.")
1599
- if prompts is not None and prompt_token_ids is not None \
1600
- and len(prompts) != len(prompt_token_ids):
1601
- raise ValueError(
1602
- "The lengths of prompts and prompt_token_ids must be the same."
1603
- )
1604
-
1605
- if prompts is not None:
1606
- prompts = [p["content"] for p in parse_and_batch_prompt(prompts)]
1607
- if prompt_token_ids is not None:
1608
- prompt_token_ids = [
1609
- p["content"] for p in parse_and_batch_prompt(prompt_token_ids)
1610
- ]
1611
- if prompts is not None:
1612
- num_requests = len(prompts)
1613
- elif prompt_token_ids is not None:
1614
- num_requests = len(prompt_token_ids)
1615
- parsed_prompts: list[PromptType] = []
1616
- for i in range(num_requests):
1617
- item: PromptType
1618
-
1619
- if prompts is not None:
1620
- item = TextPrompt(prompt=prompts[i])
1621
- elif prompt_token_ids is not None:
1622
- item = TokensPrompt(prompt_token_ids=prompt_token_ids[i])
1623
- else:
1624
- raise AssertionError
1625
-
1626
- parsed_prompts.append(item)
1627
-
1628
- return parsed_prompts
1629
-
1363
def _validate_and_add_requests(
1364
self,
1365
prompts: Union[PromptType, Sequence[PromptType]],