A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 00:30 UTC
[Core] Consolidate prompt arguments to LLM engines (#4328)
Co-authored-by: Roger Wang <ywang@roblox.com>
Cyrus Leung committed
May 29, 2024 at 04:29 UTC
5ae5ed1e6047d4095149e26526a618be0529a118
43 files changed
+1404
-439
.buildkite/test-pipeline.yaml
+6
-3
@@ -63,9 +63,9 @@ steps:
63
mirror_hardwares: [amd]
64
65
commands:
66
- # these tests have to be separated, because each one will allocate all posible GPU memory
67
- - pytest -v -s entrypoints --ignore=entrypoints/test_server_oot_registration.py
68
- - pytest -v -s entrypoints/test_server_oot_registration.py
66
+ - pytest -v -s test_inputs.py
67
+ - pytest -v -s entrypoints -m llm
68
+ - pytest -v -s entrypoints -m openai
69
70
- label: Examples Test
71
working_dir: "/vllm-workspace/examples"
@@ -110,6 +110,9 @@ steps:
110
mirror_hardwares: [amd]
111
command: pytest -v -s test_logits_processor.py
112
113
+- label: Utils Test
114
+ command: pytest -v -s test_utils.py
115
+
116
- label: Worker Test
117
mirror_hardwares: [amd]
118
command: pytest -v -s worker
benchmarks/benchmark_latency.py
+7
-4
@@ -3,13 +3,14 @@ import argparse
3
import json
4
import time
5
from pathlib import Path
6
-from typing import Optional
6
+from typing import List, Optional
7
8
import numpy as np
9
import torch
10
from tqdm import tqdm
11
12
from vllm import LLM, SamplingParams
13
+from vllm.inputs import PromptStrictInputs
14
from vllm.model_executor.layers.quantization import QUANTIZATION_METHODS
15
16
@@ -48,7 +49,9 @@ def main(args: argparse.Namespace):
49
dummy_prompt_token_ids = np.random.randint(10000,
50
size=(args.batch_size,
51
args.input_len))
51
- dummy_prompt_token_ids = dummy_prompt_token_ids.tolist()
52
+ dummy_inputs: List[PromptStrictInputs] = [{
53
+ "prompt_token_ids": batch
54
+ } for batch in dummy_prompt_token_ids.tolist()]
55
56
def run_to_completion(profile_dir: Optional[str] = None):
57
if profile_dir:
@@ -59,13 +62,13 @@ def main(args: argparse.Namespace):
62
],
63
on_trace_ready=torch.profiler.tensorboard_trace_handler(
64
str(profile_dir))) as p:
62
- llm.generate(prompt_token_ids=dummy_prompt_token_ids,
65
+ llm.generate(dummy_inputs,
66
sampling_params=sampling_params,
67
use_tqdm=False)
68
print(p.key_averages())
69
else:
70
start_time = time.perf_counter()
68
- llm.generate(prompt_token_ids=dummy_prompt_token_ids,
71
+ llm.generate(dummy_inputs,
72
sampling_params=sampling_params,
73
use_tqdm=False)
74
end_time = time.perf_counter()
docs/source/dev/offline_inference/llm.rst
renamed
+1
-1
@@ -1,5 +1,5 @@
1
LLM Class
2
-==========
2
+=========
3
4
.. autoclass:: vllm.LLM
5
:members:
docs/source/dev/offline_inference/llm_inputs.rst
new
+14
@@ -0,0 +1,14 @@
1
+LLM Inputs
2
+==========
3
+
4
+.. autodata:: vllm.inputs.PromptStrictInputs
5
+
6
+.. autoclass:: vllm.inputs.TextPrompt
7
+ :show-inheritance:
8
+ :members:
9
+ :member-order: bysource
10
+
11
+.. autoclass:: vllm.inputs.TokensPrompt
12
+ :show-inheritance:
13
+ :members:
14
+ :member-order: bysource
docs/source/dev/offline_inference/offline_index.rst
new
+8
@@ -0,0 +1,8 @@
1
+Offline Inference
2
+=================================
3
+
4
+.. toctree::
5
+ :maxdepth: 1
6
+
7
+ llm
8
+ llm_inputs
docs/source/dev/sampling_params.rst
renamed
docs/source/index.rst
+3
-8
@@ -68,13 +68,6 @@ Documentation
68
getting_started/quickstart
69
getting_started/examples/examples_index
70
71
-.. toctree::
72
- :maxdepth: 1
73
- :caption: Offline Inference
74
-
75
- offline_inference/llm
76
- offline_inference/sampling_params
77
-
71
.. toctree::
72
:maxdepth: 1
73
:caption: Serving
@@ -108,7 +101,9 @@ Documentation
101
.. toctree::
102
:maxdepth: 2
103
:caption: Developer Documentation
111
-
104
+
105
+ dev/sampling_params
106
+ dev/offline_inference/offline_index
107
dev/engine/engine_index
108
dev/kernel/paged_attention
109
dev/dockerfile/dockerfile
docs/source/serving/openai_compatible_server.md
+2
-2
@@ -48,7 +48,7 @@ completion = client.chat.completions.create(
48
```
49
50
### Extra Parameters for Chat API
51
-The following [sampling parameters (click through to see documentation)](../offline_inference/sampling_params.rst) are supported.
51
+The following [sampling parameters (click through to see documentation)](../dev/sampling_params.rst) are supported.
52
53
```{literalinclude} ../../../vllm/entrypoints/openai/protocol.py
54
:language: python
@@ -65,7 +65,7 @@ The following extra parameters are supported:
65
```
66
67
### Extra Parameters for Completions API
68
-The following [sampling parameters (click through to see documentation)](../offline_inference/sampling_params.rst) are supported.
68
+The following [sampling parameters (click through to see documentation)](../dev/sampling_params.rst) are supported.
69
70
```{literalinclude} ../../../vllm/entrypoints/openai/protocol.py
71
:language: python
examples/llava_example.py
+16
-9
@@ -23,11 +23,15 @@ def run_llava_pixel_values():
23
"\nUSER: What is the content of this image?\nASSISTANT:")
24
25
# This should be provided by another online or offline component.
26
- images = torch.load("images/stop_sign_pixel_values.pt")
26
+ image = torch.load("images/stop_sign_pixel_values.pt")
27
+
28
+ outputs = llm.generate({
29
+ "prompt":
30
+ prompt,
31
+ "multi_modal_data":
32
+ MultiModalData(type=MultiModalData.Type.IMAGE, data=image),
33
+ })
34
28
- outputs = llm.generate(prompt,
29
- multi_modal_data=MultiModalData(
30
- type=MultiModalData.Type.IMAGE, data=images))
35
for o in outputs:
36
generated_text = o.outputs[0].text
37
print(generated_text)
@@ -46,11 +50,14 @@ def run_llava_image_features():
50
"\nUSER: What is the content of this image?\nASSISTANT:")
51
52
# This should be provided by another online or offline component.
49
- images = torch.load("images/stop_sign_image_features.pt")
50
-
51
- outputs = llm.generate(prompt,
52
- multi_modal_data=MultiModalData(
53
- type=MultiModalData.Type.IMAGE, data=images))
53
+ image = torch.load("images/stop_sign_image_features.pt")
54
+
55
+ outputs = llm.generate({
56
+ "prompt":
57
+ prompt,
58
+ "multi_modal_data":
59
+ MultiModalData(type=MultiModalData.Type.IMAGE, data=image),
60
+ })
61
for o in outputs:
62
generated_text = o.outputs[0].text
63
print(generated_text)
pyproject.toml
+7
@@ -65,3 +65,10 @@ skip = "./tests/prompts,./benchmarks/sonnet.txt,./tests/lora/data,./build"
65
[tool.isort]
66
use_parentheses = true
67
skip_gitignore = true
68
+
69
+[tool.pytest.ini_options]
70
+markers = [
71
+ "skip_global_cleanup",
72
+ "llm: run tests for vLLM API only",
73
+ "openai: run tests for OpenAI API only",
74
+]
tests/async_engine/test_async_llm_engine.py
+1
-1
@@ -25,7 +25,7 @@ class MockEngine:
25
return [RequestOutput(
26
request_id=self.request_id)] if self.request_id else []
27
28
- async def encode_request_async(self, *args, **kwargs):
28
+ async def process_model_inputs_async(self, *args, **kwargs):
29
pass
30
31
def generate(self, request_id):
tests/async_engine/test_openapi_server_ray.py
+1
-1
@@ -29,7 +29,7 @@ def server():
29
ray.shutdown()
30
31
32
-@pytest.fixture(scope="session")
32
+@pytest.fixture(scope="module")
33
def client():
34
client = openai.AsyncOpenAI(
35
base_url="http://localhost:8000/v1",
tests/conftest.py
+17
-6
@@ -12,6 +12,7 @@ from transformers import (AutoModelForCausalLM, AutoProcessor, AutoTokenizer,
12
from vllm import LLM, SamplingParams
13
from vllm.config import TokenizerPoolConfig, VisionLanguageConfig
14
from vllm.distributed import destroy_model_parallel
15
+from vllm.inputs import PromptInputs
16
from vllm.logger import init_logger
17
from vllm.sequence import MultiModalData
18
@@ -402,12 +403,22 @@ class VllmRunner:
403
) -> List[Tuple[List[int], str]]:
404
if images is not None:
405
assert len(prompts) == images.shape[0]
405
- req_outputs = self.model.generate(
406
- prompts,
407
- sampling_params=sampling_params,
408
- multi_modal_data=MultiModalData(type=MultiModalData.Type.IMAGE,
409
- data=images)
410
- if images is not None else None)
406
+
407
+ prompt_inputs: List[PromptInputs] = []
408
+ for i, prompt in enumerate(prompts):
409
+ image = None if images is None else images[i:i + 1]
410
+ mm_data = None if image is None else MultiModalData(
411
+ type=MultiModalData.Type.IMAGE,
412
+ data=image,
413
+ )
414
+
415
+ prompt_inputs.append({
416
+ "prompt": prompt,
417
+ "multi_modal_data": mm_data,
418
+ })
419
+
420
+ req_outputs = self.model.generate(prompt_inputs,
421
+ sampling_params=sampling_params)
422
outputs = []
423
for req_output in req_outputs:
424
prompt_str = req_output.prompt
tests/core/test_block_manager.py
+12
-3
@@ -133,8 +133,11 @@ def test_append_slot_cow():
133
134
# Allocate prompt to gpu block. There is one slot left in the block.
135
prompt = Sequence(seq_id=1,
136
- prompt="one two three",
137
- prompt_token_ids=[1, 2, 3],
136
+ inputs={
137
+ "prompt": "one two three",
138
+ "prompt_token_ids": [1, 2, 3],
139
+ "multi_modal_data": None
140
+ },
141
block_size=block_size)
142
143
# Fork the sequence, such that a COW will be required when we append a new
@@ -304,7 +307,13 @@ def test_sliding_window_multi_seq():
307
308
assert block_manager.get_num_free_gpu_blocks() == num_gpu_blocks
309
307
- parent = Sequence(1, "one two three", [0, 1, 2], block_size)
310
+ parent = Sequence(seq_id=1,
311
+ inputs={
312
+ "prompt": "one two three",
313
+ "prompt_token_ids": [0, 1, 2],
314
+ "multi_modal_data": None
315
+ },
316
+ block_size=block_size)
317
seq_group = SequenceGroup(request_id="1",
318
seqs=[parent],
319
arrival_time=time.time(),
tests/core/utils.py
+12
-3
@@ -21,7 +21,13 @@ def create_dummy_prompt(
21
# and prompt "0 ... block_size".
22
prompt_tokens = list(range(prompt_length))
23
prompt_str = " ".join([str(t) for t in prompt_tokens])
24
- prompt = Sequence(int(request_id), prompt_str, prompt_tokens, block_size)
24
+ prompt = Sequence(int(request_id),
25
+ inputs={
26
+ "prompt": prompt_str,
27
+ "prompt_token_ids": prompt_tokens,
28
+ "multi_modal_data": None,
29
+ },
30
+ block_size=block_size)
31
seq_group = SequenceGroup(request_id=request_id,
32
seqs=[prompt],
33
arrival_time=time.time(),
@@ -51,8 +57,11 @@ def create_seq_group(
57
for seq_id_offset, output_len in enumerate(seq_output_lens):
58
seq = Sequence(
59
seq_id=seq_id_start + seq_id_offset,
54
- prompt="",
55
- prompt_token_ids=prompt_token_ids,
60
+ inputs={
61
+ "prompt": "",
62
+ "prompt_token_ids": prompt_token_ids,
63
+ "multi_modal_data": None,
64
+ },
65
block_size=16,
66
)
67
tests/engine/test_skip_tokenizer_init.py
+1
-1
@@ -14,7 +14,7 @@ def test_skip_tokenizer_initialization(model: str):
14
with pytest.raises(ValueError) as err:
15
llm.generate("abc", sampling_params)
16
assert "prompts must be None if" in str(err.value)
17
- outputs = llm.generate(prompt_token_ids=[[1, 2, 3]],
17
+ outputs = llm.generate({"prompt_token_ids": [1, 2, 3]},
18
sampling_params=sampling_params)
19
assert len(outputs) > 0
20
completions = outputs[0].outputs
tests/entrypoints/openai/test_serving_chat.py
+4
@@ -1,11 +1,15 @@
1
import asyncio
2
from dataclasses import dataclass
3
4
+import pytest
5
+
6
from vllm.entrypoints.openai.serving_chat import OpenAIServingChat
7
8
MODEL_NAME = "openai-community/gpt2"
9
CHAT_TEMPLATE = "Dummy chat template for testing {}"
10
11
+pytestmark = pytest.mark.openai
12
+
13
14
@dataclass
15
class MockModelConfig:
tests/entrypoints/test_guided_processors.py
+2
@@ -52,6 +52,8 @@ TEST_SCHEMA = {
52
TEST_REGEX = (r"((25[0-5]|(2[0-4]|1\d|[1-9]|)\d)\.){3}"
53
r"(25[0-5]|(2[0-4]|1\d|[1-9]|)\d)")
54
55
+pytestmark = pytest.mark.openai
56
+
57
58
def test_guided_logits_processors():
59
"""Basic unit test for RegexLogitsProcessor and JSONLogitsProcessor."""
tests/entrypoints/test_llm_encode.py
new
+144
@@ -0,0 +1,144 @@
1
+import weakref
2
+from typing import List
3
+
4
+import pytest
5
+
6
+from vllm import LLM, EmbeddingRequestOutput, PoolingParams
7
+
8
+from ..conftest import cleanup
9
+
10
+MODEL_NAME = "intfloat/e5-mistral-7b-instruct"
11
+
12
+PROMPTS = [
13
+ "Hello, my name is",
14
+ "The president of the United States is",
15
+ "The capital of France is",
16
+ "The future of AI is",
17
+]
18
+
19
+TOKEN_IDS = [
20
+ # Using ID={0, 1, 2, 3} results in NaN values,
21
+ # so we add this offset of 1000
22
+ [1000],
23
+ [1000, 1001],
24
+ [1000, 1002, 1001],
25
+ [1000, 1003, 1001, 1002],
26
+]
27
+
28
+pytestmark = pytest.mark.llm
29
+
30
+
31
+@pytest.fixture(scope="module")
32
+def llm():
33
+ # pytest caches the fixture so we use weakref.proxy to
34
+ # enable garbage collection
35
+ llm = LLM(model=MODEL_NAME,
36
+ max_num_batched_tokens=32768,
37
+ tensor_parallel_size=1,
38
+ gpu_memory_utilization=0.75,
39
+ enforce_eager=True)
40
+
41
+ with llm.deprecate_legacy_api():
42
+ yield weakref.proxy(llm)
43
+
44
+ del llm
45
+
46
+ cleanup()
47
+
48
+
49
+def assert_outputs_equal(o1: List[EmbeddingRequestOutput],
50
+ o2: List[EmbeddingRequestOutput]):
51
+ assert [o.outputs for o in o1] == [o.outputs for o in o2]
52
+
53
+
54
+@pytest.mark.skip_global_cleanup
55
+@pytest.mark.parametrize('prompt', PROMPTS)
56
+def test_v1_v2_api_consistency_single_prompt_string(llm: LLM, prompt):
57
+ pooling_params = PoolingParams()
58
+
59
+ with pytest.warns(DeprecationWarning, match="'prompts'"):
60
+ v1_output = llm.encode(prompts=prompt, pooling_params=pooling_params)
61
+
62
+ v2_output = llm.encode(prompt, pooling_params=pooling_params)
63
+ assert_outputs_equal(v1_output, v2_output)
64
+
65
+ v2_output = llm.encode({"prompt": prompt}, pooling_params=pooling_params)
66
+ assert_outputs_equal(v1_output, v2_output)
67
+
68
+
69
+@pytest.mark.skip_global_cleanup
70
+@pytest.mark.parametrize('prompt_token_ids', TOKEN_IDS)
71
+def test_v1_v2_api_consistency_single_prompt_tokens(llm: LLM,
72
+ prompt_token_ids):
73
+ pooling_params = PoolingParams()
74
+
75
+ with pytest.warns(DeprecationWarning, match="'prompt_token_ids'"):
76
+ v1_output = llm.encode(prompt_token_ids=prompt_token_ids,
77
+ pooling_params=pooling_params)
78
+
79
+ v2_output = llm.encode({"prompt_token_ids": prompt_token_ids},
80
+ pooling_params=pooling_params)
81
+ assert_outputs_equal(v1_output, v2_output)
82
+
83
+
84
+@pytest.mark.skip_global_cleanup
85
+def test_v1_v2_api_consistency_multi_prompt_string(llm: LLM):
86
+ pooling_params = PoolingParams()
87
+
88
+ with pytest.warns(DeprecationWarning, match="'prompts'"):
89
+ v1_output = llm.encode(prompts=PROMPTS, pooling_params=pooling_params)
90
+
91
+ v2_output = llm.encode(PROMPTS, pooling_params=pooling_params)
92
+ assert_outputs_equal(v1_output, v2_output)
93
+
94
+ v2_output = llm.encode(
95
+ [{
96
+ "prompt": p
97
+ } for p in PROMPTS],
98
+ pooling_params=pooling_params,
99
+ )
100
+ assert_outputs_equal(v1_output, v2_output)
101
+
102
+
103
+@pytest.mark.skip_global_cleanup
104
+def test_v1_v2_api_consistency_multi_prompt_tokens(llm: LLM):
105
+ pooling_params = PoolingParams()
106
+
107
+ with pytest.warns(DeprecationWarning, match="'prompt_token_ids'"):
108
+ v1_output = llm.encode(prompt_token_ids=TOKEN_IDS,
109
+ pooling_params=pooling_params)
110
+
111
+ v2_output = llm.encode(
112
+ [{
113
+ "prompt_token_ids": p
114
+ } for p in TOKEN_IDS],
115
+ pooling_params=pooling_params,
116
+ )
117
+ assert_outputs_equal(v1_output, v2_output)
118
+
119
+
120
+@pytest.mark.skip_global_cleanup
121
+def test_multiple_pooling_params(llm: LLM):
122
+ pooling_params = [
123
+ PoolingParams(),
124
+ PoolingParams(),
125
+ PoolingParams(),
126
+ PoolingParams(),
127
+ ]
128
+
129
+ # Multiple PoolingParams should be matched with each prompt
130
+ outputs = llm.encode(PROMPTS, pooling_params=pooling_params)
131
+ assert len(PROMPTS) == len(outputs)
132
+
133
+ # Exception raised, if the size of params does not match the size of prompts
134
+ with pytest.raises(ValueError):
135
+ outputs = llm.encode(PROMPTS, pooling_params=pooling_params[:3])
136
+
137
+ # Single PoolingParams should be applied to every prompt
138
+ single_pooling_params = PoolingParams()
139
+ outputs = llm.encode(PROMPTS, pooling_params=single_pooling_params)
140
+ assert len(PROMPTS) == len(outputs)
141
+
142
+ # pooling_params is None, default params should be applied
143
+ outputs = llm.encode(PROMPTS, pooling_params=None)
144
+ assert len(PROMPTS) == len(outputs)
tests/entrypoints/test_llm_generate.py
+120
-17
@@ -1,21 +1,124 @@
1
+import weakref
2
+from typing import List
3
+
4
import pytest
5
3
-from vllm import LLM, SamplingParams
6
+from vllm import LLM, RequestOutput, SamplingParams
7
+
8
+from ..conftest import cleanup
9
+
10
+MODEL_NAME = "facebook/opt-125m"
11
+
12
+PROMPTS = [
13
+ "Hello, my name is",
14
+ "The president of the United States is",
15
+ "The capital of France is",
16
+ "The future of AI is",
17
+]
18
19
+TOKEN_IDS = [
20
+ [0],
21
+ [0, 1],
22
+ [0, 2, 1],
23
+ [0, 3, 1, 2],
24
+]
25
6
-def test_multiple_sampling_params():
26
+pytestmark = pytest.mark.llm
27
8
- llm = LLM(model="facebook/opt-125m",
28
+
29
+@pytest.fixture(scope="module")
30
+def llm():
31
+ # pytest caches the fixture so we use weakref.proxy to
32
+ # enable garbage collection
33
+ llm = LLM(model=MODEL_NAME,
34
max_num_batched_tokens=4096,
10
- tensor_parallel_size=1)
35
+ tensor_parallel_size=1,
36
+ gpu_memory_utilization=0.10,
37
+ enforce_eager=True)
38
+
39
+ with llm.deprecate_legacy_api():
40
+ yield weakref.proxy(llm)
41
+
42
+ del llm
43
+
44
+ cleanup()
45
+
46
+
47
+def assert_outputs_equal(o1: List[RequestOutput], o2: List[RequestOutput]):
48
+ assert [o.outputs for o in o1] == [o.outputs for o in o2]
49
+
50
+
51
+@pytest.mark.skip_global_cleanup
52
+@pytest.mark.parametrize('prompt', PROMPTS)
53
+def test_v1_v2_api_consistency_single_prompt_string(llm: LLM, prompt):
54
+ sampling_params = SamplingParams(temperature=0.0, top_p=1.0)
55
+
56
+ with pytest.warns(DeprecationWarning, match="'prompts'"):
57
+ v1_output = llm.generate(prompts=prompt,
58
+ sampling_params=sampling_params)
59
+
60
+ v2_output = llm.generate(prompt, sampling_params=sampling_params)
61
+ assert_outputs_equal(v1_output, v2_output)
62
+
63
+ v2_output = llm.generate({"prompt": prompt},
64
+ sampling_params=sampling_params)
65
+ assert_outputs_equal(v1_output, v2_output)
66
+
67
+
68
+@pytest.mark.skip_global_cleanup
69
+@pytest.mark.parametrize('prompt_token_ids', TOKEN_IDS)
70
+def test_v1_v2_api_consistency_single_prompt_tokens(llm: LLM,
71
+ prompt_token_ids):
72
+ sampling_params = SamplingParams(temperature=0.0, top_p=1.0)
73
+
74
+ with pytest.warns(DeprecationWarning, match="'prompt_token_ids'"):
75
+ v1_output = llm.generate(prompt_token_ids=prompt_token_ids,
76
+ sampling_params=sampling_params)
77
+
78
+ v2_output = llm.generate({"prompt_token_ids": prompt_token_ids},
79
+ sampling_params=sampling_params)
80
+ assert_outputs_equal(v1_output, v2_output)
81
+
82
+
83
+@pytest.mark.skip_global_cleanup
84
+def test_v1_v2_api_consistency_multi_prompt_string(llm: LLM):
85
+ sampling_params = SamplingParams(temperature=0.0, top_p=1.0)
86
+
87
+ with pytest.warns(DeprecationWarning, match="'prompts'"):
88
+ v1_output = llm.generate(prompts=PROMPTS,
89
+ sampling_params=sampling_params)
90
+
91
+ v2_output = llm.generate(PROMPTS, sampling_params=sampling_params)
92
+ assert_outputs_equal(v1_output, v2_output)
93
+
94
+ v2_output = llm.generate(
95
+ [{
96
+ "prompt": p
97
+ } for p in PROMPTS],
98
+ sampling_params=sampling_params,
99
+ )
100
+ assert_outputs_equal(v1_output, v2_output)
101
+
102
+
103
+@pytest.mark.skip_global_cleanup
104
+def test_v1_v2_api_consistency_multi_prompt_tokens(llm: LLM):
105
+ sampling_params = SamplingParams(temperature=0.0, top_p=1.0)
106
+
107
+ with pytest.warns(DeprecationWarning, match="'prompt_token_ids'"):
108
+ v1_output = llm.generate(prompt_token_ids=TOKEN_IDS,
109
+ sampling_params=sampling_params)
110
+
111
+ v2_output = llm.generate(
112
+ [{
113
+ "prompt_token_ids": p
114
+ } for p in TOKEN_IDS],
115
+ sampling_params=sampling_params,
116
+ )
117
+ assert_outputs_equal(v1_output, v2_output)
118
12
- prompts = [
13
- "Hello, my name is",
14
- "The president of the United States is",
15
- "The capital of France is",
16
- "The future of AI is",
17
- ]
119
120
+@pytest.mark.skip_global_cleanup
121
+def test_multiple_sampling_params(llm: LLM):
122
sampling_params = [
123
SamplingParams(temperature=0.01, top_p=0.95),
124
SamplingParams(temperature=0.3, top_p=0.95),
@@ -24,18 +127,18 @@ def test_multiple_sampling_params():
127
]
128
129
# Multiple SamplingParams should be matched with each prompt
27
- outputs = llm.generate(prompts, sampling_params=sampling_params)
28
- assert len(prompts) == len(outputs)
130
+ outputs = llm.generate(PROMPTS, sampling_params=sampling_params)
131
+ assert len(PROMPTS) == len(outputs)
132
133
# Exception raised, if the size of params does not match the size of prompts
134
with pytest.raises(ValueError):
32
- outputs = llm.generate(prompts, sampling_params=sampling_params[:3])
135
+ outputs = llm.generate(PROMPTS, sampling_params=sampling_params[:3])
136
137
# Single SamplingParams should be applied to every prompt
138
single_sampling_params = SamplingParams(temperature=0.3, top_p=0.95)
36
- outputs = llm.generate(prompts, sampling_params=single_sampling_params)
37
- assert len(prompts) == len(outputs)
139
+ outputs = llm.generate(PROMPTS, sampling_params=single_sampling_params)
140
+ assert len(PROMPTS) == len(outputs)
141
142
# sampling_params is None, default params should be applied
40
- outputs = llm.generate(prompts, sampling_params=None)
41
- assert len(prompts) == len(outputs)
\ No newline at end of file
143
+ outputs = llm.generate(PROMPTS, sampling_params=None)
144
+ assert len(PROMPTS) == len(outputs)
tests/entrypoints/test_openai_server.py
+32
-2
@@ -71,7 +71,7 @@ TEST_CHOICE = [
71
"Swift", "Kotlin"
72
]
73
74
-pytestmark = pytest.mark.asyncio
74
+pytestmark = pytest.mark.openai
75
76
77
@pytest.fixture(scope="session")
@@ -91,6 +91,8 @@ def server(zephyr_lora_files):
91
"--max-model-len",
92
"8192",
93
"--enforce-eager",
94
+ "--gpu-memory-utilization",
95
+ "0.75",
96
# lora config below
97
"--enable-lora",
98
"--lora-modules",
@@ -118,9 +120,11 @@ def embedding_server(zephyr_lora_files):
120
# use half precision for speed and memory savings in CI environment
121
"--dtype",
122
"bfloat16",
123
+ "--enforce-eager",
124
+ "--gpu-memory-utilization",
125
+ "0.75",
126
"--max-model-len",
127
"8192",
123
- "--enforce-eager",
128
])
129
ray.get(server_runner.ready.remote())
130
yield server_runner
@@ -136,6 +140,7 @@ def client():
140
yield client
141
142
143
+@pytest.mark.asyncio
144
async def test_check_models(server, client: openai.AsyncOpenAI):
145
models = await client.models.list()
146
models = models.data
@@ -147,6 +152,7 @@ async def test_check_models(server, client: openai.AsyncOpenAI):
152
assert lora_models[1].id == "zephyr-lora2"
153
154
155
+@pytest.mark.asyncio
156
@pytest.mark.parametrize(
157
# first test base model, then test loras
158
"model_name",
@@ -178,6 +184,7 @@ async def test_single_completion(server, client: openai.AsyncOpenAI,
184
completion.choices[0].text) >= 5
185
186
187
+@pytest.mark.asyncio
188
@pytest.mark.parametrize(
189
# first test base model, then test loras
190
"model_name",
@@ -199,6 +206,7 @@ async def test_zero_logprobs(server, client: openai.AsyncOpenAI,
206
assert choice.logprobs.top_logprobs is None
207
208
209
+@pytest.mark.asyncio
210
@pytest.mark.parametrize(
211
# just test 1 lora hereafter
212
"model_name",
@@ -243,6 +251,7 @@ async def test_single_chat_session(server, client: openai.AsyncOpenAI,
251
assert message.content is not None and len(message.content) >= 0
252
253
254
+@pytest.mark.asyncio
255
@pytest.mark.parametrize("model_name", [MODEL_NAME])
256
async def test_too_many_logprobs(server, client: openai.AsyncOpenAI,
257
model_name: str):
@@ -298,6 +307,7 @@ async def test_too_many_logprobs(server, client: openai.AsyncOpenAI,
307
assert message.content is not None and len(message.content) >= 0
308
309
310
+@pytest.mark.asyncio
311
@pytest.mark.parametrize(
312
# just test 1 lora hereafter
313
"model_name",
@@ -335,6 +345,7 @@ async def test_completion_streaming(server, client: openai.AsyncOpenAI,
345
assert "".join(chunks) == single_output
346
347
348
+@pytest.mark.asyncio
349
@pytest.mark.parametrize(
350
# just test 1 lora hereafter
351
"model_name",
@@ -385,6 +396,7 @@ async def test_chat_streaming(server, client: openai.AsyncOpenAI,
396
assert "".join(chunks) == output
397
398
399
+@pytest.mark.asyncio
400
@pytest.mark.parametrize(
401
# just test 1 lora hereafter
402
"model_name",
@@ -438,6 +450,7 @@ async def test_batch_completions(server, client: openai.AsyncOpenAI,
450
assert texts[0] == texts[1]
451
452
453
+@pytest.mark.asyncio
454
async def test_logits_bias(server, client: openai.AsyncOpenAI):
455
prompt = "Hello, my name is"
456
max_tokens = 5
@@ -485,6 +498,7 @@ async def test_logits_bias(server, client: openai.AsyncOpenAI):
498
assert first_response != completion.choices[0].text
499
500
501
+@pytest.mark.asyncio
502
@pytest.mark.parametrize("guided_decoding_backend",
503
["outlines", "lm-format-enforcer"])
504
async def test_guided_json_completion(server, client: openai.AsyncOpenAI,
@@ -507,6 +521,7 @@ async def test_guided_json_completion(server, client: openai.AsyncOpenAI,
521
jsonschema.validate(instance=output_json, schema=TEST_SCHEMA)
522
523
524
+@pytest.mark.asyncio
525
@pytest.mark.parametrize("guided_decoding_backend",
526
["outlines", "lm-format-enforcer"])
527
async def test_guided_json_chat(server, client: openai.AsyncOpenAI,
@@ -553,6 +568,7 @@ async def test_guided_json_chat(server, client: openai.AsyncOpenAI,
568
assert json1["age"] != json2["age"]
569
570
571
+@pytest.mark.asyncio
572
@pytest.mark.parametrize("guided_decoding_backend",
573
["outlines", "lm-format-enforcer"])
574
async def test_guided_regex_completion(server, client: openai.AsyncOpenAI,
@@ -573,6 +589,7 @@ async def test_guided_regex_completion(server, client: openai.AsyncOpenAI,
589
assert re.fullmatch(TEST_REGEX, completion.choices[i].text) is not None
590
591
592
+@pytest.mark.asyncio
593
@pytest.mark.parametrize("guided_decoding_backend",
594
["outlines", "lm-format-enforcer"])
595
async def test_guided_regex_chat(server, client: openai.AsyncOpenAI,
@@ -610,6 +627,7 @@ async def test_guided_regex_chat(server, client: openai.AsyncOpenAI,
627
assert ip1 != ip2
628
629
630
+@pytest.mark.asyncio
631
@pytest.mark.parametrize("guided_decoding_backend",
632
["outlines", "lm-format-enforcer"])
633
async def test_guided_choice_completion(server, client: openai.AsyncOpenAI,
@@ -629,6 +647,7 @@ async def test_guided_choice_completion(server, client: openai.AsyncOpenAI,
647
assert completion.choices[i].text in TEST_CHOICE
648
649
650
+@pytest.mark.asyncio
651
@pytest.mark.parametrize("guided_decoding_backend",
652
["outlines", "lm-format-enforcer"])
653
async def test_guided_choice_chat(server, client: openai.AsyncOpenAI,
@@ -667,6 +686,7 @@ async def test_guided_choice_chat(server, client: openai.AsyncOpenAI,
686
assert choice1 != choice2
687
688
689
+@pytest.mark.asyncio
690
@pytest.mark.parametrize("guided_decoding_backend",
691
["outlines", "lm-format-enforcer"])
692
async def test_guided_decoding_type_error(server, client: openai.AsyncOpenAI,
@@ -702,6 +722,7 @@ async def test_guided_decoding_type_error(server, client: openai.AsyncOpenAI,
722
extra_body=dict(guided_regex=TEST_REGEX, guided_json=TEST_SCHEMA))
723
724
725
+@pytest.mark.asyncio
726
@pytest.mark.parametrize("guided_decoding_backend",
727
["outlines", "lm-format-enforcer"])
728
async def test_guided_choice_chat_logprobs(server, client: openai.AsyncOpenAI,
@@ -732,6 +753,7 @@ async def test_guided_choice_chat_logprobs(server, client: openai.AsyncOpenAI,
753
for token, logprob in token_dict.items())
754
755
756
+@pytest.mark.asyncio
757
async def test_response_format_json_object(server, client: openai.AsyncOpenAI):
758
for _ in range(2):
759
resp = await client.chat.completions.create(
@@ -749,6 +771,7 @@ async def test_response_format_json_object(server, client: openai.AsyncOpenAI):
771
assert loaded == {"result": 2}, loaded
772
773
774
+@pytest.mark.asyncio
775
async def test_extra_fields(server, client: openai.AsyncOpenAI):
776
with pytest.raises(BadRequestError) as exc_info:
777
await client.chat.completions.create(
@@ -764,6 +787,7 @@ async def test_extra_fields(server, client: openai.AsyncOpenAI):
787
assert "extra_forbidden" in exc_info.value.message
788
789
790
+@pytest.mark.asyncio
791
async def test_complex_message_content(server, client: openai.AsyncOpenAI):
792
resp = await client.chat.completions.create(
793
model=MODEL_NAME,
@@ -783,6 +807,7 @@ async def test_complex_message_content(server, client: openai.AsyncOpenAI):
807
assert content == "2"
808
809
810
+@pytest.mark.asyncio
811
async def test_custom_role(server, client: openai.AsyncOpenAI):
812
# Not sure how the model handles custom roles so we just check that
813
# both string and complex message content are handled in the same way
@@ -813,6 +838,7 @@ async def test_custom_role(server, client: openai.AsyncOpenAI):
838
assert content1 == content2
839
840
841
+@pytest.mark.asyncio
842
async def test_guided_grammar(server, client: openai.AsyncOpenAI):
843
simple_sql_grammar = """
844
start: select_statement
@@ -847,6 +873,7 @@ number: "1" | "2"
873
assert content.strip() == ground_truth
874
875
876
+@pytest.mark.asyncio
877
@pytest.mark.parametrize(
878
# first test base model, then test loras
879
"model_name",
@@ -878,6 +905,7 @@ async def test_echo_logprob_completion(server, client: openai.AsyncOpenAI,
905
assert len(logprobs.tokens) > 5
906
907
908
+@pytest.mark.asyncio
909
async def test_long_seed(server, client: openai.AsyncOpenAI):
910
for seed in [
911
torch.iinfo(torch.long).min - 1,
@@ -897,6 +925,7 @@ async def test_long_seed(server, client: openai.AsyncOpenAI):
925
or "less_than_equal" in exc_info.value.message)
926
927
928
+@pytest.mark.asyncio
929
@pytest.mark.parametrize(
930
"model_name",
931
[EMBEDDING_MODEL_NAME],
@@ -935,6 +964,7 @@ async def test_single_embedding(embedding_server, client: openai.AsyncOpenAI,
964
assert embeddings.usage.total_tokens == 5
965
966
967
+@pytest.mark.asyncio
968
@pytest.mark.parametrize(
969
"model_name",
970
[EMBEDDING_MODEL_NAME],
tests/entrypoints/test_server_oot_registration.py
+7
-4
@@ -1,7 +1,7 @@
1
-import multiprocessing
1
import sys
2
import time
3
4
+import pytest
5
import torch
6
from openai import OpenAI, OpenAIError
7
@@ -10,6 +10,8 @@ from vllm.model_executor.models.opt import OPTForCausalLM
10
from vllm.model_executor.sampling_metadata import SamplingMetadata
11
from vllm.utils import get_open_port
12
13
+pytestmark = pytest.mark.openai
14
+
15
16
class MyOPTForCausalLM(OPTForCausalLM):
17
@@ -26,15 +28,16 @@ def server_function(port):
28
# register our dummy model
29
ModelRegistry.register_model("OPTForCausalLM", MyOPTForCausalLM)
30
sys.argv = ["placeholder.py"] + \
29
- ("--model facebook/opt-125m --dtype"
30
- f" float32 --api-key token-abc123 --port {port}").split()
31
+ ("--model facebook/opt-125m --gpu-memory-utilization 0.10 "
32
+ f"--dtype float32 --api-key token-abc123 --port {port}").split()
33
import runpy
34
runpy.run_module('vllm.entrypoints.openai.api_server', run_name='__main__')
35
36
37
def test_oot_registration_for_api_server():
38
port = get_open_port()
37
- server = multiprocessing.Process(target=server_function, args=(port, ))
39
+ ctx = torch.multiprocessing.get_context()
40
+ server = ctx.Process(target=server_function, args=(port, ))
41
server.start()
42
client = OpenAI(
43
base_url=f"http://localhost:{port}/v1",
tests/lora/test_long_context.py
+3
-5
@@ -86,20 +86,18 @@ def generate(
86
87
88
def batched_generate(
89
- llm,
89
+ llm: vllm.LLM,
90
inputs: List[Tuple[str, SamplingParams, Optional[LoRARequest]]],
91
):
92
for input in inputs:
93
prompt, sampling_param, lora_req = input
94
- requests_data = llm._validate_and_prepare_requests(
94
+ # Add requests to the engine and run the engine
95
+ llm._validate_and_add_requests(
96
prompt,
97
sampling_param,
98
lora_request=lora_req,
99
)
100
100
- # Add requests to the engine and run the engine
101
- for request_data in requests_data:
102
- llm._add_request(**request_data)
101
outputs = llm._run_engine(use_tqdm=True)
102
return [outputs[i].outputs[0].text.strip() for i in range(len(outputs))]
103
tests/samplers/test_logits_processor.py
+4
-7
@@ -35,28 +35,25 @@ def test_logits_processor_force_generate(
35
36
# test logits_processors when prompt_logprobs is not None
37
vllm_model.model._add_request(
38
- prompt=example_prompts[0],
38
+ example_prompts[0],
39
params=params_with_logprobs,
40
- prompt_token_ids=None,
40
)
41
42
# test prompt_logprobs is not None
43
vllm_model.model._add_request(
45
- prompt=example_prompts[1],
44
+ example_prompts[1],
45
params=SamplingParams(
46
prompt_logprobs=3,
47
max_tokens=max_tokens,
48
),
50
- prompt_token_ids=None,
49
)
50
51
# test grouped requests
52
vllm_model.model._add_request(
55
- prompt=example_prompts[2],
53
+ example_prompts[2],
54
params=SamplingParams(max_tokens=max_tokens),
57
- prompt_token_ids=None,
55
)
56
60
- outputs = vllm_model.model._run_engine(False)
57
+ outputs = vllm_model.model._run_engine(use_tqdm=False)
58
59
assert outputs[0].outputs[0].text == enforced_answers * repeat_times
tests/samplers/test_seeded_generate.py
+1
-5
@@ -57,11 +57,7 @@ def test_random_sample_with_seed(
57
sampling_params_seed_1,
58
sampling_params_seed_2,
59
):
60
- llm._add_request(
61
- prompt=prompt,
62
- prompt_token_ids=None,
63
- params=params,
64
- )
60
+ llm._add_request(prompt, params=params)
61
62
results = llm._run_engine(use_tqdm=False)
63
all_outputs = [[out.token_ids for out in output.outputs]
tests/test_cache_block_hashing.py
+9
-2
@@ -70,8 +70,15 @@ def test_auto_prefix_caching(model: str, block_size: int, max_num_seqs: int,
70
for prompt in prompts:
71
hashes[-1].append([])
72
prompt_token_ids = tokenizer.encode(prompt)
73
- seq = Sequence(seq_id, prompt, prompt_token_ids, block_size,
74
- tokenizer.tokenizer.eos_token_id, lora_request)
73
+ seq = Sequence(seq_id,
74
+ inputs={
75
+ "prompt": prompt,
76
+ "prompt_token_ids": prompt_token_ids,
77
+ "multi_modal_data": None,
78
+ },
79
+ block_size=block_size,
80
+ eos_token_id=tokenizer.tokenizer.eos_token_id,
81
+ lora_request=lora_request)
82
83
num_blocks = len(prompt_token_ids) // block_size
84
for idx in range(num_blocks):
tests/test_inputs.py
new
+53
@@ -0,0 +1,53 @@
1
+from typing import List
2
+
3
+import pytest
4
+
5
+from vllm.inputs import parse_and_batch_prompt
6
+
7
+STRING_INPUTS = [
8
+ '',
9
+ 'foo',
10
+ 'foo bar',
11
+ 'foo baz bar',
12
+ 'foo bar qux baz',
13
+]
14
+
15
+TOKEN_INPUTS = [
16
+ [-1],
17
+ [1],
18
+ [1, 2],
19
+ [1, 3, 4],
20
+ [1, 2, 4, 3],
21
+]
22
+
23
+INPUTS_SLICES = [
24
+ slice(None, None, -1),
25
+ slice(None, None, 2),
26
+ slice(None, None, -2),
27
+]
28
+
29
+
30
+def test_parse_single_batch_empty():
31
+ with pytest.raises(ValueError, match="at least one prompt"):
32
+ parse_and_batch_prompt([])
33
+
34
+ with pytest.raises(ValueError, match="at least one prompt"):
35
+ parse_and_batch_prompt([[]])
36
+
37
+
38
+@pytest.mark.parametrize('string_input', STRING_INPUTS)
39
+def test_parse_single_batch_string_consistent(string_input: str):
40
+ assert parse_and_batch_prompt(string_input) \
41
+ == parse_and_batch_prompt([string_input])
42
+
43
+
44
+@pytest.mark.parametrize('token_input', TOKEN_INPUTS)
45
+def test_parse_single_batch_token_consistent(token_input: List[int]):
46
+ assert parse_and_batch_prompt(token_input) \
47
+ == parse_and_batch_prompt([token_input])
48
+
49
+
50
+@pytest.mark.parametrize('inputs_slice', INPUTS_SLICES)
51
+def test_parse_single_batch_string_slice(inputs_slice: slice):
52
+ assert parse_and_batch_prompt(STRING_INPUTS)[inputs_slice] \
53
+ == parse_and_batch_prompt(STRING_INPUTS[inputs_slice])
tests/test_utils.py
new
+63
@@ -0,0 +1,63 @@
1
+import pytest
2
+
3
+from vllm.utils import deprecate_kwargs
4
+
5
+from .utils import error_on_warning
6
+
7
+
8
+def test_deprecate_kwargs_always():
9
+
10
+ @deprecate_kwargs("old_arg", is_deprecated=True)
11
+ def dummy(*, old_arg: object = None, new_arg: object = None):
12
+ pass
13
+
14
+ with pytest.warns(DeprecationWarning, match="'old_arg'"):
15
+ dummy(old_arg=1)
16
+
17
+ with error_on_warning():
18
+ dummy(new_arg=1)
19
+
20
+
21
+def test_deprecate_kwargs_never():
22
+
23
+ @deprecate_kwargs("old_arg", is_deprecated=False)
24
+ def dummy(*, old_arg: object = None, new_arg: object = None):
25
+ pass
26
+
27
+ with error_on_warning():
28
+ dummy(old_arg=1)
29
+
30
+ with error_on_warning():
31
+ dummy(new_arg=1)
32
+
33
+
34
+def test_deprecate_kwargs_dynamic():
35
+ is_deprecated = True
36
+
37
+ @deprecate_kwargs("old_arg", is_deprecated=lambda: is_deprecated)
38
+ def dummy(*, old_arg: object = None, new_arg: object = None):
39
+ pass
40
+
41
+ with pytest.warns(DeprecationWarning, match="'old_arg'"):
42
+ dummy(old_arg=1)
43
+
44
+ with error_on_warning():
45
+ dummy(new_arg=1)
46
+
47
+ is_deprecated = False
48
+
49
+ with error_on_warning():
50
+ dummy(old_arg=1)
51
+
52
+ with error_on_warning():
53
+ dummy(new_arg=1)
54
+
55
+
56
+def test_deprecate_kwargs_additional_message():
57
+
58
+ @deprecate_kwargs("old_arg", is_deprecated=True, additional_message="abcd")
59
+ def dummy(*, old_arg: object = None, new_arg: object = None):
60
+ pass
61
+
62
+ with pytest.warns(DeprecationWarning, match="abcd"):
63
+ dummy(old_arg=1)
tests/tokenization/test_detokenize.py
+5
-2
@@ -123,8 +123,11 @@ def create_sequence(prompt_token_ids=None):
123
prompt_token_ids = prompt_token_ids or [1]
124
return Sequence(
125
seq_id=0,
126
- prompt="<s>",
127
- prompt_token_ids=prompt_token_ids,
126
+ inputs={
127
+ "prompt": "<s>",
128
+ "prompt_token_ids": prompt_token_ids,
129
+ "multi_modal_data": None,
130
+ },
131
block_size=16,
132
)
133
tests/utils.py
+14
@@ -2,6 +2,8 @@ import os
2
import subprocess
3
import sys
4
import time
5
+import warnings
6
+from contextlib import contextmanager
7
8
import ray
9
import requests
@@ -87,3 +89,15 @@ def multi_process_tensor_parallel(
89
ray.get(refs)
90
91
ray.shutdown()
92
+
93
+
94
+@contextmanager
95
+def error_on_warning():
96
+ """
97
+ Within the scope of this context manager, tests will fail if any warning
98
+ is emitted.
99
+ """
100
+ with warnings.catch_warnings():
101
+ warnings.simplefilter("error")
102
+
103
+ yield
vllm/__init__.py
+4
@@ -5,6 +5,7 @@ from vllm.engine.async_llm_engine import AsyncLLMEngine
5
from vllm.engine.llm_engine import LLMEngine
6
from vllm.entrypoints.llm import LLM
7
from vllm.executor.ray_utils import initialize_ray_cluster
8
+from vllm.inputs import PromptStrictInputs, TextPrompt, TokensPrompt
9
from vllm.model_executor.models import ModelRegistry
10
from vllm.outputs import (CompletionOutput, EmbeddingOutput,
11
EmbeddingRequestOutput, RequestOutput)
@@ -16,6 +17,9 @@ __version__ = "0.4.2"
17
__all__ = [
18
"LLM",
19
"ModelRegistry",
20
+ "PromptStrictInputs",
21
+ "TextPrompt",
22
+ "TokensPrompt",
23
"SamplingParams",
24
"RequestOutput",
25
"CompletionOutput",
vllm/engine/async_llm_engine.py
+81
-90
@@ -12,12 +12,13 @@ from vllm.core.scheduler import SchedulerOutputs
12
from vllm.engine.arg_utils import AsyncEngineArgs
13
from vllm.engine.llm_engine import LLMEngine
14
from vllm.executor.ray_utils import initialize_ray_cluster, ray
15
+from vllm.inputs import LLMInputs, PromptInputs
16
from vllm.logger import init_logger
17
from vllm.lora.request import LoRARequest
18
from vllm.outputs import EmbeddingRequestOutput, RequestOutput
19
from vllm.pooling_params import PoolingParams
20
from vllm.sampling_params import SamplingParams
20
-from vllm.sequence import ExecuteModelRequest, MultiModalData, SamplerOutput
21
+from vllm.sequence import ExecuteModelRequest, SamplerOutput
22
from vllm.usage.usage_lib import UsageContext
23
24
logger = init_logger(__name__)
@@ -244,64 +245,69 @@ class _AsyncLLMEngine(LLMEngine):
245
246
return request_outputs
247
247
- async def encode_request_async(
248
+ async def process_model_inputs_async(
249
self,
249
- request_id: str, # pylint: disable=unused-argument
250
- prompt: Optional[str],
251
- prompt_token_ids: Optional[List[int]] = None,
250
+ request_id: str,
251
+ inputs: PromptInputs,
252
lora_request: Optional[LoRARequest] = None,
253
- ):
254
- if prompt_token_ids is None:
255
- assert prompt is not None
256
- prompt_token_ids = await self.tokenizer.encode_async(
253
+ ) -> LLMInputs:
254
+ if isinstance(inputs, str):
255
+ inputs = {"prompt": inputs}
256
+
257
+ if "prompt_token_ids" not in inputs:
258
+ tokenizer = self.get_tokenizer_group("prompts must be None if "
259
+ "skip_tokenizer_init is True")
260
+
261
+ prompt_token_ids = await tokenizer.encode_async(
262
request_id=request_id,
258
- prompt=prompt,
263
+ prompt=inputs["prompt"],
264
lora_request=lora_request)
260
- return prompt_token_ids
265
+ else:
266
+ prompt_token_ids = inputs["prompt_token_ids"]
267
+
268
+ return LLMInputs(prompt_token_ids=prompt_token_ids,
269
+ prompt=inputs.get("prompt"),
270
+ multi_modal_data=inputs.get("multi_modal_data"))
271
272
async def add_request_async(
273
self,
274
request_id: str,
265
- prompt: Optional[str],
275
+ inputs: PromptInputs,
276
params: Union[SamplingParams, PoolingParams],
267
- prompt_token_ids: Optional[List[int]] = None,
277
arrival_time: Optional[float] = None,
278
lora_request: Optional[LoRARequest] = None,
270
- multi_modal_data: Optional[MultiModalData] = None,
279
) -> None:
280
if lora_request is not None and not self.lora_config:
281
raise ValueError(f"Got lora_request {lora_request} but LoRA is "
282
"not enabled!")
283
if arrival_time is None:
284
arrival_time = time.time()
277
- prompt_token_ids = await self.encode_request_async(
285
+
286
+ processed_inputs = await self.process_model_inputs_async(
287
+ request_id=request_id, inputs=inputs, lora_request=lora_request)
288
+
289
+ self._add_processed_request(
290
request_id=request_id,
279
- prompt=prompt,
280
- prompt_token_ids=prompt_token_ids,
281
- lora_request=lora_request)
282
-
283
- return self.add_request(request_id,
284
- prompt=prompt,
285
- params=params,
286
- prompt_token_ids=prompt_token_ids,
287
- arrival_time=arrival_time,
288
- lora_request=lora_request,
289
- multi_modal_data=multi_modal_data)
291
+ processed_inputs=processed_inputs,
292
+ params=params,
293
+ arrival_time=arrival_time,
294
+ lora_request=lora_request,
295
+ )
296
297
async def check_health_async(self) -> None:
298
self.model_executor.check_health()
299
300
301
class AsyncLLMEngine:
296
- """An asynchronous wrapper for LLMEngine.
302
+ """An asynchronous wrapper for :class:`LLMEngine`.
303
298
- This class is used to wrap the LLMEngine class to make it asynchronous. It
299
- uses asyncio to create a background loop that keeps processing incoming
300
- requests. The LLMEngine is kicked by the generate method when there
301
- are requests in the waiting queue. The generate method yields the outputs
302
- from the LLMEngine to the caller.
304
+ This class is used to wrap the :class:`LLMEngine` class to make it
305
+ asynchronous. It uses asyncio to create a background loop that keeps
306
+ processing incoming requests. The :class:`LLMEngine` is kicked by the
307
+ generate method when there are requests in the waiting queue. The generate
308
+ method yields the outputs from the :class:`LLMEngine` to the caller.
309
304
- NOTE: For the comprehensive list of arguments, see `LLMEngine`.
310
+ NOTE: For the comprehensive list of arguments, see :class:`LLMEngine`.
311
312
Args:
313
worker_use_ray: Whether to use Ray for model workers. Required for
@@ -315,8 +321,8 @@ class AsyncLLMEngine:
321
being printed in log.
322
start_engine_loop: If True, the background task to run the engine
323
will be automatically started in the generate call.
318
- *args: Arguments for LLMEngine.
319
- *kwargs: Arguments for LLMEngine.
324
+ *args: Arguments for :class:`LLMEngine`.
325
+ **kwargs: Arguments for :class:`LLMEngine`.
326
"""
327
328
_engine_class: Type[_AsyncLLMEngine] = _AsyncLLMEngine
@@ -526,22 +532,26 @@ class AsyncLLMEngine:
532
async def add_request(
533
self,
534
request_id: str,
529
- prompt: Optional[str],
535
+ inputs: PromptInputs,
536
params: Union[SamplingParams, PoolingParams],
531
- prompt_token_ids: Optional[List[int]] = None,
537
arrival_time: Optional[float] = None,
538
lora_request: Optional[LoRARequest] = None,
534
- multi_modal_data: Optional[MultiModalData] = None,
539
) -> AsyncStream:
540
if self.log_requests:
537
- shortened_prompt = prompt
538
- shortened_token_ids = prompt_token_ids
539
- if self.max_log_len is not None:
541
+ if isinstance(inputs, str):
542
+ shortened_prompt = inputs
543
+ shortened_token_ids = None
544
+ else:
545
+ shortened_prompt = inputs.get("prompt")
546
+ shortened_token_ids = inputs.get("prompt_token_ids")
547
+
548
+ max_log_len = self.max_log_len
549
+ if max_log_len is not None:
550
if shortened_prompt is not None:
541
- shortened_prompt = shortened_prompt[:self.max_log_len]
551
+ shortened_prompt = shortened_prompt[:max_log_len]
552
if shortened_token_ids is not None:
543
- shortened_token_ids = shortened_token_ids[:self.
544
- max_log_len]
553
+ shortened_token_ids = shortened_token_ids[:max_log_len]
554
+
555
logger.info(
556
"Received request %s: prompt: %r, "
557
"params: %s, prompt_token_ids: %s, "
@@ -562,39 +572,33 @@ class AsyncLLMEngine:
572
arrival_time = time.time()
573
574
if self.engine_use_ray:
565
- prompt_token_ids = await (
566
- self.engine.encode_request_async.remote( # type: ignore
575
+ processed_inputs = await self.engine.process_model_inputs_async \
576
+ .remote( # type: ignore
577
request_id=request_id,
568
- prompt=prompt,
569
- prompt_token_ids=prompt_token_ids,
570
- lora_request=lora_request))
578
+ inputs=inputs,
579
+ lora_request=lora_request)
580
else:
572
- prompt_token_ids = await self.engine.encode_request_async(
581
+ processed_inputs = await self.engine.process_model_inputs_async(
582
request_id=request_id,
574
- prompt=prompt,
575
- prompt_token_ids=prompt_token_ids,
583
+ inputs=inputs,
584
lora_request=lora_request)
585
586
stream = self._request_tracker.add_request(
587
request_id,
580
- prompt=prompt,
588
+ inputs=processed_inputs,
589
params=params,
582
- prompt_token_ids=prompt_token_ids,
590
arrival_time=arrival_time,
591
lora_request=lora_request,
585
- multi_modal_data=multi_modal_data,
592
)
593
594
return stream
595
596
async def generate(
597
self,
592
- prompt: Optional[str],
598
+ inputs: PromptInputs,
599
sampling_params: SamplingParams,
600
request_id: str,
595
- prompt_token_ids: Optional[List[int]] = None,
601
lora_request: Optional[LoRARequest] = None,
597
- multi_modal_data: Optional[MultiModalData] = None
602
) -> AsyncIterator[RequestOutput]:
603
"""Generate outputs for a request.
604
@@ -603,14 +607,12 @@ class AsyncLLMEngine:
607
from the LLMEngine to the caller.
608
609
Args:
606
- prompt: The prompt string. Can be None if prompt_token_ids is
607
- provided.
610
+ inputs: The inputs to the LLM. See
611
+ :class:`~vllm.inputs.PromptInputs`
612
+ for more details about the format of each input.
613
sampling_params: The sampling parameters of the request.
614
request_id: The unique id of the request.
610
- prompt_token_ids: The token IDs of the prompt. If None, we
611
- use the tokenizer to convert the prompts to token IDs.
615
lora_request: LoRA request to use for generation, if any.
613
- multi_modal_data: Multi modal data per request.
616
617
Yields:
618
The output `RequestOutput` objects from the LLMEngine
@@ -659,24 +661,20 @@ class AsyncLLMEngine:
661
>>> # Process and return the final output
662
>>> ...
663
"""
662
- async for output in self.process_request(
664
+ async for output in self._process_request(
665
request_id,
664
- prompt,
666
+ inputs,
667
sampling_params,
666
- prompt_token_ids,
667
- lora_request,
668
- multi_modal_data,
668
+ lora_request=lora_request,
669
):
670
- yield output
670
+ yield LLMEngine.validate_output(output, RequestOutput)
671
672
async def encode(
673
self,
674
- prompt: Optional[str],
674
+ inputs: PromptInputs,
675
pooling_params: PoolingParams,
676
request_id: str,
677
- prompt_token_ids: Optional[List[int]] = None,
677
lora_request: Optional[LoRARequest] = None,
679
- multi_modal_data: Optional[MultiModalData] = None
678
) -> AsyncIterator[EmbeddingRequestOutput]:
679
"""Generate outputs for a request from an embedding model.
680
@@ -685,14 +683,12 @@ class AsyncLLMEngine:
683
from the LLMEngine to the caller.
684
685
Args:
688
- prompt: The prompt string. Can be None if prompt_token_ids is
689
- provided.
686
+ inputs: The inputs to the LLM. See
687
+ :class:`~vllm.inputs.PromptInputs`
688
+ for more details about the format of each input.
689
pooling_params: The pooling parameters of the request.
690
request_id: The unique id of the request.
692
- prompt_token_ids: The token IDs of the prompt. If None, we
693
- use the tokenizer to convert the prompts to token IDs.
691
lora_request: LoRA request to use for generation, if any.
695
- multi_modal_data: Multi modal data per request.
692
693
Yields:
694
The output `EmbeddingRequestOutput` objects from the LLMEngine
@@ -739,24 +735,21 @@ class AsyncLLMEngine:
735
>>> # Process and return the final output
736
>>> ...
737
"""
742
- async for output in self.process_request(
738
+ async for output in self._process_request(
739
request_id,
744
- prompt,
740
+ inputs,
741
pooling_params,
746
- prompt_token_ids,
747
- lora_request,
748
- multi_modal_data,
742
+ lora_request=lora_request,
743
):
750
- yield output
744
+ yield LLMEngine.validate_output(output, EmbeddingRequestOutput)
745
752
- async def process_request(
746
+ async def _process_request(
747
self,
748
request_id: str,
755
- prompt: Optional[str],
749
+ inputs: PromptInputs,
750
params: Union[SamplingParams, PoolingParams],
757
- prompt_token_ids: Optional[List[int]] = None,
751
+ *,
752
lora_request: Optional[LoRARequest] = None,
759
- multi_modal_data: Optional[MultiModalData] = None,
753
) -> AsyncIterator[Union[RequestOutput, EmbeddingRequestOutput]]:
754
"""Common logic to process requests with SamplingParams or
755
PoolingParams."""
@@ -764,12 +757,10 @@ class AsyncLLMEngine:
757
758
stream = await self.add_request(
759
request_id,
767
- prompt,
760
+ inputs,
761
params,
769
- prompt_token_ids=prompt_token_ids,
762
arrival_time=arrival_time,
763
lora_request=lora_request,
772
- multi_modal_data=multi_modal_data,
764
)
765
766
try:
vllm/engine/llm_engine.py
+179
-90
@@ -1,5 +1,8 @@
1
import time
2
-from typing import Iterable, List, Optional, Type, Union
2
+from contextlib import contextmanager
3
+from typing import TYPE_CHECKING, ClassVar, Iterable, List, Optional
4
+from typing import Sequence as GenericSequence
5
+from typing import Type, TypeVar, Union
6
7
from transformers import GenerationConfig, PreTrainedTokenizer
8
@@ -18,6 +21,7 @@ from vllm.engine.output_processor.stop_checker import StopChecker
21
from vllm.engine.output_processor.util import create_output_by_sequence_group
22
from vllm.executor.executor_base import ExecutorBase
23
from vllm.executor.ray_utils import initialize_ray_cluster
24
+from vllm.inputs import LLMInputs, PromptInputs
25
from vllm.logger import init_logger
26
from vllm.lora.request import LoRARequest
27
from vllm.outputs import (EmbeddingRequestOutput, RequestOutput,
@@ -25,8 +29,8 @@ from vllm.outputs import (EmbeddingRequestOutput, RequestOutput,
29
from vllm.pooling_params import PoolingParams
30
from vllm.sampling_params import SamplingParams
31
from vllm.sequence import (EmbeddingSequenceGroupOutput, ExecuteModelRequest,
28
- MultiModalData, PoolerOutput, SamplerOutput,
29
- Sequence, SequenceGroup, SequenceGroupMetadata,
32
+ PoolerOutput, SamplerOutput, Sequence,
33
+ SequenceGroup, SequenceGroupMetadata,
34
SequenceStatus)
35
from vllm.transformers_utils.detokenizer import Detokenizer
36
from vllm.transformers_utils.tokenizer_group import (BaseTokenizerGroup,
@@ -50,6 +54,9 @@ def _load_generation_config_dict(model_config: ModelConfig):
54
return {}
55
56
57
+_O = TypeVar("_O", RequestOutput, EmbeddingRequestOutput)
58
+
59
+
60
class LLMEngine:
61
"""An LLM engine that receives requests and generates texts.
62
@@ -60,11 +67,11 @@ class LLMEngine:
67
iteration-level scheduling and efficient memory management to maximize the
68
serving throughput.
69
63
- The `LLM` class wraps this class for offline batched inference and the
64
- `AsyncLLMEngine` class wraps this class for online serving.
70
+ The :class:`~vllm.LLM` class wraps this class for offline batched inference
71
+ and the :class:`AsyncLLMEngine` class wraps this class for online serving.
72
66
- NOTE: The config arguments are derived from the `EngineArgs` class. For the
67
- comprehensive list of arguments, see `EngineArgs`.
73
+ NOTE: The config arguments are derived from the :class:`~vllm.EngineArgs`
74
+ class. For the comprehensive list of arguments, see :ref:`engine_args`.
75
76
Args:
77
model_config: The configuration related to the LLM model.
@@ -81,9 +88,60 @@ class LLMEngine:
88
executor_class: The model executor class for managing distributed
89
execution.
90
log_stats: Whether to log statistics.
84
- usage_context: Specified entry point, used for usage info collection
91
+ usage_context: Specified entry point, used for usage info collection.
92
"""
93
94
+ DO_VALIDATE_OUTPUT: ClassVar[bool] = False
95
+ """A flag to toggle whether to validate the type of request output."""
96
+
97
+ @classmethod
98
+ @contextmanager
99
+ def enable_output_validation(cls):
100
+ cls.DO_VALIDATE_OUTPUT = True
101
+
102
+ yield
103
+
104
+ cls.DO_VALIDATE_OUTPUT = False
105
+
106
+ @classmethod
107
+ def validate_output(
108
+ cls,
109
+ output: object,
110
+ output_type: Type[_O],
111
+ ) -> _O:
112
+ do_validate = cls.DO_VALIDATE_OUTPUT
113
+
114
+ if ((TYPE_CHECKING or do_validate)
115
+ and not isinstance(output, output_type)):
116
+ raise TypeError(f"Expected output of type {output_type}, "
117
+ f"but found type {type(output)}")
118
+
119
+ return output
120
+
121
+ @classmethod
122
+ def validate_outputs(
123
+ cls,
124
+ outputs: GenericSequence[object],
125
+ output_type: Type[_O],
126
+ ) -> List[_O]:
127
+ do_validate = cls.DO_VALIDATE_OUTPUT
128
+
129
+ outputs_: List[_O]
130
+ if TYPE_CHECKING or do_validate:
131
+ outputs_ = []
132
+ for output in outputs:
133
+ if not isinstance(output, output_type):
134
+ raise TypeError(f"Expected output of type {output_type}, "
135
+ f"but found type {type(output)}")
136
+
137
+ outputs_.append(output)
138
+ else:
139
+ outputs_ = outputs
140
+
141
+ return outputs_
142
+
143
+ tokenizer: Optional[BaseTokenizerGroup]
144
+
145
def __init__(
146
self,
147
model_config: ModelConfig,
@@ -151,12 +209,11 @@ class LLMEngine:
209
self.log_stats = log_stats
210
211
if not self.model_config.skip_tokenizer_init:
154
- self.tokenizer: BaseTokenizerGroup
155
- self._init_tokenizer()
212
+ self.tokenizer = self._init_tokenizer()
213
self.detokenizer = Detokenizer(self.tokenizer)
214
else:
158
- self.detokenizer = None
215
self.tokenizer = None
216
+ self.detokenizer = None
217
218
self.seq_counter = Counter()
219
self.generation_config_fields = _load_generation_config_dict(
@@ -318,14 +375,26 @@ class LLMEngine:
375
if model_executor := getattr(self, "model_executor", None):
376
model_executor.shutdown()
377
378
+ MISSING_TOKENIZER_GROUP_MSG = ("Unable to get tokenizer because "
379
+ "skip_tokenizer_init is True")
380
+
381
+ def get_tokenizer_group(
382
+ self,
383
+ fail_msg: str = MISSING_TOKENIZER_GROUP_MSG) -> BaseTokenizerGroup:
384
+ if self.tokenizer is None:
385
+ raise ValueError(fail_msg)
386
+
387
+ return self.tokenizer
388
+
389
def get_tokenizer(self) -> "PreTrainedTokenizer":
322
- return self.tokenizer.get_lora_tokenizer(None)
390
+ return self.get_tokenizer_group().get_lora_tokenizer(None)
391
392
def get_tokenizer_for_seq(self,
393
sequence: Sequence) -> "PreTrainedTokenizer":
326
- return self.tokenizer.get_lora_tokenizer(sequence.lora_request)
394
+ return self.get_tokenizer_group().get_lora_tokenizer(
395
+ sequence.lora_request)
396
328
- def _init_tokenizer(self, **tokenizer_init_kwargs):
397
+ def _init_tokenizer(self, **tokenizer_init_kwargs) -> BaseTokenizerGroup:
398
init_kwargs = dict(
399
tokenizer_id=self.model_config.tokenizer,
400
enable_lora=bool(self.lora_config),
@@ -335,8 +404,9 @@ class LLMEngine:
404
trust_remote_code=self.model_config.trust_remote_code,
405
revision=self.model_config.tokenizer_revision)
406
init_kwargs.update(tokenizer_init_kwargs)
338
- self.tokenizer = get_tokenizer_group(
339
- self.parallel_config.tokenizer_pool_config, **init_kwargs)
407
+
408
+ return get_tokenizer_group(self.parallel_config.tokenizer_pool_config,
409
+ **init_kwargs)
410
411
def _verify_args(self) -> None:
412
self.model_config.verify_with_parallel_config(self.parallel_config)
@@ -346,29 +416,85 @@ class LLMEngine:
416
self.lora_config.verify_with_scheduler_config(
417
self.scheduler_config)
418
349
- def encode_request(
419
+ def _get_eos_token_id(
420
+ self, lora_request: Optional[LoRARequest]) -> Optional[int]:
421
+ if self.tokenizer is None:
422
+ logger.warning("Using None for EOS token id because tokenizer "
423
+ "is not initialized")
424
+ return None
425
+
426
+ return self.tokenizer.get_lora_tokenizer(lora_request).eos_token_id
427
+
428
+ def _add_processed_request(
429
self,
351
- request_id: str, # pylint: disable=unused-argument
352
- prompt: Optional[str],
353
- prompt_token_ids: Optional[List[int]] = None,
430
+ request_id: str,
431
+ processed_inputs: LLMInputs,
432
+ params: Union[SamplingParams, PoolingParams],
433
+ arrival_time: float,
434
+ lora_request: Optional[LoRARequest],
435
+ ) -> None:
436
+ # Create the sequences.
437
+ block_size = self.cache_config.block_size
438
+ seq_id = next(self.seq_counter)
439
+ eos_token_id = self._get_eos_token_id(lora_request)
440
+
441
+ seq = Sequence(seq_id, processed_inputs, block_size, eos_token_id,
442
+ lora_request)
443
+
444
+ # Create a SequenceGroup based on SamplingParams or PoolingParams
445
+ if isinstance(params, SamplingParams):
446
+ seq_group = self._create_sequence_group_with_sampling(
447
+ request_id,
448
+ seq,
449
+ params,
450
+ arrival_time=arrival_time,
451
+ lora_request=lora_request,
452
+ )
453
+ elif isinstance(params, PoolingParams):
454
+ seq_group = self._create_sequence_group_with_pooling(
455
+ request_id,
456
+ seq,
457
+ params,
458
+ arrival_time=arrival_time,
459
+ lora_request=lora_request,
460
+ )
461
+ else:
462
+ raise ValueError(
463
+ "Either SamplingParams or PoolingParams must be provided.")
464
+
465
+ # Add the sequence group to the scheduler.
466
+ self.scheduler.add_seq_group(seq_group)
467
+
468
+ def process_model_inputs(
469
+ self,
470
+ request_id: str,
471
+ inputs: PromptInputs,
472
lora_request: Optional[LoRARequest] = None,
355
- ):
356
- if prompt_token_ids is None:
357
- assert prompt is not None
358
- prompt_token_ids = self.tokenizer.encode(request_id=request_id,
359
- prompt=prompt,
360
- lora_request=lora_request)
361
- return prompt_token_ids
473
+ ) -> LLMInputs:
474
+ if isinstance(inputs, str):
475
+ inputs = {"prompt": inputs}
476
+
477
+ if "prompt_token_ids" not in inputs:
478
+ tokenizer = self.get_tokenizer_group("prompts must be None if "
479
+ "skip_tokenizer_init is True")
480
+
481
+ prompt_token_ids = tokenizer.encode(request_id=request_id,
482
+ prompt=inputs["prompt"],
483
+ lora_request=lora_request)
484
+ else:
485
+ prompt_token_ids = inputs["prompt_token_ids"]
486
+
487
+ return LLMInputs(prompt_token_ids=prompt_token_ids,
488
+ prompt=inputs.get("prompt"),
489
+ multi_modal_data=inputs.get("multi_modal_data"))
490
491
def add_request(
492
self,
493
request_id: str,
366
- prompt: Optional[str],
494
+ inputs: PromptInputs,
495
params: Union[SamplingParams, PoolingParams],
368
- prompt_token_ids: Optional[List[int]] = None,
496
arrival_time: Optional[float] = None,
497
lora_request: Optional[LoRARequest] = None,
371
- multi_modal_data: Optional[MultiModalData] = None,
498
) -> None:
499
"""Add a request to the engine's request pool.
500
@@ -378,15 +504,14 @@ class LLMEngine:
504
505
Args:
506
request_id: The unique ID of the request.
381
- prompt: The prompt string. Can be None if prompt_token_ids is
382
- provided.
383
- params: Parameters for sampling or pooling. SamplingParams
384
- for text generation. PoolingParams for pooling.
385
- prompt_token_ids: The token IDs of the prompt. If None, we
386
- use the tokenizer to convert the prompts to token IDs.
507
+ inputs: The inputs to the LLM. See
508
+ :class:`~vllm.inputs.PromptInputs`
509
+ for more details about the format of each input.
510
+ params: Parameters for sampling or pooling.
511
+ :class:`~vllm.SamplingParams` for text generation.
512
+ :class:`~vllm.PoolingParams` for pooling.
513
arrival_time: The arrival time of the request. If None, we use
514
the current monotonic time.
389
- multi_modal_data: Multi modal data per request.
515
516
Details:
517
- Set arrival_time to the current time if it is None.
@@ -417,59 +542,26 @@ class LLMEngine:
542
"not enabled!")
543
if arrival_time is None:
544
arrival_time = time.time()
420
- prompt_token_ids = self.encode_request(
421
- request_id=request_id,
422
- prompt=prompt,
423
- prompt_token_ids=prompt_token_ids,
424
- lora_request=lora_request)
425
-
426
- # Create the sequences.
427
- block_size = self.cache_config.block_size
428
- seq_id = next(self.seq_counter)
429
- eos_token_id = None
430
- if self.tokenizer:
431
- eos_token_id = self.tokenizer.get_lora_tokenizer(
432
- lora_request).eos_token_id
433
- else:
434
- logger.warning("Use None for EOS token id because tokenizer is "
435
- "not initialized")
436
- seq = Sequence(seq_id, prompt, prompt_token_ids, block_size,
437
- eos_token_id, lora_request)
545
439
- # Create a SequenceGroup based on SamplingParams or PoolingParams
440
- if isinstance(params, SamplingParams):
441
- seq_group = self._create_sequence_group_with_sampling(
442
- request_id,
443
- seq,
444
- params,
445
- arrival_time,
446
- lora_request,
447
- multi_modal_data,
448
- )
449
- elif isinstance(params, PoolingParams):
450
- seq_group = self._create_sequence_group_with_pooling(
451
- request_id,
452
- seq,
453
- params,
454
- arrival_time,
455
- lora_request,
456
- multi_modal_data,
457
- )
458
- else:
459
- raise ValueError(
460
- "Either SamplingParams or PoolingParams must be provided.")
546
+ processed_inputs = self.process_model_inputs(request_id=request_id,
547
+ inputs=inputs,
548
+ lora_request=lora_request)
549
462
- # Add the sequence group to the scheduler.
463
- self.scheduler.add_seq_group(seq_group)
550
+ self._add_processed_request(
551
+ request_id=request_id,
552
+ processed_inputs=processed_inputs,
553
+ params=params,
554
+ arrival_time=arrival_time,
555
+ lora_request=lora_request,
556
+ )
557
558
def _create_sequence_group_with_sampling(
559
self,
560
request_id: str,
561
seq: Sequence,
562
sampling_params: SamplingParams,
470
- arrival_time: Optional[float] = None,
471
- lora_request: Optional[LoRARequest] = None,
472
- multi_modal_data: Optional[MultiModalData] = None,
563
+ arrival_time: float,
564
+ lora_request: Optional[LoRARequest],
565
) -> SequenceGroup:
566
"""Creates a SequenceGroup with SamplingParams."""
567
max_logprobs = self.get_model_config().max_logprobs
@@ -495,8 +587,7 @@ class LLMEngine:
587
seqs=[seq],
588
arrival_time=arrival_time,
589
sampling_params=sampling_params,
498
- lora_request=lora_request,
499
- multi_modal_data=multi_modal_data)
590
+ lora_request=lora_request)
591
592
return seq_group
593
@@ -505,9 +596,8 @@ class LLMEngine:
596
request_id: str,
597
seq: Sequence,
598
pooling_params: PoolingParams,
508
- arrival_time: Optional[float] = None,
509
- lora_request: Optional[LoRARequest] = None,
510
- multi_modal_data: Optional[MultiModalData] = None,
599
+ arrival_time: float,
600
+ lora_request: Optional[LoRARequest],
601
) -> SequenceGroup:
602
"""Creates a SequenceGroup with PoolingParams."""
603
# Defensive copy of PoolingParams, which are used by the pooler
@@ -517,7 +607,6 @@ class LLMEngine:
607
seqs=[seq],
608
arrival_time=arrival_time,
609
lora_request=lora_request,
520
- multi_modal_data=multi_modal_data,
610
pooling_params=pooling_params)
611
return seq_group
612
@@ -570,7 +659,7 @@ class LLMEngine:
659
660
def _process_model_outputs(
661
self,
573
- output: List[Union[SamplerOutput, PoolerOutput]],
662
+ output: GenericSequence[Union[SamplerOutput, PoolerOutput]],
663
scheduled_seq_groups: List[ScheduledSequenceGroup],
664
ignored_seq_groups: List[SequenceGroup],
665
seq_group_metadata_list: List[SequenceGroupMetadata],
@@ -585,7 +674,7 @@ class LLMEngine:
674
# Organize outputs by [sequence group][step] instead of
675
# [step][sequence group].
676
output_by_sequence_group = create_output_by_sequence_group(
588
- sampler_outputs=output, num_seq_groups=len(scheduled_seq_groups))
677
+ output, num_seq_groups=len(scheduled_seq_groups))
678
679
# Update the scheduled sequence groups with the model outputs.
680
for scheduled_seq_group, outputs, seq_group_meta in zip(
vllm/engine/output_processor/util.py
+6
-4
@@ -1,18 +1,20 @@
1
from typing import List
2
+from typing import Sequence as GenericSequence
3
+from typing import Union
4
3
-from vllm.sequence import SamplerOutput, SequenceGroupOutput
5
+from vllm.sequence import PoolerOutput, SamplerOutput, SequenceGroupOutput
6
7
8
def create_output_by_sequence_group(
7
- sampler_outputs: List[SamplerOutput],
9
+ outputs: GenericSequence[Union[SamplerOutput, PoolerOutput]],
10
num_seq_groups: int) -> List[List[SequenceGroupOutput]]:
11
"""Helper method which transforms a 2d list organized by
12
[step][sequence group] into [sequence group][step].
13
"""
12
- output_by_sequence_group: List[List[SamplerOutput]] = [
14
+ output_by_sequence_group: List[List[SequenceGroupOutput]] = [
15
[] for _ in range(num_seq_groups)
16
]
15
- for step in sampler_outputs:
17
+ for step in outputs:
18
for i, sequence_group_output in enumerate(step):
19
output_by_sequence_group[i].append(sequence_group_output)
20
vllm/entrypoints/llm.py
+302
-102
@@ -1,11 +1,14 @@
1
-from typing import List, Optional, Union
1
+from contextlib import contextmanager
2
+from typing import ClassVar, List, Optional, Sequence, Union, cast, overload
3
3
-import torch
4
from tqdm import tqdm
5
from transformers import PreTrainedTokenizer, PreTrainedTokenizerFast
6
7
from vllm.engine.arg_utils import EngineArgs
8
from vllm.engine.llm_engine import LLMEngine
9
+from vllm.inputs import (PromptInputs, PromptStrictInputs, TextPrompt,
10
+ TextTokensPrompt, TokensPrompt,
11
+ parse_and_batch_prompt)
12
from vllm.logger import init_logger
13
from vllm.lora.request import LoRARequest
14
from vllm.outputs import EmbeddingRequestOutput, RequestOutput
@@ -13,7 +16,7 @@ from vllm.pooling_params import PoolingParams
16
from vllm.sampling_params import SamplingParams
17
from vllm.sequence import MultiModalData
18
from vllm.usage.usage_lib import UsageContext
16
-from vllm.utils import Counter
19
+from vllm.utils import Counter, deprecate_kwargs
20
21
logger = init_logger(__name__)
22
@@ -28,8 +31,10 @@ class LLM:
31
mechanism and efficient memory management.
32
33
NOTE: This class is intended to be used for offline inference. For online
31
- serving, use the `AsyncLLMEngine` class instead.
32
- NOTE: For the comprehensive list of arguments, see `EngineArgs`.
34
+ serving, use the :class:`~vllm.AsyncLLMEngine` class instead.
35
+
36
+ NOTE: For the comprehensive list of arguments, see
37
+ :class:`~vllm.EngineArgs`.
38
39
Args:
40
model: The name or path of a HuggingFace Transformers model.
@@ -81,6 +86,18 @@ class LLM:
86
disable_custom_all_reduce: See ParallelConfig
87
"""
88
89
+ DEPRECATE_LEGACY: ClassVar[bool] = False
90
+ """A flag to toggle whether to deprecate the legacy generate/encode API."""
91
+
92
+ @classmethod
93
+ @contextmanager
94
+ def deprecate_legacy_api(cls):
95
+ cls.DEPRECATE_LEGACY = True
96
+
97
+ yield
98
+
99
+ cls.DEPRECATE_LEGACY = False
100
+
101
def __init__(
102
self,
103
model: str,
@@ -138,15 +155,101 @@ class LLM:
155
) -> None:
156
self.llm_engine.tokenizer.tokenizer = tokenizer
157
158
+ @overload # LEGACY: single (prompt + optional token ids)
159
+ def generate(
160
+ self,
161
+ prompts: str,
162
+ sampling_params: Optional[Union[SamplingParams,
163
+ List[SamplingParams]]] = None,
164
+ prompt_token_ids: Optional[List[int]] = None,
165
+ use_tqdm: bool = True,
166
+ lora_request: Optional[LoRARequest] = None,
167
+ multi_modal_data: Optional[MultiModalData] = None,
168
+ ) -> List[RequestOutput]:
169
+ ...
170
+
171
+ @overload # LEGACY: multi (prompt + optional token ids)
172
def generate(
173
self,
143
- prompts: Optional[Union[str, List[str]]] = None,
174
+ prompts: List[str],
175
sampling_params: Optional[Union[SamplingParams,
176
List[SamplingParams]]] = None,
177
prompt_token_ids: Optional[List[List[int]]] = None,
178
use_tqdm: bool = True,
179
lora_request: Optional[LoRARequest] = None,
180
multi_modal_data: Optional[MultiModalData] = None,
181
+ ) -> List[RequestOutput]:
182
+ ...
183
+
184
+ @overload # LEGACY: single (token ids + optional prompt)
185
+ def generate(
186
+ self,
187
+ prompts: Optional[str] = None,
188
+ sampling_params: Optional[Union[SamplingParams,
189
+ List[SamplingParams]]] = None,
190
+ *,
191
+ prompt_token_ids: List[int],
192
+ use_tqdm: bool = True,
193
+ lora_request: Optional[LoRARequest] = None,
194
+ multi_modal_data: Optional[MultiModalData] = None,
195
+ ) -> List[RequestOutput]:
196
+ ...
197
+
198
+ @overload # LEGACY: multi (token ids + optional prompt)
199
+ def generate(
200
+ self,
201
+ prompts: Optional[List[str]] = None,
202
+ sampling_params: Optional[Union[SamplingParams,
203
+ List[SamplingParams]]] = None,
204
+ *,
205
+ prompt_token_ids: List[List[int]],
206
+ use_tqdm: bool = True,
207
+ lora_request: Optional[LoRARequest] = None,
208
+ multi_modal_data: Optional[MultiModalData] = None,
209
+ ) -> List[RequestOutput]:
210
+ ...
211
+
212
+ @overload # LEGACY: single or multi token ids [pos-only]
213
+ def generate(
214
+ self,
215
+ prompts: None,
216
+ sampling_params: None,
217
+ prompt_token_ids: Union[List[int], List[List[int]]],
218
+ use_tqdm: bool = True,
219
+ lora_request: Optional[LoRARequest] = None,
220
+ multi_modal_data: Optional[MultiModalData] = None,
221
+ ) -> List[RequestOutput]:
222
+ ...
223
+
224
+ @overload
225
+ def generate(
226
+ self,
227
+ inputs: Union[PromptStrictInputs, Sequence[PromptStrictInputs]],
228
+ /, # We may enable `inputs` keyword after removing the old API
229
+ *,
230
+ sampling_params: Optional[Union[SamplingParams,
231
+ Sequence[SamplingParams]]] = None,
232
+ use_tqdm: bool = True,
233
+ lora_request: Optional[LoRARequest] = None,
234
+ ) -> List[RequestOutput]:
235
+ ...
236
+
237
+ @deprecate_kwargs("prompts",
238
+ "prompt_token_ids",
239
+ "multi_modal_data",
240
+ is_deprecated=lambda: LLM.DEPRECATE_LEGACY,
241
+ additional_message="Please use the 'inputs' parameter "
242
+ "instead.")
243
+ def generate(
244
+ self,
245
+ prompts: Union[Union[PromptStrictInputs, Sequence[PromptStrictInputs]],
246
+ Optional[Union[str, List[str]]]] = None,
247
+ sampling_params: Optional[Union[SamplingParams,
248
+ Sequence[SamplingParams]]] = None,
249
+ prompt_token_ids: Optional[Union[List[int], List[List[int]]]] = None,
250
+ use_tqdm: bool = True,
251
+ lora_request: Optional[LoRARequest] = None,
252
+ multi_modal_data: Optional[MultiModalData] = None,
253
) -> List[RequestOutput]:
254
"""Generates the completions for the input prompts.
255
@@ -155,49 +258,138 @@ class LLM:
258
into a single list and pass it to this method.
259
260
Args:
158
- prompts: A list of prompts to generate completions for.
261
+ inputs: A list of inputs to generate completions for.
262
sampling_params: The sampling parameters for text generation. If
263
None, we use the default sampling parameters.
264
When it is a single value, it is applied to every prompt.
265
When it is a list, the list must have the same length as the
266
prompts and it is paired one by one with the prompt.
164
- prompt_token_ids: A list of token IDs for the prompts. If None, we
165
- use the tokenizer to convert the prompts to token IDs.
267
use_tqdm: Whether to use tqdm to display the progress bar.
268
lora_request: LoRA request to use for generation, if any.
168
- multi_modal_data: Multi modal data.
269
270
Returns:
271
A list of `RequestOutput` objects containing the
272
generated completions in the same order as the input prompts.
273
"""
274
+ if prompt_token_ids is not None or multi_modal_data is not None:
275
+ inputs = self._convert_v1_inputs(
276
+ prompts=cast(Optional[Union[str, List[str]]], prompts),
277
+ prompt_token_ids=prompt_token_ids,
278
+ multi_modal_data=multi_modal_data,
279
+ )
280
+ else:
281
+ inputs = cast(
282
+ Union[PromptStrictInputs, Sequence[PromptStrictInputs]],
283
+ prompts)
284
+
285
if sampling_params is None:
286
# Use default sampling params.
287
sampling_params = SamplingParams()
288
178
- requests_data = self._validate_and_prepare_requests(
179
- prompts,
180
- sampling_params,
181
- prompt_token_ids,
182
- lora_request,
183
- multi_modal_data,
289
+ self._validate_and_add_requests(
290
+ inputs=inputs,
291
+ params=sampling_params,
292
+ lora_request=lora_request,
293
)
294
186
- # Add requests to the engine and run the engine
187
- for request_data in requests_data:
188
- self._add_request(**request_data)
295
+ outputs = self._run_engine(use_tqdm=use_tqdm)
296
+ return LLMEngine.validate_outputs(outputs, RequestOutput)
297
190
- return self._run_engine(use_tqdm)
298
+ @overload # LEGACY: single (prompt + optional token ids)
299
+ def encode(
300
+ self,
301
+ prompts: str,
302
+ pooling_params: Optional[Union[PoolingParams,
303
+ Sequence[PoolingParams]]] = None,
304
+ prompt_token_ids: Optional[List[int]] = None,
305
+ use_tqdm: bool = True,
306
+ lora_request: Optional[LoRARequest] = None,
307
+ multi_modal_data: Optional[MultiModalData] = None,
308
+ ) -> List[EmbeddingRequestOutput]:
309
+ ...
310
311
+ @overload # LEGACY: multi (prompt + optional token ids)
312
def encode(
313
self,
194
- prompts: Optional[Union[str, List[str]]] = None,
314
+ prompts: List[str],
315
pooling_params: Optional[Union[PoolingParams,
196
- List[PoolingParams]]] = None,
316
+ Sequence[PoolingParams]]] = None,
317
prompt_token_ids: Optional[List[List[int]]] = None,
318
use_tqdm: bool = True,
319
lora_request: Optional[LoRARequest] = None,
320
multi_modal_data: Optional[MultiModalData] = None,
321
+ ) -> List[EmbeddingRequestOutput]:
322
+ ...
323
+
324
+ @overload # LEGACY: single (token ids + optional prompt)
325
+ def encode(
326
+ self,
327
+ prompts: Optional[str] = None,
328
+ pooling_params: Optional[Union[PoolingParams,
329
+ Sequence[PoolingParams]]] = None,
330
+ *,
331
+ prompt_token_ids: List[int],
332
+ use_tqdm: bool = True,
333
+ lora_request: Optional[LoRARequest] = None,
334
+ multi_modal_data: Optional[MultiModalData] = None,
335
+ ) -> List[EmbeddingRequestOutput]:
336
+ ...
337
+
338
+ @overload # LEGACY: multi (token ids + optional prompt)
339
+ def encode(
340
+ self,
341
+ prompts: Optional[List[str]] = None,
342
+ pooling_params: Optional[Union[PoolingParams,
343
+ Sequence[PoolingParams]]] = None,
344
+ *,
345
+ prompt_token_ids: List[List[int]],
346
+ use_tqdm: bool = True,
347
+ lora_request: Optional[LoRARequest] = None,
348
+ multi_modal_data: Optional[MultiModalData] = None,
349
+ ) -> List[EmbeddingRequestOutput]:
350
+ ...
351
+
352
+ @overload # LEGACY: single or multi token ids [pos-only]
353
+ def encode(
354
+ self,
355
+ prompts: None,
356
+ pooling_params: None,
357
+ prompt_token_ids: Union[List[int], List[List[int]]],
358
+ use_tqdm: bool = True,
359
+ lora_request: Optional[LoRARequest] = None,
360
+ multi_modal_data: Optional[MultiModalData] = None,
361
+ ) -> List[EmbeddingRequestOutput]:
362
+ ...
363
+
364
+ @overload
365
+ def encode(
366
+ self,
367
+ inputs: Union[PromptStrictInputs, Sequence[PromptStrictInputs]],
368
+ /, # We may enable `inputs` keyword after removing the old API
369
+ *,
370
+ pooling_params: Optional[Union[PoolingParams,
371
+ Sequence[PoolingParams]]] = None,
372
+ use_tqdm: bool = True,
373
+ lora_request: Optional[LoRARequest] = None,
374
+ ) -> List[EmbeddingRequestOutput]:
375
+ ...
376
+
377
+ @deprecate_kwargs("prompts",
378
+ "prompt_token_ids",
379
+ "multi_modal_data",
380
+ is_deprecated=lambda: LLM.DEPRECATE_LEGACY,
381
+ additional_message="Please use the 'inputs' parameter "
382
+ "instead.")
383
+ def encode(
384
+ self,
385
+ prompts: Union[Union[PromptStrictInputs, Sequence[PromptStrictInputs]],
386
+ Optional[Union[str, List[str]]]] = None,
387
+ pooling_params: Optional[Union[PoolingParams,
388
+ Sequence[PoolingParams]]] = None,
389
+ prompt_token_ids: Optional[Union[List[int], List[List[int]]]] = None,
390
+ use_tqdm: bool = True,
391
+ lora_request: Optional[LoRARequest] = None,
392
+ multi_modal_data: Optional[MultiModalData] = None,
393
) -> List[EmbeddingRequestOutput]:
394
"""Generates the completions for the input prompts.
395
@@ -206,124 +398,133 @@ class LLM:
398
into a single list and pass it to this method.
399
400
Args:
209
- prompts: A list of prompts to generate completions for.
401
+ inputs: The inputs to the LLM. You may pass a sequence of inputs for
402
+ batch inference. See :class:`~vllm.inputs.PromptStrictInputs`
403
+ for more details about the format of each input.
404
pooling_params: The pooling parameters for pooling. If None, we
405
use the default pooling parameters.
212
- prompt_token_ids: A list of token IDs for the prompts. If None, we
213
- use the tokenizer to convert the prompts to token IDs.
406
use_tqdm: Whether to use tqdm to display the progress bar.
407
lora_request: LoRA request to use for generation, if any.
216
- multi_modal_data: Multi modal data.
408
409
Returns:
410
A list of `EmbeddingRequestOutput` objects containing the
411
generated embeddings in the same order as the input prompts.
412
"""
413
+ if prompt_token_ids is not None or multi_modal_data is not None:
414
+ inputs = self._convert_v1_inputs(
415
+ prompts=cast(Optional[Union[str, List[str]]], prompts),
416
+ prompt_token_ids=prompt_token_ids,
417
+ multi_modal_data=multi_modal_data,
418
+ )
419
+ else:
420
+ inputs = cast(
421
+ Union[PromptStrictInputs, Sequence[PromptStrictInputs]],
422
+ prompts)
423
+
424
if pooling_params is None:
425
# Use default pooling params.
426
pooling_params = PoolingParams()
427
226
- requests_data = self._validate_and_prepare_requests(
227
- prompts,
228
- pooling_params,
229
- prompt_token_ids,
230
- lora_request,
231
- multi_modal_data,
428
+ self._validate_and_add_requests(
429
+ inputs=inputs,
430
+ params=pooling_params,
431
+ lora_request=lora_request,
432
)
433
234
- # Add requests to the engine and run the engine
235
- for request_data in requests_data:
236
- self._add_request(**request_data)
434
+ outputs = self._run_engine(use_tqdm=use_tqdm)
435
+ return LLMEngine.validate_outputs(outputs, EmbeddingRequestOutput)
436
238
- return self._run_engine(use_tqdm)
239
-
240
- def _validate_and_prepare_requests(
437
+ # LEGACY
438
+ def _convert_v1_inputs(
439
self,
440
prompts: Optional[Union[str, List[str]]],
243
- params: Union[Union[SamplingParams, PoolingParams],
244
- List[Union[SamplingParams,
245
- PoolingParams]]], # Unified parameter
246
- prompt_token_ids: Optional[List[List[int]]] = None,
247
- lora_request: Optional[LoRARequest] = None,
248
- multi_modal_data: Optional[MultiModalData] = None,
249
- ) -> List[dict]:
250
- """Validates and prepares request data for adding to the engine.
441
+ prompt_token_ids: Optional[Union[List[int], List[List[int]]]],
442
+ multi_modal_data: Optional[MultiModalData],
443
+ ):
444
+ # skip_tokenizer_init is now checked in engine
445
252
- Ensures prompts and token IDs are consistent, and returns a list of
253
- dictionaries with request data for further processing.
254
- """
255
- if prompts is None and prompt_token_ids is None:
256
- raise ValueError("Either prompts or prompt_token_ids must be "
257
- "provided.")
258
- if self.llm_engine.model_config.skip_tokenizer_init \
259
- and prompts is not None:
260
- raise ValueError("prompts must be None if skip_tokenizer_init "
261
- "is True")
262
- if isinstance(prompts, str):
263
- # Convert a single prompt to a list.
264
- prompts = [prompts]
265
- if (prompts is not None and prompt_token_ids is not None
266
- and len(prompts) != len(prompt_token_ids)):
267
- raise ValueError("The lengths of prompts and prompt_token_ids "
268
- "must be the same.")
446
+ if prompts is not None:
447
+ prompts = [p["content"] for p in parse_and_batch_prompt(prompts)]
448
+ if prompt_token_ids is not None:
449
+ prompt_token_ids = [
450
+ p["content"] for p in parse_and_batch_prompt(prompt_token_ids)
451
+ ]
452
453
+ num_requests = None
454
if prompts is not None:
455
num_requests = len(prompts)
272
- else:
273
- assert prompt_token_ids is not None
456
+ if prompt_token_ids is not None:
457
+ if (num_requests is not None
458
+ and num_requests != len(prompt_token_ids)):
459
+ raise ValueError("The lengths of prompts and prompt_token_ids "
460
+ "must be the same.")
461
+
462
num_requests = len(prompt_token_ids)
463
+ if num_requests is None:
464
+ raise ValueError("Either prompts or prompt_token_ids must be "
465
+ "provided.")
466
+
467
+ inputs: List[PromptInputs] = []
468
+ for i in range(num_requests):
469
+ if prompts is not None:
470
+ if prompt_token_ids is not None:
471
+ item = TextTokensPrompt(
472
+ prompt=prompts[i],
473
+ prompt_token_ids=prompt_token_ids[i])
474
+ else:
475
+ item = TextPrompt(prompt=prompts[i])
476
+ else:
477
+ if prompt_token_ids is not None:
478
+ item = TokensPrompt(prompt_token_ids=prompt_token_ids[i])
479
+ else:
480
+ raise AssertionError
481
+
482
+ if multi_modal_data is not None:
483
+ item["multi_modal_data"] = multi_modal_data
484
+
485
+ inputs.append(item)
486
+
487
+ return inputs
488
+
489
+ def _validate_and_add_requests(
490
+ self,
491
+ inputs: Union[PromptStrictInputs, Sequence[PromptStrictInputs]],
492
+ params: Union[SamplingParams, Sequence[SamplingParams], PoolingParams,
493
+ Sequence[PoolingParams]],
494
+ lora_request: Optional[LoRARequest],
495
+ ) -> None:
496
+ if isinstance(inputs, (str, dict)):
497
+ # Convert a single prompt to a list.
498
+ inputs = [inputs]
499
+
500
+ num_requests = len(inputs)
501
502
if isinstance(params, list) and len(params) != num_requests:
503
raise ValueError("The lengths of prompts and params "
504
"must be the same.")
279
- if multi_modal_data:
280
- multi_modal_data.data = multi_modal_data.data.to(torch.float16)
505
506
# Add requests to the engine.
283
- requests_data = []
284
- for i in range(num_requests):
285
- prompt = prompts[i] if prompts is not None else None
286
- token_ids = None if prompt_token_ids is None else prompt_token_ids[
287
- i]
288
-
289
- multi_modal_item = MultiModalData(
290
- type=multi_modal_data.type,
291
- data=multi_modal_data.data[i].unsqueeze(0),
292
- ) if multi_modal_data else None
293
-
294
- requests_data.append({
295
- "prompt":
296
- prompt,
297
- "params":
298
- params[i] if isinstance(params, list) else params,
299
- "prompt_token_ids":
300
- token_ids,
301
- "lora_request":
302
- lora_request,
303
- "multi_modal_data":
304
- multi_modal_item,
305
- })
306
-
307
- return requests_data
507
+ for i, request_inputs in enumerate(inputs):
508
+ self._add_request(
509
+ request_inputs,
510
+ params[i] if isinstance(params, Sequence) else params,
511
+ lora_request=lora_request,
512
+ )
513
514
def _add_request(
515
self,
311
- prompt: Optional[str],
516
+ inputs: PromptInputs,
517
params: Union[SamplingParams, PoolingParams],
313
- prompt_token_ids: Optional[List[int]],
518
lora_request: Optional[LoRARequest] = None,
315
- multi_modal_data: Optional[MultiModalData] = None,
519
) -> None:
520
request_id = str(next(self.request_counter))
521
self.llm_engine.add_request(request_id,
319
- prompt,
522
+ inputs,
523
params,
321
- prompt_token_ids,
322
- lora_request=lora_request,
323
- multi_modal_data=multi_modal_data)
524
+ lora_request=lora_request)
525
526
def _run_engine(
326
- self, use_tqdm: bool
527
+ self, *, use_tqdm: bool
528
) -> List[Union[RequestOutput, EmbeddingRequestOutput]]:
529
# Initialize tqdm.
530
if use_tqdm:
@@ -355,5 +556,4 @@ class LLM:
556
# Sort the outputs by request ID.
557
# This is necessary because some requests may be finished earlier than
558
# its previous requests.
358
- outputs = sorted(outputs, key=lambda x: int(x.request_id))
359
- return outputs
559
+ return sorted(outputs, key=lambda x: int(x.request_id))
vllm/entrypoints/openai/serving_chat.py
+9
-3
@@ -176,9 +176,15 @@ class OpenAIServingChat(OpenAIServing):
176
except ValueError as e:
177
return self.create_error_response(str(e))
178
179
- result_generator = self.engine.generate(prompt_text, sampling_params,
180
- request_id, prompt_ids,
181
- lora_request)
179
+ result_generator = self.engine.generate(
180
+ {
181
+ "prompt": prompt_text,
182
+ "prompt_token_ids": prompt_ids
183
+ },
184
+ sampling_params,
185
+ request_id,
186
+ lora_request,
187
+ )
188
# Streaming response
189
if request.stream:
190
return self.chat_completion_stream_generator(
vllm/entrypoints/openai/serving_completion.py
+11
-6
@@ -119,12 +119,17 @@ class OpenAIServingCompletion(OpenAIServing):
119
truncate_prompt_tokens)
120
prompt_ids, prompt_text = prompt_formats
121
122
- generators.append(
123
- self.engine.generate(prompt_text,
124
- sampling_params,
125
- f"{request_id}-{i}",
126
- prompt_token_ids=prompt_ids,
127
- lora_request=lora_request))
122
+ generator = self.engine.generate(
123
+ {
124
+ "prompt": prompt_text,
125
+ "prompt_token_ids": prompt_ids
126
+ },
127
+ sampling_params,
128
+ f"{request_id}-{i}",
129
+ lora_request=lora_request,
130
+ )
131
+
132
+ generators.append(generator)
133
except ValueError as e:
134
# TODO: Use a vllm-specific Validation Error
135
return self.create_error_response(str(e))
vllm/entrypoints/openai/serving_embedding.py
+26
-16
@@ -1,5 +1,5 @@
1
import time
2
-from typing import AsyncIterator, List, Tuple
2
+from typing import AsyncIterator, List, Optional, Tuple
3
4
from fastapi import Request
5
@@ -100,11 +100,16 @@ class OpenAIServingEmbedding(OpenAIServing):
100
101
prompt_ids, prompt_text = prompt_formats
102
103
- generators.append(
104
- self.engine.generate(prompt_text,
105
- pooling_params,
106
- f"{request_id}-{i}",
107
- prompt_token_ids=prompt_ids))
103
+ generator = self.engine.encode(
104
+ {
105
+ "prompt": prompt_text,
106
+ "prompt_token_ids": prompt_ids
107
+ },
108
+ pooling_params,
109
+ f"{request_id}-{i}",
110
+ )
111
+
112
+ generators.append(generator)
113
except ValueError as e:
114
# TODO: Use a vllm-specific Validation Error
115
return self.create_error_response(str(e))
@@ -113,16 +118,21 @@ class OpenAIServingEmbedding(OpenAIServing):
118
int, EmbeddingRequestOutput]] = merge_async_iterators(*generators)
119
120
# Non-streaming response
116
- final_res_batch: EmbeddingRequestOutput = [None] * len(prompts)
117
- async for i, res in result_generator:
118
- if await raw_request.is_disconnected():
119
- # Abort the request if the client disconnects.
120
- await self.engine.abort(f"{request_id}-{i}")
121
- # TODO: Use a vllm-specific Validation Error
122
- return self.create_error_response("Client disconnected")
123
- final_res_batch[i] = res
124
- response = request_output_to_embedding_response(
125
- final_res_batch, request_id, created_time, model_name)
121
+ final_res_batch: List[Optional[EmbeddingRequestOutput]]
122
+ final_res_batch = [None] * len(prompts)
123
+ try:
124
+ async for i, res in result_generator:
125
+ if await raw_request.is_disconnected():
126
+ # Abort the request if the client disconnects.
127
+ await self.engine.abort(f"{request_id}-{i}")
128
+ # TODO: Use a vllm-specific Validation Error
129
+ return self.create_error_response("Client disconnected")
130
+ final_res_batch[i] = res
131
+ response = request_output_to_embedding_response(
132
+ final_res_batch, request_id, created_time, model_name)
133
+ except ValueError as e:
134
+ # TODO: Use a vllm-specific Validation Error
135
+ return self.create_error_response(str(e))
136
137
return response
138
vllm/entrypoints/openai/serving_engine.py
+4
-2
@@ -143,7 +143,8 @@ class OpenAIServing:
143
return json_str
144
145
async def _check_model(
146
- self, request: Union[CompletionRequest, ChatCompletionRequest]
146
+ self, request: Union[CompletionRequest, ChatCompletionRequest,
147
+ EmbeddingRequest]
148
) -> Optional[ErrorResponse]:
149
if request.model in self.served_model_names:
150
return None
@@ -155,7 +156,8 @@ class OpenAIServing:
156
status_code=HTTPStatus.NOT_FOUND)
157
158
def _maybe_get_lora(
158
- self, request: Union[CompletionRequest, ChatCompletionRequest]
159
+ self, request: Union[CompletionRequest, ChatCompletionRequest,
160
+ EmbeddingRequest]
161
) -> Optional[LoRARequest]:
162
if request.model in self.served_model_names:
163
return None
vllm/inputs.py
new
+130
@@ -0,0 +1,130 @@
1
+from typing import (TYPE_CHECKING, List, Literal, Optional, Sequence,
2
+ TypedDict, Union, cast, overload)
3
+
4
+from typing_extensions import NotRequired
5
+
6
+if TYPE_CHECKING:
7
+ from vllm.sequence import MultiModalData
8
+
9
+
10
+class ParsedText(TypedDict):
11
+ content: str
12
+ is_tokens: Literal[False]
13
+
14
+
15
+class ParsedTokens(TypedDict):
16
+ content: List[int]
17
+ is_tokens: Literal[True]
18
+
19
+
20
+# https://github.com/vllm-project/vllm/pull/4028
21
+@overload
22
+def parse_and_batch_prompt(
23
+ prompt: Union[str, List[str]]) -> Sequence[ParsedText]:
24
+ ...
25
+
26
+
27
+@overload
28
+def parse_and_batch_prompt(
29
+ prompt: Union[List[int], List[List[int]]]) -> Sequence[ParsedTokens]:
30
+ ...
31
+
32
+
33
+def parse_and_batch_prompt(
34
+ prompt: Union[str, List[str], List[int], List[List[int]]],
35
+) -> Union[Sequence[ParsedText], Sequence[ParsedTokens]]:
36
+ if isinstance(prompt, str):
37
+ # case 1: a string
38
+ return [ParsedText(content=prompt, is_tokens=False)]
39
+
40
+ if isinstance(prompt, list):
41
+ if len(prompt) == 0:
42
+ raise ValueError("please provide at least one prompt")
43
+
44
+ if isinstance(prompt[0], str):
45
+ # case 2: array of strings
46
+ return [
47
+ ParsedText(content=elem, is_tokens=False)
48
+ for elem in cast(List[str], prompt)
49
+ ]
50
+ if isinstance(prompt[0], int):
51
+ # case 3: array of tokens
52
+ elem = cast(List[int], prompt)
53
+ return [ParsedTokens(content=elem, is_tokens=True)]
54
+ if isinstance(prompt[0], list):
55
+ if len(prompt[0]) == 0:
56
+ raise ValueError("please provide at least one prompt")
57
+
58
+ if isinstance(prompt[0][0], int):
59
+ # case 4: array of token arrays
60
+ return [
61
+ ParsedTokens(content=elem, is_tokens=True)
62
+ for elem in cast(List[List[int]], prompt)
63
+ ]
64
+
65
+ raise ValueError("prompt must be a string, array of strings, "
66
+ "array of tokens, or array of token arrays")
67
+
68
+
69
+class TextPrompt(TypedDict):
70
+ """Schema for a text prompt."""
71
+
72
+ prompt: str
73
+ """The input text to be tokenized before passing to the model."""
74
+
75
+ multi_modal_data: NotRequired["MultiModalData"]
76
+ """
77
+ Optional multi-modal data to pass to the model,
78
+ if the model supports it.
79
+ """
80
+
81
+
82
+class TokensPrompt(TypedDict):
83
+ """Schema for a tokenized prompt."""
84
+
85
+ prompt_token_ids: List[int]
86
+ """A list of token IDs to pass to the model."""
87
+
88
+ multi_modal_data: NotRequired["MultiModalData"]
89
+ """
90
+ Optional multi-modal data to pass to the model,
91
+ if the model supports it.
92
+ """
93
+
94
+
95
+class TextTokensPrompt(TypedDict):
96
+ """It is assumed that :attr:`prompt` is consistent with
97
+ :attr:`prompt_token_ids`. This is currently used in
98
+ :class:`AsyncLLMEngine` for logging both the text and token IDs."""
99
+
100
+ prompt: str
101
+ """The prompt text."""
102
+
103
+ prompt_token_ids: List[int]
104
+ """The token IDs of the prompt. If None, we use the
105
+ tokenizer to convert the prompts to token IDs."""
106
+
107
+ multi_modal_data: NotRequired["MultiModalData"]
108
+ """
109
+ Optional multi-modal data to pass to the model,
110
+ if the model supports it.
111
+ """
112
+
113
+
114
+PromptStrictInputs = Union[str, TextPrompt, TokensPrompt]
115
+"""
116
+The inputs to the LLM, which can take one of the following forms:
117
+
118
+- A text prompt (:class:`str` or :class:`TextPrompt`)
119
+- A tokenized prompt (:class:`TokensPrompt`)
120
+"""
121
+
122
+PromptInputs = Union[str, TextPrompt, TokensPrompt, TextTokensPrompt]
123
+"""Same as :const:`PromptStrictInputs` but additionally accepts
124
+:class:`TextTokensPrompt`."""
125
+
126
+
127
+class LLMInputs(TypedDict):
128
+ prompt_token_ids: List[int]
129
+ prompt: Optional[str]
130
+ multi_modal_data: Optional["MultiModalData"]
vllm/outputs.py
+15
-27
@@ -1,4 +1,5 @@
1
import time
2
+from dataclasses import dataclass
3
from typing import List, Optional, Union
4
5
from vllm.lora.request import LoRARequest
@@ -6,6 +7,7 @@ from vllm.sequence import (PromptLogprobs, RequestMetrics, SampleLogprobs,
7
SequenceGroup, SequenceStatus)
8
9
10
+@dataclass
11
class CompletionOutput:
12
"""The output data of one completion output of a request.
13
@@ -24,25 +26,14 @@ class CompletionOutput:
26
lora_request: The LoRA request that was used to generate the output.
27
"""
28
27
- def __init__(
28
- self,
29
- index: int,
30
- text: str,
31
- token_ids: List[int],
32
- cumulative_logprob: float,
33
- logprobs: Optional[SampleLogprobs],
34
- finish_reason: Optional[str] = None,
35
- stop_reason: Union[int, str, None] = None,
36
- lora_request: Optional[LoRARequest] = None,
37
- ) -> None:
38
- self.index = index
39
- self.text = text
40
- self.token_ids = token_ids
41
- self.cumulative_logprob = cumulative_logprob
42
- self.logprobs = logprobs
43
- self.finish_reason = finish_reason
44
- self.stop_reason = stop_reason
45
- self.lora_request = lora_request
29
+ index: int
30
+ text: str
31
+ token_ids: List[int]
32
+ cumulative_logprob: float
33
+ logprobs: Optional[SampleLogprobs]
34
+ finish_reason: Optional[str] = None
35
+ stop_reason: Union[int, str, None] = None
36
+ lora_request: Optional[LoRARequest] = None
37
38
def finished(self) -> bool:
39
return self.finish_reason is not None
@@ -57,6 +48,7 @@ class CompletionOutput:
48
f"stop_reason={self.stop_reason})")
49
50
51
+@dataclass
52
class EmbeddingOutput:
53
"""The output data of one completion output of a request.
54
@@ -65,15 +57,11 @@ class EmbeddingOutput:
57
length of vector depends on the model as listed in the embedding guide.
58
"""
59
68
- def __init__(
69
- self,
70
- embedding: List[float],
71
- ) -> None:
72
- self.embedding = embedding
60
+ embedding: List[float]
61
62
def __repr__(self) -> str:
63
return (f"EmbeddingOutput("
76
- f"embedding={len(self.embedding)}")
64
+ f"embedding={len(self.embedding)})")
65
66
67
class RequestOutput:
@@ -93,7 +81,7 @@ class RequestOutput:
81
def __init__(
82
self,
83
request_id: str,
96
- prompt: str,
84
+ prompt: Optional[str],
85
prompt_token_ids: List[int],
86
prompt_logprobs: Optional[PromptLogprobs],
87
outputs: List[CompletionOutput],
@@ -183,7 +171,7 @@ class EmbeddingRequestOutput:
171
finished (bool): A flag indicating whether the embedding is completed.
172
"""
173
186
- def __init__(self, request_id: str, outputs: 'EmbeddingOutput',
174
+ def __init__(self, request_id: str, outputs: "EmbeddingOutput",
175
prompt_token_ids: List[int], finished: bool):
176
self.request_id = request_id
177
self.prompt_token_ids = prompt_token_ids
vllm/sequence.py
+26
-12
@@ -6,6 +6,7 @@ from dataclasses import dataclass, field
6
from typing import TYPE_CHECKING, Dict, List, Optional, Tuple, Union
7
8
from vllm.block import LogicalTokenBlock
9
+from vllm.inputs import LLMInputs
10
from vllm.lora.request import LoRARequest
11
from vllm.pooling_params import PoolingParams
12
from vllm.sampling_params import SamplingParams
@@ -210,8 +211,7 @@ class Sequence:
211
212
Args:
213
seq_id: The ID of the sequence.
213
- prompt: The prompt of the sequence.
214
- prompt_token_ids: The token IDs of the prompt.
214
+ inputs: The inputs of the sequence.
215
block_size: The block size of the sequence. Should be the same as the
216
block size used by the block manager and cache engine.
217
lora_request: LoRA request.
@@ -220,25 +220,24 @@ class Sequence:
220
def __init__(
221
self,
222
seq_id: int,
223
- prompt: str,
224
- prompt_token_ids: List[int],
223
+ inputs: LLMInputs,
224
block_size: int,
225
eos_token_id: Optional[int] = None,
226
lora_request: Optional[LoRARequest] = None,
227
) -> None:
228
self.seq_id = seq_id
230
- self.prompt = prompt
229
+ self.inputs = inputs
230
self.block_size = block_size
231
self.eos_token_id = eos_token_id
232
self.lora_request = lora_request
233
235
- self.data: SequenceData = SequenceData(prompt_token_ids)
234
+ self.data = SequenceData(self.prompt_token_ids)
235
self.output_logprobs: SampleLogprobs = []
236
self.output_text = ""
237
238
self.logical_token_blocks: List[LogicalTokenBlock] = []
239
# Initialize the logical token blocks with the prompt token ids.
241
- self._append_tokens_to_blocks(prompt_token_ids)
240
+ self._append_tokens_to_blocks(self.prompt_token_ids)
241
self.status = SequenceStatus.WAITING
242
self.stop_reason: Union[int, str, None] = None
243
@@ -248,6 +247,18 @@ class Sequence:
247
# Input + output tokens
248
self.tokens: Optional[List[str]] = None
249
250
+ @property
251
+ def prompt(self) -> Optional[str]:
252
+ return self.inputs["prompt"]
253
+
254
+ @property
255
+ def prompt_token_ids(self) -> List[int]:
256
+ return self.inputs["prompt_token_ids"]
257
+
258
+ @property
259
+ def multi_modal_data(self) -> Optional["MultiModalData"]:
260
+ return self.inputs["multi_modal_data"]
261
+
262
@property
263
def lora_int_id(self) -> int:
264
return self.lora_request.lora_int_id if self.lora_request else 0
@@ -415,7 +426,6 @@ class SequenceGroup:
426
sampling_params: The sampling parameters used to generate the outputs.
427
arrival_time: The arrival time of the request.
428
lora_request: LoRA request.
418
- multi_modal_data: Multi modal data associated with the request.
429
embeddings: The embeddings vectors of the prompt of the sequence group
430
for an embedding model.
431
pooling_params: The pooling parameters used to generate the pooling
@@ -429,7 +439,6 @@ class SequenceGroup:
439
arrival_time: float,
440
sampling_params: Optional[SamplingParams] = None,
441
lora_request: Optional[LoRARequest] = None,
432
- multi_modal_data: Optional[MultiModalData] = None,
442
embeddings: Optional[List[float]] = None,
443
pooling_params: Optional[PoolingParams] = None,
444
) -> None:
@@ -444,12 +453,11 @@ class SequenceGroup:
453
self.lora_request = lora_request
454
self.prompt_logprobs: Optional[PromptLogprobs] = None
455
self.state = SequenceGroupState()
447
- self.multi_modal_data = multi_modal_data
456
self.embeddings = embeddings
457
self.pooling_params = pooling_params
458
459
@property
452
- def prompt(self) -> str:
460
+ def prompt(self) -> Optional[str]:
461
# All sequences in the group should have the same prompt.
462
# We use the prompt of an arbitrary sequence.
463
return next(iter(self.seqs_dict.values())).prompt
@@ -458,7 +466,13 @@ class SequenceGroup:
466
def prompt_token_ids(self) -> List[int]:
467
# All sequences in the group should have the same prompt.
468
# We use the prompt of an arbitrary sequence.
461
- return next(iter(self.seqs_dict.values())).data.prompt_token_ids
469
+ return next(iter(self.seqs_dict.values())).prompt_token_ids
470
+
471
+ @property
472
+ def multi_modal_data(self) -> Optional[MultiModalData]:
473
+ # All sequences in the group should have the same multi-modal data.
474
+ # We use the multi-modal data of an arbitrary sequence.
475
+ return next(iter(self.seqs_dict.values())).multi_modal_data
476
477
@property
478
def lora_int_id(self) -> int:
vllm/utils.py
+42
-1
@@ -11,7 +11,7 @@ import threading
11
import uuid
12
import warnings
13
from collections import defaultdict
14
-from functools import lru_cache, partial
14
+from functools import lru_cache, partial, wraps
15
from platform import uname
16
from typing import (Any, AsyncIterator, Awaitable, Callable, Dict, Generic,
17
Hashable, List, Optional, OrderedDict, Tuple, TypeVar,
@@ -658,3 +658,44 @@ def enable_trace_function_call_for_thread() -> None:
658
filename)
659
os.makedirs(os.path.dirname(log_path), exist_ok=True)
660
enable_trace_function_call(log_path)
661
+
662
+
663
+def identity(value: T) -> T:
664
+ return value
665
+
666
+
667
+F = TypeVar('F', bound=Callable[..., Any])
668
+
669
+
670
+def deprecate_kwargs(
671
+ *kws: str,
672
+ is_deprecated: Union[bool, Callable[[], bool]] = True,
673
+ additional_message: Optional[str] = None) -> Callable[[F], F]:
674
+ deprecated_kws = set(kws)
675
+
676
+ if not callable(is_deprecated):
677
+ is_deprecated = partial(identity, is_deprecated)
678
+
679
+ def wrapper(fn: F) -> F:
680
+
681
+ @wraps(fn)
682
+ def inner(*args, **kwargs):
683
+ if is_deprecated():
684
+ deprecated_kwargs = kwargs.keys() & deprecated_kws
685
+ if deprecated_kwargs:
686
+ msg = (
687
+ f"The keyword arguments {deprecated_kwargs} are "
688
+ "deprecated and will be removed in a future update.")
689
+ if additional_message is not None:
690
+ msg += f" {additional_message}"
691
+
692
+ warnings.warn(
693
+ DeprecationWarning(msg),
694
+ stacklevel=3, # The inner function takes up one level
695
+ )
696
+
697
+ return fn(*args, **kwargs)
698
+
699
+ return inner # type: ignore
700
+
701
+ return wrapper