@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 5ae5ed1e60
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 00:30 UTC

[Core] Consolidate prompt arguments to LLM engines (#4328)

Co-authored-by: Roger Wang <ywang@roblox.com>

Cyrus Leung committed May 29, 2024 at 04:29 UTC 5ae5ed1e6047d4095149e26526a618be0529a118
43 files changed +1404 -439
.buildkite/test-pipeline.yaml
+6 -3
@@ -63,9 +63,9 @@ steps:
63 mirror_hardwares: [amd]
64
65 commands:
66 - # these tests have to be separated, because each one will allocate all posible GPU memory
67 - - pytest -v -s entrypoints --ignore=entrypoints/test_server_oot_registration.py
68 - - pytest -v -s entrypoints/test_server_oot_registration.py
66 + - pytest -v -s test_inputs.py
67 + - pytest -v -s entrypoints -m llm
68 + - pytest -v -s entrypoints -m openai
69
70 - label: Examples Test
71 working_dir: "/vllm-workspace/examples"
@@ -110,6 +110,9 @@ steps:
110 mirror_hardwares: [amd]
111 command: pytest -v -s test_logits_processor.py
112
113 +- label: Utils Test
114 + command: pytest -v -s test_utils.py
115 +
116 - label: Worker Test
117 mirror_hardwares: [amd]
118 command: pytest -v -s worker
benchmarks/benchmark_latency.py
+7 -4
@@ -3,13 +3,14 @@ import argparse
3 import json
4 import time
5 from pathlib import Path
6 -from typing import Optional
6 +from typing import List, Optional
7
8 import numpy as np
9 import torch
10 from tqdm import tqdm
11
12 from vllm import LLM, SamplingParams
13 +from vllm.inputs import PromptStrictInputs
14 from vllm.model_executor.layers.quantization import QUANTIZATION_METHODS
15
16
@@ -48,7 +49,9 @@ def main(args: argparse.Namespace):
49 dummy_prompt_token_ids = np.random.randint(10000,
50 size=(args.batch_size,
51 args.input_len))
51 - dummy_prompt_token_ids = dummy_prompt_token_ids.tolist()
52 + dummy_inputs: List[PromptStrictInputs] = [{
53 + "prompt_token_ids": batch
54 + } for batch in dummy_prompt_token_ids.tolist()]
55
56 def run_to_completion(profile_dir: Optional[str] = None):
57 if profile_dir:
@@ -59,13 +62,13 @@ def main(args: argparse.Namespace):
62 ],
63 on_trace_ready=torch.profiler.tensorboard_trace_handler(
64 str(profile_dir))) as p:
62 - llm.generate(prompt_token_ids=dummy_prompt_token_ids,
65 + llm.generate(dummy_inputs,
66 sampling_params=sampling_params,
67 use_tqdm=False)
68 print(p.key_averages())
69 else:
70 start_time = time.perf_counter()
68 - llm.generate(prompt_token_ids=dummy_prompt_token_ids,
71 + llm.generate(dummy_inputs,
72 sampling_params=sampling_params,
73 use_tqdm=False)
74 end_time = time.perf_counter()
docs/source/dev/offline_inference/llm.rst renamed
+1 -1
@@ -1,5 +1,5 @@
1 LLM Class
2 -==========
2 +=========
3
4 .. autoclass:: vllm.LLM
5 :members:
docs/source/dev/offline_inference/llm_inputs.rst new
+14
@@ -0,0 +1,14 @@
1 +LLM Inputs
2 +==========
3 +
4 +.. autodata:: vllm.inputs.PromptStrictInputs
5 +
6 +.. autoclass:: vllm.inputs.TextPrompt
7 + :show-inheritance:
8 + :members:
9 + :member-order: bysource
10 +
11 +.. autoclass:: vllm.inputs.TokensPrompt
12 + :show-inheritance:
13 + :members:
14 + :member-order: bysource
docs/source/dev/offline_inference/offline_index.rst new
+8
@@ -0,0 +1,8 @@
1 +Offline Inference
2 +=================================
3 +
4 +.. toctree::
5 + :maxdepth: 1
6 +
7 + llm
8 + llm_inputs
docs/source/dev/sampling_params.rst renamed
docs/source/index.rst
+3 -8
@@ -68,13 +68,6 @@ Documentation
68 getting_started/quickstart
69 getting_started/examples/examples_index
70
71 -.. toctree::
72 - :maxdepth: 1
73 - :caption: Offline Inference
74 -
75 - offline_inference/llm
76 - offline_inference/sampling_params
77 -
71 .. toctree::
72 :maxdepth: 1
73 :caption: Serving
@@ -108,7 +101,9 @@ Documentation
101 .. toctree::
102 :maxdepth: 2
103 :caption: Developer Documentation
111 -
104 +
105 + dev/sampling_params
106 + dev/offline_inference/offline_index
107 dev/engine/engine_index
108 dev/kernel/paged_attention
109 dev/dockerfile/dockerfile
docs/source/serving/openai_compatible_server.md
+2 -2
@@ -48,7 +48,7 @@ completion = client.chat.completions.create(
48 ```
49
50 ### Extra Parameters for Chat API
51 -The following [sampling parameters (click through to see documentation)](../offline_inference/sampling_params.rst) are supported.
51 +The following [sampling parameters (click through to see documentation)](../dev/sampling_params.rst) are supported.
52
53 ```{literalinclude} ../../../vllm/entrypoints/openai/protocol.py
54 :language: python
@@ -65,7 +65,7 @@ The following extra parameters are supported:
65 ```
66
67 ### Extra Parameters for Completions API
68 -The following [sampling parameters (click through to see documentation)](../offline_inference/sampling_params.rst) are supported.
68 +The following [sampling parameters (click through to see documentation)](../dev/sampling_params.rst) are supported.
69
70 ```{literalinclude} ../../../vllm/entrypoints/openai/protocol.py
71 :language: python
examples/llava_example.py
+16 -9
@@ -23,11 +23,15 @@ def run_llava_pixel_values():
23 "\nUSER: What is the content of this image?\nASSISTANT:")
24
25 # This should be provided by another online or offline component.
26 - images = torch.load("images/stop_sign_pixel_values.pt")
26 + image = torch.load("images/stop_sign_pixel_values.pt")
27 +
28 + outputs = llm.generate({
29 + "prompt":
30 + prompt,
31 + "multi_modal_data":
32 + MultiModalData(type=MultiModalData.Type.IMAGE, data=image),
33 + })
34
28 - outputs = llm.generate(prompt,
29 - multi_modal_data=MultiModalData(
30 - type=MultiModalData.Type.IMAGE, data=images))
35 for o in outputs:
36 generated_text = o.outputs[0].text
37 print(generated_text)
@@ -46,11 +50,14 @@ def run_llava_image_features():
50 "\nUSER: What is the content of this image?\nASSISTANT:")
51
52 # This should be provided by another online or offline component.
49 - images = torch.load("images/stop_sign_image_features.pt")
50 -
51 - outputs = llm.generate(prompt,
52 - multi_modal_data=MultiModalData(
53 - type=MultiModalData.Type.IMAGE, data=images))
53 + image = torch.load("images/stop_sign_image_features.pt")
54 +
55 + outputs = llm.generate({
56 + "prompt":
57 + prompt,
58 + "multi_modal_data":
59 + MultiModalData(type=MultiModalData.Type.IMAGE, data=image),
60 + })
61 for o in outputs:
62 generated_text = o.outputs[0].text
63 print(generated_text)
pyproject.toml
+7
@@ -65,3 +65,10 @@ skip = "./tests/prompts,./benchmarks/sonnet.txt,./tests/lora/data,./build"
65 [tool.isort]
66 use_parentheses = true
67 skip_gitignore = true
68 +
69 +[tool.pytest.ini_options]
70 +markers = [
71 + "skip_global_cleanup",
72 + "llm: run tests for vLLM API only",
73 + "openai: run tests for OpenAI API only",
74 +]
tests/async_engine/test_async_llm_engine.py
+1 -1
@@ -25,7 +25,7 @@ class MockEngine:
25 return [RequestOutput(
26 request_id=self.request_id)] if self.request_id else []
27
28 - async def encode_request_async(self, *args, **kwargs):
28 + async def process_model_inputs_async(self, *args, **kwargs):
29 pass
30
31 def generate(self, request_id):
tests/async_engine/test_openapi_server_ray.py
+1 -1
@@ -29,7 +29,7 @@ def server():
29 ray.shutdown()
30
31
32 -@pytest.fixture(scope="session")
32 +@pytest.fixture(scope="module")
33 def client():
34 client = openai.AsyncOpenAI(
35 base_url="http://localhost:8000/v1",
tests/conftest.py
+17 -6
@@ -12,6 +12,7 @@ from transformers import (AutoModelForCausalLM, AutoProcessor, AutoTokenizer,
12 from vllm import LLM, SamplingParams
13 from vllm.config import TokenizerPoolConfig, VisionLanguageConfig
14 from vllm.distributed import destroy_model_parallel
15 +from vllm.inputs import PromptInputs
16 from vllm.logger import init_logger
17 from vllm.sequence import MultiModalData
18
@@ -402,12 +403,22 @@ class VllmRunner:
403 ) -> List[Tuple[List[int], str]]:
404 if images is not None:
405 assert len(prompts) == images.shape[0]
405 - req_outputs = self.model.generate(
406 - prompts,
407 - sampling_params=sampling_params,
408 - multi_modal_data=MultiModalData(type=MultiModalData.Type.IMAGE,
409 - data=images)
410 - if images is not None else None)
406 +
407 + prompt_inputs: List[PromptInputs] = []
408 + for i, prompt in enumerate(prompts):
409 + image = None if images is None else images[i:i + 1]
410 + mm_data = None if image is None else MultiModalData(
411 + type=MultiModalData.Type.IMAGE,
412 + data=image,
413 + )
414 +
415 + prompt_inputs.append({
416 + "prompt": prompt,
417 + "multi_modal_data": mm_data,
418 + })
419 +
420 + req_outputs = self.model.generate(prompt_inputs,
421 + sampling_params=sampling_params)
422 outputs = []
423 for req_output in req_outputs:
424 prompt_str = req_output.prompt
tests/core/test_block_manager.py
+12 -3
@@ -133,8 +133,11 @@ def test_append_slot_cow():
133
134 # Allocate prompt to gpu block. There is one slot left in the block.
135 prompt = Sequence(seq_id=1,
136 - prompt="one two three",
137 - prompt_token_ids=[1, 2, 3],
136 + inputs={
137 + "prompt": "one two three",
138 + "prompt_token_ids": [1, 2, 3],
139 + "multi_modal_data": None
140 + },
141 block_size=block_size)
142
143 # Fork the sequence, such that a COW will be required when we append a new
@@ -304,7 +307,13 @@ def test_sliding_window_multi_seq():
307
308 assert block_manager.get_num_free_gpu_blocks() == num_gpu_blocks
309
307 - parent = Sequence(1, "one two three", [0, 1, 2], block_size)
310 + parent = Sequence(seq_id=1,
311 + inputs={
312 + "prompt": "one two three",
313 + "prompt_token_ids": [0, 1, 2],
314 + "multi_modal_data": None
315 + },
316 + block_size=block_size)
317 seq_group = SequenceGroup(request_id="1",
318 seqs=[parent],
319 arrival_time=time.time(),
tests/core/utils.py
+12 -3
@@ -21,7 +21,13 @@ def create_dummy_prompt(
21 # and prompt "0 ... block_size".
22 prompt_tokens = list(range(prompt_length))
23 prompt_str = " ".join([str(t) for t in prompt_tokens])
24 - prompt = Sequence(int(request_id), prompt_str, prompt_tokens, block_size)
24 + prompt = Sequence(int(request_id),
25 + inputs={
26 + "prompt": prompt_str,
27 + "prompt_token_ids": prompt_tokens,
28 + "multi_modal_data": None,
29 + },
30 + block_size=block_size)
31 seq_group = SequenceGroup(request_id=request_id,
32 seqs=[prompt],
33 arrival_time=time.time(),
@@ -51,8 +57,11 @@ def create_seq_group(
57 for seq_id_offset, output_len in enumerate(seq_output_lens):
58 seq = Sequence(
59 seq_id=seq_id_start + seq_id_offset,
54 - prompt="",
55 - prompt_token_ids=prompt_token_ids,
60 + inputs={
61 + "prompt": "",
62 + "prompt_token_ids": prompt_token_ids,
63 + "multi_modal_data": None,
64 + },
65 block_size=16,
66 )
67
tests/engine/test_skip_tokenizer_init.py
+1 -1
@@ -14,7 +14,7 @@ def test_skip_tokenizer_initialization(model: str):
14 with pytest.raises(ValueError) as err:
15 llm.generate("abc", sampling_params)
16 assert "prompts must be None if" in str(err.value)
17 - outputs = llm.generate(prompt_token_ids=[[1, 2, 3]],
17 + outputs = llm.generate({"prompt_token_ids": [1, 2, 3]},
18 sampling_params=sampling_params)
19 assert len(outputs) > 0
20 completions = outputs[0].outputs
tests/entrypoints/openai/test_serving_chat.py
+4
@@ -1,11 +1,15 @@
1 import asyncio
2 from dataclasses import dataclass
3
4 +import pytest
5 +
6 from vllm.entrypoints.openai.serving_chat import OpenAIServingChat
7
8 MODEL_NAME = "openai-community/gpt2"
9 CHAT_TEMPLATE = "Dummy chat template for testing {}"
10
11 +pytestmark = pytest.mark.openai
12 +
13
14 @dataclass
15 class MockModelConfig:
tests/entrypoints/test_guided_processors.py
+2
@@ -52,6 +52,8 @@ TEST_SCHEMA = {
52 TEST_REGEX = (r"((25[0-5]|(2[0-4]|1\d|[1-9]|)\d)\.){3}"
53 r"(25[0-5]|(2[0-4]|1\d|[1-9]|)\d)")
54
55 +pytestmark = pytest.mark.openai
56 +
57
58 def test_guided_logits_processors():
59 """Basic unit test for RegexLogitsProcessor and JSONLogitsProcessor."""
tests/entrypoints/test_llm_encode.py new
+144
@@ -0,0 +1,144 @@
1 +import weakref
2 +from typing import List
3 +
4 +import pytest
5 +
6 +from vllm import LLM, EmbeddingRequestOutput, PoolingParams
7 +
8 +from ..conftest import cleanup
9 +
10 +MODEL_NAME = "intfloat/e5-mistral-7b-instruct"
11 +
12 +PROMPTS = [
13 + "Hello, my name is",
14 + "The president of the United States is",
15 + "The capital of France is",
16 + "The future of AI is",
17 +]
18 +
19 +TOKEN_IDS = [
20 + # Using ID={0, 1, 2, 3} results in NaN values,
21 + # so we add this offset of 1000
22 + [1000],
23 + [1000, 1001],
24 + [1000, 1002, 1001],
25 + [1000, 1003, 1001, 1002],
26 +]
27 +
28 +pytestmark = pytest.mark.llm
29 +
30 +
31 +@pytest.fixture(scope="module")
32 +def llm():
33 + # pytest caches the fixture so we use weakref.proxy to
34 + # enable garbage collection
35 + llm = LLM(model=MODEL_NAME,
36 + max_num_batched_tokens=32768,
37 + tensor_parallel_size=1,
38 + gpu_memory_utilization=0.75,
39 + enforce_eager=True)
40 +
41 + with llm.deprecate_legacy_api():
42 + yield weakref.proxy(llm)
43 +
44 + del llm
45 +
46 + cleanup()
47 +
48 +
49 +def assert_outputs_equal(o1: List[EmbeddingRequestOutput],
50 + o2: List[EmbeddingRequestOutput]):
51 + assert [o.outputs for o in o1] == [o.outputs for o in o2]
52 +
53 +
54 +@pytest.mark.skip_global_cleanup
55 +@pytest.mark.parametrize('prompt', PROMPTS)
56 +def test_v1_v2_api_consistency_single_prompt_string(llm: LLM, prompt):
57 + pooling_params = PoolingParams()
58 +
59 + with pytest.warns(DeprecationWarning, match="'prompts'"):
60 + v1_output = llm.encode(prompts=prompt, pooling_params=pooling_params)
61 +
62 + v2_output = llm.encode(prompt, pooling_params=pooling_params)
63 + assert_outputs_equal(v1_output, v2_output)
64 +
65 + v2_output = llm.encode({"prompt": prompt}, pooling_params=pooling_params)
66 + assert_outputs_equal(v1_output, v2_output)
67 +
68 +
69 +@pytest.mark.skip_global_cleanup
70 +@pytest.mark.parametrize('prompt_token_ids', TOKEN_IDS)
71 +def test_v1_v2_api_consistency_single_prompt_tokens(llm: LLM,
72 + prompt_token_ids):
73 + pooling_params = PoolingParams()
74 +
75 + with pytest.warns(DeprecationWarning, match="'prompt_token_ids'"):
76 + v1_output = llm.encode(prompt_token_ids=prompt_token_ids,
77 + pooling_params=pooling_params)
78 +
79 + v2_output = llm.encode({"prompt_token_ids": prompt_token_ids},
80 + pooling_params=pooling_params)
81 + assert_outputs_equal(v1_output, v2_output)
82 +
83 +
84 +@pytest.mark.skip_global_cleanup
85 +def test_v1_v2_api_consistency_multi_prompt_string(llm: LLM):
86 + pooling_params = PoolingParams()
87 +
88 + with pytest.warns(DeprecationWarning, match="'prompts'"):
89 + v1_output = llm.encode(prompts=PROMPTS, pooling_params=pooling_params)
90 +
91 + v2_output = llm.encode(PROMPTS, pooling_params=pooling_params)
92 + assert_outputs_equal(v1_output, v2_output)
93 +
94 + v2_output = llm.encode(
95 + [{
96 + "prompt": p
97 + } for p in PROMPTS],
98 + pooling_params=pooling_params,
99 + )
100 + assert_outputs_equal(v1_output, v2_output)
101 +
102 +
103 +@pytest.mark.skip_global_cleanup
104 +def test_v1_v2_api_consistency_multi_prompt_tokens(llm: LLM):
105 + pooling_params = PoolingParams()
106 +
107 + with pytest.warns(DeprecationWarning, match="'prompt_token_ids'"):
108 + v1_output = llm.encode(prompt_token_ids=TOKEN_IDS,
109 + pooling_params=pooling_params)
110 +
111 + v2_output = llm.encode(
112 + [{
113 + "prompt_token_ids": p
114 + } for p in TOKEN_IDS],
115 + pooling_params=pooling_params,
116 + )
117 + assert_outputs_equal(v1_output, v2_output)
118 +
119 +
120 +@pytest.mark.skip_global_cleanup
121 +def test_multiple_pooling_params(llm: LLM):
122 + pooling_params = [
123 + PoolingParams(),
124 + PoolingParams(),
125 + PoolingParams(),
126 + PoolingParams(),
127 + ]
128 +
129 + # Multiple PoolingParams should be matched with each prompt
130 + outputs = llm.encode(PROMPTS, pooling_params=pooling_params)
131 + assert len(PROMPTS) == len(outputs)
132 +
133 + # Exception raised, if the size of params does not match the size of prompts
134 + with pytest.raises(ValueError):
135 + outputs = llm.encode(PROMPTS, pooling_params=pooling_params[:3])
136 +
137 + # Single PoolingParams should be applied to every prompt
138 + single_pooling_params = PoolingParams()
139 + outputs = llm.encode(PROMPTS, pooling_params=single_pooling_params)
140 + assert len(PROMPTS) == len(outputs)
141 +
142 + # pooling_params is None, default params should be applied
143 + outputs = llm.encode(PROMPTS, pooling_params=None)
144 + assert len(PROMPTS) == len(outputs)
tests/entrypoints/test_llm_generate.py
+120 -17
@@ -1,21 +1,124 @@
1 +import weakref
2 +from typing import List
3 +
4 import pytest
5
3 -from vllm import LLM, SamplingParams
6 +from vllm import LLM, RequestOutput, SamplingParams
7 +
8 +from ..conftest import cleanup
9 +
10 +MODEL_NAME = "facebook/opt-125m"
11 +
12 +PROMPTS = [
13 + "Hello, my name is",
14 + "The president of the United States is",
15 + "The capital of France is",
16 + "The future of AI is",
17 +]
18
19 +TOKEN_IDS = [
20 + [0],
21 + [0, 1],
22 + [0, 2, 1],
23 + [0, 3, 1, 2],
24 +]
25
6 -def test_multiple_sampling_params():
26 +pytestmark = pytest.mark.llm
27
8 - llm = LLM(model="facebook/opt-125m",
28 +
29 +@pytest.fixture(scope="module")
30 +def llm():
31 + # pytest caches the fixture so we use weakref.proxy to
32 + # enable garbage collection
33 + llm = LLM(model=MODEL_NAME,
34 max_num_batched_tokens=4096,
10 - tensor_parallel_size=1)
35 + tensor_parallel_size=1,
36 + gpu_memory_utilization=0.10,
37 + enforce_eager=True)
38 +
39 + with llm.deprecate_legacy_api():
40 + yield weakref.proxy(llm)
41 +
42 + del llm
43 +
44 + cleanup()
45 +
46 +
47 +def assert_outputs_equal(o1: List[RequestOutput], o2: List[RequestOutput]):
48 + assert [o.outputs for o in o1] == [o.outputs for o in o2]
49 +
50 +
51 +@pytest.mark.skip_global_cleanup
52 +@pytest.mark.parametrize('prompt', PROMPTS)
53 +def test_v1_v2_api_consistency_single_prompt_string(llm: LLM, prompt):
54 + sampling_params = SamplingParams(temperature=0.0, top_p=1.0)
55 +
56 + with pytest.warns(DeprecationWarning, match="'prompts'"):
57 + v1_output = llm.generate(prompts=prompt,
58 + sampling_params=sampling_params)
59 +
60 + v2_output = llm.generate(prompt, sampling_params=sampling_params)
61 + assert_outputs_equal(v1_output, v2_output)
62 +
63 + v2_output = llm.generate({"prompt": prompt},
64 + sampling_params=sampling_params)
65 + assert_outputs_equal(v1_output, v2_output)
66 +
67 +
68 +@pytest.mark.skip_global_cleanup
69 +@pytest.mark.parametrize('prompt_token_ids', TOKEN_IDS)
70 +def test_v1_v2_api_consistency_single_prompt_tokens(llm: LLM,
71 + prompt_token_ids):
72 + sampling_params = SamplingParams(temperature=0.0, top_p=1.0)
73 +
74 + with pytest.warns(DeprecationWarning, match="'prompt_token_ids'"):
75 + v1_output = llm.generate(prompt_token_ids=prompt_token_ids,
76 + sampling_params=sampling_params)
77 +
78 + v2_output = llm.generate({"prompt_token_ids": prompt_token_ids},
79 + sampling_params=sampling_params)
80 + assert_outputs_equal(v1_output, v2_output)
81 +
82 +
83 +@pytest.mark.skip_global_cleanup
84 +def test_v1_v2_api_consistency_multi_prompt_string(llm: LLM):
85 + sampling_params = SamplingParams(temperature=0.0, top_p=1.0)
86 +
87 + with pytest.warns(DeprecationWarning, match="'prompts'"):
88 + v1_output = llm.generate(prompts=PROMPTS,
89 + sampling_params=sampling_params)
90 +
91 + v2_output = llm.generate(PROMPTS, sampling_params=sampling_params)
92 + assert_outputs_equal(v1_output, v2_output)
93 +
94 + v2_output = llm.generate(
95 + [{
96 + "prompt": p
97 + } for p in PROMPTS],
98 + sampling_params=sampling_params,
99 + )
100 + assert_outputs_equal(v1_output, v2_output)
101 +
102 +
103 +@pytest.mark.skip_global_cleanup
104 +def test_v1_v2_api_consistency_multi_prompt_tokens(llm: LLM):
105 + sampling_params = SamplingParams(temperature=0.0, top_p=1.0)
106 +
107 + with pytest.warns(DeprecationWarning, match="'prompt_token_ids'"):
108 + v1_output = llm.generate(prompt_token_ids=TOKEN_IDS,
109 + sampling_params=sampling_params)
110 +
111 + v2_output = llm.generate(
112 + [{
113 + "prompt_token_ids": p
114 + } for p in TOKEN_IDS],
115 + sampling_params=sampling_params,
116 + )
117 + assert_outputs_equal(v1_output, v2_output)
118
12 - prompts = [
13 - "Hello, my name is",
14 - "The president of the United States is",
15 - "The capital of France is",
16 - "The future of AI is",
17 - ]
119
120 +@pytest.mark.skip_global_cleanup
121 +def test_multiple_sampling_params(llm: LLM):
122 sampling_params = [
123 SamplingParams(temperature=0.01, top_p=0.95),
124 SamplingParams(temperature=0.3, top_p=0.95),
@@ -24,18 +127,18 @@ def test_multiple_sampling_params():
127 ]
128
129 # Multiple SamplingParams should be matched with each prompt
27 - outputs = llm.generate(prompts, sampling_params=sampling_params)
28 - assert len(prompts) == len(outputs)
130 + outputs = llm.generate(PROMPTS, sampling_params=sampling_params)
131 + assert len(PROMPTS) == len(outputs)
132
133 # Exception raised, if the size of params does not match the size of prompts
134 with pytest.raises(ValueError):
32 - outputs = llm.generate(prompts, sampling_params=sampling_params[:3])
135 + outputs = llm.generate(PROMPTS, sampling_params=sampling_params[:3])
136
137 # Single SamplingParams should be applied to every prompt
138 single_sampling_params = SamplingParams(temperature=0.3, top_p=0.95)
36 - outputs = llm.generate(prompts, sampling_params=single_sampling_params)
37 - assert len(prompts) == len(outputs)
139 + outputs = llm.generate(PROMPTS, sampling_params=single_sampling_params)
140 + assert len(PROMPTS) == len(outputs)
141
142 # sampling_params is None, default params should be applied
40 - outputs = llm.generate(prompts, sampling_params=None)
41 - assert len(prompts) == len(outputs)
\ No newline at end of file
143 + outputs = llm.generate(PROMPTS, sampling_params=None)
144 + assert len(PROMPTS) == len(outputs)
tests/entrypoints/test_openai_server.py
+32 -2
@@ -71,7 +71,7 @@ TEST_CHOICE = [
71 "Swift", "Kotlin"
72 ]
73
74 -pytestmark = pytest.mark.asyncio
74 +pytestmark = pytest.mark.openai
75
76
77 @pytest.fixture(scope="session")
@@ -91,6 +91,8 @@ def server(zephyr_lora_files):
91 "--max-model-len",
92 "8192",
93 "--enforce-eager",
94 + "--gpu-memory-utilization",
95 + "0.75",
96 # lora config below
97 "--enable-lora",
98 "--lora-modules",
@@ -118,9 +120,11 @@ def embedding_server(zephyr_lora_files):
120 # use half precision for speed and memory savings in CI environment
121 "--dtype",
122 "bfloat16",
123 + "--enforce-eager",
124 + "--gpu-memory-utilization",
125 + "0.75",
126 "--max-model-len",
127 "8192",
123 - "--enforce-eager",
128 ])
129 ray.get(server_runner.ready.remote())
130 yield server_runner
@@ -136,6 +140,7 @@ def client():
140 yield client
141
142
143 +@pytest.mark.asyncio
144 async def test_check_models(server, client: openai.AsyncOpenAI):
145 models = await client.models.list()
146 models = models.data
@@ -147,6 +152,7 @@ async def test_check_models(server, client: openai.AsyncOpenAI):
152 assert lora_models[1].id == "zephyr-lora2"
153
154
155 +@pytest.mark.asyncio
156 @pytest.mark.parametrize(
157 # first test base model, then test loras
158 "model_name",
@@ -178,6 +184,7 @@ async def test_single_completion(server, client: openai.AsyncOpenAI,
184 completion.choices[0].text) >= 5
185
186
187 +@pytest.mark.asyncio
188 @pytest.mark.parametrize(
189 # first test base model, then test loras
190 "model_name",
@@ -199,6 +206,7 @@ async def test_zero_logprobs(server, client: openai.AsyncOpenAI,
206 assert choice.logprobs.top_logprobs is None
207
208
209 +@pytest.mark.asyncio
210 @pytest.mark.parametrize(
211 # just test 1 lora hereafter
212 "model_name",
@@ -243,6 +251,7 @@ async def test_single_chat_session(server, client: openai.AsyncOpenAI,
251 assert message.content is not None and len(message.content) >= 0
252
253
254 +@pytest.mark.asyncio
255 @pytest.mark.parametrize("model_name", [MODEL_NAME])
256 async def test_too_many_logprobs(server, client: openai.AsyncOpenAI,
257 model_name: str):
@@ -298,6 +307,7 @@ async def test_too_many_logprobs(server, client: openai.AsyncOpenAI,
307 assert message.content is not None and len(message.content) >= 0
308
309
310 +@pytest.mark.asyncio
311 @pytest.mark.parametrize(
312 # just test 1 lora hereafter
313 "model_name",
@@ -335,6 +345,7 @@ async def test_completion_streaming(server, client: openai.AsyncOpenAI,
345 assert "".join(chunks) == single_output
346
347
348 +@pytest.mark.asyncio
349 @pytest.mark.parametrize(
350 # just test 1 lora hereafter
351 "model_name",
@@ -385,6 +396,7 @@ async def test_chat_streaming(server, client: openai.AsyncOpenAI,
396 assert "".join(chunks) == output
397
398
399 +@pytest.mark.asyncio
400 @pytest.mark.parametrize(
401 # just test 1 lora hereafter
402 "model_name",
@@ -438,6 +450,7 @@ async def test_batch_completions(server, client: openai.AsyncOpenAI,
450 assert texts[0] == texts[1]
451
452
453 +@pytest.mark.asyncio
454 async def test_logits_bias(server, client: openai.AsyncOpenAI):
455 prompt = "Hello, my name is"
456 max_tokens = 5
@@ -485,6 +498,7 @@ async def test_logits_bias(server, client: openai.AsyncOpenAI):
498 assert first_response != completion.choices[0].text
499
500
501 +@pytest.mark.asyncio
502 @pytest.mark.parametrize("guided_decoding_backend",
503 ["outlines", "lm-format-enforcer"])
504 async def test_guided_json_completion(server, client: openai.AsyncOpenAI,
@@ -507,6 +521,7 @@ async def test_guided_json_completion(server, client: openai.AsyncOpenAI,
521 jsonschema.validate(instance=output_json, schema=TEST_SCHEMA)
522
523
524 +@pytest.mark.asyncio
525 @pytest.mark.parametrize("guided_decoding_backend",
526 ["outlines", "lm-format-enforcer"])
527 async def test_guided_json_chat(server, client: openai.AsyncOpenAI,
@@ -553,6 +568,7 @@ async def test_guided_json_chat(server, client: openai.AsyncOpenAI,
568 assert json1["age"] != json2["age"]
569
570
571 +@pytest.mark.asyncio
572 @pytest.mark.parametrize("guided_decoding_backend",
573 ["outlines", "lm-format-enforcer"])
574 async def test_guided_regex_completion(server, client: openai.AsyncOpenAI,
@@ -573,6 +589,7 @@ async def test_guided_regex_completion(server, client: openai.AsyncOpenAI,
589 assert re.fullmatch(TEST_REGEX, completion.choices[i].text) is not None
590
591
592 +@pytest.mark.asyncio
593 @pytest.mark.parametrize("guided_decoding_backend",
594 ["outlines", "lm-format-enforcer"])
595 async def test_guided_regex_chat(server, client: openai.AsyncOpenAI,
@@ -610,6 +627,7 @@ async def test_guided_regex_chat(server, client: openai.AsyncOpenAI,
627 assert ip1 != ip2
628
629
630 +@pytest.mark.asyncio
631 @pytest.mark.parametrize("guided_decoding_backend",
632 ["outlines", "lm-format-enforcer"])
633 async def test_guided_choice_completion(server, client: openai.AsyncOpenAI,
@@ -629,6 +647,7 @@ async def test_guided_choice_completion(server, client: openai.AsyncOpenAI,
647 assert completion.choices[i].text in TEST_CHOICE
648
649
650 +@pytest.mark.asyncio
651 @pytest.mark.parametrize("guided_decoding_backend",
652 ["outlines", "lm-format-enforcer"])
653 async def test_guided_choice_chat(server, client: openai.AsyncOpenAI,
@@ -667,6 +686,7 @@ async def test_guided_choice_chat(server, client: openai.AsyncOpenAI,
686 assert choice1 != choice2
687
688
689 +@pytest.mark.asyncio
690 @pytest.mark.parametrize("guided_decoding_backend",
691 ["outlines", "lm-format-enforcer"])
692 async def test_guided_decoding_type_error(server, client: openai.AsyncOpenAI,
@@ -702,6 +722,7 @@ async def test_guided_decoding_type_error(server, client: openai.AsyncOpenAI,
722 extra_body=dict(guided_regex=TEST_REGEX, guided_json=TEST_SCHEMA))
723
724
725 +@pytest.mark.asyncio
726 @pytest.mark.parametrize("guided_decoding_backend",
727 ["outlines", "lm-format-enforcer"])
728 async def test_guided_choice_chat_logprobs(server, client: openai.AsyncOpenAI,
@@ -732,6 +753,7 @@ async def test_guided_choice_chat_logprobs(server, client: openai.AsyncOpenAI,
753 for token, logprob in token_dict.items())
754
755
756 +@pytest.mark.asyncio
757 async def test_response_format_json_object(server, client: openai.AsyncOpenAI):
758 for _ in range(2):
759 resp = await client.chat.completions.create(
@@ -749,6 +771,7 @@ async def test_response_format_json_object(server, client: openai.AsyncOpenAI):
771 assert loaded == {"result": 2}, loaded
772
773
774 +@pytest.mark.asyncio
775 async def test_extra_fields(server, client: openai.AsyncOpenAI):
776 with pytest.raises(BadRequestError) as exc_info:
777 await client.chat.completions.create(
@@ -764,6 +787,7 @@ async def test_extra_fields(server, client: openai.AsyncOpenAI):
787 assert "extra_forbidden" in exc_info.value.message
788
789
790 +@pytest.mark.asyncio
791 async def test_complex_message_content(server, client: openai.AsyncOpenAI):
792 resp = await client.chat.completions.create(
793 model=MODEL_NAME,
@@ -783,6 +807,7 @@ async def test_complex_message_content(server, client: openai.AsyncOpenAI):
807 assert content == "2"
808
809
810 +@pytest.mark.asyncio
811 async def test_custom_role(server, client: openai.AsyncOpenAI):
812 # Not sure how the model handles custom roles so we just check that
813 # both string and complex message content are handled in the same way
@@ -813,6 +838,7 @@ async def test_custom_role(server, client: openai.AsyncOpenAI):
838 assert content1 == content2
839
840
841 +@pytest.mark.asyncio
842 async def test_guided_grammar(server, client: openai.AsyncOpenAI):
843 simple_sql_grammar = """
844 start: select_statement
@@ -847,6 +873,7 @@ number: "1" | "2"
873 assert content.strip() == ground_truth
874
875
876 +@pytest.mark.asyncio
877 @pytest.mark.parametrize(
878 # first test base model, then test loras
879 "model_name",
@@ -878,6 +905,7 @@ async def test_echo_logprob_completion(server, client: openai.AsyncOpenAI,
905 assert len(logprobs.tokens) > 5
906
907
908 +@pytest.mark.asyncio
909 async def test_long_seed(server, client: openai.AsyncOpenAI):
910 for seed in [
911 torch.iinfo(torch.long).min - 1,
@@ -897,6 +925,7 @@ async def test_long_seed(server, client: openai.AsyncOpenAI):
925 or "less_than_equal" in exc_info.value.message)
926
927
928 +@pytest.mark.asyncio
929 @pytest.mark.parametrize(
930 "model_name",
931 [EMBEDDING_MODEL_NAME],
@@ -935,6 +964,7 @@ async def test_single_embedding(embedding_server, client: openai.AsyncOpenAI,
964 assert embeddings.usage.total_tokens == 5
965
966
967 +@pytest.mark.asyncio
968 @pytest.mark.parametrize(
969 "model_name",
970 [EMBEDDING_MODEL_NAME],
tests/entrypoints/test_server_oot_registration.py
+7 -4
@@ -1,7 +1,7 @@
1 -import multiprocessing
1 import sys
2 import time
3
4 +import pytest
5 import torch
6 from openai import OpenAI, OpenAIError
7
@@ -10,6 +10,8 @@ from vllm.model_executor.models.opt import OPTForCausalLM
10 from vllm.model_executor.sampling_metadata import SamplingMetadata
11 from vllm.utils import get_open_port
12
13 +pytestmark = pytest.mark.openai
14 +
15
16 class MyOPTForCausalLM(OPTForCausalLM):
17
@@ -26,15 +28,16 @@ def server_function(port):
28 # register our dummy model
29 ModelRegistry.register_model("OPTForCausalLM", MyOPTForCausalLM)
30 sys.argv = ["placeholder.py"] + \
29 - ("--model facebook/opt-125m --dtype"
30 - f" float32 --api-key token-abc123 --port {port}").split()
31 + ("--model facebook/opt-125m --gpu-memory-utilization 0.10 "
32 + f"--dtype float32 --api-key token-abc123 --port {port}").split()
33 import runpy
34 runpy.run_module('vllm.entrypoints.openai.api_server', run_name='__main__')
35
36
37 def test_oot_registration_for_api_server():
38 port = get_open_port()
37 - server = multiprocessing.Process(target=server_function, args=(port, ))
39 + ctx = torch.multiprocessing.get_context()
40 + server = ctx.Process(target=server_function, args=(port, ))
41 server.start()
42 client = OpenAI(
43 base_url=f"http://localhost:{port}/v1",
tests/lora/test_long_context.py
+3 -5
@@ -86,20 +86,18 @@ def generate(
86
87
88 def batched_generate(
89 - llm,
89 + llm: vllm.LLM,
90 inputs: List[Tuple[str, SamplingParams, Optional[LoRARequest]]],
91 ):
92 for input in inputs:
93 prompt, sampling_param, lora_req = input
94 - requests_data = llm._validate_and_prepare_requests(
94 + # Add requests to the engine and run the engine
95 + llm._validate_and_add_requests(
96 prompt,
97 sampling_param,
98 lora_request=lora_req,
99 )
100
100 - # Add requests to the engine and run the engine
101 - for request_data in requests_data:
102 - llm._add_request(**request_data)
101 outputs = llm._run_engine(use_tqdm=True)
102 return [outputs[i].outputs[0].text.strip() for i in range(len(outputs))]
103
tests/samplers/test_logits_processor.py
+4 -7
@@ -35,28 +35,25 @@ def test_logits_processor_force_generate(
35
36 # test logits_processors when prompt_logprobs is not None
37 vllm_model.model._add_request(
38 - prompt=example_prompts[0],
38 + example_prompts[0],
39 params=params_with_logprobs,
40 - prompt_token_ids=None,
40 )
41
42 # test prompt_logprobs is not None
43 vllm_model.model._add_request(
45 - prompt=example_prompts[1],
44 + example_prompts[1],
45 params=SamplingParams(
46 prompt_logprobs=3,
47 max_tokens=max_tokens,
48 ),
50 - prompt_token_ids=None,
49 )
50
51 # test grouped requests
52 vllm_model.model._add_request(
55 - prompt=example_prompts[2],
53 + example_prompts[2],
54 params=SamplingParams(max_tokens=max_tokens),
57 - prompt_token_ids=None,
55 )
56
60 - outputs = vllm_model.model._run_engine(False)
57 + outputs = vllm_model.model._run_engine(use_tqdm=False)
58
59 assert outputs[0].outputs[0].text == enforced_answers * repeat_times
tests/samplers/test_seeded_generate.py
+1 -5
@@ -57,11 +57,7 @@ def test_random_sample_with_seed(
57 sampling_params_seed_1,
58 sampling_params_seed_2,
59 ):
60 - llm._add_request(
61 - prompt=prompt,
62 - prompt_token_ids=None,
63 - params=params,
64 - )
60 + llm._add_request(prompt, params=params)
61
62 results = llm._run_engine(use_tqdm=False)
63 all_outputs = [[out.token_ids for out in output.outputs]
tests/test_cache_block_hashing.py
+9 -2
@@ -70,8 +70,15 @@ def test_auto_prefix_caching(model: str, block_size: int, max_num_seqs: int,
70 for prompt in prompts:
71 hashes[-1].append([])
72 prompt_token_ids = tokenizer.encode(prompt)
73 - seq = Sequence(seq_id, prompt, prompt_token_ids, block_size,
74 - tokenizer.tokenizer.eos_token_id, lora_request)
73 + seq = Sequence(seq_id,
74 + inputs={
75 + "prompt": prompt,
76 + "prompt_token_ids": prompt_token_ids,
77 + "multi_modal_data": None,
78 + },
79 + block_size=block_size,
80 + eos_token_id=tokenizer.tokenizer.eos_token_id,
81 + lora_request=lora_request)
82
83 num_blocks = len(prompt_token_ids) // block_size
84 for idx in range(num_blocks):
tests/test_inputs.py new
+53
@@ -0,0 +1,53 @@
1 +from typing import List
2 +
3 +import pytest
4 +
5 +from vllm.inputs import parse_and_batch_prompt
6 +
7 +STRING_INPUTS = [
8 + '',
9 + 'foo',
10 + 'foo bar',
11 + 'foo baz bar',
12 + 'foo bar qux baz',
13 +]
14 +
15 +TOKEN_INPUTS = [
16 + [-1],
17 + [1],
18 + [1, 2],
19 + [1, 3, 4],
20 + [1, 2, 4, 3],
21 +]
22 +
23 +INPUTS_SLICES = [
24 + slice(None, None, -1),
25 + slice(None, None, 2),
26 + slice(None, None, -2),
27 +]
28 +
29 +
30 +def test_parse_single_batch_empty():
31 + with pytest.raises(ValueError, match="at least one prompt"):
32 + parse_and_batch_prompt([])
33 +
34 + with pytest.raises(ValueError, match="at least one prompt"):
35 + parse_and_batch_prompt([[]])
36 +
37 +
38 +@pytest.mark.parametrize('string_input', STRING_INPUTS)
39 +def test_parse_single_batch_string_consistent(string_input: str):
40 + assert parse_and_batch_prompt(string_input) \
41 + == parse_and_batch_prompt([string_input])
42 +
43 +
44 +@pytest.mark.parametrize('token_input', TOKEN_INPUTS)
45 +def test_parse_single_batch_token_consistent(token_input: List[int]):
46 + assert parse_and_batch_prompt(token_input) \
47 + == parse_and_batch_prompt([token_input])
48 +
49 +
50 +@pytest.mark.parametrize('inputs_slice', INPUTS_SLICES)
51 +def test_parse_single_batch_string_slice(inputs_slice: slice):
52 + assert parse_and_batch_prompt(STRING_INPUTS)[inputs_slice] \
53 + == parse_and_batch_prompt(STRING_INPUTS[inputs_slice])
tests/test_utils.py new
+63
@@ -0,0 +1,63 @@
1 +import pytest
2 +
3 +from vllm.utils import deprecate_kwargs
4 +
5 +from .utils import error_on_warning
6 +
7 +
8 +def test_deprecate_kwargs_always():
9 +
10 + @deprecate_kwargs("old_arg", is_deprecated=True)
11 + def dummy(*, old_arg: object = None, new_arg: object = None):
12 + pass
13 +
14 + with pytest.warns(DeprecationWarning, match="'old_arg'"):
15 + dummy(old_arg=1)
16 +
17 + with error_on_warning():
18 + dummy(new_arg=1)
19 +
20 +
21 +def test_deprecate_kwargs_never():
22 +
23 + @deprecate_kwargs("old_arg", is_deprecated=False)
24 + def dummy(*, old_arg: object = None, new_arg: object = None):
25 + pass
26 +
27 + with error_on_warning():
28 + dummy(old_arg=1)
29 +
30 + with error_on_warning():
31 + dummy(new_arg=1)
32 +
33 +
34 +def test_deprecate_kwargs_dynamic():
35 + is_deprecated = True
36 +
37 + @deprecate_kwargs("old_arg", is_deprecated=lambda: is_deprecated)
38 + def dummy(*, old_arg: object = None, new_arg: object = None):
39 + pass
40 +
41 + with pytest.warns(DeprecationWarning, match="'old_arg'"):
42 + dummy(old_arg=1)
43 +
44 + with error_on_warning():
45 + dummy(new_arg=1)
46 +
47 + is_deprecated = False
48 +
49 + with error_on_warning():
50 + dummy(old_arg=1)
51 +
52 + with error_on_warning():
53 + dummy(new_arg=1)
54 +
55 +
56 +def test_deprecate_kwargs_additional_message():
57 +
58 + @deprecate_kwargs("old_arg", is_deprecated=True, additional_message="abcd")
59 + def dummy(*, old_arg: object = None, new_arg: object = None):
60 + pass
61 +
62 + with pytest.warns(DeprecationWarning, match="abcd"):
63 + dummy(old_arg=1)
tests/tokenization/test_detokenize.py
+5 -2
@@ -123,8 +123,11 @@ def create_sequence(prompt_token_ids=None):
123 prompt_token_ids = prompt_token_ids or [1]
124 return Sequence(
125 seq_id=0,
126 - prompt="<s>",
127 - prompt_token_ids=prompt_token_ids,
126 + inputs={
127 + "prompt": "<s>",
128 + "prompt_token_ids": prompt_token_ids,
129 + "multi_modal_data": None,
130 + },
131 block_size=16,
132 )
133
tests/utils.py
+14
@@ -2,6 +2,8 @@ import os
2 import subprocess
3 import sys
4 import time
5 +import warnings
6 +from contextlib import contextmanager
7
8 import ray
9 import requests
@@ -87,3 +89,15 @@ def multi_process_tensor_parallel(
89 ray.get(refs)
90
91 ray.shutdown()
92 +
93 +
94 +@contextmanager
95 +def error_on_warning():
96 + """
97 + Within the scope of this context manager, tests will fail if any warning
98 + is emitted.
99 + """
100 + with warnings.catch_warnings():
101 + warnings.simplefilter("error")
102 +
103 + yield
vllm/__init__.py
+4
@@ -5,6 +5,7 @@ from vllm.engine.async_llm_engine import AsyncLLMEngine
5 from vllm.engine.llm_engine import LLMEngine
6 from vllm.entrypoints.llm import LLM
7 from vllm.executor.ray_utils import initialize_ray_cluster
8 +from vllm.inputs import PromptStrictInputs, TextPrompt, TokensPrompt
9 from vllm.model_executor.models import ModelRegistry
10 from vllm.outputs import (CompletionOutput, EmbeddingOutput,
11 EmbeddingRequestOutput, RequestOutput)
@@ -16,6 +17,9 @@ __version__ = "0.4.2"
17 __all__ = [
18 "LLM",
19 "ModelRegistry",
20 + "PromptStrictInputs",
21 + "TextPrompt",
22 + "TokensPrompt",
23 "SamplingParams",
24 "RequestOutput",
25 "CompletionOutput",
vllm/engine/async_llm_engine.py
+81 -90
@@ -12,12 +12,13 @@ from vllm.core.scheduler import SchedulerOutputs
12 from vllm.engine.arg_utils import AsyncEngineArgs
13 from vllm.engine.llm_engine import LLMEngine
14 from vllm.executor.ray_utils import initialize_ray_cluster, ray
15 +from vllm.inputs import LLMInputs, PromptInputs
16 from vllm.logger import init_logger
17 from vllm.lora.request import LoRARequest
18 from vllm.outputs import EmbeddingRequestOutput, RequestOutput
19 from vllm.pooling_params import PoolingParams
20 from vllm.sampling_params import SamplingParams
20 -from vllm.sequence import ExecuteModelRequest, MultiModalData, SamplerOutput
21 +from vllm.sequence import ExecuteModelRequest, SamplerOutput
22 from vllm.usage.usage_lib import UsageContext
23
24 logger = init_logger(__name__)
@@ -244,64 +245,69 @@ class _AsyncLLMEngine(LLMEngine):
245
246 return request_outputs
247
247 - async def encode_request_async(
248 + async def process_model_inputs_async(
249 self,
249 - request_id: str, # pylint: disable=unused-argument
250 - prompt: Optional[str],
251 - prompt_token_ids: Optional[List[int]] = None,
250 + request_id: str,
251 + inputs: PromptInputs,
252 lora_request: Optional[LoRARequest] = None,
253 - ):
254 - if prompt_token_ids is None:
255 - assert prompt is not None
256 - prompt_token_ids = await self.tokenizer.encode_async(
253 + ) -> LLMInputs:
254 + if isinstance(inputs, str):
255 + inputs = {"prompt": inputs}
256 +
257 + if "prompt_token_ids" not in inputs:
258 + tokenizer = self.get_tokenizer_group("prompts must be None if "
259 + "skip_tokenizer_init is True")
260 +
261 + prompt_token_ids = await tokenizer.encode_async(
262 request_id=request_id,
258 - prompt=prompt,
263 + prompt=inputs["prompt"],
264 lora_request=lora_request)
260 - return prompt_token_ids
265 + else:
266 + prompt_token_ids = inputs["prompt_token_ids"]
267 +
268 + return LLMInputs(prompt_token_ids=prompt_token_ids,
269 + prompt=inputs.get("prompt"),
270 + multi_modal_data=inputs.get("multi_modal_data"))
271
272 async def add_request_async(
273 self,
274 request_id: str,
265 - prompt: Optional[str],
275 + inputs: PromptInputs,
276 params: Union[SamplingParams, PoolingParams],
267 - prompt_token_ids: Optional[List[int]] = None,
277 arrival_time: Optional[float] = None,
278 lora_request: Optional[LoRARequest] = None,
270 - multi_modal_data: Optional[MultiModalData] = None,
279 ) -> None:
280 if lora_request is not None and not self.lora_config:
281 raise ValueError(f"Got lora_request {lora_request} but LoRA is "
282 "not enabled!")
283 if arrival_time is None:
284 arrival_time = time.time()
277 - prompt_token_ids = await self.encode_request_async(
285 +
286 + processed_inputs = await self.process_model_inputs_async(
287 + request_id=request_id, inputs=inputs, lora_request=lora_request)
288 +
289 + self._add_processed_request(
290 request_id=request_id,
279 - prompt=prompt,
280 - prompt_token_ids=prompt_token_ids,
281 - lora_request=lora_request)
282 -
283 - return self.add_request(request_id,
284 - prompt=prompt,
285 - params=params,
286 - prompt_token_ids=prompt_token_ids,
287 - arrival_time=arrival_time,
288 - lora_request=lora_request,
289 - multi_modal_data=multi_modal_data)
291 + processed_inputs=processed_inputs,
292 + params=params,
293 + arrival_time=arrival_time,
294 + lora_request=lora_request,
295 + )
296
297 async def check_health_async(self) -> None:
298 self.model_executor.check_health()
299
300
301 class AsyncLLMEngine:
296 - """An asynchronous wrapper for LLMEngine.
302 + """An asynchronous wrapper for :class:`LLMEngine`.
303
298 - This class is used to wrap the LLMEngine class to make it asynchronous. It
299 - uses asyncio to create a background loop that keeps processing incoming
300 - requests. The LLMEngine is kicked by the generate method when there
301 - are requests in the waiting queue. The generate method yields the outputs
302 - from the LLMEngine to the caller.
304 + This class is used to wrap the :class:`LLMEngine` class to make it
305 + asynchronous. It uses asyncio to create a background loop that keeps
306 + processing incoming requests. The :class:`LLMEngine` is kicked by the
307 + generate method when there are requests in the waiting queue. The generate
308 + method yields the outputs from the :class:`LLMEngine` to the caller.
309
304 - NOTE: For the comprehensive list of arguments, see `LLMEngine`.
310 + NOTE: For the comprehensive list of arguments, see :class:`LLMEngine`.
311
312 Args:
313 worker_use_ray: Whether to use Ray for model workers. Required for
@@ -315,8 +321,8 @@ class AsyncLLMEngine:
321 being printed in log.
322 start_engine_loop: If True, the background task to run the engine
323 will be automatically started in the generate call.
318 - *args: Arguments for LLMEngine.
319 - *kwargs: Arguments for LLMEngine.
324 + *args: Arguments for :class:`LLMEngine`.
325 + **kwargs: Arguments for :class:`LLMEngine`.
326 """
327
328 _engine_class: Type[_AsyncLLMEngine] = _AsyncLLMEngine
@@ -526,22 +532,26 @@ class AsyncLLMEngine:
532 async def add_request(
533 self,
534 request_id: str,
529 - prompt: Optional[str],
535 + inputs: PromptInputs,
536 params: Union[SamplingParams, PoolingParams],
531 - prompt_token_ids: Optional[List[int]] = None,
537 arrival_time: Optional[float] = None,
538 lora_request: Optional[LoRARequest] = None,
534 - multi_modal_data: Optional[MultiModalData] = None,
539 ) -> AsyncStream:
540 if self.log_requests:
537 - shortened_prompt = prompt
538 - shortened_token_ids = prompt_token_ids
539 - if self.max_log_len is not None:
541 + if isinstance(inputs, str):
542 + shortened_prompt = inputs
543 + shortened_token_ids = None
544 + else:
545 + shortened_prompt = inputs.get("prompt")
546 + shortened_token_ids = inputs.get("prompt_token_ids")
547 +
548 + max_log_len = self.max_log_len
549 + if max_log_len is not None:
550 if shortened_prompt is not None:
541 - shortened_prompt = shortened_prompt[:self.max_log_len]
551 + shortened_prompt = shortened_prompt[:max_log_len]
552 if shortened_token_ids is not None:
543 - shortened_token_ids = shortened_token_ids[:self.
544 - max_log_len]
553 + shortened_token_ids = shortened_token_ids[:max_log_len]
554 +
555 logger.info(
556 "Received request %s: prompt: %r, "
557 "params: %s, prompt_token_ids: %s, "
@@ -562,39 +572,33 @@ class AsyncLLMEngine:
572 arrival_time = time.time()
573
574 if self.engine_use_ray:
565 - prompt_token_ids = await (
566 - self.engine.encode_request_async.remote( # type: ignore
575 + processed_inputs = await self.engine.process_model_inputs_async \
576 + .remote( # type: ignore
577 request_id=request_id,
568 - prompt=prompt,
569 - prompt_token_ids=prompt_token_ids,
570 - lora_request=lora_request))
578 + inputs=inputs,
579 + lora_request=lora_request)
580 else:
572 - prompt_token_ids = await self.engine.encode_request_async(
581 + processed_inputs = await self.engine.process_model_inputs_async(
582 request_id=request_id,
574 - prompt=prompt,
575 - prompt_token_ids=prompt_token_ids,
583 + inputs=inputs,
584 lora_request=lora_request)
585
586 stream = self._request_tracker.add_request(
587 request_id,
580 - prompt=prompt,
588 + inputs=processed_inputs,
589 params=params,
582 - prompt_token_ids=prompt_token_ids,
590 arrival_time=arrival_time,
591 lora_request=lora_request,
585 - multi_modal_data=multi_modal_data,
592 )
593
594 return stream
595
596 async def generate(
597 self,
592 - prompt: Optional[str],
598 + inputs: PromptInputs,
599 sampling_params: SamplingParams,
600 request_id: str,
595 - prompt_token_ids: Optional[List[int]] = None,
601 lora_request: Optional[LoRARequest] = None,
597 - multi_modal_data: Optional[MultiModalData] = None
602 ) -> AsyncIterator[RequestOutput]:
603 """Generate outputs for a request.
604
@@ -603,14 +607,12 @@ class AsyncLLMEngine:
607 from the LLMEngine to the caller.
608
609 Args:
606 - prompt: The prompt string. Can be None if prompt_token_ids is
607 - provided.
610 + inputs: The inputs to the LLM. See
611 + :class:`~vllm.inputs.PromptInputs`
612 + for more details about the format of each input.
613 sampling_params: The sampling parameters of the request.
614 request_id: The unique id of the request.
610 - prompt_token_ids: The token IDs of the prompt. If None, we
611 - use the tokenizer to convert the prompts to token IDs.
615 lora_request: LoRA request to use for generation, if any.
613 - multi_modal_data: Multi modal data per request.
616
617 Yields:
618 The output `RequestOutput` objects from the LLMEngine
@@ -659,24 +661,20 @@ class AsyncLLMEngine:
661 >>> # Process and return the final output
662 >>> ...
663 """
662 - async for output in self.process_request(
664 + async for output in self._process_request(
665 request_id,
664 - prompt,
666 + inputs,
667 sampling_params,
666 - prompt_token_ids,
667 - lora_request,
668 - multi_modal_data,
668 + lora_request=lora_request,
669 ):
670 - yield output
670 + yield LLMEngine.validate_output(output, RequestOutput)
671
672 async def encode(
673 self,
674 - prompt: Optional[str],
674 + inputs: PromptInputs,
675 pooling_params: PoolingParams,
676 request_id: str,
677 - prompt_token_ids: Optional[List[int]] = None,
677 lora_request: Optional[LoRARequest] = None,
679 - multi_modal_data: Optional[MultiModalData] = None
678 ) -> AsyncIterator[EmbeddingRequestOutput]:
679 """Generate outputs for a request from an embedding model.
680
@@ -685,14 +683,12 @@ class AsyncLLMEngine:
683 from the LLMEngine to the caller.
684
685 Args:
688 - prompt: The prompt string. Can be None if prompt_token_ids is
689 - provided.
686 + inputs: The inputs to the LLM. See
687 + :class:`~vllm.inputs.PromptInputs`
688 + for more details about the format of each input.
689 pooling_params: The pooling parameters of the request.
690 request_id: The unique id of the request.
692 - prompt_token_ids: The token IDs of the prompt. If None, we
693 - use the tokenizer to convert the prompts to token IDs.
691 lora_request: LoRA request to use for generation, if any.
695 - multi_modal_data: Multi modal data per request.
692
693 Yields:
694 The output `EmbeddingRequestOutput` objects from the LLMEngine
@@ -739,24 +735,21 @@ class AsyncLLMEngine:
735 >>> # Process and return the final output
736 >>> ...
737 """
742 - async for output in self.process_request(
738 + async for output in self._process_request(
739 request_id,
744 - prompt,
740 + inputs,
741 pooling_params,
746 - prompt_token_ids,
747 - lora_request,
748 - multi_modal_data,
742 + lora_request=lora_request,
743 ):
750 - yield output
744 + yield LLMEngine.validate_output(output, EmbeddingRequestOutput)
745
752 - async def process_request(
746 + async def _process_request(
747 self,
748 request_id: str,
755 - prompt: Optional[str],
749 + inputs: PromptInputs,
750 params: Union[SamplingParams, PoolingParams],
757 - prompt_token_ids: Optional[List[int]] = None,
751 + *,
752 lora_request: Optional[LoRARequest] = None,
759 - multi_modal_data: Optional[MultiModalData] = None,
753 ) -> AsyncIterator[Union[RequestOutput, EmbeddingRequestOutput]]:
754 """Common logic to process requests with SamplingParams or
755 PoolingParams."""
@@ -764,12 +757,10 @@ class AsyncLLMEngine:
757
758 stream = await self.add_request(
759 request_id,
767 - prompt,
760 + inputs,
761 params,
769 - prompt_token_ids=prompt_token_ids,
762 arrival_time=arrival_time,
763 lora_request=lora_request,
772 - multi_modal_data=multi_modal_data,
764 )
765
766 try:
vllm/engine/llm_engine.py
+179 -90
@@ -1,5 +1,8 @@
1 import time
2 -from typing import Iterable, List, Optional, Type, Union
2 +from contextlib import contextmanager
3 +from typing import TYPE_CHECKING, ClassVar, Iterable, List, Optional
4 +from typing import Sequence as GenericSequence
5 +from typing import Type, TypeVar, Union
6
7 from transformers import GenerationConfig, PreTrainedTokenizer
8
@@ -18,6 +21,7 @@ from vllm.engine.output_processor.stop_checker import StopChecker
21 from vllm.engine.output_processor.util import create_output_by_sequence_group
22 from vllm.executor.executor_base import ExecutorBase
23 from vllm.executor.ray_utils import initialize_ray_cluster
24 +from vllm.inputs import LLMInputs, PromptInputs
25 from vllm.logger import init_logger
26 from vllm.lora.request import LoRARequest
27 from vllm.outputs import (EmbeddingRequestOutput, RequestOutput,
@@ -25,8 +29,8 @@ from vllm.outputs import (EmbeddingRequestOutput, RequestOutput,
29 from vllm.pooling_params import PoolingParams
30 from vllm.sampling_params import SamplingParams
31 from vllm.sequence import (EmbeddingSequenceGroupOutput, ExecuteModelRequest,
28 - MultiModalData, PoolerOutput, SamplerOutput,
29 - Sequence, SequenceGroup, SequenceGroupMetadata,
32 + PoolerOutput, SamplerOutput, Sequence,
33 + SequenceGroup, SequenceGroupMetadata,
34 SequenceStatus)
35 from vllm.transformers_utils.detokenizer import Detokenizer
36 from vllm.transformers_utils.tokenizer_group import (BaseTokenizerGroup,
@@ -50,6 +54,9 @@ def _load_generation_config_dict(model_config: ModelConfig):
54 return {}
55
56
57 +_O = TypeVar("_O", RequestOutput, EmbeddingRequestOutput)
58 +
59 +
60 class LLMEngine:
61 """An LLM engine that receives requests and generates texts.
62
@@ -60,11 +67,11 @@ class LLMEngine:
67 iteration-level scheduling and efficient memory management to maximize the
68 serving throughput.
69
63 - The `LLM` class wraps this class for offline batched inference and the
64 - `AsyncLLMEngine` class wraps this class for online serving.
70 + The :class:`~vllm.LLM` class wraps this class for offline batched inference
71 + and the :class:`AsyncLLMEngine` class wraps this class for online serving.
72
66 - NOTE: The config arguments are derived from the `EngineArgs` class. For the
67 - comprehensive list of arguments, see `EngineArgs`.
73 + NOTE: The config arguments are derived from the :class:`~vllm.EngineArgs`
74 + class. For the comprehensive list of arguments, see :ref:`engine_args`.
75
76 Args:
77 model_config: The configuration related to the LLM model.
@@ -81,9 +88,60 @@ class LLMEngine:
88 executor_class: The model executor class for managing distributed
89 execution.
90 log_stats: Whether to log statistics.
84 - usage_context: Specified entry point, used for usage info collection
91 + usage_context: Specified entry point, used for usage info collection.
92 """
93
94 + DO_VALIDATE_OUTPUT: ClassVar[bool] = False
95 + """A flag to toggle whether to validate the type of request output."""
96 +
97 + @classmethod
98 + @contextmanager
99 + def enable_output_validation(cls):
100 + cls.DO_VALIDATE_OUTPUT = True
101 +
102 + yield
103 +
104 + cls.DO_VALIDATE_OUTPUT = False
105 +
106 + @classmethod
107 + def validate_output(
108 + cls,
109 + output: object,
110 + output_type: Type[_O],
111 + ) -> _O:
112 + do_validate = cls.DO_VALIDATE_OUTPUT
113 +
114 + if ((TYPE_CHECKING or do_validate)
115 + and not isinstance(output, output_type)):
116 + raise TypeError(f"Expected output of type {output_type}, "
117 + f"but found type {type(output)}")
118 +
119 + return output
120 +
121 + @classmethod
122 + def validate_outputs(
123 + cls,
124 + outputs: GenericSequence[object],
125 + output_type: Type[_O],
126 + ) -> List[_O]:
127 + do_validate = cls.DO_VALIDATE_OUTPUT
128 +
129 + outputs_: List[_O]
130 + if TYPE_CHECKING or do_validate:
131 + outputs_ = []
132 + for output in outputs:
133 + if not isinstance(output, output_type):
134 + raise TypeError(f"Expected output of type {output_type}, "
135 + f"but found type {type(output)}")
136 +
137 + outputs_.append(output)
138 + else:
139 + outputs_ = outputs
140 +
141 + return outputs_
142 +
143 + tokenizer: Optional[BaseTokenizerGroup]
144 +
145 def __init__(
146 self,
147 model_config: ModelConfig,
@@ -151,12 +209,11 @@ class LLMEngine:
209 self.log_stats = log_stats
210
211 if not self.model_config.skip_tokenizer_init:
154 - self.tokenizer: BaseTokenizerGroup
155 - self._init_tokenizer()
212 + self.tokenizer = self._init_tokenizer()
213 self.detokenizer = Detokenizer(self.tokenizer)
214 else:
158 - self.detokenizer = None
215 self.tokenizer = None
216 + self.detokenizer = None
217
218 self.seq_counter = Counter()
219 self.generation_config_fields = _load_generation_config_dict(
@@ -318,14 +375,26 @@ class LLMEngine:
375 if model_executor := getattr(self, "model_executor", None):
376 model_executor.shutdown()
377
378 + MISSING_TOKENIZER_GROUP_MSG = ("Unable to get tokenizer because "
379 + "skip_tokenizer_init is True")
380 +
381 + def get_tokenizer_group(
382 + self,
383 + fail_msg: str = MISSING_TOKENIZER_GROUP_MSG) -> BaseTokenizerGroup:
384 + if self.tokenizer is None:
385 + raise ValueError(fail_msg)
386 +
387 + return self.tokenizer
388 +
389 def get_tokenizer(self) -> "PreTrainedTokenizer":
322 - return self.tokenizer.get_lora_tokenizer(None)
390 + return self.get_tokenizer_group().get_lora_tokenizer(None)
391
392 def get_tokenizer_for_seq(self,
393 sequence: Sequence) -> "PreTrainedTokenizer":
326 - return self.tokenizer.get_lora_tokenizer(sequence.lora_request)
394 + return self.get_tokenizer_group().get_lora_tokenizer(
395 + sequence.lora_request)
396
328 - def _init_tokenizer(self, **tokenizer_init_kwargs):
397 + def _init_tokenizer(self, **tokenizer_init_kwargs) -> BaseTokenizerGroup:
398 init_kwargs = dict(
399 tokenizer_id=self.model_config.tokenizer,
400 enable_lora=bool(self.lora_config),
@@ -335,8 +404,9 @@ class LLMEngine:
404 trust_remote_code=self.model_config.trust_remote_code,
405 revision=self.model_config.tokenizer_revision)
406 init_kwargs.update(tokenizer_init_kwargs)
338 - self.tokenizer = get_tokenizer_group(
339 - self.parallel_config.tokenizer_pool_config, **init_kwargs)
407 +
408 + return get_tokenizer_group(self.parallel_config.tokenizer_pool_config,
409 + **init_kwargs)
410
411 def _verify_args(self) -> None:
412 self.model_config.verify_with_parallel_config(self.parallel_config)
@@ -346,29 +416,85 @@ class LLMEngine:
416 self.lora_config.verify_with_scheduler_config(
417 self.scheduler_config)
418
349 - def encode_request(
419 + def _get_eos_token_id(
420 + self, lora_request: Optional[LoRARequest]) -> Optional[int]:
421 + if self.tokenizer is None:
422 + logger.warning("Using None for EOS token id because tokenizer "
423 + "is not initialized")
424 + return None
425 +
426 + return self.tokenizer.get_lora_tokenizer(lora_request).eos_token_id
427 +
428 + def _add_processed_request(
429 self,
351 - request_id: str, # pylint: disable=unused-argument
352 - prompt: Optional[str],
353 - prompt_token_ids: Optional[List[int]] = None,
430 + request_id: str,
431 + processed_inputs: LLMInputs,
432 + params: Union[SamplingParams, PoolingParams],
433 + arrival_time: float,
434 + lora_request: Optional[LoRARequest],
435 + ) -> None:
436 + # Create the sequences.
437 + block_size = self.cache_config.block_size
438 + seq_id = next(self.seq_counter)
439 + eos_token_id = self._get_eos_token_id(lora_request)
440 +
441 + seq = Sequence(seq_id, processed_inputs, block_size, eos_token_id,
442 + lora_request)
443 +
444 + # Create a SequenceGroup based on SamplingParams or PoolingParams
445 + if isinstance(params, SamplingParams):
446 + seq_group = self._create_sequence_group_with_sampling(
447 + request_id,
448 + seq,
449 + params,
450 + arrival_time=arrival_time,
451 + lora_request=lora_request,
452 + )
453 + elif isinstance(params, PoolingParams):
454 + seq_group = self._create_sequence_group_with_pooling(
455 + request_id,
456 + seq,
457 + params,
458 + arrival_time=arrival_time,
459 + lora_request=lora_request,
460 + )
461 + else:
462 + raise ValueError(
463 + "Either SamplingParams or PoolingParams must be provided.")
464 +
465 + # Add the sequence group to the scheduler.
466 + self.scheduler.add_seq_group(seq_group)
467 +
468 + def process_model_inputs(
469 + self,
470 + request_id: str,
471 + inputs: PromptInputs,
472 lora_request: Optional[LoRARequest] = None,
355 - ):
356 - if prompt_token_ids is None:
357 - assert prompt is not None
358 - prompt_token_ids = self.tokenizer.encode(request_id=request_id,
359 - prompt=prompt,
360 - lora_request=lora_request)
361 - return prompt_token_ids
473 + ) -> LLMInputs:
474 + if isinstance(inputs, str):
475 + inputs = {"prompt": inputs}
476 +
477 + if "prompt_token_ids" not in inputs:
478 + tokenizer = self.get_tokenizer_group("prompts must be None if "
479 + "skip_tokenizer_init is True")
480 +
481 + prompt_token_ids = tokenizer.encode(request_id=request_id,
482 + prompt=inputs["prompt"],
483 + lora_request=lora_request)
484 + else:
485 + prompt_token_ids = inputs["prompt_token_ids"]
486 +
487 + return LLMInputs(prompt_token_ids=prompt_token_ids,
488 + prompt=inputs.get("prompt"),
489 + multi_modal_data=inputs.get("multi_modal_data"))
490
491 def add_request(
492 self,
493 request_id: str,
366 - prompt: Optional[str],
494 + inputs: PromptInputs,
495 params: Union[SamplingParams, PoolingParams],
368 - prompt_token_ids: Optional[List[int]] = None,
496 arrival_time: Optional[float] = None,
497 lora_request: Optional[LoRARequest] = None,
371 - multi_modal_data: Optional[MultiModalData] = None,
498 ) -> None:
499 """Add a request to the engine's request pool.
500
@@ -378,15 +504,14 @@ class LLMEngine:
504
505 Args:
506 request_id: The unique ID of the request.
381 - prompt: The prompt string. Can be None if prompt_token_ids is
382 - provided.
383 - params: Parameters for sampling or pooling. SamplingParams
384 - for text generation. PoolingParams for pooling.
385 - prompt_token_ids: The token IDs of the prompt. If None, we
386 - use the tokenizer to convert the prompts to token IDs.
507 + inputs: The inputs to the LLM. See
508 + :class:`~vllm.inputs.PromptInputs`
509 + for more details about the format of each input.
510 + params: Parameters for sampling or pooling.
511 + :class:`~vllm.SamplingParams` for text generation.
512 + :class:`~vllm.PoolingParams` for pooling.
513 arrival_time: The arrival time of the request. If None, we use
514 the current monotonic time.
389 - multi_modal_data: Multi modal data per request.
515
516 Details:
517 - Set arrival_time to the current time if it is None.
@@ -417,59 +542,26 @@ class LLMEngine:
542 "not enabled!")
543 if arrival_time is None:
544 arrival_time = time.time()
420 - prompt_token_ids = self.encode_request(
421 - request_id=request_id,
422 - prompt=prompt,
423 - prompt_token_ids=prompt_token_ids,
424 - lora_request=lora_request)
425 -
426 - # Create the sequences.
427 - block_size = self.cache_config.block_size
428 - seq_id = next(self.seq_counter)
429 - eos_token_id = None
430 - if self.tokenizer:
431 - eos_token_id = self.tokenizer.get_lora_tokenizer(
432 - lora_request).eos_token_id
433 - else:
434 - logger.warning("Use None for EOS token id because tokenizer is "
435 - "not initialized")
436 - seq = Sequence(seq_id, prompt, prompt_token_ids, block_size,
437 - eos_token_id, lora_request)
545
439 - # Create a SequenceGroup based on SamplingParams or PoolingParams
440 - if isinstance(params, SamplingParams):
441 - seq_group = self._create_sequence_group_with_sampling(
442 - request_id,
443 - seq,
444 - params,
445 - arrival_time,
446 - lora_request,
447 - multi_modal_data,
448 - )
449 - elif isinstance(params, PoolingParams):
450 - seq_group = self._create_sequence_group_with_pooling(
451 - request_id,
452 - seq,
453 - params,
454 - arrival_time,
455 - lora_request,
456 - multi_modal_data,
457 - )
458 - else:
459 - raise ValueError(
460 - "Either SamplingParams or PoolingParams must be provided.")
546 + processed_inputs = self.process_model_inputs(request_id=request_id,
547 + inputs=inputs,
548 + lora_request=lora_request)
549
462 - # Add the sequence group to the scheduler.
463 - self.scheduler.add_seq_group(seq_group)
550 + self._add_processed_request(
551 + request_id=request_id,
552 + processed_inputs=processed_inputs,
553 + params=params,
554 + arrival_time=arrival_time,
555 + lora_request=lora_request,
556 + )
557
558 def _create_sequence_group_with_sampling(
559 self,
560 request_id: str,
561 seq: Sequence,
562 sampling_params: SamplingParams,
470 - arrival_time: Optional[float] = None,
471 - lora_request: Optional[LoRARequest] = None,
472 - multi_modal_data: Optional[MultiModalData] = None,
563 + arrival_time: float,
564 + lora_request: Optional[LoRARequest],
565 ) -> SequenceGroup:
566 """Creates a SequenceGroup with SamplingParams."""
567 max_logprobs = self.get_model_config().max_logprobs
@@ -495,8 +587,7 @@ class LLMEngine:
587 seqs=[seq],
588 arrival_time=arrival_time,
589 sampling_params=sampling_params,
498 - lora_request=lora_request,
499 - multi_modal_data=multi_modal_data)
590 + lora_request=lora_request)
591
592 return seq_group
593
@@ -505,9 +596,8 @@ class LLMEngine:
596 request_id: str,
597 seq: Sequence,
598 pooling_params: PoolingParams,
508 - arrival_time: Optional[float] = None,
509 - lora_request: Optional[LoRARequest] = None,
510 - multi_modal_data: Optional[MultiModalData] = None,
599 + arrival_time: float,
600 + lora_request: Optional[LoRARequest],
601 ) -> SequenceGroup:
602 """Creates a SequenceGroup with PoolingParams."""
603 # Defensive copy of PoolingParams, which are used by the pooler
@@ -517,7 +607,6 @@ class LLMEngine:
607 seqs=[seq],
608 arrival_time=arrival_time,
609 lora_request=lora_request,
520 - multi_modal_data=multi_modal_data,
610 pooling_params=pooling_params)
611 return seq_group
612
@@ -570,7 +659,7 @@ class LLMEngine:
659
660 def _process_model_outputs(
661 self,
573 - output: List[Union[SamplerOutput, PoolerOutput]],
662 + output: GenericSequence[Union[SamplerOutput, PoolerOutput]],
663 scheduled_seq_groups: List[ScheduledSequenceGroup],
664 ignored_seq_groups: List[SequenceGroup],
665 seq_group_metadata_list: List[SequenceGroupMetadata],
@@ -585,7 +674,7 @@ class LLMEngine:
674 # Organize outputs by [sequence group][step] instead of
675 # [step][sequence group].
676 output_by_sequence_group = create_output_by_sequence_group(
588 - sampler_outputs=output, num_seq_groups=len(scheduled_seq_groups))
677 + output, num_seq_groups=len(scheduled_seq_groups))
678
679 # Update the scheduled sequence groups with the model outputs.
680 for scheduled_seq_group, outputs, seq_group_meta in zip(
vllm/engine/output_processor/util.py
+6 -4
@@ -1,18 +1,20 @@
1 from typing import List
2 +from typing import Sequence as GenericSequence
3 +from typing import Union
4
3 -from vllm.sequence import SamplerOutput, SequenceGroupOutput
5 +from vllm.sequence import PoolerOutput, SamplerOutput, SequenceGroupOutput
6
7
8 def create_output_by_sequence_group(
7 - sampler_outputs: List[SamplerOutput],
9 + outputs: GenericSequence[Union[SamplerOutput, PoolerOutput]],
10 num_seq_groups: int) -> List[List[SequenceGroupOutput]]:
11 """Helper method which transforms a 2d list organized by
12 [step][sequence group] into [sequence group][step].
13 """
12 - output_by_sequence_group: List[List[SamplerOutput]] = [
14 + output_by_sequence_group: List[List[SequenceGroupOutput]] = [
15 [] for _ in range(num_seq_groups)
16 ]
15 - for step in sampler_outputs:
17 + for step in outputs:
18 for i, sequence_group_output in enumerate(step):
19 output_by_sequence_group[i].append(sequence_group_output)
20
vllm/entrypoints/llm.py
+302 -102
@@ -1,11 +1,14 @@
1 -from typing import List, Optional, Union
1 +from contextlib import contextmanager
2 +from typing import ClassVar, List, Optional, Sequence, Union, cast, overload
3
3 -import torch
4 from tqdm import tqdm
5 from transformers import PreTrainedTokenizer, PreTrainedTokenizerFast
6
7 from vllm.engine.arg_utils import EngineArgs
8 from vllm.engine.llm_engine import LLMEngine
9 +from vllm.inputs import (PromptInputs, PromptStrictInputs, TextPrompt,
10 + TextTokensPrompt, TokensPrompt,
11 + parse_and_batch_prompt)
12 from vllm.logger import init_logger
13 from vllm.lora.request import LoRARequest
14 from vllm.outputs import EmbeddingRequestOutput, RequestOutput
@@ -13,7 +16,7 @@ from vllm.pooling_params import PoolingParams
16 from vllm.sampling_params import SamplingParams
17 from vllm.sequence import MultiModalData
18 from vllm.usage.usage_lib import UsageContext
16 -from vllm.utils import Counter
19 +from vllm.utils import Counter, deprecate_kwargs
20
21 logger = init_logger(__name__)
22
@@ -28,8 +31,10 @@ class LLM:
31 mechanism and efficient memory management.
32
33 NOTE: This class is intended to be used for offline inference. For online
31 - serving, use the `AsyncLLMEngine` class instead.
32 - NOTE: For the comprehensive list of arguments, see `EngineArgs`.
34 + serving, use the :class:`~vllm.AsyncLLMEngine` class instead.
35 +
36 + NOTE: For the comprehensive list of arguments, see
37 + :class:`~vllm.EngineArgs`.
38
39 Args:
40 model: The name or path of a HuggingFace Transformers model.
@@ -81,6 +86,18 @@ class LLM:
86 disable_custom_all_reduce: See ParallelConfig
87 """
88
89 + DEPRECATE_LEGACY: ClassVar[bool] = False
90 + """A flag to toggle whether to deprecate the legacy generate/encode API."""
91 +
92 + @classmethod
93 + @contextmanager
94 + def deprecate_legacy_api(cls):
95 + cls.DEPRECATE_LEGACY = True
96 +
97 + yield
98 +
99 + cls.DEPRECATE_LEGACY = False
100 +
101 def __init__(
102 self,
103 model: str,
@@ -138,15 +155,101 @@ class LLM:
155 ) -> None:
156 self.llm_engine.tokenizer.tokenizer = tokenizer
157
158 + @overload # LEGACY: single (prompt + optional token ids)
159 + def generate(
160 + self,
161 + prompts: str,
162 + sampling_params: Optional[Union[SamplingParams,
163 + List[SamplingParams]]] = None,
164 + prompt_token_ids: Optional[List[int]] = None,
165 + use_tqdm: bool = True,
166 + lora_request: Optional[LoRARequest] = None,
167 + multi_modal_data: Optional[MultiModalData] = None,
168 + ) -> List[RequestOutput]:
169 + ...
170 +
171 + @overload # LEGACY: multi (prompt + optional token ids)
172 def generate(
173 self,
143 - prompts: Optional[Union[str, List[str]]] = None,
174 + prompts: List[str],
175 sampling_params: Optional[Union[SamplingParams,
176 List[SamplingParams]]] = None,
177 prompt_token_ids: Optional[List[List[int]]] = None,
178 use_tqdm: bool = True,
179 lora_request: Optional[LoRARequest] = None,
180 multi_modal_data: Optional[MultiModalData] = None,
181 + ) -> List[RequestOutput]:
182 + ...
183 +
184 + @overload # LEGACY: single (token ids + optional prompt)
185 + def generate(
186 + self,
187 + prompts: Optional[str] = None,
188 + sampling_params: Optional[Union[SamplingParams,
189 + List[SamplingParams]]] = None,
190 + *,
191 + prompt_token_ids: List[int],
192 + use_tqdm: bool = True,
193 + lora_request: Optional[LoRARequest] = None,
194 + multi_modal_data: Optional[MultiModalData] = None,
195 + ) -> List[RequestOutput]:
196 + ...
197 +
198 + @overload # LEGACY: multi (token ids + optional prompt)
199 + def generate(
200 + self,
201 + prompts: Optional[List[str]] = None,
202 + sampling_params: Optional[Union[SamplingParams,
203 + List[SamplingParams]]] = None,
204 + *,
205 + prompt_token_ids: List[List[int]],
206 + use_tqdm: bool = True,
207 + lora_request: Optional[LoRARequest] = None,
208 + multi_modal_data: Optional[MultiModalData] = None,
209 + ) -> List[RequestOutput]:
210 + ...
211 +
212 + @overload # LEGACY: single or multi token ids [pos-only]
213 + def generate(
214 + self,
215 + prompts: None,
216 + sampling_params: None,
217 + prompt_token_ids: Union[List[int], List[List[int]]],
218 + use_tqdm: bool = True,
219 + lora_request: Optional[LoRARequest] = None,
220 + multi_modal_data: Optional[MultiModalData] = None,
221 + ) -> List[RequestOutput]:
222 + ...
223 +
224 + @overload
225 + def generate(
226 + self,
227 + inputs: Union[PromptStrictInputs, Sequence[PromptStrictInputs]],
228 + /, # We may enable `inputs` keyword after removing the old API
229 + *,
230 + sampling_params: Optional[Union[SamplingParams,
231 + Sequence[SamplingParams]]] = None,
232 + use_tqdm: bool = True,
233 + lora_request: Optional[LoRARequest] = None,
234 + ) -> List[RequestOutput]:
235 + ...
236 +
237 + @deprecate_kwargs("prompts",
238 + "prompt_token_ids",
239 + "multi_modal_data",
240 + is_deprecated=lambda: LLM.DEPRECATE_LEGACY,
241 + additional_message="Please use the 'inputs' parameter "
242 + "instead.")
243 + def generate(
244 + self,
245 + prompts: Union[Union[PromptStrictInputs, Sequence[PromptStrictInputs]],
246 + Optional[Union[str, List[str]]]] = None,
247 + sampling_params: Optional[Union[SamplingParams,
248 + Sequence[SamplingParams]]] = None,
249 + prompt_token_ids: Optional[Union[List[int], List[List[int]]]] = None,
250 + use_tqdm: bool = True,
251 + lora_request: Optional[LoRARequest] = None,
252 + multi_modal_data: Optional[MultiModalData] = None,
253 ) -> List[RequestOutput]:
254 """Generates the completions for the input prompts.
255
@@ -155,49 +258,138 @@ class LLM:
258 into a single list and pass it to this method.
259
260 Args:
158 - prompts: A list of prompts to generate completions for.
261 + inputs: A list of inputs to generate completions for.
262 sampling_params: The sampling parameters for text generation. If
263 None, we use the default sampling parameters.
264 When it is a single value, it is applied to every prompt.
265 When it is a list, the list must have the same length as the
266 prompts and it is paired one by one with the prompt.
164 - prompt_token_ids: A list of token IDs for the prompts. If None, we
165 - use the tokenizer to convert the prompts to token IDs.
267 use_tqdm: Whether to use tqdm to display the progress bar.
268 lora_request: LoRA request to use for generation, if any.
168 - multi_modal_data: Multi modal data.
269
270 Returns:
271 A list of `RequestOutput` objects containing the
272 generated completions in the same order as the input prompts.
273 """
274 + if prompt_token_ids is not None or multi_modal_data is not None:
275 + inputs = self._convert_v1_inputs(
276 + prompts=cast(Optional[Union[str, List[str]]], prompts),
277 + prompt_token_ids=prompt_token_ids,
278 + multi_modal_data=multi_modal_data,
279 + )
280 + else:
281 + inputs = cast(
282 + Union[PromptStrictInputs, Sequence[PromptStrictInputs]],
283 + prompts)
284 +
285 if sampling_params is None:
286 # Use default sampling params.
287 sampling_params = SamplingParams()
288
178 - requests_data = self._validate_and_prepare_requests(
179 - prompts,
180 - sampling_params,
181 - prompt_token_ids,
182 - lora_request,
183 - multi_modal_data,
289 + self._validate_and_add_requests(
290 + inputs=inputs,
291 + params=sampling_params,
292 + lora_request=lora_request,
293 )
294
186 - # Add requests to the engine and run the engine
187 - for request_data in requests_data:
188 - self._add_request(**request_data)
295 + outputs = self._run_engine(use_tqdm=use_tqdm)
296 + return LLMEngine.validate_outputs(outputs, RequestOutput)
297
190 - return self._run_engine(use_tqdm)
298 + @overload # LEGACY: single (prompt + optional token ids)
299 + def encode(
300 + self,
301 + prompts: str,
302 + pooling_params: Optional[Union[PoolingParams,
303 + Sequence[PoolingParams]]] = None,
304 + prompt_token_ids: Optional[List[int]] = None,
305 + use_tqdm: bool = True,
306 + lora_request: Optional[LoRARequest] = None,
307 + multi_modal_data: Optional[MultiModalData] = None,
308 + ) -> List[EmbeddingRequestOutput]:
309 + ...
310
311 + @overload # LEGACY: multi (prompt + optional token ids)
312 def encode(
313 self,
194 - prompts: Optional[Union[str, List[str]]] = None,
314 + prompts: List[str],
315 pooling_params: Optional[Union[PoolingParams,
196 - List[PoolingParams]]] = None,
316 + Sequence[PoolingParams]]] = None,
317 prompt_token_ids: Optional[List[List[int]]] = None,
318 use_tqdm: bool = True,
319 lora_request: Optional[LoRARequest] = None,
320 multi_modal_data: Optional[MultiModalData] = None,
321 + ) -> List[EmbeddingRequestOutput]:
322 + ...
323 +
324 + @overload # LEGACY: single (token ids + optional prompt)
325 + def encode(
326 + self,
327 + prompts: Optional[str] = None,
328 + pooling_params: Optional[Union[PoolingParams,
329 + Sequence[PoolingParams]]] = None,
330 + *,
331 + prompt_token_ids: List[int],
332 + use_tqdm: bool = True,
333 + lora_request: Optional[LoRARequest] = None,
334 + multi_modal_data: Optional[MultiModalData] = None,
335 + ) -> List[EmbeddingRequestOutput]:
336 + ...
337 +
338 + @overload # LEGACY: multi (token ids + optional prompt)
339 + def encode(
340 + self,
341 + prompts: Optional[List[str]] = None,
342 + pooling_params: Optional[Union[PoolingParams,
343 + Sequence[PoolingParams]]] = None,
344 + *,
345 + prompt_token_ids: List[List[int]],
346 + use_tqdm: bool = True,
347 + lora_request: Optional[LoRARequest] = None,
348 + multi_modal_data: Optional[MultiModalData] = None,
349 + ) -> List[EmbeddingRequestOutput]:
350 + ...
351 +
352 + @overload # LEGACY: single or multi token ids [pos-only]
353 + def encode(
354 + self,
355 + prompts: None,
356 + pooling_params: None,
357 + prompt_token_ids: Union[List[int], List[List[int]]],
358 + use_tqdm: bool = True,
359 + lora_request: Optional[LoRARequest] = None,
360 + multi_modal_data: Optional[MultiModalData] = None,
361 + ) -> List[EmbeddingRequestOutput]:
362 + ...
363 +
364 + @overload
365 + def encode(
366 + self,
367 + inputs: Union[PromptStrictInputs, Sequence[PromptStrictInputs]],
368 + /, # We may enable `inputs` keyword after removing the old API
369 + *,
370 + pooling_params: Optional[Union[PoolingParams,
371 + Sequence[PoolingParams]]] = None,
372 + use_tqdm: bool = True,
373 + lora_request: Optional[LoRARequest] = None,
374 + ) -> List[EmbeddingRequestOutput]:
375 + ...
376 +
377 + @deprecate_kwargs("prompts",
378 + "prompt_token_ids",
379 + "multi_modal_data",
380 + is_deprecated=lambda: LLM.DEPRECATE_LEGACY,
381 + additional_message="Please use the 'inputs' parameter "
382 + "instead.")
383 + def encode(
384 + self,
385 + prompts: Union[Union[PromptStrictInputs, Sequence[PromptStrictInputs]],
386 + Optional[Union[str, List[str]]]] = None,
387 + pooling_params: Optional[Union[PoolingParams,
388 + Sequence[PoolingParams]]] = None,
389 + prompt_token_ids: Optional[Union[List[int], List[List[int]]]] = None,
390 + use_tqdm: bool = True,
391 + lora_request: Optional[LoRARequest] = None,
392 + multi_modal_data: Optional[MultiModalData] = None,
393 ) -> List[EmbeddingRequestOutput]:
394 """Generates the completions for the input prompts.
395
@@ -206,124 +398,133 @@ class LLM:
398 into a single list and pass it to this method.
399
400 Args:
209 - prompts: A list of prompts to generate completions for.
401 + inputs: The inputs to the LLM. You may pass a sequence of inputs for
402 + batch inference. See :class:`~vllm.inputs.PromptStrictInputs`
403 + for more details about the format of each input.
404 pooling_params: The pooling parameters for pooling. If None, we
405 use the default pooling parameters.
212 - prompt_token_ids: A list of token IDs for the prompts. If None, we
213 - use the tokenizer to convert the prompts to token IDs.
406 use_tqdm: Whether to use tqdm to display the progress bar.
407 lora_request: LoRA request to use for generation, if any.
216 - multi_modal_data: Multi modal data.
408
409 Returns:
410 A list of `EmbeddingRequestOutput` objects containing the
411 generated embeddings in the same order as the input prompts.
412 """
413 + if prompt_token_ids is not None or multi_modal_data is not None:
414 + inputs = self._convert_v1_inputs(
415 + prompts=cast(Optional[Union[str, List[str]]], prompts),
416 + prompt_token_ids=prompt_token_ids,
417 + multi_modal_data=multi_modal_data,
418 + )
419 + else:
420 + inputs = cast(
421 + Union[PromptStrictInputs, Sequence[PromptStrictInputs]],
422 + prompts)
423 +
424 if pooling_params is None:
425 # Use default pooling params.
426 pooling_params = PoolingParams()
427
226 - requests_data = self._validate_and_prepare_requests(
227 - prompts,
228 - pooling_params,
229 - prompt_token_ids,
230 - lora_request,
231 - multi_modal_data,
428 + self._validate_and_add_requests(
429 + inputs=inputs,
430 + params=pooling_params,
431 + lora_request=lora_request,
432 )
433
234 - # Add requests to the engine and run the engine
235 - for request_data in requests_data:
236 - self._add_request(**request_data)
434 + outputs = self._run_engine(use_tqdm=use_tqdm)
435 + return LLMEngine.validate_outputs(outputs, EmbeddingRequestOutput)
436
238 - return self._run_engine(use_tqdm)
239 -
240 - def _validate_and_prepare_requests(
437 + # LEGACY
438 + def _convert_v1_inputs(
439 self,
440 prompts: Optional[Union[str, List[str]]],
243 - params: Union[Union[SamplingParams, PoolingParams],
244 - List[Union[SamplingParams,
245 - PoolingParams]]], # Unified parameter
246 - prompt_token_ids: Optional[List[List[int]]] = None,
247 - lora_request: Optional[LoRARequest] = None,
248 - multi_modal_data: Optional[MultiModalData] = None,
249 - ) -> List[dict]:
250 - """Validates and prepares request data for adding to the engine.
441 + prompt_token_ids: Optional[Union[List[int], List[List[int]]]],
442 + multi_modal_data: Optional[MultiModalData],
443 + ):
444 + # skip_tokenizer_init is now checked in engine
445
252 - Ensures prompts and token IDs are consistent, and returns a list of
253 - dictionaries with request data for further processing.
254 - """
255 - if prompts is None and prompt_token_ids is None:
256 - raise ValueError("Either prompts or prompt_token_ids must be "
257 - "provided.")
258 - if self.llm_engine.model_config.skip_tokenizer_init \
259 - and prompts is not None:
260 - raise ValueError("prompts must be None if skip_tokenizer_init "
261 - "is True")
262 - if isinstance(prompts, str):
263 - # Convert a single prompt to a list.
264 - prompts = [prompts]
265 - if (prompts is not None and prompt_token_ids is not None
266 - and len(prompts) != len(prompt_token_ids)):
267 - raise ValueError("The lengths of prompts and prompt_token_ids "
268 - "must be the same.")
446 + if prompts is not None:
447 + prompts = [p["content"] for p in parse_and_batch_prompt(prompts)]
448 + if prompt_token_ids is not None:
449 + prompt_token_ids = [
450 + p["content"] for p in parse_and_batch_prompt(prompt_token_ids)
451 + ]
452
453 + num_requests = None
454 if prompts is not None:
455 num_requests = len(prompts)
272 - else:
273 - assert prompt_token_ids is not None
456 + if prompt_token_ids is not None:
457 + if (num_requests is not None
458 + and num_requests != len(prompt_token_ids)):
459 + raise ValueError("The lengths of prompts and prompt_token_ids "
460 + "must be the same.")
461 +
462 num_requests = len(prompt_token_ids)
463 + if num_requests is None:
464 + raise ValueError("Either prompts or prompt_token_ids must be "
465 + "provided.")
466 +
467 + inputs: List[PromptInputs] = []
468 + for i in range(num_requests):
469 + if prompts is not None:
470 + if prompt_token_ids is not None:
471 + item = TextTokensPrompt(
472 + prompt=prompts[i],
473 + prompt_token_ids=prompt_token_ids[i])
474 + else:
475 + item = TextPrompt(prompt=prompts[i])
476 + else:
477 + if prompt_token_ids is not None:
478 + item = TokensPrompt(prompt_token_ids=prompt_token_ids[i])
479 + else:
480 + raise AssertionError
481 +
482 + if multi_modal_data is not None:
483 + item["multi_modal_data"] = multi_modal_data
484 +
485 + inputs.append(item)
486 +
487 + return inputs
488 +
489 + def _validate_and_add_requests(
490 + self,
491 + inputs: Union[PromptStrictInputs, Sequence[PromptStrictInputs]],
492 + params: Union[SamplingParams, Sequence[SamplingParams], PoolingParams,
493 + Sequence[PoolingParams]],
494 + lora_request: Optional[LoRARequest],
495 + ) -> None:
496 + if isinstance(inputs, (str, dict)):
497 + # Convert a single prompt to a list.
498 + inputs = [inputs]
499 +
500 + num_requests = len(inputs)
501
502 if isinstance(params, list) and len(params) != num_requests:
503 raise ValueError("The lengths of prompts and params "
504 "must be the same.")
279 - if multi_modal_data:
280 - multi_modal_data.data = multi_modal_data.data.to(torch.float16)
505
506 # Add requests to the engine.
283 - requests_data = []
284 - for i in range(num_requests):
285 - prompt = prompts[i] if prompts is not None else None
286 - token_ids = None if prompt_token_ids is None else prompt_token_ids[
287 - i]
288 -
289 - multi_modal_item = MultiModalData(
290 - type=multi_modal_data.type,
291 - data=multi_modal_data.data[i].unsqueeze(0),
292 - ) if multi_modal_data else None
293 -
294 - requests_data.append({
295 - "prompt":
296 - prompt,
297 - "params":
298 - params[i] if isinstance(params, list) else params,
299 - "prompt_token_ids":
300 - token_ids,
301 - "lora_request":
302 - lora_request,
303 - "multi_modal_data":
304 - multi_modal_item,
305 - })
306 -
307 - return requests_data
507 + for i, request_inputs in enumerate(inputs):
508 + self._add_request(
509 + request_inputs,
510 + params[i] if isinstance(params, Sequence) else params,
511 + lora_request=lora_request,
512 + )
513
514 def _add_request(
515 self,
311 - prompt: Optional[str],
516 + inputs: PromptInputs,
517 params: Union[SamplingParams, PoolingParams],
313 - prompt_token_ids: Optional[List[int]],
518 lora_request: Optional[LoRARequest] = None,
315 - multi_modal_data: Optional[MultiModalData] = None,
519 ) -> None:
520 request_id = str(next(self.request_counter))
521 self.llm_engine.add_request(request_id,
319 - prompt,
522 + inputs,
523 params,
321 - prompt_token_ids,
322 - lora_request=lora_request,
323 - multi_modal_data=multi_modal_data)
524 + lora_request=lora_request)
525
526 def _run_engine(
326 - self, use_tqdm: bool
527 + self, *, use_tqdm: bool
528 ) -> List[Union[RequestOutput, EmbeddingRequestOutput]]:
529 # Initialize tqdm.
530 if use_tqdm:
@@ -355,5 +556,4 @@ class LLM:
556 # Sort the outputs by request ID.
557 # This is necessary because some requests may be finished earlier than
558 # its previous requests.
358 - outputs = sorted(outputs, key=lambda x: int(x.request_id))
359 - return outputs
559 + return sorted(outputs, key=lambda x: int(x.request_id))
vllm/entrypoints/openai/serving_chat.py
+9 -3
@@ -176,9 +176,15 @@ class OpenAIServingChat(OpenAIServing):
176 except ValueError as e:
177 return self.create_error_response(str(e))
178
179 - result_generator = self.engine.generate(prompt_text, sampling_params,
180 - request_id, prompt_ids,
181 - lora_request)
179 + result_generator = self.engine.generate(
180 + {
181 + "prompt": prompt_text,
182 + "prompt_token_ids": prompt_ids
183 + },
184 + sampling_params,
185 + request_id,
186 + lora_request,
187 + )
188 # Streaming response
189 if request.stream:
190 return self.chat_completion_stream_generator(
vllm/entrypoints/openai/serving_completion.py
+11 -6
@@ -119,12 +119,17 @@ class OpenAIServingCompletion(OpenAIServing):
119 truncate_prompt_tokens)
120 prompt_ids, prompt_text = prompt_formats
121
122 - generators.append(
123 - self.engine.generate(prompt_text,
124 - sampling_params,
125 - f"{request_id}-{i}",
126 - prompt_token_ids=prompt_ids,
127 - lora_request=lora_request))
122 + generator = self.engine.generate(
123 + {
124 + "prompt": prompt_text,
125 + "prompt_token_ids": prompt_ids
126 + },
127 + sampling_params,
128 + f"{request_id}-{i}",
129 + lora_request=lora_request,
130 + )
131 +
132 + generators.append(generator)
133 except ValueError as e:
134 # TODO: Use a vllm-specific Validation Error
135 return self.create_error_response(str(e))
vllm/entrypoints/openai/serving_embedding.py
+26 -16
@@ -1,5 +1,5 @@
1 import time
2 -from typing import AsyncIterator, List, Tuple
2 +from typing import AsyncIterator, List, Optional, Tuple
3
4 from fastapi import Request
5
@@ -100,11 +100,16 @@ class OpenAIServingEmbedding(OpenAIServing):
100
101 prompt_ids, prompt_text = prompt_formats
102
103 - generators.append(
104 - self.engine.generate(prompt_text,
105 - pooling_params,
106 - f"{request_id}-{i}",
107 - prompt_token_ids=prompt_ids))
103 + generator = self.engine.encode(
104 + {
105 + "prompt": prompt_text,
106 + "prompt_token_ids": prompt_ids
107 + },
108 + pooling_params,
109 + f"{request_id}-{i}",
110 + )
111 +
112 + generators.append(generator)
113 except ValueError as e:
114 # TODO: Use a vllm-specific Validation Error
115 return self.create_error_response(str(e))
@@ -113,16 +118,21 @@ class OpenAIServingEmbedding(OpenAIServing):
118 int, EmbeddingRequestOutput]] = merge_async_iterators(*generators)
119
120 # Non-streaming response
116 - final_res_batch: EmbeddingRequestOutput = [None] * len(prompts)
117 - async for i, res in result_generator:
118 - if await raw_request.is_disconnected():
119 - # Abort the request if the client disconnects.
120 - await self.engine.abort(f"{request_id}-{i}")
121 - # TODO: Use a vllm-specific Validation Error
122 - return self.create_error_response("Client disconnected")
123 - final_res_batch[i] = res
124 - response = request_output_to_embedding_response(
125 - final_res_batch, request_id, created_time, model_name)
121 + final_res_batch: List[Optional[EmbeddingRequestOutput]]
122 + final_res_batch = [None] * len(prompts)
123 + try:
124 + async for i, res in result_generator:
125 + if await raw_request.is_disconnected():
126 + # Abort the request if the client disconnects.
127 + await self.engine.abort(f"{request_id}-{i}")
128 + # TODO: Use a vllm-specific Validation Error
129 + return self.create_error_response("Client disconnected")
130 + final_res_batch[i] = res
131 + response = request_output_to_embedding_response(
132 + final_res_batch, request_id, created_time, model_name)
133 + except ValueError as e:
134 + # TODO: Use a vllm-specific Validation Error
135 + return self.create_error_response(str(e))
136
137 return response
138
vllm/entrypoints/openai/serving_engine.py
+4 -2
@@ -143,7 +143,8 @@ class OpenAIServing:
143 return json_str
144
145 async def _check_model(
146 - self, request: Union[CompletionRequest, ChatCompletionRequest]
146 + self, request: Union[CompletionRequest, ChatCompletionRequest,
147 + EmbeddingRequest]
148 ) -> Optional[ErrorResponse]:
149 if request.model in self.served_model_names:
150 return None
@@ -155,7 +156,8 @@ class OpenAIServing:
156 status_code=HTTPStatus.NOT_FOUND)
157
158 def _maybe_get_lora(
158 - self, request: Union[CompletionRequest, ChatCompletionRequest]
159 + self, request: Union[CompletionRequest, ChatCompletionRequest,
160 + EmbeddingRequest]
161 ) -> Optional[LoRARequest]:
162 if request.model in self.served_model_names:
163 return None
vllm/inputs.py new
+130
@@ -0,0 +1,130 @@
1 +from typing import (TYPE_CHECKING, List, Literal, Optional, Sequence,
2 + TypedDict, Union, cast, overload)
3 +
4 +from typing_extensions import NotRequired
5 +
6 +if TYPE_CHECKING:
7 + from vllm.sequence import MultiModalData
8 +
9 +
10 +class ParsedText(TypedDict):
11 + content: str
12 + is_tokens: Literal[False]
13 +
14 +
15 +class ParsedTokens(TypedDict):
16 + content: List[int]
17 + is_tokens: Literal[True]
18 +
19 +
20 +# https://github.com/vllm-project/vllm/pull/4028
21 +@overload
22 +def parse_and_batch_prompt(
23 + prompt: Union[str, List[str]]) -> Sequence[ParsedText]:
24 + ...
25 +
26 +
27 +@overload
28 +def parse_and_batch_prompt(
29 + prompt: Union[List[int], List[List[int]]]) -> Sequence[ParsedTokens]:
30 + ...
31 +
32 +
33 +def parse_and_batch_prompt(
34 + prompt: Union[str, List[str], List[int], List[List[int]]],
35 +) -> Union[Sequence[ParsedText], Sequence[ParsedTokens]]:
36 + if isinstance(prompt, str):
37 + # case 1: a string
38 + return [ParsedText(content=prompt, is_tokens=False)]
39 +
40 + if isinstance(prompt, list):
41 + if len(prompt) == 0:
42 + raise ValueError("please provide at least one prompt")
43 +
44 + if isinstance(prompt[0], str):
45 + # case 2: array of strings
46 + return [
47 + ParsedText(content=elem, is_tokens=False)
48 + for elem in cast(List[str], prompt)
49 + ]
50 + if isinstance(prompt[0], int):
51 + # case 3: array of tokens
52 + elem = cast(List[int], prompt)
53 + return [ParsedTokens(content=elem, is_tokens=True)]
54 + if isinstance(prompt[0], list):
55 + if len(prompt[0]) == 0:
56 + raise ValueError("please provide at least one prompt")
57 +
58 + if isinstance(prompt[0][0], int):
59 + # case 4: array of token arrays
60 + return [
61 + ParsedTokens(content=elem, is_tokens=True)
62 + for elem in cast(List[List[int]], prompt)
63 + ]
64 +
65 + raise ValueError("prompt must be a string, array of strings, "
66 + "array of tokens, or array of token arrays")
67 +
68 +
69 +class TextPrompt(TypedDict):
70 + """Schema for a text prompt."""
71 +
72 + prompt: str
73 + """The input text to be tokenized before passing to the model."""
74 +
75 + multi_modal_data: NotRequired["MultiModalData"]
76 + """
77 + Optional multi-modal data to pass to the model,
78 + if the model supports it.
79 + """
80 +
81 +
82 +class TokensPrompt(TypedDict):
83 + """Schema for a tokenized prompt."""
84 +
85 + prompt_token_ids: List[int]
86 + """A list of token IDs to pass to the model."""
87 +
88 + multi_modal_data: NotRequired["MultiModalData"]
89 + """
90 + Optional multi-modal data to pass to the model,
91 + if the model supports it.
92 + """
93 +
94 +
95 +class TextTokensPrompt(TypedDict):
96 + """It is assumed that :attr:`prompt` is consistent with
97 + :attr:`prompt_token_ids`. This is currently used in
98 + :class:`AsyncLLMEngine` for logging both the text and token IDs."""
99 +
100 + prompt: str
101 + """The prompt text."""
102 +
103 + prompt_token_ids: List[int]
104 + """The token IDs of the prompt. If None, we use the
105 + tokenizer to convert the prompts to token IDs."""
106 +
107 + multi_modal_data: NotRequired["MultiModalData"]
108 + """
109 + Optional multi-modal data to pass to the model,
110 + if the model supports it.
111 + """
112 +
113 +
114 +PromptStrictInputs = Union[str, TextPrompt, TokensPrompt]
115 +"""
116 +The inputs to the LLM, which can take one of the following forms:
117 +
118 +- A text prompt (:class:`str` or :class:`TextPrompt`)
119 +- A tokenized prompt (:class:`TokensPrompt`)
120 +"""
121 +
122 +PromptInputs = Union[str, TextPrompt, TokensPrompt, TextTokensPrompt]
123 +"""Same as :const:`PromptStrictInputs` but additionally accepts
124 +:class:`TextTokensPrompt`."""
125 +
126 +
127 +class LLMInputs(TypedDict):
128 + prompt_token_ids: List[int]
129 + prompt: Optional[str]
130 + multi_modal_data: Optional["MultiModalData"]
vllm/outputs.py
+15 -27
@@ -1,4 +1,5 @@
1 import time
2 +from dataclasses import dataclass
3 from typing import List, Optional, Union
4
5 from vllm.lora.request import LoRARequest
@@ -6,6 +7,7 @@ from vllm.sequence import (PromptLogprobs, RequestMetrics, SampleLogprobs,
7 SequenceGroup, SequenceStatus)
8
9
10 +@dataclass
11 class CompletionOutput:
12 """The output data of one completion output of a request.
13
@@ -24,25 +26,14 @@ class CompletionOutput:
26 lora_request: The LoRA request that was used to generate the output.
27 """
28
27 - def __init__(
28 - self,
29 - index: int,
30 - text: str,
31 - token_ids: List[int],
32 - cumulative_logprob: float,
33 - logprobs: Optional[SampleLogprobs],
34 - finish_reason: Optional[str] = None,
35 - stop_reason: Union[int, str, None] = None,
36 - lora_request: Optional[LoRARequest] = None,
37 - ) -> None:
38 - self.index = index
39 - self.text = text
40 - self.token_ids = token_ids
41 - self.cumulative_logprob = cumulative_logprob
42 - self.logprobs = logprobs
43 - self.finish_reason = finish_reason
44 - self.stop_reason = stop_reason
45 - self.lora_request = lora_request
29 + index: int
30 + text: str
31 + token_ids: List[int]
32 + cumulative_logprob: float
33 + logprobs: Optional[SampleLogprobs]
34 + finish_reason: Optional[str] = None
35 + stop_reason: Union[int, str, None] = None
36 + lora_request: Optional[LoRARequest] = None
37
38 def finished(self) -> bool:
39 return self.finish_reason is not None
@@ -57,6 +48,7 @@ class CompletionOutput:
48 f"stop_reason={self.stop_reason})")
49
50
51 +@dataclass
52 class EmbeddingOutput:
53 """The output data of one completion output of a request.
54
@@ -65,15 +57,11 @@ class EmbeddingOutput:
57 length of vector depends on the model as listed in the embedding guide.
58 """
59
68 - def __init__(
69 - self,
70 - embedding: List[float],
71 - ) -> None:
72 - self.embedding = embedding
60 + embedding: List[float]
61
62 def __repr__(self) -> str:
63 return (f"EmbeddingOutput("
76 - f"embedding={len(self.embedding)}")
64 + f"embedding={len(self.embedding)})")
65
66
67 class RequestOutput:
@@ -93,7 +81,7 @@ class RequestOutput:
81 def __init__(
82 self,
83 request_id: str,
96 - prompt: str,
84 + prompt: Optional[str],
85 prompt_token_ids: List[int],
86 prompt_logprobs: Optional[PromptLogprobs],
87 outputs: List[CompletionOutput],
@@ -183,7 +171,7 @@ class EmbeddingRequestOutput:
171 finished (bool): A flag indicating whether the embedding is completed.
172 """
173
186 - def __init__(self, request_id: str, outputs: 'EmbeddingOutput',
174 + def __init__(self, request_id: str, outputs: "EmbeddingOutput",
175 prompt_token_ids: List[int], finished: bool):
176 self.request_id = request_id
177 self.prompt_token_ids = prompt_token_ids
vllm/sequence.py
+26 -12
@@ -6,6 +6,7 @@ from dataclasses import dataclass, field
6 from typing import TYPE_CHECKING, Dict, List, Optional, Tuple, Union
7
8 from vllm.block import LogicalTokenBlock
9 +from vllm.inputs import LLMInputs
10 from vllm.lora.request import LoRARequest
11 from vllm.pooling_params import PoolingParams
12 from vllm.sampling_params import SamplingParams
@@ -210,8 +211,7 @@ class Sequence:
211
212 Args:
213 seq_id: The ID of the sequence.
213 - prompt: The prompt of the sequence.
214 - prompt_token_ids: The token IDs of the prompt.
214 + inputs: The inputs of the sequence.
215 block_size: The block size of the sequence. Should be the same as the
216 block size used by the block manager and cache engine.
217 lora_request: LoRA request.
@@ -220,25 +220,24 @@ class Sequence:
220 def __init__(
221 self,
222 seq_id: int,
223 - prompt: str,
224 - prompt_token_ids: List[int],
223 + inputs: LLMInputs,
224 block_size: int,
225 eos_token_id: Optional[int] = None,
226 lora_request: Optional[LoRARequest] = None,
227 ) -> None:
228 self.seq_id = seq_id
230 - self.prompt = prompt
229 + self.inputs = inputs
230 self.block_size = block_size
231 self.eos_token_id = eos_token_id
232 self.lora_request = lora_request
233
235 - self.data: SequenceData = SequenceData(prompt_token_ids)
234 + self.data = SequenceData(self.prompt_token_ids)
235 self.output_logprobs: SampleLogprobs = []
236 self.output_text = ""
237
238 self.logical_token_blocks: List[LogicalTokenBlock] = []
239 # Initialize the logical token blocks with the prompt token ids.
241 - self._append_tokens_to_blocks(prompt_token_ids)
240 + self._append_tokens_to_blocks(self.prompt_token_ids)
241 self.status = SequenceStatus.WAITING
242 self.stop_reason: Union[int, str, None] = None
243
@@ -248,6 +247,18 @@ class Sequence:
247 # Input + output tokens
248 self.tokens: Optional[List[str]] = None
249
250 + @property
251 + def prompt(self) -> Optional[str]:
252 + return self.inputs["prompt"]
253 +
254 + @property
255 + def prompt_token_ids(self) -> List[int]:
256 + return self.inputs["prompt_token_ids"]
257 +
258 + @property
259 + def multi_modal_data(self) -> Optional["MultiModalData"]:
260 + return self.inputs["multi_modal_data"]
261 +
262 @property
263 def lora_int_id(self) -> int:
264 return self.lora_request.lora_int_id if self.lora_request else 0
@@ -415,7 +426,6 @@ class SequenceGroup:
426 sampling_params: The sampling parameters used to generate the outputs.
427 arrival_time: The arrival time of the request.
428 lora_request: LoRA request.
418 - multi_modal_data: Multi modal data associated with the request.
429 embeddings: The embeddings vectors of the prompt of the sequence group
430 for an embedding model.
431 pooling_params: The pooling parameters used to generate the pooling
@@ -429,7 +439,6 @@ class SequenceGroup:
439 arrival_time: float,
440 sampling_params: Optional[SamplingParams] = None,
441 lora_request: Optional[LoRARequest] = None,
432 - multi_modal_data: Optional[MultiModalData] = None,
442 embeddings: Optional[List[float]] = None,
443 pooling_params: Optional[PoolingParams] = None,
444 ) -> None:
@@ -444,12 +453,11 @@ class SequenceGroup:
453 self.lora_request = lora_request
454 self.prompt_logprobs: Optional[PromptLogprobs] = None
455 self.state = SequenceGroupState()
447 - self.multi_modal_data = multi_modal_data
456 self.embeddings = embeddings
457 self.pooling_params = pooling_params
458
459 @property
452 - def prompt(self) -> str:
460 + def prompt(self) -> Optional[str]:
461 # All sequences in the group should have the same prompt.
462 # We use the prompt of an arbitrary sequence.
463 return next(iter(self.seqs_dict.values())).prompt
@@ -458,7 +466,13 @@ class SequenceGroup:
466 def prompt_token_ids(self) -> List[int]:
467 # All sequences in the group should have the same prompt.
468 # We use the prompt of an arbitrary sequence.
461 - return next(iter(self.seqs_dict.values())).data.prompt_token_ids
469 + return next(iter(self.seqs_dict.values())).prompt_token_ids
470 +
471 + @property
472 + def multi_modal_data(self) -> Optional[MultiModalData]:
473 + # All sequences in the group should have the same multi-modal data.
474 + # We use the multi-modal data of an arbitrary sequence.
475 + return next(iter(self.seqs_dict.values())).multi_modal_data
476
477 @property
478 def lora_int_id(self) -> int:
vllm/utils.py
+42 -1
@@ -11,7 +11,7 @@ import threading
11 import uuid
12 import warnings
13 from collections import defaultdict
14 -from functools import lru_cache, partial
14 +from functools import lru_cache, partial, wraps
15 from platform import uname
16 from typing import (Any, AsyncIterator, Awaitable, Callable, Dict, Generic,
17 Hashable, List, Optional, OrderedDict, Tuple, TypeVar,
@@ -658,3 +658,44 @@ def enable_trace_function_call_for_thread() -> None:
658 filename)
659 os.makedirs(os.path.dirname(log_path), exist_ok=True)
660 enable_trace_function_call(log_path)
661 +
662 +
663 +def identity(value: T) -> T:
664 + return value
665 +
666 +
667 +F = TypeVar('F', bound=Callable[..., Any])
668 +
669 +
670 +def deprecate_kwargs(
671 + *kws: str,
672 + is_deprecated: Union[bool, Callable[[], bool]] = True,
673 + additional_message: Optional[str] = None) -> Callable[[F], F]:
674 + deprecated_kws = set(kws)
675 +
676 + if not callable(is_deprecated):
677 + is_deprecated = partial(identity, is_deprecated)
678 +
679 + def wrapper(fn: F) -> F:
680 +
681 + @wraps(fn)
682 + def inner(*args, **kwargs):
683 + if is_deprecated():
684 + deprecated_kwargs = kwargs.keys() & deprecated_kws
685 + if deprecated_kwargs:
686 + msg = (
687 + f"The keyword arguments {deprecated_kwargs} are "
688 + "deprecated and will be removed in a future update.")
689 + if additional_message is not None:
690 + msg += f" {additional_message}"
691 +
692 + warnings.warn(
693 + DeprecationWarning(msg),
694 + stacklevel=3, # The inner function takes up one level
695 + )
696 +
697 + return fn(*args, **kwargs)
698 +
699 + return inner # type: ignore
700 +
701 + return wrapper