A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 08:15 UTC
[V1] `AsyncLLM` Implementation (#9826)
Signed-off-by: Nick Hill <nickhill@us.ibm.com> Signed-off-by: rshaw@neuralmagic.com <rshaw@neuralmagic.com> Signed-off-by: Nick Hill <nhill@redhat.com> Co-authored-by: Nick Hill <nickhill@us.ibm.com> Co-authored-by: Varun Sundar Rabindranath <varun@neuralmagic.com> Co-authored-by: Nick Hill <nhill@redhat.com> Co-authored-by: Tyler Michael Smith <tyler@neuralmagic.com>
Robert Shaw committed
Nov 11, 2024 at 18:05 UTC
6ace6fba2ca42b79a948a9b47af00487b5f73868
29 files changed
+2412
-727
.buildkite/test-pipeline.yaml
+8
@@ -165,6 +165,14 @@ steps:
165
# OOM in the CI unless we run this separately
166
- pytest -v -s tokenization
167
168
+- label: V1 Test
169
+ #mirror_hardwares: [amd]
170
+ source_file_dependencies:
171
+ - vllm/
172
+ - tests/v1
173
+ commands:
174
+ - pytest -v -s v1
175
+
176
- label: Examples Test # 15min
177
working_dir: "/vllm-workspace/examples"
178
#mirror_hardwares: [amd]
tests/entrypoints/llm/test_accuracy.py
new
+56
@@ -0,0 +1,56 @@
1
+"""
2
+This file test accuracy of the vLLM server via LMEval.
3
+It uses local-completions, which interacts with vLLM
4
+through the OAI API with N concurrent connections.
5
+This simulates real work usage of the API and makes
6
+sure that the zmq frontend mp RPC message passing and
7
+AsyncLLMEngine are working correctly.
8
+"""
9
+
10
+import lm_eval
11
+import pytest
12
+
13
+from vllm.platforms import current_platform
14
+
15
+MODEL_NAME = "Qwen/Qwen2-1.5B-Instruct"
16
+NUM_CONCURRENT = 500
17
+TASK = "gsm8k"
18
+FILTER = "exact_match,strict-match"
19
+RTOL = 0.03
20
+EXPECTED_VALUE = 0.58
21
+
22
+
23
+def run_test():
24
+ """Run the end to end accuracy test."""
25
+
26
+ model_args = f"pretrained={MODEL_NAME},max_model_len=2048"
27
+
28
+ results = lm_eval.simple_evaluate(
29
+ model="vllm",
30
+ model_args=model_args,
31
+ tasks="gsm8k",
32
+ batch_size="auto",
33
+ )
34
+
35
+ measured_value = results["results"][TASK][FILTER]
36
+ assert (measured_value - RTOL < EXPECTED_VALUE
37
+ and measured_value + RTOL > EXPECTED_VALUE
38
+ ), f"Expected: {EXPECTED_VALUE} | Measured: {measured_value}"
39
+
40
+
41
+@pytest.mark.skipif(not current_platform.is_cuda(),
42
+ reason="V1 is currently only supported on CUDA.")
43
+def test_lm_eval_accuracy_v1_engine(monkeypatch):
44
+ """Run with the V1 Engine."""
45
+
46
+ with monkeypatch.context() as m:
47
+ m.setenv("VLLM_USE_V1", "1")
48
+ run_test()
49
+
50
+
51
+def test_lm_eval_accuracy_v0_engine(monkeypatch):
52
+ """Run with the V0 Engine."""
53
+
54
+ with monkeypatch.context() as m:
55
+ m.setenv("VLLM_USE_V1", "0")
56
+ run_test()
tests/entrypoints/openai/test_accuracy.py
+22
-3
@@ -37,11 +37,11 @@ if current_platform.is_tpu():
37
MAX_WAIT_SECONDS = 600
38
39
40
-@pytest.mark.parametrize("more_args", MORE_ARGS_LIST)
41
-def test_lm_eval_accuracy(more_args):
40
+def run_test(more_args):
41
+ """Run the end to end accuracy test."""
42
+
43
args = list(DEFAULT_ARGS)
44
args.extend(more_args)
44
-
45
print(f"Running with: {args}")
46
47
with RemoteOpenAIServer(
@@ -64,3 +64,22 @@ def test_lm_eval_accuracy(more_args):
64
assert (measured_value - RTOL < EXPECTED_VALUE
65
and measured_value + RTOL > EXPECTED_VALUE
66
), f"Expected: {EXPECTED_VALUE} | Measured: {measured_value}"
67
+
68
+
69
+@pytest.mark.skipif(not current_platform.is_cuda(),
70
+ reason="V1 currently only supported on CUDA")
71
+def test_lm_eval_accuracy_v1_engine(monkeypatch):
72
+ """Run with the V1 Engine."""
73
+
74
+ with monkeypatch.context() as m:
75
+ m.setenv("VLLM_USE_V1", "1")
76
+ run_test([])
77
+
78
+
79
+@pytest.mark.parametrize("more_args", MORE_ARGS_LIST)
80
+def test_lm_eval_accuracy_v0_engine(monkeypatch, more_args):
81
+ """Run with the V0 Engine."""
82
+
83
+ with monkeypatch.context() as m:
84
+ m.setenv("VLLM_USE_V1", "0")
85
+ run_test(more_args)
tests/v1/__init__.py
renamed
tests/v1/engine/__init__.py
tests/v1/engine/test_async_llm.py
new
+66
@@ -0,0 +1,66 @@
1
+import asyncio
2
+from typing import Tuple
3
+
4
+import pytest
5
+
6
+from vllm import SamplingParams
7
+from vllm.engine.arg_utils import AsyncEngineArgs
8
+from vllm.platforms import current_platform
9
+from vllm.v1.engine.async_llm import AsyncLLM
10
+
11
+if not current_platform.is_cuda():
12
+ pytest.skip(reason="V1 currently only supported on CUDA.",
13
+ allow_module_level=True)
14
+
15
+ENGINE_ARGS = AsyncEngineArgs(model="meta-llama/Llama-3.2-1B",
16
+ disable_log_requests=True)
17
+
18
+
19
+async def generate(engine: AsyncLLM, request_id: str,
20
+ max_tokens: int) -> Tuple[int, str]:
21
+ count = 0
22
+ async for _ in engine.generate(request_id=request_id,
23
+ prompt="Hello my name is Robert and",
24
+ sampling_params=SamplingParams(
25
+ max_tokens=max_tokens, temperature=0)):
26
+
27
+ count += 1
28
+ await asyncio.sleep(0.)
29
+
30
+ return count, request_id
31
+
32
+
33
+@pytest.mark.asyncio
34
+async def test_load(monkeypatch):
35
+ with monkeypatch.context() as m:
36
+ m.setenv("VLLM_USE_V1", "1")
37
+
38
+ engine = AsyncLLM.from_engine_args(ENGINE_ARGS)
39
+
40
+ NUM_REQUESTS = 10000
41
+ NUM_EXPECTED_TOKENS = 10
42
+
43
+ request_ids = [f"request-{i}" for i in range(NUM_REQUESTS)]
44
+
45
+ # Create concurrent requests.
46
+ tasks = []
47
+ for request_id in request_ids:
48
+ tasks.append(
49
+ asyncio.create_task(
50
+ generate(engine, request_id, NUM_EXPECTED_TOKENS)))
51
+
52
+ # Confirm that we got all the EXPECTED tokens from the requests.
53
+ failed_request_id = None
54
+ tokens = None
55
+ for task in tasks:
56
+ num_generated_tokens, request_id = await task
57
+ if (num_generated_tokens != NUM_EXPECTED_TOKENS
58
+ and failed_request_id is None):
59
+ failed_request_id = request_id
60
+ tokens = num_generated_tokens
61
+
62
+ assert failed_request_id is None, (
63
+ f"{failed_request_id} generated {tokens} but "
64
+ f"expected {NUM_EXPECTED_TOKENS}")
65
+
66
+ engine.shutdown()
tests/v1/engine/test_detokenizer.py
new
+205
@@ -0,0 +1,205 @@
1
+from typing import List
2
+
3
+import pytest
4
+from transformers import AutoTokenizer
5
+
6
+from vllm.sampling_params import RequestOutputKind
7
+from vllm.v1.engine import EngineCoreOutput
8
+from vllm.v1.engine.detokenizer import Detokenizer, DetokenizerRequest
9
+
10
+TOKENIZER_NAME = "mistralai/Mistral-7B-Instruct-v0.3"
11
+tokenizer = AutoTokenizer.from_pretrained(TOKENIZER_NAME)
12
+
13
+FULL_STRINGS = [
14
+ "My name is Robert from Neural Magic and I love working on vLLM so much!",
15
+ "Red Hat is the best open source company by far across Linux, K8s, and AI.",
16
+ "Nick is the name of my brother in addition to my colleague from Red Hat.",
17
+]
18
+
19
+STOP_STRINGS = ["I love working on", "company by far", "brother in"]
20
+
21
+FULL_TOKENS = [tokenizer(text).input_ids for text in FULL_STRINGS]
22
+PROMPT_LEN = 5
23
+PROMPT_TOKENS = [
24
+ tokenizer(text).input_ids[:PROMPT_LEN] for text in FULL_STRINGS
25
+]
26
+GENERATION_TOKENS = [
27
+ tokenizer(text).input_ids[PROMPT_LEN:] for text in FULL_STRINGS
28
+]
29
+PROMPT_STRINGS = [
30
+ tokenizer.decode(prompt_tokens, skip_special_tokens=True)
31
+ for prompt_tokens in PROMPT_TOKENS
32
+]
33
+PROMPT_STRINGS_LEN = [len(prompt_string) for prompt_string in PROMPT_STRINGS]
34
+GENERATION_STRINGS = [
35
+ text[prompt_len:]
36
+ for text, prompt_len in zip(FULL_STRINGS, PROMPT_STRINGS_LEN)
37
+]
38
+
39
+
40
+class MockEngineCore:
41
+ """Mock outputs form premade tokens lists."""
42
+
43
+ def __init__(self, tokens_list: List[List[int]]):
44
+ self.tokens_list = tokens_list
45
+ self.current_idx = 0
46
+
47
+ def get_outputs(self) -> List[EngineCoreOutput]:
48
+ token_idx = self.current_idx
49
+ self.current_idx += 1
50
+
51
+ outputs = []
52
+ for req_idx, token_ids in enumerate(self.tokens_list):
53
+ if len(token_ids) > token_idx:
54
+ output = EngineCoreOutput(request_id=f"request-{req_idx}",
55
+ new_token_ids=[token_ids[token_idx]],
56
+ finished=False)
57
+ if token_idx == len(token_ids) - 1:
58
+ output.finished = True
59
+ output.finish_reason = "stopped"
60
+ outputs.append(output)
61
+
62
+ return outputs
63
+
64
+
65
+@pytest.mark.parametrize(
66
+ "request_output_kind",
67
+ [RequestOutputKind.DELTA, RequestOutputKind.FINAL_ONLY])
68
+def test_incremental_detokenization(request_output_kind: RequestOutputKind):
69
+ detokenizer = Detokenizer(TOKENIZER_NAME)
70
+ engine_core = MockEngineCore(GENERATION_TOKENS)
71
+
72
+ # Make N requests.
73
+ requests = [
74
+ DetokenizerRequest(
75
+ request_id=f"request-{idx}",
76
+ prompt=prompt,
77
+ prompt_token_ids=prompt_tokens,
78
+ skip_special_tokens=False,
79
+ spaces_between_special_tokens=False,
80
+ output_kind=request_output_kind,
81
+ stop=[],
82
+ include_stop_str_in_output=False,
83
+ ) for idx, (
84
+ prompt,
85
+ prompt_tokens) in enumerate(zip(PROMPT_STRINGS, PROMPT_TOKENS))
86
+ ]
87
+
88
+ # Add requests to the detokenizer.
89
+ for request in requests:
90
+ detokenizer.add_request(request)
91
+
92
+ gen_strings = {}
93
+ gen_tokens = {}
94
+ while True:
95
+ # Mock output from the EngineCore.
96
+ outputs = engine_core.get_outputs()
97
+ if len(outputs) == 0:
98
+ break
99
+
100
+ # Step the Detokenizer.
101
+ request_outputs, requests_to_abort = detokenizer.step(outputs)
102
+ assert len(requests_to_abort) == 0
103
+
104
+ # Update tracking.
105
+ for request_output in request_outputs:
106
+ request_id = request_output.request_id
107
+ new_text = request_output.outputs[0].text
108
+ new_tokens = request_output.outputs[0].token_ids
109
+ if request_id not in gen_strings:
110
+ gen_strings[request_id] = new_text
111
+ gen_tokens[request_id] = new_tokens
112
+ else:
113
+ gen_strings[request_id] += new_text
114
+ gen_tokens[request_id].extend(new_tokens)
115
+
116
+ # Confirmed tracked values matches what we expected.
117
+ for idx, (ref_gen_str, ref_gen_toks) in enumerate(
118
+ zip(GENERATION_STRINGS, GENERATION_TOKENS)):
119
+ gen_str = gen_strings[f"request-{idx}"]
120
+ gen_toks = gen_tokens[f"request-{idx}"]
121
+
122
+ assert gen_str == ref_gen_str, f"{gen_str=}, {ref_gen_str=}"
123
+ assert gen_toks == ref_gen_toks, f"{gen_toks=}, {ref_gen_toks=}"
124
+
125
+ assert detokenizer.get_num_unfinished_requests() == 0
126
+ assert not detokenizer.has_unfinished_requests()
127
+
128
+
129
+@pytest.mark.parametrize("include_stop_str_in_output", [True, False])
130
+def test_stop_string(include_stop_str_in_output: bool):
131
+ detokenizer = Detokenizer(TOKENIZER_NAME)
132
+ engine_core = MockEngineCore(GENERATION_TOKENS)
133
+
134
+ # Make N requests.
135
+ requests = [
136
+ DetokenizerRequest(
137
+ request_id=f"request-{idx}",
138
+ prompt=prompt,
139
+ prompt_token_ids=prompt_tokens,
140
+ skip_special_tokens=False,
141
+ spaces_between_special_tokens=False,
142
+ output_kind=RequestOutputKind.DELTA,
143
+ stop=STOP_STRINGS,
144
+ include_stop_str_in_output=include_stop_str_in_output,
145
+ ) for idx, (
146
+ prompt,
147
+ prompt_tokens) in enumerate(zip(PROMPT_STRINGS, PROMPT_TOKENS))
148
+ ]
149
+
150
+ # Add requests to the detokenizer.
151
+ for request in requests:
152
+ detokenizer.add_request(request)
153
+
154
+ gen_strings = {}
155
+ aborted = []
156
+ while True:
157
+ # Mock output from the EngineCore.
158
+ outputs = engine_core.get_outputs()
159
+ if len(outputs) == 0:
160
+ break
161
+
162
+ # Step the Detokenizer.
163
+ request_outputs, requests_to_abort = detokenizer.step(outputs)
164
+ for request_output in request_outputs:
165
+ # If aborted, we should not get a request output.
166
+ assert request_output.request_id not in aborted
167
+ aborted.extend(requests_to_abort)
168
+
169
+ # Update tracking.
170
+ for request_output in request_outputs:
171
+ if request_output.finished:
172
+ assert request_output.outputs[0].finish_reason == "stop"
173
+
174
+ request_id = request_output.request_id
175
+ new_text = request_output.outputs[0].text
176
+ if request_id not in gen_strings:
177
+ gen_strings[request_id] = new_text
178
+ else:
179
+ gen_strings[request_id] += new_text
180
+
181
+ # Confirmed tracked values matches what we expected.
182
+ for idx, (ref_gen_str,
183
+ stop_str) in enumerate(zip(GENERATION_STRINGS, STOP_STRINGS)):
184
+
185
+ # Request should be aborted.
186
+ request_id = f"request-{idx}"
187
+ assert request_id in aborted
188
+
189
+ # Collected values that were generated.
190
+ gen_str = gen_strings[request_id]
191
+
192
+ # Construct reference strings.
193
+ stop_str_idx = ref_gen_str.find(stop_str)
194
+ ref_str_exc_stop = ref_gen_str[:stop_str_idx]
195
+ ref_str_inc_stop = ref_gen_str[:stop_str_idx] + stop_str
196
+
197
+ if include_stop_str_in_output:
198
+ assert gen_str == ref_str_inc_stop, (
199
+ f"{gen_str=}, {ref_str_inc_stop=}")
200
+ else:
201
+ assert gen_str == ref_str_exc_stop, (
202
+ f"{gen_str=}, {ref_str_exc_stop=}")
203
+
204
+ assert detokenizer.get_num_unfinished_requests() == 0
205
+ assert not detokenizer.has_unfinished_requests()
tests/v1/engine/test_engine_core.py
new
+137
@@ -0,0 +1,137 @@
1
+import time
2
+import uuid
3
+
4
+import pytest
5
+from transformers import AutoTokenizer
6
+
7
+from vllm import SamplingParams
8
+from vllm.engine.arg_utils import EngineArgs
9
+from vllm.platforms import current_platform
10
+from vllm.usage.usage_lib import UsageContext
11
+from vllm.v1.engine import EngineCoreRequest
12
+from vllm.v1.engine.async_llm import AsyncLLM
13
+from vllm.v1.engine.core import EngineCore
14
+
15
+if not current_platform.is_cuda():
16
+ pytest.skip(reason="V1 currently only supported on CUDA.",
17
+ allow_module_level=True)
18
+
19
+MODEL_NAME = "meta-llama/Llama-3.2-1B-Instruct"
20
+TOKENIZER = AutoTokenizer.from_pretrained(MODEL_NAME)
21
+PROMPT = "Hello my name is Robert and I love quantization kernels"
22
+PROMPT_TOKENS = TOKENIZER(PROMPT).input_ids
23
+
24
+
25
+def make_request() -> EngineCoreRequest:
26
+ return EngineCoreRequest(
27
+ request_id=uuid.uuid4(),
28
+ prompt=PROMPT,
29
+ prompt_token_ids=PROMPT_TOKENS,
30
+ sampling_params=SamplingParams(),
31
+ eos_token_id=None,
32
+ arrival_time=time.time(),
33
+ lora_request=None,
34
+ )
35
+
36
+
37
+def test_engine_core(monkeypatch):
38
+
39
+ with monkeypatch.context() as m:
40
+ m.setenv("VLLM_USE_V1", "1")
41
+ """Setup the EngineCore."""
42
+ engine_args = EngineArgs(model=MODEL_NAME)
43
+ vllm_config = engine_args.create_engine_config()
44
+ executor_class = AsyncLLM._get_executor_cls(vllm_config)
45
+
46
+ engine_core = EngineCore(vllm_config=vllm_config,
47
+ executor_class=executor_class,
48
+ usage_context=UsageContext.UNKNOWN_CONTEXT)
49
+ """Test basic request lifecycle."""
50
+
51
+ # First request.
52
+ engine_core.add_request(make_request())
53
+ assert len(engine_core.scheduler.waiting) == 1
54
+ assert len(engine_core.scheduler.running) == 0
55
+
56
+ _ = engine_core.step()
57
+ assert len(engine_core.scheduler.waiting) == 0
58
+ assert len(engine_core.scheduler.running) == 1
59
+
60
+ # Second request.
61
+ engine_core.add_request(make_request())
62
+ assert len(engine_core.scheduler.waiting) == 1
63
+ assert len(engine_core.scheduler.running) == 1
64
+
65
+ _ = engine_core.step()
66
+ assert len(engine_core.scheduler.waiting) == 0
67
+ assert len(engine_core.scheduler.running) == 2
68
+
69
+ # Add two requests in a row.
70
+ engine_core.add_request(make_request())
71
+ engine_core.add_request(make_request())
72
+ assert len(engine_core.scheduler.waiting) == 2
73
+ assert len(engine_core.scheduler.running) == 2
74
+
75
+ _ = engine_core.step()
76
+ assert len(engine_core.scheduler.waiting) == 0
77
+ assert len(engine_core.scheduler.running) == 4
78
+
79
+ # Loop through until they are all done.
80
+ while len(engine_core.step()) > 0:
81
+ pass
82
+
83
+ assert len(engine_core.scheduler.waiting) == 0
84
+ assert len(engine_core.scheduler.running) == 0
85
+ """Test abort cycle."""
86
+
87
+ # Basic abort.
88
+ req = make_request()
89
+ request_id = req.request_id
90
+
91
+ engine_core.add_request(req)
92
+ assert len(engine_core.scheduler.waiting) == 1
93
+ assert len(engine_core.scheduler.running) == 0
94
+
95
+ _ = engine_core.step()
96
+ assert len(engine_core.scheduler.waiting) == 0
97
+ assert len(engine_core.scheduler.running) == 1
98
+
99
+ engine_core.abort_requests([request_id])
100
+ assert len(engine_core.scheduler.waiting) == 0
101
+ assert len(engine_core.scheduler.running) == 0
102
+
103
+ # Add, step, abort 1 of the 3.
104
+ req0 = make_request()
105
+ req1 = make_request()
106
+ req2 = make_request()
107
+
108
+ engine_core.add_request(req0)
109
+ engine_core.add_request(req1)
110
+ assert len(engine_core.scheduler.waiting) == 2
111
+ assert len(engine_core.scheduler.running) == 0
112
+
113
+ _ = engine_core.step()
114
+ assert len(engine_core.scheduler.waiting) == 0
115
+ assert len(engine_core.scheduler.running) == 2
116
+
117
+ engine_core.add_request(req2)
118
+ assert len(engine_core.scheduler.waiting) == 1
119
+ assert len(engine_core.scheduler.running) == 2
120
+
121
+ _ = engine_core.step()
122
+ assert len(engine_core.scheduler.waiting) == 0
123
+ assert len(engine_core.scheduler.running) == 3
124
+
125
+ # Abort just one.
126
+ engine_core.abort_requests([req1.request_id])
127
+ assert len(engine_core.scheduler.waiting) == 0
128
+ assert len(engine_core.scheduler.running) == 2
129
+
130
+ _ = engine_core.step()
131
+ assert len(engine_core.scheduler.waiting) == 0
132
+ assert len(engine_core.scheduler.running) == 2
133
+
134
+ # Abort the other requests at the same time.
135
+ engine_core.abort_requests([req2.request_id, req0.request_id])
136
+ assert len(engine_core.scheduler.waiting) == 0
137
+ assert len(engine_core.scheduler.running) == 0
tests/v1/engine/test_engine_core_client.py
new
+202
@@ -0,0 +1,202 @@
1
+import asyncio
2
+import time
3
+import uuid
4
+from typing import Dict, List
5
+
6
+import pytest
7
+from transformers import AutoTokenizer
8
+
9
+from vllm import SamplingParams
10
+from vllm.engine.arg_utils import EngineArgs
11
+from vllm.platforms import current_platform
12
+from vllm.usage.usage_lib import UsageContext
13
+from vllm.v1.engine import EngineCoreRequest
14
+from vllm.v1.engine.async_llm import AsyncLLM
15
+from vllm.v1.engine.core_client import EngineCoreClient
16
+
17
+if not current_platform.is_cuda():
18
+ pytest.skip(reason="V1 currently only supported on CUDA.",
19
+ allow_module_level=True)
20
+
21
+MODEL_NAME = "meta-llama/Llama-3.2-1B-Instruct"
22
+TOKENIZER = AutoTokenizer.from_pretrained(MODEL_NAME)
23
+PROMPT = "Hello my name is Robert and I love quantization kernels"
24
+PROMPT_TOKENS = TOKENIZER(PROMPT).input_ids
25
+
26
+
27
+def make_request(params: SamplingParams) -> EngineCoreRequest:
28
+ return EngineCoreRequest(
29
+ request_id=str(uuid.uuid4()),
30
+ prompt=PROMPT,
31
+ prompt_token_ids=PROMPT_TOKENS,
32
+ sampling_params=params,
33
+ eos_token_id=None,
34
+ arrival_time=time.time(),
35
+ lora_request=None,
36
+ )
37
+
38
+
39
+def loop_until_done(client: EngineCoreClient, outputs: Dict):
40
+
41
+ while True:
42
+ engine_core_outputs = client.get_output()
43
+
44
+ if len(engine_core_outputs) == 0:
45
+ break
46
+
47
+ all_finished = True
48
+ for out in engine_core_outputs:
49
+ outputs[out.request_id].append(out)
50
+ if not out.finished:
51
+ all_finished = False
52
+
53
+ if all_finished:
54
+ break
55
+
56
+
57
+async def loop_until_done_async(client: EngineCoreClient, outputs: Dict):
58
+
59
+ while True:
60
+ engine_core_outputs = await client.get_output_async()
61
+
62
+ if len(engine_core_outputs) == 0:
63
+ break
64
+
65
+ all_finished = True
66
+ for out in engine_core_outputs:
67
+ outputs[out.request_id].append(out)
68
+ if not out.finished:
69
+ all_finished = False
70
+
71
+ if all_finished:
72
+ break
73
+
74
+
75
+@pytest.mark.parametrize("multiprocessing_mode", [True, False])
76
+def test_engine_core_client(monkeypatch, multiprocessing_mode: bool):
77
+
78
+ with monkeypatch.context() as m:
79
+ m.setenv("VLLM_USE_V1", "1")
80
+
81
+ engine_args = EngineArgs(model=MODEL_NAME)
82
+ vllm_config = engine_args.create_engine_config()
83
+ executor_class = AsyncLLM._get_executor_cls(vllm_config)
84
+ client = EngineCoreClient.make_client(
85
+ vllm_config,
86
+ executor_class,
87
+ UsageContext.UNKNOWN_CONTEXT,
88
+ multiprocess_mode=multiprocessing_mode,
89
+ asyncio_mode=False,
90
+ )
91
+
92
+ MAX_TOKENS = 20
93
+ params = SamplingParams(max_tokens=MAX_TOKENS)
94
+ """Normal Request Cycle."""
95
+ requests = [make_request(params) for _ in range(10)]
96
+ request_ids = [req.request_id for req in requests]
97
+
98
+ # Add requests to the engine.
99
+ for request in requests:
100
+ client.add_request(request)
101
+ time.sleep(0.01)
102
+
103
+ outputs: Dict[str, List] = {req_id: [] for req_id in request_ids}
104
+ loop_until_done(client, outputs)
105
+
106
+ for req_id in request_ids:
107
+ assert len(outputs[req_id]) == MAX_TOKENS, (
108
+ f"{outputs[req_id]=}, {MAX_TOKENS=}")
109
+ """Abort Request Cycle."""
110
+
111
+ # Note: this code pathway will only work for multiprocessing
112
+ # since we have to call get_output() explicitly
113
+
114
+ # Add requests to the engine.
115
+ for idx, request in enumerate(requests):
116
+ client.add_request(request)
117
+ time.sleep(0.01)
118
+ if idx % 2 == 0:
119
+ client.abort_requests([request.request_id])
120
+
121
+ outputs = {req_id: [] for req_id in request_ids}
122
+ loop_until_done(client, outputs)
123
+
124
+ for idx, req_id in enumerate(request_ids):
125
+ if idx % 2 == 0:
126
+ assert len(outputs[req_id]) < MAX_TOKENS, (
127
+ f"{len(outputs[req_id])=}, {MAX_TOKENS=}")
128
+ else:
129
+ assert len(outputs[req_id]) == MAX_TOKENS, (
130
+ f"{len(outputs[req_id])=}, {MAX_TOKENS=}")
131
+ """Abort after request is finished."""
132
+
133
+ # Note: this code pathway will only work for multiprocessing
134
+ # since we have to call get_output() explicitly
135
+
136
+ request = requests[0]
137
+ client.add_request(request)
138
+ time.sleep(10.)
139
+
140
+ client.abort_requests([request.request_id])
141
+
142
+ # Shutdown the client.
143
+ client.shutdown()
144
+
145
+
146
+@pytest.mark.asyncio
147
+async def test_engine_core_client_asyncio(monkeypatch):
148
+
149
+ with monkeypatch.context() as m:
150
+ m.setenv("VLLM_USE_V1", "1")
151
+
152
+ engine_args = EngineArgs(model=MODEL_NAME)
153
+ vllm_config = engine_args.create_engine_config()
154
+ executor_class = AsyncLLM._get_executor_cls(vllm_config)
155
+ client = EngineCoreClient.make_client(
156
+ vllm_config,
157
+ executor_class,
158
+ UsageContext.UNKNOWN_CONTEXT,
159
+ multiprocess_mode=True,
160
+ asyncio_mode=True,
161
+ )
162
+
163
+ MAX_TOKENS = 20
164
+ params = SamplingParams(max_tokens=MAX_TOKENS)
165
+ """Normal Request Cycle."""
166
+
167
+ requests = [make_request(params) for _ in range(10)]
168
+ request_ids = [req.request_id for req in requests]
169
+
170
+ # Add requests to the engine.
171
+ for request in requests:
172
+ await client.add_request_async(request)
173
+ await asyncio.sleep(0.01)
174
+
175
+ outputs: Dict[str, List] = {req_id: [] for req_id in request_ids}
176
+ await loop_until_done_async(client, outputs)
177
+
178
+ for req_id in request_ids:
179
+ assert len(outputs[req_id]) == MAX_TOKENS, (
180
+ f"{outputs[req_id]=}, {MAX_TOKENS=}")
181
+ """Abort Request Cycle."""
182
+
183
+ # Add requests to the engine.
184
+ for idx, request in enumerate(requests):
185
+ await client.add_request_async(request)
186
+ await asyncio.sleep(0.01)
187
+ if idx % 2 == 0:
188
+ await client.abort_requests_async([request.request_id])
189
+
190
+ outputs = {req_id: [] for req_id in request_ids}
191
+ await loop_until_done_async(client, outputs)
192
+
193
+ for idx, req_id in enumerate(request_ids):
194
+ if idx % 2 == 0:
195
+ assert len(outputs[req_id]) < MAX_TOKENS, (
196
+ f"{len(outputs[req_id])=}, {MAX_TOKENS=}")
197
+ else:
198
+ assert len(outputs[req_id]) == MAX_TOKENS, (
199
+ f"{len(outputs[req_id])=}, {MAX_TOKENS=}")
200
+
201
+ # Shutdown the client.
202
+ client.shutdown()
vllm/config.py
+41
@@ -2106,3 +2106,44 @@ class VllmConfig:
2106
self.model_config is not None and self.load_config is not None:
2107
self.quant_config = VllmConfig._get_quantization_config(
2108
self.model_config, self.load_config)
2109
+
2110
+ def __str__(self):
2111
+ return ("model=%r, speculative_config=%r, tokenizer=%r, "
2112
+ "skip_tokenizer_init=%s, tokenizer_mode=%s, revision=%s, "
2113
+ "override_neuron_config=%s, tokenizer_revision=%s, "
2114
+ "trust_remote_code=%s, dtype=%s, max_seq_len=%d, "
2115
+ "download_dir=%r, load_format=%s, tensor_parallel_size=%d, "
2116
+ "pipeline_parallel_size=%d, "
2117
+ "disable_custom_all_reduce=%s, quantization=%s, "
2118
+ "enforce_eager=%s, kv_cache_dtype=%s, "
2119
+ "quantization_param_path=%s, device_config=%s, "
2120
+ "decoding_config=%r, observability_config=%r, "
2121
+ "seed=%d, served_model_name=%s, "
2122
+ "num_scheduler_steps=%d, enable_prefix_caching=%s, "
2123
+ "use_async_output_proc=%s, mm_processor_kwargs=%s") % \
2124
+ (self.model_config.model, self.speculative_config,
2125
+ self.model_config.tokenizer,
2126
+ self.model_config.skip_tokenizer_init,
2127
+ self.model_config.tokenizer_mode,
2128
+ self.model_config.revision,
2129
+ self.model_config.override_neuron_config,
2130
+ self.model_config.tokenizer_revision,
2131
+ self.model_config.trust_remote_code,
2132
+ self.model_config.dtype,
2133
+ self.model_config.max_model_len,
2134
+ self.load_config.download_dir,
2135
+ self.load_config.load_format,
2136
+ self.parallel_config.tensor_parallel_size,
2137
+ self.parallel_config.pipeline_parallel_size,
2138
+ self.parallel_config.disable_custom_all_reduce,
2139
+ self.model_config.quantization,
2140
+ self.model_config.enforce_eager,
2141
+ self.cache_config.cache_dtype,
2142
+ self.model_config.quantization_param_path,
2143
+ self.device_config.device, self.decoding_config,
2144
+ self.observability_config, self.model_config.seed,
2145
+ self.model_config.served_model_name,
2146
+ self.scheduler_config.num_scheduler_steps,
2147
+ self.cache_config.enable_prefix_caching,
2148
+ self.model_config.use_async_output_proc,
2149
+ self.model_config.mm_processor_kwargs)
\ No newline at end of file
vllm/engine/multiprocessing/engine.py
+2
-11
@@ -6,7 +6,6 @@ from typing import Iterator, List, Optional, Union
6
import cloudpickle
7
import zmq
8
9
-import vllm.envs
9
from vllm import AsyncEngineArgs, SamplingParams
10
from vllm.engine.llm_engine import LLMEngine
11
# yapf conflicts with isort for this block
@@ -113,17 +112,9 @@ class MQLLMEngine:
112
load_general_plugins()
113
114
engine_config = engine_args.create_engine_config()
116
- if vllm.envs.VLLM_USE_V1:
117
- # Lazy import: the v1 package isn't distributed
118
- from vllm.v1.engine.llm_engine import LLMEngine as V1LLMEngine
119
- engine_class = V1LLMEngine
120
- else:
121
- engine_class = LLMEngine
122
-
123
- executor_class = engine_class._get_executor_cls(engine_config)
115
+ executor_class = LLMEngine._get_executor_cls(engine_config)
116
125
- use_async_sockets = (engine_config.model_config.use_async_output_proc
126
- and not vllm.envs.VLLM_USE_V1)
117
+ use_async_sockets = engine_config.model_config.use_async_output_proc
118
119
return cls(ipc_path=ipc_path,
120
use_async_sockets=use_async_sockets,
vllm/engine/output_processor/stop_checker.py
+27
-16
@@ -1,4 +1,4 @@
1
-from typing import Callable, Optional
1
+from typing import Callable, List, Optional, Tuple
2
3
from vllm.lora.request import LoRARequest
4
from vllm.sampling_params import SamplingParams
@@ -67,9 +67,13 @@ class StopChecker:
67
return
68
69
# Check if any stop strings are matched.
70
- stop_str = self._check_stop_strings(seq, new_char_count,
71
- sampling_params)
72
- if stop_str is not None:
70
+ stop = self.check_stop_strings(
71
+ seq.output_text, new_char_count, sampling_params.stop,
72
+ sampling_params.include_stop_str_in_output)
73
+ if stop is not None:
74
+ stop_str, truncate_to = stop
75
+ if truncate_to != -1:
76
+ seq.output_text = seq.output_text[:truncate_to]
77
seq.status = SequenceStatus.FINISHED_STOPPED
78
seq.stop_reason = stop_str
79
return
@@ -85,33 +89,40 @@ class StopChecker:
89
return
90
91
@staticmethod
88
- def _check_stop_strings(seq: Sequence, new_char_count: int,
89
- sampling_params: SamplingParams) -> Optional[str]:
92
+ def check_stop_strings(
93
+ output_text: str,
94
+ new_char_count: int,
95
+ stop: List[str],
96
+ include_in_output: bool,
97
+ ) -> Optional[Tuple[str, int]]:
98
"""Check if any stop strings are matched and truncate sequence
99
output text accordingly.
100
93
- Returns the stop string if matched or else None.
101
+ Returns tuple (stop_string, offset) if matched or else None.
102
+
103
+ Where stop_string is the matched stop string and offset is the
104
+ length to which output_text should be truncated, or -1 for no
105
+ truncation.
106
"""
95
- if not new_char_count or not sampling_params.stop:
107
+ if not new_char_count or not stop:
108
return None
109
98
- for stop_str in sampling_params.stop:
110
+ for stop_str in stop:
111
stop_string_len = len(stop_str)
112
# Avoid searching already-searched text.
101
- stop_index = seq.output_text.find(
102
- stop_str, -new_char_count - stop_string_len)
113
+ stop_index = output_text.find(stop_str,
114
+ -new_char_count - stop_string_len)
115
if stop_index == -1:
116
continue
117
106
- if sampling_params.include_stop_str_in_output:
118
+ if include_in_output:
119
# Truncate to end of stop string.
120
stop_index += stop_string_len
109
- if stop_index >= len(seq.output_text):
121
+ if stop_index >= len(output_text):
122
# No truncation required.
111
- return stop_str
123
+ return stop_str, -1
124
125
# Truncate the output text to either the beginning
126
# or end of the stop string.
115
- seq.output_text = seq.output_text[:stop_index]
116
- return stop_str
127
+ return stop_str, stop_index
128
return None
vllm/entrypoints/llm.py
+3
@@ -210,8 +210,11 @@ class LLM:
210
# Logic to switch between engines is done at runtime instead of import
211
# to avoid import order issues
212
self.engine_class = self.get_engine_class()
213
+
214
+ # TODO(rob): enable mp by default (issue with fork vs spawn)
215
self.llm_engine = self.engine_class.from_engine_args(
216
engine_args, usage_context=UsageContext.LLM_CLASS)
217
+
218
self.request_counter = Counter()
219
220
@staticmethod
vllm/entrypoints/openai/api_server.py
+9
-2
@@ -26,7 +26,6 @@ from typing_extensions import assert_never
26
import vllm.envs as envs
27
from vllm.config import ModelConfig
28
from vllm.engine.arg_utils import AsyncEngineArgs
29
-from vllm.engine.async_llm_engine import AsyncLLMEngine
29
from vllm.engine.multiprocessing.client import MQLLMEngineClient
30
from vllm.engine.multiprocessing.engine import run_mp_engine
31
from vllm.engine.protocol import EngineClient
@@ -61,6 +60,11 @@ from vllm.usage.usage_lib import UsageContext
60
from vllm.utils import FlexibleArgumentParser, get_open_zmq_ipc_path
61
from vllm.version import __version__ as VLLM_VERSION
62
63
+if envs.VLLM_USE_V1:
64
+ from vllm.v1.engine.async_llm import AsyncLLMEngine # type: ignore
65
+else:
66
+ from vllm.engine.async_llm_engine import AsyncLLMEngine # type: ignore
67
+
68
TIMEOUT_KEEP_ALIVE = 5 # seconds
69
70
prometheus_multiproc_dir: tempfile.TemporaryDirectory
@@ -126,7 +130,8 @@ async def build_async_engine_client_from_engine_args(
130
# Fall back
131
# TODO: fill out feature matrix.
132
if (MQLLMEngineClient.is_unsupported_config(engine_args)
129
- or disable_frontend_multiprocessing):
133
+ or envs.VLLM_USE_V1 or disable_frontend_multiprocessing):
134
+
135
engine_config = engine_args.create_engine_config()
136
uses_ray = getattr(AsyncLLMEngine._get_executor_cls(engine_config),
137
"uses_ray", False)
@@ -143,6 +148,8 @@ async def build_async_engine_client_from_engine_args(
148
None, build_engine)
149
150
yield engine_client
151
+ if hasattr(engine_client, "shutdown"):
152
+ engine_client.shutdown()
153
return
154
155
# Otherwise, use the multiprocessing AsyncLLMEngine.
vllm/envs.py
+5
@@ -72,6 +72,7 @@ if TYPE_CHECKING:
72
VLLM_CUSTOM_OPS: List[str] = []
73
VLLM_DISABLED_KERNELS: List[str] = []
74
VLLM_USE_V1: bool = False
75
+ VLLM_ENABLE_V1_MULTIPROCESSING: bool = False
76
77
78
def get_default_cache_root():
@@ -473,6 +474,10 @@ environment_variables: Dict[str, Callable[[], Any]] = {
474
# If set, use the V1 code path.
475
"VLLM_USE_V1":
476
lambda: bool(int(os.getenv("VLLM_USE_V1", "0"))),
477
+
478
+ # If set, enable multiprocessing in LLM for the V1 code path.
479
+ "VLLM_ENABLE_V1_MULTIPROCESSING":
480
+ lambda: bool(int(os.getenv("VLLM_ENABLE_V1_MULTIPROCESSING", "0"))),
481
}
482
483
# end-env-vars-definition
vllm/outputs.py
+30
@@ -113,6 +113,36 @@ class RequestOutput:
113
self.encoder_prompt = encoder_prompt
114
self.encoder_prompt_token_ids = encoder_prompt_token_ids
115
116
+ @classmethod
117
+ def new(
118
+ cls,
119
+ request_id: str,
120
+ prompt: Optional[str],
121
+ prompt_token_ids: Optional[List[int]],
122
+ text: str,
123
+ token_ids: List[int],
124
+ finished: bool = False,
125
+ ) -> "RequestOutput":
126
+ """Initialize a new RequestOutput object."""
127
+
128
+ # TODO: Support `n` > 1.
129
+ completion_output = CompletionOutput(
130
+ index=0,
131
+ text=text,
132
+ token_ids=token_ids,
133
+ cumulative_logprob=None,
134
+ logprobs=None, # TODO
135
+ )
136
+
137
+ return RequestOutput(
138
+ request_id=request_id,
139
+ prompt=prompt,
140
+ prompt_token_ids=prompt_token_ids,
141
+ prompt_logprobs=None, # TODO
142
+ outputs=[completion_output],
143
+ finished=finished,
144
+ )
145
+
146
@classmethod
147
def from_seq_group(
148
cls, seq_group: SequenceGroup, use_cache: bool,
vllm/v1/__init__.py
vllm/v1/core/kv_cache_manager.py
+7
-6
@@ -70,7 +70,7 @@ class KVCacheManager:
70
71
Args:
72
request: The request to get the computed blocks.
73
-
73
+
74
Returns:
75
A list of blocks that are computed for the request.
76
"""
@@ -105,7 +105,7 @@ class KVCacheManager:
105
Args:
106
request: The request to append slots.
107
num_tokens: The number of tokens to append.
108
-
108
+
109
Returns:
110
A list of new blocks if new blocks are allocated, or None
111
if new blocks are required but cannot be allocated.
@@ -176,7 +176,7 @@ class KVCacheManager:
176
num_tokens: The number of tokens to allocate. Note that this does
177
not include the tokens that have already been computed.
178
computed_blocks: The blocks that have already been computed.
179
-
179
+
180
Returns:
181
A list of new allocated blocks.
182
"""
@@ -240,7 +240,8 @@ class KVCacheManager:
240
Args:
241
request: The request to free the blocks.
242
"""
243
- blocks = self.req_to_blocks.pop(request.request_id)
243
+ # Default to [] in case a request is freed (aborted) before alloc.
244
+ blocks = self.req_to_blocks.pop(request.request_id, [])
245
if self.enable_caching:
246
# Free blocks in reverse order so that the tail blocks are
247
# freed first.
@@ -259,13 +260,13 @@ class KVCacheManager:
260
"""Get new blocks from the free block pool, and add token IDs to
261
allocated blocks if caching is enabled.
262
Note that we do not check block cache in this function.
262
-
263
+
264
Args:
265
num_blocks: The number of blocks to allocate.
266
token_ids: The token IDs in the blocks. None if caching is disabled.
267
parent_block: The parent block. Used to include block chain
268
in the block hash.
268
-
269
+
270
Returns:
271
A list of new block.
272
"""
vllm/v1/core/scheduler.py
+19
-7
@@ -1,12 +1,13 @@
1
from collections import deque
2
from dataclasses import dataclass
3
-from typing import Deque, Dict, Iterable, List, Optional, Set, Tuple, Union
3
+from typing import Deque, Dict, Iterable, List, Optional, Set, Union
4
5
from vllm.config import CacheConfig, LoRAConfig, SchedulerConfig
6
from vllm.logger import init_logger
7
from vllm.multimodal import MultiModalDataDict
8
from vllm.sampling_params import SamplingParams
9
from vllm.v1.core.kv_cache_manager import KVCacheManager
10
+from vllm.v1.engine import EngineCoreOutput
11
from vllm.v1.outputs import ModelRunnerOutput
12
from vllm.v1.request import Request, RequestStatus
13
@@ -237,13 +238,12 @@ class Scheduler:
238
self,
239
scheduler_output: "SchedulerOutput",
240
model_runner_output: "ModelRunnerOutput",
240
- ) -> List[Tuple[Request, int]]:
241
+ ) -> List[EngineCoreOutput]:
242
# NOTE(woosuk): This method doesn't consider speculative decoding.
243
sampled_token_ids = model_runner_output.sampled_token_ids_cpu.tolist()
244
num_scheduled_tokens = scheduler_output.num_scheduled_tokens
245
new_running: List[Request] = []
245
- # (request, num_sampled_tokens)
246
- sampled: List[Tuple[Request, int]] = []
246
+ engine_core_outputs: List[EngineCoreOutput] = []
247
for request in self.running:
248
req_id = request.request_id
249
request.num_computed_tokens += num_scheduled_tokens[req_id]
@@ -257,17 +257,29 @@ class Scheduler:
257
# generates at most one token at each step.
258
token_id = sampled_token_ids[req_index]
259
request.append_output_token_ids(token_id)
260
- sampled.append((request, 1))
260
+ num_new_tokens = 1
261
# TODO: Update the KV cache manager for prefix caching.
262
263
- # Check if the request is finished.
263
+ # Check for stop and update request state.
264
+ # This must be called before me make the EngineCoreOutput.
265
stopped = self._check_stop(request)
266
+
267
+ # Add EngineCoreOutput for this Request.
268
+ output = EngineCoreOutput(
269
+ request_id=req_id,
270
+ new_token_ids=request.output_token_ids[-num_new_tokens:],
271
+ finished=request.is_finished(),
272
+ finish_reason=request.get_finished_reason(),
273
+ stop_reason=request.stop_reason)
274
+ engine_core_outputs.append(output)
275
+
276
+ # Breakout of the loop.
277
if stopped:
278
continue
279
280
new_running.append(request)
281
self.running = new_running
270
- return sampled
282
+ return engine_core_outputs
283
284
def _check_stop(self, request: Request) -> bool:
285
if (request.num_tokens >= self.max_model_len
vllm/v1/engine/__init__.py
+72
@@ -0,0 +1,72 @@
1
+import enum
2
+from dataclasses import dataclass
3
+from typing import List, Optional, Union
4
+
5
+import msgspec
6
+
7
+from vllm.lora.request import LoRARequest
8
+from vllm.sampling_params import RequestOutputKind, SamplingParams
9
+
10
+
11
+@dataclass
12
+class DetokenizerRequest:
13
+
14
+ request_id: str
15
+ prompt: Optional[str]
16
+ prompt_token_ids: List[int]
17
+ skip_special_tokens: bool
18
+ spaces_between_special_tokens: bool
19
+ output_kind: RequestOutputKind
20
+
21
+ stop: List[str]
22
+ include_stop_str_in_output: bool
23
+
24
+
25
+class EngineCoreRequest(msgspec.Struct, omit_defaults=True):
26
+
27
+ # NOTE: prompt and prompt_token_ids should be DecoderOnlyInput,
28
+ # but this object is currently not playing well with msgspec
29
+ # due to circular imports and typing we have in data.py
30
+
31
+ request_id: str
32
+ #NOTE(Nick): I don't think we need to pass prompt here since it should
33
+ # always be tokenized?
34
+ prompt: Optional[str]
35
+ prompt_token_ids: List[int]
36
+ sampling_params: SamplingParams
37
+ eos_token_id: Optional[int]
38
+ arrival_time: float
39
+ lora_request: Optional[LoRARequest]
40
+
41
+
42
+class EngineCoreOutput(msgspec.Struct,
43
+ array_like=True,
44
+ omit_defaults=True,
45
+ gc=False):
46
+
47
+ request_id: str
48
+ new_token_ids: List[int]
49
+ finished: bool
50
+ finish_reason: Optional[str] = None
51
+ stop_reason: Union[int, str, None] = None
52
+
53
+
54
+class EngineCoreOutputs(msgspec.Struct,
55
+ array_like=True,
56
+ omit_defaults=True,
57
+ gc=False):
58
+
59
+ #NOTE(Nick): We could consider ways to make this more compact,
60
+ # e.g. columnwise layout and using an int enum for finish/stop reason
61
+
62
+ # [num_reqs]
63
+ outputs: List[EngineCoreOutput]
64
+
65
+
66
+class EngineCoreRequestType(enum.Enum):
67
+ """
68
+ Request types defined as hex byte strings, so it can be sent over sockets
69
+ without separate encoding step.
70
+ """
71
+ ADD = b'\x00'
72
+ ABORT = b'\x01'
vllm/v1/engine/async_llm.py
new
+368
@@ -0,0 +1,368 @@
1
+import asyncio
2
+from typing import AsyncGenerator, Dict, List, Mapping, Optional, Type, Union
3
+
4
+from vllm.config import ModelConfig, VllmConfig
5
+from vllm.engine.arg_utils import AsyncEngineArgs
6
+from vllm.engine.metrics_types import StatLoggerBase
7
+from vllm.engine.protocol import EngineClient
8
+from vllm.inputs import INPUT_REGISTRY, InputRegistry, PromptType
9
+from vllm.logger import init_logger
10
+from vllm.lora.request import LoRARequest
11
+from vllm.outputs import EmbeddingRequestOutput, RequestOutput
12
+from vllm.pooling_params import PoolingParams
13
+from vllm.prompt_adapter.request import PromptAdapterRequest
14
+from vllm.sampling_params import SamplingParams
15
+from vllm.transformers_utils.tokenizer import AnyTokenizer
16
+from vllm.transformers_utils.tokenizer_group import init_tokenizer_from_configs
17
+from vllm.usage.usage_lib import UsageContext
18
+from vllm.v1.engine.async_stream import AsyncStream
19
+from vllm.v1.engine.core_client import EngineCoreClient
20
+from vllm.v1.engine.detokenizer import Detokenizer
21
+from vllm.v1.engine.processor import Processor
22
+from vllm.v1.executor.gpu_executor import GPUExecutor
23
+
24
+logger = init_logger(__name__)
25
+
26
+
27
+class AsyncLLM(EngineClient):
28
+
29
+ def __init__(
30
+ self,
31
+ vllm_config: VllmConfig,
32
+ executor_class: Type[GPUExecutor],
33
+ log_stats: bool,
34
+ usage_context: UsageContext = UsageContext.ENGINE_CONTEXT,
35
+ stat_loggers: Optional[Dict[str, StatLoggerBase]] = None,
36
+ input_registry: InputRegistry = INPUT_REGISTRY,
37
+ use_cached_outputs: bool = False,
38
+ log_requests: bool = True,
39
+ start_engine_loop: bool = True,
40
+ ) -> None:
41
+ assert start_engine_loop
42
+
43
+ self.log_requests = log_requests
44
+ self.log_stats = log_stats
45
+ self.stat_loggers = stat_loggers
46
+ self.model_config = vllm_config.model_config
47
+
48
+ # Tokenizer (+ ensure liveness if running in another process).
49
+ self.tokenizer = init_tokenizer_from_configs(
50
+ model_config=vllm_config.model_config,
51
+ scheduler_config=vllm_config.scheduler_config,
52
+ parallel_config=vllm_config.parallel_config,
53
+ enable_lora=bool(vllm_config.lora_config))
54
+ self.tokenizer.ping()
55
+
56
+ # Request streams (map of request_id -> AsyncStream).
57
+ self.request_streams: Dict[str, AsyncStream] = {}
58
+ # List of cancelled request ids to be aborted.
59
+ self.client_aborted_requests: List[str] = []
60
+
61
+ # Processor (converts Inputs --> EngineCoreRequests).
62
+ self.processor = Processor(vllm_config.model_config,
63
+ vllm_config.lora_config, self.tokenizer,
64
+ input_registry)
65
+
66
+ # Detokenizer (converts EngineCoreOutputs --> RequestOutput).
67
+ self.detokenizer = Detokenizer(vllm_config.model_config.tokenizer)
68
+
69
+ # EngineCore (starts the engine in background process).
70
+ self.engine_core = EngineCoreClient.make_client(
71
+ vllm_config=vllm_config,
72
+ executor_class=executor_class,
73
+ usage_context=usage_context,
74
+ multiprocess_mode=True,
75
+ asyncio_mode=True,
76
+ )
77
+
78
+ self.output_handler = None
79
+
80
+ def __del__(self):
81
+ self.shutdown()
82
+
83
+ @classmethod
84
+ def from_engine_args(
85
+ cls,
86
+ engine_args: AsyncEngineArgs,
87
+ engine_config: Optional[VllmConfig] = None,
88
+ start_engine_loop: bool = True,
89
+ usage_context: UsageContext = UsageContext.ENGINE_CONTEXT,
90
+ stat_loggers: Optional[Dict[str, StatLoggerBase]] = None,
91
+ ) -> "AsyncLLMEngine":
92
+ """Create an AsyncLLM from the EngineArgs."""
93
+
94
+ # Create the engine configs.
95
+ if engine_config is None:
96
+ vllm_config = engine_args.create_engine_config()
97
+ else:
98
+ vllm_config = engine_config
99
+
100
+ executor_class = cls._get_executor_cls(vllm_config)
101
+
102
+ # Create the AsyncLLM.
103
+ return cls(
104
+ vllm_config=vllm_config,
105
+ executor_class=executor_class,
106
+ log_requests=not engine_args.disable_log_requests,
107
+ log_stats=not engine_args.disable_log_stats,
108
+ start_engine_loop=start_engine_loop,
109
+ usage_context=usage_context,
110
+ stat_loggers=stat_loggers,
111
+ )
112
+
113
+ def shutdown(self):
114
+ """Shutdown, cleaning up the background proc and IPC."""
115
+
116
+ self.engine_core.shutdown()
117
+
118
+ if handler := getattr(self, "output_handler", None):
119
+ handler.cancel()
120
+
121
+ @classmethod
122
+ def _get_executor_cls(cls, vllm_config: VllmConfig):
123
+ return GPUExecutor
124
+
125
+ async def add_request(
126
+ self,
127
+ request_id: str,
128
+ prompt: PromptType,
129
+ params: Union[SamplingParams, PoolingParams],
130
+ arrival_time: Optional[float] = None,
131
+ lora_request: Optional[LoRARequest] = None,
132
+ trace_headers: Optional[Mapping[str, str]] = None,
133
+ prompt_adapter_request: Optional[PromptAdapterRequest] = None,
134
+ priority: int = 0,
135
+ ) -> AsyncGenerator[Union[RequestOutput, EmbeddingRequestOutput], None]:
136
+ """Add new request to the AsyncLLM."""
137
+
138
+ if self.detokenizer.is_request_active(request_id):
139
+ raise KeyError(f"Request {request_id} already exists.")
140
+
141
+ # 1) Create a new AsyncStream for the request.
142
+ stream = self._add_request_to_streams(request_id)
143
+
144
+ # 2) Convert input --> DetokenizerRequest / EngineCoreRequest.
145
+ detokenizer_req, engine_core_req = self.processor.process_inputs(
146
+ request_id, prompt, params, arrival_time, lora_request,
147
+ trace_headers, prompt_adapter_request, priority)
148
+
149
+ # 3) Add the request to Detokenizer (this process).
150
+ self.detokenizer.add_request(detokenizer_req)
151
+
152
+ # 4) Add the EngineCoreRequest to EngineCore (separate process).
153
+ await self.engine_core.add_request_async(engine_core_req)
154
+
155
+ # 5) Return the generator.
156
+ return stream.generator()
157
+
158
+ # TODO: we should support multiple prompts in one call, as you
159
+ # can do with LLM.generate. So that for multi-prompt completion
160
+ # requests we don't need to send multiple messages to core proc,
161
+ # and so we don't need multiple streams which then get
162
+ # re-multiplexed in the API server anyhow.
163
+ async def generate(
164
+ self,
165
+ prompt: PromptType,
166
+ sampling_params: SamplingParams,
167
+ request_id: str,
168
+ lora_request: Optional[LoRARequest] = None,
169
+ trace_headers: Optional[Mapping[str, str]] = None,
170
+ prompt_adapter_request: Optional[PromptAdapterRequest] = None,
171
+ priority: int = 0,
172
+ ) -> AsyncGenerator[RequestOutput, None]:
173
+ """
174
+ Main function called by the API server to kick off a request
175
+ * 1) Making an AsyncStream corresponding to the Request.
176
+ # 2) Processing the Input.
177
+ * 3) Adding the Request to the Detokenizer.
178
+ * 4) Adding the Request to the EngineCore (separate process).
179
+
180
+ A separate output_handler loop runs in a background AsyncIO task,
181
+ pulling outputs from EngineCore and putting them into the
182
+ per-request AsyncStream.
183
+
184
+ The caller of generate() iterates the returned AsyncGenerator,
185
+ returning the RequestOutput back to the caller.
186
+ """
187
+
188
+ # We start the output_handler on the first call to generate() so that
189
+ # we can call __init__ before the event loop starts, which enables us
190
+ # to handle startup failure gracefully in the OpenAI server.
191
+ if self.output_handler is None:
192
+ self.output_handler = asyncio.create_task(
193
+ self._run_output_handler())
194
+
195
+ async for output in await self.add_request(
196
+ request_id,
197
+ prompt,
198
+ sampling_params,
199
+ lora_request=lora_request,
200
+ trace_headers=trace_headers,
201
+ prompt_adapter_request=prompt_adapter_request,
202
+ priority=priority,
203
+ ):
204
+ yield output
205
+
206
+ def _finish_stream(self, request_id: str):
207
+ stream = self.request_streams.pop(request_id, None)
208
+ if stream is not None:
209
+ stream.finish()
210
+
211
+ def _add_request_to_streams(
212
+ self,
213
+ request_id: str,
214
+ ) -> AsyncStream:
215
+
216
+ if request_id in self.request_streams:
217
+ raise ValueError(f"Request id {request_id} already running.")
218
+
219
+ # Avoid streams having circular ref to parent AsyncLLM object.
220
+ aborted_reqs = self.client_aborted_requests
221
+ stream = AsyncStream(request_id, aborted_reqs.append)
222
+ self.request_streams[request_id] = stream
223
+
224
+ if self.log_requests:
225
+ logger.info("Added request %s.", request_id)
226
+
227
+ return stream
228
+
229
+ async def _process_cancellations(self) -> None:
230
+ """
231
+ Process requests cancelled from user disconnecting.
232
+
233
+ When a client disconnects, AsyncStream._cancel() is called.
234
+ We passed a callback to AsyncStream(), which appends to
235
+ self.client_aborted_requests.
236
+
237
+ As a result, if any requests are canceled from the user side
238
+ the request_id will show up in self.client_aborted_requests.
239
+ """
240
+
241
+ # Avoid streams having circular ref to parent AsyncLLM object.
242
+ if not self.client_aborted_requests:
243
+ return
244
+ reqs_to_abort = self.client_aborted_requests.copy()
245
+ self.client_aborted_requests.clear()
246
+
247
+ # Remove from Detokenizer.
248
+ self.detokenizer.abort_requests(reqs_to_abort)
249
+
250
+ # Remove from RequestStreams.
251
+ for request_id in reqs_to_abort:
252
+ if self.log_requests:
253
+ logger.info("User-cancelled request %s.", request_id)
254
+ self._finish_stream(request_id)
255
+
256
+ # Remove from EngineCore.
257
+ await self.engine_core.abort_requests_async(reqs_to_abort)
258
+
259
+ def _process_request_outputs(self, request_outputs: List[RequestOutput]):
260
+ """Process outputs by putting them into per-request AsyncStreams."""
261
+
262
+ for request_output in request_outputs:
263
+ request_id = request_output.request_id
264
+ assert request_id in self.request_streams
265
+
266
+ # Each request in the API server pulls from the per-request stream.
267
+ stream = self.request_streams.get(request_id)
268
+ if stream is not None:
269
+ stream.put(request_output)
270
+
271
+ # If finished, remove from the tracker.
272
+ if request_output.finished:
273
+ if self.log_requests:
274
+ logger.info("Finished request %s.", request_id)
275
+ self._finish_stream(request_id)
276
+
277
+ async def _run_output_handler(self):
278
+ """Background loop: pulls from EngineCore and pushes to AsyncStreams."""
279
+
280
+ try:
281
+ while True:
282
+ # 1) Pull EngineCoreOutput from the EngineCore.
283
+ outputs = await self.engine_core.get_output_async()
284
+
285
+ # 2) Detokenize based on the output.
286
+ request_outputs, reqs_to_abort = self.detokenizer.step(outputs)
287
+
288
+ # 3) Put the RequestOutputs into the per-request AsyncStreams.
289
+ self._process_request_outputs(request_outputs)
290
+
291
+ # 4) Abort any requests that finished due to stop strings.
292
+ await self.engine_core.abort_requests_async(reqs_to_abort)
293
+
294
+ # 5) Abort any requests due to client cancellations.
295
+ await self._process_cancellations()
296
+
297
+ except BaseException as e:
298
+ logger.error(e)
299
+ raise e
300
+
301
+ # TODO: can we eliminate these?
302
+
303
+ async def abort(self, request_id: str) -> None:
304
+ # Note: Who Calls this? I dont think this is actually used.
305
+ raise ValueError("Not Supported on V1 yet.")
306
+
307
+ def encode(
308
+ self,
309
+ prompt: PromptType,
310
+ pooling_params: PoolingParams,
311
+ request_id: str,
312
+ lora_request: Optional[LoRARequest] = None,
313
+ trace_headers: Optional[Mapping[str, str]] = None,
314
+ priority: int = 0,
315
+ ):
316
+ raise ValueError("Not Supported on V1 yet.")
317
+
318
+ async def get_model_config(self) -> ModelConfig:
319
+ return self.model_config
320
+
321
+ async def get_decoding_config(self):
322
+ raise ValueError("Not Supported on V1 yet.")
323
+
324
+ async def get_tokenizer(
325
+ self,
326
+ lora_request: Optional[LoRARequest] = None,
327
+ ) -> AnyTokenizer:
328
+ assert lora_request is None
329
+ return self.detokenizer.tokenizer
330
+
331
+ async def is_tracing_enabled(self) -> bool:
332
+ return False
333
+
334
+ async def do_log_stats(
335
+ self,
336
+ scheduler_outputs=None,
337
+ model_output=None,
338
+ ) -> None:
339
+ logger.debug("Called do_log_stats.")
340
+
341
+ async def check_health(self) -> None:
342
+ logger.debug("Called check_health.")
343
+
344
+ async def start_profile(self) -> None:
345
+ raise ValueError("Not supported on V1 yet.")
346
+
347
+ async def stop_profile(self) -> None:
348
+ raise ValueError("Not supported on V1 yet.")
349
+
350
+ @property
351
+ def is_running(self) -> bool:
352
+ return True
353
+
354
+ @property
355
+ def is_stopped(self) -> bool:
356
+ return False
357
+
358
+ @property
359
+ def errored(self) -> bool:
360
+ return False
361
+
362
+ @property
363
+ def dead_error(self) -> BaseException:
364
+ return Exception
365
+
366
+
367
+# Retain V0 name for backwards compatibility.
368
+AsyncLLMEngine = AsyncLLM
vllm/v1/engine/async_stream.py
new
+55
@@ -0,0 +1,55 @@
1
+import asyncio
2
+from typing import Any, AsyncGenerator, Callable, Optional, Type, Union
3
+
4
+from vllm.outputs import EmbeddingRequestOutput, RequestOutput
5
+
6
+
7
+class AsyncStream:
8
+ """A stream of RequestOutputs or EmbeddingRequestOutputs for a request
9
+ that can be iterated over asynchronously via an async generator."""
10
+
11
+ STOP_ITERATION = Exception() # Sentinel
12
+
13
+ def __init__(self, request_id: str, cancel: Callable[[str], None]) -> None:
14
+ self.request_id = request_id
15
+ self._cancel = cancel
16
+ self._queue: asyncio.Queue = asyncio.Queue()
17
+ self._finished = False
18
+
19
+ def put(self, item: Union[RequestOutput, EmbeddingRequestOutput,
20
+ Exception]) -> None:
21
+ if not self._finished:
22
+ self._queue.put_nowait(item)
23
+
24
+ def finish(
25
+ self,
26
+ exception: Optional[Union[BaseException, Type[BaseException]]] = None,
27
+ ) -> None:
28
+ if not self._finished:
29
+ self._finished = True
30
+ self._queue.put_nowait(exception if self._is_raisable(exception)
31
+ else AsyncStream.STOP_ITERATION)
32
+
33
+ async def generator(
34
+ self
35
+ ) -> AsyncGenerator[Union[RequestOutput, EmbeddingRequestOutput], None]:
36
+ finished = False
37
+ try:
38
+ while True:
39
+ result = await self._queue.get()
40
+ if self._is_raisable(result):
41
+ finished = True
42
+ if result == AsyncStream.STOP_ITERATION:
43
+ return
44
+ raise result
45
+ yield result
46
+ finally:
47
+ self._finished = True
48
+ if not finished:
49
+ self._cancel(self.request_id)
50
+
51
+ @staticmethod
52
+ def _is_raisable(value: Any):
53
+ return isinstance(value, BaseException) or \
54
+ (isinstance(value, type) and \
55
+ issubclass(value, BaseException))
vllm/v1/engine/core.py
new
+352
@@ -0,0 +1,352 @@
1
+import multiprocessing
2
+import queue
3
+import threading
4
+import time
5
+from contextlib import contextmanager
6
+from multiprocessing.process import BaseProcess
7
+from multiprocessing.sharedctypes import Synchronized
8
+from typing import Any, Iterator, List, Tuple, Type, Union
9
+
10
+import zmq
11
+import zmq.asyncio
12
+from msgspec import msgpack
13
+
14
+from vllm.config import CacheConfig, VllmConfig
15
+from vllm.logger import init_logger
16
+from vllm.usage.usage_lib import UsageContext
17
+from vllm.v1.core.scheduler import Scheduler
18
+from vllm.v1.engine import (EngineCoreOutput, EngineCoreOutputs,
19
+ EngineCoreRequest, EngineCoreRequestType)
20
+from vllm.v1.executor.gpu_executor import GPUExecutor
21
+from vllm.v1.request import Request, RequestStatus
22
+from vllm.version import __version__ as VLLM_VERSION
23
+
24
+logger = init_logger(__name__)
25
+
26
+POLLING_TIMEOUT_MS = 5000
27
+POLLING_TIMEOUT_S = POLLING_TIMEOUT_MS // 1000
28
+LOGGING_TIME_S = 5000
29
+
30
+
31
+class EngineCore:
32
+ """Inner loop of vLLM's Engine."""
33
+
34
+ def __init__(
35
+ self,
36
+ vllm_config: VllmConfig,
37
+ executor_class: Type[GPUExecutor],
38
+ usage_context: UsageContext,
39
+ ):
40
+ # Override the configs for V1.
41
+ # FIXME
42
+ if usage_context == UsageContext.LLM_CLASS:
43
+ vllm_config.scheduler_config.max_num_seqs = 1024
44
+ vllm_config.scheduler_config.max_num_batched_tokens = 8192
45
+ elif usage_context == UsageContext.OPENAI_API_SERVER:
46
+ vllm_config.scheduler_config.max_num_seqs = 1024
47
+ vllm_config.scheduler_config.max_num_batched_tokens = 2048
48
+
49
+ # TODO (ywang96): Enable APC by default when VLM supports it.
50
+ if not vllm_config.model_config.is_multimodal_model:
51
+ vllm_config.cache_config.enable_prefix_caching = True
52
+
53
+ assert vllm_config.model_config.task != "embedding"
54
+
55
+ logger.info("Initializing an LLM engine (v%s) with config: %s",
56
+ VLLM_VERSION, vllm_config)
57
+
58
+ # Setup Model.
59
+ self.model_executor = executor_class(vllm_config)
60
+
61
+ # Setup KV Caches and update CacheConfig after profiling.
62
+ num_gpu_blocks, num_cpu_blocks = self._initialize_kv_caches(
63
+ vllm_config.cache_config)
64
+ vllm_config.cache_config.num_gpu_blocks = num_gpu_blocks
65
+ vllm_config.cache_config.num_cpu_blocks = num_cpu_blocks
66
+
67
+ # Setup scheduler.
68
+ self.scheduler = Scheduler(vllm_config.scheduler_config,
69
+ vllm_config.cache_config,
70
+ vllm_config.lora_config)
71
+
72
+ self._last_logging_time = time.time()
73
+
74
+ def _initialize_kv_caches(self,
75
+ cache_config: CacheConfig) -> Tuple[int, int]:
76
+ num_gpu_blocks, _ = self.model_executor.determine_num_available_blocks(
77
+ )
78
+
79
+ if cache_config.num_gpu_blocks_override is not None:
80
+ num_gpu_blocks_override = cache_config.num_gpu_blocks_override
81
+ logger.info(
82
+ "Overriding num_gpu_blocks=%d with "
83
+ "num_gpu_blocks_override=%d", num_gpu_blocks,
84
+ num_gpu_blocks_override)
85
+ num_gpu_blocks = num_gpu_blocks_override
86
+
87
+ num_cpu_blocks = 0
88
+ self.model_executor.initialize_cache(num_gpu_blocks)
89
+ return num_gpu_blocks, num_cpu_blocks
90
+
91
+ def add_request(self, request: EngineCoreRequest):
92
+ """Add request to the scheduler."""
93
+
94
+ req = Request.from_engine_core_request(request)
95
+ self.scheduler.add_request(req)
96
+
97
+ def abort_requests(self, request_ids: List[str]):
98
+ """Abort requests from the scheduler."""
99
+
100
+ # TODO: The scheduler doesn't really need to know the
101
+ # specific finish reason, TBD whether we propagate that
102
+ # (i.e. client-aborted vs stop criteria met).
103
+ self.scheduler.finish_requests(request_ids,
104
+ RequestStatus.FINISHED_ABORTED)
105
+
106
+ def step(self) -> List[EngineCoreOutput]:
107
+ """Schedule, execute, and make output."""
108
+
109
+ if not self.scheduler.has_unfinished_requests():
110
+ return []
111
+
112
+ scheduler_output = self.scheduler.schedule()
113
+ output = self.model_executor.execute_model(scheduler_output)
114
+ engine_core_outputs = self.scheduler.update_from_output(
115
+ scheduler_output, output)
116
+ return engine_core_outputs
117
+
118
+
119
+class EngineCoreProc(EngineCore):
120
+ """ZMQ-wrapper for running EngineCore in background process."""
121
+
122
+ READY_STR = "READY"
123
+
124
+ def __init__(
125
+ self,
126
+ vllm_config: VllmConfig,
127
+ executor_class: Type[GPUExecutor],
128
+ usage_context: UsageContext,
129
+ input_path: str,
130
+ output_path: str,
131
+ ready_path: str,
132
+ should_shutdown: Synchronized,
133
+ ):
134
+ super().__init__(vllm_config, executor_class, usage_context)
135
+
136
+ # Signal from main process to shutdown (multiprocessing.Value).
137
+ self.should_shutdown = should_shutdown
138
+
139
+ # Background Threads and Queues for IO. These enable us to
140
+ # overlap ZMQ socket IO with GPU since they release the GIL,
141
+ # and to overlap some serialization/deserialization with the
142
+ # model forward pass.
143
+ # Threads handle Socket <-> Queues and core_busy_loop uses Queue.
144
+ self.input_queue = queue.Queue()
145
+ self.output_queue = queue.Queue()
146
+ threading.Thread(target=self.process_input_socket,
147
+ args=(input_path, ),
148
+ daemon=True).start()
149
+ threading.Thread(target=self.process_output_socket,
150
+ args=(output_path, ),
151
+ daemon=True).start()
152
+
153
+ # Send Readiness signal to EngineClient.
154
+ with self.make_socket(ready_path, zmq.constants.PUSH) as ready_socket:
155
+ ready_socket.send_string(EngineCoreProc.READY_STR)
156
+
157
+ @contextmanager
158
+ def make_socket(self, path: str, type: Any) -> Iterator[zmq.Socket]:
159
+ """Context manager for use """
160
+
161
+ ctx = zmq.Context()
162
+ try:
163
+ socket = ctx.socket(type)
164
+
165
+ if type == zmq.constants.PULL:
166
+ socket.connect(path)
167
+ elif type == zmq.constants.PUSH:
168
+ socket.bind(path)
169
+ else:
170
+ raise ValueError(f"Unknown Socket Type: {type}")
171
+
172
+ yield socket
173
+
174
+ except KeyboardInterrupt:
175
+ logger.debug("EngineCore had Keyboard Interrupt.")
176
+
177
+ finally:
178
+ ctx.destroy(linger=0)
179
+
180
+ @staticmethod
181
+ def wait_for_startup(
182
+ proc: BaseProcess,
183
+ ready_path: str,
184
+ ) -> None:
185
+ """Wait until the EngineCore is ready."""
186
+
187
+ try:
188
+ sync_ctx = zmq.Context() # type: ignore[attr-defined]
189
+ socket = sync_ctx.socket(zmq.constants.PULL)
190
+ socket.connect(ready_path)
191
+
192
+ # Wait for EngineCore to send EngineCoreProc.READY_STR.
193
+ while socket.poll(timeout=POLLING_TIMEOUT_MS) == 0:
194
+ logger.debug("Waiting for EngineCoreProc to startup.")
195
+
196
+ if not proc.is_alive():
197
+ raise RuntimeError("EngineCoreProc failed to start.")
198
+
199
+ message = socket.recv_string()
200
+ assert message == EngineCoreProc.READY_STR
201
+
202
+ except BaseException as e:
203
+ logger.exception(e)
204
+ raise e
205
+
206
+ finally:
207
+ sync_ctx.destroy(linger=0)
208
+
209
+ @staticmethod
210
+ def make_engine_core_process(
211
+ vllm_config: VllmConfig,
212
+ executor_class: Type[GPUExecutor],
213
+ usage_context: UsageContext,
214
+ input_path: str,
215
+ output_path: str,
216
+ ready_path: str,
217
+ should_shutdown: Synchronized,
218
+ ) -> BaseProcess:
219
+ # The current process might have CUDA context,
220
+ # so we need to spawn a new process.
221
+ # NOTE(rob): this is a problem for using EngineCoreProc w/
222
+ # LLM, since we need a if __name__ == "__main__" guard.
223
+ context = multiprocessing.get_context("spawn")
224
+
225
+ process_kwargs = {
226
+ "input_path": input_path,
227
+ "output_path": output_path,
228
+ "ready_path": ready_path,
229
+ "vllm_config": vllm_config,
230
+ "executor_class": executor_class,
231
+ "usage_context": usage_context,
232
+ "should_shutdown": should_shutdown
233
+ }
234
+ # Run EngineCore busy loop in background process.
235
+ proc = context.Process(target=EngineCoreProc.run_engine_core,
236
+ kwargs=process_kwargs)
237
+ proc.start()
238
+
239
+ # Wait for startup
240
+ EngineCoreProc.wait_for_startup(proc, ready_path)
241
+ return proc
242
+
243
+ @staticmethod
244
+ def run_engine_core(*args, **kwargs):
245
+ """Launch EngineCore busy loop in background process."""
246
+
247
+ try:
248
+ engine_core = EngineCoreProc(*args, **kwargs)
249
+ engine_core.run_busy_loop()
250
+
251
+ except KeyboardInterrupt:
252
+ logger.debug("EngineCore interrupted.")
253
+
254
+ except BaseException as e:
255
+ logger.exception(e)
256
+ raise e
257
+
258
+ def run_busy_loop(self):
259
+ """Core busy loop of the EngineCore."""
260
+
261
+ # Loop until we get a shutdown signal.
262
+ while not self.should_shutdown:
263
+ # 1) Poll the input queue until there is work to do.
264
+ if not self.scheduler.has_unfinished_requests():
265
+ while True:
266
+ try:
267
+ req = self.input_queue.get(timeout=POLLING_TIMEOUT_S)
268
+ self._handle_client_request(req)
269
+ break
270
+ except queue.Empty:
271
+ self._log_stats()
272
+ logger.debug("EngineCore busy loop waiting.")
273
+ if self.should_shutdown:
274
+ return
275
+
276
+ # 2) Handle any new client requests (Abort or Add).
277
+ while not self.input_queue.empty():
278
+ req = self.input_queue.get_nowait()
279
+ self._handle_client_request(req)
280
+
281
+ # 3) Step the engine core.
282
+ outputs = self.step()
283
+
284
+ # 4) Put EngineCoreOutputs into the output queue.
285
+ self.output_queue.put_nowait(outputs)
286
+
287
+ self._log_stats()
288
+
289
+ def _log_stats(self):
290
+ """Log basic stats every LOGGING_TIME_S"""
291
+
292
+ now = time.time()
293
+
294
+ if now - self._last_logging_time > LOGGING_TIME_S:
295
+ logger.info(
296
+ "RUNNING: %s | WAITING: %s",
297
+ len(self.scheduler.running),
298
+ len(self.scheduler.waiting),
299
+ )
300
+
301
+ self._last_logging_time = now
302
+
303
+ def _handle_client_request(
304
+ self, request: Union[EngineCoreRequest, List[str]]) -> None:
305
+ """Handle EngineCoreRequest or EngineCoreABORT from Client."""
306
+
307
+ if isinstance(request, EngineCoreRequest):
308
+ self.add_request(request)
309
+ else:
310
+ # TODO: make an EngineCoreAbort wrapper
311
+ assert isinstance(request, list)
312
+ self.abort_requests(request)
313
+
314
+ def process_input_socket(self, input_path: str):
315
+ """Input socket IO thread."""
316
+
317
+ # Msgpack serialization decoding.
318
+ decoder_add_req = msgpack.Decoder(EngineCoreRequest)
319
+ decoder_abort_req = msgpack.Decoder(list[str])
320
+
321
+ with self.make_socket(input_path, zmq.constants.PULL) as socket:
322
+ while True:
323
+ # (RequestType, RequestData)
324
+ type_frame, data_frame = socket.recv_multipart(copy=False)
325
+ request_type = type_frame.buffer
326
+ request_data = data_frame.buffer
327
+
328
+ # Deserialize the request data.
329
+ if request_type == EngineCoreRequestType.ADD.value:
330
+ request = decoder_add_req.decode(request_data)
331
+ elif request_type == EngineCoreRequestType.ABORT.value:
332
+ request = decoder_abort_req.decode(request_data)
333
+ else:
334
+ raise ValueError(f"Unknown RequestType: {request_type}")
335
+
336
+ # Push to input queue for core busy loop.
337
+ self.input_queue.put_nowait(request)
338
+
339
+ def process_output_socket(self, output_path: str):
340
+ """Output socket IO thread."""
341
+
342
+ # Msgpack serialization encoding.
343
+ encoder = msgpack.Encoder()
344
+ # Reuse send buffer.
345
+ buffer = bytearray()
346
+
347
+ with self.make_socket(output_path, zmq.constants.PUSH) as socket:
348
+ while True:
349
+ engine_core_outputs = self.output_queue.get()
350
+ outputs = EngineCoreOutputs(outputs=engine_core_outputs)
351
+ encoder.encode_into(outputs, buffer)
352
+ socket.send_multipart((buffer, ), copy=False)
vllm/v1/engine/core_client.py
new
+218
@@ -0,0 +1,218 @@
1
+import multiprocessing
2
+import time
3
+from typing import List, Union
4
+
5
+import msgspec
6
+import zmq
7
+import zmq.asyncio
8
+
9
+from vllm.logger import init_logger
10
+from vllm.utils import get_open_zmq_ipc_path
11
+from vllm.v1.engine import (EngineCoreOutput, EngineCoreOutputs,
12
+ EngineCoreRequest, EngineCoreRequestType)
13
+from vllm.v1.engine.core import EngineCore, EngineCoreProc
14
+
15
+logger = init_logger(__name__)
16
+
17
+
18
+class EngineCoreClient:
19
+ """
20
+ EngineCoreClient: subclasses handle different methods for pushing
21
+ and pulling from the EngineCore for asyncio / multiprocessing.
22
+
23
+ Subclasses:
24
+ * InprocClient: In process EngineCore (for V0-style LLMEngine use)
25
+ * SyncMPClient: ZMQ + background proc EngineCore (for LLM)
26
+ * AsyncMPClient: ZMQ + background proc EngineCore w/ asyncio (for AsyncLLM)
27
+ """
28
+
29
+ @staticmethod
30
+ def make_client(
31
+ *args,
32
+ multiprocess_mode: bool,
33
+ asyncio_mode: bool,
34
+ **kwargs,
35
+ ) -> "EngineCoreClient":
36
+
37
+ # TODO: support this for debugging purposes.
38
+ if asyncio_mode and not multiprocess_mode:
39
+ raise NotImplementedError(
40
+ "Running EngineCore in asyncio without multiprocessing "
41
+ "is not currently supported.")
42
+
43
+ if multiprocess_mode and asyncio_mode:
44
+ return AsyncMPClient(*args, **kwargs)
45
+
46
+ if multiprocess_mode and not asyncio_mode:
47
+ return SyncMPClient(*args, **kwargs)
48
+
49
+ return InprocClient(*args, **kwargs)
50
+
51
+ def shutdown(self):
52
+ pass
53
+
54
+ def get_output(self) -> List[EngineCoreOutput]:
55
+ raise NotImplementedError
56
+
57
+ def add_request(self, request: EngineCoreRequest) -> None:
58
+ raise NotImplementedError
59
+
60
+ def abort_requests(self, request_ids: List[str]) -> None:
61
+ raise NotImplementedError
62
+
63
+ async def get_output_async(self) -> List[EngineCoreOutput]:
64
+ raise NotImplementedError
65
+
66
+ async def add_request_async(self, request: EngineCoreRequest) -> None:
67
+ raise NotImplementedError
68
+
69
+ async def abort_requests_async(self, request_ids: List[str]) -> None:
70
+ raise NotImplementedError
71
+
72
+
73
+class InprocClient(EngineCoreClient):
74
+ """
75
+ InprocClient: client for in-process EngineCore. Intended
76
+ for use in LLMEngine for V0-style add_request() and step()
77
+ EngineCore setup in this process (no busy loop).
78
+
79
+ * pushes EngineCoreRequest directly into the EngineCore
80
+ * pulls EngineCoreOutputs by stepping the EngineCore
81
+
82
+ TODO: support asyncio-mode for debugging.
83
+ """
84
+
85
+ def __init__(self, *args, **kwargs):
86
+ self.engine_core = EngineCore(*args, **kwargs)
87
+
88
+ def get_output(self) -> List[EngineCoreOutput]:
89
+ return self.engine_core.step()
90
+
91
+ def add_request(self, request: EngineCoreRequest) -> None:
92
+ self.engine_core.add_request(request)
93
+
94
+ def abort_requests(self, request_ids: List[str]) -> None:
95
+ self.engine_core.abort_requests(request_ids)
96
+
97
+
98
+class MPClient(EngineCoreClient):
99
+ """
100
+ MPClient: base client for multi-proc EngineCore.
101
+ EngineCore runs in a background process busy loop, getting
102
+ new EngineCoreRequests and returning EngineCoreOutputs
103
+
104
+ * pushes EngineCoreRequests via input_socket
105
+ * pulls EngineCoreOutputs via output_socket
106
+
107
+ * AsyncMPClient subclass for AsyncLLM usage
108
+ * SyncMPClient subclass for LLM usage
109
+ """
110
+
111
+ def __init__(
112
+ self,
113
+ *args,
114
+ asyncio_mode: bool,
115
+ **kwargs,
116
+ ):
117
+ # Serialization setup.
118
+ self.encoder = msgspec.msgpack.Encoder()
119
+ self.decoder = msgspec.msgpack.Decoder(EngineCoreOutputs)
120
+
121
+ # ZMQ setup.
122
+ self.ctx = (zmq.asyncio.Context() if asyncio_mode else zmq.Context())
123
+
124
+ # Path for IPC.
125
+ ready_path = get_open_zmq_ipc_path()
126
+ output_path = get_open_zmq_ipc_path()
127
+ input_path = get_open_zmq_ipc_path()
128
+
129
+ # Get output (EngineCoreOutput) from EngineCore.
130
+ self.output_socket = self.ctx.socket(zmq.constants.PULL)
131
+ self.output_socket.connect(output_path)
132
+
133
+ # Send input (EngineCoreRequest) to EngineCore.
134
+ self.input_socket = self.ctx.socket(zmq.constants.PUSH)
135
+ self.input_socket.bind(input_path)
136
+
137
+ # Start EngineCore in background process.
138
+ self.should_shutdown = multiprocessing.Value('b', False, lock=False)
139
+ self.proc = EngineCoreProc.make_engine_core_process(
140
+ *args,
141
+ input_path=input_path,
142
+ output_path=output_path,
143
+ ready_path=ready_path,
144
+ should_shutdown=self.should_shutdown,
145
+ **kwargs,
146
+ )
147
+
148
+ def shutdown(self):
149
+ # Send shutdown signal to background process.
150
+ self.should_shutdown = True
151
+
152
+ # Shut down the zmq context.
153
+ self.ctx.destroy(linger=0)
154
+
155
+ # Shutdown the process if needed.
156
+ if hasattr(self, "proc") and self.proc.is_alive():
157
+ self.proc.terminate()
158
+
159
+ time.sleep(5)
160
+ if self.proc.is_alive():
161
+ self.proc.kill()
162
+
163
+ def __del__(self):
164
+ self.shutdown()
165
+
166
+
167
+class SyncMPClient(MPClient):
168
+ """Synchronous client for multi-proc EngineCore."""
169
+
170
+ def __init__(self, *args, **kwargs):
171
+ super().__init__(*args, asyncio_mode=False, **kwargs)
172
+
173
+ def get_output(self) -> List[EngineCoreOutput]:
174
+
175
+ (frame, ) = self.output_socket.recv_multipart(copy=False)
176
+ engine_core_outputs = self.decoder.decode(frame.buffer).outputs
177
+ return engine_core_outputs
178
+
179
+ def _send_input(self, request_type: EngineCoreRequestType,
180
+ request: Union[EngineCoreRequest, List[str]]) -> None:
181
+
182
+ # (RequestType, SerializedRequest)
183
+ msg = (request_type.value, self.encoder.encode(request))
184
+ self.input_socket.send_multipart(msg, copy=False)
185
+
186
+ def add_request(self, request: EngineCoreRequest) -> None:
187
+ self._send_input(EngineCoreRequestType.ADD, request)
188
+
189
+ def abort_requests(self, request_ids: List[str]) -> None:
190
+ self._send_input(EngineCoreRequestType.ABORT, request_ids)
191
+
192
+
193
+class AsyncMPClient(MPClient):
194
+ """Asyncio-compatible client for multi-proc EngineCore."""
195
+
196
+ def __init__(self, *args, **kwargs):
197
+ super().__init__(*args, asyncio_mode=True, **kwargs)
198
+
199
+ async def get_output_async(self) -> List[EngineCoreOutput]:
200
+
201
+ frames = await self.output_socket.recv_multipart(copy=False)
202
+ engine_core_outputs = self.decoder.decode(frames[0].buffer).outputs
203
+
204
+ return engine_core_outputs
205
+
206
+ async def _send_input(
207
+ self, request_type: EngineCoreRequestType,
208
+ request: Union[EngineCoreRequest, List[str]]) -> None:
209
+
210
+ msg = (request_type.value, self.encoder.encode(request))
211
+ await self.input_socket.send_multipart(msg, copy=False)
212
+
213
+ async def add_request_async(self, request: EngineCoreRequest) -> None:
214
+ await self._send_input(EngineCoreRequestType.ADD, request)
215
+
216
+ async def abort_requests_async(self, request_ids: List[str]) -> None:
217
+ if len(request_ids) > 0:
218
+ await self._send_input(EngineCoreRequestType.ABORT, request_ids)
vllm/v1/engine/detokenizer.py
new
+265
@@ -0,0 +1,265 @@
1
+from dataclasses import dataclass
2
+from typing import Dict, Iterable, List, Optional, Tuple
3
+
4
+from vllm.engine.output_processor.stop_checker import StopChecker
5
+from vllm.logger import init_logger
6
+from vllm.outputs import RequestOutput
7
+from vllm.sampling_params import RequestOutputKind
8
+from vllm.transformers_utils.detokenizer_utils import (
9
+ AnyTokenizer, convert_prompt_ids_to_tokens, detokenize_incrementally)
10
+from vllm.transformers_utils.tokenizer import get_tokenizer
11
+from vllm.v1.engine import DetokenizerRequest, EngineCoreOutput
12
+
13
+logger = init_logger(__name__)
14
+
15
+
16
+@dataclass
17
+class IncrementalDetokenizer:
18
+
19
+ # Generation data
20
+ output_text: str
21
+ tokens: List[str]
22
+ token_ids: List[int]
23
+
24
+ # Stop strings
25
+ stop: List[str]
26
+ include_stop_str_in_output: bool
27
+
28
+ # Metadata for incremental detokenization
29
+ prefix_offset: int
30
+ read_offset: int
31
+
32
+ # Parameters for detokenization
33
+ skip_special_tokens: bool
34
+ spaces_between_special_tokens: bool
35
+ output_kind: RequestOutputKind
36
+
37
+ # TODO: Probably decouple these
38
+ request_id: str
39
+ prompt: Optional[str]
40
+ prompt_token_ids: List[int]
41
+
42
+ # Tokenizer for this request
43
+ tokenizer: AnyTokenizer
44
+
45
+ # Accounting for stop string buffering
46
+ stop_buffer_length: int
47
+ _last_output_text_offset: int = 0
48
+
49
+ @property
50
+ def output_token_ids(self) -> List[int]:
51
+ assert len(self.token_ids) >= len(self.prompt_token_ids)
52
+ return self.token_ids[len(self.prompt_token_ids):]
53
+
54
+ @classmethod
55
+ def from_new_request(
56
+ cls,
57
+ tokenizer: AnyTokenizer,
58
+ request: DetokenizerRequest,
59
+ ) -> "IncrementalDetokenizer":
60
+
61
+ tokens, prefix_offset, read_offset = convert_prompt_ids_to_tokens(
62
+ tokenizer=tokenizer,
63
+ prompt_ids=request.prompt_token_ids,
64
+ skip_special_tokens=request.skip_special_tokens,
65
+ )
66
+
67
+ stops = request.stop
68
+ # Number of chars to hold back when stop strings are to be excluded
69
+ # from streamed output.
70
+ if stops and not request.include_stop_str_in_output:
71
+ stop_buffer_length = max(len(s) for s in stops) - 1
72
+ else:
73
+ stop_buffer_length = 0
74
+
75
+ return cls(
76
+ output_text="",
77
+ tokens=tokens,
78
+ # Detokenizer mutates this list, so need a unique copy.
79
+ # NOTE(Nick): could we take ownership of it though?
80
+ token_ids=request.prompt_token_ids.copy(),
81
+ stop=stops,
82
+ include_stop_str_in_output=request.include_stop_str_in_output,
83
+ prefix_offset=prefix_offset,
84
+ read_offset=read_offset,
85
+ skip_special_tokens=request.skip_special_tokens,
86
+ spaces_between_special_tokens=request.
87
+ spaces_between_special_tokens,
88
+ output_kind=request.output_kind,
89
+ request_id=request.request_id,
90
+ prompt=request.prompt,
91
+ prompt_token_ids=request.prompt_token_ids,
92
+ tokenizer=tokenizer,
93
+ stop_buffer_length=stop_buffer_length,
94
+ )
95
+
96
+ def add_tokens(
97
+ self,
98
+ new_token_ids: List[int],
99
+ finish_reason: Optional[str],
100
+ stop_reason: Optional[str],
101
+ ) -> Optional[RequestOutput]:
102
+ """
103
+ Update RequestState for the request_id by:
104
+ 1) Detokenize the new token ids incrementally.
105
+ 2) Update the RequestOutput with the new text.
106
+ """
107
+
108
+ # 1) Detokenize the new token ids incrementally.
109
+ # TODO(woosuk): This method becomes very inefficient when the number of
110
+ # new_token_ids is more than 1. We need to optimize this.
111
+ decoded_text = ""
112
+ for new_token_id in new_token_ids:
113
+ self.token_ids.append(new_token_id)
114
+ (new_tokens, new_decoded_token_text, prefix_offset,
115
+ read_offset) = detokenize_incrementally(
116
+ tokenizer=self.tokenizer,
117
+ all_input_ids=self.token_ids,
118
+ prev_tokens=self.tokens,
119
+ prefix_offset=self.prefix_offset,
120
+ read_offset=self.read_offset,
121
+ skip_special_tokens=self.skip_special_tokens,
122
+ spaces_between_special_tokens=self.
123
+ spaces_between_special_tokens,
124
+ )
125
+
126
+ self.tokens.extend(new_tokens)
127
+ self.prefix_offset = prefix_offset
128
+ self.read_offset = read_offset
129
+ self.output_text += new_decoded_token_text
130
+
131
+ decoded_text += new_decoded_token_text
132
+
133
+ # 2) Evaluate stop criteria.
134
+ if self.stop:
135
+ stop = StopChecker.check_stop_strings(
136
+ output_text=self.output_text,
137
+ new_char_count=len(decoded_text),
138
+ stop=self.stop,
139
+ include_in_output=self.include_stop_str_in_output,
140
+ )
141
+ if stop is not None:
142
+ stop_str, truncate_to = stop
143
+ if truncate_to != -1:
144
+ self.output_text = self.output_text[:truncate_to]
145
+ finish_reason = "stop" # TODO: use constant
146
+ stop_reason = stop_str
147
+
148
+ # TODO: handle stop_token_ids here too?
149
+
150
+ # 3) Update the RequestOutput object with the new text.
151
+ finished = bool(finish_reason)
152
+ if self.output_kind == RequestOutputKind.FINAL_ONLY \
153
+ and not finished:
154
+ return None
155
+
156
+ delta = self.output_kind == RequestOutputKind.DELTA
157
+ output_text = self._get_next_output_text(finished, delta)
158
+ token_ids = new_token_ids if delta else self.output_token_ids
159
+
160
+ request_output = RequestOutput.new(
161
+ self.request_id,
162
+ self.prompt,
163
+ self.prompt_token_ids,
164
+ output_text,
165
+ token_ids,
166
+ finished,
167
+ )
168
+
169
+ if finished:
170
+ completion_output = request_output.outputs[0]
171
+ completion_output.finish_reason = finish_reason
172
+ completion_output.stop_reason = stop_reason
173
+
174
+ return request_output
175
+
176
+ def _get_next_output_text(self, finished: bool, delta: bool) -> str:
177
+ """If delta is True, only new text since the last call to
178
+ this method is returned"""
179
+
180
+ # We return the full output text if the sequence is finished.
181
+ buffer_length = 0 if finished else self.stop_buffer_length
182
+ if not delta:
183
+ return self.output_text[:-buffer_length] if buffer_length else (
184
+ self.output_text)
185
+ length = len(self.output_text) - buffer_length
186
+ last_offset = self._last_output_text_offset
187
+ if last_offset < length:
188
+ self._last_output_text_offset = length
189
+ return self.output_text[last_offset:length]
190
+ return ""
191
+
192
+
193
+class Detokenizer:
194
+
195
+ def __init__(self, tokenizer_name: str):
196
+ # TODO: once we support LoRA, we should should pass the tokenizer
197
+ # here. We currently have two copies (this + in the LLMEngine).
198
+ self.tokenizer = get_tokenizer(tokenizer_name)
199
+
200
+ # Request id -> IncrementalDetokenizer
201
+ self.request_states: Dict[str, IncrementalDetokenizer] = {}
202
+
203
+ def is_request_active(self, request_id: str):
204
+ return request_id in self.request_states
205
+
206
+ def get_num_unfinished_requests(self):
207
+ return len(self.request_states)
208
+
209
+ def has_unfinished_requests(self) -> bool:
210
+ return len(self.request_states) > 0
211
+
212
+ def abort_requests(
213
+ self,
214
+ request_ids: Iterable[str],
215
+ ) -> None:
216
+ """Remove the request_ids from the Detokenizer."""
217
+
218
+ for request_id in request_ids:
219
+ self.request_states.pop(request_id, None)
220
+
221
+ def add_request(
222
+ self,
223
+ request: DetokenizerRequest,
224
+ ):
225
+ """Add new request to the Detokenizer."""
226
+
227
+ assert (request.request_id not in self.request_states)
228
+
229
+ request_state = IncrementalDetokenizer.from_new_request(
230
+ self.tokenizer, request)
231
+ self.request_states[request.request_id] = request_state
232
+
233
+ def step(
234
+ self, encore_core_outputs: List[EngineCoreOutput]
235
+ ) -> Tuple[List[RequestOutput], List[str]]:
236
+ """Update state and request the RequestOutputs to the LLMEngine."""
237
+
238
+ request_outputs: List[RequestOutput] = []
239
+ requests_to_abort: List[str] = []
240
+ for engine_core_output in encore_core_outputs:
241
+ request_id = engine_core_output.request_id
242
+ detokenizer = self.request_states.get(request_id)
243
+ if detokenizer is None:
244
+ # Ignore output for already-aborted request.
245
+ continue
246
+
247
+ # Detokenize and update state.
248
+ request_output = detokenizer.add_tokens(
249
+ new_token_ids=engine_core_output.new_token_ids,
250
+ finish_reason=engine_core_output.finish_reason,
251
+ stop_reason=engine_core_output.stop_reason,
252
+ )
253
+
254
+ if request_output is not None:
255
+ # Add to RequestOutputs list.
256
+ request_outputs.append(request_output)
257
+
258
+ # Free completed requests.
259
+ if request_output.finished:
260
+ self.request_states.pop(request_id)
261
+ if not engine_core_output.finished:
262
+ requests_to_abort.append(request_id)
263
+
264
+ # Return to EngineClient.
265
+ return request_outputs, requests_to_abort
vllm/v1/engine/llm_engine.py
+99
-453
@@ -1,35 +1,28 @@
1
-import time
2
-from typing import (Any, Dict, Iterable, List, Mapping, Optional, Tuple, Type,
3
- Union)
1
+from typing import Dict, List, Mapping, Optional, Type, Union
2
5
-from vllm.config import (DecodingConfig, LoRAConfig, ModelConfig,
6
- ObservabilityConfig, ParallelConfig, SchedulerConfig,
7
- VllmConfig)
3
+from vllm.config import VllmConfig
4
from vllm.engine.arg_utils import EngineArgs
5
from vllm.engine.metrics_types import StatLoggerBase
10
-from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs,
11
- EncoderDecoderLLMInputs, InputRegistry, PromptType)
12
-from vllm.inputs.preprocess import InputPreprocessor
6
+from vllm.envs import VLLM_ENABLE_V1_MULTIPROCESSING
7
+from vllm.inputs import INPUT_REGISTRY, InputRegistry, PromptType
8
from vllm.logger import init_logger
9
from vllm.lora.request import LoRARequest
15
-from vllm.outputs import CompletionOutput, RequestOutput
10
+from vllm.outputs import RequestOutput
11
from vllm.pooling_params import PoolingParams
12
from vllm.prompt_adapter.request import PromptAdapterRequest
18
-from vllm.sampling_params import RequestOutputKind, SamplingParams
19
-from vllm.transformers_utils.config import try_get_generation_config
20
-from vllm.transformers_utils.tokenizer_group import (
21
- BaseTokenizerGroup, init_tokenizer_from_configs)
13
+from vllm.sampling_params import SamplingParams
14
+from vllm.transformers_utils.tokenizer_group import init_tokenizer_from_configs
15
from vllm.usage.usage_lib import UsageContext
23
-from vllm.v1.core.scheduler import Scheduler
16
+from vllm.v1.engine.core_client import EngineCoreClient
17
+from vllm.v1.engine.detokenizer import Detokenizer
18
+from vllm.v1.engine.processor import Processor
19
from vllm.v1.executor.gpu_executor import GPUExecutor
25
-from vllm.v1.request import Request, RequestStatus
26
-from vllm.v1.tokenizer.detokenizer import Detokenizer, DetokenizerInputs
27
-from vllm.version import __version__ as VLLM_VERSION
20
21
logger = init_logger(__name__)
22
23
24
class LLMEngine:
25
+ """Legacy LLMEngine for backwards compatibility."""
26
27
def __init__(
28
self,
@@ -40,146 +33,36 @@ class LLMEngine:
33
stat_loggers: Optional[Dict[str, StatLoggerBase]] = None,
34
input_registry: InputRegistry = INPUT_REGISTRY,
35
use_cached_outputs: bool = False,
36
+ multiprocess_mode: bool = False,
37
) -> None:
38
45
- # TODO: remove the local variables and use self.* throughout the class.
46
- model_config = self.model_config = vllm_config.model_config
47
- cache_config = self.cache_config = vllm_config.cache_config
48
- lora_config = self.lora_config = vllm_config.lora_config
49
- parallel_config = self.parallel_config = vllm_config.parallel_config
50
- scheduler_config = self.scheduler_config = vllm_config.scheduler_config
51
- device_config = self.device_config = vllm_config.device_config
52
- speculative_config = self.speculative_config = vllm_config.speculative_config # noqa
53
- load_config = self.load_config = vllm_config.load_config
54
- decoding_config = self.decoding_config = vllm_config.decoding_config or DecodingConfig( # noqa
39
+ # TODO: Can we avoid this?
40
+ self.model_config = vllm_config.model_config
41
+
42
+ # Tokenizer (+ ensure liveness if running in another process).
43
+ self.tokenizer = init_tokenizer_from_configs(
44
+ model_config=vllm_config.model_config,
45
+ scheduler_config=vllm_config.scheduler_config,
46
+ parallel_config=vllm_config.parallel_config,
47
+ enable_lora=bool(vllm_config.lora_config))
48
+ self.tokenizer.ping()
49
+
50
+ # Processor (convert Inputs --> EngineCoreRequests)
51
+ self.processor = Processor(vllm_config.model_config,
52
+ vllm_config.lora_config, self.tokenizer,
53
+ input_registry)
54
+
55
+ # Detokenizer (converts EngineCoreOutputs --> RequestOutput)
56
+ self.detokenizer = Detokenizer(vllm_config.model_config.tokenizer)
57
+
58
+ # EngineCore (gets EngineCoreRequests and gives EngineCoreOutputs)
59
+ self.engine_core = EngineCoreClient.make_client(
60
+ vllm_config,
61
+ executor_class,
62
+ usage_context,
63
+ multiprocess_mode=multiprocess_mode,
64
+ asyncio_mode=False,
65
)
56
- prompt_adapter_config = self.prompt_adapter_config = vllm_config.prompt_adapter_config # noqa
57
- observability_config = self.observability_config = vllm_config.observability_config or ObservabilityConfig( # noqa
58
- )
59
-
60
- # Override the configs for V1.
61
- # FIXME
62
- if usage_context == UsageContext.LLM_CLASS:
63
- scheduler_config.max_num_seqs = 1024
64
- scheduler_config.max_num_batched_tokens = 8192
65
- elif usage_context == UsageContext.OPENAI_API_SERVER:
66
- scheduler_config.max_num_seqs = 1024
67
- scheduler_config.max_num_batched_tokens = 2048
68
-
69
- # TODO (ywang96): Enable APC by default when VLM supports it.
70
- if not model_config.is_multimodal_model:
71
- cache_config.enable_prefix_caching = True
72
-
73
- logger.info(
74
- "Initializing an LLM engine (v%s) with config: "
75
- "model=%r, speculative_config=%r, tokenizer=%r, "
76
- "skip_tokenizer_init=%s, tokenizer_mode=%s, revision=%s, "
77
- "override_neuron_config=%s, tokenizer_revision=%s, "
78
- "trust_remote_code=%s, dtype=%s, max_seq_len=%d, "
79
- "download_dir=%r, load_format=%s, tensor_parallel_size=%d, "
80
- "pipeline_parallel_size=%d, "
81
- "disable_custom_all_reduce=%s, quantization=%s, "
82
- "enforce_eager=%s, kv_cache_dtype=%s, "
83
- "quantization_param_path=%s, device_config=%s, "
84
- "decoding_config=%r, observability_config=%r, "
85
- "seed=%d, served_model_name=%s, "
86
- "num_scheduler_steps=%d, enable_prefix_caching=%s, "
87
- "use_async_output_proc=%s, mm_processor_kwargs=%s)",
88
- VLLM_VERSION,
89
- model_config.model,
90
- speculative_config,
91
- model_config.tokenizer,
92
- model_config.skip_tokenizer_init,
93
- model_config.tokenizer_mode,
94
- model_config.revision,
95
- model_config.override_neuron_config,
96
- model_config.tokenizer_revision,
97
- model_config.trust_remote_code,
98
- model_config.dtype,
99
- model_config.max_model_len,
100
- load_config.download_dir,
101
- load_config.load_format,
102
- parallel_config.tensor_parallel_size,
103
- parallel_config.pipeline_parallel_size,
104
- parallel_config.disable_custom_all_reduce,
105
- model_config.quantization,
106
- model_config.enforce_eager,
107
- cache_config.cache_dtype,
108
- model_config.quantization_param_path,
109
- device_config.device,
110
- decoding_config,
111
- observability_config,
112
- model_config.seed,
113
- model_config.served_model_name,
114
- scheduler_config.num_scheduler_steps,
115
- cache_config.enable_prefix_caching,
116
- model_config.use_async_output_proc,
117
- model_config.mm_processor_kwargs,
118
- )
119
-
120
- self.log_stats = log_stats
121
-
122
- assert not self.model_config.skip_tokenizer_init
123
- self.tokenizer = self._init_tokenizer()
124
- if self.tokenizer:
125
- # Ping the tokenizer to ensure liveness if it runs in a
126
- # different process.
127
- self.tokenizer.ping()
128
- self.detokenizer = Detokenizer(
129
- tokenizer_name=self.model_config.tokenizer,
130
- tokenizer_mode=self.model_config.tokenizer_mode,
131
- trust_remote_code=self.model_config.trust_remote_code)
132
-
133
- self.generation_config_fields = _load_generation_config_dict(
134
- model_config)
135
- self.input_preprocessor = InputPreprocessor(model_config,
136
- self.tokenizer)
137
- self.input_registry = input_registry
138
- self.input_processor = input_registry.create_input_processor(
139
- model_config)
140
-
141
- # Request id -> Request
142
- self.requests: Dict[str, Request] = {}
143
- # NOTE(woosuk): Now that the detokenizer works asynchronously, we need
144
- # to keep track of how many steps each request has been lagged behind
145
- # in terms of detokenization.
146
- # Request id -> how many detokenizer steps the request should wait for.
147
- self.num_lagged_steps: Dict[str, int] = {}
148
- # OPTIMIZATION: Cache the request output and update it incrementally.
149
- # This is used to avoid creating a new RequestOutput object every step.
150
- # Request id -> RequestOutput
151
- self.request_outputs: Dict[str, RequestOutput] = {}
152
-
153
- self.model_executor = executor_class(vllm_config=vllm_config)
154
- assert self.model_config.task != "embedding"
155
- self._initialize_kv_caches()
156
-
157
- # Create the scheduler.
158
- # NOTE: the cache_config here have been updated with the numbers of
159
- # GPU and CPU blocks, which are profiled in the distributed executor.
160
- self.scheduler = Scheduler(scheduler_config, cache_config, lora_config)
161
-
162
- def __del__(self):
163
- # Small hack- implicit clean up of resources on garbage collect
164
- # TODO: this should probably be explicitly invoked when we're done with
165
- # the engine
166
- self.terminate_detokenizer()
167
-
168
- def _initialize_kv_caches(self) -> None:
169
- num_gpu_blocks, _ = self.model_executor.determine_num_available_blocks(
170
- )
171
-
172
- if self.cache_config.num_gpu_blocks_override is not None:
173
- num_gpu_blocks_override = self.cache_config.num_gpu_blocks_override
174
- logger.info(
175
- "Overriding num_gpu_blocks=%d with "
176
- "num_gpu_blocks_override=%d", num_gpu_blocks,
177
- num_gpu_blocks_override)
178
- num_gpu_blocks = num_gpu_blocks_override
179
-
180
- self.cache_config.num_gpu_blocks = num_gpu_blocks
181
- self.cache_config.num_cpu_blocks = 0
182
- self.model_executor.initialize_cache(num_gpu_blocks)
66
67
@classmethod
68
def from_engine_args(
@@ -187,71 +70,49 @@ class LLMEngine:
70
engine_args: EngineArgs,
71
usage_context: UsageContext = UsageContext.ENGINE_CONTEXT,
72
stat_loggers: Optional[Dict[str, StatLoggerBase]] = None,
73
+ enable_multiprocessing: bool = False,
74
) -> "LLMEngine":
75
"""Creates an LLM engine from the engine arguments."""
76
+
77
# Create the engine configs.
193
- engine_config = engine_args.create_engine_config()
194
- executor_class = cls._get_executor_cls(engine_config)
195
- # Create the LLM engine.
196
- engine = cls(
197
- vllm_config=engine_config,
198
- executor_class=executor_class,
199
- log_stats=not engine_args.disable_log_stats,
200
- usage_context=usage_context,
201
- stat_loggers=stat_loggers,
202
- )
203
- return engine
204
-
205
- def _init_tokenizer(self) -> BaseTokenizerGroup:
206
- return init_tokenizer_from_configs(
207
- model_config=self.model_config,
208
- scheduler_config=self.scheduler_config,
209
- parallel_config=self.parallel_config,
210
- enable_lora=bool(self.lora_config))
211
-
212
- def _verify_args(self) -> None:
213
- self.model_config.verify_with_parallel_config(self.parallel_config)
214
- self.cache_config.verify_with_parallel_config(self.parallel_config)
215
- if self.lora_config:
216
- self.lora_config.verify_with_model_config(self.model_config)
217
- self.lora_config.verify_with_scheduler_config(
218
- self.scheduler_config)
219
- if self.prompt_adapter_config:
220
- self.prompt_adapter_config.verify_with_model_config(
221
- self.model_config)
222
-
223
- def _add_processed_request(
224
- self,
225
- request_id: str,
226
- processed_inputs: Union[DecoderOnlyInputs, EncoderDecoderLLMInputs],
227
- params: Union[SamplingParams, PoolingParams],
228
- arrival_time: float,
229
- lora_request: Optional[LoRARequest],
230
- prompt_adapter_request: Optional[PromptAdapterRequest],
231
- trace_headers: Optional[Mapping[str, str]] = None,
232
- ) -> None:
233
- assert prompt_adapter_request is None
234
- assert trace_headers is None
235
- self._validate_model_inputs(processed_inputs)
236
- eos_token_id = self.input_preprocessor.get_eos_token_id(lora_request)
237
-
238
- # TODO(woosuk): Support embedding mode.
239
- assert isinstance(params, SamplingParams)
240
- sampling_params = params.clone()
241
- sampling_params.update_from_generation_config(
242
- self.generation_config_fields, eos_token_id)
243
-
244
- # TODO(woosuk): Check max_logprobs
245
- # TODO(woosuk): Support encoder-decoder models.
246
- req = Request(request_id, processed_inputs, params, eos_token_id,
247
- arrival_time)
248
- self.requests[request_id] = req
249
- self.num_lagged_steps[request_id] = 0
250
- self.scheduler.add_request(req)
78
+ vllm_config = engine_args.create_engine_config()
79
+ executor_class = cls._get_executor_cls(vllm_config)
80
+
81
+ if VLLM_ENABLE_V1_MULTIPROCESSING:
82
+ logger.debug("Enabling multiprocessing for LLMEngine.")
83
+ enable_multiprocessing = True
84
+
85
+ # Create the LLMEngine.
86
+ return cls(vllm_config=vllm_config,
87
+ executor_class=executor_class,
88
+ log_stats=not engine_args.disable_log_stats,
89
+ usage_context=usage_context,
90
+ stat_loggers=stat_loggers,
91
+ multiprocess_mode=enable_multiprocessing)
92
+
93
+ @classmethod
94
+ def _get_executor_cls(cls, vllm_config: VllmConfig):
95
+ return GPUExecutor
96
97
def stop_remote_worker_execution_loop(self) -> None:
98
raise NotImplementedError("TP not implemented yet.")
99
100
+ def get_num_unfinished_requests(self) -> int:
101
+ return self.detokenizer.get_num_unfinished_requests()
102
+
103
+ def has_unfinished_requests(self) -> bool:
104
+ return self.detokenizer.has_unfinished_requests()
105
+
106
+ @classmethod
107
+ def validate_outputs(cls, outputs, output_type):
108
+ return outputs
109
+
110
+ def abort_request(self, request_ids: List[str]) -> None:
111
+ """Remove request_ids from EngineCore and Detokenizer."""
112
+
113
+ self.engine_core.abort_requests(request_ids)
114
+ self.detokenizer.abort_requests(request_ids)
115
+
116
def add_request(
117
self,
118
request_id: str,
@@ -263,261 +124,46 @@ class LLMEngine:
124
prompt_adapter_request: Optional[PromptAdapterRequest] = None,
125
priority: int = 0,
126
) -> None:
266
- if lora_request is not None and not self.lora_config:
267
- raise ValueError(f"Got lora_request {lora_request} but LoRA is "
268
- "not enabled!")
269
- if arrival_time is None:
270
- arrival_time = time.time()
271
- assert priority == 0, "vLLM V1 does not support priority at the moment."
272
-
273
- preprocessed_inputs = self.input_preprocessor.preprocess(
274
- prompt,
275
- request_id=request_id,
276
- lora_request=lora_request,
277
- prompt_adapter_request=prompt_adapter_request,
278
- )
279
- processed_inputs = self.input_processor(preprocessed_inputs)
280
-
281
- self._add_processed_request(
282
- request_id=request_id,
283
- processed_inputs=processed_inputs,
284
- params=params,
285
- arrival_time=arrival_time,
286
- lora_request=lora_request,
287
- prompt_adapter_request=prompt_adapter_request,
288
- trace_headers=trace_headers,
289
- )
127
291
- def abort_request(self, request_id: Union[str, Iterable[str]]) -> None:
292
- self.scheduler.finish_requests(request_id,
293
- RequestStatus.FINISHED_ABORTED)
294
- self._free_request(request_id)
128
+ # 1) Process raw inputs into the request.
129
+ detokenizer_req, engine_core_req = self.processor.process_inputs(
130
+ request_id, prompt, params, arrival_time, lora_request,
131
+ trace_headers, prompt_adapter_request, priority)
132
296
- def get_num_unfinished_requests(self) -> int:
297
- """Gets the number of unfinished requests."""
298
- return len(self.requests)
133
+ # 2) Add the request to Detokenizer.
134
+ self.detokenizer.add_request(detokenizer_req)
135
300
- def has_unfinished_requests(self) -> bool:
301
- """Returns True if there are unfinished requests."""
302
- return len(self.requests) > 0
136
+ # 3) Add the request to EngineCore.
137
+ self.engine_core.add_request(engine_core_req)
138
139
def step(self) -> List[RequestOutput]:
305
- # NOTE(woosuk): This method may return an empty list when the
306
- # detokenizer is still processing the outputs. This should not be
307
- # considered as the end of the generation process.
308
- # FIXME(woosuk): Currently, the step method is inefficient because it
309
- # creates RequestOutput objects for all running requests, while they
310
- # may not be needed unless the output is streamed to the client.
311
- if self.scheduler.has_unfinished_requests():
312
- scheduler_output = self.scheduler.schedule()
313
- output = self.model_executor.execute_model(scheduler_output)
314
- sampled = self.scheduler.update_from_output(
315
- scheduler_output, output)
316
- self.send_to_detokenizer(sampled)
317
- req_outputs = self.recv_from_detokenizer()
318
- return req_outputs
319
-
320
- def send_to_detokenizer(self, sampled: List[Tuple[Request, int]]) -> None:
321
- inputs = DetokenizerInputs(
322
- req_ids=[],
323
- prompt_token_ids=[],
324
- new_token_ids=[],
325
- skip_special_tokens=[],
326
- spaces_between_special_tokens=[],
327
- free_req_ids=[], # TODO(woosuk): Implement freeing.
328
- )
329
- for req, num_tokens in sampled:
330
- inputs.req_ids.append(req.request_id)
331
- if req.num_output_tokens == num_tokens:
332
- # The request is first detokenized.
333
- inputs.prompt_token_ids.append(req.prompt_token_ids)
334
- else:
335
- # The prompt token ids are already cached in the detokenizer.
336
- inputs.prompt_token_ids.append([])
337
- inputs.new_token_ids.append(req.output_token_ids[-num_tokens:])
338
- inputs.skip_special_tokens.append(
339
- req.sampling_params.skip_special_tokens)
340
- inputs.spaces_between_special_tokens.append(
341
- req.sampling_params.spaces_between_special_tokens)
342
-
343
- # Update the number of lagged steps.
344
- self.num_lagged_steps[req.request_id] += 1
345
- self.detokenizer.send(inputs)
346
-
347
- def recv_from_detokenizer(self) -> List[RequestOutput]:
348
- detokenizer_output = self.detokenizer.recv()
349
- if detokenizer_output is None:
350
- return []
351
-
352
- req_outputs: List[RequestOutput] = []
353
- num_reqs = len(detokenizer_output.req_ids)
354
- for i in range(num_reqs):
355
- req_id = detokenizer_output.req_ids[i]
356
- if req_id not in self.requests:
357
- # The request has been aborted while the detokenizer was
358
- # processing the outputs.
359
- continue
360
-
361
- req = self.requests[req_id]
362
- req.output_text += detokenizer_output.detokenized_texts[i]
363
-
364
- self.num_lagged_steps[req_id] -= 1
365
- finished = (self.num_lagged_steps[req_id] == 0
366
- and req.is_finished())
367
- req_output = self._make_request_output(
368
- req, detokenizer_output.num_output_token_ids[i],
369
- detokenizer_output.detokenized_texts[i], finished)
370
- req_outputs.append(req_output)
371
-
372
- if finished:
373
- self._free_request(req_id)
374
- return req_outputs
375
-
376
- def terminate_detokenizer(self) -> None:
377
- self.detokenizer.terminate()
378
-
379
- def _make_request_output(
380
- self,
381
- request: Request,
382
- num_output_tokens: int,
383
- new_output_text: str,
384
- finished: bool,
385
- ) -> RequestOutput:
386
- req_output = self.request_outputs.get(request.request_id)
387
- if req_output is None:
388
- # TODO: Support `n` > 1.
389
- completion_output = CompletionOutput(
390
- index=0,
391
- text="",
392
- token_ids=[],
393
- cumulative_logprob=None,
394
- logprobs=None, # TODO
395
- finish_reason=None,
396
- stop_reason=None,
397
- lora_request=None,
398
- )
399
- req_output = RequestOutput(
400
- request_id=request.request_id,
401
- prompt=request.prompt,
402
- prompt_token_ids=request.prompt_token_ids,
403
- prompt_logprobs=None, # TODO
404
- outputs=[completion_output],
405
- finished=False,
406
- metrics=None,
407
- lora_request=None,
408
- encoder_prompt=None,
409
- encoder_prompt_token_ids=None,
410
- )
411
- self.request_outputs[request.request_id] = req_output
412
-
413
- completion_output = req_output.outputs[0]
414
- if request.sampling_params.output_kind == RequestOutputKind.CUMULATIVE:
415
- completion_output.text += new_output_text
416
- completion_output.token_ids = (
417
- request.output_token_ids[:num_output_tokens])
418
- elif request.sampling_params.output_kind == RequestOutputKind.DELTA:
419
- completion_output.text = new_output_text
420
- num_prev_tokens = len(completion_output.token_ids)
421
- completion_output.token_ids = request.output_token_ids[
422
- num_prev_tokens:num_output_tokens]
423
- elif (request.sampling_params.output_kind ==
424
- RequestOutputKind.FINAL_ONLY):
425
- if finished:
426
- completion_output.text = request.output_text
427
- completion_output.token_ids = request.output_token_ids
428
- else:
429
- completion_output.text = ""
430
- completion_output.token_ids = []
431
-
432
- if finished:
433
- completion_output.finish_reason = request.get_finished_reason()
434
- completion_output.stop_reason = request.stop_reason
435
- req_output.finished = finished
436
- return req_output
437
-
438
- def _free_request(self, request_id: str) -> None:
439
- self.requests.pop(request_id, None)
440
- self.num_lagged_steps.pop(request_id, None)
441
- self.request_outputs.pop(request_id, None)
442
-
443
- def check_health(self) -> None:
444
- if self.tokenizer:
445
- self.tokenizer.check_health()
446
- self.model_executor.check_health()
447
-
448
- def _validate_model_inputs(self, inputs: Union[DecoderOnlyInputs,
449
- EncoderDecoderLLMInputs]):
450
- prompt_ids = inputs.get("prompt_token_ids")
451
- if prompt_ids is None or len(prompt_ids) == 0:
452
- raise ValueError("Prompt cannot be empty")
453
-
454
- if self.model_config.is_multimodal_model:
455
- max_prompt_len = self.model_config.max_model_len
456
-
457
- if len(prompt_ids) > max_prompt_len:
458
- raise ValueError(
459
- f"The prompt (total length {len(prompt_ids)}) is too long "
460
- f"to fit into the model (context length {max_prompt_len}). "
461
- "Make sure that `max_model_len` is no smaller than the "
462
- "number of text tokens plus multimodal tokens. For image "
463
- "inputs, the number of image tokens depends on the number "
464
- "of images, and possibly their aspect ratios as well.")
465
-
466
- @classmethod
467
- def validate_outputs(cls, outputs, output_type):
468
- return outputs
469
-
470
- def get_model_config(self) -> ModelConfig:
471
- """Gets the model configuration."""
472
- return self.model_config
140
474
- def get_parallel_config(self) -> ParallelConfig:
475
- """Gets the parallel configuration."""
476
- return self.parallel_config
141
+ # 1) Get EngineCoreOutput from the EngineCore.
142
+ engine_core_outputs = self.engine_core.get_output()
143
478
- def get_decoding_config(self) -> DecodingConfig:
479
- """Gets the decoding configuration."""
480
- return self.decoding_config
144
+ # 2) Detokenizer the EngineCoreOutput.
145
+ request_outputs, requests_to_abort = self.detokenizer.step(
146
+ engine_core_outputs)
147
482
- def get_scheduler_config(self) -> SchedulerConfig:
483
- """Gets the scheduler configuration."""
484
- return self.scheduler_config
148
+ # 3) Abort requests that finished due to stopping criteria.
149
+ if requests_to_abort:
150
+ self.abort_request(requests_to_abort)
151
486
- def get_lora_config(self) -> LoRAConfig:
487
- """Gets the LoRA configuration."""
488
- return self.lora_config
489
-
490
- @classmethod
491
- def _get_executor_cls(cls, engine_config: VllmConfig):
492
- return GPUExecutor
152
+ return request_outputs
153
494
- def is_tracing_enabled(self) -> bool:
495
- return False
154
+ # TODO(rob): Can we get rid of these?
155
497
- def do_log_stats(self, *args, **kwargs) -> None:
156
+ def get_model_config(self):
157
pass
158
500
- def is_encoder_decoder_model(self) -> bool:
501
- return False
502
-
503
- def start_profile(self) -> None:
159
+ def is_encoder_decoder_model(self):
160
pass
161
506
- def stop_profile(self) -> None:
162
+ def start_profile(self):
163
pass
164
509
- def get_tokenizer_group(self, *args, **kwargs):
510
- return self.tokenizer
511
-
512
-
513
-def _load_generation_config_dict(model_config: ModelConfig) -> Dict[str, Any]:
514
- config = try_get_generation_config(
515
- model_config.model,
516
- trust_remote_code=model_config.trust_remote_code,
517
- revision=model_config.revision,
518
- )
519
-
520
- if config is None:
521
- return {}
165
+ def stop_profile(self):
166
+ pass
167
523
- return config.to_diff_dict()
168
+ def get_tokenizer_group(self, group_type):
169
+ pass
vllm/v1/engine/processor.py
new
+128
@@ -0,0 +1,128 @@
1
+import time
2
+from typing import Any, Dict, Mapping, Optional, Tuple, Union
3
+
4
+from vllm.config import LoRAConfig, ModelConfig
5
+from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs,
6
+ EncoderDecoderLLMInputs, InputRegistry, PromptType)
7
+from vllm.inputs.preprocess import InputPreprocessor
8
+from vllm.lora.request import LoRARequest
9
+from vllm.pooling_params import PoolingParams
10
+from vllm.prompt_adapter.request import PromptAdapterRequest
11
+from vllm.sampling_params import SamplingParams
12
+from vllm.transformers_utils.config import try_get_generation_config
13
+from vllm.transformers_utils.tokenizer_group import AnyTokenizer
14
+from vllm.v1.engine import DetokenizerRequest, EngineCoreRequest
15
+
16
+
17
+class Processor:
18
+
19
+ def __init__(
20
+ self,
21
+ model_config: ModelConfig,
22
+ lora_config: Optional[LoRAConfig],
23
+ tokenizer: AnyTokenizer,
24
+ input_registry: InputRegistry = INPUT_REGISTRY,
25
+ ):
26
+
27
+ self.model_config = model_config
28
+ self.lora_config = lora_config
29
+ self.tokenizer = tokenizer
30
+
31
+ self.generation_config_fields = _load_generation_config_dict(
32
+ model_config)
33
+ self.input_preprocessor = InputPreprocessor(model_config,
34
+ self.tokenizer)
35
+ self.input_processor = input_registry.create_input_processor(
36
+ model_config)
37
+
38
+ # TODO: run in an ThreadpoolExecutor or BackgroundProcess.
39
+ # This ideally should releases the GIL, so we should not block the
40
+ # asyncio loop while this is running.
41
+ def process_inputs(
42
+ self,
43
+ request_id: str,
44
+ prompt: PromptType,
45
+ params: Union[SamplingParams, PoolingParams],
46
+ arrival_time: float,
47
+ lora_request: Optional[LoRARequest] = None,
48
+ trace_headers: Optional[Mapping[str, str]] = None,
49
+ prompt_adapter_request: Optional[PromptAdapterRequest] = None,
50
+ priority: int = 0,
51
+ ) -> Tuple[DetokenizerRequest, EngineCoreRequest]:
52
+
53
+ # TODO(woosuk): Support embedding mode.
54
+ # TODO(woosuk): Check max_logprobs
55
+ # TODO(woosuk): Support encoder-decoder models.
56
+
57
+ if lora_request is not None and not self.lora_config:
58
+ raise ValueError(f"Got lora_request {lora_request} but LoRA is "
59
+ "not enabled!")
60
+ if arrival_time is None:
61
+ arrival_time = time.time()
62
+ assert priority == 0, "vLLM V1 does not support priority at the moment."
63
+ assert trace_headers is None, "vLLM V1 does not support tracing yet."
64
+
65
+ # Process inputs.
66
+ preprocessed_inputs = self.input_preprocessor.preprocess(
67
+ prompt,
68
+ request_id=request_id,
69
+ lora_request=lora_request,
70
+ prompt_adapter_request=prompt_adapter_request,
71
+ )
72
+ processed_inputs = self.input_processor(preprocessed_inputs)
73
+ self._validate_model_inputs(processed_inputs)
74
+ eos_token_id = self.input_preprocessor.get_eos_token_id(lora_request)
75
+
76
+ assert isinstance(params, SamplingParams)
77
+ # TODO: can we avoid cloning here in multiproc case
78
+ sampling_params = params.clone()
79
+ sampling_params.update_from_generation_config(
80
+ self.generation_config_fields, eos_token_id)
81
+
82
+ # Make Request for Detokenizer.
83
+ detokenizer_request = DetokenizerRequest(
84
+ request_id, processed_inputs.get("prompt"),
85
+ processed_inputs.get("prompt_token_ids"),
86
+ sampling_params.skip_special_tokens,
87
+ sampling_params.spaces_between_special_tokens,
88
+ sampling_params.output_kind, sampling_params.stop,
89
+ sampling_params.include_stop_str_in_output)
90
+
91
+ # Make Request for EngineCore.
92
+ engine_core_request = EngineCoreRequest(
93
+ request_id, processed_inputs.get("prompt"),
94
+ processed_inputs.get("prompt_token_ids"), sampling_params,
95
+ eos_token_id, arrival_time, lora_request)
96
+
97
+ return detokenizer_request, engine_core_request
98
+
99
+ def _validate_model_inputs(self, inputs: Union[DecoderOnlyInputs,
100
+ EncoderDecoderLLMInputs]):
101
+ prompt_ids = inputs.get("prompt_token_ids")
102
+ if prompt_ids is None or len(prompt_ids) == 0:
103
+ raise ValueError("Prompt cannot be empty")
104
+
105
+ if self.model_config.is_multimodal_model:
106
+ max_prompt_len = self.model_config.max_model_len
107
+
108
+ if len(prompt_ids) > max_prompt_len:
109
+ raise ValueError(
110
+ f"The prompt (total length {len(prompt_ids)}) is too long "
111
+ f"to fit into the model (context length {max_prompt_len}). "
112
+ "Make sure that `max_model_len` is no smaller than the "
113
+ "number of text tokens plus multimodal tokens. For image "
114
+ "inputs, the number of image tokens depends on the number "
115
+ "of images, and possibly their aspect ratios as well.")
116
+
117
+
118
+def _load_generation_config_dict(model_config: ModelConfig) -> Dict[str, Any]:
119
+ config = try_get_generation_config(
120
+ model_config.model,
121
+ trust_remote_code=model_config.trust_remote_code,
122
+ revision=model_config.revision,
123
+ )
124
+
125
+ if config is None:
126
+ return {}
127
+
128
+ return config.to_diff_dict()
vllm/v1/request.py
+16
-1
@@ -1,9 +1,11 @@
1
import enum
2
from typing import TYPE_CHECKING, List, Optional, Union
3
4
+from vllm.inputs.data import DecoderOnlyInputs
5
from vllm.lora.request import LoRARequest
6
from vllm.sampling_params import SamplingParams
7
from vllm.sequence import RequestMetrics
8
+from vllm.v1.engine import EngineCoreRequest
9
from vllm.v1.utils import ConstantList
10
11
if TYPE_CHECKING:
@@ -43,9 +45,22 @@ class Request:
45
self.num_prompt_tokens = len(self.prompt_token_ids)
46
self._output_token_ids: List[int] = []
47
self._all_token_ids: List[int] = self.prompt_token_ids.copy()
46
- self.output_text = ""
48
self.num_computed_tokens = 0
49
50
+ @classmethod
51
+ def from_engine_core_request(cls, request: EngineCoreRequest) -> "Request":
52
+
53
+ return cls(
54
+ request_id=request.request_id,
55
+ inputs=DecoderOnlyInputs(type="token",
56
+ prompt_token_ids=request.prompt_token_ids,
57
+ prompt=request.prompt),
58
+ sampling_params=request.sampling_params,
59
+ eos_token_id=request.eos_token_id,
60
+ arrival_time=request.arrival_time,
61
+ lora_request=request.lora_request,
62
+ )
63
+
64
@property
65
def output_token_ids(self) -> ConstantList[int]:
66
# Prevent directly appending to the output_token_ids since
vllm/v1/tokenizer/detokenizer.py
deleted
-228
@@ -1,228 +0,0 @@
1
-import multiprocessing
2
-from dataclasses import dataclass
3
-from typing import Dict, List, Optional
4
-
5
-import msgspec
6
-import zmq
7
-from msgspec import msgpack
8
-
9
-from vllm.transformers_utils.detokenizer_utils import (
10
- convert_prompt_ids_to_tokens, detokenize_incrementally)
11
-from vllm.transformers_utils.tokenizer import get_tokenizer
12
-from vllm.utils import get_open_port
13
-
14
-
15
-class DetokenizerInputs(msgspec.Struct):
16
-
17
- # [num_reqs]
18
- req_ids: List[str]
19
- # A request's prompt token ids is sent to the detokenizer only when
20
- # the request is first detokenized. Otherwise, an empty list is sent.
21
- prompt_token_ids: List[List[int]]
22
- new_token_ids: List[List[int]]
23
- skip_special_tokens: List[bool]
24
- spaces_between_special_tokens: List[bool]
25
-
26
- # [num_free_reqs]
27
- free_req_ids: List[str]
28
-
29
-
30
-class DetokenizerOutputs(msgspec.Struct):
31
-
32
- # [num_reqs]
33
- req_ids: List[str]
34
- detokenized_texts: List[str]
35
- # NOTE(woosuk): The number of the output token ids of each request
36
- # at the time of detokenization. The detokenizer returns this to the engine
37
- # because the request state (including the output token ids) is
38
- # asynchronously updated in the engine, while RequestOutput requires the
39
- # output token ids to be consistent with the detokenized text.
40
- num_output_token_ids: List[int]
41
-
42
-
43
-class Detokenizer:
44
-
45
- def __init__(self, tokenizer_name: str, tokenizer_mode: str,
46
- trust_remote_code: bool):
47
- # FIXME(woosuk): Currently, the detokenizer is just a hacky prototype.
48
- # For example, it does not terminate properly. We need to improve this.
49
- self.push_port = get_open_port()
50
- self.pull_port = get_open_port()
51
- # NOTE: The push port of the engine process should be the same as the
52
- # pull port of the detokenizer process. Vice versa.
53
- self.detokenizer = DetokenizerProc(tokenizer_name=tokenizer_name,
54
- tokenizer_mode=tokenizer_mode,
55
- trust_remote_code=trust_remote_code,
56
- push_port=self.pull_port,
57
- pull_port=self.push_port)
58
- self.detokenizer.start()
59
-
60
- self.zmq_context = zmq.Context()
61
- self.push_socket = self.zmq_context.socket(zmq.PUSH)
62
- self.push_socket.connect(f"tcp://localhost:{self.push_port}")
63
- self.pull_socket = self.zmq_context.socket(zmq.PULL)
64
- self.pull_socket.connect(f"tcp://localhost:{self.pull_port}")
65
- self.poller = zmq.Poller()
66
- self.poller.register(self.pull_socket, zmq.POLLIN)
67
- self.msgpack_encoder = msgpack.Encoder()
68
- self.msgpack_decoder = msgpack.Decoder(DetokenizerOutputs)
69
-
70
- def send(self, inputs: DetokenizerInputs) -> None:
71
- self.push_socket.send(self.msgpack_encoder.encode(inputs),
72
- flags=zmq.NOBLOCK)
73
-
74
- def recv(self) -> Optional[DetokenizerOutputs]:
75
- socks = dict(self.poller.poll(timeout=0))
76
- if self.pull_socket in socks and socks[self.pull_socket] == zmq.POLLIN:
77
- msg = self.pull_socket.recv()
78
- return self.msgpack_decoder.decode(msg)
79
- return None
80
-
81
- def terminate(self) -> None:
82
- self.detokenizer.kill()
83
- self.detokenizer.join()
84
-
85
-
86
-class DetokenizerProc(multiprocessing.Process):
87
-
88
- def __init__(
89
- self,
90
- tokenizer_name: str,
91
- tokenizer_mode: str,
92
- trust_remote_code: bool,
93
- pull_port: int,
94
- push_port: int,
95
- ):
96
- super().__init__()
97
- self.tokenizer_name = tokenizer_name
98
- self.tokenizer_mode = tokenizer_mode
99
- self.trust_remote_code = trust_remote_code
100
- # NOTE: The pull_port of the detokenizer process should be the same as
101
- # the push_port of the engine process. Vice versa.
102
- self.pull_port = pull_port
103
- self.push_port = push_port
104
-
105
- def run(self):
106
- # Initialize these objects after the process is forked since they are
107
- # not picklable.
108
- self.msgpack_encoder = msgpack.Encoder()
109
- self.msgpack_decoder = msgpack.Decoder(DetokenizerInputs)
110
- self.tokenizer = get_tokenizer(
111
- tokenizer_name=self.tokenizer_name,
112
- tokenizer_mode=self.tokenizer_mode,
113
- trust_remote_code=self.trust_remote_code)
114
- # req_id -> RequestState
115
- self.request_states: Dict[str, RequestState] = {}
116
-
117
- self.zmq_context = zmq.Context()
118
- self.pull_socket = self.zmq_context.socket(zmq.PULL)
119
- self.pull_socket.bind(f"tcp://*:{self.pull_port}")
120
- self.push_socket = self.zmq_context.socket(zmq.PUSH)
121
- self.push_socket.bind(f"tcp://*:{self.push_port}")
122
-
123
- while True:
124
- if self.pull_socket.poll(timeout=1000) == 0:
125
- # Nothing to read
126
- continue
127
- message = self.pull_socket.recv()
128
- inputs = self.msgpack_decoder.decode(message)
129
-
130
- for req_id in inputs.free_req_ids:
131
- self.free(req_id)
132
-
133
- detokenized_texts: List[str] = []
134
- num_output_token_ids: List[int] = []
135
- num_reqs = len(inputs.req_ids)
136
- for i in range(num_reqs):
137
- req_id = inputs.req_ids[i]
138
- if req_id not in self.request_states:
139
- self.add_request(
140
- request_id=req_id,
141
- prompt_token_ids=inputs.prompt_token_ids[i],
142
- skip_special_tokens=inputs.skip_special_tokens[i],
143
- spaces_between_special_tokens=inputs.
144
- spaces_between_special_tokens[i],
145
- )
146
- new_str = self.detokenize(req_id, inputs.new_token_ids[i])
147
- detokenized_texts.append(new_str)
148
- req_state = self.request_states[req_id]
149
- num_output_token_ids.append(
150
- len(req_state.token_ids) - req_state.num_prompt_tokens)
151
-
152
- detokenized = DetokenizerOutputs(
153
- req_ids=inputs.req_ids,
154
- detokenized_texts=detokenized_texts,
155
- num_output_token_ids=num_output_token_ids,
156
- )
157
- self.push_socket.send(self.msgpack_encoder.encode(detokenized),
158
- flags=zmq.NOBLOCK)
159
-
160
- def add_request(
161
- self,
162
- request_id: str,
163
- prompt_token_ids: List[int],
164
- skip_special_tokens: bool,
165
- spaces_between_special_tokens: bool,
166
- ) -> None:
167
- tokens, prefix_offset, read_offset = convert_prompt_ids_to_tokens(
168
- tokenizer=self.tokenizer,
169
- prompt_ids=prompt_token_ids,
170
- skip_special_tokens=skip_special_tokens,
171
- )
172
- self.request_states[request_id] = RequestState(
173
- req_id=request_id,
174
- token_ids=prompt_token_ids,
175
- tokens=tokens,
176
- num_prompt_tokens=len(prompt_token_ids),
177
- prefix_offset=prefix_offset,
178
- read_offset=read_offset,
179
- skip_special_tokens=skip_special_tokens,
180
- spaces_between_special_tokens=spaces_between_special_tokens,
181
- )
182
-
183
- def free(self, request_id: str) -> None:
184
- del self.request_states[request_id]
185
-
186
- def detokenize(self, request_id: str, new_token_ids: List[int]) -> str:
187
- # TODO(woosuk): This method becomes very inefficient when the number of
188
- # new_token_ids is more than 1. We need to optimize this.
189
- req_state = self.request_states[request_id]
190
- decoded_text = ""
191
- for new_token_id in new_token_ids:
192
- req_state.token_ids.append(new_token_id)
193
- (new_tokens, new_decoded_token_text, prefix_offset,
194
- read_offset) = detokenize_incrementally(
195
- tokenizer=self.tokenizer,
196
- all_input_ids=req_state.token_ids,
197
- prev_tokens=req_state.tokens,
198
- prefix_offset=req_state.prefix_offset,
199
- read_offset=req_state.read_offset,
200
- skip_special_tokens=req_state.skip_special_tokens,
201
- spaces_between_special_tokens=req_state.
202
- spaces_between_special_tokens,
203
- )
204
-
205
- req_state.tokens.extend(new_tokens)
206
- req_state.prefix_offset = prefix_offset
207
- req_state.read_offset = read_offset
208
- req_state.output_text += new_decoded_token_text
209
- decoded_text += new_decoded_token_text
210
- return decoded_text
211
-
212
-
213
-@dataclass
214
-class RequestState:
215
-
216
- req_id: str
217
-
218
- token_ids: List[int]
219
- tokens: List[str]
220
- num_prompt_tokens: int
221
-
222
- prefix_offset: int
223
- read_offset: int
224
-
225
- skip_special_tokens: bool
226
- spaces_between_special_tokens: bool
227
-
228
- output_text: str = ""