A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 01:45 UTC
[BugFix] fix num_lookahead_slots missing in async executor (#4165)
Co-authored-by: Lei Wen <wenlei03@qiyi.com>
leiwen83 committed
May 1, 2024 at 01:12 UTC
4bb53e2dde809ea5727b8cac95a080893733a1ef
9 files changed
+163
-19
tests/spec_decode/e2e/conftest.py
+123
-2
@@ -1,10 +1,127 @@
1
-from typing import List, Tuple
1
+import asyncio
2
+from typing import List, Optional, Tuple, Union
3
4
import pytest
5
+import ray
6
7
from tests.conftest import cleanup
8
from vllm import LLM
9
+from vllm.engine.arg_utils import AsyncEngineArgs
10
+from vllm.engine.async_llm_engine import AsyncLLMEngine
11
+from vllm.lora.request import LoRARequest
12
from vllm.model_executor.utils import set_random_seed
13
+from vllm.outputs import RequestOutput
14
+from vllm.sampling_params import SamplingParams
15
+from vllm.sequence import MultiModalData
16
+from vllm.usage.usage_lib import UsageContext
17
+from vllm.utils import Counter, random_uuid
18
+
19
+
20
+class AsyncLLM:
21
+ """AsyncLLM
22
+
23
+ Note: Current LLM class in vllm don't support async mode, for test purpose,
24
+ we implement async one in here. Maybe we could move to
25
+ vllm/entrypoints/llm.py in future.
26
+
27
+ Below AsyncLLM is directly borrow from vllm/entrypoints/llm.py with changes
28
+ to make to work in async mode.
29
+ """
30
+
31
+ def __init__(
32
+ self,
33
+ model: str,
34
+ tokenizer: Optional[str] = None,
35
+ tokenizer_mode: str = "auto",
36
+ skip_tokenizer_init: bool = False,
37
+ trust_remote_code: bool = False,
38
+ tensor_parallel_size: int = 1,
39
+ dtype: str = "auto",
40
+ quantization: Optional[str] = None,
41
+ revision: Optional[str] = None,
42
+ tokenizer_revision: Optional[str] = None,
43
+ seed: int = 0,
44
+ gpu_memory_utilization: float = 0.9,
45
+ swap_space: int = 4,
46
+ enforce_eager: bool = False,
47
+ max_context_len_to_capture: int = 8192,
48
+ disable_custom_all_reduce: bool = False,
49
+ **kwargs,
50
+ ) -> None:
51
+ if "disable_log_stats" not in kwargs:
52
+ kwargs["disable_log_stats"] = True
53
+ self.engine_args = AsyncEngineArgs(
54
+ model=model,
55
+ tokenizer=tokenizer,
56
+ tokenizer_mode=tokenizer_mode,
57
+ skip_tokenizer_init=skip_tokenizer_init,
58
+ trust_remote_code=trust_remote_code,
59
+ tensor_parallel_size=tensor_parallel_size,
60
+ dtype=dtype,
61
+ quantization=quantization,
62
+ revision=revision,
63
+ tokenizer_revision=tokenizer_revision,
64
+ seed=seed,
65
+ gpu_memory_utilization=gpu_memory_utilization,
66
+ swap_space=swap_space,
67
+ enforce_eager=enforce_eager,
68
+ max_context_len_to_capture=max_context_len_to_capture,
69
+ engine_use_ray=True,
70
+ disable_custom_all_reduce=disable_custom_all_reduce,
71
+ **kwargs,
72
+ )
73
+ self.request_counter = Counter()
74
+
75
+ def generate(
76
+ self,
77
+ prompts: Optional[Union[str, List[str]]] = None,
78
+ sampling_params: Optional[Union[SamplingParams,
79
+ List[SamplingParams]]] = None,
80
+ prompt_token_ids: Optional[List[List[int]]] = None,
81
+ use_tqdm: bool = True,
82
+ lora_request: Optional[LoRARequest] = None,
83
+ multi_modal_data: Optional[MultiModalData] = None,
84
+ ) -> List[RequestOutput]:
85
+
86
+ llm_engine = AsyncLLMEngine.from_engine_args(
87
+ self.engine_args, usage_context=UsageContext.LLM_CLASS)
88
+
89
+ if prompts is None:
90
+ raise ValueError("prompts must be provided.")
91
+ if isinstance(prompts, str):
92
+ # Convert a single prompt to a list.
93
+ prompts = [prompts]
94
+
95
+ if prompts is not None:
96
+ num_requests = len(prompts)
97
+
98
+ if sampling_params is None:
99
+ # Use default sampling params.
100
+ sampling_params = SamplingParams()
101
+
102
+ elif isinstance(sampling_params,
103
+ list) and len(sampling_params) != num_requests:
104
+ raise ValueError("The lengths of prompts and "
105
+ "sampling_params must be the same.")
106
+
107
+ async def get_output(prompt, sampling_param) -> str:
108
+ request_id = random_uuid()
109
+ results_generator = llm_engine.generate(prompt, sampling_param,
110
+ request_id)
111
+ final_output = None
112
+ async for request_output in results_generator:
113
+ final_output = request_output
114
+ return final_output
115
+
116
+ outputs = []
117
+ try:
118
+ for i in range(num_requests):
119
+ prompt = prompts[i] if prompts is not None else None
120
+ res = asyncio.run(get_output(prompt, sampling_params))
121
+ outputs.append(res)
122
+ finally:
123
+ ray.shutdown()
124
+ return outputs
125
126
127
@pytest.fixture
@@ -36,8 +153,12 @@ def create_llm_generator(baseline_or_test, request, common_llm_kwargs,
153
154
def generator_inner():
155
print(f'Creating {baseline_or_test=} LLM for {test_name=}. {kwargs=}')
39
- llm = LLM(**kwargs)
156
157
+ use_async = False
158
+ if "use_async" in kwargs:
159
+ use_async = kwargs.pop("use_async")
160
+
161
+ llm = AsyncLLM(**kwargs) if use_async else LLM(**kwargs)
162
set_random_seed(seed)
163
164
yield llm
tests/spec_decode/e2e/test_compatibility.py
+11
-4
@@ -42,10 +42,17 @@ def test_spec_decode_xfail_ray(test_llm_generator):
42
temperature=temperature,
43
)
44
45
- with pytest.raises(AssertionError,
46
- match="Speculative decoding not yet supported for "):
47
- get_output_from_llm_generator(test_llm_generator, prompts,
48
- sampling_params)
45
+ try:
46
+ with pytest.raises(
47
+ AssertionError,
48
+ match="Speculative decoding not yet supported for "):
49
+ get_output_from_llm_generator(test_llm_generator, prompts,
50
+ sampling_params)
51
+ finally:
52
+ # we need to free up ray resource,
53
+ # so that latter test could use the gpu we allocated here
54
+ import ray
55
+ ray.shutdown()
56
57
58
@pytest.mark.parametrize(
tests/spec_decode/e2e/test_correctness.py
+16
-9
@@ -40,17 +40,24 @@ from .conftest import get_output_from_llm_generator
40
41
@pytest.mark.parametrize(
42
"common_llm_kwargs",
43
- [{
44
- # Use a small model for a fast test.
45
- # Note this is repeated in the test body; to initialize a tokenizer.
46
- "model": "JackFram/llama-68m",
43
+ [
44
+ {
45
+ # Use a small model for a fast test.
46
+ # Note this is repeated in the test body; to initialize a tokenizer.
47
+ "model": "JackFram/llama-68m",
48
48
- # Skip cuda graph recording for fast test.
49
- "enforce_eager": True,
49
+ # Skip cuda graph recording for fast test.
50
+ "enforce_eager": True,
51
51
- # Required for spec decode.
52
- "use_v2_block_manager": True
53
- }])
52
+ # Required for spec decode.
53
+ "use_v2_block_manager": True,
54
+
55
+ # whether use AsyncLLM engine
56
+ "use_async": async_mode,
57
+ }
58
+ # Try both async and sync engine execution
59
+ for async_mode in [True, False]
60
+ ])
61
@pytest.mark.parametrize(
62
"per_test_common_llm_kwargs",
63
[
vllm/engine/async_llm_engine.py
+4
-2
@@ -211,9 +211,11 @@ class _AsyncLLMEngine(LLMEngine):
211
if not scheduler_outputs.is_empty():
212
# Execute the model.
213
output = await self.model_executor.execute_model_async(
214
- seq_group_metadata_list, scheduler_outputs.blocks_to_swap_in,
214
+ seq_group_metadata_list,
215
+ scheduler_outputs.blocks_to_swap_in,
216
scheduler_outputs.blocks_to_swap_out,
216
- scheduler_outputs.blocks_to_copy)
217
+ scheduler_outputs.blocks_to_copy,
218
+ num_lookahead_slots=scheduler_outputs.num_lookahead_slots)
219
else:
220
output = []
221
vllm/executor/cpu_executor.py
+3
-1
@@ -109,12 +109,14 @@ class CPUExecutorAsync(CPUExecutor, ExecutorAsyncBase):
109
blocks_to_swap_in: Dict[int, int],
110
blocks_to_swap_out: Dict[int, int],
111
blocks_to_copy: Dict[int, List[int]],
112
+ num_lookahead_slots: int,
113
) -> List[SamplerOutput]:
114
output = await make_async(self.driver_worker.execute_model)(
115
seq_group_metadata_list=seq_group_metadata_list,
116
blocks_to_swap_in=blocks_to_swap_in,
117
blocks_to_swap_out=blocks_to_swap_out,
117
- blocks_to_copy=blocks_to_copy)
118
+ blocks_to_copy=blocks_to_copy,
119
+ num_lookahead_slots=num_lookahead_slots)
120
return output
121
122
async def check_health_async(self) -> None:
vllm/executor/executor_base.py
+1
@@ -112,6 +112,7 @@ class ExecutorAsyncBase(ExecutorBase):
112
blocks_to_swap_in: Dict[int, int],
113
blocks_to_swap_out: Dict[int, int],
114
blocks_to_copy: Dict[int, List[int]],
115
+ num_lookahead_slots: int,
116
) -> List[SamplerOutput]:
117
"""Executes one model step on the given sequences."""
118
raise NotImplementedError
vllm/executor/gpu_executor.py
+3
-1
@@ -163,10 +163,12 @@ class GPUExecutorAsync(GPUExecutor, ExecutorAsyncBase):
163
blocks_to_swap_in: Dict[int, int],
164
blocks_to_swap_out: Dict[int, int],
165
blocks_to_copy: Dict[int, List[int]],
166
+ num_lookahead_slots: int,
167
) -> List[SamplerOutput]:
168
output = await make_async(self.driver_worker.execute_model)(
169
seq_group_metadata_list=seq_group_metadata_list,
170
blocks_to_swap_in=blocks_to_swap_in,
171
blocks_to_swap_out=blocks_to_swap_out,
171
- blocks_to_copy=blocks_to_copy)
172
+ blocks_to_copy=blocks_to_copy,
173
+ num_lookahead_slots=num_lookahead_slots)
174
return output
vllm/executor/neuron_executor.py
+1
@@ -84,6 +84,7 @@ class NeuronExecutorAsync(NeuronExecutor, ExecutorAsyncBase):
84
blocks_to_swap_in: Dict[int, int],
85
blocks_to_swap_out: Dict[int, int],
86
blocks_to_copy: Dict[int, List[int]],
87
+ num_lookahead_slots: int,
88
) -> List[SamplerOutput]:
89
output = await make_async(self.driver_worker.execute_model)(
90
seq_group_metadata_list=seq_group_metadata_list, )
vllm/executor/ray_gpu_executor.py
+1
@@ -196,6 +196,7 @@ class RayGPUExecutor(DistributedGPUExecutor):
196
"blocks_to_swap_in": blocks_to_swap_in,
197
"blocks_to_swap_out": blocks_to_swap_out,
198
"blocks_to_copy": blocks_to_copy,
199
+ "num_lookahead_slots": num_lookahead_slots,
200
},
201
use_ray_compiled_dag=USE_RAY_COMPILED_DAG)
202