@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 4bb53e2dde
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 01:45 UTC

[BugFix] fix num_lookahead_slots missing in async executor (#4165)

Co-authored-by: Lei Wen <wenlei03@qiyi.com>

leiwen83 committed May 1, 2024 at 01:12 UTC 4bb53e2dde809ea5727b8cac95a080893733a1ef
9 files changed +163 -19
tests/spec_decode/e2e/conftest.py
+123 -2
@@ -1,10 +1,127 @@
1 -from typing import List, Tuple
1 +import asyncio
2 +from typing import List, Optional, Tuple, Union
3
4 import pytest
5 +import ray
6
7 from tests.conftest import cleanup
8 from vllm import LLM
9 +from vllm.engine.arg_utils import AsyncEngineArgs
10 +from vllm.engine.async_llm_engine import AsyncLLMEngine
11 +from vllm.lora.request import LoRARequest
12 from vllm.model_executor.utils import set_random_seed
13 +from vllm.outputs import RequestOutput
14 +from vllm.sampling_params import SamplingParams
15 +from vllm.sequence import MultiModalData
16 +from vllm.usage.usage_lib import UsageContext
17 +from vllm.utils import Counter, random_uuid
18 +
19 +
20 +class AsyncLLM:
21 + """AsyncLLM
22 +
23 + Note: Current LLM class in vllm don't support async mode, for test purpose,
24 + we implement async one in here. Maybe we could move to
25 + vllm/entrypoints/llm.py in future.
26 +
27 + Below AsyncLLM is directly borrow from vllm/entrypoints/llm.py with changes
28 + to make to work in async mode.
29 + """
30 +
31 + def __init__(
32 + self,
33 + model: str,
34 + tokenizer: Optional[str] = None,
35 + tokenizer_mode: str = "auto",
36 + skip_tokenizer_init: bool = False,
37 + trust_remote_code: bool = False,
38 + tensor_parallel_size: int = 1,
39 + dtype: str = "auto",
40 + quantization: Optional[str] = None,
41 + revision: Optional[str] = None,
42 + tokenizer_revision: Optional[str] = None,
43 + seed: int = 0,
44 + gpu_memory_utilization: float = 0.9,
45 + swap_space: int = 4,
46 + enforce_eager: bool = False,
47 + max_context_len_to_capture: int = 8192,
48 + disable_custom_all_reduce: bool = False,
49 + **kwargs,
50 + ) -> None:
51 + if "disable_log_stats" not in kwargs:
52 + kwargs["disable_log_stats"] = True
53 + self.engine_args = AsyncEngineArgs(
54 + model=model,
55 + tokenizer=tokenizer,
56 + tokenizer_mode=tokenizer_mode,
57 + skip_tokenizer_init=skip_tokenizer_init,
58 + trust_remote_code=trust_remote_code,
59 + tensor_parallel_size=tensor_parallel_size,
60 + dtype=dtype,
61 + quantization=quantization,
62 + revision=revision,
63 + tokenizer_revision=tokenizer_revision,
64 + seed=seed,
65 + gpu_memory_utilization=gpu_memory_utilization,
66 + swap_space=swap_space,
67 + enforce_eager=enforce_eager,
68 + max_context_len_to_capture=max_context_len_to_capture,
69 + engine_use_ray=True,
70 + disable_custom_all_reduce=disable_custom_all_reduce,
71 + **kwargs,
72 + )
73 + self.request_counter = Counter()
74 +
75 + def generate(
76 + self,
77 + prompts: Optional[Union[str, List[str]]] = None,
78 + sampling_params: Optional[Union[SamplingParams,
79 + List[SamplingParams]]] = None,
80 + prompt_token_ids: Optional[List[List[int]]] = None,
81 + use_tqdm: bool = True,
82 + lora_request: Optional[LoRARequest] = None,
83 + multi_modal_data: Optional[MultiModalData] = None,
84 + ) -> List[RequestOutput]:
85 +
86 + llm_engine = AsyncLLMEngine.from_engine_args(
87 + self.engine_args, usage_context=UsageContext.LLM_CLASS)
88 +
89 + if prompts is None:
90 + raise ValueError("prompts must be provided.")
91 + if isinstance(prompts, str):
92 + # Convert a single prompt to a list.
93 + prompts = [prompts]
94 +
95 + if prompts is not None:
96 + num_requests = len(prompts)
97 +
98 + if sampling_params is None:
99 + # Use default sampling params.
100 + sampling_params = SamplingParams()
101 +
102 + elif isinstance(sampling_params,
103 + list) and len(sampling_params) != num_requests:
104 + raise ValueError("The lengths of prompts and "
105 + "sampling_params must be the same.")
106 +
107 + async def get_output(prompt, sampling_param) -> str:
108 + request_id = random_uuid()
109 + results_generator = llm_engine.generate(prompt, sampling_param,
110 + request_id)
111 + final_output = None
112 + async for request_output in results_generator:
113 + final_output = request_output
114 + return final_output
115 +
116 + outputs = []
117 + try:
118 + for i in range(num_requests):
119 + prompt = prompts[i] if prompts is not None else None
120 + res = asyncio.run(get_output(prompt, sampling_params))
121 + outputs.append(res)
122 + finally:
123 + ray.shutdown()
124 + return outputs
125
126
127 @pytest.fixture
@@ -36,8 +153,12 @@ def create_llm_generator(baseline_or_test, request, common_llm_kwargs,
153
154 def generator_inner():
155 print(f'Creating {baseline_or_test=} LLM for {test_name=}. {kwargs=}')
39 - llm = LLM(**kwargs)
156
157 + use_async = False
158 + if "use_async" in kwargs:
159 + use_async = kwargs.pop("use_async")
160 +
161 + llm = AsyncLLM(**kwargs) if use_async else LLM(**kwargs)
162 set_random_seed(seed)
163
164 yield llm
tests/spec_decode/e2e/test_compatibility.py
+11 -4
@@ -42,10 +42,17 @@ def test_spec_decode_xfail_ray(test_llm_generator):
42 temperature=temperature,
43 )
44
45 - with pytest.raises(AssertionError,
46 - match="Speculative decoding not yet supported for "):
47 - get_output_from_llm_generator(test_llm_generator, prompts,
48 - sampling_params)
45 + try:
46 + with pytest.raises(
47 + AssertionError,
48 + match="Speculative decoding not yet supported for "):
49 + get_output_from_llm_generator(test_llm_generator, prompts,
50 + sampling_params)
51 + finally:
52 + # we need to free up ray resource,
53 + # so that latter test could use the gpu we allocated here
54 + import ray
55 + ray.shutdown()
56
57
58 @pytest.mark.parametrize(
tests/spec_decode/e2e/test_correctness.py
+16 -9
@@ -40,17 +40,24 @@ from .conftest import get_output_from_llm_generator
40
41 @pytest.mark.parametrize(
42 "common_llm_kwargs",
43 - [{
44 - # Use a small model for a fast test.
45 - # Note this is repeated in the test body; to initialize a tokenizer.
46 - "model": "JackFram/llama-68m",
43 + [
44 + {
45 + # Use a small model for a fast test.
46 + # Note this is repeated in the test body; to initialize a tokenizer.
47 + "model": "JackFram/llama-68m",
48
48 - # Skip cuda graph recording for fast test.
49 - "enforce_eager": True,
49 + # Skip cuda graph recording for fast test.
50 + "enforce_eager": True,
51
51 - # Required for spec decode.
52 - "use_v2_block_manager": True
53 - }])
52 + # Required for spec decode.
53 + "use_v2_block_manager": True,
54 +
55 + # whether use AsyncLLM engine
56 + "use_async": async_mode,
57 + }
58 + # Try both async and sync engine execution
59 + for async_mode in [True, False]
60 + ])
61 @pytest.mark.parametrize(
62 "per_test_common_llm_kwargs",
63 [
vllm/engine/async_llm_engine.py
+4 -2
@@ -211,9 +211,11 @@ class _AsyncLLMEngine(LLMEngine):
211 if not scheduler_outputs.is_empty():
212 # Execute the model.
213 output = await self.model_executor.execute_model_async(
214 - seq_group_metadata_list, scheduler_outputs.blocks_to_swap_in,
214 + seq_group_metadata_list,
215 + scheduler_outputs.blocks_to_swap_in,
216 scheduler_outputs.blocks_to_swap_out,
216 - scheduler_outputs.blocks_to_copy)
217 + scheduler_outputs.blocks_to_copy,
218 + num_lookahead_slots=scheduler_outputs.num_lookahead_slots)
219 else:
220 output = []
221
vllm/executor/cpu_executor.py
+3 -1
@@ -109,12 +109,14 @@ class CPUExecutorAsync(CPUExecutor, ExecutorAsyncBase):
109 blocks_to_swap_in: Dict[int, int],
110 blocks_to_swap_out: Dict[int, int],
111 blocks_to_copy: Dict[int, List[int]],
112 + num_lookahead_slots: int,
113 ) -> List[SamplerOutput]:
114 output = await make_async(self.driver_worker.execute_model)(
115 seq_group_metadata_list=seq_group_metadata_list,
116 blocks_to_swap_in=blocks_to_swap_in,
117 blocks_to_swap_out=blocks_to_swap_out,
117 - blocks_to_copy=blocks_to_copy)
118 + blocks_to_copy=blocks_to_copy,
119 + num_lookahead_slots=num_lookahead_slots)
120 return output
121
122 async def check_health_async(self) -> None:
vllm/executor/executor_base.py
+1
@@ -112,6 +112,7 @@ class ExecutorAsyncBase(ExecutorBase):
112 blocks_to_swap_in: Dict[int, int],
113 blocks_to_swap_out: Dict[int, int],
114 blocks_to_copy: Dict[int, List[int]],
115 + num_lookahead_slots: int,
116 ) -> List[SamplerOutput]:
117 """Executes one model step on the given sequences."""
118 raise NotImplementedError
vllm/executor/gpu_executor.py
+3 -1
@@ -163,10 +163,12 @@ class GPUExecutorAsync(GPUExecutor, ExecutorAsyncBase):
163 blocks_to_swap_in: Dict[int, int],
164 blocks_to_swap_out: Dict[int, int],
165 blocks_to_copy: Dict[int, List[int]],
166 + num_lookahead_slots: int,
167 ) -> List[SamplerOutput]:
168 output = await make_async(self.driver_worker.execute_model)(
169 seq_group_metadata_list=seq_group_metadata_list,
170 blocks_to_swap_in=blocks_to_swap_in,
171 blocks_to_swap_out=blocks_to_swap_out,
171 - blocks_to_copy=blocks_to_copy)
172 + blocks_to_copy=blocks_to_copy,
173 + num_lookahead_slots=num_lookahead_slots)
174 return output
vllm/executor/neuron_executor.py
+1
@@ -84,6 +84,7 @@ class NeuronExecutorAsync(NeuronExecutor, ExecutorAsyncBase):
84 blocks_to_swap_in: Dict[int, int],
85 blocks_to_swap_out: Dict[int, int],
86 blocks_to_copy: Dict[int, List[int]],
87 + num_lookahead_slots: int,
88 ) -> List[SamplerOutput]:
89 output = await make_async(self.driver_worker.execute_model)(
90 seq_group_metadata_list=seq_group_metadata_list, )
vllm/executor/ray_gpu_executor.py
+1
@@ -196,6 +196,7 @@ class RayGPUExecutor(DistributedGPUExecutor):
196 "blocks_to_swap_in": blocks_to_swap_in,
197 "blocks_to_swap_out": blocks_to_swap_out,
198 "blocks_to_copy": blocks_to_copy,
199 + "num_lookahead_slots": num_lookahead_slots,
200 },
201 use_ray_compiled_dag=USE_RAY_COMPILED_DAG)
202