A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 01:45 UTC
[Bug fix][Core] fixup ngram not setup correctly (#4551)
Co-authored-by: Lei Wen <wenlei03@qiyi.com> Co-authored-by: Cade Daniel <edacih@gmail.com> Co-authored-by: Cody Yu <hao.yu.cody@gmail.com>
leiwen83 committed
May 8, 2024 at 02:40 UTC
8344f7742b794ca6ec9bcb891c178cd0551f23d0
3 files changed
+29
-13
tests/spec_decode/e2e/conftest.py
+18
-6
@@ -55,7 +55,7 @@ class AsyncLLM:
55
) -> None:
56
if "disable_log_stats" not in kwargs:
57
kwargs["disable_log_stats"] = True
58
- self.engine_args = AsyncEngineArgs(
58
+ engine_args = AsyncEngineArgs(
59
model=model,
60
tokenizer=tokenizer,
61
tokenizer_mode=tokenizer_mode,
@@ -76,6 +76,8 @@ class AsyncLLM:
76
**kwargs,
77
)
78
self.request_counter = Counter()
79
+ self.llm_engine = AsyncLLMEngine.from_engine_args(
80
+ engine_args, usage_context=UsageContext.LLM_CLASS)
81
82
def generate(
83
self,
@@ -88,9 +90,6 @@ class AsyncLLM:
90
multi_modal_data: Optional[MultiModalData] = None,
91
) -> List[RequestOutput]:
92
91
- llm_engine = AsyncLLMEngine.from_engine_args(
92
- self.engine_args, usage_context=UsageContext.LLM_CLASS)
93
-
93
if prompts is None:
94
raise ValueError("prompts must be provided.")
95
if isinstance(prompts, str):
@@ -111,8 +110,8 @@ class AsyncLLM:
110
111
async def get_output(prompt, sampling_param) -> str:
112
request_id = random_uuid()
114
- results_generator = llm_engine.generate(prompt, sampling_param,
115
- request_id)
113
+ results_generator = self.llm_engine.generate(
114
+ prompt, sampling_param, request_id)
115
final_output = None
116
async for request_output in results_generator:
117
final_output = request_output
@@ -185,12 +184,25 @@ def create_llm_generator(baseline_or_test, request, common_llm_kwargs,
184
return generator_outer
185
186
187
+def maybe_assert_ngram_worker(llm):
188
+ # Verify the proposer worker is ngram if ngram is specified.
189
+ if (not isinstance(llm, AsyncLLM)
190
+ and llm.llm_engine.speculative_config is not None
191
+ and llm.llm_engine.speculative_config.ngram_prompt_lookup_max > 0):
192
+ from vllm.spec_decode.ngram_worker import NGramWorker
193
+ assert isinstance(
194
+ llm.llm_engine.model_executor.driver_worker.proposer_worker,
195
+ NGramWorker)
196
+
197
+
198
def get_output_from_llm_generator(
199
llm_generator, prompts,
200
sampling_params) -> Tuple[List[str], List[List[int]]]:
201
tokens = []
202
token_ids = []
203
for llm in llm_generator():
204
+ maybe_assert_ngram_worker(llm)
205
+
206
outputs = llm.generate(prompts, sampling_params, use_tqdm=True)
207
token_ids = [output.outputs[0].token_ids for output in outputs]
208
tokens = [output.outputs[0].text for output in outputs]
vllm/executor/gpu_executor.py
+4
@@ -82,6 +82,10 @@ class GPUExecutor(ExecutorBase):
82
draft_worker_kwargs.update(
83
model_config=self.speculative_config.draft_model_config,
84
parallel_config=self.speculative_config.draft_parallel_config,
85
+ ngram_prompt_lookup_max=self.speculative_config.
86
+ ngram_prompt_lookup_max,
87
+ ngram_prompt_lookup_min=self.speculative_config.
88
+ ngram_prompt_lookup_min,
89
# TODO allow draft-model specific load config.
90
#load_config=self.load_config,
91
)
vllm/spec_decode/spec_decode_worker.py
+7
-7
@@ -57,13 +57,10 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
57
draft_worker_kwargs,
58
) -> "SpecDecodeWorker":
59
60
- if "ngram_prompt_lookup_max" in draft_worker_kwargs:
61
- ngram_prompt_lookup_max = (
62
- draft_worker_kwargs.pop("ngram_prompt_lookup_max"))
63
- ngram_prompt_lookup_min = (
64
- draft_worker_kwargs.pop("ngram_prompt_lookup_min"))
65
- else:
66
- ngram_prompt_lookup_max = 0
60
+ ngram_prompt_lookup_max = (
61
+ draft_worker_kwargs.pop("ngram_prompt_lookup_max"))
62
+ ngram_prompt_lookup_min = (
63
+ draft_worker_kwargs.pop("ngram_prompt_lookup_min"))
64
65
if ngram_prompt_lookup_max > 0:
66
proposer_worker = NGramWorker(**draft_worker_kwargs)
@@ -72,6 +69,9 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
69
else:
70
proposer_worker = MultiStepWorker(**draft_worker_kwargs)
71
72
+ logger.info("Configuring SpecDecodeWorker with proposer=%s",
73
+ type(proposer_worker))
74
+
75
return SpecDecodeWorker(
76
proposer_worker,
77
scorer_worker,