@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 8344f7742b
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 01:45 UTC

[Bug fix][Core] fixup ngram not setup correctly (#4551)

Co-authored-by: Lei Wen <wenlei03@qiyi.com> Co-authored-by: Cade Daniel <edacih@gmail.com> Co-authored-by: Cody Yu <hao.yu.cody@gmail.com>

leiwen83 committed May 8, 2024 at 02:40 UTC 8344f7742b794ca6ec9bcb891c178cd0551f23d0
3 files changed +29 -13
tests/spec_decode/e2e/conftest.py
+18 -6
@@ -55,7 +55,7 @@ class AsyncLLM:
55 ) -> None:
56 if "disable_log_stats" not in kwargs:
57 kwargs["disable_log_stats"] = True
58 - self.engine_args = AsyncEngineArgs(
58 + engine_args = AsyncEngineArgs(
59 model=model,
60 tokenizer=tokenizer,
61 tokenizer_mode=tokenizer_mode,
@@ -76,6 +76,8 @@ class AsyncLLM:
76 **kwargs,
77 )
78 self.request_counter = Counter()
79 + self.llm_engine = AsyncLLMEngine.from_engine_args(
80 + engine_args, usage_context=UsageContext.LLM_CLASS)
81
82 def generate(
83 self,
@@ -88,9 +90,6 @@ class AsyncLLM:
90 multi_modal_data: Optional[MultiModalData] = None,
91 ) -> List[RequestOutput]:
92
91 - llm_engine = AsyncLLMEngine.from_engine_args(
92 - self.engine_args, usage_context=UsageContext.LLM_CLASS)
93 -
93 if prompts is None:
94 raise ValueError("prompts must be provided.")
95 if isinstance(prompts, str):
@@ -111,8 +110,8 @@ class AsyncLLM:
110
111 async def get_output(prompt, sampling_param) -> str:
112 request_id = random_uuid()
114 - results_generator = llm_engine.generate(prompt, sampling_param,
115 - request_id)
113 + results_generator = self.llm_engine.generate(
114 + prompt, sampling_param, request_id)
115 final_output = None
116 async for request_output in results_generator:
117 final_output = request_output
@@ -185,12 +184,25 @@ def create_llm_generator(baseline_or_test, request, common_llm_kwargs,
184 return generator_outer
185
186
187 +def maybe_assert_ngram_worker(llm):
188 + # Verify the proposer worker is ngram if ngram is specified.
189 + if (not isinstance(llm, AsyncLLM)
190 + and llm.llm_engine.speculative_config is not None
191 + and llm.llm_engine.speculative_config.ngram_prompt_lookup_max > 0):
192 + from vllm.spec_decode.ngram_worker import NGramWorker
193 + assert isinstance(
194 + llm.llm_engine.model_executor.driver_worker.proposer_worker,
195 + NGramWorker)
196 +
197 +
198 def get_output_from_llm_generator(
199 llm_generator, prompts,
200 sampling_params) -> Tuple[List[str], List[List[int]]]:
201 tokens = []
202 token_ids = []
203 for llm in llm_generator():
204 + maybe_assert_ngram_worker(llm)
205 +
206 outputs = llm.generate(prompts, sampling_params, use_tqdm=True)
207 token_ids = [output.outputs[0].token_ids for output in outputs]
208 tokens = [output.outputs[0].text for output in outputs]
vllm/executor/gpu_executor.py
+4
@@ -82,6 +82,10 @@ class GPUExecutor(ExecutorBase):
82 draft_worker_kwargs.update(
83 model_config=self.speculative_config.draft_model_config,
84 parallel_config=self.speculative_config.draft_parallel_config,
85 + ngram_prompt_lookup_max=self.speculative_config.
86 + ngram_prompt_lookup_max,
87 + ngram_prompt_lookup_min=self.speculative_config.
88 + ngram_prompt_lookup_min,
89 # TODO allow draft-model specific load config.
90 #load_config=self.load_config,
91 )
vllm/spec_decode/spec_decode_worker.py
+7 -7
@@ -57,13 +57,10 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
57 draft_worker_kwargs,
58 ) -> "SpecDecodeWorker":
59
60 - if "ngram_prompt_lookup_max" in draft_worker_kwargs:
61 - ngram_prompt_lookup_max = (
62 - draft_worker_kwargs.pop("ngram_prompt_lookup_max"))
63 - ngram_prompt_lookup_min = (
64 - draft_worker_kwargs.pop("ngram_prompt_lookup_min"))
65 - else:
66 - ngram_prompt_lookup_max = 0
60 + ngram_prompt_lookup_max = (
61 + draft_worker_kwargs.pop("ngram_prompt_lookup_max"))
62 + ngram_prompt_lookup_min = (
63 + draft_worker_kwargs.pop("ngram_prompt_lookup_min"))
64
65 if ngram_prompt_lookup_max > 0:
66 proposer_worker = NGramWorker(**draft_worker_kwargs)
@@ -72,6 +69,9 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
69 else:
70 proposer_worker = MultiStepWorker(**draft_worker_kwargs)
71
72 + logger.info("Configuring SpecDecodeWorker with proposer=%s",
73 + type(proposer_worker))
74 +
75 return SpecDecodeWorker(
76 proposer_worker,
77 scorer_worker,