@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 19d02ff938
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 15:45 UTC

[Bugfix] Fix PP for Multi-Step (#8887)

Varun Sundar Rabindranath committed Sep 28, 2024 at 11:52 UTC 19d02ff93812fb6a28f0f1a0a0f9233e9388d616
5 files changed +130 -15
tests/multi_step/test_correctness_async_llm.py
+82
@@ -142,3 +142,85 @@ async def test_multi_step(
142 name_0="hf",
143 name_1="vllm",
144 )
145 +
146 +
147 +@pytest.mark.parametrize(("tp_size, pp_size"), [
148 + (1, 2),
149 +])
150 +@pytest.mark.asyncio
151 +async def test_multi_step_pp_smoke(
152 + tp_size: int,
153 + pp_size: int,
154 + monkeypatch,
155 +) -> None:
156 + """
157 + Smoke test for the vLLM engine with multi-step scheduling in an
158 + OpenAI-protocol client/server environment.
159 +
160 + This tests compares the outputs between multi-step scheduling and
161 + single-step scheduling. Notably, this test lets the engines generate
162 + more tokens (default is 5) and test for an exact match over all the
163 + tokens.
164 +
165 + Args:
166 + tp_size: degree of tensor-parallelism
167 + pp_size: degree of pipeline-parallelism
168 + eager_mode
169 + """
170 +
171 + model = "JackFram/llama-160m"
172 + num_scheduler_steps = 8
173 + attention_backend = "FLASH_ATTN"
174 + max_num_seqs = 3
175 +
176 + override_backend_env_variable(monkeypatch, attention_backend)
177 +
178 + # Prompt from the ShareGPT dataset
179 + prompts = [
180 + "in the jtbd context whats a push?", # codespell:ignore
181 + "in the jtbd context whats a push?", # codespell:ignore
182 + "in the jtbd context whats a push?", # codespell:ignore
183 + "in the jtbd context whats a push?", # codespell:ignore
184 + ]
185 + # Use varying max_tokens to introduce scheduling randomness.
186 + max_tokens = [10 * i for i in range(1, len(prompts) + 1)]
187 + assert len(prompts) == len(max_tokens)
188 +
189 + test_args = [
190 + "--tensor-parallel-size",
191 + str(tp_size), "--pipeline-parallel-size",
192 + str(pp_size), "--max-num-seqs",
193 + str(max_num_seqs)
194 + ]
195 +
196 + server_args = DEFAULT_SERVER_ARGS + test_args
197 + ms_server_args = DEFAULT_SERVER_ARGS + \
198 + ["--num-scheduler-steps", f"{num_scheduler_steps}"] + \
199 + test_args
200 +
201 + # Spin up client/server & issue completion API requests.
202 + # Default `max_wait_seconds` is 240 but was empirically
203 + # was raised 3x to 720 *just for this test* due to
204 + # observed timeouts in GHA CI
205 + ref_completions = await completions_with_server_args(
206 + prompts=prompts,
207 + model_name=model,
208 + server_cli_args=server_args,
209 + num_logprobs=None,
210 + max_wait_seconds=5 * 240,
211 + max_tokens=max_tokens)
212 +
213 + test_completions = await completions_with_server_args(
214 + prompts=prompts,
215 + model_name=model,
216 + server_cli_args=ms_server_args,
217 + num_logprobs=None,
218 + max_wait_seconds=5 * 240,
219 + max_tokens=max_tokens)
220 +
221 + # Assert multi-step scheduling produces identical tokens
222 + # to single-step scheduling.
223 + ref_generations = get_client_text_generations(ref_completions)
224 + test_generations = get_client_text_generations(test_completions)
225 +
226 + assert ref_generations == test_generations
tests/utils.py
+26 -12
@@ -1,3 +1,4 @@
1 +import asyncio
2 import functools
3 import os
4 import signal
@@ -7,7 +8,7 @@ import time
8 import warnings
9 from contextlib import contextmanager
10 from pathlib import Path
10 -from typing import Any, Callable, Dict, List, Optional
11 +from typing import Any, Callable, Dict, List, Optional, Union
12
13 import openai
14 import pytest
@@ -476,7 +477,8 @@ async def completions_with_server_args(
477 server_cli_args: List[str],
478 num_logprobs: Optional[int],
479 max_wait_seconds: int = 240,
479 -) -> Completion:
480 + max_tokens: Union[int, list] = 5,
481 +) -> List[Completion]:
482 '''Construct a remote OpenAI server, obtain an async client to the
483 server & invoke the completions API to obtain completions.
484
@@ -487,37 +489,49 @@ async def completions_with_server_args(
489 num_logprobs: Number of logprobs to report (or `None`)
490 max_wait_seconds: timeout interval for bringing up server.
491 Default: 240sec
492 + max_tokens: max_tokens value for each of the given input prompts.
493 + if only one max_token value is given, the same value is used
494 + for all the prompts.
495
496 Returns:
497 OpenAI Completion instance
498 '''
499
500 + if isinstance(max_tokens, int):
501 + max_tokens = [max_tokens] * len(prompts)
502 +
503 + assert len(max_tokens) == len(prompts)
504 +
505 outputs = None
506 max_wait_seconds = 240 * 3 # 240 is default
507 with RemoteOpenAIServer(model_name,
508 server_cli_args,
509 max_wait_seconds=max_wait_seconds) as server:
510 client = server.get_async_client()
501 - outputs = await client.completions.create(model=model_name,
502 - prompt=prompts,
503 - temperature=0,
504 - stream=False,
505 - max_tokens=5,
506 - logprobs=num_logprobs)
511 + outputs = [ client.completions.create(model=model_name,
512 + prompt=[p],
513 + temperature=0,
514 + stream=False,
515 + max_tokens=max_tok,
516 + logprobs=num_logprobs) \
517 + for p, max_tok in zip(prompts, max_tokens) ]
518 + outputs = await asyncio.gather(*outputs)
519 +
520 assert outputs is not None, "Completion API call failed."
521
522 return outputs
523
524
512 -def get_client_text_generations(completions: Completion) -> List[str]:
525 +def get_client_text_generations(completions: List[Completion]) -> List[str]:
526 '''Extract generated tokens from the output of a
527 request made to an Open-AI-protocol completions endpoint.
528 '''
516 - return [x.text for x in completions.choices]
529 + assert all([len(x.choices) == 1 for x in completions])
530 + return [x.choices[0].text for x in completions]
531
532
533 def get_client_text_logprob_generations(
520 - completions: Completion) -> List[TextTextLogprobs]:
534 + completions: List[Completion]) -> List[TextTextLogprobs]:
535 '''Operates on the output of a request made to an Open-AI-protocol
536 completions endpoint; obtains top-rank logprobs for each token in
537 each :class:`SequenceGroup`
@@ -526,4 +540,4 @@ def get_client_text_logprob_generations(
540 text = ''.join(text_generations)
541 return [(text_generations, text,
542 (None if x.logprobs is None else x.logprobs.top_logprobs))
529 - for x in completions.choices]
543 + for completion in completions for x in completion.choices]
vllm/engine/output_processor/multi_step.py
+3
@@ -97,6 +97,9 @@ class MultiStepOutputProcessor(SequenceGroupOutputProcessor):
97 assert len(seqs) == 1, (
98 "Beam search not supported in multi-step decoding.")
99 seq = seqs[0]
100 + seq_id = seq.seq_id
101 + assert all(
102 + [seq_id == output.samples[0].parent_seq_id for output in outputs])
103
104 if is_async:
105 # Async case: We process tokens one by one. Here, we know the token
vllm/worker/model_runner.py
+9 -1
@@ -1007,8 +1007,16 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
1007
1008 # Used to cache python objects
1009 self.inter_data_cache: Dict[int, PyObjectCache] = {}
1010 +
1011 + # Using the PythonizationCache in Pipeline-Parallel clobbers the
1012 + # SequenceGroupToSample object. In Pipeline-Parallel, we have
1013 + # more than 1 Scheduler, resulting in a potential back-to-back
1014 + # prepare_model_inputs() call. This clobbers the cached
1015 + # SequenceGroupToSample objects, as we reset the cache during
1016 + # every prepare_model_inputs() call.
1017 self.sampling_metadata_cache: SamplingMetadataCache = \
1011 - SamplingMetadataCache()
1018 + SamplingMetadataCache() \
1019 + if self.parallel_config.pipeline_parallel_size == 1 else None
1020
1021 def load_model(self) -> None:
1022 logger.info("Starting to load model %s...", self.model_config.model)
vllm/worker/multi_step_model_runner.py
+10 -2
@@ -326,7 +326,14 @@ class MultiStepModelRunner(GPUModelRunnerBase[StatefulModelInput]):
326 self.is_multi_step = self.scheduler_config.is_multi_step
327 self.pinned_sampled_token_ids: Optional[torch.Tensor] = None
328
329 - self.pythonization_cache = PythonizationCache()
329 + # Using the PythonizationCache in Pipeline-Parallel clobbers the
330 + # SequenceOutput and CompletionSequenceGroupOutput object.
331 + # When cache-reset happens at the last step of a multi-step
332 + # execution, there may be other on-going single-step/multi-step
333 + # executions. The current caching implementation does not check
334 + # for this.
335 + self.pythonization_cache = PythonizationCache() \
336 + if self.parallel_config.pipeline_parallel_size == 1 else None
337
338 @functools.cached_property
339 def _copy_stream(self):
@@ -577,7 +584,8 @@ class MultiStepModelRunner(GPUModelRunnerBase[StatefulModelInput]):
584 if model_input.is_last_step:
585 outputs = self._final_process_outputs(
586 model_input, model_input.base_output_proc_callback)
580 - self.pythonization_cache.reset()
587 + if self.pythonization_cache:
588 + self.pythonization_cache.reset()
589 return outputs
590
591 # should be [SamplerOutput]