@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / cbc2ef5529
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 10:45 UTC

[misc] hide best_of from engine (#9261)

Co-authored-by: Brendan Wong <bjwpokemon@gmail.com>

youkaichao committed Oct 10, 2024 at 21:30 UTC cbc2ef55292b2af6ff742095c030e8425124c005
14 files changed +46 -73
tests/entrypoints/openai/test_metrics.py
-4
@@ -70,7 +70,6 @@ EXPECTED_VALUES = {
70 [("_sum", _NUM_REQUESTS * _NUM_GENERATION_TOKENS_PER_REQUEST),
71 ("_count", _NUM_REQUESTS)],
72 "vllm:request_params_n": [("_count", _NUM_REQUESTS)],
73 - "vllm:request_params_best_of": [("_count", _NUM_REQUESTS)],
73 "vllm:prompt_tokens": [("_total",
74 _NUM_REQUESTS * _NUM_PROMPT_TOKENS_PER_REQUEST)],
75 "vllm:generation_tokens":
@@ -151,9 +150,6 @@ EXPECTED_METRICS = [
150 "vllm:request_params_n_sum",
151 "vllm:request_params_n_bucket",
152 "vllm:request_params_n_count",
154 - "vllm:request_params_best_of_sum",
155 - "vllm:request_params_best_of_bucket",
156 - "vllm:request_params_best_of_count",
153 "vllm:num_preemptions_total",
154 "vllm:prompt_tokens_total",
155 "vllm:generation_tokens_total",
tests/metrics/test_metrics.py
-1
@@ -326,7 +326,6 @@ def assert_metrics(engine: LLMEngine, disable_log_stats: bool,
326 "vllm:e2e_request_latency_seconds",
327 "vllm:request_prompt_tokens",
328 "vllm:request_generation_tokens",
329 - "vllm:request_params_best_of",
329 "vllm:request_params_n",
330 ]
331 for metric_name in request_histogram_metrics:
tests/tracing/test_tracing.py
-4
@@ -98,8 +98,6 @@ def test_traces(trace_service):
98 SpanAttributes.LLM_REQUEST_TOP_P) == sampling_params.top_p
99 assert attributes.get(
100 SpanAttributes.LLM_REQUEST_MAX_TOKENS) == sampling_params.max_tokens
101 - assert attributes.get(
102 - SpanAttributes.LLM_REQUEST_BEST_OF) == sampling_params.best_of
101 assert attributes.get(SpanAttributes.LLM_REQUEST_N) == sampling_params.n
102 assert attributes.get(SpanAttributes.LLM_USAGE_PROMPT_TOKENS) == len(
103 outputs[0].prompt_token_ids)
@@ -155,8 +153,6 @@ def test_traces_with_detailed_steps(trace_service):
153 SpanAttributes.LLM_REQUEST_TOP_P) == sampling_params.top_p
154 assert attributes.get(
155 SpanAttributes.LLM_REQUEST_MAX_TOKENS) == sampling_params.max_tokens
158 - assert attributes.get(
159 - SpanAttributes.LLM_REQUEST_BEST_OF) == sampling_params.best_of
156 assert attributes.get(SpanAttributes.LLM_REQUEST_N) == sampling_params.n
157 assert attributes.get(SpanAttributes.LLM_USAGE_PROMPT_TOKENS) == len(
158 outputs[0].prompt_token_ids)
vllm/core/scheduler.py
+1 -1
@@ -1205,7 +1205,7 @@ class Scheduler:
1205 # async_output_proc is allowed only when we have a single sequence
1206 # in the sequence group
1207 no_single_seq = seq_group.sampling_params is None or (
1208 - seq_group.sampling_params.best_of == 1)
1208 + seq_group.sampling_params.n == 1)
1209 return no_single_seq
1210
1211 def schedule(
vllm/engine/llm_engine.py
+2 -9
@@ -767,7 +767,7 @@ class LLMEngine:
767 Details:
768 - Set arrival_time to the current time if it is None.
769 - Set prompt_token_ids to the encoded prompt if it is None.
770 - - Create `best_of` number of :class:`~vllm.Sequence` objects.
770 + - Create `n` number of :class:`~vllm.Sequence` objects.
771 - Create a :class:`~vllm.SequenceGroup` object
772 from the list of :class:`~vllm.Sequence`.
773 - Add the :class:`~vllm.SequenceGroup` object to the scheduler.
@@ -1242,8 +1242,7 @@ class LLMEngine:
1242 if seq_group_metadata.do_sample:
1243 assert len(sequence_group_outputs.samples) == 1, (
1244 "Async output processor expects a single sample"
1245 - " (i.e sampling_params.n == 1 and no "
1246 - "sampling_params.best_of > 1)")
1245 + " (i.e sampling_params.n == 1)")
1246 sample = sequence_group_outputs.samples[0]
1247
1248 assert len(seq_group.seqs) == 1
@@ -1612,7 +1611,6 @@ class LLMEngine:
1611 # Metadata
1612 num_prompt_tokens_requests: List[int] = []
1613 num_generation_tokens_requests: List[int] = []
1615 - best_of_requests: List[int] = []
1614 n_requests: List[int] = []
1615 finished_reason_requests: List[str] = []
1616
@@ -1683,8 +1681,6 @@ class LLMEngine:
1681 for seq in seq_group.get_finished_seqs()
1682 ])
1683 if seq_group.sampling_params is not None:
1686 - best_of_requests.append(
1687 - seq_group.sampling_params.best_of)
1684 n_requests.append(seq_group.sampling_params.n)
1685 finished_reason_requests.extend([
1686 SequenceStatus.get_finished_reason(seq.status)
@@ -1737,7 +1733,6 @@ class LLMEngine:
1733 # Metadata
1734 num_prompt_tokens_requests=num_prompt_tokens_requests,
1735 num_generation_tokens_requests=num_generation_tokens_requests,
1740 - best_of_requests=best_of_requests,
1736 n_requests=n_requests,
1737 finished_reason_requests=finished_reason_requests,
1738 )
@@ -1824,8 +1819,6 @@ class LLMEngine:
1819 seq_group.sampling_params.top_p)
1820 seq_span.set_attribute(SpanAttributes.LLM_REQUEST_MAX_TOKENS,
1821 seq_group.sampling_params.max_tokens)
1827 - seq_span.set_attribute(SpanAttributes.LLM_REQUEST_BEST_OF,
1828 - seq_group.sampling_params.best_of)
1822 seq_span.set_attribute(SpanAttributes.LLM_REQUEST_N,
1823 seq_group.sampling_params.n)
1824 seq_span.set_attribute(SpanAttributes.LLM_USAGE_NUM_SEQUENCES,
vllm/engine/metrics.py
-8
@@ -134,12 +134,6 @@ class Metrics:
134 labelnames=labelnames,
135 buckets=build_1_2_5_buckets(max_model_len),
136 )
137 - self.histogram_best_of_request = self._histogram_cls(
138 - name="vllm:request_params_best_of",
139 - documentation="Histogram of the best_of request parameter.",
140 - labelnames=labelnames,
141 - buckets=[1, 2, 5, 10, 20],
142 - )
137 self.histogram_n_request = self._histogram_cls(
138 name="vllm:request_params_n",
139 documentation="Histogram of the n request parameter.",
@@ -473,8 +467,6 @@ class PrometheusStatLogger(StatLoggerBase):
467 self.metrics.histogram_num_generation_tokens_request,
468 stats.num_generation_tokens_requests)
469 self._log_histogram(self.metrics.histogram_n_request, stats.n_requests)
476 - self._log_histogram(self.metrics.histogram_best_of_request,
477 - stats.best_of_requests)
470
471 def _log_prometheus_interval(self, prompt_throughput: float,
472 generation_throughput: float) -> None:
vllm/engine/metrics_types.py
-1
@@ -49,7 +49,6 @@ class Stats:
49 # Metadata
50 num_prompt_tokens_requests: List[int]
51 num_generation_tokens_requests: List[int]
52 - best_of_requests: List[int]
52 n_requests: List[int]
53 finished_reason_requests: List[str]
54
vllm/engine/output_processor/single_step.py
+1 -1
@@ -112,7 +112,7 @@ class SingleStepOutputProcessor(SequenceGroupOutputProcessor):
112 outputs: SequenceGroupOutput,
113 is_async: bool) -> None:
114 sampling_params = seq_group.sampling_params
115 - if sampling_params.best_of == 1:
115 + if sampling_params.n == 1:
116 # only have one output sample
117 sample = outputs.samples[0]
118 # only have one sequence
vllm/model_executor/layers/sampler.py
+8 -9
@@ -508,7 +508,7 @@ def _random_sample(
508 same as the length of selected_seq_groups. If the corresponding
509 seq_group has do_sample=False, tuple contains ([], [])
510 """
511 - # Find the maximum best_of value of the prompt phase requests.
511 + # Find the maximum n value of the prompt phase requests.
512 random_samples = random_samples.cpu()
513 sample_idx = 0
514 results: SampleResultType = []
@@ -523,9 +523,9 @@ def _random_sample(
523 num_parent_seqs = len(seq_ids)
524 if is_prompt:
525 # Prompt phase.
526 - parent_ids = [0] * sampling_params.best_of
526 + parent_ids = [0] * sampling_params.n
527 next_token_ids = random_samples[
528 - sample_idx, :sampling_params.best_of].tolist()
528 + sample_idx, :sampling_params.n].tolist()
529 else:
530 # Generation phase.
531 parent_ids = list(range(num_parent_seqs))
@@ -570,7 +570,7 @@ def _beam_search_sample(
570 is_prompt = seq_group.is_prompt
571 seq_ids, sampling_params = seq_group.seq_ids, seq_group.sampling_params
572 num_parent_seqs = len(seq_ids)
573 - beam_width = sampling_params.best_of
573 + beam_width = sampling_params.n
574 seq_group_logprobs = logprobs[sample_idx:sample_idx + num_parent_seqs]
575 if is_prompt:
576 # Prompt phase.
@@ -797,12 +797,11 @@ def _sample_with_torch(
797 greedy_samples)
798
799 elif sampling_type in (SamplingType.RANDOM, SamplingType.RANDOM_SEED):
800 - max_best_of_in_batch = 1
800 + max_n_in_batch = 1
801 for seq_group in seq_groups:
802 if seq_group.is_prompt:
803 sampling_params = seq_group.sampling_params
804 - max_best_of_in_batch = max(max_best_of_in_batch,
805 - sampling_params.best_of)
804 + max_n_in_batch = max(max_n_in_batch, sampling_params.n)
805 seq_groups_arg = (None if sampling_type == SamplingType.RANDOM else
806 seq_groups)
807
@@ -812,13 +811,13 @@ def _sample_with_torch(
811 probs[long_sample_indices],
812 sampling_tensors.top_ks[long_sample_indices],
813 sampling_tensors.top_ps[long_sample_indices],
815 - max_best_of_in_batch,
814 + max_n_in_batch,
815 seq_groups_arg,
816 )
817 else:
818 multinomial_samples[sampling_type] = _multinomial(
819 probs[long_sample_indices],
821 - max_best_of_in_batch,
820 + max_n_in_batch,
821 seq_groups=seq_groups_arg)
822
823 if sampled_token_ids_tensor is not None:
vllm/outputs.py
+1 -1
@@ -141,7 +141,7 @@ class RequestOutput:
141 top_n_seqs = seqs
142 else:
143 # Get the top-n sequences.
144 - n = sampling_params.n
144 + n = sampling_params._real_n or sampling_params.n
145 sorting_key = lambda seq: seq.get_cumulative_logprob()
146 sorted_seqs = sorted(seqs, key=sorting_key, reverse=True)
147 top_n_seqs = sorted_seqs[:n]
vllm/sampling_params.py
+17 -16
@@ -106,9 +106,8 @@ class SamplingParams(
106 n: Number of output sequences to return for the given prompt.
107 best_of: Number of output sequences that are generated from the prompt.
108 From these `best_of` sequences, the top `n` sequences are returned.
109 - `best_of` must be greater than or equal to `n`. This is treated as
110 - the beam width when `use_beam_search` is True. By default, `best_of`
111 - is set to `n`.
109 + `best_of` must be greater than or equal to `n`. By default,
110 + `best_of` is set to `n`.
111 presence_penalty: Float that penalizes new tokens based on whether they
112 appear in the generated text so far. Values > 0 encourage the model
113 to use new tokens, while values < 0 encourage the model to repeat
@@ -173,6 +172,7 @@ class SamplingParams(
172
173 n: int = 1
174 best_of: Optional[int] = None
175 + _real_n: Optional[int] = None
176 presence_penalty: float = 0.0
177 frequency_penalty: float = 0.0
178 repetition_penalty: float = 1.0
@@ -282,7 +282,19 @@ class SamplingParams(
282 )
283
284 def __post_init__(self) -> None:
285 - self.best_of = self.best_of or self.n
285 + # how we deal with `best_of``:
286 + # if `best_of`` is not set, we default to `n`;
287 + # if `best_of`` is set, we set `n`` to `best_of`,
288 + # and set `_real_n`` to the original `n`.
289 + # when we return the result, we will check
290 + # if we need to return `n` or `_real_n` results
291 + if self.best_of:
292 + if self.best_of < self.n:
293 + raise ValueError(
294 + f"best_of must be greater than or equal to n, "
295 + f"got n={self.n} and best_of={self.best_of}.")
296 + self._real_n = self.n
297 + self.n = self.best_of
298 if 0 < self.temperature < _MAX_TEMP:
299 logger.warning(
300 "temperature %s is less than %s, which may cause numerical "
@@ -329,12 +341,6 @@ class SamplingParams(
341 f"type {type(self.n)}")
342 if self.n < 1:
343 raise ValueError(f"n must be at least 1, got {self.n}.")
332 - if not isinstance(self.best_of, int):
333 - raise ValueError(f"best_of must be an int, but is of "
334 - f"type {type(self.best_of)}")
335 - if self.best_of < self.n:
336 - raise ValueError(f"best_of must be greater than or equal to n, "
337 - f"got n={self.n} and best_of={self.best_of}.")
344 if not -2.0 <= self.presence_penalty <= 2.0:
345 raise ValueError("presence_penalty must be in [-2, 2], got "
346 f"{self.presence_penalty}.")
@@ -385,7 +391,7 @@ class SamplingParams(
391 raise ValueError(
392 "stop strings are only supported when detokenize is True. "
393 "Set detokenize=True to use stop.")
388 - if self.best_of != self.n and self.output_kind == (
394 + if self.best_of != self._real_n and self.output_kind == (
395 RequestOutputKind.DELTA):
396 raise ValueError("best_of must equal n to use output_kind=DELTA")
397
@@ -393,10 +399,6 @@ class SamplingParams(
399 if self.n > 1:
400 raise ValueError("n must be 1 when using greedy sampling, "
401 f"got {self.n}.")
396 - assert isinstance(self.best_of, int)
397 - if self.best_of > 1:
398 - raise ValueError("best_of must be 1 when using greedy sampling, "
399 - f"got {self.best_of}.")
402
403 def update_from_generation_config(
404 self,
@@ -453,7 +455,6 @@ class SamplingParams(
455 def __repr__(self) -> str:
456 return (
457 f"SamplingParams(n={self.n}, "
456 - f"best_of={self.best_of}, "
458 f"presence_penalty={self.presence_penalty}, "
459 f"frequency_penalty={self.frequency_penalty}, "
460 f"repetition_penalty={self.repetition_penalty}, "
vllm/sequence.py
+5 -5
@@ -803,14 +803,14 @@ class SequenceGroup:
803 """The maximum number of sequences running in parallel in the remaining
804 lifetime of the request."""
805 if self.sampling_params:
806 - best_of = self.sampling_params.best_of
807 - assert isinstance(best_of, int)
808 - if best_of > self.num_seqs():
806 + n = self.sampling_params.n
807 + assert isinstance(n, int)
808 + if n > self.num_seqs():
809 # At prompt stage, the sequence group is not yet filled up
810 # and only have one sequence running. However, in the
811 - # generation stage, we will have `best_of` sequences
811 + # generation stage, we will have `n` sequences
812 # running.
813 - return best_of
813 + return n
814 # At sampling stages, return the number of actual sequences
815 # that are not finished yet.
816 return self.num_unfinished_seqs()
vllm/tracing.py
-1
@@ -96,7 +96,6 @@ class SpanAttributes(BaseSpanAttributes):
96 # The following span attribute names are added here because they are missing
97 # from the Semantic Conventions for LLM.
98 LLM_REQUEST_ID = "gen_ai.request.id"
99 - LLM_REQUEST_BEST_OF = "gen_ai.request.best_of"
99 LLM_REQUEST_N = "gen_ai.request.n"
100 LLM_USAGE_NUM_SEQUENCES = "gen_ai.usage.num_sequences"
101 LLM_LATENCY_TIME_IN_QUEUE = "gen_ai.latency.time_in_queue"
vllm/worker/tpu_model_runner.py
+11 -12
@@ -49,7 +49,7 @@ class ModelInputForTPU(ModelRunnerInputBase):
49 t: torch.Tensor
50 p: torch.Tensor
51 num_samples: int
52 - best_of: List[int]
52 + n: List[int]
53 seq_groups: List[List[int]]
54 is_first_multi_step: bool = True
55 is_last_step: bool = True
@@ -65,7 +65,7 @@ class ModelInputForTPU(ModelRunnerInputBase):
65 "t": self.t,
66 "p": self.p,
67 "num_samples": self.num_samples,
68 - "best_of": self.best_of,
68 + "n": self.n,
69 "seq_groups": self.seq_groups,
70 "is_first_multi_step": self.is_first_multi_step,
71 "is_last_step": self.is_last_step,
@@ -435,7 +435,7 @@ class TPUModelRunner(ModelRunnerBase[ModelInputForTPU]):
435 assert len(seq_group_metadata_list) > 0
436 t = []
437 p = []
438 - best_of = []
438 + n = []
439 for seq_group_metadata in seq_group_metadata_list:
440 sampling_params = seq_group_metadata.sampling_params
441 t.append(sampling_params.temperature)
@@ -448,11 +448,11 @@ class TPUModelRunner(ModelRunnerBase[ModelInputForTPU]):
448 raise NotImplementedError(
449 "Top-k sampling is currently disabled for the TPU backend "
450 "due to performance issues.")
451 - if sampling_params.best_of > _MAX_NUM_SAMPLES:
451 + if sampling_params.n > _MAX_NUM_SAMPLES:
452 raise NotImplementedError(
453 f"Best of > {_MAX_NUM_SAMPLES} is not supported by the TPU "
454 "backend.")
455 - best_of.append(sampling_params.best_of)
455 + n.append(sampling_params.n)
456 if sampling_params.logprobs is not None:
457 raise NotImplementedError(
458 "logprobs is not currently supported by the TPU backend.")
@@ -465,7 +465,7 @@ class TPUModelRunner(ModelRunnerBase[ModelInputForTPU]):
465 num_seqs = len(seq_group_metadata.seq_data)
466 t += [t[-1]] * (num_seqs - 1)
467 p += [p[-1]] * (num_seqs - 1)
468 - best_of += [best_of[-1]] * (num_seqs - 1)
468 + n += [n[-1]] * (num_seqs - 1)
469
470 num_paddings = padded_batch_size - len(t)
471 t += [1.0] * num_paddings
@@ -473,7 +473,7 @@ class TPUModelRunner(ModelRunnerBase[ModelInputForTPU]):
473
474 t = torch.tensor(t, dtype=torch.float32, device="cpu")
475 p = torch.tensor(p, dtype=torch.float32, device="cpu")
476 - return t, p, best_of
476 + return t, p, n
477
478 def prepare_model_input(
479 self,
@@ -493,8 +493,8 @@ class TPUModelRunner(ModelRunnerBase[ModelInputForTPU]):
493 inputs = self._prepare_decode(seq_group_metadata_list)
494 input_tokens, input_positions, attn_metadata, input_lens = inputs
495 padded_batch_size = input_tokens.shape[0]
496 - t, p, best_of = self._prepare_sample(seq_group_metadata_list,
497 - padded_batch_size)
496 + t, p, n = self._prepare_sample(seq_group_metadata_list,
497 + padded_batch_size)
498 num_samples = _MAX_NUM_SAMPLES if is_prompt else 1
499
500 seq_groups = [
@@ -502,8 +502,7 @@ class TPUModelRunner(ModelRunnerBase[ModelInputForTPU]):
502 for metadata in seq_group_metadata_list
503 ]
504 return ModelInputForTPU(input_tokens, input_positions, attn_metadata,
505 - input_lens, t, p, num_samples, best_of,
506 - seq_groups)
505 + input_lens, t, p, num_samples, n, seq_groups)
506
507 def make_model_input_from_broadcasted_tensor_dict(
508 self, tensor_dict: Dict[str, Any]) -> ModelInputForTPU:
@@ -609,7 +608,7 @@ class TPUModelRunner(ModelRunnerBase[ModelInputForTPU]):
608 assert len(seq_ids) == 1
609 seq_id = seq_ids[0]
610 seq_outputs = []
612 - for j in range(model_input.best_of[i]):
611 + for j in range(model_input.n[i]):
612 next_token_id = next_token_ids[i][j]
613 seq_outputs.append(
614 SequenceOutput(seq_id, next_token_id,