A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 10:45 UTC
[misc] hide best_of from engine (#9261)
Co-authored-by: Brendan Wong <bjwpokemon@gmail.com>
youkaichao committed
Oct 10, 2024 at 21:30 UTC
cbc2ef55292b2af6ff742095c030e8425124c005
14 files changed
+46
-73
tests/entrypoints/openai/test_metrics.py
-4
@@ -70,7 +70,6 @@ EXPECTED_VALUES = {
70
[("_sum", _NUM_REQUESTS * _NUM_GENERATION_TOKENS_PER_REQUEST),
71
("_count", _NUM_REQUESTS)],
72
"vllm:request_params_n": [("_count", _NUM_REQUESTS)],
73
- "vllm:request_params_best_of": [("_count", _NUM_REQUESTS)],
73
"vllm:prompt_tokens": [("_total",
74
_NUM_REQUESTS * _NUM_PROMPT_TOKENS_PER_REQUEST)],
75
"vllm:generation_tokens":
@@ -151,9 +150,6 @@ EXPECTED_METRICS = [
150
"vllm:request_params_n_sum",
151
"vllm:request_params_n_bucket",
152
"vllm:request_params_n_count",
154
- "vllm:request_params_best_of_sum",
155
- "vllm:request_params_best_of_bucket",
156
- "vllm:request_params_best_of_count",
153
"vllm:num_preemptions_total",
154
"vllm:prompt_tokens_total",
155
"vllm:generation_tokens_total",
tests/metrics/test_metrics.py
-1
@@ -326,7 +326,6 @@ def assert_metrics(engine: LLMEngine, disable_log_stats: bool,
326
"vllm:e2e_request_latency_seconds",
327
"vllm:request_prompt_tokens",
328
"vllm:request_generation_tokens",
329
- "vllm:request_params_best_of",
329
"vllm:request_params_n",
330
]
331
for metric_name in request_histogram_metrics:
tests/tracing/test_tracing.py
-4
@@ -98,8 +98,6 @@ def test_traces(trace_service):
98
SpanAttributes.LLM_REQUEST_TOP_P) == sampling_params.top_p
99
assert attributes.get(
100
SpanAttributes.LLM_REQUEST_MAX_TOKENS) == sampling_params.max_tokens
101
- assert attributes.get(
102
- SpanAttributes.LLM_REQUEST_BEST_OF) == sampling_params.best_of
101
assert attributes.get(SpanAttributes.LLM_REQUEST_N) == sampling_params.n
102
assert attributes.get(SpanAttributes.LLM_USAGE_PROMPT_TOKENS) == len(
103
outputs[0].prompt_token_ids)
@@ -155,8 +153,6 @@ def test_traces_with_detailed_steps(trace_service):
153
SpanAttributes.LLM_REQUEST_TOP_P) == sampling_params.top_p
154
assert attributes.get(
155
SpanAttributes.LLM_REQUEST_MAX_TOKENS) == sampling_params.max_tokens
158
- assert attributes.get(
159
- SpanAttributes.LLM_REQUEST_BEST_OF) == sampling_params.best_of
156
assert attributes.get(SpanAttributes.LLM_REQUEST_N) == sampling_params.n
157
assert attributes.get(SpanAttributes.LLM_USAGE_PROMPT_TOKENS) == len(
158
outputs[0].prompt_token_ids)
vllm/core/scheduler.py
+1
-1
@@ -1205,7 +1205,7 @@ class Scheduler:
1205
# async_output_proc is allowed only when we have a single sequence
1206
# in the sequence group
1207
no_single_seq = seq_group.sampling_params is None or (
1208
- seq_group.sampling_params.best_of == 1)
1208
+ seq_group.sampling_params.n == 1)
1209
return no_single_seq
1210
1211
def schedule(
vllm/engine/llm_engine.py
+2
-9
@@ -767,7 +767,7 @@ class LLMEngine:
767
Details:
768
- Set arrival_time to the current time if it is None.
769
- Set prompt_token_ids to the encoded prompt if it is None.
770
- - Create `best_of` number of :class:`~vllm.Sequence` objects.
770
+ - Create `n` number of :class:`~vllm.Sequence` objects.
771
- Create a :class:`~vllm.SequenceGroup` object
772
from the list of :class:`~vllm.Sequence`.
773
- Add the :class:`~vllm.SequenceGroup` object to the scheduler.
@@ -1242,8 +1242,7 @@ class LLMEngine:
1242
if seq_group_metadata.do_sample:
1243
assert len(sequence_group_outputs.samples) == 1, (
1244
"Async output processor expects a single sample"
1245
- " (i.e sampling_params.n == 1 and no "
1246
- "sampling_params.best_of > 1)")
1245
+ " (i.e sampling_params.n == 1)")
1246
sample = sequence_group_outputs.samples[0]
1247
1248
assert len(seq_group.seqs) == 1
@@ -1612,7 +1611,6 @@ class LLMEngine:
1611
# Metadata
1612
num_prompt_tokens_requests: List[int] = []
1613
num_generation_tokens_requests: List[int] = []
1615
- best_of_requests: List[int] = []
1614
n_requests: List[int] = []
1615
finished_reason_requests: List[str] = []
1616
@@ -1683,8 +1681,6 @@ class LLMEngine:
1681
for seq in seq_group.get_finished_seqs()
1682
])
1683
if seq_group.sampling_params is not None:
1686
- best_of_requests.append(
1687
- seq_group.sampling_params.best_of)
1684
n_requests.append(seq_group.sampling_params.n)
1685
finished_reason_requests.extend([
1686
SequenceStatus.get_finished_reason(seq.status)
@@ -1737,7 +1733,6 @@ class LLMEngine:
1733
# Metadata
1734
num_prompt_tokens_requests=num_prompt_tokens_requests,
1735
num_generation_tokens_requests=num_generation_tokens_requests,
1740
- best_of_requests=best_of_requests,
1736
n_requests=n_requests,
1737
finished_reason_requests=finished_reason_requests,
1738
)
@@ -1824,8 +1819,6 @@ class LLMEngine:
1819
seq_group.sampling_params.top_p)
1820
seq_span.set_attribute(SpanAttributes.LLM_REQUEST_MAX_TOKENS,
1821
seq_group.sampling_params.max_tokens)
1827
- seq_span.set_attribute(SpanAttributes.LLM_REQUEST_BEST_OF,
1828
- seq_group.sampling_params.best_of)
1822
seq_span.set_attribute(SpanAttributes.LLM_REQUEST_N,
1823
seq_group.sampling_params.n)
1824
seq_span.set_attribute(SpanAttributes.LLM_USAGE_NUM_SEQUENCES,
vllm/engine/metrics.py
-8
@@ -134,12 +134,6 @@ class Metrics:
134
labelnames=labelnames,
135
buckets=build_1_2_5_buckets(max_model_len),
136
)
137
- self.histogram_best_of_request = self._histogram_cls(
138
- name="vllm:request_params_best_of",
139
- documentation="Histogram of the best_of request parameter.",
140
- labelnames=labelnames,
141
- buckets=[1, 2, 5, 10, 20],
142
- )
137
self.histogram_n_request = self._histogram_cls(
138
name="vllm:request_params_n",
139
documentation="Histogram of the n request parameter.",
@@ -473,8 +467,6 @@ class PrometheusStatLogger(StatLoggerBase):
467
self.metrics.histogram_num_generation_tokens_request,
468
stats.num_generation_tokens_requests)
469
self._log_histogram(self.metrics.histogram_n_request, stats.n_requests)
476
- self._log_histogram(self.metrics.histogram_best_of_request,
477
- stats.best_of_requests)
470
471
def _log_prometheus_interval(self, prompt_throughput: float,
472
generation_throughput: float) -> None:
vllm/engine/metrics_types.py
-1
@@ -49,7 +49,6 @@ class Stats:
49
# Metadata
50
num_prompt_tokens_requests: List[int]
51
num_generation_tokens_requests: List[int]
52
- best_of_requests: List[int]
52
n_requests: List[int]
53
finished_reason_requests: List[str]
54
vllm/engine/output_processor/single_step.py
+1
-1
@@ -112,7 +112,7 @@ class SingleStepOutputProcessor(SequenceGroupOutputProcessor):
112
outputs: SequenceGroupOutput,
113
is_async: bool) -> None:
114
sampling_params = seq_group.sampling_params
115
- if sampling_params.best_of == 1:
115
+ if sampling_params.n == 1:
116
# only have one output sample
117
sample = outputs.samples[0]
118
# only have one sequence
vllm/model_executor/layers/sampler.py
+8
-9
@@ -508,7 +508,7 @@ def _random_sample(
508
same as the length of selected_seq_groups. If the corresponding
509
seq_group has do_sample=False, tuple contains ([], [])
510
"""
511
- # Find the maximum best_of value of the prompt phase requests.
511
+ # Find the maximum n value of the prompt phase requests.
512
random_samples = random_samples.cpu()
513
sample_idx = 0
514
results: SampleResultType = []
@@ -523,9 +523,9 @@ def _random_sample(
523
num_parent_seqs = len(seq_ids)
524
if is_prompt:
525
# Prompt phase.
526
- parent_ids = [0] * sampling_params.best_of
526
+ parent_ids = [0] * sampling_params.n
527
next_token_ids = random_samples[
528
- sample_idx, :sampling_params.best_of].tolist()
528
+ sample_idx, :sampling_params.n].tolist()
529
else:
530
# Generation phase.
531
parent_ids = list(range(num_parent_seqs))
@@ -570,7 +570,7 @@ def _beam_search_sample(
570
is_prompt = seq_group.is_prompt
571
seq_ids, sampling_params = seq_group.seq_ids, seq_group.sampling_params
572
num_parent_seqs = len(seq_ids)
573
- beam_width = sampling_params.best_of
573
+ beam_width = sampling_params.n
574
seq_group_logprobs = logprobs[sample_idx:sample_idx + num_parent_seqs]
575
if is_prompt:
576
# Prompt phase.
@@ -797,12 +797,11 @@ def _sample_with_torch(
797
greedy_samples)
798
799
elif sampling_type in (SamplingType.RANDOM, SamplingType.RANDOM_SEED):
800
- max_best_of_in_batch = 1
800
+ max_n_in_batch = 1
801
for seq_group in seq_groups:
802
if seq_group.is_prompt:
803
sampling_params = seq_group.sampling_params
804
- max_best_of_in_batch = max(max_best_of_in_batch,
805
- sampling_params.best_of)
804
+ max_n_in_batch = max(max_n_in_batch, sampling_params.n)
805
seq_groups_arg = (None if sampling_type == SamplingType.RANDOM else
806
seq_groups)
807
@@ -812,13 +811,13 @@ def _sample_with_torch(
811
probs[long_sample_indices],
812
sampling_tensors.top_ks[long_sample_indices],
813
sampling_tensors.top_ps[long_sample_indices],
815
- max_best_of_in_batch,
814
+ max_n_in_batch,
815
seq_groups_arg,
816
)
817
else:
818
multinomial_samples[sampling_type] = _multinomial(
819
probs[long_sample_indices],
821
- max_best_of_in_batch,
820
+ max_n_in_batch,
821
seq_groups=seq_groups_arg)
822
823
if sampled_token_ids_tensor is not None:
vllm/outputs.py
+1
-1
@@ -141,7 +141,7 @@ class RequestOutput:
141
top_n_seqs = seqs
142
else:
143
# Get the top-n sequences.
144
- n = sampling_params.n
144
+ n = sampling_params._real_n or sampling_params.n
145
sorting_key = lambda seq: seq.get_cumulative_logprob()
146
sorted_seqs = sorted(seqs, key=sorting_key, reverse=True)
147
top_n_seqs = sorted_seqs[:n]
vllm/sampling_params.py
+17
-16
@@ -106,9 +106,8 @@ class SamplingParams(
106
n: Number of output sequences to return for the given prompt.
107
best_of: Number of output sequences that are generated from the prompt.
108
From these `best_of` sequences, the top `n` sequences are returned.
109
- `best_of` must be greater than or equal to `n`. This is treated as
110
- the beam width when `use_beam_search` is True. By default, `best_of`
111
- is set to `n`.
109
+ `best_of` must be greater than or equal to `n`. By default,
110
+ `best_of` is set to `n`.
111
presence_penalty: Float that penalizes new tokens based on whether they
112
appear in the generated text so far. Values > 0 encourage the model
113
to use new tokens, while values < 0 encourage the model to repeat
@@ -173,6 +172,7 @@ class SamplingParams(
172
173
n: int = 1
174
best_of: Optional[int] = None
175
+ _real_n: Optional[int] = None
176
presence_penalty: float = 0.0
177
frequency_penalty: float = 0.0
178
repetition_penalty: float = 1.0
@@ -282,7 +282,19 @@ class SamplingParams(
282
)
283
284
def __post_init__(self) -> None:
285
- self.best_of = self.best_of or self.n
285
+ # how we deal with `best_of``:
286
+ # if `best_of`` is not set, we default to `n`;
287
+ # if `best_of`` is set, we set `n`` to `best_of`,
288
+ # and set `_real_n`` to the original `n`.
289
+ # when we return the result, we will check
290
+ # if we need to return `n` or `_real_n` results
291
+ if self.best_of:
292
+ if self.best_of < self.n:
293
+ raise ValueError(
294
+ f"best_of must be greater than or equal to n, "
295
+ f"got n={self.n} and best_of={self.best_of}.")
296
+ self._real_n = self.n
297
+ self.n = self.best_of
298
if 0 < self.temperature < _MAX_TEMP:
299
logger.warning(
300
"temperature %s is less than %s, which may cause numerical "
@@ -329,12 +341,6 @@ class SamplingParams(
341
f"type {type(self.n)}")
342
if self.n < 1:
343
raise ValueError(f"n must be at least 1, got {self.n}.")
332
- if not isinstance(self.best_of, int):
333
- raise ValueError(f"best_of must be an int, but is of "
334
- f"type {type(self.best_of)}")
335
- if self.best_of < self.n:
336
- raise ValueError(f"best_of must be greater than or equal to n, "
337
- f"got n={self.n} and best_of={self.best_of}.")
344
if not -2.0 <= self.presence_penalty <= 2.0:
345
raise ValueError("presence_penalty must be in [-2, 2], got "
346
f"{self.presence_penalty}.")
@@ -385,7 +391,7 @@ class SamplingParams(
391
raise ValueError(
392
"stop strings are only supported when detokenize is True. "
393
"Set detokenize=True to use stop.")
388
- if self.best_of != self.n and self.output_kind == (
394
+ if self.best_of != self._real_n and self.output_kind == (
395
RequestOutputKind.DELTA):
396
raise ValueError("best_of must equal n to use output_kind=DELTA")
397
@@ -393,10 +399,6 @@ class SamplingParams(
399
if self.n > 1:
400
raise ValueError("n must be 1 when using greedy sampling, "
401
f"got {self.n}.")
396
- assert isinstance(self.best_of, int)
397
- if self.best_of > 1:
398
- raise ValueError("best_of must be 1 when using greedy sampling, "
399
- f"got {self.best_of}.")
402
403
def update_from_generation_config(
404
self,
@@ -453,7 +455,6 @@ class SamplingParams(
455
def __repr__(self) -> str:
456
return (
457
f"SamplingParams(n={self.n}, "
456
- f"best_of={self.best_of}, "
458
f"presence_penalty={self.presence_penalty}, "
459
f"frequency_penalty={self.frequency_penalty}, "
460
f"repetition_penalty={self.repetition_penalty}, "
vllm/sequence.py
+5
-5
@@ -803,14 +803,14 @@ class SequenceGroup:
803
"""The maximum number of sequences running in parallel in the remaining
804
lifetime of the request."""
805
if self.sampling_params:
806
- best_of = self.sampling_params.best_of
807
- assert isinstance(best_of, int)
808
- if best_of > self.num_seqs():
806
+ n = self.sampling_params.n
807
+ assert isinstance(n, int)
808
+ if n > self.num_seqs():
809
# At prompt stage, the sequence group is not yet filled up
810
# and only have one sequence running. However, in the
811
- # generation stage, we will have `best_of` sequences
811
+ # generation stage, we will have `n` sequences
812
# running.
813
- return best_of
813
+ return n
814
# At sampling stages, return the number of actual sequences
815
# that are not finished yet.
816
return self.num_unfinished_seqs()
vllm/tracing.py
-1
@@ -96,7 +96,6 @@ class SpanAttributes(BaseSpanAttributes):
96
# The following span attribute names are added here because they are missing
97
# from the Semantic Conventions for LLM.
98
LLM_REQUEST_ID = "gen_ai.request.id"
99
- LLM_REQUEST_BEST_OF = "gen_ai.request.best_of"
99
LLM_REQUEST_N = "gen_ai.request.n"
100
LLM_USAGE_NUM_SEQUENCES = "gen_ai.usage.num_sequences"
101
LLM_LATENCY_TIME_IN_QUEUE = "gen_ai.latency.time_in_queue"
vllm/worker/tpu_model_runner.py
+11
-12
@@ -49,7 +49,7 @@ class ModelInputForTPU(ModelRunnerInputBase):
49
t: torch.Tensor
50
p: torch.Tensor
51
num_samples: int
52
- best_of: List[int]
52
+ n: List[int]
53
seq_groups: List[List[int]]
54
is_first_multi_step: bool = True
55
is_last_step: bool = True
@@ -65,7 +65,7 @@ class ModelInputForTPU(ModelRunnerInputBase):
65
"t": self.t,
66
"p": self.p,
67
"num_samples": self.num_samples,
68
- "best_of": self.best_of,
68
+ "n": self.n,
69
"seq_groups": self.seq_groups,
70
"is_first_multi_step": self.is_first_multi_step,
71
"is_last_step": self.is_last_step,
@@ -435,7 +435,7 @@ class TPUModelRunner(ModelRunnerBase[ModelInputForTPU]):
435
assert len(seq_group_metadata_list) > 0
436
t = []
437
p = []
438
- best_of = []
438
+ n = []
439
for seq_group_metadata in seq_group_metadata_list:
440
sampling_params = seq_group_metadata.sampling_params
441
t.append(sampling_params.temperature)
@@ -448,11 +448,11 @@ class TPUModelRunner(ModelRunnerBase[ModelInputForTPU]):
448
raise NotImplementedError(
449
"Top-k sampling is currently disabled for the TPU backend "
450
"due to performance issues.")
451
- if sampling_params.best_of > _MAX_NUM_SAMPLES:
451
+ if sampling_params.n > _MAX_NUM_SAMPLES:
452
raise NotImplementedError(
453
f"Best of > {_MAX_NUM_SAMPLES} is not supported by the TPU "
454
"backend.")
455
- best_of.append(sampling_params.best_of)
455
+ n.append(sampling_params.n)
456
if sampling_params.logprobs is not None:
457
raise NotImplementedError(
458
"logprobs is not currently supported by the TPU backend.")
@@ -465,7 +465,7 @@ class TPUModelRunner(ModelRunnerBase[ModelInputForTPU]):
465
num_seqs = len(seq_group_metadata.seq_data)
466
t += [t[-1]] * (num_seqs - 1)
467
p += [p[-1]] * (num_seqs - 1)
468
- best_of += [best_of[-1]] * (num_seqs - 1)
468
+ n += [n[-1]] * (num_seqs - 1)
469
470
num_paddings = padded_batch_size - len(t)
471
t += [1.0] * num_paddings
@@ -473,7 +473,7 @@ class TPUModelRunner(ModelRunnerBase[ModelInputForTPU]):
473
474
t = torch.tensor(t, dtype=torch.float32, device="cpu")
475
p = torch.tensor(p, dtype=torch.float32, device="cpu")
476
- return t, p, best_of
476
+ return t, p, n
477
478
def prepare_model_input(
479
self,
@@ -493,8 +493,8 @@ class TPUModelRunner(ModelRunnerBase[ModelInputForTPU]):
493
inputs = self._prepare_decode(seq_group_metadata_list)
494
input_tokens, input_positions, attn_metadata, input_lens = inputs
495
padded_batch_size = input_tokens.shape[0]
496
- t, p, best_of = self._prepare_sample(seq_group_metadata_list,
497
- padded_batch_size)
496
+ t, p, n = self._prepare_sample(seq_group_metadata_list,
497
+ padded_batch_size)
498
num_samples = _MAX_NUM_SAMPLES if is_prompt else 1
499
500
seq_groups = [
@@ -502,8 +502,7 @@ class TPUModelRunner(ModelRunnerBase[ModelInputForTPU]):
502
for metadata in seq_group_metadata_list
503
]
504
return ModelInputForTPU(input_tokens, input_positions, attn_metadata,
505
- input_lens, t, p, num_samples, best_of,
506
- seq_groups)
505
+ input_lens, t, p, num_samples, n, seq_groups)
506
507
def make_model_input_from_broadcasted_tensor_dict(
508
self, tensor_dict: Dict[str, Any]) -> ModelInputForTPU:
@@ -609,7 +608,7 @@ class TPUModelRunner(ModelRunnerBase[ModelInputForTPU]):
608
assert len(seq_ids) == 1
609
seq_id = seq_ids[0]
610
seq_outputs = []
612
- for j in range(model_input.best_of[i]):
611
+ for j in range(model_input.n[i]):
612
next_token_id = next_token_ids[i][j]
613
seq_outputs.append(
614
SequenceOutput(seq_id, next_token_id,