A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 06:45 UTC
Implement prompt logprobs & Batched topk for computing logprobs (#1328)
Co-authored-by: Yunmo Chen <16273544+wanmok@users.noreply.github.com>
Zhuohan Li committed
Oct 16, 2023 at 10:56 UTC
9d9072a069202e7892a40ef94e9085019e73f370
14 files changed
+371
-132
examples/llm_engine_example.py
+1
-1
@@ -11,7 +11,7 @@ def main(args: argparse.Namespace):
11
# Test the following prompts.
12
test_prompts = [
13
("A robot may not injure a human being",
14
- SamplingParams(temperature=0.0)),
14
+ SamplingParams(temperature=0.0, logprobs=1, prompt_logprobs=1)),
15
("To be or not to be,",
16
SamplingParams(temperature=0.8, top_k=5, presence_penalty=0.2)),
17
("What is the meaning of life?",
tests/async_engine/test_request_tracker.py
+1
-1
@@ -64,7 +64,7 @@ def test_request_tracker():
64
stream_5 = tracker.add_request("5")
65
assert tracker.new_requests_event.flag
66
tracker.process_request_output(
67
- RequestOutput("2", "output", [], [], finished=True))
67
+ RequestOutput("2", "output", [], [], [], finished=True))
68
new, finished = tracker.get_new_and_finished_requests()
69
assert not tracker.new_requests_event.flag
70
assert len(finished) == 1
tests/conftest.py
+33
@@ -107,6 +107,39 @@ class HfRunner:
107
outputs[i] = (output_ids, output_str)
108
return outputs
109
110
+ def generate_greedy_logprobs(
111
+ self,
112
+ prompts: List[str],
113
+ max_tokens: int,
114
+ ) -> List[List[torch.Tensor]]:
115
+ all_logprobs = []
116
+ for prompt in prompts:
117
+ input_ids = self.tokenizer(prompt, return_tensors="pt").input_ids
118
+ output = self.model.generate(
119
+ input_ids.cuda(),
120
+ use_cache=True,
121
+ do_sample=False,
122
+ max_new_tokens=max_tokens,
123
+ output_hidden_states=True,
124
+ return_dict_in_generate=True,
125
+ )
126
+ seq_logprobs = []
127
+ for hidden_states in output.hidden_states:
128
+ last_hidden_states = hidden_states[-1][0]
129
+ logits = torch.matmul(
130
+ last_hidden_states,
131
+ self.model.get_output_embeddings().weight.t(),
132
+ )
133
+ if self.model.get_output_embeddings().bias is not None:
134
+ logits += self.model.get_output_embeddings(
135
+ ).bias.unsqueeze(0)
136
+ logprobs = torch.nn.functional.log_softmax(logits,
137
+ dim=-1,
138
+ dtype=torch.float32)
139
+ seq_logprobs.append(logprobs)
140
+ all_logprobs.append(seq_logprobs)
141
+ return all_logprobs
142
+
143
144
@pytest.fixture
145
def hf_runner():
tests/samplers/test_logprobs.py
new
+55
@@ -0,0 +1,55 @@
1
+import pytest
2
+import torch
3
+
4
+from vllm import SamplingParams
5
+
6
+MODELS = ["facebook/opt-125m"]
7
+
8
+
9
+@pytest.mark.parametrize("model", MODELS)
10
+@pytest.mark.parametrize("dtype", ["half"])
11
+def test_get_prompt_logprobs(
12
+ hf_runner,
13
+ vllm_runner,
14
+ model,
15
+ dtype,
16
+ example_prompts,
17
+):
18
+ max_tokens = 5
19
+ hf_model = hf_runner(model, dtype=dtype)
20
+ hf_logprobs = hf_model.generate_greedy_logprobs(
21
+ example_prompts,
22
+ max_tokens=max_tokens,
23
+ )
24
+ del hf_model
25
+
26
+ vllm_model = vllm_runner(model, dtype=dtype)
27
+ vllm_sampling_params = SamplingParams(max_tokens=max_tokens,
28
+ logprobs=5,
29
+ prompt_logprobs=5,
30
+ temperature=0.0)
31
+ vllm_results = vllm_model.model.generate(
32
+ example_prompts, sampling_params=vllm_sampling_params)
33
+
34
+ # Test whether logprobs are included in the results.
35
+ for result in vllm_results:
36
+ assert result.prompt_logprobs is not None
37
+ assert result.outputs[0].logprobs is not None
38
+
39
+ # Test whether prompt logprobs are consistent with HF
40
+ for vllm_result, hf_logprob in zip(vllm_results, hf_logprobs):
41
+ # Check prompt logprobs
42
+ vllm_prompt_logprobs = vllm_result.prompt_logprobs[1:]
43
+ for i, vllm_prompt_logprob_dict in enumerate(vllm_prompt_logprobs):
44
+ for token_id, logprob in vllm_prompt_logprob_dict.items():
45
+ torch.testing.assert_close(logprob,
46
+ hf_logprob[0][i][token_id].item(),
47
+ atol=1e-2,
48
+ rtol=1e-2)
49
+ vllm_sample_logprobs = vllm_result.outputs[0].logprobs
50
+ for i, vllm_sample_logprob_dict in enumerate(vllm_sample_logprobs):
51
+ for token_id, logprob in vllm_sample_logprob_dict.items():
52
+ torch.testing.assert_close(logprob,
53
+ hf_logprob[i][-1][token_id].item(),
54
+ atol=1e-2,
55
+ rtol=1e-2)
vllm/config.py
+1
-1
@@ -143,7 +143,7 @@ class ModelConfig:
143
def get_num_kv_heads(self, parallel_config: "ParallelConfig") -> int:
144
"""Returns the number of KV heads per GPU worker."""
145
# For GPTBigCode & Falcon:
146
- # Note: for falcon, when new_decoder_architecture is True, the
146
+ # NOTE: for falcon, when new_decoder_architecture is True, the
147
# multi_query flag is ignored and we use n_head_kv for the number of
148
# KV heads.
149
falcon_model_types = ["falcon", "RefinedWeb", "RefinedWebModel"]
vllm/engine/llm_engine.py
+13
-7
@@ -12,8 +12,8 @@ from vllm.logger import init_logger
12
from vllm.outputs import RequestOutput
13
from vllm.sampling_params import SamplingParams
14
from vllm.sequence import (SamplerOutput, Sequence, SequenceGroup,
15
- SequenceGroupMetadata, SequenceOutputs,
16
- SequenceStatus)
15
+ SequenceGroupMetadata, SequenceGroupOutputs,
16
+ SequenceOutputs, SequenceStatus)
17
from vllm.transformers_utils.tokenizer import (detokenize_incrementally,
18
get_tokenizer)
19
from vllm.utils import Counter
@@ -350,9 +350,15 @@ class LLMEngine:
350
eos_token_id=self.tokenizer.eos_token_id))
351
return current_worst_score >= highest_attainable_score
352
353
- def _process_sequence_group_samples(
354
- self, seq_group: SequenceGroup,
355
- samples: List[SequenceOutputs]) -> None:
353
+ def _process_sequence_group_outputs(self, seq_group: SequenceGroup,
354
+ outputs: SequenceGroupOutputs) -> None:
355
+ # Process prompt logprobs
356
+ prompt_logprobs = outputs.prompt_logprobs
357
+ if prompt_logprobs is not None:
358
+ seq_group.prompt_logprobs = prompt_logprobs
359
+
360
+ # Process samples
361
+ samples = outputs.samples
362
parent_seqs = seq_group.get_seqs(status=SequenceStatus.RUNNING)
363
existing_finished_seqs = seq_group.get_finished_seqs()
364
parent_child_dict = {
@@ -520,8 +526,8 @@ class LLMEngine:
526
scheduler_outputs: SchedulerOutputs) -> List[RequestOutput]:
527
# Update the scheduled sequence groups with the model outputs.
528
scheduled_seq_groups = scheduler_outputs.scheduled_seq_groups
523
- for seq_group, samples in zip(scheduled_seq_groups, output):
524
- self._process_sequence_group_samples(seq_group, samples)
529
+ for seq_group, outputs in zip(scheduled_seq_groups, output):
530
+ self._process_sequence_group_outputs(seq_group, outputs)
531
532
# Free the finished sequence groups.
533
self.scheduler.free_finished_seq_groups()
vllm/model_executor/layers/attention.py
+1
-1
@@ -420,7 +420,7 @@ class PagedAttentionWithALiBi(PagedAttention):
420
# Generates ALiBi mask for each prompt.
421
for prompt_len in input_metadata.prompt_lens:
422
bias = torch.arange(prompt_len, dtype=dtype)
423
- # Note(zhuohan): HF uses
423
+ # NOTE(zhuohan): HF uses
424
# `bias = bias[None, :].repeat(prompt_len, 1)`
425
# here. We find that both biases give the same results, but
426
# the bias below more accurately follows the original ALiBi
vllm/model_executor/layers/sampler.py
+203
-107
@@ -8,7 +8,8 @@ from vllm.model_executor.input_metadata import InputMetadata
8
from vllm.model_executor.parallel_utils.communication_op import (
9
tensor_model_parallel_all_gather)
10
from vllm.sampling_params import SamplingParams, SamplingType
11
-from vllm.sequence import SamplerOutput, SequenceData, SequenceOutputs
11
+from vllm.sequence import (PromptLogprobs, SampleLogprobs, SamplerOutput,
12
+ SequenceData, SequenceGroupOutputs, SequenceOutputs)
13
14
_SAMPLING_EPS = 1e-5
15
@@ -82,7 +83,12 @@ class Sampler(nn.Module):
83
logprobs = torch.log_softmax(logits, dim=-1, dtype=torch.float)
84
85
# Sample the next tokens.
85
- return _sample(probs, logprobs, input_metadata)
86
+ sample_results = _sample(probs, logprobs, input_metadata)
87
+ # Get the logprobs query results.
88
+ prompt_logprobs, sample_logprobs = _get_logprobs(
89
+ logprobs, input_metadata, sample_results)
90
+ return _build_sampler_output(sample_results, input_metadata,
91
+ prompt_logprobs, sample_logprobs)
92
93
94
def _get_logits(hidden_states: torch.Tensor, embedding: torch.Tensor,
@@ -102,24 +108,28 @@ def _prune_hidden_states(
108
hidden_states: torch.Tensor,
109
input_metadata: InputMetadata,
110
) -> torch.Tensor:
105
- last_token_indices = []
111
+ selected_token_indices: List[int] = []
112
start_idx = 0
113
for i, seq_group in enumerate(input_metadata.seq_groups):
108
- seq_ids, _ = seq_group
114
+ seq_ids, sampling_params = seq_group
115
if i < input_metadata.num_prompts:
116
assert len(seq_ids) == 1, "Prompt input should have only one seq."
117
prompt_len = input_metadata.prompt_lens[i]
112
- last_token_indices.append(start_idx + prompt_len - 1)
118
+ if sampling_params.prompt_logprobs is not None:
119
+ selected_token_indices.extend(
120
+ range(start_idx, start_idx + prompt_len - 1))
121
+ selected_token_indices.append(start_idx + prompt_len - 1)
122
start_idx += prompt_len
123
else:
124
num_seqs = len(seq_ids)
116
- last_token_indices.extend(range(start_idx, start_idx + num_seqs))
125
+ selected_token_indices.extend(
126
+ range(start_idx, start_idx + num_seqs))
127
start_idx += num_seqs
128
119
- last_token_indices = torch.tensor(last_token_indices,
120
- dtype=torch.long,
121
- device=hidden_states.device)
122
- return hidden_states.index_select(0, last_token_indices)
129
+ selected_token_indices = torch.tensor(selected_token_indices,
130
+ dtype=torch.long,
131
+ device=hidden_states.device)
132
+ return hidden_states.index_select(0, selected_token_indices)
133
134
135
def _get_penalties(
@@ -127,10 +137,17 @@ def _get_penalties(
137
# Collect the presence and frequency penalties.
138
presence_penalties: List[float] = []
139
frequency_penalties: List[float] = []
130
- for seq_group in input_metadata.seq_groups:
140
+ for i, seq_group in enumerate(input_metadata.seq_groups):
141
seq_ids, sampling_params = seq_group
142
p = sampling_params.presence_penalty
143
f = sampling_params.frequency_penalty
144
+ if (i < input_metadata.num_prompts
145
+ and sampling_params.prompt_logprobs is not None):
146
+ # NOTE: We do not apply presence and frequency penalties for the
147
+ # prompt token positions where we don't sample new tokens.
148
+ prompt_len = input_metadata.prompt_lens[i]
149
+ presence_penalties += [0] * (prompt_len - 1)
150
+ frequency_penalties += [0] * (prompt_len - 1)
151
presence_penalties += [p] * len(seq_ids)
152
frequency_penalties += [f] * len(seq_ids)
153
return presence_penalties, frequency_penalties
@@ -138,8 +155,14 @@ def _get_penalties(
155
156
def _get_output_tokens(input_metadata: InputMetadata) -> List[List[int]]:
157
output_tokens: List[List[int]] = []
141
- for seq_group in input_metadata.seq_groups:
142
- seq_ids, _ = seq_group
158
+ for i, seq_group in enumerate(input_metadata.seq_groups):
159
+ seq_ids, sampling_params = seq_group
160
+ if (i < input_metadata.num_prompts
161
+ and sampling_params.prompt_logprobs is not None):
162
+ # NOTE: prompt token positions do not need output tokens to
163
+ # compute penalties.
164
+ prompt_len = input_metadata.prompt_lens[i]
165
+ output_tokens.extend([] for _ in range(prompt_len - 1))
166
for seq_id in seq_ids:
167
seq_data = input_metadata.seq_data[seq_id]
168
output_tokens.append(seq_data.output_token_ids)
@@ -200,7 +223,7 @@ def _apply_penalties(
223
def _get_temperatures(input_metadata: InputMetadata) -> List[float]:
224
# Collect the temperatures for the logits.
225
temperatures: List[float] = []
203
- for seq_group in input_metadata.seq_groups:
226
+ for i, seq_group in enumerate(input_metadata.seq_groups):
227
seq_ids, sampling_params = seq_group
228
temperature = sampling_params.temperature
229
if temperature < _SAMPLING_EPS:
@@ -208,6 +231,10 @@ def _get_temperatures(input_metadata: InputMetadata) -> List[float]:
231
# (i.e., greedy sampling or beam search).
232
# Set the temperature to 1 to avoid division by zero.
233
temperature = 1.0
234
+ if (i < input_metadata.num_prompts
235
+ and sampling_params.prompt_logprobs is not None):
236
+ prompt_len = input_metadata.prompt_lens[i]
237
+ temperatures += [temperature] * (prompt_len - 1)
238
temperatures += [temperature] * len(seq_ids)
239
return temperatures
240
@@ -218,13 +245,18 @@ def _get_top_p_top_k(
245
) -> Tuple[List[float], List[int]]:
246
top_ps: List[float] = []
247
top_ks: List[int] = []
221
- for seq_group in input_metadata.seq_groups:
248
+ for i, seq_group in enumerate(input_metadata.seq_groups):
249
seq_ids, sampling_params = seq_group
250
top_p = sampling_params.top_p
251
# k should not be greater than the vocab size.
252
top_k = min(sampling_params.top_k, vocab_size)
253
# k=-1 means no truncation.
254
top_k = vocab_size if top_k == -1 else top_k
255
+ if (i < input_metadata.num_prompts
256
+ and sampling_params.prompt_logprobs is not None):
257
+ prompt_len = input_metadata.prompt_lens[i]
258
+ top_ps += [top_p] * (prompt_len - 1)
259
+ top_ks += [top_k] * (prompt_len - 1)
260
top_ps += [top_p] * len(seq_ids)
261
top_ks += [top_k] * len(seq_ids)
262
return top_ps, top_ks
@@ -259,49 +291,6 @@ def _apply_top_p_top_k(
291
return logits
292
293
262
-def _get_topk_logprobs(
263
- logprobs: torch.Tensor,
264
- num_logprobs: Optional[int],
265
-) -> List[Dict[int, float]]:
266
- num_seqs = logprobs.size(0)
267
- if num_logprobs is None or num_logprobs == 0:
268
- return [{} for _ in range(num_seqs)]
269
-
270
- all_topk_logprobs, all_topk_ids = torch.topk(logprobs,
271
- num_logprobs,
272
- dim=-1)
273
- all_topk_logprobs = all_topk_logprobs.cpu()
274
- all_topk_ids = all_topk_ids.cpu()
275
- all_token_to_logprob = []
276
- for topk_logprobs, topk_ids in zip(all_topk_logprobs, all_topk_ids):
277
- token_to_logprob: Dict[int, float] = {}
278
- for token_id, logprob in zip(topk_ids, topk_logprobs):
279
- token_to_logprob[token_id.item()] = logprob.item()
280
- all_token_to_logprob.append(token_to_logprob)
281
- return all_token_to_logprob
282
-
283
-
284
-def _build_sequence_outputs(
285
- parent_ids: List[int],
286
- next_token_ids: List[int],
287
- selected_token_logprobs: List[float],
288
- parent_seq_ids: List[int],
289
- parent_logprobs: torch.Tensor,
290
- num_output_logprobs: Optional[int],
291
-) -> List[SequenceOutputs]:
292
- # Get top-k log probabilities for the next tokens.
293
- next_logprobs = _get_topk_logprobs(parent_logprobs, num_output_logprobs)
294
- seq_outputs: List[SequenceOutputs] = []
295
- for parent_id, next_token_id, token_logprob in zip(
296
- parent_ids, next_token_ids, selected_token_logprobs):
297
- output_logprobs = next_logprobs[parent_id].copy()
298
- output_logprobs[next_token_id] = token_logprob
299
- seq_outputs.append(
300
- SequenceOutputs(parent_seq_ids[parent_id], next_token_id,
301
- output_logprobs))
302
- return seq_outputs
303
-
304
-
294
def _greedy_sample(
295
selected_seq_groups: List[Tuple[List[int], SamplingParams]],
296
logprobs: torch.Tensor,
@@ -372,7 +361,7 @@ def _beam_search_sample(
361
# for details. See also HF reference:
362
# https://github.com/huggingface/transformers/blob/a4dd53d88e4852f023332d284ff07a01afcd5681/src/transformers/generation/utils.py#L3063-L3065
363
#
375
- # Note: Beam search is not vectorized, so its speed can be slower than
364
+ # NOTE: Beam search is not vectorized, so its speed can be slower than
365
# other sampling methods.
366
sample_idx = 0
367
results = []
@@ -416,79 +405,186 @@ def _sample(
405
probs: torch.Tensor,
406
logprobs: torch.Tensor,
407
input_metadata: InputMetadata,
419
-) -> SamplerOutput:
408
+) -> List[Tuple[List[int], List[int]]]:
409
categorized_seq_group_ids = {t: [] for t in SamplingType}
410
+ categorized_sample_indices = {t: [] for t in SamplingType}
411
start_idx = 0
422
- categorized_seq_ids = {t: [] for t in SamplingType}
412
for i, seq_group in enumerate(input_metadata.seq_groups):
413
seq_ids, sampling_params = seq_group
414
sampling_type = sampling_params.sampling_type
415
+ if (i < input_metadata.num_prompts
416
+ and sampling_params.prompt_logprobs is not None):
417
+ # NOTE: prompt token positions do not need sample, skip
418
+ prompt_len = input_metadata.prompt_lens[i]
419
+ start_idx += prompt_len - 1
420
categorized_seq_group_ids[sampling_type].append(i)
421
num_seqs = len(seq_ids)
428
- categorized_seq_ids[sampling_type].extend(
422
+ categorized_sample_indices[sampling_type].extend(
423
range(start_idx, start_idx + num_seqs))
424
start_idx += num_seqs
431
- seq_outputs_dict: Dict[int, List[SequenceOutputs]] = {}
425
+
426
+ sample_results_dict: Dict[int, Tuple[List[int], List[int]]] = {}
427
for sampling_type in SamplingType:
428
seq_group_ids = categorized_seq_group_ids[sampling_type]
429
seq_groups = [input_metadata.seq_groups[i] for i in seq_group_ids]
430
is_prompts = [i < input_metadata.num_prompts for i in seq_group_ids]
436
- num_tokens = len(categorized_seq_ids[sampling_type])
431
+ sample_indices = categorized_sample_indices[sampling_type]
432
+ num_tokens = len(sample_indices)
433
if num_tokens == 0:
434
continue
439
- category_logprobs = logprobs[categorized_seq_ids[sampling_type]]
440
- category_probs = probs[categorized_seq_ids[sampling_type]]
435
if sampling_type == SamplingType.GREEDY:
436
+ category_logprobs = logprobs[sample_indices]
437
sample_results = _greedy_sample(seq_groups, category_logprobs)
438
elif sampling_type == SamplingType.RANDOM:
439
+ category_probs = probs[sample_indices]
440
sample_results = _random_sample(seq_groups, is_prompts,
441
category_probs)
442
elif sampling_type == SamplingType.BEAM:
443
+ category_logprobs = logprobs[sample_indices]
444
sample_results = _beam_search_sample(seq_groups, is_prompts,
445
input_metadata.seq_data,
446
category_logprobs)
447
else:
448
raise ValueError(f"Unsupported sampling type: {sampling_type}")
449
+ sample_results_dict.update(zip(seq_group_ids, sample_results))
450
453
- # Batched query for logprobs of selected token
454
- batched_logprobs_query_seq_indices: List[int] = []
455
- batched_logprobs_query_token_indices: List[int] = []
456
- sample_idx = 0
457
- for seq_group_id, seq_group, sample_result in zip(
458
- seq_group_ids, seq_groups, sample_results):
459
- seq_ids, sampling_params = seq_group
460
- next_token_ids, parent_ids = sample_result
461
- num_parent_seqs = len(seq_ids)
451
+ sample_results = [
452
+ sample_results_dict[i] for i in range(len(input_metadata.seq_groups))
453
+ ]
454
+ return sample_results
455
+
456
+
457
+def _get_logprobs(
458
+ logprobs: torch.Tensor,
459
+ input_metadata: InputMetadata,
460
+ sample_results: List[Tuple[List[int], List[int]]],
461
+) -> Tuple[List[Optional[List[Optional[Dict[int, float]]]]], List[List[Dict[
462
+ int, float]]]]:
463
+ # Prepare query indices
464
+ batched_logprobs_query_seq_indices: List[int] = []
465
+ batched_logprobs_query_token_indices: List[int] = []
466
+ largest_num_logprobs = 0
467
+ sample_idx = 0
468
+ for i, (seq_group, sample_result) in enumerate(
469
+ zip(input_metadata.seq_groups, sample_results)):
470
+ seq_ids, sampling_params = seq_group
471
+ next_token_ids, parent_ids = sample_result
472
+ num_parent_seqs = len(seq_ids)
473
+ if (i < input_metadata.num_prompts
474
+ and sampling_params.prompt_logprobs is not None):
475
+ largest_num_logprobs = max(largest_num_logprobs,
476
+ sampling_params.prompt_logprobs)
477
+ prompt_len = input_metadata.prompt_lens[i]
478
+ prompt_tokens = input_metadata.seq_data[
479
+ seq_ids[0]].prompt_token_ids
480
batched_logprobs_query_seq_indices.extend(
463
- [sample_idx + parent_id for parent_id in parent_ids])
464
- batched_logprobs_query_token_indices.extend(next_token_ids)
465
- sample_idx += num_parent_seqs
466
- assert sample_idx == num_tokens
467
- batched_logprobs_query_result = category_logprobs[[
468
- batched_logprobs_query_seq_indices,
469
- batched_logprobs_query_token_indices
470
- ]].tolist()
471
-
472
- # Build the sequence outputs.
473
- sample_idx = 0
474
- result_idx = 0
475
- for seq_group_id, seq_group, sample_result in zip(
476
- seq_group_ids, seq_groups, sample_results):
477
- seq_ids, sampling_params = seq_group
478
- next_token_ids, parent_ids = sample_result
479
- num_results = len(next_token_ids)
480
- num_parent_seqs = len(seq_ids)
481
- parent_logprobs = category_logprobs[sample_idx:sample_idx +
482
- num_parent_seqs]
483
- selected_token_logprobs = batched_logprobs_query_result[
484
- result_idx:result_idx + num_results]
485
- seq_output = _build_sequence_outputs(parent_ids, next_token_ids,
486
- selected_token_logprobs,
487
- seq_ids, parent_logprobs,
488
- sampling_params.logprobs)
489
- seq_outputs_dict[seq_group_id] = seq_output
490
- sample_idx += num_parent_seqs
491
- result_idx += num_results
492
- assert sample_idx == num_tokens
493
-
494
- return [seq_outputs_dict[i] for i in range(len(input_metadata.seq_groups))]
481
+ sample_idx + j for j in range(prompt_len - 1))
482
+ batched_logprobs_query_token_indices.extend(
483
+ token_id for token_id in prompt_tokens[1:])
484
+ sample_idx += prompt_len - 1
485
+ batched_logprobs_query_seq_indices.extend(
486
+ [sample_idx + parent_id for parent_id in parent_ids])
487
+ batched_logprobs_query_token_indices.extend(next_token_ids)
488
+ if sampling_params.logprobs is not None:
489
+ largest_num_logprobs = max(largest_num_logprobs,
490
+ sampling_params.logprobs)
491
+ sample_idx += num_parent_seqs
492
+ assert sample_idx == logprobs.size(0)
493
+
494
+ # Batched query for logprobs of selected token
495
+ batched_logprobs_query_result = logprobs[[
496
+ batched_logprobs_query_seq_indices,
497
+ batched_logprobs_query_token_indices
498
+ ]].cpu()
499
+
500
+ # Batched query for logprobs of topk tokens
501
+ if largest_num_logprobs > 0:
502
+ top_logprobs, top_token_ids = torch.topk(logprobs,
503
+ largest_num_logprobs,
504
+ dim=-1)
505
+ top_logprobs = top_logprobs.cpu()
506
+ top_token_ids = top_token_ids.cpu()
507
+ else:
508
+ top_logprobs, top_token_ids = None, None
509
+
510
+ # Gather results
511
+ result_prompt_logprobs: List[Optional[PromptLogprobs]] = []
512
+ result_sample_logprobs: List[SampleLogprobs] = []
513
+ sample_idx = 0
514
+ query_result_idx = 0
515
+ for i, (seq_group, sample_result) in enumerate(
516
+ zip(input_metadata.seq_groups, sample_results)):
517
+ seq_ids, sampling_params = seq_group
518
+ next_token_ids, parent_ids = sample_result
519
+
520
+ # Prompt logprobs
521
+ if (i < input_metadata.num_prompts
522
+ and sampling_params.prompt_logprobs is not None):
523
+ num_logprobs = sampling_params.prompt_logprobs
524
+ prompt_len = input_metadata.prompt_lens[i]
525
+ prompt_tokens = input_metadata.seq_data[
526
+ seq_ids[0]].prompt_token_ids
527
+ group_prompt_logprobs: PromptLogprobs = [None]
528
+ for token_id in prompt_tokens[1:]:
529
+ prompt_logprobs_dict = {
530
+ token_id:
531
+ batched_logprobs_query_result[query_result_idx].item()
532
+ }
533
+ if num_logprobs > 0:
534
+ prompt_logprobs_dict.update(
535
+ zip(top_token_ids[sample_idx, :num_logprobs].tolist(),
536
+ top_logprobs[sample_idx, :num_logprobs].tolist()))
537
+ group_prompt_logprobs.append(prompt_logprobs_dict)
538
+ sample_idx += 1
539
+ query_result_idx += 1
540
+ result_prompt_logprobs.append(group_prompt_logprobs)
541
+ else:
542
+ result_prompt_logprobs.append(None)
543
+
544
+ # Sample logprobs
545
+ num_logprobs = sampling_params.logprobs
546
+ if num_logprobs is None:
547
+ num_logprobs = 0
548
+ group_sample_logprobs: SampleLogprobs = []
549
+ for next_token_id, parent_id in zip(next_token_ids, parent_ids):
550
+ sample_logprobs_dict = {
551
+ next_token_id:
552
+ batched_logprobs_query_result[query_result_idx].item()
553
+ }
554
+ query_result_idx += 1
555
+ if num_logprobs > 0:
556
+ sample_logprobs_dict.update(
557
+ zip(
558
+ top_token_ids[sample_idx +
559
+ parent_id, :num_logprobs].tolist(),
560
+ top_logprobs[sample_idx +
561
+ parent_id, :num_logprobs].tolist()))
562
+ group_sample_logprobs.append(sample_logprobs_dict)
563
+ result_sample_logprobs.append(group_sample_logprobs)
564
+ sample_idx += len(seq_ids)
565
+
566
+ return result_prompt_logprobs, result_sample_logprobs
567
+
568
+
569
+def _build_sampler_output(
570
+ sample_results: List[Tuple[List[int], List[int]]],
571
+ input_metadata: InputMetadata,
572
+ prompt_logprobs: List[Optional[PromptLogprobs]],
573
+ sample_logprobs: List[SampleLogprobs],
574
+) -> SamplerOutput:
575
+ sampler_output = []
576
+ for (seq_group, sample_result, group_prompt_logprobs,
577
+ group_sample_logprobs) in zip(input_metadata.seq_groups,
578
+ sample_results, prompt_logprobs,
579
+ sample_logprobs):
580
+ seq_ids, _ = seq_group
581
+ next_token_ids, parent_ids = sample_result
582
+ seq_outputs = []
583
+ for parent_id, next_token_id, logprobs in zip(parent_ids,
584
+ next_token_ids,
585
+ group_sample_logprobs):
586
+ seq_outputs.append(
587
+ SequenceOutputs(seq_ids[parent_id], next_token_id, logprobs))
588
+ sampler_output.append(
589
+ SequenceGroupOutputs(seq_outputs, group_prompt_logprobs))
590
+ return sampler_output
vllm/model_executor/parallel_utils/communication_op.py
+1
-1
@@ -9,7 +9,7 @@ from vllm.model_executor.parallel_utils.parallel_state import (
9
def tensor_model_parallel_all_reduce(input_):
10
"""All-reduce the input tensor across model parallel group.
11
12
- Note: This operation is applied in-place on the input tensor.
12
+ NOTE: This operation is applied in-place on the input tensor.
13
"""
14
# Bypass the function if we are using only 1 GPU.
15
if get_tensor_model_parallel_world_size() == 1:
vllm/model_executor/parallel_utils/layers.py
+1
-1
@@ -133,7 +133,7 @@ class ColumnParallelLinear(torch.nn.Module):
133
params_dtype = torch.get_default_dtype()
134
135
# Parameters.
136
- # Note: torch.nn.functional.linear performs XA^T + b and as a result
136
+ # NOTE: torch.nn.functional.linear performs XA^T + b and as a result
137
# we allocate the transpose.
138
self.create_weights(params_dtype)
139
vllm/model_executor/parallel_utils/utils.py
+1
-1
@@ -41,7 +41,7 @@ def split_tensor_along_last_dim(
41
last_dim_size = divide(tensor.size()[last_dim], num_partitions)
42
# Split.
43
tensor_list = torch.split(tensor, last_dim_size, dim=last_dim)
44
- # Note: torch.split does not create contiguous tensors by default.
44
+ # NOTE: torch.split does not create contiguous tensors by default.
45
if contiguous_split_chunks:
46
return tuple(chunk.contiguous() for chunk in tensor_list)
47
vllm/outputs.py
+13
-8
@@ -1,6 +1,7 @@
1
-from typing import Dict, List, Optional
1
+from typing import List, Optional
2
3
-from vllm.sequence import SequenceGroup, SequenceStatus
3
+from vllm.sequence import (PromptLogprobs, SampleLogprobs, SequenceGroup,
4
+ SequenceStatus)
5
6
7
class CompletionOutput:
@@ -23,7 +24,7 @@ class CompletionOutput:
24
text: str,
25
token_ids: List[int],
26
cumulative_logprob: float,
26
- logprobs: Optional[List[Dict[int, float]]],
27
+ logprobs: Optional[SampleLogprobs],
28
finish_reason: Optional[str] = None,
29
) -> None:
30
self.index = index
@@ -61,12 +62,14 @@ class RequestOutput:
62
request_id: str,
63
prompt: str,
64
prompt_token_ids: List[int],
65
+ prompt_logprobs: Optional[PromptLogprobs],
66
outputs: List[CompletionOutput],
67
finished: bool,
68
) -> None:
69
self.request_id = request_id
70
self.prompt = prompt
71
self.prompt_token_ids = prompt_token_ids
72
+ self.prompt_logprobs = prompt_logprobs
73
self.outputs = outputs
74
self.finished = finished
75
@@ -91,7 +94,7 @@ class RequestOutput:
94
# NOTE: We need to take care of this case because the sequence
95
# always has the logprobs of the sampled tokens even if the
96
# logprobs are not requested.
94
- logprobs = {}
97
+ logprobs = None
98
finshed_reason = SequenceStatus.get_finished_reason(seq.status)
99
output = CompletionOutput(seqs.index(seq), seq.output_text,
100
seq.get_output_token_ids(),
@@ -100,15 +103,17 @@ class RequestOutput:
103
outputs.append(output)
104
105
# Every sequence in the sequence group should have the same prompt.
103
- prompt = top_n_seqs[0].prompt
104
- prompt_token_ids = top_n_seqs[0].data.prompt_token_ids
106
+ prompt = seq_group.prompt
107
+ prompt_token_ids = seq_group.prompt_token_ids
108
+ prompt_logprobs = seq_group.prompt_logprobs
109
finished = seq_group.is_finished()
106
- return cls(seq_group.request_id, prompt, prompt_token_ids, outputs,
107
- finished)
110
+ return cls(seq_group.request_id, prompt, prompt_token_ids,
111
+ prompt_logprobs, outputs, finished)
112
113
def __repr__(self) -> str:
114
return (f"RequestOutput(request_id={self.request_id}, "
115
f"prompt={self.prompt!r}, "
116
f"prompt_token_ids={self.prompt_token_ids}, "
117
+ f"prompt_logprobs={self.prompt_logprobs}, "
118
f"outputs={self.outputs}, "
119
f"finished={self.finished})")
vllm/sampling_params.py
+12
@@ -60,6 +60,12 @@ class SamplingParams:
60
tokens after the EOS token is generated.
61
max_tokens: Maximum number of tokens to generate per output sequence.
62
logprobs: Number of log probabilities to return per output token.
63
+ Note that the implementation follows the OpenAI API: The return
64
+ result includes the log probabilities on the `logprobs` most likely
65
+ tokens, as well the chosen tokens. The API will always return the
66
+ log probability of the sampled token, so there may be up to
67
+ `logprobs+1` elements in the response.
68
+ prompt_logprobs: Number of log probabilities to return per prompt token.
69
skip_special_tokens: Whether to skip special tokens in the output.
70
"""
71
@@ -80,6 +86,7 @@ class SamplingParams:
86
ignore_eos: bool = False,
87
max_tokens: int = 16,
88
logprobs: Optional[int] = None,
89
+ prompt_logprobs: Optional[int] = None,
90
skip_special_tokens: bool = True,
91
) -> None:
92
self.n = n
@@ -105,6 +112,7 @@ class SamplingParams:
112
self.ignore_eos = ignore_eos
113
self.max_tokens = max_tokens
114
self.logprobs = logprobs
115
+ self.prompt_logprobs = prompt_logprobs
116
self.skip_special_tokens = skip_special_tokens
117
118
self._verify_args()
@@ -142,6 +150,9 @@ class SamplingParams:
150
if self.logprobs is not None and self.logprobs < 0:
151
raise ValueError(
152
f"logprobs must be non-negative, got {self.logprobs}.")
153
+ if self.prompt_logprobs is not None and self.prompt_logprobs < 0:
154
+ raise ValueError(f"prompt_logprobs must be non-negative, got "
155
+ f"{self.prompt_logprobs}.")
156
157
def _verify_beam_search(self) -> None:
158
if self.best_of == 1:
@@ -200,4 +211,5 @@ class SamplingParams:
211
f"ignore_eos={self.ignore_eos}, "
212
f"max_tokens={self.max_tokens}, "
213
f"logprobs={self.logprobs}, "
214
+ f"prompt_logprobs={self.prompt_logprobs}, "
215
f"skip_special_tokens={self.skip_special_tokens})")
vllm/sequence.py
+35
-3
@@ -6,6 +6,9 @@ from typing import Dict, List, Optional, Union
6
from vllm.block import LogicalTokenBlock
7
from vllm.sampling_params import SamplingParams
8
9
+PromptLogprobs = List[Optional[Dict[int, float]]]
10
+SampleLogprobs = List[Dict[int, float]]
11
+
12
13
class SequenceStatus(enum.Enum):
14
"""Status of a sequence."""
@@ -116,7 +119,7 @@ class Sequence:
119
self.block_size = block_size
120
121
self.data = SequenceData(prompt_token_ids)
119
- self.output_logprobs: List[Dict[int, float]] = []
122
+ self.output_logprobs: SampleLogprobs = []
123
self.output_text = ""
124
125
self.logical_token_blocks: List[LogicalTokenBlock] = []
@@ -196,7 +199,7 @@ class Sequence:
199
"""
200
if seq_len is None:
201
seq_len = self.get_len()
199
- # Note: HF implementation does not count the EOS token
202
+ # NOTE: HF implementation does not count the EOS token
203
# towards the length, we align with that here for testing.
204
if (eos_token_id is not None
205
and self.get_last_token_id() == eos_token_id):
@@ -238,6 +241,19 @@ class SequenceGroup:
241
self.seqs_dict = {seq.seq_id: seq for seq in seqs}
242
self.sampling_params = sampling_params
243
self.arrival_time = arrival_time
244
+ self.prompt_logprobs: Optional[PromptLogprobs] = None
245
+
246
+ @property
247
+ def prompt(self) -> str:
248
+ # All sequences in the group should have the same prompt.
249
+ # We use the prompt of an arbitrary sequence.
250
+ return next(iter(self.seqs_dict.values())).prompt
251
+
252
+ @property
253
+ def prompt_token_ids(self) -> List[int]:
254
+ # All sequences in the group should have the same prompt.
255
+ # We use the prompt of an arbitrary sequence.
256
+ return next(iter(self.seqs_dict.values())).data.prompt_token_ids
257
258
def get_max_num_running_seqs(self) -> int:
259
"""The maximum number of sequences running in parallel in the remaining
@@ -370,6 +386,22 @@ class SequenceOutputs:
386
and self.logprobs == other.logprobs)
387
388
389
+class SequenceGroupOutputs:
390
+ """The model outputs associated with a sequence group."""
391
+
392
+ def __init__(
393
+ self,
394
+ samples: List[SequenceOutputs],
395
+ prompt_logprobs: Optional[PromptLogprobs],
396
+ ) -> None:
397
+ self.samples = samples
398
+ self.prompt_logprobs = prompt_logprobs
399
+
400
+ def __repr__(self) -> str:
401
+ return (f"SequenceGroupOutputs(samples={self.samples}, "
402
+ f"prompt_logprobs={self.prompt_logprobs})")
403
+
404
+
405
# For each sequence group, we generate a list of SequenceOutputs object,
406
# each of which contains one possible candidate for the next token.
375
-SamplerOutput = List[List[SequenceOutputs]]
407
+SamplerOutput = List[SequenceGroupOutputs]